From 4e85684aef8d67e0c9c0fe8f06207e5eddb7c96f Mon Sep 17 00:00:00 2001 From: Jerome Carnis Date: Tue, 26 Apr 2022 12:20:36 +0200 Subject: [PATCH 1/6] update_saved_attributes.py: use new format --- kamzik3/example/saved_attributes_example.yml | 22 ++++---------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/kamzik3/example/saved_attributes_example.yml b/kamzik3/example/saved_attributes_example.yml index 43af711..7a48dff 100644 --- a/kamzik3/example/saved_attributes_example.yml +++ b/kamzik3/example/saved_attributes_example.yml @@ -1,18 +1,4 @@ -{ - "/Default/Detector": { - "Color": "None", - "Time": "None", - "Comment": "None", - "Z ['Position'][mm]": "None", - "Y ['Position'][mm]": "None", - "X ['Position'][mm]": "None", - }, - "/Default/Sample": { - "Color": "None", - "Time": "None", - "Comment": "None", - "Z ['Position'][mm]": "None", - "Y ['Position'][mm]": "None", - "X ['Position'][mm]": "None", - }, -} \ No newline at end of file +groups: [Default, Dummy] +motors: + "Detector": ["Z ['Position'][mm]", "Y ['Position'][mm]", "X ['Position'][mm]"] + "Sample": ["Z ['Position'][mm]", "Y ['Position'][mm]", "X ['Position'][mm]"] \ No newline at end of file -- GitLab From 93f6ae3a4aa4087f6f3eac04afbf2292c4dfb3d4 Mon Sep 17 00:00:00 2001 From: Jerome Carnis Date: Tue, 26 Apr 2022 15:49:26 +0200 Subject: [PATCH 2/6] update_saved_attributes.py: implement parsing configs with a simplified format, update tests --- kamzik3/snippets/update_saved_attributes.py | 47 ++++++++++--- kamzik3/test/test_update_saved_attributes.py | 74 +++++++++++--------- 2 files changed, 80 insertions(+), 41 deletions(-) diff --git a/kamzik3/snippets/update_saved_attributes.py b/kamzik3/snippets/update_saved_attributes.py index bc604c2..6fd7217 100755 --- a/kamzik3/snippets/update_saved_attributes.py +++ b/kamzik3/snippets/update_saved_attributes.py @@ -5,7 +5,7 @@ from h5py import File import oyaml as yaml from pathlib import Path import pandas as pd -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional def generate_config( @@ -98,11 +98,36 @@ def generate_saved_attributes(path: Path, groups: Optional[Dict[str, Any]]) -> b return True -def get_yaml_config(filename: Path) -> Any: +def get_yaml_config(filename: Path) -> Dict[str, Any]: """Open and load a yaml configuration file for the saved attributes.""" with filename.open(mode="r", encoding="utf-8") as config_file: new_conf = yaml.load(config_file, Loader=yaml.SafeLoader) - return new_conf + return check_config(new_conf) + + +def check_config(config: Dict[str, Any]) -> Dict[str, Any]: + """ + Format the new config. + + :param config: the config loaded from the yaml configuration file. + """ + result: Dict[str, Any] = {} + groups = config.get("groups", ["/Default"]) + if "motors" not in config: + raise ValueError("mandatory key 'motors' absent, invalid config") + + for group in groups: + new_group = group if group.startswith("/") else "/" + group + for motor in config["motors"]: + new_motor = motor if motor.startswith("/") else "/" + motor + result[new_group + new_motor] = { + "Color": "None", + "Time": "None", + "Comment": "None", + } + for attribute in config["motors"][motor]: + result[new_group + new_motor][attribute] = "None" + return result def get_existing_config(filename: Path) -> Dict[str, Any]: @@ -131,14 +156,20 @@ def save_existing_model(config: Dict[str, Any], path: Path) -> None: """ path.parent.mkdir(parents=True, exist_ok=True) - out: Dict[str, Any] = {} + groups: List[str] = [] + motors: Dict[str, List[str]] = {} + for key in config: - out[key] = copy.deepcopy(config[key][0]) - if "index" in out[key]: - del out[key]["index"] + group, motor = key.split("/")[1:] # expected ["", "group_name", "motor_name"] + if group not in groups: + groups.append(group) + if motor not in motors: + motors[motor] = list(copy.deepcopy(config[key][0]).keys()) + for item in ["index", "Color", "Time", "Comment"]: + motors[motor].remove(item) with path.open(mode="w", encoding="utf-8") as file: - yaml.dump(out, stream=file) + yaml.dump({"groups": groups, "motors": motors}, stream=file) if __name__ == "__main__": diff --git a/kamzik3/test/test_update_saved_attributes.py b/kamzik3/test/test_update_saved_attributes.py index 015ba51..e2a27d6 100644 --- a/kamzik3/test/test_update_saved_attributes.py +++ b/kamzik3/test/test_update_saved_attributes.py @@ -1,4 +1,5 @@ from pathlib import Path +import yaml import kamzik3.snippets.update_saved_attributes as upd @@ -49,36 +50,11 @@ existing_config = { } new_config = { - "/Default/Detector": { - "Color": "None", - "Time": "None", - "Comment": "None", - "Z ['Position'][mm]": "None", - "Y ['Position'][mm]": "None", - "X ['Position'][mm]": "None", - }, - "/Default/Sample": { - "Color": "None", - "Time": "None", - "Comment": "None", - "Z ['Position'][mm]": "None", - "Y ['Position'][mm]": "None", - "X ['Position'][mm]": "None", - }, - "/Default/Chiller": { - "Color": ["None"], - "Time": ["None"], - "Comment": ["None"], - "Ika ['Temperature'][degC]": "None", - "Ika ['Velocity'][rpm]": "None", - }, - "/Lab099/Detector": { - "Color": "None", - "Time": "None", - "Comment": "None", - "Z ['Position'][mm]": "None", - "Y ['Position'][mm]": "None", - "X ['Position'][mm]": "None", + "groups": ["Default", "Lab099"], + "motors": { + "Detector": ["Z ['Position'][mm]", "Y ['Position'][mm]", "X ['Position'][mm]"], + "Sample": ["X ['Position'][mm]", "Y ['Position'][mm]", "Z ['Position'][mm]"], + "Chiller": ["Ika ['Temperature'][degC]", "Ika ['Velocity'][rpm]"], }, } @@ -100,10 +76,36 @@ def test_get_yaml_config_ok(): assert new_conf["/Default/Sample"].get("X ['Position'][mm]") == "None" +def test_check_config_no_group(): + with yml_file.open(mode="r", encoding="utf-8") as config_file: + new_conf = yaml.load(config_file, Loader=yaml.SafeLoader) + del new_conf["groups"] + result = upd.check_config(new_conf) + assert set(result.keys()) == {"/Default/Detector", "/Default/Sample"} + assert set(result["/Default/Detector"]) == { + "Color", + "Time", + "Comment", + "X ['Position'][mm]", + "Y ['Position'][mm]", + "Z ['Position'][mm]", + } + + def test_save_existing_config_path_ok(tmp_path): filename = tmp_path / "existing_config.yml" upd.save_existing_model(config=existing_config, path=filename) assert filename.is_file() + with filename.open(mode="r", encoding="utf-8") as file: + conf = yaml.load(file, Loader=yaml.SafeLoader) + assert set(conf.keys()) == {"groups", "motors"} + assert conf["groups"] == ["Default"] + assert set(conf["motors"].keys()) == {"Detector", "Sample"} + assert set(conf["motors"]["Sample"]) == { + "Z ['Position'][mm]", + "Y ['Position'][mm]", + "X ['Position'][mm]", + } def test_generate_saved_attributes_groups_none(tmp_path): @@ -113,7 +115,12 @@ def test_generate_saved_attributes_groups_none(tmp_path): def test_generate_saved_attributes_groups_no_record(tmp_path): new_h5file = tmp_path / "saved_attributes_updated.h5" - assert upd.generate_saved_attributes(path=new_h5file, groups=new_config) is True + assert ( + upd.generate_saved_attributes( + path=new_h5file, groups=upd.check_config(new_config) + ) + is True + ) assert new_h5file.is_file() @@ -133,12 +140,13 @@ def test_generate_config_yaml_none(): def test_generate_config_no_existing_config(): - assert upd.generate_config(yaml_config=new_config, h5_config=None) is new_config + new_conf = upd.check_config(new_config) + assert upd.generate_config(yaml_config=new_conf, h5_config=None) is new_conf def test_generate_config_ok(): final_config = upd.generate_config( - yaml_config=new_config, h5_config=existing_config + yaml_config=upd.check_config(new_config), h5_config=existing_config ) assert "/Default/Chiller" in final_config assert final_config["/Default/Detector"][1].get("Time") == "31-05-2021 12:05:59" -- GitLab From 61c13c084053a9fd2526134b07fa8355e0e40faa Mon Sep 17 00:00:00 2001 From: Jerome Carnis Date: Tue, 26 Apr 2022 15:57:43 +0200 Subject: [PATCH 3/6] saved_attributes_example.yml: use yaml format --- kamzik3/example/saved_attributes_example.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/kamzik3/example/saved_attributes_example.yml b/kamzik3/example/saved_attributes_example.yml index 7a48dff..4f80e59 100644 --- a/kamzik3/example/saved_attributes_example.yml +++ b/kamzik3/example/saved_attributes_example.yml @@ -1,4 +1,11 @@ -groups: [Default, Dummy] +groups: +- Default motors: - "Detector": ["Z ['Position'][mm]", "Y ['Position'][mm]", "X ['Position'][mm]"] - "Sample": ["Z ['Position'][mm]", "Y ['Position'][mm]", "X ['Position'][mm]"] \ No newline at end of file + Detector: + - Z ['Position'][mm] + - Y ['Position'][mm] + - X ['Position'][mm] + Sample: + - X ['Position'][mm] + - Y ['Position'][mm] + - Z ['Position'][mm] -- GitLab From 218e0be37293b4ab8ea272cd1199de3fbfbbee83 Mon Sep 17 00:00:00 2001 From: Jerome Carnis Date: Wed, 27 Apr 2022 15:11:44 +0200 Subject: [PATCH 4/6] update_saved_attributes: remove anti-patterns --- kamzik3/snippets/update_saved_attributes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/kamzik3/snippets/update_saved_attributes.py b/kamzik3/snippets/update_saved_attributes.py index 6fd7217..49236e0 100755 --- a/kamzik3/snippets/update_saved_attributes.py +++ b/kamzik3/snippets/update_saved_attributes.py @@ -113,19 +113,21 @@ def check_config(config: Dict[str, Any]) -> Dict[str, Any]: """ result: Dict[str, Any] = {} groups = config.get("groups", ["/Default"]) - if "motors" not in config: + motors = config.get("motors", None) + + if motors is None: raise ValueError("mandatory key 'motors' absent, invalid config") for group in groups: new_group = group if group.startswith("/") else "/" + group - for motor in config["motors"]: + for motor, attributes in motors.items(): new_motor = motor if motor.startswith("/") else "/" + motor result[new_group + new_motor] = { "Color": "None", "Time": "None", "Comment": "None", } - for attribute in config["motors"][motor]: + for attribute in attributes: result[new_group + new_motor][attribute] = "None" return result -- GitLab From acb8dca0de542a5e4c498aba578108bcf8af64bb Mon Sep 17 00:00:00 2001 From: Jerome Carnis Date: Wed, 27 Apr 2022 15:24:18 +0200 Subject: [PATCH 5/6] update_saved_attributes: make `save_existing_model` more robust --- kamzik3/snippets/update_saved_attributes.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kamzik3/snippets/update_saved_attributes.py b/kamzik3/snippets/update_saved_attributes.py index 49236e0..8dd45b4 100755 --- a/kamzik3/snippets/update_saved_attributes.py +++ b/kamzik3/snippets/update_saved_attributes.py @@ -154,7 +154,6 @@ def save_existing_model(config: Dict[str, Any], path: Path) -> None: :param config: dict, config containing the model at index 0 and then records :param path: Path, where to save the extracted model - """ path.parent.mkdir(parents=True, exist_ok=True) @@ -162,7 +161,11 @@ def save_existing_model(config: Dict[str, Any], path: Path) -> None: motors: Dict[str, List[str]] = {} for key in config: - group, motor = key.split("/")[1:] # expected ["", "group_name", "motor_name"] + try: + group, motor = key.split("/", maxsplit=2)[1:] + # expected ["", "group_name", "motor_name"] + except ValueError: + raise ValueError(f"Invalid key '{key}', expecting '/group_name/motor_name'") if group not in groups: groups.append(group) if motor not in motors: -- GitLab From 94324d1c9a6600fa2cdfae7251a4e90d7b4bcba1 Mon Sep 17 00:00:00 2001 From: Jerome Carnis Date: Wed, 27 Apr 2022 15:38:06 +0200 Subject: [PATCH 6/6] update_saved_attributes: use more precise type hints --- kamzik3/snippets/update_saved_attributes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kamzik3/snippets/update_saved_attributes.py b/kamzik3/snippets/update_saved_attributes.py index 8dd45b4..19bb0e7 100755 --- a/kamzik3/snippets/update_saved_attributes.py +++ b/kamzik3/snippets/update_saved_attributes.py @@ -98,14 +98,14 @@ def generate_saved_attributes(path: Path, groups: Optional[Dict[str, Any]]) -> b return True -def get_yaml_config(filename: Path) -> Dict[str, Any]: +def get_yaml_config(filename: Path) -> Dict[str, Dict[str, str]]: """Open and load a yaml configuration file for the saved attributes.""" with filename.open(mode="r", encoding="utf-8") as config_file: new_conf = yaml.load(config_file, Loader=yaml.SafeLoader) return check_config(new_conf) -def check_config(config: Dict[str, Any]) -> Dict[str, Any]: +def check_config(config: Dict[str, Any]) -> Dict[str, Dict[str, str]]: """ Format the new config. -- GitLab