From d18934ce3e39be6718b5140bb25b4ef70ea785b0 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 14 Oct 2024 22:11:26 +0200 Subject: [PATCH 1/9] move code to src folder --- {sed/config => config}/NXmpes_config.json | 0 {sed/config => config}/default.yaml | 0 {sed/config => config}/flash_example_config.yaml | 0 {sed/config => config}/mpes_example_config.yaml | 0 {sed/config => config}/sxp_example_config.yaml | 0 pyproject.toml | 6 ++---- {sed => src/sed}/__init__.py | 0 {sed => src/sed}/binning/__init__.py | 0 {sed => src/sed}/binning/binning.py | 0 {sed => src/sed}/binning/numba_bin.py | 0 {sed => src/sed}/binning/utils.py | 0 {sed => src/sed}/calibrator/__init__.py | 0 {sed => src/sed}/calibrator/delay.py | 0 {sed => src/sed}/calibrator/energy.py | 0 {sed => src/sed}/calibrator/momentum.py | 0 {sed => src/sed}/core/__init__.py | 0 {sed => src/sed}/core/config.py | 0 {sed/config => src/sed/core}/config_model.py | 0 {sed => src/sed}/core/dfops.py | 0 {sed => src/sed}/core/logging.py | 0 {sed => src/sed}/core/metadata.py | 0 {sed => src/sed}/core/processor.py | 0 {sed => src/sed}/dataset/__init__.py | 0 {sed => src/sed}/dataset/dataset.py | 0 {sed => src/sed}/dataset/datasets.json | 0 {sed => src/sed}/diagnostics.py | 0 {sed => src/sed}/io/__init__.py | 0 {sed => src/sed}/io/hdf5.py | 0 {sed => src/sed}/io/nexus.py | 0 {sed => src/sed}/io/tiff.py | 0 {sed => src/sed}/loader/__init__.py | 0 {sed => src/sed}/loader/base/README.md | 0 {sed => src/sed}/loader/base/__init__.py | 0 {sed => src/sed}/loader/base/loader.py | 0 {sed => src/sed}/loader/flash/__init__.py | 0 {sed => src/sed}/loader/flash/buffer_handler.py | 0 {sed => src/sed}/loader/flash/dataframe.py | 0 {sed => src/sed}/loader/flash/instruments.py | 0 {sed => src/sed}/loader/flash/loader.py | 0 {sed => src/sed}/loader/flash/metadata.py | 0 {sed => src/sed}/loader/flash/utils.py | 0 {sed => src/sed}/loader/generic/__init__.py | 0 {sed => src/sed}/loader/generic/loader.py | 0 {sed => src/sed}/loader/loader_interface.py | 0 {sed => src/sed}/loader/mirrorutil.py | 0 {sed => src/sed}/loader/mpes/__init__.py | 0 {sed => src/sed}/loader/mpes/loader.py | 0 {sed => src/sed}/loader/sxp/__init__.py | 0 {sed => src/sed}/loader/sxp/loader.py | 0 {sed => src/sed}/loader/utils.py | 0 50 files changed, 2 insertions(+), 4 deletions(-) rename {sed/config => config}/NXmpes_config.json (100%) rename {sed/config => config}/default.yaml (100%) rename {sed/config => config}/flash_example_config.yaml (100%) rename {sed/config => config}/mpes_example_config.yaml (100%) rename {sed/config => config}/sxp_example_config.yaml (100%) rename {sed => src/sed}/__init__.py (100%) rename {sed => src/sed}/binning/__init__.py (100%) rename {sed => src/sed}/binning/binning.py (100%) rename {sed => src/sed}/binning/numba_bin.py (100%) rename {sed => src/sed}/binning/utils.py (100%) rename {sed => src/sed}/calibrator/__init__.py (100%) rename {sed => src/sed}/calibrator/delay.py (100%) rename {sed => src/sed}/calibrator/energy.py (100%) rename {sed => src/sed}/calibrator/momentum.py (100%) rename {sed => src/sed}/core/__init__.py (100%) rename {sed => src/sed}/core/config.py (100%) rename {sed/config => src/sed/core}/config_model.py (100%) rename {sed => src/sed}/core/dfops.py (100%) rename {sed => src/sed}/core/logging.py (100%) rename {sed => src/sed}/core/metadata.py (100%) rename {sed => src/sed}/core/processor.py (100%) rename {sed => src/sed}/dataset/__init__.py (100%) rename {sed => src/sed}/dataset/dataset.py (100%) rename {sed => src/sed}/dataset/datasets.json (100%) rename {sed => src/sed}/diagnostics.py (100%) rename {sed => src/sed}/io/__init__.py (100%) rename {sed => src/sed}/io/hdf5.py (100%) rename {sed => src/sed}/io/nexus.py (100%) rename {sed => src/sed}/io/tiff.py (100%) rename {sed => src/sed}/loader/__init__.py (100%) rename {sed => src/sed}/loader/base/README.md (100%) rename {sed => src/sed}/loader/base/__init__.py (100%) rename {sed => src/sed}/loader/base/loader.py (100%) rename {sed => src/sed}/loader/flash/__init__.py (100%) rename {sed => src/sed}/loader/flash/buffer_handler.py (100%) rename {sed => src/sed}/loader/flash/dataframe.py (100%) rename {sed => src/sed}/loader/flash/instruments.py (100%) rename {sed => src/sed}/loader/flash/loader.py (100%) rename {sed => src/sed}/loader/flash/metadata.py (100%) rename {sed => src/sed}/loader/flash/utils.py (100%) rename {sed => src/sed}/loader/generic/__init__.py (100%) rename {sed => src/sed}/loader/generic/loader.py (100%) rename {sed => src/sed}/loader/loader_interface.py (100%) rename {sed => src/sed}/loader/mirrorutil.py (100%) rename {sed => src/sed}/loader/mpes/__init__.py (100%) rename {sed => src/sed}/loader/mpes/loader.py (100%) rename {sed => src/sed}/loader/sxp/__init__.py (100%) rename {sed => src/sed}/loader/sxp/loader.py (100%) rename {sed => src/sed}/loader/utils.py (100%) diff --git a/sed/config/NXmpes_config.json b/config/NXmpes_config.json similarity index 100% rename from sed/config/NXmpes_config.json rename to config/NXmpes_config.json diff --git a/sed/config/default.yaml b/config/default.yaml similarity index 100% rename from sed/config/default.yaml rename to config/default.yaml diff --git a/sed/config/flash_example_config.yaml b/config/flash_example_config.yaml similarity index 100% rename from sed/config/flash_example_config.yaml rename to config/flash_example_config.yaml diff --git a/sed/config/mpes_example_config.yaml b/config/mpes_example_config.yaml similarity index 100% rename from sed/config/mpes_example_config.yaml rename to config/mpes_example_config.yaml diff --git a/sed/config/sxp_example_config.yaml b/config/sxp_example_config.yaml similarity index 100% rename from sed/config/sxp_example_config.yaml rename to config/sxp_example_config.yaml diff --git a/pyproject.toml b/pyproject.toml index 4d71473e..fd670628 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,6 @@ [tool.poetry] name = "sed-processor" -packages = [ - {include = "sed"} -] +packages = [{include = "sed", from = "src"}] version = "1.0.0a0" description = "Single Event Data Frame Processor: Backend to handle photoelectron resolved datastreams" authors = ["OpenCOMPES team "] @@ -84,7 +82,7 @@ omit = [ ] [tool.ruff] -include = ["sed/*.py", "tests/*.py"] +include = ["src/*.py", "tests/*.py"] lint.select = [ "E", # pycodestyle "W", # pycodestyle diff --git a/sed/__init__.py b/src/sed/__init__.py similarity index 100% rename from sed/__init__.py rename to src/sed/__init__.py diff --git a/sed/binning/__init__.py b/src/sed/binning/__init__.py similarity index 100% rename from sed/binning/__init__.py rename to src/sed/binning/__init__.py diff --git a/sed/binning/binning.py b/src/sed/binning/binning.py similarity index 100% rename from sed/binning/binning.py rename to src/sed/binning/binning.py diff --git a/sed/binning/numba_bin.py b/src/sed/binning/numba_bin.py similarity index 100% rename from sed/binning/numba_bin.py rename to src/sed/binning/numba_bin.py diff --git a/sed/binning/utils.py b/src/sed/binning/utils.py similarity index 100% rename from sed/binning/utils.py rename to src/sed/binning/utils.py diff --git a/sed/calibrator/__init__.py b/src/sed/calibrator/__init__.py similarity index 100% rename from sed/calibrator/__init__.py rename to src/sed/calibrator/__init__.py diff --git a/sed/calibrator/delay.py b/src/sed/calibrator/delay.py similarity index 100% rename from sed/calibrator/delay.py rename to src/sed/calibrator/delay.py diff --git a/sed/calibrator/energy.py b/src/sed/calibrator/energy.py similarity index 100% rename from sed/calibrator/energy.py rename to src/sed/calibrator/energy.py diff --git a/sed/calibrator/momentum.py b/src/sed/calibrator/momentum.py similarity index 100% rename from sed/calibrator/momentum.py rename to src/sed/calibrator/momentum.py diff --git a/sed/core/__init__.py b/src/sed/core/__init__.py similarity index 100% rename from sed/core/__init__.py rename to src/sed/core/__init__.py diff --git a/sed/core/config.py b/src/sed/core/config.py similarity index 100% rename from sed/core/config.py rename to src/sed/core/config.py diff --git a/sed/config/config_model.py b/src/sed/core/config_model.py similarity index 100% rename from sed/config/config_model.py rename to src/sed/core/config_model.py diff --git a/sed/core/dfops.py b/src/sed/core/dfops.py similarity index 100% rename from sed/core/dfops.py rename to src/sed/core/dfops.py diff --git a/sed/core/logging.py b/src/sed/core/logging.py similarity index 100% rename from sed/core/logging.py rename to src/sed/core/logging.py diff --git a/sed/core/metadata.py b/src/sed/core/metadata.py similarity index 100% rename from sed/core/metadata.py rename to src/sed/core/metadata.py diff --git a/sed/core/processor.py b/src/sed/core/processor.py similarity index 100% rename from sed/core/processor.py rename to src/sed/core/processor.py diff --git a/sed/dataset/__init__.py b/src/sed/dataset/__init__.py similarity index 100% rename from sed/dataset/__init__.py rename to src/sed/dataset/__init__.py diff --git a/sed/dataset/dataset.py b/src/sed/dataset/dataset.py similarity index 100% rename from sed/dataset/dataset.py rename to src/sed/dataset/dataset.py diff --git a/sed/dataset/datasets.json b/src/sed/dataset/datasets.json similarity index 100% rename from sed/dataset/datasets.json rename to src/sed/dataset/datasets.json diff --git a/sed/diagnostics.py b/src/sed/diagnostics.py similarity index 100% rename from sed/diagnostics.py rename to src/sed/diagnostics.py diff --git a/sed/io/__init__.py b/src/sed/io/__init__.py similarity index 100% rename from sed/io/__init__.py rename to src/sed/io/__init__.py diff --git a/sed/io/hdf5.py b/src/sed/io/hdf5.py similarity index 100% rename from sed/io/hdf5.py rename to src/sed/io/hdf5.py diff --git a/sed/io/nexus.py b/src/sed/io/nexus.py similarity index 100% rename from sed/io/nexus.py rename to src/sed/io/nexus.py diff --git a/sed/io/tiff.py b/src/sed/io/tiff.py similarity index 100% rename from sed/io/tiff.py rename to src/sed/io/tiff.py diff --git a/sed/loader/__init__.py b/src/sed/loader/__init__.py similarity index 100% rename from sed/loader/__init__.py rename to src/sed/loader/__init__.py diff --git a/sed/loader/base/README.md b/src/sed/loader/base/README.md similarity index 100% rename from sed/loader/base/README.md rename to src/sed/loader/base/README.md diff --git a/sed/loader/base/__init__.py b/src/sed/loader/base/__init__.py similarity index 100% rename from sed/loader/base/__init__.py rename to src/sed/loader/base/__init__.py diff --git a/sed/loader/base/loader.py b/src/sed/loader/base/loader.py similarity index 100% rename from sed/loader/base/loader.py rename to src/sed/loader/base/loader.py diff --git a/sed/loader/flash/__init__.py b/src/sed/loader/flash/__init__.py similarity index 100% rename from sed/loader/flash/__init__.py rename to src/sed/loader/flash/__init__.py diff --git a/sed/loader/flash/buffer_handler.py b/src/sed/loader/flash/buffer_handler.py similarity index 100% rename from sed/loader/flash/buffer_handler.py rename to src/sed/loader/flash/buffer_handler.py diff --git a/sed/loader/flash/dataframe.py b/src/sed/loader/flash/dataframe.py similarity index 100% rename from sed/loader/flash/dataframe.py rename to src/sed/loader/flash/dataframe.py diff --git a/sed/loader/flash/instruments.py b/src/sed/loader/flash/instruments.py similarity index 100% rename from sed/loader/flash/instruments.py rename to src/sed/loader/flash/instruments.py diff --git a/sed/loader/flash/loader.py b/src/sed/loader/flash/loader.py similarity index 100% rename from sed/loader/flash/loader.py rename to src/sed/loader/flash/loader.py diff --git a/sed/loader/flash/metadata.py b/src/sed/loader/flash/metadata.py similarity index 100% rename from sed/loader/flash/metadata.py rename to src/sed/loader/flash/metadata.py diff --git a/sed/loader/flash/utils.py b/src/sed/loader/flash/utils.py similarity index 100% rename from sed/loader/flash/utils.py rename to src/sed/loader/flash/utils.py diff --git a/sed/loader/generic/__init__.py b/src/sed/loader/generic/__init__.py similarity index 100% rename from sed/loader/generic/__init__.py rename to src/sed/loader/generic/__init__.py diff --git a/sed/loader/generic/loader.py b/src/sed/loader/generic/loader.py similarity index 100% rename from sed/loader/generic/loader.py rename to src/sed/loader/generic/loader.py diff --git a/sed/loader/loader_interface.py b/src/sed/loader/loader_interface.py similarity index 100% rename from sed/loader/loader_interface.py rename to src/sed/loader/loader_interface.py diff --git a/sed/loader/mirrorutil.py b/src/sed/loader/mirrorutil.py similarity index 100% rename from sed/loader/mirrorutil.py rename to src/sed/loader/mirrorutil.py diff --git a/sed/loader/mpes/__init__.py b/src/sed/loader/mpes/__init__.py similarity index 100% rename from sed/loader/mpes/__init__.py rename to src/sed/loader/mpes/__init__.py diff --git a/sed/loader/mpes/loader.py b/src/sed/loader/mpes/loader.py similarity index 100% rename from sed/loader/mpes/loader.py rename to src/sed/loader/mpes/loader.py diff --git a/sed/loader/sxp/__init__.py b/src/sed/loader/sxp/__init__.py similarity index 100% rename from sed/loader/sxp/__init__.py rename to src/sed/loader/sxp/__init__.py diff --git a/sed/loader/sxp/loader.py b/src/sed/loader/sxp/loader.py similarity index 100% rename from sed/loader/sxp/loader.py rename to src/sed/loader/sxp/loader.py diff --git a/sed/loader/utils.py b/src/sed/loader/utils.py similarity index 100% rename from sed/loader/utils.py rename to src/sed/loader/utils.py From 43756924bd2eda28c4d88b160be3991820f7f12e Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 14 Oct 2024 23:16:53 +0200 Subject: [PATCH 2/9] fix tests --- src/sed/core/config.py | 4 ++-- tests/calibrator/test_delay.py | 2 +- tests/calibrator/test_energy.py | 4 ++-- tests/calibrator/test_momentum.py | 4 ++-- tests/loader/flash/conftest.py | 11 ++++++----- tests/loader/mpes/test_mpes_loader.py | 2 +- tests/loader/sxp/test_sxp_loader.py | 2 +- tests/loader/test_loaders.py | 2 +- tests/loader/test_mirrorutil.py | 6 +++--- tests/test_config.py | 6 +++--- tests/test_dataset.py | 4 ++-- tests/test_diagnostics.py | 6 +++--- tests/test_processor.py | 14 +++++++------- 13 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/sed/core/config.py b/src/sed/core/config.py index ca29285c..d24b1b7b 100644 --- a/src/sed/core/config.py +++ b/src/sed/core/config.py @@ -12,7 +12,7 @@ import yaml from platformdirs import user_config_path -from sed.config.config_model import ConfigModel +from sed.core.config_model import ConfigModel from sed.core.logging import setup_logging package_dir = os.path.dirname(find_spec("sed").origin) @@ -28,7 +28,7 @@ def parse_config( folder_config: dict | str = None, user_config: dict | str = None, system_config: dict | str = None, - default_config: (dict | str) = f"{package_dir}/config/default.yaml", + default_config: (dict | str) = f"{package_dir}/../../config/default.yaml", verbose: bool = True, verify_config: bool = True, ) -> dict: diff --git a/tests/calibrator/test_delay.py b/tests/calibrator/test_delay.py index b5bbe49c..5b6838ed 100644 --- a/tests/calibrator/test_delay.py +++ b/tests/calibrator/test_delay.py @@ -16,7 +16,7 @@ from sed.loader.loader_interface import get_loader package_dir = os.path.dirname(find_spec("sed").origin) -file = package_dir + "/../tests/data/loader/mpes/Scan0030_2.h5" +file = package_dir + "/../../tests/data/loader/mpes/Scan0030_2.h5" def test_delay_parameters_from_file() -> None: diff --git a/tests/calibrator/test_energy.py b/tests/calibrator/test_energy.py index 21bac331..fdbda343 100644 --- a/tests/calibrator/test_energy.py +++ b/tests/calibrator/test_energy.py @@ -22,8 +22,8 @@ from sed.loader.loader_interface import get_loader package_dir = os.path.dirname(find_spec("sed").origin) -df_folder = package_dir + "/../tests/data/loader/mpes/" -folder = package_dir + "/../tests/data/calibrator/" +df_folder = package_dir + "/../../tests/data/loader/mpes/" +folder = package_dir + "/../../tests/data/calibrator/" files = glob.glob(df_folder + "*.h5") traces_list = [] diff --git a/tests/calibrator/test_momentum.py b/tests/calibrator/test_momentum.py index de229e9f..a8e2f166 100644 --- a/tests/calibrator/test_momentum.py +++ b/tests/calibrator/test_momentum.py @@ -18,8 +18,8 @@ # pylint: disable=duplicate-code package_dir = os.path.dirname(find_spec("sed").origin) -df_folder = package_dir + "/../tests/data/loader/mpes/" -folder = package_dir + "/../tests/data/calibrator/" +df_folder = package_dir + "/../../tests/data/loader/mpes/" +folder = package_dir + "/../../tests/data/calibrator/" files = glob.glob(df_folder + "*.h5") momentum_map_list = [] diff --git a/tests/loader/flash/conftest.py b/tests/loader/flash/conftest.py index 8fa9c3b7..3cd98275 100644 --- a/tests/loader/flash/conftest.py +++ b/tests/loader/flash/conftest.py @@ -11,7 +11,7 @@ from sed.core.config import parse_config package_dir = os.path.dirname(find_spec("sed").origin) -config_path = os.path.join(package_dir, "../tests/data/loader/flash/config.yaml") +config_path = os.path.join(package_dir, "../../tests/data/loader/flash/config.yaml") H5_PATH = "FLASH1_USER3_stream_2_run43878_file1_20230130T153807.1.h5" H5_PATHS = [H5_PATH, "FLASH1_USER3_stream_2_run43879_file1_20230130T153807.1.h5"] @@ -45,7 +45,7 @@ def fixture_h5_file() -> h5py.File: Returns: h5py.File: The open h5 file. """ - return h5py.File(os.path.join(package_dir, f"../tests/data/loader/flash/{H5_PATH}"), "r") + return h5py.File(os.path.join(package_dir, f"../../tests/data/loader/flash/{H5_PATH}"), "r") @pytest.fixture(name="h5_file_copy") @@ -56,7 +56,7 @@ def fixture_h5_file_copy(tmp_path: Path) -> h5py.File: h5py.File: The open h5 file copy. """ # Create a copy of the h5 file in a temporary directory - original_file_path = os.path.join(package_dir, f"../tests/data/loader/flash/{H5_PATH}") + original_file_path = os.path.join(package_dir, f"../../tests/data/loader/flash/{H5_PATH}") copy_file_path = tmp_path / "copy.h5" shutil.copyfile(original_file_path, copy_file_path) @@ -72,7 +72,7 @@ def fixture_h5_file2_copy(tmp_path: Path) -> h5py.File: h5py.File: The open h5 file copy. """ # Create a copy of the h5 file in a temporary directory - original_file_path = os.path.join(package_dir, f"../tests/data/loader/flash/{H5_PATHS[1]}") + original_file_path = os.path.join(package_dir, f"../../tests/data/loader/flash/{H5_PATHS[1]}") copy_file_path = tmp_path / "copy2.h5" shutil.copyfile(original_file_path, copy_file_path) @@ -88,5 +88,6 @@ def fixture_h5_paths() -> list[Path]: list: A list of h5 file paths. """ return [ - Path(os.path.join(package_dir, f"../tests/data/loader/flash/{path}")) for path in H5_PATHS + Path(os.path.join(package_dir, f"../../tests/data/loader/flash/{path}")) + for path in H5_PATHS ] diff --git a/tests/loader/mpes/test_mpes_loader.py b/tests/loader/mpes/test_mpes_loader.py index 936efe44..0bc15b6f 100644 --- a/tests/loader/mpes/test_mpes_loader.py +++ b/tests/loader/mpes/test_mpes_loader.py @@ -13,7 +13,7 @@ package_dir = os.path.dirname(find_spec("sed").origin) -test_data_dir = os.path.join(package_dir, "..", "tests", "data", "loader", "mpes") +test_data_dir = os.path.join(package_dir, "../../tests/data/loader/mpes") config = parse_config( os.path.join(test_data_dir, "config.yaml"), diff --git a/tests/loader/sxp/test_sxp_loader.py b/tests/loader/sxp/test_sxp_loader.py index 1ee06d41..8ae3f64a 100644 --- a/tests/loader/sxp/test_sxp_loader.py +++ b/tests/loader/sxp/test_sxp_loader.py @@ -11,7 +11,7 @@ from sed.loader.sxp.loader import SXPLoader package_dir = os.path.dirname(find_spec("sed").origin) -config_path = os.path.join(package_dir, "../tests/data/loader/sxp/config.yaml") +config_path = os.path.join(package_dir, "../../tests/data/loader/sxp/config.yaml") H5_PATH = "RAW-R0016-DA03-S00000.h5" diff --git a/tests/loader/test_loaders.py b/tests/loader/test_loaders.py index f3e0bf84..705f8ae8 100644 --- a/tests/loader/test_loaders.py +++ b/tests/loader/test_loaders.py @@ -21,7 +21,7 @@ package_dir = os.path.dirname(find_spec("sed").origin) -test_data_dir = os.path.join(package_dir, "..", "tests", "data") +test_data_dir = os.path.join(package_dir, "../../tests/data") read_types = ["one_file", "files", "one_folder", "folders", "one_run", "runs"] runs = {"generic": None, "mpes": ["30", "50"], "flash": ["43878", "43878"], "sxp": ["0016", "0016"]} diff --git a/tests/loader/test_mirrorutil.py b/tests/loader/test_mirrorutil.py index d7d522c8..5aa57477 100644 --- a/tests/loader/test_mirrorutil.py +++ b/tests/loader/test_mirrorutil.py @@ -16,9 +16,9 @@ package_dir = os.path.dirname(find_spec("sed").origin) -source_folder = package_dir + "/../" -folder = package_dir + "/../tests/data/loader/mpes" -file = package_dir + "/../tests/data/loader/mpes/Scan0030_2.h5" +source_folder = package_dir + "/../../" +folder = package_dir + "/../../tests/data/loader/mpes" +file = package_dir + "/../../tests/data/loader/mpes/Scan0030_2.h5" def test_copy_tool_folder() -> None: diff --git a/tests/test_config.py b/tests/test_config.py index bdbdd9c8..d94a44f1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -92,10 +92,10 @@ def test_load_does_not_modify() -> None: def test_load_config() -> None: """Test if the config loader can handle json and yaml files.""" config_json = load_config( - f"{package_dir}/../tests/data/config/config.json", + f"{package_dir}/../../tests/data/config/config.json", ) config_yaml = load_config( - f"{package_dir}/../tests/data/config/config.yaml", + f"{package_dir}/../../tests/data/config/config.yaml", ) assert config_json == config_yaml @@ -103,7 +103,7 @@ def test_load_config() -> None: def test_load_config_raise() -> None: """Test if the config loader raises an error for a wrong file type.""" with pytest.raises(TypeError): - load_config(f"{package_dir}/../README.md") + load_config(f"{package_dir}/../../README.md") def test_complete_dictionary() -> None: diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 4ff70ede..653b97e1 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -15,7 +15,7 @@ from sed.dataset import DatasetsManager as dm package_dir = os.path.dirname(find_spec("sed").origin) -json_path = os.path.join(package_dir, "datasets.json") +json_path = os.path.join(package_dir, "../../config/datasets.json") @pytest.fixture @@ -120,7 +120,7 @@ def test_rearrange_data(zip_file): # noqa: ARG001 ds._rearrange_data() assert os.path.exists("test/datasets/Test/test_file.txt") assert os.path.exists("test/datasets/Test/test_subdir.txt") - assert ~os.path.exists("test/datasets/Test/subdir") + assert not os.path.exists("test/datasets/Test/subdir") with pytest.raises(FileNotFoundError): ds._subdirs = ["non_existing_subdir"] diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index cb5bcce6..e99fdc1e 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -15,11 +15,11 @@ # pylint: disable=duplicate-code package_dir = os.path.dirname(find_spec("sed").origin) -df_folder = package_dir + "/../tests/data/loader/mpes/" -folder = package_dir + "/../tests/data/calibrator/" +df_folder = package_dir + "/../../tests/data/loader/mpes/" +folder = package_dir + "/../../tests/data/calibrator/" files = glob.glob(df_folder + "*.h5") config = parse_config( - package_dir + "/../tests/data/loader/mpes/config.yaml", + package_dir + "/../../tests/data/loader/mpes/config.yaml", folder_config={}, user_config={}, system_config={}, diff --git a/tests/test_processor.py b/tests/test_processor.py index af204eec..ca489f38 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -26,14 +26,14 @@ # pylint: disable=duplicate-code package_dir = os.path.dirname(find_spec("sed").origin) -df_folder = package_dir + "/../tests/data/loader/mpes/" -df_folder_generic = package_dir + "/../tests/data/loader/generic/" -folder = package_dir + "/../tests/data/calibrator/" +df_folder = package_dir + "/../../tests/data/loader/mpes/" +df_folder_generic = package_dir + "/../../tests/data/loader/generic/" +folder = package_dir + "/../../tests/data/calibrator/" files = glob.glob(df_folder + "*.h5") runs = ["30", "50"] runs_flash = ["43878", "43878"] loader = get_loader(loader_name="mpes") -source_folder = package_dir + "/../" +source_folder = package_dir + "/../../" dest_folder = tempfile.mkdtemp() gid = os.getgid() @@ -852,7 +852,7 @@ def test_add_time_stamped_data() -> None: """Test the function to add time-stamped data""" processor = SedProcessor( folder=df_folder + "../mpes/", - config=package_dir + "/config/mpes_example_config.yaml", + config=package_dir + "/../../config/mpes_example_config.yaml", folder_config={}, user_config={}, system_config={}, @@ -1083,7 +1083,7 @@ def test_save(caplog) -> None: config = parse_config( config={"dataframe": {"tof_binning": 1}}, folder_config={}, - user_config=package_dir + "/../sed/config/mpes_example_config.yaml", + user_config=package_dir + "/../../config/mpes_example_config.yaml", system_config={}, verify_config=False, ) @@ -1140,7 +1140,7 @@ def test_save(caplog) -> None: yaml.dump({"Instrument": {"undocumented_field": "undocumented entry"}}, f) with open("temp_config.json", "w") as f: with open( - package_dir + "/../sed/config/NXmpes_config.json", + package_dir + "/../../config/NXmpes_config.json", encoding="utf-8", ) as stream: config_dict = json.load(stream) From f2931ef46ec198f01ce39ab78335fdf06d1ef2fc Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 14 Oct 2024 23:25:29 +0200 Subject: [PATCH 3/9] update workflows --- .github/workflows/documentation.yml | 4 ++-- .github/workflows/linting.yml | 6 +++--- .github/workflows/release.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index dec7767a..6afef727 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -5,7 +5,7 @@ on: branches: [ main ] tags: [ v* ] paths: - - sed/**/* + - src/sed/**/* - tutorial/** - .github/workflows/documentation.yml # Allows you to run this workflow manually from the Actions tab @@ -50,7 +50,7 @@ jobs: - name: copy tutorial files to docs run: | cp -r $GITHUB_WORKSPACE/tutorial $GITHUB_WORKSPACE/docs/ - cp -r $GITHUB_WORKSPACE/sed/config $GITHUB_WORKSPACE/docs/sed + cp -r $GITHUB_WORKSPACE/config $GITHUB_WORKSPACE/docs/ - name: download RAW data # if: steps.cache-primes.outputs.cache-hit != 'true' diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 593077cc..89c92048 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -25,15 +25,15 @@ jobs: # Linting steps, execute all linters even if one fails - name: ruff run: - poetry run ruff sed tests + poetry run ruff src/sed tests - name: ruff formatting if: ${{ always() }} run: - poetry run ruff format --check sed tests + poetry run ruff format --check src/sed tests - name: mypy if: ${{ always() }} run: - poetry run mypy sed tests + poetry run mypy src/sed tests - name: spellcheck if: ${{ always() }} uses: streetsidesoftware/cspell-action@v6 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aed41825..ce492666 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ on: tags: - v[0-9]+.[0-9]+.[0-9]+ paths: - - sed/**/* + - src/sed/**/* - .github/workflows/release.yml # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 35e49660ad498beaafa3c25d785a9ec85e7184e6 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 14 Oct 2024 23:56:09 +0200 Subject: [PATCH 4/9] update tutorials --- {src/sed/dataset => config}/datasets.json | 0 src/sed/dataset/dataset.py | 2 +- ...nversion_pipeline_for_example_time-resolved_ARPES_data.ipynb | 2 +- tutorial/3_metadata_collection_and_export_to_NeXus.ipynb | 2 +- tutorial/4_hextof_workflow.ipynb | 2 +- tutorial/5_sxp_workflow.ipynb | 2 +- tutorial/6_binning_with_time-stamped_data.ipynb | 2 +- tutorial/7_correcting_orthorhombic_symmetry.ipynb | 2 +- tutorial/8_jittering_tutorial.ipynb | 2 +- 9 files changed, 8 insertions(+), 8 deletions(-) rename {src/sed/dataset => config}/datasets.json (100%) diff --git a/src/sed/dataset/datasets.json b/config/datasets.json similarity index 100% rename from src/sed/dataset/datasets.json rename to config/datasets.json diff --git a/src/sed/dataset/dataset.py b/src/sed/dataset/dataset.py index fca7fc83..ad445924 100644 --- a/src/sed/dataset/dataset.py +++ b/src/sed/dataset/dataset.py @@ -32,7 +32,7 @@ class DatasetsManager: FILENAME = NAME + ".json" json_path = {} json_path["user"] = os.path.join(USER_CONFIG_PATH, FILENAME) - json_path["module"] = os.path.join(os.path.dirname(__file__), FILENAME) + json_path["module"] = os.path.join(os.path.dirname(__file__), "../../../config", FILENAME) json_path["folder"] = "./" + FILENAME @staticmethod diff --git a/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb b/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb index 19fb3b74..2a10dbba 100644 --- a/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb +++ b/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb @@ -59,7 +59,7 @@ "outputs": [], "source": [ "# create sed processor using the config file:\n", - "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", system_config={}, verbose=True)" + "sp = sed.SedProcessor(folder=scandir, config=\"../config/mpes_example_config.yaml\", system_config={}, verbose=True)" ] }, { diff --git a/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb b/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb index 7ac5d39f..53b58a05 100644 --- a/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb +++ b/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb @@ -143,7 +143,7 @@ "outputs": [], "source": [ "# create sed processor using the config file, and collect the meta data from the files:\n", - "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", system_config={}, metadata=metadata, collect_metadata=True)" + "sp = sed.SedProcessor(folder=scandir, config=\"../config/mpes_example_config.yaml\", system_config={}, metadata=metadata, collect_metadata=True)" ] }, { diff --git a/tutorial/4_hextof_workflow.ipynb b/tutorial/4_hextof_workflow.ipynb index 273e09e1..ad3d2e5c 100644 --- a/tutorial/4_hextof_workflow.ipynb +++ b/tutorial/4_hextof_workflow.ipynb @@ -103,7 +103,7 @@ "outputs": [], "source": [ "# pick the default configuration file for hextof@FLASH\n", - "config_file = Path('../sed/config/flash_example_config.yaml')\n", + "config_file = Path('../config/flash_example_config.yaml')\n", "assert config_file.exists()" ] }, diff --git a/tutorial/5_sxp_workflow.ipynb b/tutorial/5_sxp_workflow.ipynb index 713cef9c..b225a96e 100644 --- a/tutorial/5_sxp_workflow.ipynb +++ b/tutorial/5_sxp_workflow.ipynb @@ -85,7 +85,7 @@ "outputs": [], "source": [ "# pick the default configuration file for SXP@XFEL\n", - "config_file = Path('../sed/config/sxp_example_config.yaml')\n", + "config_file = Path('../config/sxp_example_config.yaml')\n", "assert config_file.exists()" ] }, diff --git a/tutorial/6_binning_with_time-stamped_data.ipynb b/tutorial/6_binning_with_time-stamped_data.ipynb index 7978568f..20b9e335 100644 --- a/tutorial/6_binning_with_time-stamped_data.ipynb +++ b/tutorial/6_binning_with_time-stamped_data.ipynb @@ -68,7 +68,7 @@ "outputs": [], "source": [ "# create sed processor using the config file with time-stamps:\n", - "sp = sed.SedProcessor(folder=scandir, user_config=\"../sed/config/mpes_example_config.yaml\", system_config={}, time_stamps=True, verbose=True)" + "sp = sed.SedProcessor(folder=scandir, user_config=\"../config/mpes_example_config.yaml\", system_config={}, time_stamps=True, verbose=True)" ] }, { diff --git a/tutorial/7_correcting_orthorhombic_symmetry.ipynb b/tutorial/7_correcting_orthorhombic_symmetry.ipynb index faba30b6..47bd1530 100644 --- a/tutorial/7_correcting_orthorhombic_symmetry.ipynb +++ b/tutorial/7_correcting_orthorhombic_symmetry.ipynb @@ -60,7 +60,7 @@ "outputs": [], "source": [ "# create sed processor using the config file with time-stamps:\n", - "sp = sed.SedProcessor(folder=scandir, user_config=\"../sed/config/mpes_example_config.yaml\", system_config={}, time_stamps=True, verbose=True)\n", + "sp = sed.SedProcessor(folder=scandir, user_config=\"../config/mpes_example_config.yaml\", system_config={}, time_stamps=True, verbose=True)\n", "sp.add_jitter()" ] }, diff --git a/tutorial/8_jittering_tutorial.ipynb b/tutorial/8_jittering_tutorial.ipynb index d98a19b2..5b94dd08 100644 --- a/tutorial/8_jittering_tutorial.ipynb +++ b/tutorial/8_jittering_tutorial.ipynb @@ -58,7 +58,7 @@ "outputs": [], "source": [ "# create sed processor using the config file:\n", - "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", system_config={})" + "sp = sed.SedProcessor(folder=scandir, config=\"../config/mpes_example_config.yaml\", system_config={})" ] }, { From 07cbc896316e65f51772b9ec839a070e033dd584 Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 15 Oct 2024 00:04:40 +0200 Subject: [PATCH 5/9] add config to build specs --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index fd670628..f579cce9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [tool.poetry] name = "sed-processor" packages = [{include = "sed", from = "src"}] +include = ["config"] version = "1.0.0a0" description = "Single Event Data Frame Processor: Backend to handle photoelectron resolved datastreams" authors = ["OpenCOMPES team "] From 757e19ef60c0e7f48dbff57ef6a7a2f03fe9d279 Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 15 Oct 2024 12:36:30 +0200 Subject: [PATCH 6/9] fix flash build script again --- docs/scripts/build_flash_parquets.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/scripts/build_flash_parquets.py b/docs/scripts/build_flash_parquets.py index 152df4de..0200bfd1 100644 --- a/docs/scripts/build_flash_parquets.py +++ b/docs/scripts/build_flash_parquets.py @@ -1,10 +1,12 @@ -from pathlib import Path +import os +from importlib.util import find_spec -import sed from sed import SedProcessor from sed.dataset import dataset -config_file = Path(sed.__file__).parent / "config/flash_example_config.yaml" +package_dir = os.path.dirname(find_spec("sed").origin) + +config_file = package_dir + "/../../config/flash_example_config.yaml" dataset.get("Gd_W110", root_dir="./tutorial") data_path = dataset.dir From 1499550b6b4aea41a135e74b9928040cc64f406a Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 15 Oct 2024 15:54:39 +0200 Subject: [PATCH 7/9] update path to Nexus config file --- config/mpes_example_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/mpes_example_config.yaml b/config/mpes_example_config.yaml index c45ca865..fe81c28f 100644 --- a/config/mpes_example_config.yaml +++ b/config/mpes_example_config.yaml @@ -310,4 +310,4 @@ nexus: definition: "NXmpes" # List containing additional input files to be handed to the pynxtools converter tool, # e.g. containing a configuration file, and additional metadata. - input_files: ["../sed/config/NXmpes_config.json"] + input_files: ["../config/NXmpes_config.json"] From a1010310fed622d410059e7255b57d2fe5de5719 Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 15 Oct 2024 16:38:11 +0200 Subject: [PATCH 8/9] fix tests --- tests/test_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_processor.py b/tests/test_processor.py index ca489f38..18a42756 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -1092,7 +1092,7 @@ def test_save(caplog) -> None: ] = 21.0 config["metadata"]["lens_mode_config"]["6kV_kmodem4.0_30VTOF_453ns_focus.sav"]["Z1"] = 2450 config["metadata"]["lens_mode_config"]["6kV_kmodem4.0_30VTOF_453ns_focus.sav"]["F"] = 69.23 - config["nexus"]["input_files"] = [package_dir + "/../sed/config/NXmpes_config.json"] + config["nexus"]["input_files"] = [package_dir + "/../../config/NXmpes_config.json"] processor = SedProcessor( folder=df_folder, config=config, From 0d1b805dee0d232ea333f6e78d354ccb3318d396 Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 22 Oct 2024 20:24:48 +0200 Subject: [PATCH 9/9] fix benchmarks --- benchmarks/benchmark_sed.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/benchmark_sed.py b/benchmarks/benchmark_sed.py index f181dec5..af016608 100644 --- a/benchmarks/benchmark_sed.py +++ b/benchmarks/benchmark_sed.py @@ -35,7 +35,7 @@ test_data_dir = os.path.join(package_dir, "..", "tests", "data") runs = {"generic": None, "mpes": ["30", "50"], "flash": ["43878"], "sxp": ["0016"]} -targets = load_config(package_dir + "/../benchmarks/benchmark_targets.yaml") +targets = load_config(package_dir + "/../../benchmarks/benchmark_targets.yaml") def test_binning_1d() -> None: @@ -59,7 +59,7 @@ def test_binning_1d() -> None: if np.mean(result) < 0.8 * targets["binning_1d"]: print(f"Updating targets for 'binning_1d' to {float(np.mean(result))}") targets["binning_1d"] = float(np.mean(result)) - save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml") + save_config(targets, package_dir + "/../../benchmarks/benchmark_targets.yaml") def test_binning_4d() -> None: @@ -83,14 +83,14 @@ def test_binning_4d() -> None: if np.mean(result) < 0.8 * targets["binning_4d"]: print(f"Updating targets for 'binning_4d' to {float(np.mean(result))}") targets["binning_4d"] = float(np.mean(result)) - save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml") + save_config(targets, package_dir + "/../../benchmarks/benchmark_targets.yaml") def test_splinewarp() -> None: """Run a benchmark for the generation of the inverse dfield correction""" processor = SedProcessor( dataframe=dataframe.copy(), - config=package_dir + "/config/mpes_example_config.yaml", + config=package_dir + "/../../config/mpes_example_config.yaml", folder_config={}, user_config={}, system_config={}, @@ -109,14 +109,14 @@ def test_splinewarp() -> None: if np.mean(result) < 0.8 * targets["inv_dfield"]: print(f"Updating targets for 'inv_dfield' to {float(np.mean(result))}") targets["inv_dfield"] = float(np.mean(result)) - save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml") + save_config(targets, package_dir + "/../../benchmarks/benchmark_targets.yaml") def test_workflow_1d() -> None: """Run a benchmark for 1d binning of converted data""" processor = SedProcessor( dataframe=dataframe.copy(), - config=package_dir + "/config/mpes_example_config.yaml", + config=package_dir + "/../../config/mpes_example_config.yaml", folder_config={}, user_config={}, system_config={}, @@ -145,14 +145,14 @@ def test_workflow_1d() -> None: if np.mean(result) < 0.8 * targets["workflow_1d"]: print(f"Updating targets for 'workflow_1d' to {float(np.mean(result))}") targets["workflow_1d"] = float(np.mean(result)) - save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml") + save_config(targets, package_dir + "/../../benchmarks/benchmark_targets.yaml") def test_workflow_4d() -> None: """Run a benchmark for 4d binning of converted data""" processor = SedProcessor( dataframe=dataframe.copy(), - config=package_dir + "/config/mpes_example_config.yaml", + config=package_dir + "/../../config/mpes_example_config.yaml", folder_config={}, user_config={}, system_config={}, @@ -181,7 +181,7 @@ def test_workflow_4d() -> None: if np.mean(result) < 0.8 * targets["workflow_4d"]: print(f"Updating targets for 'workflow_4d' to {float(np.mean(result))}") targets["workflow_4d"] = float(np.mean(result)) - save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml") + save_config(targets, package_dir + "/../../benchmarks/benchmark_targets.yaml") @pytest.mark.parametrize("loader", get_all_loaders()) @@ -210,4 +210,4 @@ def test_loader_compute(loader: BaseLoader) -> None: f"to {float(np.mean(result))}", ) targets[f"loader_compute_{loader_name}"] = float(np.mean(result)) - save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml") + save_config(targets, package_dir + "/../../benchmarks/benchmark_targets.yaml")