diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt index 0d832b1f..7e3d25e3 100644 --- a/.cspell/custom-dictionary.txt +++ b/.cspell/custom-dictionary.txt @@ -99,13 +99,17 @@ dtype dtypes easimon ecalibdict -electronanalyser +elab +elabapi +elabid +electronanalyzer Elektronen endstation energycal energycalfolder ENERGYDISPERSION ENOSPC +entityid equiscale Eref errorbar @@ -266,6 +270,7 @@ ontop OPCPA openmp OPTICALDELAY +orcid otherax packetcoders Pandoc @@ -340,6 +345,7 @@ sdir segs setp sfile +sharelink shutil Sixten sohail @@ -373,6 +379,7 @@ toctree tofseg tqdm traceseg +trarpes trseg Tsec txtsize @@ -385,6 +392,7 @@ ufunc unbinned uncategorised undoc +userid utime varnames venv diff --git a/docs/sed/loader.rst b/docs/sed/loader.rst index 468f34e4..b3d1f48a 100644 --- a/docs/sed/loader.rst +++ b/docs/sed/loader.rst @@ -29,6 +29,10 @@ MpesLoader :members: :undoc-members: +.. automodule:: sed.loader.mpes.metadata + :members: + :undoc-members: + FlashLoader ################################################### .. automodule:: sed.loader.flash.loader diff --git a/pyproject.toml b/pyproject.toml index cd832ea4..a6c549a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,9 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "bokeh>=2.4.2", + "bokeh>=2.4.2,<3.7.0", "dask>=2021.12.0,<2024.8", + "elabapi-python>=5.0", "fastdtw>=0.3.4", "h5py>=3.6.0", "ipympl>=0.9.1", @@ -43,8 +44,8 @@ dependencies = [ "pandas>=1.4.1", "photutils<2.0", "psutil>=5.9.0", - "pynxtools-mpes>=0.2.0", - "pynxtools>=0.9.0", + "pynxtools-mpes>=0.2.2", + "pynxtools>=0.10.1", "pyyaml>=6.0.0", "scipy>=1.8.0", "symmetrize>=0.5.5", @@ -95,8 +96,8 @@ all = [ [tool.coverage.report] omit = [ - "config.py", - "config-3.py", + "./config.py", + "./config-3.py", ] [tool.ruff] diff --git a/src/sed/calibrator/momentum.py b/src/sed/calibrator/momentum.py index 391270b0..fdb572b3 100644 --- a/src/sed/calibrator/momentum.py +++ b/src/sed/calibrator/momentum.py @@ -2004,9 +2004,9 @@ def gather_calibration_metadata(self, calibration: dict = None) -> dict: metadata["calibration"] = calibration # create empty calibrated axis entries, if they are not present. if "kx_axis" not in metadata["calibration"]: - metadata["calibration"]["kx_axis"] = 0 + metadata["calibration"]["kx_axis"] = 0.0 if "ky_axis" not in metadata["calibration"]: - metadata["calibration"]["ky_axis"] = 0 + metadata["calibration"]["ky_axis"] = 0.0 return metadata diff --git a/src/sed/config/NXmpes_config-HEXTOF.json b/src/sed/config/NXmpes_config-HEXTOF.json index 3bd38a41..8eca3e78 100755 --- a/src/sed/config/NXmpes_config-HEXTOF.json +++ b/src/sed/config/NXmpes_config-HEXTOF.json @@ -6,9 +6,7 @@ "/ENTRY[entry]/experiment_institution": "Deutsches Elektronen-Synchrotron DESY", "/ENTRY[entry]/experiment_facility": "Free-Electron Laser FLASH", "/ENTRY[entry]/experiment_laboratory": "@attrs:metadata/creationLocation", - "/ENTRY/entry_identifier": { - "identifier":"@attrs:metadata/pid" - }, + "/ENTRY/identifierNAME[entry_identifier]": "@attrs:metadata/pid", "/ENTRY[entry]/USER[user0]": { "name": "!@attrs:metadata/principalInvestigator", "role": "Principal investigator", @@ -48,7 +46,7 @@ "value/@units": "@attrs:metadata/scientificMetadata/sample/sample_pressure/unit" } }, - "/ENTRY[entry]/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]": { + "/ENTRY[entry]/INSTRUMENT[instrument]/ELECTRONANALYZER[electronanalyzer]": { "description": "HEXTOF Momentum Microscope", "device_information": { "vendor": "None", @@ -78,7 +76,7 @@ "type": "estimated" } }, - "/ENTRY[entry]/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]/COLLECTIONCOLUMN[collectioncolumn]": { + "/ENTRY[entry]/INSTRUMENT[instrument]/ELECTRONANALYZER[electronanalyzer]/COLLECTIONCOLUMN[collectioncolumn]": { "projection": "@attrs:metadata/scientificMetadata/Collection/projection", "scheme": "momentum dispersive", "lens_mode": "@attrs:metadata/scientificMetadata/Collection/lens_mode", @@ -107,19 +105,18 @@ } } }, - "/ENTRY[entry]/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]/ENERGYDISPERSION[energydispersion]": { + "/ENTRY[entry]/INSTRUMENT[instrument]/ELECTRONANALYZER[electronanalyzer]/ENERGYDISPERSION[energydispersion]": { "pass_energy": "@attrs:metadata/scientificMetadata/Collection/tof_voltage", "pass_energy/@units": "V", "scheme": "tof", "tof_distance": 0.8, "tof_distance/@units": "m" }, - "/ENTRY[entry]/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]/DETECTOR[detector]": { + "/ENTRY[entry]/INSTRUMENT[instrument]/ELECTRONANALYZER[electronanalyzer]/ELECTRON_DETECTOR[detector]": { "amplifier_type": "MCP", - "detector_type": "DLD", - "sensor_pixels": [2024, 2048] + "detector_type": "DLD" }, - "/ENTRY[entry]/INSTRUMENT[instrument]/sourceTYPE[source_probe]": { + "/ENTRY[entry]/INSTRUMENT[instrument]/source_probe": { "name": "@attrs:metadata/scientificMetadata/Source/name", "probe": "@attrs:metadata/scientificMetadata/Source/probe", "type": "@attrs:metadata/scientificMetadata/Source/type", @@ -128,7 +125,7 @@ "frequency/@units": "@attrs:metadata/scientificMetadata/Source/repetition_rate/unit", "associated_beam": "/entry/instrument/beam_probe" }, - "/ENTRY[entry]/INSTRUMENT[instrument]/beamTYPE[beam_probe]": { + "/ENTRY[entry]/INSTRUMENT[instrument]/beam_probe": { "distance": 0.0, "distance/@units": "mm", "incident_energy": "@attrs:metadata/scientificMetadata/Source/photon_energy/value", @@ -146,16 +143,16 @@ "energy_dispersion": "@attrs:metadata/scientificMetadata/Source/dispersion/value", "energy_dispersion/@units": "@attrs:metadata/scientificMetadata/Source/dispersion/unit" }, - "/ENTRY[entry]/INSTRUMENT[instrument]/sourceTYPE[source_pump]": { + "/ENTRY[entry]/INSTRUMENT[instrument]/source_pump": { "name": "PIGLET @ FLASH @ DESY", "probe": "visible light", "type": "Optical Laser", "mode": "Single Bunch", - "frequency": 1000, + "frequency": 1000.0, "frequency/@units": "kHz", "associated_beam": "/entry/instrument/beam_pump" }, - "/ENTRY[entry]/INSTRUMENT[instrument]/beamTYPE[beam_pump]": { + "/ENTRY[entry]/INSTRUMENT[instrument]/beam_pump": { "distance": 0.0, "distance/@units": "mm", "incident_wavelength": "@attrs:metadata/scientificMetadata/Laser/wavelength/value", @@ -195,7 +192,7 @@ "gas_pressure_env": { "pressure_gauge": "@link:/entry/instrument/pressure_gauge" }, - "bias": { + "bias_env": { "voltmeter": "@link:/entry/instrument/manipulator/sample_bias_voltmeter" } }, @@ -207,6 +204,6 @@ "data/@units": "counts", "AXISNAME[*]": "@data:*.data", "AXISNAME[*]/@units": "@data:*.unit", - "energy/@type": "@attrs:metadata/energy_calibration/calibration/energy_scale" + "AXISNAME[energy]/@type": "@attrs:metadata/energy_calibration/calibration/energy_scale" } } diff --git a/src/sed/config/NXmpes_config.json b/src/sed/config/NXmpes_config.json old mode 100755 new mode 100644 index 62110337..996be008 --- a/src/sed/config/NXmpes_config.json +++ b/src/sed/config/NXmpes_config.json @@ -1,43 +1,41 @@ { "/@default": "entry", "/ENTRY/@default": "data", - "/ENTRY/title": "['@eln:/ENTRY/title', '@attrs:metadata/entry_title']", + "/ENTRY/title": "['@eln:/ENTRY/title', '@attrs:metadata/entry_title', '@attrs:metadata/elabFTW/scan/title']", "/ENTRY/start_time": "@attrs:metadata/timing/acquisition_start", "/ENTRY/experiment_institution": "Fritz Haber Institute - Max Planck Society", "/ENTRY/experiment_facility": "Time Resolved ARPES", "/ENTRY/experiment_laboratory": "Clean Room 4", - "/ENTRY/entry_identifier": { - "identifier":"@attrs:metadata/entry_identifier" - }, + "/ENTRY/identifierNAME[entry_identifier]": "@attrs:metadata/entry_identifier", "/ENTRY/end_time": "@attrs:metadata/timing/acquisition_stop", "/ENTRY/duration": "@attrs:metadata/timing/acquisition_duration", "/ENTRY/duration/@units": "s", "/ENTRY/collection_time": "@attrs:metadata/timing/collection_time", "/ENTRY/collection_time/@units": "s", "/ENTRY/USER[user]": { - "name": "!['@eln:/ENTRY/User/name', '@attrs:metadata/user0/name']", - "role": "['@eln:/ENTRY/User/role', '@attrs:metadata/user0/role']", - "affiliation": "!['@eln:/ENTRY/User/affiliation', '@attrs:metadata/user0/affiliation']", - "address": "['@eln:/ENTRY/User/address', '@attrs:metadata/user0/address']", - "email": "['@eln:/ENTRY/User/email', '@attrs:metadata/user0/email']" + "name": "!['@eln:/ENTRY/User/name', '@attrs:metadata/user0/name', '@attrs:metadata/elabFTW/user/name']", + "role": "['@eln:/ENTRY/User/role', '@attrs:metadata/user0/role', 'Principal Investigator']", + "affiliation": "['@eln:/ENTRY/User/affiliation', '@attrs:metadata/user0/affiliation', 'Fritz Haber Institute of the Max Planck Society']", + "address": "['@eln:/ENTRY/User/address', '@attrs:metadata/user0/address', 'Faradayweg 4-6, 14195 Berlin, Germany']", + "email": "['@eln:/ENTRY/User/email', '@attrs:metadata/user0/email', '@attrs:metadata/elabFTW/user/email']" }, "/ENTRY/INSTRUMENT[instrument]": { "name": "Time-of-flight momentum microscope equipped delay line detector, at the endstation of the high rep-rate HHG source at FHI", "name/@short_name": "TR-ARPES @ FHI", "energy_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/energy_resolution']", + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/energy_resolution', '@attrs:metadata/elabFTW/trarpes_metis/energy_resolution']", "resolution/@units": "meV", "physical_quantity": "energy", "type": "estimated" }, "RESOLUTION[temporal_resolution]": { - "resolution": 35.0, + "resolution": "!['@attrs:metadata/elabFTW/laser_status/temporal_resolution', '35.0']", "resolution/@units": "fs", "physical_quantity": "time", "type": "estimated" }, "RESOLUTION[momentum_resolution]": { - "resolution": "@link:/entry/instrument/electronanalyser/momentum_resolution", + "resolution": "@link:/entry/instrument/electronanalyzer/momentum_resolution", "resolution/@units": "1/angstrom", "physical_quantity": "momentum", "type": "estimated" @@ -48,33 +46,33 @@ "value": "!['@eln:/ENTRY/Sample/gas_pressure', '@attrs:metadata/file/trARPES:XGS600:PressureAC:P_RD']", "value/@units": "mbar" }, - "ELECTRONANALYSER[electronanalyser]": { + "ELECTRONANALYZER[electronanalyzer]": { "description": "SPECS Metis 1000 Momentum Microscope", "device_information": { "vendor": "SPECS GmbH", "model": "Metis 1000 Momentum Microscope" }, "fast_axes": ["kx", "ky", "E"], - "slow_axes": "@attrs:metadata/instrument/analyzer/slow_axes", + "slow_axes": "['@attrs:metadata/instrument/analyzer/slow_axes', '@attrs:metadata/elabFTW/scan/scan_type']", "energy_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/analyzer/energy_resolution']", + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/analyzer/energy_resolution', '@attrs:metadata/elabFTW/trarpes_metis/energy_resolution']", "resolution/@units": "meV", "physical_quantity": "energy", "type": "estimated" }, "momentum_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/momentum_resolution', '@attrs:metadata/instrument/analyzer/momentum_resolution']", + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/momentum_resolution', '@attrs:metadata/instrument/analyzer/momentum_resolution', '@attrs:metadata/elabFTW/trarpes_metis/momentum_resolution']", "resolution/@units": "1/angstrom", "physical_quantity": "momentum", "type": "estimated" }, "spatial_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/spatial_resolution', '@attrs:metadata/instrument/analyzer/spatial_resolution']", + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/spatial_resolution', '@attrs:metadata/instrument/analyzer/spatial_resolution', '@attrs:metadata/elabFTW/trarpes_metis/spatial_resolution']", "resolution/@units": "µm", "physical_quantity": "length", "type": "estimated" }, - "depends_on": "/entry/instrument/electronanalyser/transformations/trans_z", + "depends_on": "/entry/instrument/electronanalyzer/transformations/trans_z", "TRANSFORMATIONS[transformations]": { "AXISNAME[trans_z]": 4.0, "AXISNAME[trans_z]/@depends_on": "rot_y", @@ -89,7 +87,7 @@ }, "COLLECTIONCOLUMN[collectioncolumn]": { "projection": "@attrs:metadata/instrument/analyzer/projection", - "scheme": "@attrs:metadata/instrument/analyzer/scheme", + "scheme": "['@attrs:metadata/instrument/analyzer/scheme', 'momentum dispersive']", "lens_mode": "@attrs:metadata/instrument/analyzer/lens_mode", "extractor_voltage": "@attrs:metadata/file/KTOF:Lens:Extr:V", "extractor_voltage/@units": "V", @@ -132,11 +130,9 @@ "tof_distance": 0.9, "tof_distance/@units": "m" }, - "DETECTOR[detector]": { + "ELECTRON_DETECTOR[detector]": { "amplifier_type": "MCP", "detector_type": "DLD", - "sensor_pixels": [1800, 1800], - "sensor_pixels/@units": "", "amplifier_bias": "@attrs:metadata/file/KTOF:Lens:MCPfront:V", "amplifier_bias/@units": "V", "amplifier_voltage": "@attrs:metadata/file/KTOF:Lens:MCPback:V", @@ -145,59 +141,59 @@ "detector_voltage/@units": "V" } }, - "sourceTYPE[source_probe]": { + "source_probe": { "name": "HHG @ TR-ARPES @ FHI", "probe": "photon", "type": "HHG laser", "mode": "Single Bunch", - "frequency": "['@eln:/ENTRY/Instrument/Source/Probe/frequency', '@attrs:metadata/instrument/beam/probe/frequency']", + "frequency": "!['@eln:/ENTRY/Instrument/Source/Probe/frequency', '@attrs:metadata/instrument/beam/probe/frequency', '@attrs:metadata/elabFTW/laser_status/probe_repetition_rate']", "frequency/@units": "kHz", "associated_beam": "/entry/instrument/beam_probe" }, - "beamTYPE[beam_probe]": { + "beam_probe": { "distance": 0.0, "distance/@units": "mm", - "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy', '@attrs:metadata/instrument/beam/probe/incident_energy']", + "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy', '@attrs:metadata/instrument/beam/probe/incident_energy', '@attrs:metadata/elabFTW/laser_status/probe_photon_energy']", "incident_energy/@units": "eV", - "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy_spread', '@attrs:metadata/instrument/beam/probe/incident_energy_spread']", + "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy_spread', '@attrs:metadata/instrument/beam/probe/incident_energy_spread', '@attrs:metadata/elabFTW/laser_status/probe_photon_energy_spread']", "incident_energy_spread/@units": "eV", - "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Probe/pulse_duration', '@attrs:metadata/instrument/beam/probe/pulse_duration']", + "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Probe/pulse_duration', '@attrs:metadata/instrument/beam/probe/pulse_duration', '@attrs:metadata/elabFTW/laser_status/probe_pulse_duration']", "pulse_duration/@units": "fs", - "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_polarization', '@attrs:metadata/instrument/beam/probe/incident_polarization']", + "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_polarization', '@attrs:metadata/instrument/beam/probe/incident_polarization', '@attrs:metadata/elabFTW/scan/probe_polarization']", "incident_polarization/@units": "V^2/mm^2", - "extent": "['@eln:/ENTRY/Instrument/Beam/Probe/extent', '@attrs:metadata/instrument/beam/probe/extent']", + "extent": "['@eln:/ENTRY/Instrument/Beam/Probe/extent', '@attrs:metadata/instrument/beam/probe/extent', '@attrs:metadata/elabFTW/laser_status/probe_profile']", "extent/@units": "µm", "associated_source": "/entry/instrument/source_probe" }, - "sourceTYPE[source_pump]": { + "source_pump": { "name": "OPCPA @ TR-ARPES @ FHI", "probe": "visible light", "type": "Optical Laser", "mode": "Single Bunch", - "frequency": "['@eln:/ENTRY/Instrument/Source/Pump/frequency', '@attrs:metadata/instrument/beam/pump/frequency']", + "frequency": "!['@eln:/ENTRY/Instrument/Source/Pump/frequency', '@attrs:metadata/instrument/beam/pump/frequency', '@attrs:metadata/elabFTW/laser_status/pump_repetition_rate']", "frequency/@units": "kHz", "associated_beam": "/entry/instrument/beam_pump" }, - "beamTYPE[beam_pump]": { + "beam_pump": { "distance": 0.0, "distance/@units": "mm", - "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy', '@attrs:metadata/instrument/beam/pump/incident_energy']", + "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy', '@attrs:metadata/instrument/beam/pump/incident_energy', '@attrs:metadata/elabFTW/laser_status/pump_photon_energy']", "incident_energy/@units": "eV", - "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy_spread', '@attrs:metadata/instrument/beam/pump/incident_energy_spread']", + "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy_spread', '@attrs:metadata/instrument/beam/pump/incident_energy_spread', '@attrs:metadata/elabFTW/laser_status/pump_photon_energy_spread']", "incident_energy_spread/@units": "eV", - "incident_wavelength": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_wavelength', '@attrs:metadata/instrument/beam/pump/incident_wavelength']", + "incident_wavelength": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_wavelength', '@attrs:metadata/instrument/beam/pump/incident_wavelength', '@attrs:metadata/file/trARPES:Orpheus:Wavelength']", "incident_wavelength/@units": "nm", - "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_duration', '@attrs:metadata/instrument/beam/pump/pulse_duration']", + "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_duration', '@attrs:metadata/instrument/beam/pump/pulse_duration', '@attrs:metadata/elabFTW/laser_status/pump_pulse_duration']", "pulse_duration/@units": "fs", - "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_polarization', '@attrs:metadata/instrument/beam/pump/incident_polarization']", + "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_polarization', '@attrs:metadata/instrument/beam/pump/incident_polarization', '@attrs:metadata/elabFTW/scan/pump_polarization']", "incident_polarization/@units": "V^2/mm^2", - "pulse_energy": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_energy', '@attrs:metadata/instrument/beam/pump/pulse_energy']", + "pulse_energy": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_energy', '@attrs:metadata/instrument/beam/pump/pulse_energy', '@attrs:metadata/elabFTW/scan/pump_pulse_energy']", "pulse_energy/@units": "µJ", - "average_power": "['@eln:/ENTRY/Instrument/Beam/Pump/average_power', '@attrs:metadata/instrument/beam/pump/average_power']", + "average_power": "['@eln:/ENTRY/Instrument/Beam/Pump/average_power', '@attrs:metadata/instrument/beam/pump/average_power', '@attrs:metadata/file/trARPES:Pump:Power.RBV']", "average_power/@units": "mW", - "extent": "['@eln:/ENTRY/Instrument/Beam/Pump/extent', '@attrs:metadata/instrument/beam/pump/extent']", + "extent": "['@eln:/ENTRY/Instrument/Beam/Pump/extent', '@attrs:metadata/instrument/beam/pump/extent', '@attrs:metadata/elabFTW/laser_status/pump_profile']", "extent/@units": "µm", - "fluence": "['@eln:/ENTRY/Instrument/Beam/Pump/fluence', '@attrs:metadata/instrument/beam/pump/fluence']", + "fluence": "['@eln:/ENTRY/Instrument/Beam/Pump/fluence', '@attrs:metadata/instrument/beam/pump/fluence', '@attrs:metadata/elabFTW/scan/pump_fluence']", "fluence/@units": "mJ/cm^2", "associated_source": "/entry/instrument/source_pump" }, @@ -235,13 +231,13 @@ } }, "/ENTRY/SAMPLE[sample]": { - "preparation_date": "['@eln:/ENTRY/Sample/preparation_date', '@attrs:metadata/sample/preparation_date']", - "history/notes/description": "['@eln:/ENTRY/Sample/sample_history', '@attrs:metadata/sample/sample_history']", - "history/notes/type": "text/plain", - "description": "['@eln:/ENTRY/Sample/description', '@attrs:metadata/sample/chemical_formula']", - "name": "['@eln:/ENTRY/Sample/name', '@attrs:metadata/sample/name']", + "preparation_date": "['@eln:/ENTRY/Sample/preparation_date', '@attrs:metadata/sample/preparation_date', '@attrs:metadata/elabFTW/sample/preparation_date']", + "history/sample_preparation/start_time": "['@eln:/ENTRY/Sample/preparation_date', '@attrs:metadata/sample/preparation_date', '@attrs:metadata/elabFTW/sample/preparation_date']", + "history/sample_preparation/description": "['@eln:/ENTRY/Sample/sample_history', '@attrs:metadata/sample/sample_history', '@attrs:metadata/elabFTW/sample/sample_history']", + "description": "['@eln:/ENTRY/Sample/description', '@attrs:metadata/sample/chemical_formula', '@attrs:metadata/elabFTW/sample/summary']", + "name": "['@eln:/ENTRY/Sample/name', '@attrs:metadata/sample/name', '@attrs:metadata/elabFTW/sample/title']", "situation": "vacuum", - "SUBSTANCE[substance]/molecular_formula_hill": "['@eln:/ENTRY/Sample/chemical_formula', '@attrs:metadata/sample/chemical_formula']", + "chemical_formula": "['@eln:/ENTRY/Sample/chemical_formula', '@attrs:metadata/sample/chemical_formula', '@attrs:metadata/elabFTW/sample/sample_formula']", "temperature_env": { "temperature_sensor": "@link:/entry/instrument/manipulator/temperature_sensor" }, @@ -290,7 +286,8 @@ "AXISNAME[trans_x]/@vector": [1, 0, 0] } }, - "/ENTRY/PROCESS_MPES[process]/DISTORTION[distortion]": { + "/ENTRY/DISTORTION[distortion]": { + "applied": "!@attrs:metadata/momentum_correction/correction/applied", "symmetry": "!@attrs:metadata/momentum_correction/correction/rotation_symmetry", "symmetry/@units": "", "original_centre": "@attrs:metadata/momentum_correction/correction/center_point", @@ -302,7 +299,8 @@ "rdeform_field": "@attrs:metadata/momentum_correction/correction/rdeform_field", "rdeform_field/@units": "" }, - "/ENTRY/PROCESS_MPES[process]/REGISTRATION[registration]": { + "/ENTRY/REGISTRATION[registration]": { + "applied": "!@attrs:metadata/momentum_correction/registration/applied", "depends_on": "/entry/process/registration/transformations/rot_z", "TRANSFORMATIONS[transformations]": { "AXISNAME[trans_x]": "@attrs:metadata/momentum_correction/registration/trans_x/value", @@ -323,39 +321,45 @@ "AXISNAME[rot_z]/@depends_on": "@attrs:metadata/momentum_correction/registration/rot_z/depends_on" } }, - "/ENTRY/PROCESS_MPES[process]/energy_calibration":{ - "coefficients": "@attrs:metadata/energy_calibration/calibration/coefficients", - "coefficients/@units": "", - "fit_function": "@attrs:metadata/energy_calibration/calibration/fit_function", + "/ENTRY/CALIBRATION[energy_calibration]":{ + "applied": "!@attrs:metadata/energy_calibration/applied", + "fit_formula_inputs/TERM[coefficients]": "@attrs:metadata/energy_calibration/calibration/coefficients", + "fit_formula_inputs/TERM[coefficients]/@units": "", + "fit_formula_description": "@attrs:metadata/energy_calibration/calibration/fit_function", "original_axis": "@attrs:metadata/energy_calibration/tof", - "original_axis/@units": "ns", + "original_axis/@units": "", "calibrated_axis": "@attrs:metadata/energy_calibration/calibration/axis", "calibrated_axis/@units": "eV", "physical_quantity": "energy" }, - "/ENTRY/PROCESS_MPES[process]/kx_calibration": { - "scaling": "@attrs:metadata/momentum_calibration/calibration/kx_scale", - "scaling/@units": "", + "/ENTRY/CALIBRATION[kx_calibration]": { + "applied": "!@attrs:metadata/momentum_calibration/applied", + "scaling_factor": "@attrs:metadata/momentum_calibration/calibration/kx_scale", + "scaling_factor/@units": "", "offset": "@attrs:metadata/momentum_calibration/calibration/x_center", "offset/@units": "", "calibrated_axis": "@attrs:metadata/momentum_calibration/calibration/kx_axis", "calibrated_axis/@units": "1/angstrom", "physical_quantity": "momentum" }, - "/ENTRY/PROCESS_MPES[process]/ky_calibration": { - "scaling": "@attrs:metadata/momentum_calibration/calibration/ky_scale", + "/ENTRY/CALIBRATION[ky_calibration]": { + "applied": "!@attrs:metadata/momentum_calibration/applied", + "scaling_factor": "@attrs:metadata/momentum_calibration/calibration/ky_scale", + "scaling_factor/@units": "", "offset": "@attrs:metadata/momentum_calibration/calibration/y_center", + "offset/@units": "", "calibrated_axis": "@attrs:metadata/momentum_calibration/calibration/ky_axis", - "calibrated_axis/@units": "Angstrom^-1" + "calibrated_axis/@units": "1/angstrom", + "physical_quantity": "momentum" }, "/ENTRY/data": { "@axes": "@data:dims", - "@*_indices": "@data:*.index", + "AXISNAME_indices[@*_indices]": "@data:*.index", "@signal": "data", "data": "@data:data", "data/@units": "counts", - "*": "@data:*.data", - "*/@units": "@data:*.unit", - "energy/@type": "@attrs:metadata/energy_calibration/calibration/energy_scale" + "AXISNAME[*]": "@data:*.data", + "AXISNAME[*]/@units": "@data:*.unit", + "AXISNAME[energy]/@type": "['@attrs:metadata/energy_calibration/calibration/energy_scale', 'kinetic']" } } diff --git a/src/sed/core/config_model.py b/src/sed/core/config_model.py index 6379b639..bca9f959 100644 --- a/src/sed/core/config_model.py +++ b/src/sed/core/config_model.py @@ -33,6 +33,7 @@ class CopyToolModel(BaseModel): source: DirectoryPath dest: DirectoryPath + use: Optional[bool] = None safety_margin: Optional[float] = None gid: Optional[int] = None scheduler: Optional[str] = None @@ -321,6 +322,7 @@ class OffsetColumn(BaseModel): class MetadataModel(BaseModel): model_config = ConfigDict(extra="forbid") + elab_url: Optional[HttpUrl] = None archiver_url: Optional[HttpUrl] = None epics_pvs: Optional[Sequence[str]] = None fa_in_channel: Optional[str] = None diff --git a/src/sed/core/processor.py b/src/sed/core/processor.py index 3ea0eda9..b968364e 100644 --- a/src/sed/core/processor.py +++ b/src/sed/core/processor.py @@ -38,8 +38,8 @@ from sed.io import to_tiff from sed.loader import CopyTool from sed.loader import get_loader -from sed.loader.mpes.loader import get_archiver_data from sed.loader.mpes.loader import MpesLoader +from sed.loader.mpes.metadata import get_archiver_data N_CPU = psutil.cpu_count() @@ -162,7 +162,9 @@ def __init__( verbose=self._verbose, ) - self.use_copy_tool = "copy_tool" in self._config["core"] + self.use_copy_tool = "copy_tool" in self._config["core"] and self._config["core"][ + "copy_tool" + ].pop("use", True) if self.use_copy_tool: try: self.ct = CopyTool( diff --git a/src/sed/loader/mpes/loader.py b/src/sed/loader/mpes/loader.py index e3e75cda..98fba9a6 100644 --- a/src/sed/loader/mpes/loader.py +++ b/src/sed/loader/mpes/loader.py @@ -8,13 +8,9 @@ import datetime import glob import io -import json import os from collections.abc import Sequence from typing import Any -from urllib.error import HTTPError -from urllib.error import URLError -from urllib.request import urlopen import dask import dask.array as da @@ -27,6 +23,7 @@ from sed.core.logging import set_verbosity from sed.core.logging import setup_logging from sed.loader.base.loader import BaseLoader +from sed.loader.mpes.metadata import MetadataRetriever # Configure logging @@ -224,7 +221,6 @@ def hdf5_to_timed_dataframe( electron_channels = [] column_names = [] - for name, channel in channels.items(): if channel["format"] == "per_electron": if channel["dataset_key"] in test_proc: @@ -468,16 +464,13 @@ def hdf5_to_timed_array( # Delayed array for loading an HDF5 file of reasonable size (e.g. < 1GB) h5file = load_h5_in_memory(h5filename) - # Read out groups: data_list = [] ms_marker = np.asarray(h5file[ms_markers_key]) for channel in channels: - timed_dataset = np.zeros_like(ms_marker) if channel["format"] == "per_electron": g_dataset = np.asarray(h5file[channel["dataset_key"]]) - for i, point in enumerate(ms_marker): - timed_dataset[i] = g_dataset[int(point) - 1] + timed_dataset = g_dataset[np.maximum(ms_marker - 1, 0)] else: raise ValueError( f"Invalid 'format':{channel['format']} for channel {channel['dataset_key']}.", @@ -580,34 +573,6 @@ def get_elapsed_time( return secs -def get_archiver_data( - archiver_url: str, - archiver_channel: str, - ts_from: float, - ts_to: float, -) -> tuple[np.ndarray, np.ndarray]: - """Extract time stamps and corresponding data from and EPICS archiver instance - - Args: - archiver_url (str): URL of the archiver data extraction interface - archiver_channel (str): EPICS channel to extract data for - ts_from (float): starting time stamp of the range of interest - ts_to (float): ending time stamp of the range of interest - - Returns: - tuple[np.ndarray, np.ndarray]: The extracted time stamps and corresponding data - """ - iso_from = datetime.datetime.utcfromtimestamp(ts_from).isoformat() - iso_to = datetime.datetime.utcfromtimestamp(ts_to).isoformat() - req_str = archiver_url + archiver_channel + "&from=" + iso_from + "Z&to=" + iso_to + "Z" - with urlopen(req_str) as req: - data = json.load(req) - secs = [x["secs"] + x["nanos"] * 1e-9 for x in data[0]["data"]] - vals = [x["val"] for x in data[0]["data"]] - - return (np.asarray(secs), np.asarray(vals)) - - class MpesLoader(BaseLoader): """Mpes implementation of the Loader. Reads from h5 files or folders of the SPECS Metis 1000 (FHI Berlin) @@ -729,6 +694,7 @@ def read_dataframe( metadata=metadata, ) + token = kwds.pop("token", None) channels = kwds.pop( "channels", self._config.get("dataframe", {}).get("channels", None), @@ -777,6 +743,7 @@ def read_dataframe( metadata = self.gather_metadata( files=self.files, metadata=self.metadata, + token=token, ) else: metadata = self.metadata @@ -821,6 +788,14 @@ def get_files_from_run_id( recursive=True, ), ) + # Compatibility for old scan format + if not run_files: + run_files = natsorted( + glob.glob( + folder + "/**/Scan" + str(run_id).zfill(3) + "_*." + extension, + recursive=True, + ), + ) files.extend(run_files) # Check if any files are found @@ -877,6 +852,7 @@ def gather_metadata( self, files: Sequence[str], metadata: dict = None, + token: str = None, ) -> dict: """Collect meta data from files @@ -884,6 +860,7 @@ def gather_metadata( files (Sequence[str]): List of files loaded metadata (dict, optional): Manual meta data dictionary. Auto-generated meta data are added to it. Defaults to None. + token (str, optional):: The elabFTW api token to use for fetching metadata Returns: dict: The completed metadata dictionary. @@ -921,140 +898,21 @@ def gather_metadata( os.path.realpath(files[0]), ) - logger.info("Collecting data from the EPICS archive...") - # Get metadata from Epics archive if not present already - epics_channels = self._config["metadata"]["epics_pvs"] - - start = datetime.datetime.utcfromtimestamp(ts_from) + metadata_retriever = MetadataRetriever(self._config["metadata"], token) - channels_missing = set(epics_channels) - set( - metadata["file"].keys(), + metadata = metadata_retriever.fetch_epics_metadata( + ts_from=ts_from, + ts_to=ts_to, + metadata=metadata, ) - for channel in channels_missing: - try: - _, vals = get_archiver_data( - archiver_url=str(self._config["metadata"].get("archiver_url")), - archiver_channel=channel, - ts_from=ts_from, - ts_to=ts_to, - ) - metadata["file"][f"{channel}"] = np.mean(vals) - except IndexError: - metadata["file"][f"{channel}"] = np.nan - logger.info( - f"Data for channel {channel} doesn't exist for time {start}", - ) - except HTTPError as exc: - logger.warning( - f"Incorrect URL for the archive channel {channel}. " - "Make sure that the channel name and file start and end times are " - "correct.", - ) - logger.warning(f"Error code: {exc}") - except URLError as exc: - logger.warning( - f"Cannot access the archive URL for channel {channel}. " - f"Make sure that you are within the FHI network." - f"Skipping over channels {channels_missing}.", - ) - logger.warning(f"Error code: {exc}") - break - - # Determine the correct aperture_config - stamps = sorted( - list(self._config["metadata"]["aperture_config"].keys()) + [start], - ) - current_index = stamps.index(start) - timestamp = stamps[current_index - 1] # pick last configuration before file date - - # Aperture metadata - if "instrument" not in metadata.keys(): - metadata["instrument"] = {"analyzer": {}} - metadata["instrument"]["analyzer"]["fa_shape"] = "circle" - metadata["instrument"]["analyzer"]["ca_shape"] = "circle" - metadata["instrument"]["analyzer"]["fa_size"] = np.nan - metadata["instrument"]["analyzer"]["ca_size"] = np.nan - # get field aperture shape and size - if { - self._config["metadata"]["fa_in_channel"], - self._config["metadata"]["fa_hor_channel"], - }.issubset(set(metadata["file"].keys())): - fa_in = metadata["file"][self._config["metadata"]["fa_in_channel"]] - fa_hor = metadata["file"][self._config["metadata"]["fa_hor_channel"]] - for key, value in self._config["metadata"]["aperture_config"][timestamp][ - "fa_size" - ].items(): - if value[0][0] < fa_in < value[0][1] and value[1][0] < fa_hor < value[1][1]: - try: - k_float = float(key) - metadata["instrument"]["analyzer"]["fa_size"] = k_float - except ValueError: # store string if numeric interpretation fails - metadata["instrument"]["analyzer"]["fa_shape"] = key - break - else: - logger.warning("Field aperture size not found.") - - # get contrast aperture shape and size - if self._config["metadata"]["ca_in_channel"] in metadata["file"]: - ca_in = metadata["file"][self._config["metadata"]["ca_in_channel"]] - for key, value in self._config["metadata"]["aperture_config"][timestamp][ - "ca_size" - ].items(): - if value[0] < ca_in < value[1]: - try: - k_float = float(key) - metadata["instrument"]["analyzer"]["ca_size"] = k_float - except ValueError: # store string if numeric interpretation fails - metadata["instrument"]["analyzer"]["ca_shape"] = key - break - else: - logger.warning("Contrast aperture size not found.") - - # Storing the lens modes corresponding to lens voltages. - # Use lens voltages present in first lens_mode entry. - lens_list = self._config["metadata"]["lens_mode_config"][ - next(iter(self._config["metadata"]["lens_mode_config"])) - ].keys() - - lens_volts = np.array( - [metadata["file"].get(f"KTOF:Lens:{lens}:V", np.nan) for lens in lens_list], - ) - for mode, value in self._config["metadata"]["lens_mode_config"].items(): - lens_volts_config = np.array([value[k] for k in lens_list]) - if np.allclose( - lens_volts, - lens_volts_config, - rtol=0.005, - ): # Equal upto 0.5% tolerance - metadata["instrument"]["analyzer"]["lens_mode"] = mode - break - else: - logger.warning( - "Lens mode for given lens voltages not found. " - "Storing lens mode from the user, if provided.", - ) - - # Determining projection from the lens mode - try: - lens_mode = metadata["instrument"]["analyzer"]["lens_mode"] - if "spatial" in lens_mode.split("_")[1]: - metadata["instrument"]["analyzer"]["projection"] = "real" - metadata["instrument"]["analyzer"]["scheme"] = "momentum dispersive" - else: - metadata["instrument"]["analyzer"]["projection"] = "reciprocal" - metadata["instrument"]["analyzer"]["scheme"] = "spatial dispersive" - except IndexError: - logger.warning( - "Lens mode must have the form, '6kV_kmodem4.0_20VTOF_v3.sav'. " - "Can't determine projection. " - "Storing projection from the user, if provided.", - ) - except KeyError: - logger.warning( - "Lens mode not found. Can't determine projection. " - "Storing projection from the user, if provided.", + if self.runs: + metadata = metadata_retriever.fetch_elab_metadata( + runs=self.runs, + metadata=metadata, ) + else: + logger.warning('Fetching elabFTW metadata only supported for loading from "runs"') return metadata diff --git a/src/sed/loader/mpes/metadata.py b/src/sed/loader/mpes/metadata.py new file mode 100644 index 00000000..06bf90ad --- /dev/null +++ b/src/sed/loader/mpes/metadata.py @@ -0,0 +1,449 @@ +""" +The module provides a MetadataRetriever class for retrieving metadata +from an EPICS archiver and an elabFTW instance. +""" +from __future__ import annotations + +import datetime +import json +from copy import deepcopy +from urllib.error import HTTPError +from urllib.error import URLError +from urllib.request import urlopen + +import elabapi_python +import numpy as np + +from sed.core.config import read_env_var +from sed.core.config import save_env_var +from sed.core.logging import setup_logging + +logger = setup_logging("mpes_metadata_retriever") + + +class MetadataRetriever: + """ + A class for retrieving metadata from an EPICS archiver and an elabFTW instance. + """ + + def __init__(self, metadata_config: dict, token: str = None) -> None: + """ + Initializes the MetadataRetriever class. + + Args: + metadata_config (dict): Takes a dict containing at least url for the EPICS archiver and + elabFTW instance. + token (str, optional): The token to use for fetching metadata. If provided, + will be saved to .env file for future use. + """ + self._config = deepcopy(metadata_config) + # Token handling + if token: + self.token = token + save_env_var("ELAB_TOKEN", self.token) + else: + # Try to load token from config or .env file + self.token = read_env_var("ELAB_TOKEN") + + if not self.token: + logger.warning( + "No valid token provided for elabFTW. Fetching elabFTW metadata will be skipped.", + ) + return + + self.url = self._config.get("elab_url") + if not self.url: + logger.warning( + "No URL provided for elabFTW. Fetching elabFTW metadata will be skipped.", + ) + return + + # Config + self.configuration = elabapi_python.Configuration() + self.configuration.api_key["api_key"] = self.token + self.configuration.api_key_prefix["api_key"] = "Authorization" + self.configuration.host = str(self.url) + self.configuration.debug = False + self.configuration.verify_ssl = False + + # create an instance of the API class + self.api_client = elabapi_python.ApiClient(self.configuration) + # fix issue with Authorization header not being properly set by the generated lib + self.api_client.set_default_header(header_name="Authorization", header_value=self.token) + + # create an instance of Items + self.itemsApi = elabapi_python.ItemsApi(self.api_client) + self.experimentsApi = elabapi_python.ExperimentsApi(self.api_client) + self.linksApi = elabapi_python.LinksToItemsApi(self.api_client) + self.experimentsLinksApi = elabapi_python.LinksToExperimentsApi(self.api_client) + self.usersApi = elabapi_python.UsersApi(self.api_client) + + def fetch_epics_metadata(self, ts_from: float, ts_to: float, metadata: dict) -> dict: + """Fetch metadata from an EPICS archiver instance for times between ts_from and ts_to. + Channels are defined in the config. + + Args: + ts_from (float): Start timestamp of the range to collect data from. + ts_to (float): End timestamp of the range to collect data from. + metadata (dict): Input metadata dictionary. Will be updated + + Returns: + dict: Updated metadata dictionary. + """ + if not self._config.get("archiver_url"): + logger.warning( + "No URL provided for fetching metadata from the EPICS archiver. " + "Fetching EPICS metadata will be skipped.", + ) + return metadata + + logger.info("Collecting data from the EPICS archive...") + + start = datetime.datetime.utcfromtimestamp(ts_from) + + # Get metadata from Epics archive if not present already + epics_channels = self._config["epics_pvs"] + + channels_missing = set(epics_channels) - set( + metadata["file"].keys(), + ) + for channel in channels_missing: + try: + _, vals = get_archiver_data( + archiver_url=str(self._config.get("archiver_url")), + archiver_channel=channel, + ts_from=ts_from, + ts_to=ts_to, + ) + metadata["file"][f"{channel}"] = np.mean(vals) + + except IndexError: + logger.info( + f"Data for channel {channel} doesn't exist for time {start}", + ) + except HTTPError as exc: + logger.warning( + f"Incorrect URL for the archive channel {channel}. " + "Make sure that the channel name and file start and end times are " + "correct.", + ) + logger.warning(f"Error code: {exc}") + except URLError as exc: + logger.warning( + f"Cannot access the archive URL for channel {channel}. " + f"Make sure that you are within the FHI network." + f"Skipping over channels {channels_missing}.", + ) + logger.warning(f"Error code: {exc}") + break + + # Determine the correct aperture_config + stamps = sorted( + list(self._config["aperture_config"].keys()) + [start], + ) + current_index = stamps.index(start) + timestamp = stamps[current_index - 1] # pick last configuration before file date + + # Aperture metadata + if "instrument" not in metadata.keys(): + metadata["instrument"] = {"analyzer": {}} + metadata["instrument"]["analyzer"]["fa_shape"] = "circle" + metadata["instrument"]["analyzer"]["ca_shape"] = "circle" + metadata["instrument"]["analyzer"]["fa_size"] = np.nan + metadata["instrument"]["analyzer"]["ca_size"] = np.nan + # get field aperture shape and size + if { + self._config["fa_in_channel"], + self._config["fa_hor_channel"], + }.issubset(set(metadata["file"].keys())): + fa_in = metadata["file"][self._config["fa_in_channel"]] + fa_hor = metadata["file"][self._config["fa_hor_channel"]] + for key, value in self._config["aperture_config"][timestamp]["fa_size"].items(): + if value[0][0] < fa_in < value[0][1] and value[1][0] < fa_hor < value[1][1]: + try: + metadata["instrument"]["analyzer"]["fa_size"] = float(key) + except ValueError: # store string if numeric interpretation fails + metadata["instrument"]["analyzer"]["fa_shape"] = key + break + else: + logger.warning("Field aperture size not found.") + + # get contrast aperture shape and size + if self._config["ca_in_channel"] in metadata["file"]: + ca_in = metadata["file"][self._config["ca_in_channel"]] + for key, value in self._config["aperture_config"][timestamp]["ca_size"].items(): + if value[0] < ca_in < value[1]: + try: + metadata["instrument"]["analyzer"]["ca_size"] = float(key) + except ValueError: # store string if numeric interpretation fails + metadata["instrument"]["analyzer"]["ca_shape"] = key + break + else: + logger.warning("Contrast aperture size not found.") + + # Storing the lens modes corresponding to lens voltages. + # Use lens voltages present in first lens_mode entry. + lens_list = self._config["lens_mode_config"][ + next(iter(self._config["lens_mode_config"])) + ].keys() + + lens_volts = np.array( + [metadata["file"].get(f"KTOF:Lens:{lens}:V", np.nan) for lens in lens_list], + ) + for mode, value in self._config["lens_mode_config"].items(): + lens_volts_config = np.array([value[k] for k in lens_list]) + if np.allclose( + lens_volts, + lens_volts_config, + rtol=0.005, + ): # Equal upto 0.5% tolerance + metadata["instrument"]["analyzer"]["lens_mode"] = mode + break + else: + logger.warning( + "Lens mode for given lens voltages not found. " + "Storing lens mode from the user, if provided.", + ) + + # Determining projection from the lens mode + try: + lens_mode = metadata["instrument"]["analyzer"]["lens_mode"] + if "spatial" in lens_mode.split("_")[1]: + metadata["instrument"]["analyzer"]["projection"] = "real" + metadata["instrument"]["analyzer"]["scheme"] = "spatial dispersive" + else: + metadata["instrument"]["analyzer"]["projection"] = "reciprocal" + metadata["instrument"]["analyzer"]["scheme"] = "momentum dispersive" + except IndexError: + logger.warning( + "Lens mode must have the form, '6kV_kmodem4.0_20VTOF_v3.sav'. " + "Can't determine projection. " + "Storing projection from the user, if provided.", + ) + except KeyError: + logger.warning( + "Lens mode not found. Can't determine projection. " + "Storing projection from the user, if provided.", + ) + + return metadata + + def fetch_elab_metadata(self, runs: list[str], metadata: dict) -> dict: + """Fetch metadata from an elabFTW instance + + Args: + runs (list[str]): List of runs for which to fetch metadata + metadata (dict): Input metadata dictionary. Will be updated + + Returns: + dict: Updated metadata dictionary + """ + if not self.token: + logger.warning( + "No valid token found. Token is required for metadata collection. Either provide " + "a token parameter or set the ELAB_TOKEN environment variable.", + ) + return metadata + + if not self.url: + logger.warning( + "No URL provided for fetching metadata from elabFTW. " + "Fetching elabFTW metadata will be skipped.", + ) + return metadata + + logger.info("Collecting data from the elabFTW instance...") + # Get the experiment + try: + experiment = self.experimentsApi.read_experiments(q=f"'Metis scan {runs[0]}'")[0] + except IndexError: + logger.warning(f"No elabFTW entry found for run {runs[0]}") + return metadata + + if "elabFTW" not in metadata: + metadata["elabFTW"] = {} + + exp_id = experiment.id + # Get user information + user = self.usersApi.read_user(experiment.userid) + metadata["elabFTW"]["user"] = {} + metadata["elabFTW"]["user"]["name"] = user.fullname + metadata["elabFTW"]["user"]["email"] = user.email + metadata["elabFTW"]["user"]["id"] = user.userid + if user.orcid: + metadata["elabFTW"]["user"]["orcid"] = user.orcid + # Get the links to items + links = self.linksApi.read_entity_items_links(entity_type="experiments", id=exp_id) + # Get the items + items = [self.itemsApi.get_item(link.entityid) for link in links] + items_dict = {item.category_title: item for item in items} + items_dict["scan"] = experiment + + # Sort the metadata + for category, item in items_dict.items(): + category = category.replace(":", "").replace(" ", "_").lower() + if category not in metadata["elabFTW"]: + metadata["elabFTW"][category] = {} + metadata["elabFTW"][category]["title"] = item.title + metadata["elabFTW"][category]["summary"] = item.body + metadata["elabFTW"][category]["id"] = item.id + metadata["elabFTW"][category]["elabid"] = item.elabid + if item.sharelink: + metadata["elabFTW"][category]["link"] = item.sharelink + if item.metadata is not None: + metadata_json = json.loads(item.metadata) + for key, val in metadata_json["extra_fields"].items(): + if val["value"] is not None and val["value"] != "" and val["value"] != ["None"]: + try: + metadata["elabFTW"][category][key] = float(val["value"]) + except ValueError: + metadata["elabFTW"][category][key] = val["value"] + + # group beam profiles: + if ( + "laser_status" in metadata["elabFTW"] + and "pump_profile_x" in metadata["elabFTW"]["laser_status"] + and "pump_profile_y" in metadata["elabFTW"]["laser_status"] + ): + metadata["elabFTW"]["laser_status"]["pump_profile"] = [ + float(metadata["elabFTW"]["laser_status"]["pump_profile_x"]), + float(metadata["elabFTW"]["laser_status"]["pump_profile_y"]), + ] + if ( + "laser_status" in metadata["elabFTW"] + and "probe_profile_x" in metadata["elabFTW"]["laser_status"] + and "probe_profile_y" in metadata["elabFTW"]["laser_status"] + ): + metadata["elabFTW"]["laser_status"]["probe_profile"] = [ + float(metadata["elabFTW"]["laser_status"]["probe_profile_x"]), + float(metadata["elabFTW"]["laser_status"]["probe_profile_y"]), + ] + + # fix preparation date + if "sample" in metadata["elabFTW"] and "preparation_date" in metadata["elabFTW"]["sample"]: + metadata["elabFTW"]["sample"]["preparation_date"] = ( + datetime.datetime.strptime( + metadata["elabFTW"]["sample"]["preparation_date"], + "%Y-%m-%d", + ) + .replace(tzinfo=datetime.timezone.utc) + .isoformat() + ) + + # fix polarizations + if ( + "scan" in metadata["elabFTW"] + and "pump_polarization" in metadata["elabFTW"]["scan"] + and isinstance(metadata["elabFTW"]["scan"]["pump_polarization"], str) + ): + if metadata["elabFTW"]["scan"]["pump_polarization"] == "s": + metadata["elabFTW"]["scan"]["pump_polarization"] = 90 + elif metadata["elabFTW"]["scan"]["pump_polarization"] == "p": + metadata["elabFTW"]["scan"]["pump_polarization"] = 0 + else: + try: + metadata["elabFTW"]["scan"]["pump_polarization"] = float( + metadata["elabFTW"]["scan"]["pump_polarization"], + ) + except ValueError: + pass + + if ( + "scan" in metadata["elabFTW"] + and "probe_polarization" in metadata["elabFTW"]["scan"] + and isinstance(metadata["elabFTW"]["scan"]["probe_polarization"], str) + ): + if metadata["elabFTW"]["scan"]["probe_polarization"] == "s": + metadata["elabFTW"]["scan"]["probe_polarization"] = 90 + elif metadata["elabFTW"]["scan"]["probe_polarization"] == "p": + metadata["elabFTW"]["scan"]["probe_polarization"] = 0 + else: + try: + metadata["elabFTW"]["scan"]["probe_polarization"] = float( + metadata["elabFTW"]["scan"]["probe_polarization"], + ) + except ValueError: + pass + + if ( + "scan" in metadata["elabFTW"] + and "pump2_polarization" in metadata["elabFTW"]["scan"] + and isinstance(metadata["elabFTW"]["scan"]["pump2_polarization"], str) + ): + if metadata["elabFTW"]["scan"]["pump2_polarization"] == "s": + metadata["elabFTW"]["scan"]["pump2_polarization"] = 90 + elif metadata["elabFTW"]["scan"]["pump2_polarization"] == "p": + metadata["elabFTW"]["scan"]["pump2_polarization"] = 0 + else: + try: + metadata["elabFTW"]["scan"]["pump2_polarization"] = float( + metadata["elabFTW"]["scan"]["pump2_polarization"], + ) + except ValueError: + pass + + # fix pump status + if "scan" in metadata["elabFTW"] and "pump_status" in metadata["elabFTW"]["scan"]: + try: + metadata["elabFTW"]["scan"]["pump_status"] = ( + "open" if int(metadata["elabFTW"]["scan"]["pump_status"]) else "closed" + ) + except ValueError: + pass + if "scan" in metadata["elabFTW"] and "pump2_status" in metadata["elabFTW"]["scan"]: + try: + metadata["elabFTW"]["scan"]["pump2_status"] = ( + "open" if int(metadata["elabFTW"]["scan"]["pump2_status"]) else "closed" + ) + except ValueError: + pass + + # remove pump information if pump not applied: + if metadata["elabFTW"]["scan"].get("pump_status", "closed") == "closed": + if "pump_photon_energy" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump_photon_energy"] + if "pump_repetition_rate" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump_repetition_rate"] + else: + # add pulse energy if applicable + try: + metadata["elabFTW"]["scan"]["pump_pulse_energy"] = ( + metadata["file"]["trARPES:Pump:Power.RBV"] + / metadata["elabFTW"]["laser_status"]["pump_repetition_rate"] + ) + except KeyError: + pass + + if metadata["elabFTW"]["scan"].get("pump2_status", "closed") == "closed": + if "pump2_photon_energy" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump2_photon_energy"] + + return metadata + + +def get_archiver_data( + archiver_url: str, + archiver_channel: str, + ts_from: float, + ts_to: float, +) -> tuple[np.ndarray, np.ndarray]: + """Extract time stamps and corresponding data from and EPICS archiver instance + + Args: + archiver_url (str): URL of the archiver data extraction interface + archiver_channel (str): EPICS channel to extract data for + ts_from (float): starting time stamp of the range of interest + ts_to (float): ending time stamp of the range of interest + + Returns: + tuple[np.ndarray, np.ndarray]: The extracted time stamps and corresponding data + """ + iso_from = datetime.datetime.utcfromtimestamp(ts_from).isoformat() + iso_to = datetime.datetime.utcfromtimestamp(ts_to).isoformat() + req_str = archiver_url + archiver_channel + "&from=" + iso_from + "Z&to=" + iso_to + "Z" + with urlopen(req_str) as req: + data = json.load(req) + secs = [x["secs"] + x["nanos"] * 1e-9 for x in data[0]["data"]] + vals = [x["val"] for x in data[0]["data"]] + + return (np.asarray(secs), np.asarray(vals)) diff --git a/tests/loader/mpes/test_mpes_metadata.py b/tests/loader/mpes/test_mpes_metadata.py new file mode 100644 index 00000000..83003e40 --- /dev/null +++ b/tests/loader/mpes/test_mpes_metadata.py @@ -0,0 +1,180 @@ +"""Tests specific for Mpes loader metadata retrieval""" +from __future__ import annotations + +import datetime +import json +from unittest.mock import MagicMock +from unittest.mock import patch + +import numpy as np +import pytest + +from sed.loader.mpes.metadata import get_archiver_data +from sed.loader.mpes.metadata import MetadataRetriever +from tests.test_config import mock_env_file # noqa: F401 + + +@pytest.fixture +def metadata_config(): + return { + "elab_url": "http://example.com", + "epics_pvs": ["channel1"], + "archiver_url": "http://archiver.example.com", + "aperture_config": { + datetime.datetime.fromisoformat("2023-01-01T00:00:00"): { + "fa_size": {"1.0": [(0, 1), (0, 1)]}, + "ca_size": {"1.0": (0, 1)}, + }, + }, + "lens_mode_config": {"mode1": {"lens1": 1.0, "lens2": 2.0}}, + "fa_in_channel": "fa_in", + "fa_hor_channel": "fa_hor", + "ca_in_channel": "ca_in", + } + + +@pytest.fixture +def metadata_retriever(metadata_config, mock_env_file): # noqa: ARG001 + return MetadataRetriever(metadata_config, "dummy_token") + + +def test_metadata_retriever_init(metadata_retriever): + assert metadata_retriever.token == "dummy_token" + assert metadata_retriever.url == "http://example.com" + + +def test_metadata_retriever_no_token(metadata_config, tmp_path, monkeypatch): + monkeypatch.setattr("sed.core.config.ENV_DIR", tmp_path / ".dummy_env") + monkeypatch.setattr("sed.core.config.SYSTEM_CONFIG_PATH", tmp_path / "system") + monkeypatch.setattr("sed.core.config.USER_CONFIG_PATH", tmp_path / "user") + retriever = MetadataRetriever(metadata_config) + assert retriever.token is None + + metadata = {} + runs = ["run1"] + updated_metadata = retriever.fetch_elab_metadata(runs, metadata) + assert updated_metadata == metadata + + +def test_metadata_retriever_no_url(metadata_config, mock_env_file): # noqa: ARG001 + metadata_config.pop("elab_url") + retriever = MetadataRetriever(metadata_config, "dummy_token") + assert retriever.url is None + + metadata = {} + runs = ["run1"] + updated_metadata = retriever.fetch_elab_metadata(runs, metadata) + assert updated_metadata == metadata + + +@patch("sed.loader.mpes.metadata.urlopen") +def test_get_archiver_data(mock_urlopen): + """Test get_archiver_data using a mock of urlopen.""" + mock_response = MagicMock() + mock_response.read.return_value = json.dumps( + [{"data": [{"secs": 1, "nanos": 500000000, "val": 10}]}], + ) + mock_urlopen.return_value.__enter__.return_value = mock_response + + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + archiver_url = "http://archiver.example.com" + archiver_channel = "channel1" + + secs, vals = get_archiver_data(archiver_url, archiver_channel, ts_from, ts_to) + + assert np.array_equal(secs, np.array([1.5])) + assert np.array_equal(vals, np.array([10])) + + +@patch("sed.loader.mpes.metadata.get_archiver_data") +def test_fetch_epics_metadata(mock_get_archiver_data, metadata_retriever): + """Test fetch_epics_metadata using a mock of get_archiver_data.""" + mock_get_archiver_data.return_value = (np.array([1.5]), np.array([10])) + metadata = {"file": {}} + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + + updated_metadata = metadata_retriever.fetch_epics_metadata(ts_from, ts_to, metadata) + + assert updated_metadata["file"]["channel1"] == 10 + + +@patch("sed.loader.mpes.metadata.get_archiver_data") +def test_fetch_epics_metadata_missing_channels(mock_get_archiver_data, metadata_retriever): + """Test fetch_epics_metadata with missing EPICS channels.""" + mock_get_archiver_data.return_value = (np.array([1.5]), np.array([10])) + metadata = {"file": {"channel1": 10}} + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + + updated_metadata = metadata_retriever.fetch_epics_metadata(ts_from, ts_to, metadata) + + assert "channel1" in updated_metadata["file"] + + +@patch("sed.loader.mpes.metadata.get_archiver_data") +def test_fetch_epics_metadata_missing_aperture_config(mock_get_archiver_data, metadata_retriever): + """Test fetch_epics_metadata with missing aperture configuration.""" + mock_get_archiver_data.return_value = (np.array([1.5]), np.array([10])) + metadata = {"file": {}} + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + metadata_retriever._config["aperture_config"] = {} + + updated_metadata = metadata_retriever.fetch_epics_metadata(ts_from, ts_to, metadata) + + assert "instrument" in updated_metadata + + +@patch("sed.loader.mpes.metadata.get_archiver_data") +def test_fetch_epics_metadata_missing_field_aperture(mock_get_archiver_data, metadata_retriever): + """Test fetch_epics_metadata with missing field aperture shape and size.""" + mock_get_archiver_data.return_value = (np.array([1.5]), np.array([10])) + metadata = {"file": {}} + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + + updated_metadata = metadata_retriever.fetch_epics_metadata(ts_from, ts_to, metadata) + + assert updated_metadata["instrument"]["analyzer"]["fa_shape"] == "circle" + assert updated_metadata["instrument"]["analyzer"]["ca_shape"] == "circle" + assert np.isnan(updated_metadata["instrument"]["analyzer"]["fa_size"]) + assert np.isnan(updated_metadata["instrument"]["analyzer"]["ca_size"]) + + +@patch("sed.loader.mpes.metadata.elabapi_python") +def test_fetch_elab_metadata(mock_elabapi_python, metadata_config, mock_env_file): # noqa: ARG001 + """Test fetch_elab_metadata using a mock of elabapi_python.""" + mock_experiment = MagicMock() + mock_experiment.id = 1 + mock_experiment.userid = 1 + mock_experiment.title = "Test Experiment" + mock_experiment.body = "Test Body" + mock_experiment.metadata = json.dumps({"extra_fields": {"key": {"value": "value"}}}) + mock_elabapi_python.ExperimentsApi.return_value.read_experiments.return_value = [ + mock_experiment, + ] + mock_user = MagicMock() + mock_user.fullname = "Test User" + mock_user.email = "test@example.com" + mock_user.userid = 1 + mock_user.orcid = "0000-0000-0000-0000" + mock_elabapi_python.UsersApi.return_value.read_user.return_value = mock_user + mock_elabapi_python.LinksToItemsApi.return_value.read_entity_items_links.return_value = [] + + metadata_retriever = MetadataRetriever(metadata_config, "dummy_token") + + metadata = {} + runs = ["run1"] + + updated_metadata = metadata_retriever.fetch_elab_metadata(runs, metadata) + + assert updated_metadata["elabFTW"]["user"]["name"] == "Test User" + assert updated_metadata["elabFTW"]["user"]["email"] == "test@example.com" + assert updated_metadata["elabFTW"]["user"]["id"] == 1 + assert updated_metadata["elabFTW"]["user"]["orcid"] == "0000-0000-0000-0000" + assert updated_metadata["elabFTW"]["scan"]["title"] == "Test Experiment" + assert updated_metadata["elabFTW"]["scan"]["summary"] == "Test Body" + assert updated_metadata["elabFTW"]["scan"]["id"] == 1 + assert updated_metadata["elabFTW"]["scan"]["key"] == "value" diff --git a/tests/test_processor.py b/tests/test_processor.py index 853cd1c3..7f303a35 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -1081,11 +1081,19 @@ def test_get_normalization_histogram() -> None: metadata["instrument"]["beam"] = {} metadata["instrument"]["beam"]["probe"] = {} metadata["instrument"]["beam"]["probe"]["incident_energy"] = 21.7 +metadata["instrument"]["beam"]["probe"]["frequency"] = 500.0 +metadata["instrument"]["beam"]["probe"]["incident_energy_spread"] = 0.11 +metadata["instrument"]["beam"]["probe"]["pulse_duration"] = 20.0 +metadata["instrument"]["beam"]["probe"]["incident_polarization"] = [1, 1, 0, 0] +metadata["instrument"]["beam"]["probe"]["extent"] = [80.0, 80.0] # sample metadata["sample"] = {} metadata["sample"]["preparation_date"] = "2019-01-13T10:00:00+00:00" metadata["sample"]["name"] = "Sample Name" +metadata["file"] = {} +metadata["file"]["KTOF:Lens:Extr:I"] = -0.12877 + def test_save(caplog) -> None: """Test the save functionality""" @@ -1118,7 +1126,7 @@ def test_save(caplog) -> None: with pytest.raises(NameError): processor.save("output.tiff") axes = ["kx", "ky", "energy", "delay"] - bins = [100, 100, 200, 50] + bins = [10, 10, 20, 5] ranges = [(-2, 2), (-2, 2), (-4, 2), (-600, 1600)] processor.compute(bins=bins, axes=axes, ranges=ranges) with pytest.raises(NotImplementedError): diff --git a/tutorial/4_hextof_workflow.ipynb b/tutorial/4_hextof_workflow.ipynb index 0d0970a5..bbde1963 100644 --- a/tutorial/4_hextof_workflow.ipynb +++ b/tutorial/4_hextof_workflow.ipynb @@ -894,13 +894,21 @@ "metadata = load_config(meta_path + \"/44824_20230324T060430.json\")\n", "\n", "# Fix metadata\n", + "metadata[\"scientificMetadata\"][\"Source\"][\"photon_energy\"][\"value\"] = float(metadata[\"scientificMetadata\"][\"Source\"][\"photon_energy\"][\"value\"])\n", + "metadata[\"scientificMetadata\"][\"Source\"][\"repetition_rate\"][\"value\"] = float(metadata[\"scientificMetadata\"][\"Source\"][\"repetition_rate\"][\"value\"])\n", "metadata[\"scientificMetadata\"][\"Laser\"][\"wavelength\"][\"value\"] = float(metadata[\"scientificMetadata\"][\"Laser\"][\"wavelength\"][\"value\"][:-2])\n", + "metadata[\"scientificMetadata\"][\"Laser\"][\"pulse duration\"][\"value\"] = float(metadata[\"scientificMetadata\"][\"Laser\"][\"pulse duration\"][\"value\"])\n", + "metadata[\"scientificMetadata\"][\"Laser\"][\"pulse_energy\"][\"value\"] = float(metadata[\"scientificMetadata\"][\"Laser\"][\"pulse_energy\"][\"value\"])\n", "metadata[\"scientificMetadata\"][\"Laser\"][\"energy\"] = {\"value\": 1239.84/metadata[\"scientificMetadata\"][\"Laser\"][\"wavelength\"][\"value\"], \"unit\": \"eV\"}\n", "metadata[\"scientificMetadata\"][\"Laser\"][\"polarization\"] = [1, 1, 0, 0]\n", + "metadata[\"scientificMetadata\"][\"Manipulator\"][\"sample_bias\"] = float(metadata[\"scientificMetadata\"][\"Manipulator\"][\"sample_bias\"])\n", + "metadata[\"scientificMetadata\"][\"Collection\"][\"tof_voltage\"] = float(metadata[\"scientificMetadata\"][\"Collection\"][\"tof_voltage\"])\n", + "metadata[\"scientificMetadata\"][\"Collection\"][\"extractor_voltage\"] = float(metadata[\"scientificMetadata\"][\"Collection\"][\"extractor_voltage\"])\n", + "metadata[\"scientificMetadata\"][\"Collection\"][\"field_aperture\"] = float(metadata[\"scientificMetadata\"][\"Collection\"][\"field_aperture\"])\n", "metadata[\"scientificMetadata\"][\"Collection\"][\"field_aperture_x\"] = float(metadata[\"scientificMetadata\"][\"Collection\"][\"field_aperture_x\"])\n", "metadata[\"scientificMetadata\"][\"Collection\"][\"field_aperture_y\"] = float(metadata[\"scientificMetadata\"][\"Collection\"][\"field_aperture_y\"])\n", "metadata[\"pi\"] = {\"institute\": \"JGU Mainz\"}\n", - "metadata[\"proposer\"] = {\"institute\": \"TU Dortmund\"}\n" + "metadata[\"proposer\"] = {\"institute\": \"TU Dortmund\"}" ] }, {