From 259fdcdffd19343801d36e72357c11033a49a795 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Thu, 15 May 2025 17:02:01 -0400 Subject: [PATCH 1/6] Starting porting EigerDetector implementation --- pixi.lock | 16 +++- pyproject.toml | 3 +- src/cditools/eiger.py | 182 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 src/cditools/eiger.py diff --git a/pixi.lock b/pixi.lock index 50a6ed09..dc42997d 100644 --- a/pixi.lock +++ b/pixi.lock @@ -32,6 +32,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/cd/c883e1a7c447479d6e13985565080e3fea88ab5a107c21684c813dba1875/flexcache-0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fe/5e/3be305568fe5f34448807976dc82fc151d76c3e0e03958f34770286278c1/flexparser-0.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/41/bc2df86b72965775f6d621e0ee269a5f3ac23e8f870abf519de9c7d93b4d/h5py-3.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/79/9d/0fb148dc4d6fa4a7dd1d8378168d9b4cd8d4560a6fbf6f0121c5fc34eb68/importlib_metadata-8.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/aa/fc/ebfd32c3e124e6a1043e19c0ab0769818aa69050ce5589b63d05ff185526/numpy-2.2.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -85,6 +86,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/cd/c883e1a7c447479d6e13985565080e3fea88ab5a107c21684c813dba1875/flexcache-0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fe/5e/3be305568fe5f34448807976dc82fc151d76c3e0e03958f34770286278c1/flexparser-0.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/41/bc2df86b72965775f6d621e0ee269a5f3ac23e8f870abf519de9c7d93b4d/h5py-3.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/2b/d3/85feeba1d097b81a44bcffa6a0beab7b4dfffe78e82fc54978d3ac380736/identify-2.6.10-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/9d/0fb148dc4d6fa4a7dd1d8378168d9b4cd8d4560a6fbf6f0121c5fc34eb68/importlib_metadata-8.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl @@ -165,6 +167,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/27/cd/c883e1a7c447479d6e13985565080e3fea88ab5a107c21684c813dba1875/flexcache-0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fe/5e/3be305568fe5f34448807976dc82fc151d76c3e0e03958f34770286278c1/flexparser-0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/48/e791a7ed487dbb9729ef32bb5d1af16693d8925f4366befef54119b2e576/furo-2024.8.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/41/bc2df86b72965775f6d621e0ee269a5f3ac23e8f870abf519de9c7d93b4d/h5py-3.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/9d/0fb148dc4d6fa4a7dd1d8378168d9b4cd8d4560a6fbf6f0121c5fc34eb68/importlib_metadata-8.6.1-py3-none-any.whl @@ -237,6 +240,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/11/79/479e2194c9096b92aecdf33634ae948d2be306c6011673e98ee1917f32c2/dpkt-1.9.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/27/cd/c883e1a7c447479d6e13985565080e3fea88ab5a107c21684c813dba1875/flexcache-0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fe/5e/3be305568fe5f34448807976dc82fc151d76c3e0e03958f34770286278c1/flexparser-0.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/41/bc2df86b72965775f6d621e0ee269a5f3ac23e8f870abf519de9c7d93b4d/h5py-3.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/79/9d/0fb148dc4d6fa4a7dd1d8378168d9b4cd8d4560a6fbf6f0121c5fc34eb68/importlib_metadata-8.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/91/86a6eac449ddfae239e93ffc1918cf33fd9bab35c04d1e963b311e347a73/netifaces-0.11.0.tar.gz @@ -375,10 +379,11 @@ packages: requires_python: '>=3.8' - pypi: . name: cditools - version: 0.1.dev11+g9b82aa1.d20250429 - sha256: 19ea9cb2d7ea3b19fc923f9299e5733b3f98028dea1cf4fb9467ba41ac960810 + version: 0.1.dev18+g22d45e0.d20250515 + sha256: 2a690b8e82f56c90db4126ff52415e1fca013a04ccaa3f4e60594e428a8b7b74 requires_dist: - ophyd + - h5py - pytest>=6 ; extra == 'test' - pytest-cov>=3 ; extra == 'test' - caproto[standard]>=0.4.2rc1,!=1.2.0 ; extra == 'test' @@ -512,6 +517,13 @@ packages: - sphinx-basic-ng>=1.0.0b2 - pygments>=2.7 requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/fd/41/bc2df86b72965775f6d621e0ee269a5f3ac23e8f870abf519de9c7d93b4d/h5py-3.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: h5py + version: 3.13.0 + sha256: be949b46b7388074c5acae017fbbe3e5ba303fd9daaa52157fdfef30bbdacadd + requires_dist: + - numpy>=1.19.3 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/2b/d3/85feeba1d097b81a44bcffa6a0beab7b4dfffe78e82fc54978d3ac380736/identify-2.6.10-py2.py3-none-any.whl name: identify version: 2.6.10 diff --git a/pyproject.toml b/pyproject.toml index 98908745..d9a1fc14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,8 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "ophyd" + "ophyd", + "h5py", ] [project.optional-dependencies] diff --git a/src/cditools/eiger.py b/src/cditools/eiger.py new file mode 100644 index 00000000..5760609c --- /dev/null +++ b/src/cditools/eiger.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import datetime +import logging +import os +import time as ttime +from collections import OrderedDict, deque +from pathlib import PurePath +from types import SimpleNamespace + +import h5py +from ophyd import Component as Cpt +from ophyd import ( + Device, + EpicsPathSignal, + EpicsSignal, + ImagePlugin, + Signal, + SingleTrigger, +) +from ophyd.areadetector import EigerDetector +from ophyd.areadetector.base import ADComponent, EpicsSignalWithRBV +from ophyd.areadetector.filestore_mixins import FileStoreBase # , new_short_uid + + +logger = logging.getLogger(__name__) + +DEFAULT_DATUM_DICT = {"data": None, "omega": None} + +# TODO: convert it to Enum class. +INTERNAL_SERIES = 0 +INTERNAL_ENABLE = 1 +EXTERNAL_SERIES = 2 +EXTERNAL_ENABLE = 3 + + +class EigerDetectorWithPlugins(EigerDetector): + hdf5 = ... + +class EigerSingleTriggerV26(SingleTrigger, EigerDetectorWithPlugins): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Remove `cam.acquire` since we want to keep the camera acquiring + self.stage_sigs.pop("cam.acquire") + self.stage_sigs.update( + {"cam.compression_algo": "BS LZ4"} + ) + + def collect_asset_docs(self): + asset_docs_cache = [] + + # Get the Resource which was produced when the detector was staged. + ((name, resource),) = self.file.collect_asset_docs() + + asset_docs_cache.append(("resource", resource)) + self._datum_ids = DEFAULT_DATUM_DICT + # Generate Datum documents from scratch here, because the detector was + # triggered externally by the DeltaTau, never by ophyd. + resource_uid = resource["uid"] + # We are currently generating only one datum document for all frames, that's why + # we use the 0th index below. + # + # Uncomment & update the line below if more datum documents are needed: + # for i in range(num_points): + + seq_id = self.cam.sequence_id.get() + + self._master_file = ( + f"{resource['root']}/{resource['resource_path']}_{seq_id}_master.h5" + ) + if not os.path.isfile(self._master_file): + raise RuntimeError(f"File {self._master_file} does not exist") + + # The pseudocode below is from Tom Caswell explaining the relationship between resource, datum, and events. + # + # resource = { + # "resource_id": "RES", + # "resource_kwargs": {}, # this goes to __init__ + # "spec": "AD-EIGER-MX", + # ...: ..., + # } + # datum = { + # "datum_id": "a", + # "datum_kwargs": {"data_key": "data"}, # this goes to __call__ + # "resource": "RES", + # ...: ..., + # } + # datum = { + # "datum_id": "b", + # "datum_kwargs": {"data_key": "omega"}, + # "resource": "RES", + # ...: ..., + # } + + # event = {...: ..., "data": {"detector_img": "a", "omega": "b"}} + + for data_key in self._datum_ids.keys(): + datum_id = f"{resource_uid}/{data_key}" + self._datum_ids[data_key] = datum_id + datum = { + "resource": resource_uid, + "datum_id": datum_id, + "datum_kwargs": {"data_key": data_key}, + } + asset_docs_cache.append(("datum", datum)) + return tuple(asset_docs_cache) + + def _extract_metadata(self, field="omega"): + with h5py.File(self._master_file, "r") as hf: + return hf.get(f"entry/sample/goniometer/{field}")[()] + + def unstage(self): + ttime.sleep(1.0) + super().unstage() + + def stage(self, *args, **kwargs): + return super().stage(*args, **kwargs) + + def trigger(self, *args, **kwargs): + status = super().trigger(*args, **kwargs) + self.cam.special_trigger_button.set(1) + return status + + def read(self, *args, streaming=False, **kwargs): + """ + This is a test of using streaming read. + Ideally, this should be handled by a new _stream_attrs property. + For now, we just check for a streaming key in read and + call super() if False, or read the one key we know we should read + if True. + + Parameters + ---------- + streaming : bool, optional + whether to read streaming attrs or not + """ + if streaming: + key = self._image_name # this comes from the SingleTrigger mixin + read_dict = super().read() + ret = OrderedDict({key: read_dict[key]}) + return ret + ret = super().read(*args, **kwargs) + return ret + + def describe(self, *args, streaming=False, **kwargs): + """ + This is a test of using streaming read. + Ideally, this should be handled by a new _stream_attrs property. + For now, we just check for a streaming key in read and + call super() if False, or read the one key we know we should read + if True. + + Parameters + ---------- + streaming : bool, optional + whether to read streaming attrs or not + """ + if streaming: + key = self._image_name # this comes from the SingleTrigger mixin + read_dict = super().describe() + ret = OrderedDict({key: read_dict[key]}) + return ret + ret = super().describe(*args, **kwargs) + return ret + + def super_unstage(self): + super().unstage() + + +def set_eiger_defaults(eiger): + """Choose which attributes to read per-step (read_attrs) or + per-run (configuration attrs).""" + + eiger.read_attrs = [ + "file", + # 'stats1', 'stats2', 'stats3', 'stats4', 'stats5', + ] + # for stats in [eiger.stats1, eiger.stats2, eiger.stats3, + # eiger.stats4, eiger.stats5]: + # stats.read_attrs = ['total'] + eiger.file.read_attrs = [] + eiger.cam.read_attrs = [] From 3a1234703d7388d5f4628759ce04448af511bcf3 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Mon, 19 May 2025 11:11:27 -0400 Subject: [PATCH 2/6] Rename to EigerSingleTrigger --- src/cditools/eiger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cditools/eiger.py b/src/cditools/eiger.py index 5760609c..e05d6f20 100644 --- a/src/cditools/eiger.py +++ b/src/cditools/eiger.py @@ -37,7 +37,7 @@ class EigerDetectorWithPlugins(EigerDetector): hdf5 = ... -class EigerSingleTriggerV26(SingleTrigger, EigerDetectorWithPlugins): +class EigerSingleTrigger(SingleTrigger, EigerDetectorWithPlugins): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Remove `cam.acquire` since we want to keep the camera acquiring From eb818a7628c82354346b52f6f14d511913f7fab2 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Mon, 19 May 2025 15:00:24 -0400 Subject: [PATCH 3/6] Untested version of EigerSingleTrigger --- src/cditools/eiger.py | 279 ++++++++++++++++-------------------------- 1 file changed, 105 insertions(+), 174 deletions(-) diff --git a/src/cditools/eiger.py b/src/cditools/eiger.py index e05d6f20..323e7514 100644 --- a/src/cditools/eiger.py +++ b/src/cditools/eiger.py @@ -1,182 +1,113 @@ -from __future__ import annotations - -import datetime -import logging -import os -import time as ttime -from collections import OrderedDict, deque +from datetime import datetime from pathlib import PurePath -from types import SimpleNamespace - -import h5py -from ophyd import Component as Cpt -from ophyd import ( - Device, - EpicsPathSignal, - EpicsSignal, - ImagePlugin, - Signal, - SingleTrigger, -) -from ophyd.areadetector import EigerDetector -from ophyd.areadetector.base import ADComponent, EpicsSignalWithRBV -from ophyd.areadetector.filestore_mixins import FileStoreBase # , new_short_uid - - -logger = logging.getLogger(__name__) - -DEFAULT_DATUM_DICT = {"data": None, "omega": None} - -# TODO: convert it to Enum class. -INTERNAL_SERIES = 0 -INTERNAL_ENABLE = 1 -EXTERNAL_SERIES = 2 -EXTERNAL_ENABLE = 3 - - -class EigerDetectorWithPlugins(EigerDetector): - hdf5 = ... +from typing import Any -class EigerSingleTrigger(SingleTrigger, EigerDetectorWithPlugins): - def __init__(self, *args, **kwargs): +from ophyd import EpicsSignalRO, ROIPlugin, Device, Component as Cpt, Signal, EigerDetector, StatusBase +from ophyd.areadetector.base import ADComponent, EpicsSignalWithRBV +from ophyd.areadetector.filestore_mixins import FileStoreBase, new_short_uid +from ophyd.areadetector.trigger_mixins import SingleTrigger +from ophyd.areadetector.plugins import StatsPlugin, ProcessPlugin, ROIPlugin + + +class EigerFileHandler(Device, FileStoreBase): + """A device to handle the file writing for the Eiger detector. + + When the Eiger's FileWriter module and SaveFiles are enabled, the file writing is handled + by the detector itself. In this case, we want to generate a resource document for the + file path and file name pattern. Then, we want to generate a datum for each trigger that + enables us to get the individual frames from the file. + + The alternative to this is to use the Stream interface and configure the area detector plugins + to write to a file store. + """ + sequence_id = ADComponent(EpicsSignalRO, 'SequenceId') + file_path = ADComponent(EpicsSignalWithRBV, 'FilePath', string=True) + file_write_name_pattern = ADComponent(EpicsSignalWithRBV, 'FWNamePattern', + string=True) + file_write_images_per_file = ADComponent(EpicsSignalWithRBV, + 'FWNImagesPerFile') + current_run_start_uid = Cpt(Signal, value='', add_prefix=()) + + def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None: + self.sequence_id_offset = 1 super().__init__(*args, **kwargs) - # Remove `cam.acquire` since we want to keep the camera acquiring - self.stage_sigs.pop("cam.acquire") - self.stage_sigs.update( - {"cam.compression_algo": "BS LZ4"} - ) - - def collect_asset_docs(self): - asset_docs_cache = [] - - # Get the Resource which was produced when the detector was staged. - ((name, resource),) = self.file.collect_asset_docs() - - asset_docs_cache.append(("resource", resource)) - self._datum_ids = DEFAULT_DATUM_DICT - # Generate Datum documents from scratch here, because the detector was - # triggered externally by the DeltaTau, never by ophyd. - resource_uid = resource["uid"] - # We are currently generating only one datum document for all frames, that's why - # we use the 0th index below. - # - # Uncomment & update the line below if more datum documents are needed: - # for i in range(num_points): - - seq_id = self.cam.sequence_id.get() - - self._master_file = ( - f"{resource['root']}/{resource['resource_path']}_{seq_id}_master.h5" - ) - if not os.path.isfile(self._master_file): - raise RuntimeError(f"File {self._master_file} does not exist") - - # The pseudocode below is from Tom Caswell explaining the relationship between resource, datum, and events. - # - # resource = { - # "resource_id": "RES", - # "resource_kwargs": {}, # this goes to __init__ - # "spec": "AD-EIGER-MX", - # ...: ..., - # } - # datum = { - # "datum_id": "a", - # "datum_kwargs": {"data_key": "data"}, # this goes to __call__ - # "resource": "RES", - # ...: ..., - # } - # datum = { - # "datum_id": "b", - # "datum_kwargs": {"data_key": "omega"}, - # "resource": "RES", - # ...: ..., - # } - - # event = {...: ..., "data": {"detector_img": "a", "omega": "b"}} - - for data_key in self._datum_ids.keys(): - datum_id = f"{resource_uid}/{data_key}" - self._datum_ids[data_key] = datum_id - datum = { - "resource": resource_uid, - "datum_id": datum_id, - "datum_kwargs": {"data_key": data_key}, - } - asset_docs_cache.append(("datum", datum)) - return tuple(asset_docs_cache) - - def _extract_metadata(self, field="omega"): - with h5py.File(self._master_file, "r") as hf: - return hf.get(f"entry/sample/goniometer/{field}")[()] - - def unstage(self): - ttime.sleep(1.0) + + # NOTE: See `FileStoreBase._generate_resource` for the use of these. + self._fn = None + self.filestore_spec = "AD_EIGER2" + + def stage(self) -> list[object]: + res_uid = new_short_uid() + write_path = datetime.now().strftime(self.write_path_template) + self.file_path.set(write_path).wait(1.0) + # The name pattern must have `$id` in it. + # `$id` is replaced by the current sequence id of the acquisition. + # E.g. * _1_master.h5 + # * _1_data_000001.h5 + # * _1_data_000002.h5 + # * ... + self.file_write_name_pattern.set(f"{res_uid}_$id").wait(1.0) + + super().stage() + + # Set the filename for the resource document. + file_prefix = PurePath(self.file_path.get()) / res_uid + self._fn = file_prefix + + images_per_file = self.file_write_images_per_file.get() + resource_kwargs = {'images_per_file' : images_per_file} + + self._generate_resource(resource_kwargs) + + def generate_datum(self, key: str, timestamp: float, datum_kwargs: dict[str, Any]) -> Any: + # The detector keeps its own counter which is uses label HDF5 + # sub-files. We access that counter via the sequence_id + # signal and stash it in the datum. + seq_id = self.sequence_id_offset + self.sequence_id.get() # det writes to the NEXT one + datum_kwargs.update({'seq_id': seq_id}) + # TODO: Is this needed? + if self.frame_num is not None: + datum_kwargs.update({'frame_num': self.frame_num}) + return super().generate_datum(key, timestamp, datum_kwargs) + + +class EigerBase(EigerDetector): + """Base class for Eiger detectors that have the commonly used plugins.""" + file_handler = Cpt(EigerFileHandler, "", name="file_handler", + write_path_template="/nsls2/data/tst/legacy/mock-proposals/2025-2/pass-56789/assets/eiger/%Y/%m/%d", + root="/nsls2/data/tst/legacy/mock-proposals/2025-2/pass-56789/assets/eiger") + stats1 = Cpt(StatsPlugin, "Stats1:") + stats2 = Cpt(StatsPlugin, "Stats2:") + stats3 = Cpt(StatsPlugin, "Stats3:") + stats4 = Cpt(StatsPlugin, "Stats4:") + stats5 = Cpt(StatsPlugin, "Stats5:") + roi1 = Cpt(ROIPlugin, "ROI1:") + roi2 = Cpt(ROIPlugin, "ROI2:") + roi3 = Cpt(ROIPlugin, "ROI3:") + roi4 = Cpt(ROIPlugin, "ROI4:") + proc1 = Cpt(ProcessPlugin, "Proc1:") + + + def stage(self, *args: Any, **kwargs: dict[str, Any]) -> list[object]: + staged_devices = super().stage(*args, **kwargs) + self.cam.manual_trigger.set(1).wait(5.0) + return staged_devices + + def unstage(self) -> None: + self.cam.manual_trigger.set(0).wait(5.0) super().unstage() - def stage(self, *args, **kwargs): - return super().stage(*args, **kwargs) - def trigger(self, *args, **kwargs): +class EigerSingleTrigger(SingleTrigger, EigerBase): + """Eiger detector that uses the single trigger acquisition mode.""" + def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None: + super().__init__(*args, **kwargs) + self.stage_sigs["cam.trigger_mode"] = 0 + + def trigger(self, *args: Any, **kwargs: dict[str, Any]) -> StatusBase: status = super().trigger(*args, **kwargs) - self.cam.special_trigger_button.set(1) + # If the manual trigger is enabled, we need to press the special trigger button + # to actually trigger the detector. + if self.cam.manual_trigger.get() == 1: + self.cam.special_trigger_button.set(1).wait(5.0) return status - - def read(self, *args, streaming=False, **kwargs): - """ - This is a test of using streaming read. - Ideally, this should be handled by a new _stream_attrs property. - For now, we just check for a streaming key in read and - call super() if False, or read the one key we know we should read - if True. - - Parameters - ---------- - streaming : bool, optional - whether to read streaming attrs or not - """ - if streaming: - key = self._image_name # this comes from the SingleTrigger mixin - read_dict = super().read() - ret = OrderedDict({key: read_dict[key]}) - return ret - ret = super().read(*args, **kwargs) - return ret - - def describe(self, *args, streaming=False, **kwargs): - """ - This is a test of using streaming read. - Ideally, this should be handled by a new _stream_attrs property. - For now, we just check for a streaming key in read and - call super() if False, or read the one key we know we should read - if True. - - Parameters - ---------- - streaming : bool, optional - whether to read streaming attrs or not - """ - if streaming: - key = self._image_name # this comes from the SingleTrigger mixin - read_dict = super().describe() - ret = OrderedDict({key: read_dict[key]}) - return ret - ret = super().describe(*args, **kwargs) - return ret - - def super_unstage(self): - super().unstage() - - -def set_eiger_defaults(eiger): - """Choose which attributes to read per-step (read_attrs) or - per-run (configuration attrs).""" - - eiger.read_attrs = [ - "file", - # 'stats1', 'stats2', 'stats3', 'stats4', 'stats5', - ] - # for stats in [eiger.stats1, eiger.stats2, eiger.stats3, - # eiger.stats4, eiger.stats5]: - # stats.read_attrs = ['total'] - eiger.file.read_attrs = [] - eiger.cam.read_attrs = [] From 7c929bd4885dbd67b49eb1330525c6e706ad3625 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Tue, 27 May 2025 16:28:53 -0400 Subject: [PATCH 4/6] Check for mutliple master files for each datum --- src/cditools/eiger.py | 65 ++++++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/src/cditools/eiger.py b/src/cditools/eiger.py index 323e7514..c4962bea 100644 --- a/src/cditools/eiger.py +++ b/src/cditools/eiger.py @@ -1,6 +1,7 @@ +import os from datetime import datetime from pathlib import PurePath -from typing import Any +from typing import Any, Optional from ophyd import EpicsSignalRO, ROIPlugin, Device, Component as Cpt, Signal, EigerDetector, StatusBase from ophyd.areadetector.base import ADComponent, EpicsSignalWithRBV @@ -20,13 +21,15 @@ class EigerFileHandler(Device, FileStoreBase): The alternative to this is to use the Stream interface and configure the area detector plugins to write to a file store. """ - sequence_id = ADComponent(EpicsSignalRO, 'SequenceId') - file_path = ADComponent(EpicsSignalWithRBV, 'FilePath', string=True) - file_write_name_pattern = ADComponent(EpicsSignalWithRBV, 'FWNamePattern', + sequence_id = ADComponent(EpicsSignalRO, "SequenceId") + file_path = ADComponent(EpicsSignalWithRBV, "FilePath", string=True) + file_write_name_pattern = ADComponent(EpicsSignalWithRBV, "FWNamePattern", string=True) file_write_images_per_file = ADComponent(EpicsSignalWithRBV, - 'FWNImagesPerFile') - current_run_start_uid = Cpt(Signal, value='', add_prefix=()) + "FWNImagesPerFile") + enable = Cpt(EpicsSignalWithRBV, "FWEnable") + data_source = Cpt(EpicsSignalWithRBV, "DataSource", string=True) + save_files = Cpt(EpicsSignalWithRBV, "SaveFiles") def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None: self.sequence_id_offset = 1 @@ -34,12 +37,24 @@ def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None: # NOTE: See `FileStoreBase._generate_resource` for the use of these. self._fn = None - self.filestore_spec = "AD_EIGER2" + self.filestore_spec = "AD_EIGER" + self._master_file_paths: list[PurePath] = [] + + @property + def master_file_paths(self) -> list[PurePath]: + if len(self._master_file_paths) == 0: + raise ValueError("Master file path has not been set. Call stage() first.") + return self._master_file_paths + + @property + def sequence_number(self) -> int: + return self.sequence_id_offset + self.sequence_id.get() def stage(self) -> list[object]: res_uid = new_short_uid() - write_path = datetime.now().strftime(self.write_path_template) + write_path = f"{datetime.now().strftime(self.write_path_template)}/" self.file_path.set(write_path).wait(1.0) + # The name pattern must have `$id` in it. # `$id` is replaced by the current sequence id of the acquisition. # E.g. * _1_master.h5 @@ -59,21 +74,28 @@ def stage(self) -> list[object]: self._generate_resource(resource_kwargs) + # Validate that the root path exists + if not os.path.exists(self.reg_root): + raise FileNotFoundError(f"Root path {self.reg_root} does not exist") + + # Create the templated part of the path + if not os.path.exists(write_path): + os.makedirs(write_path) + + self._master_file_paths = [] + def generate_datum(self, key: str, timestamp: float, datum_kwargs: dict[str, Any]) -> Any: # The detector keeps its own counter which is uses label HDF5 # sub-files. We access that counter via the sequence_id # signal and stash it in the datum. - seq_id = self.sequence_id_offset + self.sequence_id.get() # det writes to the NEXT one - datum_kwargs.update({'seq_id': seq_id}) - # TODO: Is this needed? - if self.frame_num is not None: - datum_kwargs.update({'frame_num': self.frame_num}) + datum_kwargs.update({'seq_id': self.sequence_number}) + self._master_file_paths.append(f"{self._fn}_{self.sequence_number}_master.h5") return super().generate_datum(key, timestamp, datum_kwargs) class EigerBase(EigerDetector): """Base class for Eiger detectors that have the commonly used plugins.""" - file_handler = Cpt(EigerFileHandler, "", name="file_handler", + file_handler = Cpt(EigerFileHandler, "cam1:", name="file_handler", write_path_template="/nsls2/data/tst/legacy/mock-proposals/2025-2/pass-56789/assets/eiger/%Y/%m/%d", root="/nsls2/data/tst/legacy/mock-proposals/2025-2/pass-56789/assets/eiger") stats1 = Cpt(StatsPlugin, "Stats1:") @@ -90,20 +112,29 @@ class EigerBase(EigerDetector): def stage(self, *args: Any, **kwargs: dict[str, Any]) -> list[object]: staged_devices = super().stage(*args, **kwargs) - self.cam.manual_trigger.set(1).wait(5.0) + self.cam.manual_trigger.set(True).wait(5.0) + file_write_path = self.file_handler.file_path.get() + if not os.path.exists(file_write_path): + raise FileNotFoundError(f"Path {file_write_path} does not exist.") return staged_devices def unstage(self) -> None: - self.cam.manual_trigger.set(0).wait(5.0) + self.cam.manual_trigger.set(False).wait(5.0) super().unstage() + if not all(os.path.exists(path) for path in self.file_handler.master_file_paths): + raise FileNotFoundError(f"Paths {self.file_handler.master_file_paths} were not written.") + class EigerSingleTrigger(SingleTrigger, EigerBase): """Eiger detector that uses the single trigger acquisition mode.""" def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None: super().__init__(*args, **kwargs) self.stage_sigs["cam.trigger_mode"] = 0 - + self.stage_sigs["file_handler.data_source"] = "FileWriter" + self.stage_sigs["file_handler.enable"] = True + self.stage_sigs["file_handler.save_files"] = True + def trigger(self, *args: Any, **kwargs: dict[str, Any]) -> StatusBase: status = super().trigger(*args, **kwargs) # If the manual trigger is enabled, we need to press the special trigger button From 6e8b9dec0833ea16270583779479e8bcb1c73369 Mon Sep 17 00:00:00 2001 From: thomashopkins32 Date: Tue, 27 May 2025 16:47:36 -0400 Subject: [PATCH 5/6] pre-commit --- src/cditools/eiger.py | 108 +++++++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 37 deletions(-) diff --git a/src/cditools/eiger.py b/src/cditools/eiger.py index c4962bea..48d88230 100644 --- a/src/cditools/eiger.py +++ b/src/cditools/eiger.py @@ -1,13 +1,30 @@ -import os -from datetime import datetime -from pathlib import PurePath -from typing import Any, Optional +from __future__ import annotations -from ophyd import EpicsSignalRO, ROIPlugin, Device, Component as Cpt, Signal, EigerDetector, StatusBase -from ophyd.areadetector.base import ADComponent, EpicsSignalWithRBV -from ophyd.areadetector.filestore_mixins import FileStoreBase, new_short_uid -from ophyd.areadetector.trigger_mixins import SingleTrigger -from ophyd.areadetector.plugins import StatsPlugin, ProcessPlugin, ROIPlugin +from datetime import datetime +from pathlib import Path, PurePath +from typing import Any + +from ophyd import Component as Cpt # type: ignore[import-not-found] +from ophyd import ( + Device, + EigerDetector, + EpicsSignalRO, + ProcessPlugin, + ROIPlugin, + StatsPlugin, + StatusBase, +) +from ophyd.areadetector.base import ( # type: ignore[import-not-found] + ADComponent, + EpicsSignalWithRBV, +) +from ophyd.areadetector.filestore_mixins import ( # type: ignore[import-not-found] + FileStoreBase, + new_short_uid, +) +from ophyd.areadetector.trigger_mixins import ( # type: ignore[import-not-found] + SingleTrigger, +) class EigerFileHandler(Device, FileStoreBase): @@ -21,12 +38,13 @@ class EigerFileHandler(Device, FileStoreBase): The alternative to this is to use the Stream interface and configure the area detector plugins to write to a file store. """ + sequence_id = ADComponent(EpicsSignalRO, "SequenceId") file_path = ADComponent(EpicsSignalWithRBV, "FilePath", string=True) - file_write_name_pattern = ADComponent(EpicsSignalWithRBV, "FWNamePattern", - string=True) - file_write_images_per_file = ADComponent(EpicsSignalWithRBV, - "FWNImagesPerFile") + file_write_name_pattern = ADComponent( + EpicsSignalWithRBV, "FWNamePattern", string=True + ) + file_write_images_per_file = ADComponent(EpicsSignalWithRBV, "FWNImagesPerFile") enable = Cpt(EpicsSignalWithRBV, "FWEnable") data_source = Cpt(EpicsSignalWithRBV, "DataSource", string=True) save_files = Cpt(EpicsSignalWithRBV, "SaveFiles") @@ -43,18 +61,19 @@ def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None: @property def master_file_paths(self) -> list[PurePath]: if len(self._master_file_paths) == 0: - raise ValueError("Master file path has not been set. Call stage() first.") + msg = "Master file path has not been set. Call stage() first." + raise ValueError(msg) return self._master_file_paths @property def sequence_number(self) -> int: - return self.sequence_id_offset + self.sequence_id.get() + return self.sequence_id_offset + int(self.sequence_id.get()) def stage(self) -> list[object]: res_uid = new_short_uid() - write_path = f"{datetime.now().strftime(self.write_path_template)}/" - self.file_path.set(write_path).wait(1.0) - + write_path = Path(f"{datetime.now().strftime(self.write_path_template)}/") + self.file_path.set(write_path.as_posix()).wait(1.0) + # The name pattern must have `$id` in it. # `$id` is replaced by the current sequence id of the acquisition. # E.g. * _1_master.h5 @@ -63,41 +82,54 @@ def stage(self) -> list[object]: # * ... self.file_write_name_pattern.set(f"{res_uid}_$id").wait(1.0) - super().stage() + ret: list[object] = super().stage() # Set the filename for the resource document. file_prefix = PurePath(self.file_path.get()) / res_uid self._fn = file_prefix images_per_file = self.file_write_images_per_file.get() - resource_kwargs = {'images_per_file' : images_per_file} + resource_kwargs = {"images_per_file": images_per_file} self._generate_resource(resource_kwargs) # Validate that the root path exists - if not os.path.exists(self.reg_root): - raise FileNotFoundError(f"Root path {self.reg_root} does not exist") - + if not Path.exists(Path(self.reg_root)): + msg = f"Root path {self.reg_root} does not exist" + raise FileNotFoundError(msg) + # Create the templated part of the path - if not os.path.exists(write_path): - os.makedirs(write_path) + if not Path.exists(write_path): + Path.mkdir(write_path, parents=True) + # Reset the list of master file paths self._master_file_paths = [] - def generate_datum(self, key: str, timestamp: float, datum_kwargs: dict[str, Any]) -> Any: + return ret + + def generate_datum( + self, key: str, timestamp: float, datum_kwargs: dict[str, Any] + ) -> Any: # The detector keeps its own counter which is uses label HDF5 # sub-files. We access that counter via the sequence_id # signal and stash it in the datum. - datum_kwargs.update({'seq_id': self.sequence_number}) - self._master_file_paths.append(f"{self._fn}_{self.sequence_number}_master.h5") + datum_kwargs.update({"seq_id": self.sequence_number}) + self._master_file_paths.append( + PurePath(f"{self._fn}_{self.sequence_number}_master.h5") + ) return super().generate_datum(key, timestamp, datum_kwargs) class EigerBase(EigerDetector): """Base class for Eiger detectors that have the commonly used plugins.""" - file_handler = Cpt(EigerFileHandler, "cam1:", name="file_handler", - write_path_template="/nsls2/data/tst/legacy/mock-proposals/2025-2/pass-56789/assets/eiger/%Y/%m/%d", - root="/nsls2/data/tst/legacy/mock-proposals/2025-2/pass-56789/assets/eiger") + + file_handler = Cpt( + EigerFileHandler, + "cam1:", + name="file_handler", + write_path_template="/nsls2/data/tst/legacy/mock-proposals/2025-2/pass-56789/assets/eiger/%Y/%m/%d", + root="/nsls2/data/tst/legacy/mock-proposals/2025-2/pass-56789/assets/eiger", + ) stats1 = Cpt(StatsPlugin, "Stats1:") stats2 = Cpt(StatsPlugin, "Stats2:") stats3 = Cpt(StatsPlugin, "Stats3:") @@ -108,26 +140,28 @@ class EigerBase(EigerDetector): roi3 = Cpt(ROIPlugin, "ROI3:") roi4 = Cpt(ROIPlugin, "ROI4:") proc1 = Cpt(ProcessPlugin, "Proc1:") - def stage(self, *args: Any, **kwargs: dict[str, Any]) -> list[object]: - staged_devices = super().stage(*args, **kwargs) + staged_devices: list[object] = super().stage(*args, **kwargs) self.cam.manual_trigger.set(True).wait(5.0) file_write_path = self.file_handler.file_path.get() - if not os.path.exists(file_write_path): - raise FileNotFoundError(f"Path {file_write_path} does not exist.") + if not Path.exists(file_write_path): + msg = f"Path {file_write_path} does not exist." + raise FileNotFoundError(msg) return staged_devices def unstage(self) -> None: self.cam.manual_trigger.set(False).wait(5.0) super().unstage() - if not all(os.path.exists(path) for path in self.file_handler.master_file_paths): - raise FileNotFoundError(f"Paths {self.file_handler.master_file_paths} were not written.") + if not all(Path.exists(path) for path in self.file_handler.master_file_paths): + msg = f"Paths {self.file_handler.master_file_paths} were not written." + raise FileNotFoundError(msg) class EigerSingleTrigger(SingleTrigger, EigerBase): """Eiger detector that uses the single trigger acquisition mode.""" + def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None: super().__init__(*args, **kwargs) self.stage_sigs["cam.trigger_mode"] = 0 From 3dda87922b434506cf275db4fbcda69e9ef3820e Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 9 Jul 2025 10:25:40 -0400 Subject: [PATCH 6/6] Add a note about CDI deployment --- src/cditools/eiger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cditools/eiger.py b/src/cditools/eiger.py index 48d88230..5614d875 100644 --- a/src/cditools/eiger.py +++ b/src/cditools/eiger.py @@ -127,6 +127,7 @@ class EigerBase(EigerDetector): EigerFileHandler, "cam1:", name="file_handler", + # TODO: These paths need to be changed once the detector is deployed at CDI. write_path_template="/nsls2/data/tst/legacy/mock-proposals/2025-2/pass-56789/assets/eiger/%Y/%m/%d", root="/nsls2/data/tst/legacy/mock-proposals/2025-2/pass-56789/assets/eiger", )