APS data ingestor

tests/test_ingest_nexus.py

@@ -1,20 +1,43 @@
 from pathlib import Path
 from xicam.core.data import load_header
+from xicam.plugins import manager as plugin_manager
 
 from xicam.XPCS.ingestors import ingest_nxXPCS
 from xicam.XPCS.projectors.nexus import project_nxXPCS
 
+#TODO use githublfs to store (test)data
+
 f = 'B009_Aerogel_1mm_025C_att1_Lq0_001_0001-10000.nxs'
-p = Path('/home/ihumphrey/Downloads') / f
+p = Path('.') / f
 
 
 def test_ingest_nexus(path=p):
     docs = list(ingest_nxXPCS([path]))
+    #TODO check document keys even if multiple events per document
+
+    # expected_doc_keys = ["start", "descriptor", "event", "stop"]
+    # for i, doc in enumerate(docs):
+    #     assert doc[0] == expected_doc_keys[i]
+
+    expected_projection_keys = ["entry/XPCS/data/g2",
+                                "entry/XPCS/data/tau",
+                                "entry/XPCS/data/g2_errors",
+                                "entry/data/masks/mask/mask_names",
+                                'entry/XPCS/data/masks',
+                                "entry/SAXS_2D/data/I",
+                                "entry/SAXS_1D/data/I",
+                                "entry/SAXS_1D/data/Q",
+                                "entry/data/raw"]
+
+    start_doc = docs[0][-1]
+    for i in start_doc["projections"][0]["projection"].keys():
+        assert i in expected_projection_keys
+
 
-    print(docs)
 
 
-def test_project_nexus():
-    cat = load_header([p])
-    g2_arr = project_nxXPCS(cat)
-    print(g2_arr)
+# def test_project_nexus():
+#
+#     cat = load_header([p])
+#     g2_arr = project_nxXPCS(cat)
+#     print(g2_arr)

xicam/XPCS/__init__.py

@@ -1,16 +1,15 @@
+from xicam.gui.plugins.ensembleguiplugin import EnsembleGUIPlugin
 from xicam.plugins import GUILayout, GUIPlugin, manager as pluginmanager
 from . import ingestors
 
 
-class XPCS(GUIPlugin):
+class XPCS(EnsembleGUIPlugin):
     name = 'XPCS'
 
     def __init__(self):
+        super(XPCS, self).__init__()
         saxsplugin = pluginmanager.get_plugin_by_name('SAXS', 'GUIPlugin')
 
-        self.stages = saxsplugin.stages['Correlate']
+        self.stages = saxsplugin.stages
 
         self.appendCatalog = saxsplugin.appendCatalog
-        self.appendHeader = saxsplugin.appendHeader
-
-        super(XPCS, self).__init__()

xicam/XPCS/hints/__init__.py

@@ -0,0 +1,55 @@
+from xarray import Dataset
+from pyqtgraph import ImageView, PlotWidget
+
+
+class Hint:
+    def __init__(self, name="", category=""):
+        self._name = name
+        self._category = category
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def category(self):
+        return self._category
+
+
+class ImageHintCanvas(ImageView):
+    def __init__(self, *args, **kwargs):
+        super(ImageHintCanvas, self).__init__(*args, **kwargs)
+
+    def render(self, hint):
+        self.setImage(hint.image)
+
+
+class ImageHint(Hint):
+    canvas = ImageHintCanvas
+
+    def __init__(self, image, *args, **kwargs):
+        super(ImageHint, self).__init__(*args, **kwargs)
+        self.image = image
+
+
+class PlotHintCanvas(PlotWidget):
+    def __init__(self, *args, **kwargs):
+        super(PlotHintCanvas, self).__init__(*args, **kwargs)
+
+    def render(self, hint):
+        self.plot(x=hint.x.compute(), y=hint.y.compute())
+
+
+class PlotHint(Hint):
+    # Model that we can pull hints (EnsembleModel)
+    # View interprets these hints by init'ng canvas into its display
+    canvas = PlotHintCanvas
+
+    def __init__(self, x: Dataset, y: Dataset, *args, **kwargs):
+        super(PlotHint, self).__init__(*args, **kwargs)
+        self.x = x
+        self.y = y
+
+    @property
+    def name(self):
+        return self.x.name + ", " + self.y.name

xicam/XPCS/ingestors/__init__.py

@@ -1,4 +1,5 @@
 import time
+import numpy as np
 import h5py
 import event_model
 from pathlib import Path
@@ -8,15 +9,19 @@
 
 print("MIMETYPE ADDED")
 mimetypes.add_type('application/x-hdf5', '.nxs')
+mimetypes.add_type('application/x-hdf5', '.nx')
 
+#TODO separate the keys in external projections/config file and parse that here
 g2_projection_key = 'entry/XPCS/data/g2'
-tau_projection_key = 'entry/XPCS/data/t_el'  # FIXME: replace with tau once available in h5 file
+tau_projection_key = 'entry/XPCS/data/tau'
 g2_error_projection_key = 'entry/XPCS/data/g2_errors'
 g2_roi_names_key = 'entry/data/masks/mask/mask_names'
+XPCS_mask_names_key = 'entry/XPCS/data/masks'
 
 SAXS_2D_I_projection_key = 'entry/SAXS_2D/data/I'
 SAXS_1D_I_projection_key = 'entry/SAXS_1D/data/I'
 SAXS_1D_Q_projection_key = 'entry/SAXS_1D/data/Q'
+SAXS_1D_I_partial_projection_key = 'entry/SAXS_1D/data/I_partial'
 
 raw_data_projection_key = 'entry/data/raw'
 # TODO: add var for rest of projection keys
@@ -36,7 +41,7 @@
                                                'stream': 'primary',
                                                'location': 'event',
                                                'field': 'g2_error_bars'},
-                     'entry/XPCS/data/masks': {'type': 'linked',
+                     XPCS_mask_names_key: {'type': 'linked',
                                                'stream': 'primary',
                                                'location': 'event',
                                                'field': 'masks'},
@@ -57,7 +62,10 @@
                                                 'stream': 'SAXS_1D',
                                                 'location': 'event',
                                                 'field': 'SAXS_1D_Q'},
-
+                     SAXS_1D_I_partial_projection_key: {'type': 'linked',
+                                                'stream': 'SAXS_1D_I_partial',
+                                                'location': 'event',
+                                                'field': 'SAXS_1D_I_partial'},
                      raw_data_projection_key: {'type': 'linked',
                                                 'stream': 'raw',
                                                 'location': 'event',
@@ -89,9 +97,10 @@ def ingest_nxXPCS(paths):
     # masks = h5['entry/XPCS/data/masks']
     # rois = h5['entry/XPCS/data/rois']
     g2_roi_names = list(map(lambda bytestring: bytestring.decode('UTF-8'), h5[g2_roi_names_key][()]))
-    SAXS_2D_I = da.from_array(h5['entry/SAXS_2D/data/I'])
+    SAXS_2D_I = da.from_array(h5[SAXS_2D_I_projection_key])
     SAXS_1D_I = h5[SAXS_1D_I_projection_key]
     SAXS_1D_Q = h5[SAXS_1D_Q_projection_key]
+    SAXS_1D_I_partial = da.from_array(h5[SAXS_1D_I_partial_projection_key])
 
     try:
         raw_data = h5[raw_data_projection_key]
@@ -143,6 +152,12 @@ def ingest_nxXPCS(paths):
                                   'shape': SAXS_1D_Q.shape},
                     }
 
+    SAXS_1D_I_partial_keys = {'SAXS_1D_I_partial': {'source': source,
+                             'dtype': 'array',
+                             'dims': ('N', 'I'),
+                             'shape': SAXS_1D_I_partial.shape},
+                    }
+
 
     #TODO: How to add multiple streams?
     g2_stream_bundle = run_bundle.compose_descriptor(data_keys=g2_data_keys,
@@ -157,11 +172,16 @@ def ingest_nxXPCS(paths):
                                                         name='SAXS_1D'
                                                         # configuration=_metadata(path)
                                                         )
+    SAXS_1D_I_partial_stream_bundle = run_bundle.compose_descriptor(data_keys=SAXS_1D_I_partial_keys,
+                                                        name='SAXS_1D_I_partial'
+                                                        # configuration=_metadata(path)
+                                                        )
 
 
     yield 'descriptor', g2_stream_bundle.descriptor_doc
     yield 'descriptor', SAXS_2D_stream_bundle.descriptor_doc
     yield 'descriptor', SAXS_1D_stream_bundle.descriptor_doc
+    yield 'descriptor', SAXS_1D_I_partial_stream_bundle.descriptor_doc
 
 
     num_events = g2.shape[1]
@@ -183,5 +203,10 @@ def ingest_nxXPCS(paths):
                                                              'SAXS_1D_Q': SAXS_1D_Q},
                                                        timestamps={'SAXS_1D_I': t,
                                                                    'SAXS_1D_Q': t})
+    # num_curves = SAXS_1D_I_partial.shape[1]
+    # for i in range(num_curves):
+    t = time.time()
+    yield 'event', SAXS_1D_I_partial_stream_bundle.compose_event(data={'SAXS_1D_I_partial': SAXS_1D_I_partial},
+                                                                 timestamps={'SAXS_1D_I_partial': t})
 
     yield 'stop', run_bundle.compose_stop()

xicam/XPCS/projectors/nexus.py

@@ -6,7 +6,7 @@
 from xicam.core.intents import Intent, PlotIntent, ImageIntent, ErrorBarIntent
 from ..ingestors import g2_projection_key, g2_error_projection_key, g2_roi_names_key, tau_projection_key, \
                         SAXS_2D_I_projection_key, SAXS_1D_I_projection_key, SAXS_1D_Q_projection_key, \
-                        raw_data_projection_key
+                        SAXS_1D_I_partial_projection_key, raw_data_projection_key
 from scipy.misc import face
 
 
@@ -37,20 +37,28 @@ def project_nxXPCS(run_catalog: BlueskyRun) -> List[Intent]:
 
     SAXS_2D_I_stream = projection['projection'][SAXS_2D_I_projection_key]['stream']
     SAXS_2D_I_field = projection['projection'][SAXS_2D_I_projection_key]['field']
-    SAXS_2D_I = getattr(run_catalog, SAXS_2D_I_stream).to_dask().rename({SAXS_2D_I_field: SAXS_2D_I_projection_key})[SAXS_2D_I_projection_key]
+    SAXS_2D_I = getattr(run_catalog, SAXS_2D_I_stream).to_dask().\
+                        rename({SAXS_2D_I_field: SAXS_2D_I_projection_key})[SAXS_2D_I_projection_key]
 
     SAXS_1D_I_stream = projection['projection'][SAXS_1D_I_projection_key]['stream']
     SAXS_1D_I_field = projection['projection'][SAXS_1D_I_projection_key]['field']
     SAXS_1D_Q_field = projection['projection'][SAXS_1D_Q_projection_key]['field']
     SAXS_1D_I = getattr(run_catalog, SAXS_1D_I_stream).to_dask().rename({SAXS_1D_I_field: SAXS_1D_I_projection_key,
                                                                          SAXS_1D_Q_field: SAXS_1D_Q_projection_key})
     SAXS_1D_I = np.squeeze(SAXS_1D_I)
+
+    SAXS_1D_I_partial_stream = projection['projection'][SAXS_1D_I_partial_projection_key]['stream']
+    SAXS_1D_I_partial_field = projection['projection'][SAXS_1D_I_partial_projection_key]['field']
+    SAXS_1D_I_partial = getattr(run_catalog, SAXS_1D_I_partial_stream).to_dask().\
+                                rename({SAXS_1D_I_partial_field: SAXS_1D_I_partial_projection_key})[SAXS_1D_I_partial_projection_key]
+    SAXS_1D_I_partial = np.squeeze(SAXS_1D_I_partial)
+
     try:
         raw_data_stream = projection['projection'][raw_data_projection_key]['stream']
         raw_data_field = projection['projection'][raw_data_projection_key]['field']
         raw_data = getattr(run_catalog, raw_data_stream).to_dask().rename({raw_data_field: raw_data_projection_key})[raw_data_projection_key]
         raw_data = np.squeeze(raw_data)
-        l.append(SAXSImageIntent(image=raw_data, item_name="Raw frame {}".format(catalog_name)), )
+        l.append(SAXSImageIntent(image=raw_data, item_name="Raw frame {}".format(catalog_name), mixins=("SAXSImageIntentBlend",)), )
     except:
         print('No raw data available')
 
@@ -78,10 +86,15 @@ def project_nxXPCS(run_catalog: BlueskyRun) -> List[Intent]:
                                 labels={"left": "g₂", "bottom": "τ"}))
 
     #l.append(ImageIntent(image=face(True), item_name='SAXS 2D'),)
-    l.append(SAXSImageIntent(image=SAXS_2D_I, item_name="AVG frame {}".format(catalog_name)), )
+    l.append(SAXSImageIntent(image=SAXS_2D_I, item_name="AVG frame {}".format(catalog_name), mixins=("SAXSImageIntentBlend",)), )
     l.append(PlotIntent(y=SAXS_1D_I[SAXS_1D_I_projection_key],
                         x=SAXS_1D_I[SAXS_1D_Q_projection_key],
                         labels={"left": "I", "bottom": "Q"},
                         item_name='AVG SAXS curve {}'.format(catalog_name)))
+
+
+    l.append(PlotIntent(y=SAXS_1D_I_partial, x=SAXS_1D_I[SAXS_1D_Q_projection_key],
+                        labels = {"left": "I", "bottom": "Q"},
+                        item_name = 'Stability Plot {}'.format(catalog_name)))
     return l
     # TODO: additionally return intents for masks, rois