diff --git a/build-deb.sh b/build-deb.sh index d771e06..e32e7ab 100755 --- a/build-deb.sh +++ b/build-deb.sh @@ -68,12 +68,12 @@ then bookworm) debian_version=12 ;; - trixie) - debian_version=13 - ;; - sid) - debian_version=13 - ;; + trixie) + debian_version=13 + ;; + sid) + debian_version=13 + ;; esac fi @@ -109,6 +109,7 @@ optional arguments: --debian10 Simulate a debian 10 Buster system --debian11 Simulate a debian 11 Bullseye system --debian12 Simulate a debian 12 Bookworm system + --debian13 Simulate a debian 13 Trixie system " install=0 diff --git a/doc/source/dahu.rst b/doc/source/dahu.rst index d94ed72..c9d5684 100644 --- a/doc/source/dahu.rst +++ b/doc/source/dahu.rst @@ -11,15 +11,15 @@ The *dahu* server executes **jobs**: * The job (de-) serializes JSON strings coming from/returning to Tango * Jobs are executed asynchronously, the request for calculation is answered instantaneously with a *jobid* (an integer, unique for the process). * The *jobid* can be used to poll the server for the status of the job or for manual synchronization (mind that Tango can time-out!). -* When jobs are finished, the client is notified via Tango events about the status +* When jobs are finished, the client is notified via **Tango events** about the status change * Results can be retrieved after the job has finished. Jobs execute **plugin**: ------------------------ -* Plugins are written in Python (extension in Cython or OpenCL are common) +* Plugins are written in Python (extensions in Cython or OpenCL are common) * Plugins can be classes or simple functions -* The input and output MUST be JSON-seriablisable as simple dictionnaries +* The input and output MUST be JSON-serializable as simple dictionaries * Plugins are dynamically loaded from Python modules * Plugins can be profiled for performance analysis diff --git a/plugins/bm29/__init__.py b/plugins/bm29/__init__.py index 20fc1a4..23e75b0 100644 --- a/plugins/bm29/__init__.py +++ b/plugins/bm29/__init__.py @@ -11,7 +11,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "03/12/2024" +__date__ = "05/05/2025" __status__ = "development" __version__ = "0.2.0" @@ -19,6 +19,8 @@ from .integrate import IntegrateMultiframe from .subtracte import SubtractBuffer from .hplc import HPLC +from .mesh import Mesh register(IntegrateMultiframe, fqn="bm29.integratemultiframe") register(SubtractBuffer, fqn="bm29.subtractbuffer") register(HPLC, fqn="bm29.hplc") +register(Mesh, fqn="bm29.mesh") \ No newline at end of file diff --git a/plugins/bm29/common.py b/plugins/bm29/common.py index 39f34e8..900a9a4 100644 --- a/plugins/bm29/common.py +++ b/plugins/bm29/common.py @@ -4,16 +4,16 @@ """Data Analysis plugin for BM29: BioSaxs Common data structures: Sample, Ispyb - + """ __authors__ = ["Jérôme Kieffer"] __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "20/02/2025" +__date__ = "09/03/2026" __status__ = "development" -version = "0.0.2" +__version__ = "0.0.2" import os from pathlib import Path @@ -21,28 +21,27 @@ from typing import NamedTuple import json import logging -logger = logging.getLogger("bm29.common") import numpy from dahu.cache import DataCache from hdf5plugin import Bitshuffle, Zfp -import pyFAI, pyFAI.units +import pyFAI +import pyFAI.integrator.load_engines #noqa +import pyFAI.units from pyFAI.method_registry import IntegrationMethod import fabio -from .nexus import Nexus, get_isotime -# else: -# from pyFAI.io import Nexus, get_isotime - -#cmp contains the compression options, shared by all plugins. Used mainly for images -cmp = cmp_int = Bitshuffle() -cmp_float = Zfp(reversible=True) +logger = logging.getLogger("bm29.common") +#cmp contains the compression options, shared by all plugins. Used mainly for images +cmp = cmp_int = Bitshuffle() +cmp_float = Zfp(reversible=True) +version = __version__ #This is used for NXdata plot style SAXS_STYLE = json.dumps({"signal_scale_type": "log"}, - indent=2, + indent=2, separators=(",\r\n", ":\t")) NORMAL_STYLE = json.dumps({"signal_scale_type": "linear"}, - indent=2, + indent=2, separators=(",\r\n", ":\t")) @@ -89,7 +88,7 @@ def _fromdict(cls, dico): class Sample(NamedTuple): - """ This object represents the sample with the following representation + """ This object represents the sample with the following representation "sample": { "name": "bsa", "description": "protein description like Bovine Serum Albumin", @@ -97,7 +96,7 @@ class Sample(NamedTuple): "concentration": 0, "hplc": "column name and chromatography conditions", "temperature": 20, - "temperature_env": 20}, + "temperature_env": 20}, """ name: str="Unknown sample" description: str=None @@ -150,7 +149,7 @@ def get_equivalent_frames(proba, absolute=0.1, relative=0.2): ext_diag = numpy.zeros(size + 1, dtype=numpy.int16) delta = numpy.zeros(size + 1, dtype=numpy.int16) ext_diag[1:-1] = numpy.diagonal(proba, 1) >= relative - ext_diag[0] = ext_diag[1] + ext_diag[0] = ext_diag[1] delta[0] = ext_diag[1] delta[1:] = ext_diag[1:] - ext_diag[:-1] start = numpy.where(delta > 0)[0] diff --git a/plugins/bm29/hplc.py b/plugins/bm29/hplc.py index 5e3a8da..9e9c1bc 100644 --- a/plugins/bm29/hplc.py +++ b/plugins/bm29/hplc.py @@ -10,9 +10,9 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "27/05/2025" +__date__ = "20/04/2026" __status__ = "development" -__version__ = "0.3.0" +__version__ = "0.4.1" import time import os @@ -21,32 +21,65 @@ from math import log, pi import posixpath import copy -from collections import namedtuple +import zipfile from urllib3.util import parse_url from dahu.plugin import Plugin + # from dahu.utils import fully_qualified_name import logging -logger = logging.getLogger("bm29.hplc") import numpy import h5py -import pyFAI, pyFAI.azimuthalIntegrator, pyFAI.units +import pyFAI +import pyFAI.integrator.azimuthal +import pyFAI.units from pyFAI.method_registry import IntegrationMethod -import freesas, freesas.cormap, freesas.invariants +import freesas +import freesas.cormap +import freesas.invariants from freesas.autorg import auto_gpa, autoRg, auto_guinier from freesas.bift import BIFT +from freesas.app.extract_ascii import write_ascii +from freesas.containers import UVJuice from scipy.optimize import minimize import scipy.signal import scipy.ndimage import sklearn from sklearn.decomposition import NMF -from .common import Sample, Ispyb, get_equivalent_frames, cmp_float, get_integrator, KeyCache, \ - polarization_factor, method, Nexus, get_isotime, SAXS_STYLE, NORMAL_STYLE, \ - Sample, create_nexus_sample +from .common import Ispyb, SAXS_STYLE, NORMAL_STYLE, Sample, create_nexus_sample +from .nexus import Nexus, get_isotime from .ispyb import IspybConnector from .icat import send_icat +from typing import NamedTuple +import matplotlib.pyplot +from freesas.plot import hplc_plot + +logger = logging.getLogger("bm29.hplc") +matplotlib.use("Agg") + +class NexusJuice(NamedTuple): + """All information of an integration file""" -NexusJuice = namedtuple("NexusJuice", "filename h5path npt unit idx Isum q I sigma poni mask energy polarization method sample timestamps") + filename: str + h5path: str + npt: int + unit: pyFAI.units.Unit + idx: numpy.ndarray + Isum: numpy.ndarray + q: numpy.ndarray + I: numpy.ndarray # noqa + sigma: numpy.ndarray + poni: str + mask: numpy.ndarray + energy: float + polarization: float + method: tuple + sample: Sample + timestamps: numpy.ndarray + diode: numpy.ndarray | None = None + + +# NexusJuice = namedtuple("NexusJuice", "filename h5path npt unit idx Isum q I sigma poni mask energy polarization method sample timestamps") def smooth_chromatogram(signal, window): @@ -62,8 +95,8 @@ def smooth_chromatogram(signal, window): # w2 = int(6*sigma) # if w2%2 == 0: w2+=1 # print(wmin, sigma, w2) -# g = scipy.signal.gaussian(wodd, wodd/4) -# g /= g.sum() + # g = scipy.signal.gaussian(wodd, wodd/4) + # g /= g.sum() # small kernel smoothing to remove steps induced by medfilt # smth2 = scipy.signal.convolve(signal, g,"same") @@ -94,18 +127,18 @@ def search_peaks(signal, wmin=10, scale=0.9): if q > p: start = smth[:q] else: - start = smth[p - q:p] + start = smth[p - q : p] if p + q >= signal.size: stop = smth[-q:] else: - stop = smth[p:q + p] + stop = smth[p : q + p] pos = numpy.argmin(abs(start - stop)) - res[p + pos - q: p + pos] = 1 + res[p + pos - q : p + pos] = 1 w *= scale return scipy.ndimage.label(res) -def build_background(I, std=None, keep=0.3): +def build_background(intensity, std=None, keep=0.3): """ Build a background from a SVD and search for the frames looking most like the background. @@ -113,18 +146,23 @@ def build_background(I, std=None, keep=0.3): 2. measure the distance (cormap) of every single frame to the fundamental of the SVD 3. average frames that looks most like the coarse approximation (with deviation) - :param I: 2D array of shape (nframes, nbins) - :param std: same as I but with the standard deviation. + :param intensity: 2D array of shape (nframes, nbins) + :param std: same as intensity but with the standard deviation. :param keep: fraction of frames to consider for background (<1!), 30% looks like a good guess :return: (bg_avg, bg_std, indexes), each 1d of size nbins. + the index of the frames to keep """ - U, S, V = numpy.linalg.svd(I.T, full_matrices=False) + U, S, V = numpy.linalg.svd(intensity.T, full_matrices=False) bg1 = numpy.median(V[0]) * S[0] * U[:, 0] - Pscore = [freesas.cormap.measure_longest(numpy.ascontiguousarray(bg1 - i, dtype=numpy.float64)) for i in I] + Pscore = [ + freesas.cormap.measure_longest( + numpy.ascontiguousarray(bg1 - i, dtype=numpy.float64) + ) + for i in intensity + ] orderd = numpy.argsort(Pscore) - nkeep = int(math.ceil(keep * I.shape[0])) + nkeep = int(math.ceil(keep * intensity.shape[0])) to_keep = numpy.sort(orderd[:nkeep]) - bg_avg = I[to_keep].mean(axis=0) + bg_avg = intensity[to_keep].mean(axis=0) if std is not None: bg_std = numpy.sqrt(((std[to_keep]) ** 2).sum(axis=0)) / len(to_keep) else: @@ -132,9 +170,44 @@ def build_background(I, std=None, keep=0.3): return bg_avg, bg_std, to_keep +def save_zip(filename, config, intensity, sigma): + """Save a stack of intensity into a zipfile with each frames in a dat-file. + + :param filename: name of the zip-file + :param confif: this is some NexusJuice namedtuple. we use only q and the sample description. + :param intensity: 2D array with the intensity of the stack of curves + :param sigma: 2D array with the uncertainties of the stack of frames + :return: nothing + """ + basename = os.path.basename(filename) + base = os.path.splitext(basename)[0] + destz = base + "_%04i.dat" + common = {"q": config.q} + if config.sample: + sample = config.sample + if sample.name: + common["sample"]: sample.name + if sample.buffer: + common["buffer"] = sample.buffer + if sample.temperature_env: + common["storage temperature"] = sample.temperature_env + if sample.temperature: + common["exposure temperature"] = sample.temperature + if sample.concentration: + common["concentration"] = sample.concentration + res = [] + for i, s in zip(intensity, sigma): + r = copy.copy(common) + r["I"] = i + r["std"] = s + res.append(r) + with zipfile.ZipFile(filename, "w") as z: + for idx, frame in enumerate(res): + z.writestr(destz % idx, write_ascii(frame)) + class HPLC(Plugin): - """ Rebuild the complete chromatogram and perform basic analysis on it. + """Rebuild the complete chromatogram and perform basic analysis on it. Typical JSON file: { @@ -146,13 +219,16 @@ class HPLC(Plugin): "measurement_id": -1, "collection_id": -1 }, - "nmf_components": 5, + "nmf_components": 5, + "diode_medfilt": 0, + "uv_datafile": "path to UV .dat file in some gallery", "wait_for": [jobid_img001, jobid_img002], "plugin_name": "bm29.hplc" } """ + NMF_COMP = 5 - "Default number of Non-negative matrix factorisation components. Correspond to the number of spieces" + "Default number of Non-negative matrix factorization components. Correspond to the number of spices" def __init__(self): Plugin.__init__(self) @@ -160,10 +236,12 @@ def __init__(self): self.nxs = None self.output_file = None self.juices = [] + self.uv_data = None self.nmf_components = self.NMF_COMP self.to_pyarch = {} self.ispyb = None self._pid = 0 + self._time_digits = 0 def sequence_index(self): value = self._pid @@ -176,25 +254,41 @@ def setup(self): for job_id in self.input.get("wait_for", []): self.wait_for(job_id) - self.input_files = [os.path.abspath(i) for i in self.input.get("integrated_files", "")] + self.input_files = [ + os.path.abspath(i) for i in self.input.get("integrated_files", "") + ] self.output_file = self.input.get("output_file") if not self.output_file: - dirname, basename = os.path.split(os.path.commonprefix(self.input_files) + "_hplc.h5") + dirname, basename = os.path.split( + os.path.commonprefix(self.input_files) + "_hplc.h5" + ) dirname = os.path.dirname(dirname) -# dirname = os.path.join(dirname, "processed") + # dirname = os.path.join(dirname, "processed") dirname = os.path.join(dirname, "hplc") self.output_file = os.path.join(dirname, basename) if not os.path.isdir(dirname): try: os.makedirs(dirname) except Exception as err: - self.log_warning(f"Unable to create dir {dirname}. {type(err)}: {err}") + self.log_warning( + f"Unable to create dir {dirname}. {type(err)}: {err}" + ) self.log_warning("No output file provided, using " + self.output_file) self.nmf_components = int(self.input.get("nmf_components", self.NMF_COMP)) - #Manage gallery here + uv_datafile = self.input.get("uv_datafile") + if uv_datafile and os.path.exists(uv_datafile): + try: + self.uv_data = UVJuice.from_file(uv_datafile) + except Exception as err: + self.uv_data = None + self.log_warning( + f"Unable to parse {uv_datafile}; {err.__class__.__name__}: {err}" + ) + + # Manage gallery here dirname = os.path.dirname(self.output_file) gallery = os.path.join(dirname, "gallery") if not os.path.isdir(gallery): @@ -212,6 +306,7 @@ def process(self): self.to_pyarch["chunk_size"] = self.juices[0].Isum.size self.to_pyarch["id"] = os.path.commonprefix(self.input_files) self.to_pyarch["sample_name"] = self.juices[0].sample.name + self.build_plot() if not self.input.get("no_ispyb"): self.send_to_ispyb() # self.output["icat"] = @@ -229,25 +324,33 @@ def teardown(self): def create_nexus(self): nxs = Nexus(self.output_file, mode="w") - entry_grp = nxs.new_entry("entry", self.input.get("plugin_name", "dahu"), - title='BioSaxs HPLC experiment', - force_time=get_isotime()) + entry_grp = nxs.new_entry( + "entry", + self.input.get("plugin_name", "dahu"), + title="BioSaxs HPLC experiment", + force_time=get_isotime(), + ) entry_grp["version"] = __version__ nxs.h5.attrs["default"] = entry_grp.name.strip("/") - # Configuration + # Configuration cfg_grp = nxs.new_class(entry_grp, "configuration", "NXnote") - cfg_grp.create_dataset("data", data=json.dumps(self.input, indent=2, separators=(",\r\n", ":\t"))) + cfg_grp.create_dataset( + "data", data=json.dumps(self.input, indent=2, separators=(",\r\n", ":\t")) + ) cfg_grp.create_dataset("format", data="text/json") - # Process 0: Measurement group + # Process 0: Measurement group input_grp = nxs.new_class(entry_grp, "0_measurement", "NXcollection") input_grp["sequence_index"] = self.sequence_index() for idx, filename in enumerate(self.input_files): juice = self.read_nexus(filename) if juice is not None: - rel_path = os.path.relpath(os.path.abspath(filename), os.path.dirname(os.path.abspath(self.output_file))) + rel_path = os.path.relpath( + os.path.abspath(filename), + os.path.dirname(os.path.abspath(self.output_file)), + ) input_grp["LImA_%04i" % idx] = h5py.ExternalLink(rel_path, juice.h5path) self.juices.append(juice) @@ -258,46 +361,123 @@ def create_nexus(self): # Sample: outsourced ! create_nexus_sample(nxs, entry_grp, self.juices[0].sample) - # Process 1: Chromatogram - chroma_grp = nxs.new_class(entry_grp, "1_chromatogram", "NXprocess") - chroma_grp["sequence_index"] = self.sequence_index() nframes = max(i.idx.max() for i in self.juices) + 1 nbin = q.size - I = numpy.zeros((nframes, nbin), dtype=numpy.float32) + I = numpy.zeros((nframes, nbin), dtype=numpy.float32) # noqa sigma = numpy.zeros((nframes, nbin), dtype=numpy.float32) Isum = numpy.zeros(nframes) ids = numpy.arange(nframes) idx = numpy.concatenate([i.idx for i in self.juices]) - timestamps = self.to_pyarch["time"] = numpy.concatenate([i.timestamps for i in self.juices]) + timestamps = self.to_pyarch["time"] = numpy.concatenate( + [i.timestamps for i in self.juices] + ) I[idx] = numpy.vstack([i.I for i in self.juices]) Isum[idx] = numpy.concatenate([i.Isum for i in self.juices]) sigma[idx] = numpy.vstack([i.sigma for i in self.juices]) - hplc_data = nxs.new_class(chroma_grp, "hplc", "NXdata") - hplc_data.attrs["title"] = "Chromatogram" + if len(timestamps): + self._time_digits = len(f"{timestamps[-1]:.0f}") + else: + self._time_digits = 1 + + # Process 0.5: preprocessing + diode_raw = numpy.concatenate([i.diode for i in self.juices]) + medfilt_order = self.input.get("diode_medfilt", 0) + if medfilt_order >= 2: + preproc_grp = nxs.new_class(entry_grp, "0_pre-process", "NXprocess") + preproc_grp["sequence_index"] = self.sequence_index() + preproc_grp["filter_used"] = "scipy.ndimage.median_filter" + preproc_grp["filter_size"] = medfilt_order + + # diode_smooth = scipy.signal.medfilt(diode_raw, medfilt_order) + diode_smooth = scipy.ndimage.median_filter( + diode_raw, medfilt_order, + mode="mirror") + noise = (100.0 * (((diode_raw - diode_smooth) ** 2).mean()) ** 0.5 / + diode_raw.mean()) + preproc_grp.create_dataset("noise", data=noise).attrs["unit"] = r"%" + preproc_grp.create_dataset("diode_raw", data=diode_raw).attrs[ + "interpretation" + ] = "spectrum" + preproc_grp.create_dataset("diode_smooth", data=diode_smooth).attrs[ + "interpretation" + ] = "spectrum" + scale = diode_raw / diode_smooth + I *= numpy.atleast_2d(scale).T # noqa + Isum *= scale + sigma *= numpy.atleast_2d(scale).T + diode = diode_smooth + else: + diode = diode_raw + + # Process 1: Chromatogram + chroma_grp = nxs.new_class(entry_grp, "1_chromatogram", "NXprocess") + chroma_grp["sequence_index"] = self.sequence_index() + + # UV-chromatogram + if self.uv_data: + uv_data = nxs.new_class(chroma_grp, "UV-Vis", "NXdata") + uv_data.attrs["title"] = "UV-Vis - Chromatogram" + uv_data["sequence_index"] = self.sequence_index() + absorbance = uv_data.create_dataset( + "absorbance", data=self.uv_data.absorbance + ) + absorbance.attrs["unit"] = "∅" + absorbance.attrs["interpretation"] = "spectrum" + absorbance.attrs["SILX_style"] = NORMAL_STYLE + uv_data.create_dataset("timestamps", data=self.uv_data.timestamps).attrs[ + "unit" + ] = "s" + uv_data.create_dataset("wavelengths", data=self.uv_data.wavelengths).attrs[ + "unit" + ] = "nm" + uv_data.attrs["signal"] = "absorbance" + uv_data.attrs["axes"] = ["wavelengths", "timestamps"] + + # SAXS-chromatogram + hplc_data = nxs.new_class(chroma_grp, "SAXS", "NXdata") + hplc_data.attrs["title"] = "SAXS - Chromatogram" + hplc_data["sequence_index"] = self.sequence_index() + sum_ds = hplc_data.create_dataset("sum", data=Isum, dtype=numpy.float32) sum_ds.attrs["interpretation"] = "spectrum" sum_ds.attrs["long_name"] = "Summed Intensity" + sum_ds.attrs["SILX_style"] = NORMAL_STYLE + + sum_ds = hplc_data.create_dataset("diode", data=diode, dtype=numpy.float32) + sum_ds.attrs["interpretation"] = "spectrum" + sum_ds.attrs["long_name"] = "Beam-stop diode signal" + sum_ds.attrs["SILX_style"] = NORMAL_STYLE + frame_ds = hplc_data.create_dataset("frame_ids", data=ids, dtype=numpy.uint32) frame_ds.attrs["interpretation"] = "spectrum" frame_ds.attrs["long_name"] = "frame index" + hplc_data.attrs["signal"] = "sum" - hplc_data.attrs["axes"] = "frame_ids" + hplc_data.attrs["axes"] = "timestamps" # "frame_ids" chroma_grp.attrs["default"] = posixpath.relpath(hplc_data.name, chroma_grp.name) entry_grp.attrs["default"] = posixpath.relpath(hplc_data.name, entry_grp.name) - time_ds = hplc_data.create_dataset("timestamps", data=timestamps, dtype=numpy.uint32) + time_ds = hplc_data.create_dataset( + "timestamps", data=timestamps, dtype=numpy.float64 + ) time_ds.attrs["interpretation"] = "spectrum" time_ds.attrs["long_name"] = "Time stamps (s)" - integration_data = nxs.new_class(chroma_grp, "results", "NXdata") + integration_data = nxs.new_class(chroma_grp, "result", "NXdata") chroma_grp.attrs["title"] = str(self.juices[0].sample) - int_ds = integration_data.create_dataset("I", data=numpy.ascontiguousarray(I, dtype=numpy.float32)) - std_ds = integration_data.create_dataset("errors", data=numpy.ascontiguousarray(sigma, dtype=numpy.float32)) + int_ds = integration_data.create_dataset( + "I", data=numpy.ascontiguousarray(I, dtype=numpy.float32) + ) + std_ds = integration_data.create_dataset( + "errors", data=numpy.ascontiguousarray(sigma, dtype=numpy.float32) + ) q_ds = integration_data.create_dataset("q", data=self.juices[0].q) q_ds.attrs["interpretation"] = "spectrum" + q_ds.attrs["unit"] = unit_name + q_ds.attrs["long_name"] = "Scattering vector q (nm⁻¹)" integration_data.attrs["signal"] = "I" integration_data.attrs["axes"] = [".", "q"] integration_data.attrs["SILX_style"] = SAXS_STYLE @@ -309,16 +489,20 @@ def create_nexus(self): int_ds.attrs["scale"] = "log" std_ds.attrs["interpretation"] = "spectrum" - # Process 2: SVD decomposition + save_zip( + os.path.splitext(self.output_file)[0] + ".zip", self.juices[0], I, sigma + ) + + # Process 2: SVD decomposition svd_grp = nxs.new_class(entry_grp, "2_SVD", "NXprocess") svd_grp["sequence_index"] = self.sequence_index() logi = numpy.arcsinh(I.T) U, S, V = numpy.linalg.svd(logi, full_matrices=False) - # Number of Eignevector to keep: + # Number of Eigenvector to keep: svd_grp["Ref"] = "https://arxiv.org/pdf/1305.5870.pdf" beta = nframes / nbin if nframes <= nbin else 1.0 - omega = 0.56 * beta ** 3 - 0.95 * beta ** 2 + 1.82 * beta + 1.43 + omega = 0.56 * beta**3 - 0.95 * beta**2 + 1.82 * beta + 1.43 tau = numpy.median(S) * omega r = numpy.sum(S > tau) @@ -329,13 +513,17 @@ def create_nexus(self): U[:, nflip] = -U[:, nflip] eigen_data = nxs.new_class(svd_grp, "eigenvectors", "NXdata") - eigen_ds = eigen_data.create_dataset("U", data=numpy.ascontiguousarray(U.T[:r], dtype=numpy.float32)) + eigen_ds = eigen_data.create_dataset( + "U", data=numpy.ascontiguousarray(U.T[:r], dtype=numpy.float32) + ) eigen_ds.attrs["interpretation"] = "spectrum" eigen_data.attrs["signal"] = "U" eigen_data.attrs["SILX_style"] = SAXS_STYLE chroma_data = nxs.new_class(svd_grp, "chromatogram", "NXdata") - chroma_ds = chroma_data.create_dataset("V", data=numpy.ascontiguousarray(V[:r], dtype=numpy.float32)) + chroma_ds = chroma_data.create_dataset( + "V", data=numpy.ascontiguousarray(V[:r], dtype=numpy.float32) + ) chroma_ds.attrs["interpretation"] = "spectrum" chroma_data.attrs["signal"] = "V" chroma_data.attrs["SILX_style"] = NORMAL_STYLE @@ -343,13 +531,12 @@ def create_nexus(self): svd_grp.create_dataset("eigenvalues", data=S[:r], dtype=numpy.float32) svd_grp.attrs["default"] = posixpath.relpath(chroma_data.name, svd_grp.name) - # Process 3: NMF matrix decomposition + # Process 3: NMF matrix decomposition nmf_grp = nxs.new_class(entry_grp, "3_NMF", "NXprocess") nmf_grp["sequence_index"] = self.sequence_index() nmf_grp["program"] = "sklearn.decomposition.NMF" nmf_grp["version"] = sklearn.__version__ - nmf = NMF(n_components=self.nmf_components, init='nndsvd', - max_iter=1000) + nmf = NMF(n_components=self.nmf_components, init="nndsvd", max_iter=1000) try: W = nmf.fit_transform(I.T) except ValueError as err: @@ -357,7 +544,9 @@ def create_nexus(self): nmf_grp[err.__class__.__name__] = str(err) else: eigen_data = nxs.new_class(nmf_grp, "eigenvectors", "NXdata") - eigen_ds = eigen_data.create_dataset("W", data=numpy.ascontiguousarray(W.T, dtype=numpy.float32)) + eigen_ds = eigen_data.create_dataset( + "W", data=numpy.ascontiguousarray(W.T, dtype=numpy.float32) + ) eigen_ds.attrs["interpretation"] = "spectrum" eigen_data.attrs["signal"] = "W" eigen_data.attrs["SILX_style"] = SAXS_STYLE @@ -367,40 +556,51 @@ def create_nexus(self): H = nmf.components_ chroma_data = nxs.new_class(nmf_grp, "chromatogram", "NXdata") - chroma_ds = chroma_data.create_dataset("H", data=numpy.ascontiguousarray(H, dtype=numpy.float32)) + chroma_ds = chroma_data.create_dataset( + "H", data=numpy.ascontiguousarray(H, dtype=numpy.float32) + ) chroma_ds.attrs["interpretation"] = "spectrum" chroma_data.attrs["signal"] = "H" chroma_data.attrs["SILX_style"] = NORMAL_STYLE nmf_grp.attrs["default"] = posixpath.relpath(chroma_data.name, nmf_grp.name) - # Process 5: Background estimation + # Process 5: Background estimation bg_grp = nxs.new_class(entry_grp, "4_background", "NXprocess") bg_grp["sequence_index"] = self.sequence_index() bg_grp["keep"] = keep = 0.3 - bg_grp["keep"].attrs["info"] = "Fraction of curves to be considered as background" + bg_grp["keep"].attrs["info"] = ( + "Fraction of curves to be considered as background" + ) bg_avg, bg_std, to_keep = build_background(I, sigma, keep=keep) to_keep = numpy.ascontiguousarray(to_keep, dtype=numpy.int32) kept_ds = bg_grp.create_dataset("kept", data=to_keep) - kept_ds.attrs["info"] = "Index of curves used to calculate the background scattering" + kept_ds.attrs["info"] = ( + "Index of curves used to calculate the background scattering" + ) self.to_pyarch["buffer_frames"] = to_keep self.to_pyarch["buffer_I"] = bg_avg self.to_pyarch["buffer_Stdev"] = bg_std - bg_data = nxs.new_class(bg_grp, "results", "NXdata") + bg_data = nxs.new_class(bg_grp, "result", "NXdata") bg_data.attrs["signal"] = "I" bg_data.attrs["SILX_style"] = SAXS_STYLE bg_data.attrs["axes"] = radial_unit - bg_ds = bg_data.create_dataset("I", data=numpy.ascontiguousarray(bg_avg, dtype=numpy.float32)) + bg_ds = bg_data.create_dataset( + "I", data=numpy.ascontiguousarray(bg_avg, dtype=numpy.float32) + ) bg_ds.attrs["interpretation"] = "spectrum" - bg_q_ds = bg_data.create_dataset(radial_unit, - data=numpy.ascontiguousarray(q, dtype=numpy.float32)) + bg_q_ds = bg_data.create_dataset( + radial_unit, data=numpy.ascontiguousarray(q, dtype=numpy.float32) + ) bg_q_ds.attrs["units"] = unit_name radius_unit = "nm" if "nm" in unit_name else "Å" bg_q_ds.attrs["long_name"] = f"Scattering vector q ({radius_unit}⁻¹)" - bg_std_ds = bg_data.create_dataset("errors", data=numpy.ascontiguousarray(bg_std, dtype=numpy.float32)) + bg_std_ds = bg_data.create_dataset( + "errors", data=numpy.ascontiguousarray(bg_std, dtype=numpy.float32) + ) bg_std_ds.attrs["interpretation"] = "spectrum" bg_grp.attrs["default"] = posixpath.relpath(bg_data.name, bg_grp.name) I_sub = I - bg_avg - Istd_sub = numpy.sqrt(sigma ** 2 + bg_std ** 2) + Istd_sub = numpy.sqrt(sigma**2 + bg_std**2) self.to_pyarch["scattering_I"] = I self.to_pyarch["scattering_Stdev"] = sigma @@ -408,7 +608,7 @@ def create_nexus(self): self.to_pyarch["subtracted_Stdev"] = Istd_sub self.to_pyarch["sum_I"] = Isum - # Process 5: fraction of chromatogram analysis + # Process 5: fraction of chromatogram analysis fraction_grp = nxs.new_class(entry_grp, "5_SEC_fractions", "NXprocess") fraction_grp["sequence_index"] = self.sequence_index() fraction_grp["minimum_size"] = window = 10 @@ -416,13 +616,17 @@ def create_nexus(self): fractions, nfractions = search_peaks(Isum, window) self.to_pyarch["merge_frames"] = numpy.zeros((nfractions, 2), dtype=numpy.int32) self.to_pyarch["merge_I"] = numpy.zeros((nfractions, nbin), dtype=numpy.float32) - self.to_pyarch["merge_Stdev"] = numpy.zeros((nfractions, nbin), dtype=numpy.float32) + self.to_pyarch["merge_Stdev"] = numpy.zeros( + (nfractions, nbin), dtype=numpy.float32 + ) if nfractions: - for i, fraction in enumerate(scipy.ndimage.find_objects(fractions, nfractions)): + for i, fraction in enumerate( + scipy.ndimage.find_objects(fractions, nfractions) + ): self.one_fraction(fraction[0], i, nxs, fraction_grp) - # Process 6: All other calculation for ISPyB: + # Process 6: All other calculation for ISPyB: t = self.build_ispyb_group(nxs, entry_grp) self.log_warning(f"Ispyb structure creation took {t:.3f}s") @@ -445,7 +649,17 @@ def one_fraction(self, fraction, index, nxs, top_grp): I_sub = self.to_pyarch["subtracted_I"] sigma = self.to_pyarch["subtracted_Stdev"] - f_grp = nxs.new_class(top_grp, f"{fraction.start}-{fraction.stop}", "NXprocess") + time = self.to_pyarch["time"] + + template = f"%0{self._time_digits}.0fs-%0{self._time_digits}.0fs" + time_slice = template % (time[fraction.start], + time[min(fraction.stop, time.size-1)]) + # time_slice = f"{time[fraction.start]:.0f}s-{time[min(fraction.stop, time.size-1)]:.0f}s" + f_grp = nxs.new_class( + top_grp, + time_slice, + "NXprocess", + ) f_grp["sequence_index"] = self.sequence_index() f_grp["first_frame"] = fraction.start f_grp["last_frame"] = fraction.stop @@ -456,21 +670,27 @@ def one_fraction(self, fraction, index, nxs, top_grp): avg_data = nxs.new_class(f_grp, "1_average", "NXdata") avg_data["sequence_index"] = self.sequence_index() avg_data.attrs["SILX_style"] = SAXS_STYLE - avg_data.attrs["title"] = f"{sample.name}, frames {fraction.start}-{fraction.stop} averaged, buffer subtracted" + avg_data.attrs["title"] = ( + f"{sample.name}, frames {fraction.start}-{fraction.stop} averaged ({time_slice}), buffer subtracted" + ) avg_data.attrs["signal"] = "I" avg_data.attrs["axes"] = radial_unit f_grp.attrs["default"] = posixpath.relpath(avg_data.name, f_grp.name) - avg_q_ds = avg_data.create_dataset(radial_unit, - data=numpy.ascontiguousarray(q, dtype=numpy.float32)) + avg_q_ds = avg_data.create_dataset( + radial_unit, data=numpy.ascontiguousarray(q, dtype=numpy.float32) + ) avg_q_ds.attrs["units"] = unit_name radius_unit = "nm" if "nm" in unit_name else "Å" avg_q_ds.attrs["long_name"] = f"Scattering vector q ({radius_unit}⁻¹)" I_frc = I_sub[fraction].mean(axis=0) fsig2 = sigma[fraction] ** 2 sigma_frc = numpy.sqrt(fsig2.sum(axis=0)) / fsig2.shape[0] - ai2_int_ds = avg_data.create_dataset("I", data=numpy.ascontiguousarray(I_frc, dtype=numpy.float32)) - ai2_std_ds = avg_data.create_dataset("errors", - data=numpy.ascontiguousarray(sigma_frc, dtype=numpy.float32)) + ai2_int_ds = avg_data.create_dataset( + "I", data=numpy.ascontiguousarray(I_frc, dtype=numpy.float32) + ) + ai2_std_ds = avg_data.create_dataset( + "errors", data=numpy.ascontiguousarray(sigma_frc, dtype=numpy.float32) + ) ai2_int_ds.attrs["interpretation"] = "spectrum" ai2_int_ds.attrs["units"] = "arbitrary" @@ -484,7 +704,7 @@ def one_fraction(self, fraction, index, nxs, top_grp): self.to_pyarch["merge_I"][index] = I_frc self.to_pyarch["merge_Stdev"][index] = sigma_frc - # Process 4: Guinier analysis + # Process 4: Guinier analysis guinier_grp = nxs.new_class(f_grp, "2_Guinier_analysis", "NXprocess") guinier_grp["sequence_index"] = self.sequence_index() guinier_grp["program"] = "freesas.autorg" @@ -493,16 +713,15 @@ def one_fraction(self, fraction, index, nxs, top_grp): guinier_autorg = nxs.new_class(guinier_grp, "autorg", "NXcollection") guinier_gpa = nxs.new_class(guinier_grp, "gpa", "NXcollection") guinier_guinier = nxs.new_class(guinier_grp, "guinier", "NXcollection") - guinier_data = nxs.new_class(guinier_grp, "results", "NXdata") + guinier_data = nxs.new_class(guinier_grp, "result", "NXdata") guinier_data.attrs["SILX_style"] = NORMAL_STYLE guinier_data.attrs["title"] = "Guinier analysis" - # Stage4 processing: autorg and auto_gpa + # Stage4 processing: autorg and auto_gpa sasm = numpy.vstack((q, I_frc, sigma_frc)).T - try: gpa = auto_gpa(sasm) except Exception as error: - guinier_gpa["Failed"] = "%s: %s" % (error.__class__.__name__, error) + guinier_gpa["Failed"] = f"{error.__class__.__name__}: {error}" gpa = None else: # "Rg sigma_Rg I0 sigma_I0 start_point end_point quality aggregated" @@ -520,7 +739,7 @@ def one_fraction(self, fraction, index, nxs, top_grp): try: guinier = auto_guinier(sasm) except Exception as error: - guinier_guinier["Failed"] = "%s: %s" % (error.__class__.__name__, error) + guinier_guinier["Failed"] = f"{error.__class__.__name__}: {error}" guinier = None else: # "Rg sigma_Rg I0 sigma_I0 start_point end_point quality aggregated" @@ -540,7 +759,7 @@ def one_fraction(self, fraction, index, nxs, top_grp): try: autorg = autoRg(sasm) except Exception as err: - guinier_autorg["Failed"] = "%s: %s" % (err.__class__.__name__, err) + guinier_autorg["Failed"] = f"{err.__class__.__name__}: {err}" autorg = None else: if autorg.Rg < 0: @@ -574,16 +793,16 @@ def one_fraction(self, fraction, index, nxs, top_grp): guinier = None guinier_data["source"] = "None" - # Stage #4 Guinier plot generation: + # Stage #4 Guinier plot generation: - q, I, err = sasm.T[:3] + q, I, err = sasm.T[:3] # noqa mask = (I > 0) & numpy.isfinite(I) & (q > 0) & numpy.isfinite(q) if err is not None: mask &= (err > 0.0) & numpy.isfinite(err) mask = mask.astype(bool) if guinier: intercept = numpy.log(guinier.I0) - slope = -guinier.Rg ** 2 / 3.0 + slope = -(guinier.Rg**2) / 3.0 invalid = numpy.where(q > 1.5 / guinier.Rg)[0] if invalid.size: end = invalid[0] @@ -594,7 +813,7 @@ def one_fraction(self, fraction, index, nxs, top_grp): dlogI = err[mask] / logI q2_ds = guinier_data.create_dataset("q2", data=q2.astype(numpy.float32)) q2_ds.attrs["unit"] = radius_unit + "⁻²" - q2_ds.attrs["long_name"] = "q² (%s⁻²)" % radius_unit + q2_ds.attrs["long_name"] = f"q² ({radius_unit}⁻²)" q2_ds.attrs["interpretation"] = "spectrum" lnI_ds = guinier_data.create_dataset("logI", data=logI.astype(numpy.float32)) lnI_ds.attrs["long_name"] = "log(I)" @@ -611,24 +830,30 @@ def one_fraction(self, fraction, index, nxs, top_grp): guinier_data_attrs["signal"] = "logI" guinier_data_attrs["axes"] = "q2" guinier_data_attrs["auxiliary_signals"] = "fit" - guinier_grp.attrs["default"] = posixpath.relpath(guinier_data.name, guinier_grp.name) + guinier_grp.attrs["default"] = posixpath.relpath( + guinier_data.name, guinier_grp.name + ) if guinier is None: f_grp.attrs["default"] = posixpath.relpath(avg_data.name, f_grp.name) - self.log_error("No Guinier region found, data of dubious quality", do_raise=False) + self.log_error( + "No Guinier region found, data of dubious quality", do_raise=False + ) return - # Process 5: Kratky plot + # Process 5: Kratky plot kratky_grp = nxs.new_class(f_grp, "3_dimensionless_Kratky_plot", "NXprocess") kratky_grp["sequence_index"] = self.sequence_index() kratky_grp["program"] = "freesas.autorg" kratky_grp["version"] = freesas.version kratky_grp["date"] = get_isotime() - kratky_data = nxs.new_class(kratky_grp, "results", "NXdata") + kratky_data = nxs.new_class(kratky_grp, "result", "NXdata") kratky_data.attrs["SILX_style"] = NORMAL_STYLE kratky_data.attrs["title"] = "Dimensionless Kratky plots" - kratky_grp.attrs["default"] = posixpath.relpath(kratky_data.name, kratky_grp.name) + kratky_grp.attrs["default"] = posixpath.relpath( + kratky_data.name, kratky_grp.name + ) - # Stage #5 Kratky plot generation: + # Stage #5 Kratky plot generation: Rg = guinier.Rg I0 = guinier.I0 xdata = q * Rg @@ -637,21 +862,21 @@ def one_fraction(self, fraction, index, nxs, top_grp): qRg_ds = kratky_data.create_dataset("qRg", data=xdata.astype(numpy.float32)) qRg_ds.attrs["interpretation"] = "spectrum" qRg_ds.attrs["long_name"] = "q·Rg (unit-less)" - k_ds = kratky_data.create_dataset("q2Rg2I/I0", data=ydata.astype(numpy.float32)) + k_ds = kratky_data.create_dataset("q2Rg2I÷I0", data=ydata.astype(numpy.float32)) k_ds.attrs["interpretation"] = "spectrum" k_ds.attrs["long_name"] = "q²Rg²I(q)/I₀" ke_ds = kratky_data.create_dataset("errors", data=dy.astype(numpy.float32)) ke_ds.attrs["interpretation"] = "spectrum" kratky_data_attrs = kratky_data.attrs - kratky_data_attrs["signal"] = "q2Rg2I/I0" - kratky_data_attrs["axes"] = "qRg" + kratky_data_attrs["signal"] = k_ds.name + kratky_data_attrs["axes"] = qRg_ds.name - # stage 6: Rambo-Tainer invariant + # stage 6: Rambo-Tainer invariant rti_grp = nxs.new_class(f_grp, "4_invariants", "NXprocess") rti_grp["sequence_index"] = self.sequence_index() rti_grp["program"] = "freesas.invariants" rti_grp["version"] = freesas.version - rti_data = nxs.new_class(rti_grp, "results", "NXdata") + rti_data = nxs.new_class(rti_grp, "result", "NXdata") # average_data.attrs["SILX_style"] = SAXS_STYLE # average_data.attrs["signal"] = "intensity_normed" # Rambo_Tainer @@ -682,18 +907,20 @@ def one_fraction(self, fraction, index, nxs, top_grp): volume_ds.attrs["unit"] = "nm³" volume_ds.attrs["formula"] = "Porod: V = 2*π²I₀²/(sum_q I(q)q² dq)" - # stage 7: Pair distribution function, what is the equivalent of datgnom - bift_grp = nxs.new_class(f_grp, "5_indirect_Fourier_transformation", "NXprocess") + # stage 7: Pair distribution function, what is the equivalent of datgnom + bift_grp = nxs.new_class( + f_grp, "5_indirect_Fourier_transformation", "NXprocess" + ) bift_grp["sequence_index"] = self.sequence_index() bift_grp["program"] = "freesas.bift" bift_grp["version"] = freesas.version bift_grp["date"] = get_isotime() - bift_data = nxs.new_class(bift_grp, "results", "NXdata") + bift_data = nxs.new_class(bift_grp, "result", "NXdata") bift_data.attrs["SILX_style"] = NORMAL_STYLE bift_data.attrs["title"] = "Pair distance distribution function p(r)" cfg_grp = nxs.new_class(bift_grp, "configuration", "NXcollection") - # Process stage7, i.e. perform the IFT + # Process stage7, i.e. perform the IFT try: bo = BIFT(q, I, err) cfg_grp["Rg"] = guinier.Rg @@ -708,29 +935,32 @@ def one_fraction(self, fraction, index, nxs, top_grp): cfg_grp["alpha_inf"] = 1 / alpha_max cfg_grp["alpha_scan_steps"] = 11 - key = bo.grid_scan(Dmax, Dmax, 1, - 1.0 / alpha_max, alpha_max, 11, npt) + key = bo.grid_scan(Dmax, Dmax, 1, 1.0 / alpha_max, alpha_max, 11, npt) Dmax, alpha = key[:2] # Then scan on Dmax: cfg_grp["Dmax_sup"] = guinier.Rg * 4 cfg_grp["Dmax_inf"] = guinier.Rg * 2 cfg_grp["Dmax_scan_steps"] = 5 - key = bo.grid_scan(guinier.Rg * 2, guinier.Rg * 4, 5, - alpha, alpha, 1, npt) + key = bo.grid_scan(guinier.Rg * 2, guinier.Rg * 4, 5, alpha, alpha, 1, npt) Dmax, alpha = key[:2] if bo.evidence_cache[key].converged: bo.update_wisdom() use_wisdom = True else: use_wisdom = False - res = minimize(bo.opti_evidence, (Dmax, log(alpha)), args=(npt, use_wisdom), method="powell") + res = minimize( + bo.opti_evidence, + (Dmax, log(alpha)), + args=(npt, use_wisdom), + method="powell", + ) cfg_grp["Powell_steps"] = res.nfev cfg_grp["Monte-Carlo_steps"] = 0 + stats = bo.calc_stats() except Exception as error: - bift_grp["Failed"] = "%s: %s" % (error.__class__.__name__, error) + bift_grp["Failed"] = f"{error.__class__.__name__}: {error}" bo = None else: - stats = bo.calc_stats() bift_grp["alpha"] = stats.alpha_avg bift_grp["alpha_error"] = stats.alpha_std self.Dmax = bift_grp["Dmax"] = stats.Dmax_avg @@ -746,12 +976,16 @@ def one_fraction(self, fraction, index, nxs, top_grp): bift_grp["I0"] = stats.I0_avg bift_grp["I0_error"] = stats.I0_std # Now the plot: - r_ds = bift_data.create_dataset("r", data=stats.radius.astype(numpy.float32)) + r_ds = bift_data.create_dataset( + "r", data=stats.radius.astype(numpy.float32) + ) r_ds.attrs["interpretation"] = "spectrum" r_ds.attrs["unit"] = radius_unit - r_ds.attrs["long_name"] = "radius r(%s)" % radius_unit - p_ds = bift_data.create_dataset("p(r)", data=stats.density_avg.astype(numpy.float32)) + r_ds.attrs["long_name"] = f"radius r({radius_unit})" + p_ds = bift_data.create_dataset( + "p(r)", data=stats.density_avg.astype(numpy.float32) + ) p_ds.attrs["interpretation"] = "spectrum" bift_data["errors"] = stats.density_std bift_data.attrs["signal"] = "p(r)" @@ -760,7 +994,9 @@ def one_fraction(self, fraction, index, nxs, top_grp): r = stats.radius T = numpy.outer(q, r / pi) T = (4 * pi * (r[-1] - r[0]) / (len(r) - 1)) * numpy.sinc(T) - bift_ds = avg_data.create_dataset("BIFT", data=T.dot(stats.density_avg).astype(numpy.float32)) + bift_ds = avg_data.create_dataset( + "BIFT", data=T.dot(stats.density_avg).astype(numpy.float32) + ) bift_ds.attrs["interpretation"] = "spectrum" avg_data.attrs["auxiliary_signals"] = "BIFT" bift_grp.attrs["default"] = posixpath.relpath(bift_data.name, bift_grp.name) @@ -780,28 +1016,55 @@ def normalize(dataset, dtype=numpy.float32): dataset[mask] = 0.0 return numpy.ascontiguousarray(dataset, dtype=dtype) - keys = ["buffer_frames", "buffer_I", "buffer_Stdev", "Dmax", "gnom", "I0", "I0_Stdev", "mass", "mass_Stdev", "merge_frames", "merge_I", "merge_Stdev", - "q", "Qr", "Qr_Stdev", "quality", "Rg", "Rg_Stdev", "scattering_I", "scattering_Stdev", "subtracted_I", "subtracted_Stdev", "sum_I", - "time", "total", "Vc", "Vc_Stdev", "volume"] - keys_extra = {"Dmax": "Maximum diameter from IFT, skipped", - "gnom": "Radius of gyration from IFT, skipped", - "I0": "Forward scattering from GPA", - "I0_Stdev": "Uncertainty on forward scattering", - "mass": "Estimated protein weight from RT analysis", - "mass_Stdev": "Uncertainty on the mass", - "Qr": "RT invariant", - "Qr_Stdev":"Uncertainty on RT invariant", - "quality": "Quality estimated from GPA", - "Rg": "radius of gyration obtained from GPA", - "Rg_Stdev": "uncertainty on Rg", - "total": "skipped", - "Vc": "Volume of correlation obtained from RT", - "Vc_Stdev": "Uncertainty on volume", - "volume": "Molecular volume obtained from Porrod analysis", - "sum_I": "Total scattering of the frame", - "time": "Timestamps", - - } + keys = [ + "buffer_frames", + "buffer_I", + "buffer_Stdev", + "Dmax", + "gnom", + "I0", + "I0_Stdev", + "mass", + "mass_Stdev", + "merge_frames", + "merge_I", + "merge_Stdev", + "q", + "Qr", + "Qr_Stdev", + "quality", + "Rg", + "Rg_Stdev", + "scattering_I", + "scattering_Stdev", + "subtracted_I", + "subtracted_Stdev", + "sum_I", + "time", + "total", + "Vc", + "Vc_Stdev", + "volume", + ] + keys_extra = { + "Dmax": "Maximum diameter from IFT, skipped", + "gnom": "Radius of gyration from IFT, skipped", + "I0": "Forward scattering from GPA", + "I0_Stdev": "Uncertainty on forward scattering", + "mass": "Estimated protein weight from RT analysis", + "mass_Stdev": "Uncertainty on the mass", + "Qr": "RT invariant", + "Qr_Stdev": "Uncertainty on RT invariant", + "quality": "Quality estimated from GPA", + "Rg": "radius of gyration obtained from GPA", + "Rg_Stdev": "uncertainty on Rg", + "total": "skipped", + "Vc": "Volume of correlation obtained from RT", + "Vc_Stdev": "Uncertainty on volume", + "volume": "Molecular volume obtained from Porrod analysis", + "sum_I": "Total scattering of the frame", + "time": "Timestamps", + } start_time = time.perf_counter() ispyb_grp = nxs.new_class(top_grp, "6_ISPyB ", "NXcollection") ispyb_grp["sequence_index"] = self.sequence_index() @@ -812,29 +1075,64 @@ def normalize(dataset, dtype=numpy.float32): q = self.juices[0].q.astype(numpy.float64) ds = ispyb_grp.create_dataset("q", data=normalize(q, dtype=numpy.float32)) - ds.attrs["info"] = "Scattering vector length, common for all scattering intensities" - - ds = ispyb_grp.create_dataset("buffer_frames", data=self.to_pyarch["buffer_frames"]) - ds.attrs["info"] = "Index of frames used to calculate the background scattring (buffer frames)" - ds = ispyb_grp.create_dataset("buffer_I", data=normalize(self.to_pyarch["buffer_I"], dtype=numpy.float32)) + ds.attrs["info"] = ( + "Scattering vector length, common for all scattering intensities" + ) + + ds = ispyb_grp.create_dataset( + "buffer_frames", data=self.to_pyarch["buffer_frames"] + ) + ds.attrs["info"] = ( + "Index of frames used to calculate the background scattring (buffer frames)" + ) + ds = ispyb_grp.create_dataset( + "buffer_I", data=normalize(self.to_pyarch["buffer_I"], dtype=numpy.float32) + ) ds.attrs["info"] = "Averaged background scattering signal (buffer)" - ds = ispyb_grp.create_dataset("buffer_Stdev", data=normalize(self.to_pyarch["buffer_Stdev"], dtype=numpy.float32)) + ds = ispyb_grp.create_dataset( + "buffer_Stdev", + data=normalize(self.to_pyarch["buffer_Stdev"], dtype=numpy.float32), + ) ds.attrs["info"] = "Standard deviation of background scattering signal (buffer)" - ds = ispyb_grp.create_dataset("merge_frames", data=normalize(self.to_pyarch["merge_frames"], dtype=numpy.float32)) + ds = ispyb_grp.create_dataset( + "merge_frames", + data=normalize(self.to_pyarch["merge_frames"], dtype=numpy.float32), + ) ds.attrs["info"] = "Frames merged for each fraction" - ds = ispyb_grp.create_dataset("merge_I", data=normalize(self.to_pyarch["merge_I"], dtype=numpy.float32)) + ds = ispyb_grp.create_dataset( + "merge_I", data=normalize(self.to_pyarch["merge_I"], dtype=numpy.float32) + ) ds.attrs["info"] = "Scattering from merged frames in each fraction" - ds = ispyb_grp.create_dataset("merge_Stdev", data=normalize(self.to_pyarch["merge_Stdev"], dtype=numpy.float32)) - ds.attrs["info"] = "Uncertainties on scattering from merged frames in each fraction" + ds = ispyb_grp.create_dataset( + "merge_Stdev", + data=normalize(self.to_pyarch["merge_Stdev"], dtype=numpy.float32), + ) + ds.attrs["info"] = ( + "Uncertainties on scattering from merged frames in each fraction" + ) "", "", "", "" - ds = ispyb_grp.create_dataset("scattering_I", data=normalize(self.to_pyarch["scattering_I"], dtype=numpy.float32)) + ds = ispyb_grp.create_dataset( + "scattering_I", + data=normalize(self.to_pyarch["scattering_I"], dtype=numpy.float32), + ) ds.attrs["info"] = "Scattering of each individual frame" - ds = ispyb_grp.create_dataset("scattering_Stdev", data=normalize(self.to_pyarch["scattering_Stdev"], dtype=numpy.float32)) + ds = ispyb_grp.create_dataset( + "scattering_Stdev", + data=normalize(self.to_pyarch["scattering_Stdev"], dtype=numpy.float32), + ) ds.attrs["info"] = "Uncertainties on the scattering of each individual frame" - ds = ispyb_grp.create_dataset("subtracted_I", data=normalize(self.to_pyarch["subtracted_I"], dtype=numpy.float32)) + ds = ispyb_grp.create_dataset( + "subtracted_I", + data=normalize(self.to_pyarch["subtracted_I"], dtype=numpy.float32), + ) ds.attrs["info"] = "Background subtracted scattering of each individual frame" - ds = ispyb_grp.create_dataset("subtracted_Stdev", data=normalize(self.to_pyarch["subtracted_Stdev"], dtype=numpy.float32)) - ds.attrs["info"] = "Uncertainties on background subtracted scattering of each individual frame" + ds = ispyb_grp.create_dataset( + "subtracted_Stdev", + data=normalize(self.to_pyarch["subtracted_Stdev"], dtype=numpy.float32), + ) + ds.attrs["info"] = ( + "Uncertainties on background subtracted scattering of each individual frame" + ) for k1 in keys_extra: if k1 not in self.to_pyarch: @@ -847,26 +1145,32 @@ def normalize(dataset, dtype=numpy.float32): guinier = freesas.autorg.auto_gpa(sasm) try: rti = freesas.invariants.calc_Rambo_Tainer(sasm, guinier) - except: + except Exception: rti = None try: porod = freesas.invariants.calc_Porod(sasm, guinier) - except: + except Exception: porod = None - except: + except Exception: guinier = rti = porod = None if guinier is not None: - for k, v in zip(["Rg", "Rg_Stdev", "I0", "I0_Stdev", "quality"], - ['Rg', 'sigma_Rg', 'I0', 'sigma_I0', 'quality']): + for k, v in zip( + ["Rg", "Rg_Stdev", "I0", "I0_Stdev", "quality"], + ["Rg", "sigma_Rg", "I0", "sigma_I0", "quality"], + ): self.to_pyarch[k][i] = guinier.__getattribute__(v) if rti is not None: - for k, v in zip(["Vc", "Vc_Stdev", "Qr", "Qr_Stdev", "mass", "mass_Stdev"], - ['Vc', 'sigma_Vc', 'Qr', 'sigma_Qr', 'mass', 'sigma_mass']): + for k, v in zip( + ["Vc", "Vc_Stdev", "Qr", "Qr_Stdev", "mass", "mass_Stdev"], + ["Vc", "sigma_Vc", "Qr", "sigma_Qr", "mass", "sigma_mass"], + ): self.to_pyarch[k][i] = rti.__getattribute__(v) if porod is not None: self.to_pyarch["volume"][i] = porod for k, v in keys_extra.items(): - ds = ispyb_grp.create_dataset(k, data=normalize(self.to_pyarch[k], dtype=numpy.float32)) + ds = ispyb_grp.create_dataset( + k, data=normalize(self.to_pyarch[k], dtype=numpy.float32) + ) ds.attrs["info"] = v # create all symbolic links at the top level for Ispyb compatibility @@ -877,20 +1181,20 @@ def normalize(dataset, dtype=numpy.float32): @staticmethod def read_nexus(filename): - "return some NexusJuice from a HDF5 file " + "return some NexusJuice from a HDF5 file" with Nexus(filename, "r") as nxsr: entry_name = nxsr.h5.attrs["default"] entry_grp = nxsr.h5[entry_name] h5path = entry_grp.name - nxdata_grp = nxsr.h5[entry_grp.attrs["default"]] + nxdata_grp = entry_grp[entry_grp.attrs["default"]] assert nxdata_grp.name.endswith("hplc") # we are reading HPLC data signal = nxdata_grp.attrs["signal"] axis = nxdata_grp.attrs["axes"] Isum = nxdata_grp[signal][()] idx = nxdata_grp[axis][()] - integrated = nxdata_grp.parent["results"] + integrated = nxdata_grp.parent["result"] signal = integrated.attrs["signal"] - I = integrated[signal][()] + I = integrated[signal][()] # noqa axes = integrated.attrs["axes"][-1] q = integrated[axes][()] sigma = integrated["errors"][()] @@ -902,40 +1206,115 @@ def read_nexus(filename): if not os.path.exists(poni): poni = str(integration_grp["configuration/data"][()]).strip() polarization = integration_grp["configuration/polarization_factor"][()] - method = IntegrationMethod.select_method(**json.loads(integration_grp["configuration/integration_method"][()]))[0] + method = IntegrationMethod.select_method( + **json.loads(integration_grp["configuration/integration_method"][()]) + )[0] instrument_grp = nxsr.get_class(entry_grp, class_type="NXinstrument")[0] detector_grp = nxsr.get_class(instrument_grp, class_type="NXdetector")[0] mask = detector_grp["pixel_mask"].attrs["filename"] mono_grp = nxsr.get_class(instrument_grp, class_type="NXmonochromator")[0] energy = mono_grp["energy"][()] -# img_grp = nxsr.get_class(entry_grp["3_time_average"], class_type="NXdata")[0] -# image2d = img_grp["intensity_normed"][()] -# error2d = img_grp["intensity_std"][()] + # img_grp = nxsr.get_class(entry_grp["3_time_average"], class_type="NXdata")[0] + # image2d = img_grp["intensity_normed"][()] + # error2d = img_grp["intensity_std"][()] # Read the sample description: sample_grp = nxsr.get_class(entry_grp, class_type="NXsample")[0] sample_name = posixpath.split(sample_grp.name)[-1] buffer = sample_grp["buffer"][()] if "buffer" in sample_grp else "" - concentration = sample_grp["concentration"][()] if "concentration" in sample_grp else "" - description = sample_grp["description"][()] if "description" in sample_grp else "" + concentration = ( + sample_grp["concentration"][()] if "concentration" in sample_grp else "" + ) + description = ( + sample_grp["description"][()] if "description" in sample_grp else "" + ) hplc = sample_grp["hplc"][()] if "hplc" in sample_grp else "" - temperature = sample_grp["temperature"][()] if "temperature" in sample_grp else "" - temperature_env = sample_grp["temperature_env"][()] if "temperature_env" in sample_grp else "" - sample = Sample(sample_name, description, buffer, concentration, hplc, temperature_env, temperature) + temperature = ( + sample_grp["temperature"][()] if "temperature" in sample_grp else "" + ) + temperature_env = ( + sample_grp["temperature_env"][()] + if "temperature_env" in sample_grp + else "" + ) + sample = Sample( + sample_name, + description, + buffer, + concentration, + hplc, + temperature_env, + temperature, + ) meas_grp = nxsr.get_class(entry_grp, class_type="NXdata")[0] timestamps = [] for ts_name in ("timestamps", "time-stamps"): if ts_name in meas_grp: timestamps = meas_grp[ts_name][()] break - return NexusJuice(filename, h5path, npt, unit, idx, Isum, q, I, sigma, poni, mask, energy, polarization, method, sample, timestamps) - "filename h5path npt unit idx Isum q I sigma poni mask energy polarization method sample timestamps" + if "diode" in meas_grp: + diode = meas_grp["diode"][()] + else: + diode = [] + + return NexusJuice( + filename, + h5path, + npt, + unit, + idx, + Isum, + q, + I, + sigma, + poni, + mask, + energy, + polarization, + method, + sample, + timestamps, + diode, + ) + "filename h5path npt unit idx Isum q I sigma poni mask energy polarization method sample timestamps diode" + + def build_plot(self): + """Create a chromatogram in the gallery""" + gallery = os.path.join( + os.path.dirname(os.path.abspath(self.output_file)), "gallery" + ) + filename = os.path.join(gallery, "chromatogram.png") + if not os.path.isdir(gallery): + try: + os.makedirs(gallery) + except Exception as err: + self.log_warning( + f"Unable to create directory {gallery}; {err.__class__.__name__}: {err}" + ) + return + sample = self.to_pyarch.get("sample_name", "sample") + chromatogram = self.to_pyarch.get("sum_I") + + if chromatogram is not None: + fractions = self.to_pyarch.get("merge_frames") + if fractions is not None: + fractions.sort() + hplc_plot( + chromatogram, + timestamps=self.to_pyarch.get("time"), + fractions=fractions, + title=f"Chromatograms of {sample}", + filename=filename, + img_format="png", + uv_data=self.uv_data, + ) + self.output["chromatogram_file"] = filename def send_to_ispyb(self): """Data sent to ISPyB are: - * hdf5File - * jsonFile built from HDF5 - * hplcPlot various plots generated + * hdf5File + * jsonFile built from HDF5 + * hplcPlot various plots generated """ if self.ispyb and self.ispyb.url and parse_url(self.ispyb.url).host: ispyb = IspybConnector(*self.ispyb) @@ -948,16 +1327,25 @@ def send_to_icat(self): to_icat["experiment_type"] = "hplc" to_icat["sample"] = self.juices[0].sample if "volume" in to_icat: - to_icat.pop("volume") + to_icat.pop("volume") metadata = {"scanType": "hplc"} - gallery=self.ispyb.gallery or os.path.join(os.path.dirname(os.path.abspath(self.output_file)), "gallery") - self.save_csv(os.path.join(gallery, "chromatogram.csv"), to_icat.get("sum_I"), to_icat.get("Rg")) - return send_icat(sample=self.juices[0].sample, - raw=os.path.dirname(os.path.abspath(self.input_files[0])), - path=os.path.dirname(os.path.abspath(self.output_file)), - data=to_icat, - gallery=gallery, - metadata=metadata) + gallery = self.ispyb.gallery or os.path.join( + os.path.dirname(os.path.abspath(self.output_file)), "gallery" + ) + self.save_csv( + os.path.join(gallery, "chromatogram.csv"), + to_icat.get("sum_I"), + to_icat.get("Rg"), + ) + return send_icat( + sample=self.juices[0].sample, + raw=os.path.dirname(os.path.abspath(self.input_files[0])), + path=os.path.dirname(os.path.abspath(self.output_file)), + data=to_icat, + dataset="HPLC", + gallery=gallery, + metadata=metadata, + ) def save_csv(self, filename, sum_I, Rg): dirname = os.path.dirname(filename) @@ -965,12 +1353,9 @@ def save_csv(self, filename, sum_I, Rg): os.makedirs(dirname, exist_ok=True) lines = ["id,ΣI,Rg"] idx = 0 - for I,rg in zip(sum_I, Rg): + for I, rg in zip(sum_I, Rg): # noqa lines.append(f"{idx},{I},{rg}") - idx+=1 + idx += 1 lines.append("") with open(filename, "w") as csv: csv.write(os.linesep.join(lines)) - - - diff --git a/plugins/bm29/icat.py b/plugins/bm29/icat.py index e6eac7a..9b007d6 100644 --- a/plugins/bm29/icat.py +++ b/plugins/bm29/icat.py @@ -4,16 +4,16 @@ """Data Analysis plugin for BM29: BioSaxs Everything to send data to iCat, the data catalogue - + """ __authors__ = ["Jérôme Kieffer"] __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "21/02/2025" +__date__ = "08/04/2026" __status__ = "development" -version = "0.3.0" +__version__ = "0.3.0" import os @@ -26,6 +26,8 @@ logger.error("iCat connection will no work") IcatClient = None +version = __version__ + def _ensure_gallery(gallery): if gallery: gallery = os.path.abspath(gallery) @@ -42,11 +44,11 @@ def _ensure_gallery(gallery): def send_icat(proposal=None, beamline=None, sample=None, dataset=None, path=None, raw=None, data=None, gallery=None, metadata=None): """Send some data to icat, the data-catalogue - + :param proposal: mx1324 :param beamline: name of the beamline :param sample: sample name as registered in icat - :param dataset: name given by BLISS + :param dataset: name of the dataset: integration, subtraction, HPLC, ... :param path: directory name where processed data are staying :param raw: list of directory name of the raw data (not the processed ones) :param data: dict with all data sent to iCat @@ -56,42 +58,48 @@ def send_icat(proposal=None, beamline=None, sample=None, dataset=None, path=None """ gallery = _ensure_gallery(gallery) tmp = gallery.strip("/").split("/") - idx_process = [i for i,j in enumerate(tmp) if j.lower().startswith("process")][-1] - if tmp[idx_process] == "processed": - assert idx_process>=6 - if proposal is None: - proposal = tmp[idx_process-6] - if beamline is None: - beamline = tmp[idx_process-5] - if sample is None: - sample = tmp[idx_process-2] - if dataset is None: - dataset = tmp[idx_process+1] - if path is None: - path = os.path.dirname(gallery) - if raw is None: - raw = os.path.abspath(gallery[:gallery.lower().index("process")]) - elif tmp[idx_process] == "PROCESSED_DATA": - if proposal is None: - proposal = tmp[idx_process-3] - if beamline is None: - beamline = tmp[idx_process-2] - if sample is None: - sample = tmp[idx_process+1] - if dataset is None: - dataset = tmp[idx_process+2] - if path is None: - path = os.path.dirname(gallery) - if raw is None: - raw = os.path.dirname(os.path.dirname(os.path.abspath(gallery.replace("PROCESSED_DATA", "RAW_DATA")))) + idx_process = [i for i,j in enumerate(tmp) if j.lower().startswith("process")] + if idx_process: + idx_process=idx_process[-1] + if tmp[idx_process] == "processed": + assert idx_process>=6 + if proposal is None: + proposal = tmp[idx_process-6] + if beamline is None: + beamline = tmp[idx_process-5] + if sample is None: + sample = tmp[idx_process-2] + if dataset is None: + dataset = tmp[idx_process+1] + if path is None: + path = os.path.dirname(gallery) + if raw is None: + raw = os.path.abspath(gallery[:gallery.lower().index("process")]) + elif tmp[idx_process] == "PROCESSED_DATA": + if proposal is None: + proposal = tmp[idx_process-3] + if beamline is None: + beamline = tmp[idx_process-2] + if sample is None: + sample = tmp[idx_process+1] + if dataset is None: + dataset = tmp[idx_process+2] + if path is None: + path = os.path.dirname(gallery) + if raw is None: + raw = os.path.dirname(os.path.dirname(os.path.abspath(gallery.replace("PROCESSED_DATA", "RAW_DATA")))) + else: + logger.error("Unrecognized path layout") + return else: - logger.error("Unrecognized path layout") - + logger.error("No gallery provided") + return + if metadata is None: metadata = {} metadata["definition"] = "SAXS", # metadata["Sample_name"] = sample - + for k,v in data.items(): if isinstance(k, str) and k.startswith("SAXS_"): metadata[k] = v @@ -129,10 +137,10 @@ def send_icat(proposal=None, beamline=None, sample=None, dataset=None, path=None tomerge = data.get("merged") if tomerge: metadata["SAXS_frames_averaged"] = f"{tomerge[0]}-{tomerge[1]}" - + volume = data.get("volume") if volume: - metadata["SAXS_porod_volume"] = str(volume) + metadata["SAXS_porod_volume"] = str(volume) rti = data.get("rti") if rti: "Vc sigma_Vc Qr sigma_Qr mass sigma_mass" @@ -140,19 +148,18 @@ def send_icat(proposal=None, beamline=None, sample=None, dataset=None, path=None metadata["SAXS_vc_error"] = f"{rti.sigma_Vc:.2f}" metadata["SAXS_mass"] = f"{rti.mass:.2f}" metadata["SAXS_mass_error"] = f"{rti.sigma_mass:.2f}" - + if not isinstance(raw, list): raw = [raw] #Other metadata one may collect ... metadata["SAXS_experiment_type"]= data.get("experiment_type", "UNKNOWN") metadata["datasetName"] = dataset icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"]) - kwargs = {"beamline":beamline, - "proposal":proposal, - "dataset":dataset, - "path":path, - "metadata":metadata, + kwargs = {"beamline":beamline, + "proposal":proposal, + "dataset":dataset, + "path":path, + "metadata":metadata, "raw":raw} - #print(kwargs) icat_client.store_processed_data(**kwargs) return kwargs diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py index c447015..3f51726 100644 --- a/plugins/bm29/integrate.py +++ b/plugins/bm29/integrate.py @@ -11,9 +11,9 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "04/06/2025" +__date__ = "09/03/2026" __status__ = "development" -__version__ = "0.3.0" +__version__ = "0.3.1" import os import time @@ -21,7 +21,7 @@ import logging import copy import posixpath -from collections import namedtuple +from typing import NamedTuple from urllib3.util import parse_url from dahu.plugin import Plugin from dahu.factory import register @@ -30,18 +30,17 @@ import numpy import h5py import pyFAI -import pyFAI.azimuthalIntegrator import freesas import freesas.cormap - +from .nexus import Nexus, get_isotime from .common import Sample, Ispyb, get_equivalent_frames, cmp_int, cmp_float, get_integrator, KeyCache, \ - method, polarization_factor, Nexus, get_isotime, SAXS_STYLE, NORMAL_STYLE, \ + method, polarization_factor,SAXS_STYLE, NORMAL_STYLE, \ create_nexus_sample from .ispyb import IspybConnector, NumpyEncoder from .icat import send_icat from .memcached import to_memcached - +version = __version__ logger = logging.getLogger("bm29.integrate") try: import numexpr @@ -49,9 +48,23 @@ logger.error("Numexpr is not installed, falling back on numpy's implementations") numexpr = None -IntegrationResult = namedtuple("IntegrationResult", "radial intensity sigma") -CormapResult = namedtuple("CormapResult", "probability count tomerge") -AverageResult = namedtuple("AverageResult", "average deviation normalization") + +class IntegrationResult(NamedTuple): + radial:numpy.ndarray + intensity:numpy.ndarray + sigma:numpy.ndarray + + +class CormapResult(NamedTuple): + probability:float + count:int + tomerge:int + + +class AverageResult(NamedTuple): + average:numpy.ndarray + deviation:numpy.ndarray + normalization:numpy.ndarray @register @@ -189,6 +202,7 @@ def setup(self, kwargs=None): self.monitor_values = numpy.array(self.input.get("monitor_values", 1), dtype=numpy.float64) if self.input.get("average_out_monitor_values"): self.monitor_values = numpy.zeros_like(self.monitor_values) + self.monitor_values.mean() + self.log_warning("Averaging-out the monitor values !") self.normalization_factor = float(self.input.get("normalization_factor", 1)) self.scale_factor = float(self.input.get("exposure_time", 1)) / self.normalization_factor @@ -197,6 +211,7 @@ def teardown(self): logger.debug("IntegrateMultiframe.teardown") # export the output file location self.output["output_file"] = self.output_file + self.output["monitor_noise_%"] = 100 * self.monitor_values.std() / self.monitor_values.mean() if self.nxs is not None: self.nxs.close() if self.ai is not None: @@ -249,7 +264,7 @@ def wait_file(self, filename, timeout=None): """ timeout = self.timeout if timeout is None else timeout end_time = time.perf_counter() + timeout - dirname = os.path.dirname(filename) + dirname = os.path.dirname(filename) or "." while not os.path.isdir(dirname): if time.perf_counter() > end_time: self.log_error(f"Filename {filename} did not appear in {timeout} seconds") @@ -273,8 +288,10 @@ def wait_file(self, filename, timeout=None): def create_nexus(self): "create the nexus result file with basic structure" - if not os.path.isdir(os.path.dirname(self.output_file)): - os.makedirs(os.path.dirname(self.output_file)) + dirname = os.path.dirname(self.output_file) + if dirname: + if not os.path.isdir(dirname): + os.makedirs(dirname) creation_time = os.stat(self.input_file).st_ctime nxs = self.nxs = Nexus(self.output_file, mode="w", creator="dahu") @@ -395,15 +412,15 @@ def create_nexus(self): pol_ds = cfg_grp.create_dataset("polarization_factor", data=polarization_factor) pol_ds.attrs["comment"] = "Between -1 and +1, 0 for circular" cfg_grp.create_dataset("integration_method", data=json.dumps(method.method._asdict())) - integration_data = nxs.new_class(integration_grp, "results", "NXdata") + integration_data = nxs.new_class(integration_grp, "result", "NXdata") integration_grp.attrs["title"] = str(self.sample) # Stage 1 processing: Integration frame per frame - integrate1_results = self.process1_integration(self.input_frames) + integrate1_result = self.process1_integration(self.input_frames) radial_unit, unit_name = str(self.unit).split("_", 1) - q = numpy.ascontiguousarray(integrate1_results.radial, numpy.float32) - I = numpy.ascontiguousarray(integrate1_results.intensity, dtype=numpy.float32) - sigma = numpy.ascontiguousarray(integrate1_results.sigma, dtype=numpy.float32) + q = numpy.ascontiguousarray(integrate1_result.radial, numpy.float32) + I = numpy.ascontiguousarray(integrate1_result.intensity, dtype=numpy.float32) #noqa + sigma = numpy.ascontiguousarray(integrate1_result.sigma, dtype=numpy.float32) self.to_memcached[radial_unit] = q self.to_memcached["I"] = I @@ -428,7 +445,7 @@ def create_nexus(self): hplc_data = nxs.new_class(integration_grp, "hplc", "NXdata") hplc_data.attrs["title"] = "Chromatogram" - sum_ds = hplc_data.create_dataset("sum", data=numpy.ascontiguousarray(integrate1_results.intensity.sum(axis=-1), dtype=numpy.float32)) + sum_ds = hplc_data.create_dataset("sum", data=numpy.ascontiguousarray(integrate1_result.intensity.sum(axis=-1), dtype=numpy.float32)) sum_ds.attrs["interpretation"] = "spectrum" sum_ds.attrs["long_name"] = "Summed Intensity" hplc_data["frame_ids"] = frame_ds @@ -449,7 +466,7 @@ def create_nexus(self): cormap_grp["program"] = "freesas.cormap" cormap_grp["version"] = freesas.version cormap_grp["date"] = get_isotime() - cormap_data = nxs.new_class(cormap_grp, "results", "NXdata") + cormap_data = nxs.new_class(cormap_grp, "result", "NXdata") cormap_data.attrs["SILX_style"] = NORMAL_STYLE cfg_grp = nxs.new_class(cormap_grp, "configuration", "NXcollection") @@ -459,33 +476,33 @@ def create_nexus(self): cfg_grp["fidelity_rel"] = fidelity_rel # Stage 2 processing - cormap_results = self.process2_cormap(integrate1_results.intensity, fidelity_abs, fidelity_rel) + cormap_result = self.process2_cormap(integrate1_result.intensity, fidelity_abs, fidelity_rel) cormap_data.attrs["signal"] = "probability" - cormap_ds = cormap_data.create_dataset("probability", data=cormap_results.probability) + cormap_ds = cormap_data.create_dataset("probability", data=cormap_result.probability) cormap_ds.attrs["interpretation"] = "image" cormap_ds.attrs["long_name"] = "Probability to be the same" - count_ds = cormap_data.create_dataset("count", data=cormap_results.count) + count_ds = cormap_data.create_dataset("count", data=cormap_result.count) count_ds.attrs["interpretation"] = "image" count_ds.attrs["long_name"] = "Longest sequence where curves do not cross each other" - to_merge_ds = cormap_data.create_dataset("to_merge", data=numpy.arange(*cormap_results.tomerge, dtype=numpy.uint16)) + to_merge_ds = cormap_data.create_dataset("to_merge", data=numpy.arange(*cormap_result.tomerge, dtype=numpy.uint16)) to_merge_ds.attrs["long_name"] = "Index of equivalent frames" cormap_grp.attrs["default"] = posixpath.relpath(cormap_data.name, cormap_grp.name) if self.ispyb.url: - self.to_pyarch["merged"] = cormap_results.tomerge + self.to_pyarch["merged"] = cormap_result.tomerge # Process 3: time average and standard deviation average_grp = nxs.new_class(entry_grp, "3_time_average", "NXprocess") average_grp["sequence_index"] = 3 average_grp["program"] = fully_qualified_name(self.__class__) average_grp["version"] = __version__ - average_data = nxs.new_class(average_grp, "results", "NXdata") + average_data = nxs.new_class(average_grp, "result", "NXdata") average_data.attrs["SILX_style"] = SAXS_STYLE average_data.attrs["signal"] = "intensity_normed" # Stage 3 processing - res3 = self.process3_average(cormap_results.tomerge) + res3 = self.process3_average(cormap_result.tomerge) Iavg = numpy.ascontiguousarray(res3.average, dtype=numpy.float32) sigma_avg = numpy.ascontiguousarray(res3.deviation, dtype=numpy.float32) @@ -514,7 +531,7 @@ def create_nexus(self): ai2_grp["program"] = "pyFAI" ai2_grp["version"] = pyFAI.version ai2_grp["date"] = get_isotime() - ai2_data = nxs.new_class(ai2_grp, "results", "NXdata") + ai2_data = nxs.new_class(ai2_grp, "result", "NXdata") ai2_data.attrs["signal"] = "I" ai2_data.attrs["axes"] = radial_unit ai2_data.attrs["SILX_style"] = SAXS_STYLE @@ -656,6 +673,7 @@ def send_to_icat(self): raw=os.path.dirname(os.path.dirname(os.path.abspath(self.input_file))), path=os.path.dirname(os.path.abspath(self.output_file)), data=to_icat, + dataset = "integrate", gallery=self.ispyb.gallery or os.path.join(os.path.dirname(os.path.abspath(self.output_file)), "gallery"), metadata=metadata) diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py index d27b7ae..9684938 100644 --- a/plugins/bm29/ispyb.py +++ b/plugins/bm29/ispyb.py @@ -4,24 +4,28 @@ """Data Analysis plugin for BM29: BioSaxs Everything to send data to Ispyb - + """ __authors__ = ["Jérôme Kieffer"] __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "24/02/2025" +__date__ = "09/03/2026" __status__ = "development" -version = "0.2.3" +__version__ = "0.2.3" + import logging -logger = logging.getLogger("bm29.ispyb") import os import shutil import json import tempfile import numpy +from freesas.plot import kratky_plot, guinier_plot, scatter_plot, density_plot +import matplotlib.pyplot + +logger = logging.getLogger("bm29.ispyb") try: from suds.client import Client from suds.transport.https import HttpAuthenticated @@ -34,10 +38,8 @@ print("iCat connection will no work") IcatClient = None -import matplotlib.pyplot + matplotlib.use("Agg") -from freesas.containers import RG_RESULT, RT_RESULT, StatsResult -from freesas.plot import kratky_plot, guinier_plot, scatter_plot, density_plot, hplc_plot class NumpyEncoder(json.JSONEncoder): @@ -91,7 +93,7 @@ def __init__(self, url, login=None, passwd=None, gallery=None, pyarch=None, self.experiment_id = experiment_id self.run_number = run_number - + def __repr__(self): return f"Ispyb connector to {self._url}" @@ -99,7 +101,7 @@ def __repr__(self): def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, path=None, raw=None, data=None): """ DEPRECATED CODE ! - + :param proposal: mx1324 :param beamline: name of the beamline :param sample: sample name as registered in icat @@ -123,9 +125,9 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat dataset = tmp[idx_process+1] if path is None: path = os.path.dirname(self.gallery) - if raw is None: + if raw is None: raw = os.path.abspath(self.gallery[:self.gallery.lower().index("process")]) - elif tmp[idx_process] == "PROCESSED_DATA": + elif tmp[idx_process] == "PROCESSED_DATA": if proposal is None: proposal = tmp[idx_process-3] if beamline is None: @@ -136,11 +138,11 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat dataset = tmp[idx_process+2] if path is None: path = os.path.dirname(self.gallery) - if raw is None: + if raw is None: raw = os.path.dirname(os.path.dirname(os.path.abspath(self.gallery.replace("PROCESSED_DATA", "RAW_DATA")))) else: logger.error("Unrecognized path layout") - + metadata = {"definition": "SAXS", "Sample_name": sample} for k,v in data.items(): @@ -171,19 +173,19 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat tomerge = data.get("merged") if tomerge: metadata["SAXS_frames_averaged"] = f"{tomerge[0]}-{tomerge[1]}" - + volume = data.get("volume") if volume: - metadata["SAXS_porod_volume"] = str(volume) + metadata["SAXS_porod_volume"] = str(volume) #Other metadata one may collect ... metadata["SAXS_experiment_type"]= data.get("experiment_type", "UNKNOWN") metadata["datasetName"] = dataset icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"]) - kwargs = {"beamline":beamline, - "proposal":proposal, - "dataset":dataset, - "path":path, - "metadata":metadata, + kwargs = {"beamline":beamline, + "proposal":proposal, + "dataset":dataset, + "path":path, + "metadata":metadata, "raw":[raw]} icat_client.store_processed_data(**kwargs) return kwargs @@ -228,7 +230,17 @@ def _mk_filename(self, index, path, basename="frame", ext=".dat"): os.makedirs(dest) except Exception as err: logger.error("Unable to create directory %s: %s: %s", dest, type(err), err) - os.stat(dest) #this is to enforce the mounting of the directory + else: + # Once the directory is pretendily created, write something in is an delete it. + delete_me = os.path.join(dest, "delete.me") + res = os.system(f"touch {delete_me}") + if res: + logger.error(f"`touch {delete_me}` return error {res}, directory creation did probably not work as expected !") + try: + os.remove(delete_me) + except FileNotFoundError: + logger.error(f"`rm {delete_me}` raised FileNotFoundError, directory creation did probably not work as expected !") + os.stat(dest) # this is to enforce the mounting of the directory if isinstance(index, int): filename = os.path.join(dest, "%s_%04d%s" % (basename, index, ext)) else: @@ -371,19 +383,6 @@ def send_hplc(self, data): :param data: a dict with all information to be saved in Ispyb """ sample = data.get("sample_name", "sample") - #gallery - gallery = os.path.join(self.gallery, 'chromatogram.png') - chromatogram = data.get("sum_I") - - if chromatogram is not None: - fractions = data.get("merge_frames") - if fractions is not None: - fractions.sort() - hplc_plot(chromatogram, fractions, - title=f"Chromatogram of {sample}", - filename=gallery, - img_format="png", ) - hdf5_file = data.get("hdf5_filename") filename = self._mk_filename("hplc", ".", sample, ext=".h5") filename = os.path.abspath(filename) diff --git a/plugins/bm29/memcached.py b/plugins/bm29/memcached.py index 4bd8432..a7a95dd 100644 --- a/plugins/bm29/memcached.py +++ b/plugins/bm29/memcached.py @@ -10,7 +10,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "21/02/2025" +__date__ = "22/04/2025" __status__ = "development" __version__ = "0.3.0" @@ -29,5 +29,7 @@ def to_memcached(dico): mc = memcache.Client([(SERVER, 11211)]) rc["server"] = socket.getfqdn()+":11211" for k, v in dico.items(): + if len(k)>250: + k = k[-250:] rc[k] = mc.set(k, v) return rc diff --git a/plugins/bm29/mesh.py b/plugins/bm29/mesh.py new file mode 100644 index 0000000..697c06a --- /dev/null +++ b/plugins/bm29/mesh.py @@ -0,0 +1,572 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""Data Analysis plugin for BM29: BioSaxs + +* Mesh mode: Rebuild the complete map and performs basic analysis on it. +""" + +__authors__ = ["Jérôme Kieffer"] +__contact__ = "Jerome.Kieffer@ESRF.eu" +__license__ = "MIT" +__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" +__date__ = "09/03/2026" +__status__ = "development" +__version__ = "0.1.0" + +import os +import posixpath +import json +import glob +from dataclasses import dataclass, asdict +from typing import NamedTuple +import numpy +from dahu.plugin import Plugin +import h5py +import matplotlib +from matplotlib.pyplot import subplots +import pyFAI +from pyFAI.method_registry import IntegrationMethod +from pyFAI.io.ponifile import PoniFile +from pyFAI.io.diffmap_config import DiffmapConfig, WorkerConfig, MotorRange, ListDataSet, DataSet +from .common import Sample, Ispyb, SAXS_STYLE, create_nexus_sample +from .nexus import Nexus, get_isotime + + +matplotlib.use("Agg") + + +class NexusJuice(NamedTuple): + filename: str + h5path: str + npt: int + unit: str + idx: int + Isum: numpy.ndarray + q: numpy.ndarray + I: numpy.ndarray #noqa + sigma: numpy.ndarray + poni: str + mask: numpy.ndarray + energy: float + polarization: float + method: tuple + sample: str + timestamps:numpy.ndarray + + +class Position(NamedTuple): + index: int + slow: int + fast:int + + +@dataclass(slots=True) +class Scan: + """Class describing 2D mesh-scan""" + fast_motor_name: str = "fast" + fast_motor_start: float = 0.0 + fast_motor_stop: float = 1.0 + fast_motor_step: int = 0 # bliss-like steps, actually step+1 points + slow_motor_name: str = "slow" + slow_motor_start: float = 0.0 + slow_motor_stop: float = 1.0 + slow_motor_step: int = 0 # bliss-like steps, actually step+1 points + backnforth: bool = False + + def __repr__(self): + return json.dumps(self.as_dict(), indent=4) + + def as_dict(self): + """Like asdict, without extra features: + + :return: dict which can be JSON-serialized + """ + dico = {} + for key, value in asdict(self).items(): + dico[key] = value + return dico + + @classmethod + def from_dict(cls, dico): + """Alternative constructor, + :param dico: dict with the config + :return: instance of the dataclass + """ + to_init = dico + self = cls(**to_init) + return self + + def save(self, filename): + """Dump the content of the dataclass as JSON file""" + with open(filename, "w") as w: + w.write(json.dumps(self.as_dict(), indent=2)) + + def get_pos(self, idx=None): + """ + Calculate the position in the mesh scan fram according to its index + + :param idx: index of current frame + :return: namedtuple Position=(frame_index, slow_motor_position, fast_motor_position) + if valid else None + """ + width = self.fast_motor_step + 1 + line = idx // width + if self.backnforth and line % 2 == 1: + row = self.fast_motor_step - (idx % width) + else: + row = idx % width + if line<=self.slow_motor_step and row<=self.fast_motor_step: + return Position(idx, line, row) + else: + return None + + @property + def shape(self): + return (self.slow_motor_step + 1, self.fast_motor_step + 1) + + @classmethod + def parse(cls, text): + """Alternative constructor, + :param text: string containing the bliss command (starting with `amesh`) + :return: instance of the dataclass + """ + res = None + if text.startswith("amesh"): + words = text.split() + if len(words) >= 9: + res = cls(words[1], float(words[2]),float(words[3]), int(words[4]), + words[5], float(words[6]),float(words[7]), int(words[8]), + True) + return res + +def input_from_master(master_file): + """Convert a bliss masterfile containing one or multiple NXentry + with a 2D scan into a set of plugins to be launched + + :param master_file: path to a bliss masterfile + :return: list of job-input-dicts. + """ + result = [] + + with Nexus(master_file, mode="r", pure=True) as master: + for entry in master.get_entries(): + job = {"plugin_name":"bm29.mesh"} + # name = posixpath.split(entry.name)[-1] + filename = os.path.abspath(entry.file.filename) + dirtree = filename.split(os.sep)[:-1] + raw_idx = dirtree.index("RAW_DATA") + dirtree[raw_idx] = "PROCESSED_DATA" + job["output_file"] = os.sep.join(dirtree + ["mesh", "mesh.h5"]) + wildcard = os.sep.join(dirtree + ["integrate", "*.h5"]) + input_files = glob.glob(wildcard) + input_files.sort() + job["integrated_files"] = input_files + + title = entry.get("title", "") + if isinstance(title, h5py.Dataset): + title = title[()] + if isinstance(title, bytes): + title = title.decode() + if title: + scan = Scan.parse(title) + if scan is None: + continue + else: + job["scan"] = scan.as_dict() + result.append(job) + return result + + +def mesh_plot(mesh, + title="mesh-scan", + x_label="fast motor", + y_label="slow motor", + x_range=None, + y_range=None, + filename=None, + img_format="png", + ax=None, + labelsize=None, + fontsize=None,): + """ + Generate an image of the mesh-scan + + :param mesh: sum of integrated data + :param filename: name of the file where the cuve should be saved + :param img_format: image image format + :param ax: subplotib where to plot in + :return: the matplotlib figure + """ + if ax: + fig = ax.figure + else: + fig, ax = subplots(figsize=(12, 10)) + ax.imshow(mesh, cmap="binary") + ax.set_xlabel(x_label, fontsize=fontsize) + ax.set_ylabel(y_label, fontsize=fontsize) + ax.set_title(title) + if x_range is not None: + ax.xaxis.set_major_formatter(lambda x, pos: f"{numpy.interp(x, numpy.arange(x_range.size), x_range):.2f}") + if y_range is not None: + ax.yaxis.set_major_formatter(lambda x, pos: f"{numpy.interp(x, numpy.arange(y_range.size), y_range):.2f}") + + ax.tick_params(axis="x", labelsize=labelsize) + ax.tick_params(axis="y", labelsize=labelsize) + + if filename: + if img_format: + fig.savefig(filename, format=img_format) + else: + fig.savefig(filename) + return fig + + + +class Mesh(Plugin): + """ Rebuild the complete map and perform basic analysis on it. + + Typical JSON file: + { + "integrated_files": ["img_001.h5", "img_002.h5"], + "output_file": "mesh.h5" + "ispyb": { + "url": "http://ispyb.esrf.fr:1234", + "pyarch": "/data/pyarch/mx1234/sample", + "measurement_id": -1, + "collection_id": -1 + }, + "scan": { + "fast_motor_name": "chipz", + "fast_motor_start": -2.2, + "fast_motor_stop": -3.2, + "fast_motor_step": 3, + "slow_motor_name": "chipy", + "slow_motor_start": -5.2, + "slow_motor_stop": -12.2, + "slow_motor_step": 7, + "backnforth": False + } + "transpose": False, + "wait_for": [jobid_img001, jobid_img002], + "plugin_name": "bm29.mesh" + } + """ + + def __init__(self): + Plugin.__init__(self) + self.input_files = [] + self.nxs = None + self.output_file = None + self.scan = Scan() + self.transpose = None + self.juices = [] + self.to_pyarch = {} + self.ispyb = None + self._pid = 0 + + def sequence_index(self): + value = self._pid + self._pid += 1 + return value + + def setup(self): + Plugin.setup(self) + + for job_id in self.input.get("wait_for", []): + self.wait_for(job_id) + + self.input_files = [os.path.abspath(i) for i in self.input.get("integrated_files", "")] + self.output_file = self.input.get("output_file") + if not self.output_file: + dirname, basename = os.path.split(os.path.commonprefix(self.input_files) + "_mesh.h5") + dirname = os.path.dirname(dirname) + dirname = os.path.join(dirname, "mesh") + self.output_file = os.path.join(dirname, basename) + if not os.path.isdir(dirname): + try: + os.makedirs(dirname) + except Exception as err: + self.log_warning(f"Unable to create dir {dirname}. {type(err)}: {err}") + + self.log_warning(f"No output file provided, using {self.output_file}") + + #Manage gallery here + dirname = os.path.dirname(self.output_file) + gallery = os.path.join(dirname, "gallery") + if not os.path.isdir(gallery): + try: + os.makedirs(gallery) + except Exception as err: + self.log_warning(f"Unable to create dir {gallery}. {type(err)}: {err}") + ispydict = self.input.get("ispyb", {}) + ispydict["gallery"] = gallery + self.ispyb = Ispyb._fromdict(ispydict) + self.scan = Scan.from_dict(self.input.get("scan", {})) + + def process(self): + self.create_nexus() + self.to_pyarch["hdf5_filename"] = self.output_file + # self.to_pyarch["chunk_size"] = self.juices[0].Isum.size + self.to_pyarch["id"] = os.path.commonprefix(self.input_files) + self.to_pyarch["sample_name"] = self.juices[0].sample.name + if not self.input.get("no_ispyb"): + self.send_to_ispyb() + # self.output["icat"] = + self.send_to_icat() + + def teardown(self): + Plugin.teardown(self) + # logger.debug("HPLC.teardown") + # export the output file location + self.output["output_file"] = self.output_file + if self.nxs is not None: + self.nxs.close() + self.to_pyarch = None + self.ispyb = None + + def create_nexus(self): + nxs = Nexus(self.output_file, mode="w") + entry_grp = nxs.new_entry("entry", self.input.get("plugin_name", "dahu"), + title='BioSaxs Mesh experiment', + force_time=get_isotime()) + entry_grp["version"] = __version__ + nxs.h5.attrs["default"] = entry_grp.name.strip("/") + + # Configuration + cfg_grp = nxs.new_class(entry_grp, "configuration", "NXnote") + cfg_grp.create_dataset("data", data=json.dumps(self.input, indent=2, separators=(",\r\n", ":\t"))) + cfg_grp.create_dataset("format", data="text/json") + + # Process 0: Measurement group + input_grp = nxs.new_class(entry_grp, "0_measurement", "NXcollection") + input_grp["sequence_index"] = self.sequence_index() + + for idx, filename in enumerate(self.input_files): + juice = self.read_nexus(filename) + if juice is not None: + rel_path = os.path.relpath(os.path.abspath(filename), os.path.dirname(os.path.abspath(self.output_file))) + input_grp["LImA_%04i" % idx] = h5py.ExternalLink(rel_path, juice.h5path) + self.juices.append(juice) + + assert self.juices + q = self.juices[0].q + unit = self.juices[0].unit + radial_unit, unit_name = str(unit).split("_", 1) + + # Sample: outsourced ! + create_nexus_sample(nxs, entry_grp, self.juices[0].sample) + + # Process 1: Mesh + mesh_grp = nxs.new_class(entry_grp, "1_mesh", "NXprocess") + mesh_grp["sequence_index"] = self.sequence_index() + mesh_src = mesh_grp.create_group("sources") + input_dataset = ListDataSet() + if self.juices: + for idx, juice in enumerate(self.juices): + with h5py.File(juice.filename) as h: + grp = h[juice.h5path] + meas = grp["0_measurement"] + ds = meas["images"] + src = os.path.abspath(ds.file.filename) + name = ds.name + nframes, *img_shape = ds.shape + rel_path = os.path.relpath(src, os.path.dirname(os.path.abspath(self.output_file))) + mesh_src[f"images_{idx:04d}"] = h5py.ExternalLink(rel_path, name) + input_dataset.append(DataSet(src, name, nframes, img_shape)) + poni = PoniFile(juice.poni) + # mask = juice.mask + polarization = juice.polarization + method = juice.method + else: + poni = polarization = method = None + + nbin = q.size + # Creates a configuration NXnote in the NXProcess like diffmap would do""" + diffmap_grp = nxs.new_class(mesh_grp, "configuration", "NXnote") + diffmap_grp["type"] = "text/json" + worker = WorkerConfig(poni=poni, + nbpt_rad=nbin, + nbpt_azim=1) + worker.unit = unit + worker.method = (method.split, method.algorithm, method.implementation) + worker.opencl_device = method.target + worker.polarization_factor = polarization + + diffmap = DiffmapConfig(experiment_title="bm29.mesh", + slow_motor=MotorRange(start=self.scan.slow_motor_start, + stop=self.scan.slow_motor_stop, + points=self.scan.slow_motor_step+1, + name=self.scan.slow_motor_name), + fast_motor=MotorRange(start=self.scan.fast_motor_start, + stop=self.scan.fast_motor_stop, + points=self.scan.fast_motor_step+1, + name=self.scan.fast_motor_name), + offset=0, + zigzag_scan=self.scan.backnforth, + ai=worker, + input_data=input_dataset, + output_file=self.output_file) + # print(diffmap.as_dict()) + diffmap_grp.create_dataset("data", + data=json.dumps(diffmap.as_dict(), + indent=2, + separators=(",\r\n", ":\t"))) + + shape = self.scan.shape + (nbin,) + I_ary = numpy.zeros(shape, dtype=numpy.float32) + slow = numpy.linspace(self.scan.slow_motor_start, + self.scan.slow_motor_stop, + self.scan.slow_motor_step+1, + endpoint=True).astype("float32") + fast = numpy.linspace(self.scan.fast_motor_start, + self.scan.fast_motor_stop, + self.scan.fast_motor_step+1, + endpoint=True).astype("float32") + + sigma = numpy.zeros(shape, dtype=numpy.float32) + Isum = numpy.zeros(self.scan.shape, dtype=numpy.float32) + indices = numpy.zeros(self.scan.shape, dtype=numpy.uint32) + + timestamps = [] + + for juice in self.juices: + timestamps.append(juice.timestamps) + # print(juice.idx) + for j, i in enumerate(juice.idx): + p = self.scan.get_pos(i) + if p is None: + continue + # print(p) + indices[p.slow, p.fast] = p.index + I_ary[p.slow, p.fast] = juice.I[j] + Isum[p.slow, p.fast] = juice.Isum[j] + sigma[p.slow, p.fast] = juice.sigma[j] + + timestamps = self.to_pyarch["time"] = numpy.concatenate(timestamps) + + mesh_data = nxs.new_class(mesh_grp, "mesh", "NXdata") + mesh_data.attrs["title"] = "Mesh scan" + sum_ds = mesh_data.create_dataset("sum", data=Isum, dtype=numpy.float32) + sum_ds.attrs["interpretation"] = "image" + sum_ds.attrs["long_name"] = "Summed Intensity" + frame_ds = mesh_data.create_dataset("frame_ids", data=indices, dtype=numpy.uint32) + frame_ds.attrs["interpretation"] = "image" + frame_ds.attrs["long_name"] = "frame index" + slow_motor_ds = mesh_data.create_dataset("slow_motor", data =slow) + slow_motor_ds.attrs["interpretation"] = "spectrum" + slow_motor_ds.attrs["long_name"] = self.scan.slow_motor_name + fast_motor_ds = mesh_data.create_dataset("fast_motor", data =fast) + fast_motor_ds.attrs["interpretation"] = "spectrum" + fast_motor_ds.attrs["long_name"] = self.scan.fast_motor_name + mesh_data.attrs["signal"] = "sum" + mesh_data.attrs["axes"] = ["slow_motor", "fast_motor"] + + mesh_grp.attrs["default"] = posixpath.relpath(mesh_data.name, mesh_grp.name) + entry_grp.attrs["default"] = posixpath.relpath(mesh_data.name, entry_grp.name) + time_ds = mesh_data.create_dataset("timestamps", data=timestamps, dtype=numpy.uint32) + time_ds.attrs["interpretation"] = "spectrum" + time_ds.attrs["long_name"] = "Time stamps (s)" + + integration_data = nxs.new_class(mesh_grp, "result", "NXdata") + mesh_grp.attrs["title"] = str(self.juices[0].sample) + integration_data["map_ptr"] = frame_ds + int_ds = integration_data.create_dataset("I", data=numpy.ascontiguousarray(I_ary, dtype=numpy.float32)) + std_ds = integration_data.create_dataset("errors", data=numpy.ascontiguousarray(sigma, dtype=numpy.float32)) + q_ds = integration_data.create_dataset("q", data=self.juices[0].q) + slow_motor_ds = integration_data.create_dataset("slow_motor", data =slow) + slow_motor_ds.attrs["interpretation"] = "spectrum" + slow_motor_ds.attrs["long_name"] = self.scan.slow_motor_name + fast_motor_ds = integration_data.create_dataset("fast_motor", data =fast) + fast_motor_ds.attrs["interpretation"] = "spectrum" + fast_motor_ds.attrs["long_name"] = self.scan.fast_motor_name + q_ds.attrs["interpretation"] = "spectrum" + q_ds.attrs["unit"] = unit_name + q_ds.attrs["long_name"] = "Scattering vector q (nm⁻¹)" + integration_data.attrs["signal"] = "I" + integration_data.attrs["axes"] = ["slow_motor", "fast_motor", "q"] + integration_data.attrs["SILX_style"] = SAXS_STYLE + + int_ds.attrs["interpretation"] = "spectrum" + int_ds.attrs["units"] = "arbitrary" + int_ds.attrs["long_name"] = "Intensity (absolute, normalized on water)" + # int_ds.attrs["uncertainties"] = "errors" This does not work + int_ds.attrs["scale"] = "log" + std_ds.attrs["interpretation"] = "spectrum" + + mesh_plot(Isum, + title="Mesh scan", + x_label=self.scan.fast_motor_name, + y_label=self.scan.slow_motor_name, + x_range=fast, + y_range=slow, + filename=os.path.join(self.ispyb.gallery, "mesh.png")) + # save_zip(os.path.splitext(self.output_file)[0]+".zip", + # self.juices[0], I, sigma) + + def read_nexus(self, filename): + "return some NexusJuice from a HDF5 file " + with Nexus(filename, "r") as nxsr: + entry_name = nxsr.h5.attrs["default"] + entry_grp = nxsr.h5[entry_name] + h5path = entry_grp.name + nxdata_grp = entry_grp[entry_grp.attrs["default"]] + # assert nxdata_grp.name.endswith("hplc") # we are reading HPLC data + signal = nxdata_grp.attrs["signal"] + axis = nxdata_grp.attrs["axes"] + Isum = nxdata_grp[signal][()] + idx = nxdata_grp[axis][()] + try: + integrated = nxdata_grp.parent["result"] + except KeyError: + integrated = nxdata_grp.parent["results"] + self.log_warning(f"Parsing old file {filename} !") + signal = integrated.attrs["signal"] + I_ary = integrated[signal][()] + axes = integrated.attrs["axes"][-1] + q = integrated[axes][()] + sigma = integrated["errors"][()] + + npt = len(q) + unit = pyFAI.units.to_unit(axes + "_" + integrated[axes].attrs["units"]) + integration_grp = nxdata_grp.parent + poni = str(integration_grp["configuration/file_name"][()]).strip() + if not os.path.exists(poni): + poni = integration_grp["configuration/data"][()] + if isinstance(poni, bytes): + poni = poni.decode() + poni = json.loads(poni) + polarization = integration_grp["configuration/polarization_factor"][()] + method = IntegrationMethod.select_method(**json.loads(integration_grp["configuration/integration_method"][()]))[0] + instrument_grp = nxsr.get_class(entry_grp, class_type="NXinstrument")[0] + detector_grp = nxsr.get_class(instrument_grp, class_type="NXdetector")[0] + mask = detector_grp["pixel_mask"].attrs["filename"] + mono_grp = nxsr.get_class(instrument_grp, class_type="NXmonochromator")[0] + energy = mono_grp["energy"][()] + + # Read the sample description: + sample_grp = nxsr.get_class(entry_grp, class_type="NXsample")[0] + sample_name = posixpath.split(sample_grp.name)[-1] + + buffer = sample_grp["buffer"][()] if "buffer" in sample_grp else "" + concentration = sample_grp["concentration"][()] if "concentration" in sample_grp else "" + description = sample_grp["description"][()] if "description" in sample_grp else "" + hplc = sample_grp["hplc"][()] if "hplc" in sample_grp else "" + temperature = sample_grp["temperature"][()] if "temperature" in sample_grp else "" + temperature_env = sample_grp["temperature_env"][()] if "temperature_env" in sample_grp else "" + sample = Sample(sample_name, description, buffer, concentration, hplc, temperature_env, temperature) + meas_grp = nxsr.get_class(entry_grp, class_type="NXdata")[0] + timestamps = [] + for ts_name in ("timestamps", "time-stamps"): + if ts_name in meas_grp: + timestamps = meas_grp[ts_name][()] + break + return NexusJuice(filename, h5path, npt, unit, idx, Isum, q, I_ary, sigma, poni, mask, energy, polarization, method, sample, timestamps) + "filename h5path npt unit idx Isum q I sigma poni mask energy polarization method sample timestamps" + + def send_to_ispyb(self): + self.log_warning("send_to_ispyb: unimplemented") + + def send_to_icat(self): + self.log_warning("send_to_icat: unimplemented") + diff --git a/plugins/bm29/meson.build b/plugins/bm29/meson.build index 3c08e41..8a7730b 100644 --- a/plugins/bm29/meson.build +++ b/plugins/bm29/meson.build @@ -7,7 +7,8 @@ py.install_sources( [ 'nexus.py', 'subtracte.py', 'memcached.py', - 'icat.py' + 'icat.py', + 'mesh.py' ], pure: false, # Will be installed next to binaries subdir: 'dahu/plugins/bm29' # Folder relative to site-packages to install to diff --git a/plugins/bm29/nexus.py b/plugins/bm29/nexus.py index 411f949..b19e677 100644 --- a/plugins/bm29/nexus.py +++ b/plugins/bm29/nexus.py @@ -70,6 +70,15 @@ def is_hdf5(filename): return sig == signature +def fully_qualified_name(o): + """Return the fully qualified name of the class""" + klass = o.__class__ + module = klass.__module__ + if module == 'builtins': + return klass.__qualname__ # avoid outputs like 'builtins.str' + return module + '.' + klass.__qualname__ + + class Nexus: """ Writer class to handle Nexus/HDF5 data @@ -88,7 +97,8 @@ class Nexus: def __init__(self, filename, mode=None, creator=None, timeout=None, - start_time=None): + start_time=None, + pure=False): """ Constructor @@ -97,6 +107,8 @@ def __init__(self, filename, mode=None, :param creator: set as attr of the NXroot :param timeout: retry for that amount of time (in seconds) :param start_time: set as attr of the NXroot + :param pure: use pure h5py mode. Unless, try to be clever when + accessing write-opened files (can breaks external links) """ self.filename = os.path.abspath(filename) self.mode = mode @@ -104,18 +116,18 @@ def __init__(self, filename, mode=None, logger.error("h5py module missing: NeXus not supported") raise RuntimeError("H5py module is missing") - pre_existing = os.path.exists(self.filename) or "w" in mode - if self.mode is None: - if pre_existing: - self.mode = "r" - else: - self.mode = "a" if timeout: end = time.perf_counter() + timeout while time.perf_counter() < end : + pre_existing = os.path.exists(self.filename) + if self.mode is None: + if pre_existing: + mode = "r" + else: + mode = "a" try: - if self.mode == "r": + if mode == "r" and not pure: self.file_handle = open(self.filename, mode="rb") self.h5 = h5py.File(self.file_handle, mode="r") else: @@ -125,24 +137,36 @@ def __init__(self, filename, mode=None, os.stat(os.path.dirname(self.filename)) time.sleep(1) else: + self.mode = mode break else: raise OSError(f"Unable to open HDF5 file {self.filename}") else: - if self.mode == "r": + pre_existing = os.path.exists(self.filename) + if self.mode is None: + if pre_existing: + self.mode = "r" + else: + self.mode = "a" + + if not pure and self.mode == "r" and h5py.version.version_tuple >= (2, 9): self.file_handle = open(self.filename, mode=self.mode + "b") self.h5 = h5py.File(self.file_handle, mode=self.mode) else: self.file_handle = None self.h5 = h5py.File(self.filename, mode=self.mode) self.to_close = [] - if not pre_existing: + + if not pre_existing or "w" in self.mode: self.h5.attrs["NX_class"] = "NXroot" self.h5.attrs["file_time"] = get_isotime(start_time) self.h5.attrs["file_name"] = self.filename self.h5.attrs["HDF5_Version"] = h5py.version.hdf5_version self.h5.attrs["creator"] = creator or self.__class__.__name__ + def __repr__(self): + return f"<{fully_qualified_name(self)} file on {self.h5}>" + def __del__(self): self.close() @@ -202,13 +226,14 @@ def get_entries(self): :return: list of HDF5 groups """ - entries = [(grp, from_isotime(self.h5[grp + "/start_time"][()])) - for grp in self.h5 - if isinstance(self.h5[grp], h5py.Group) and - ("start_time" in self.h5[grp]) and - self.get_attr(self.h5[grp], "NX_class") == "NXentry"] - entries.sort(key=lambda a: a[1], reverse=True) # sort entries in decreasing time - return [self.h5[i[0]] for i in entries] + entries = [(name, grp, from_isotime(self.h5[name + "/start_time"][()])) + for name, grp in self.h5.items() + if isinstance(grp, h5py.Group) and + ("start_time" in grp) and + self.get_attr(grp, "NX_class") == "NXentry"] + # print(entries) + entries.sort(key=lambda a: a[-1], reverse=True) # sort entries in decreasing time + return [i[1] for i in entries] def find_detector(self, all=False): """ @@ -243,11 +268,14 @@ def new_entry(self, entry="entry", program_name="pyFAI", if not force_name: nb_entries = len(self.get_entries()) entry = "%s_%04i" % (entry, nb_entries) - entry_grp = self.h5.require_group(entry) + entry_grp = self.h5 + for i in entry.split("/"): + if i: + entry_grp = entry_grp.require_group(i) self.h5.attrs["default"] = entry_grp.name.strip("/") entry_grp.attrs["NX_class"] = "NXentry" entry_grp["title"] = str(title) - entry_grp["program_name"] = program_name + entry_grp["program_name"] = str(program_name) if isinstance(force_time, str): entry_grp["start_time"] = force_time else: diff --git a/plugins/bm29/subtracte.py b/plugins/bm29/subtracte.py index 4ea0cbe..3845a53 100644 --- a/plugins/bm29/subtracte.py +++ b/plugins/bm29/subtracte.py @@ -11,43 +11,127 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "27/05/2025" +__date__ = "09/03/2026" __status__ = "development" -__version__ = "0.3.0" +__version__ = "0.4.0" import os import posixpath import json import copy +import zipfile from math import log, pi -from collections import namedtuple +from typing import NamedTuple from urllib3.util import parse_url +import numpy from dahu.plugin import Plugin from dahu.utils import fully_qualified_name import logging -logger = logging.getLogger("bm29.subtract") -import numpy -try: - import numexpr -except ImportError: - logger.error("Numexpr is not installed, falling back on numpy's implementations") - numexpr = None import h5py -import pyFAI, pyFAI.azimuthalIntegrator +import pyFAI +import pyFAI.integrator.azimuthal from pyFAI.containers import Integrate1dResult from pyFAI.method_registry import IntegrationMethod -import freesas, freesas.cormap, freesas.invariants +import freesas +import freesas.cormap +import freesas.invariants from freesas.autorg import auto_gpa, autoRg, auto_guinier from freesas.bift import BIFT +from freesas.app.extract_ascii import write_ascii from scipy.optimize import minimize -from .common import Sample, Ispyb, get_equivalent_frames, cmp_float, get_integrator, KeyCache, \ - polarization_factor, method, Nexus, get_isotime, SAXS_STYLE, NORMAL_STYLE, \ +from .common import Ispyb, get_equivalent_frames, cmp_float, get_integrator, KeyCache, \ + polarization_factor, method, SAXS_STYLE, NORMAL_STYLE, \ Sample, create_nexus_sample +from .nexus import Nexus, get_isotime from .ispyb import IspybConnector, NumpyEncoder from .memcached import to_memcached from .icat import send_icat +logger = logging.getLogger("bm29.subtract") +try: + import numexpr +except ImportError: + logger.error("Numexpr is not installed, falling back on numpy's implementations") + numexpr = None + -NexusJuice = namedtuple("NexusJuice", "filename h5path npt unit q I sigma poni mask energy polarization method signal2d error2d normalization sample") +class NexusJuice(NamedTuple): + filename: str + h5path: str + npt: int + unit: str + q: numpy.ndarray + I: numpy.ndarray # noqa + sigma: numpy.ndarray + poni:str + mask: numpy.ndarray + energy: float + polarization: float + method: tuple + signal2d: numpy.ndarray + error2d: numpy.ndarray + normalization: numpy.ndarray + sample:str + I_all: numpy.ndarray + sigma_all: numpy.ndarray + + +def save_zip(filename, sample_juice, buffer_juices): + """Save a stack of I into a zipfile with each frames in a dat-file. + + :param filename: name of the zip-file + :param sample_juice: + :param buffer_juices: list of buffer juice + :return: nothing + """ + destz_sample = "sample/" + destz_buffer = "buffer_%1i/" + common = {"q": sample_juice.q} + if sample_juice.sample: + sample = sample_juice.sample + if sample.name: + common["sample"]: sample.name + destz_sample += sample.name + else: + destz_sample += "sample" + + if sample.buffer: + common["buffer"] = sample.buffer + destz_buffer += sample.buffer if isinstance(sample.buffer, str) else sample.buffer.decode() + else: + destz_buffer += "buffer" + + if sample.temperature_env: + common["storage temperature"] = sample.temperature_env + + if sample.temperature: + common["exposure temperature"] = sample.temperature + + if sample.concentration: + common["concentration"] = sample.concentration + destz_sample += "_%04i.dat" + destz_buffer += "_%04i.dat" + res = {} + # sample + idx = 0 + for i, s in zip(sample_juice.I_all, sample_juice.sigma_all): + r = copy.copy(common) + r["I"] = i + r["std"] = s + res[destz_sample % idx] = r + idx+=1 + # buffers + for buffer_idx, buffer in enumerate(buffer_juices): + idx = 0 + for i, s in zip(buffer.I_all, buffer.sigma_all): + r = copy.copy(common) + r["I"] = i + r["std"] = s + res[destz_buffer % (buffer_idx, idx)] = r + idx+=1 + + with zipfile.ZipFile(filename, "w") as z: + for name, frame in res.items(): + z.writestr(name, write_ascii(frame)) class SubtractBuffer(Plugin): @@ -209,7 +293,7 @@ def create_nexus(self): entry_grp = nxs.new_entry("entry", self.input.get("plugin_name", "dahu"), title='BioSaxs buffer subtraction', force_time=get_isotime()) - nxs.h5.attrs["default"] = entry_grp.name.strip["/"] + nxs.h5.attrs["default"] = entry_grp.name.strip("/") # Configuration cfg_grp = nxs.new_class(entry_grp, "configuration", "NXnote") @@ -232,13 +316,18 @@ def create_nexus(self): # Sample: outsourced ! create_nexus_sample(nxs, entry_grp, self.sample_juice.sample) + #save input curves as zipfile: TODO Check that this is is working: + save_zip(os.path.splitext(self.output_file)[0]+".zip", + self.sample_juice, + self.buffer_juices) + # Process 1: CorMap cormap_grp = nxs.new_class(entry_grp, "1_correlation_mapping", "NXprocess") cormap_grp["sequence_index"] = 1 cormap_grp["program"] = "freesas.cormap" cormap_grp["version"] = freesas.version cormap_grp["date"] = get_isotime() - cormap_data = nxs.new_class(cormap_grp, "results", "NXdata") + cormap_data = nxs.new_class(cormap_grp, "result", "NXdata") cormap_data.attrs["SILX_style"] = NORMAL_STYLE cfg_grp = nxs.new_class(cormap_grp, "configuration", "NXcollection") @@ -278,7 +367,7 @@ def create_nexus(self): average_grp["sequence_index"] = 2 average_grp["program"] = fully_qualified_name(self.__class__) average_grp["version"] = __version__ - average_data = nxs.new_class(average_grp, "results", "NXdata") + average_data = nxs.new_class(average_grp, "result", "NXdata") average_data.attrs["SILX_style"] = SAXS_STYLE average_data.attrs["signal"] = "intensity_normed" # Stage 2 processing @@ -341,7 +430,7 @@ def create_nexus(self): ai2_grp["version"] = pyFAI.version ai2_grp["date"] = get_isotime() radial_unit, unit_name = str(key_cache.unit).split("_", 1) - ai2_data = nxs.new_class(ai2_grp, "results", "NXdata") + ai2_data = nxs.new_class(ai2_grp, "result", "NXdata") ai2_data.attrs["SILX_style"] = SAXS_STYLE ai2_data.attrs["title"] = "%s, subtracted" % self.sample_juice.sample.name ai2_data.attrs["signal"] = "I" @@ -404,7 +493,7 @@ def create_nexus(self): guinier_autorg = nxs.new_class(guinier_grp, "autorg", "NXcollection") guinier_gpa = nxs.new_class(guinier_grp, "gpa", "NXcollection") guinier_guinier = nxs.new_class(guinier_grp, "guinier", "NXcollection") - guinier_data = nxs.new_class(guinier_grp, "results", "NXdata") + guinier_data = nxs.new_class(guinier_grp, "result", "NXdata") guinier_data.attrs["SILX_style"] = NORMAL_STYLE guinier_data.attrs["title"] = "Guinier analysis" # Stage4 processing: autorg and auto_gpa @@ -490,8 +579,8 @@ def create_nexus(self): # Stage #4 Guinier plot generation: - q, I, err = sasm.T[:3] - mask = (I > 0) & numpy.isfinite(I) & (q > 0) & numpy.isfinite(q) + q, I_ary, err = sasm.T[:3] + mask = (I_ary > 0) & numpy.isfinite(I_ary) & (q > 0) & numpy.isfinite(q) if err is not None: mask &= (err > 0.0) & numpy.isfinite(err) mask = mask.astype(bool) @@ -506,7 +595,7 @@ def create_nexus(self): mask[end:] = False q2 = q[mask] ** 2 - logI = numpy.log(I[mask]) + logI = numpy.log(I_ary[mask]) dlogI = err[mask] / logI q2_ds = guinier_data.create_dataset("q2", data=q2.astype(numpy.float32)) q2_ds.attrs["unit"] = radius_unit + "⁻²" @@ -538,7 +627,7 @@ def create_nexus(self): kratky_grp["program"] = "freesas.autorg" kratky_grp["version"] = freesas.version kratky_grp["date"] = get_isotime() - kratky_data = nxs.new_class(kratky_grp, "results", "NXdata") + kratky_data = nxs.new_class(kratky_grp, "result", "NXdata") kratky_data.attrs["SILX_style"] = NORMAL_STYLE kratky_data.attrs["title"] = "Dimensionless Kratky plots" kratky_grp.attrs["default"] = posixpath.relpath(kratky_data.name, kratky_grp.name) @@ -547,28 +636,28 @@ def create_nexus(self): Rg = guinier.Rg I0 = guinier.I0 xdata = q * Rg - ydata = xdata * xdata * I / I0 + ydata = xdata * xdata * I_ary / I0 dy = xdata * xdata * err / I0 qRg_ds = kratky_data.create_dataset("qRg", data=xdata.astype(numpy.float32)) qRg_ds.attrs["interpretation"] = "spectrum" qRg_ds.attrs["long_name"] = "q·Rg (unit-less)" #Nota the "/" hereafter is chr(8725), the division sign and not the usual slash - k_ds = kratky_data.create_dataset("q2Rg2I∕I0", data=ydata.astype(numpy.float32)) + k_ds = kratky_data.create_dataset("q2Rg2I÷I0", data=ydata.astype(numpy.float32)) k_ds.attrs["interpretation"] = "spectrum" k_ds.attrs["long_name"] = "q²Rg²I(q)/I₀" ke_ds = kratky_data.create_dataset("errors", data=dy.astype(numpy.float32)) ke_ds.attrs["interpretation"] = "spectrum" kratky_data_attrs = kratky_data.attrs - kratky_data_attrs["signal"] = "q2Rg2I∕I0" - kratky_data_attrs["axes"] = "qRg" + kratky_data_attrs["signal"] = k_ds.name + kratky_data_attrs["axes"] = qRg_ds.name # stage 6: Rambo-Tainer invariant rti_grp = nxs.new_class(entry_grp, "6_invariants", "NXprocess") rti_grp["sequence_index"] = 6 rti_grp["program"] = "freesas.invariants" rti_grp["version"] = freesas.version - rti_data = nxs.new_class(rti_grp, "results", "NXdata") + rti_data = nxs.new_class(rti_grp, "result", "NXdata") # average_data.attrs["SILX_style"] = SAXS_STYLE # average_data.attrs["signal"] = "intensity_normed" # Rambo_Tainer @@ -606,14 +695,14 @@ def create_nexus(self): bift_grp["program"] = "freesas.bift" bift_grp["version"] = freesas.version bift_grp["date"] = get_isotime() - bift_data = nxs.new_class(bift_grp, "results", "NXdata") + bift_data = nxs.new_class(bift_grp, "result", "NXdata") bift_data.attrs["SILX_style"] = NORMAL_STYLE bift_data.attrs["title"] = "Pair distance distribution function p(r)" cfg_grp = nxs.new_class(bift_grp, "configuration", "NXcollection") # Process stage7, i.e. perform the IFT try: - bo = BIFT(q, I, err) + bo = BIFT(q, I_ary, err) cfg_grp["Rg"] = guinier.Rg # Pretty limited quality as we have real time constrains cfg_grp["npt"] = npt = 64 @@ -690,23 +779,19 @@ def read_nexus(filename): with Nexus(filename, "r") as nxsr: entry_grp = nxsr.get_entries()[0] h5path = entry_grp.name - nxdata_grp = nxsr.h5[entry_grp.attrs["default"]] + nxdata_grp = entry_grp[entry_grp.attrs["default"]] signal = nxdata_grp.attrs["signal"] axis = nxdata_grp.attrs["axes"] - I = nxdata_grp[signal][()] + I_ary = nxdata_grp[signal][()] q = nxdata_grp[axis][()] sigma = nxdata_grp["errors"][()] npt = len(q) unit = pyFAI.units.to_unit(axis + "_" + nxdata_grp[axis].attrs["units"]) integration_grp = nxdata_grp.parent poni = integration_grp["configuration/file_name"][()] - if isinstance(poni, bytes): - poni = poni.decode() - else: - poni = str(poni) - poni = poni.strip() + poni = str_(poni).strip() if not os.path.exists(poni): - poni = str(integration_grp["configuration/data"][()]).strip() + poni = str_(integration_grp["configuration/data"][()]).strip() polarization = integration_grp["configuration/polarization_factor"][()] method = IntegrationMethod.select_method(**json.loads(integration_grp["configuration/integration_method"][()]))[0] instrument_grp = nxsr.get_class(entry_grp, class_type="NXinstrument")[0] @@ -722,7 +807,7 @@ def read_nexus(filename): sample_grp = nxsr.get_class(entry_grp, class_type="NXsample")[0] sample_name = posixpath.basename(sample_grp.name) - buffer = sample_grp["buffer"][()] if "buffer" in sample_grp else "" + buffer = str_(sample_grp["buffer"][()] if "buffer" in sample_grp else "") concentration = sample_grp["concentration"][()] if "concentration" in sample_grp else "" description = sample_grp["description"][()] if "description" in sample_grp else "" hplc = sample_grp["hplc"][()] if "hplc" in sample_grp else "" @@ -730,7 +815,15 @@ def read_nexus(filename): temperature_env = sample_grp["temperature_env"][()] if "temperature_env" in sample_grp else "" sample = Sample(sample_name, description, buffer, concentration, hplc, temperature_env, temperature) - return NexusJuice(filename, h5path, npt, unit, q, I, sigma, poni, mask, energy, polarization, method, image2d, error2d, norm, sample) + if "1_integration" in entry_grp: + I_all = entry_grp["1_integration/result/I"][()] + sigma_all = entry_grp["1_integration/result/errors"][()] + else: + I_all = [] + sigma_all = [] + + return NexusJuice(filename, h5path, npt, unit, q, I_ary, sigma, poni, mask, energy, polarization, + method, image2d, error2d, norm, sample, I_all, sigma_all) def send_to_ispyb(self): if self.ispyb.url and parse_url(self.ispyb.url).host: @@ -753,6 +846,7 @@ def send_to_icat(self): raw=raw, path=os.path.dirname(os.path.abspath(self.output_file)), data=to_icat, + dataset="subtraction", gallery=self.ispyb.gallery or os.path.join(os.path.dirname(os.path.abspath(self.output_file)), "gallery"), metadata=metadata) @@ -766,3 +860,8 @@ def send_to_memcached(self): return to_memcached(dico) +def str_(smth): + if isinstance(smth, bytes): + return smth.decode() + else: + return str(smth) diff --git a/src/dahu/__init__.py b/src/dahu/__init__.py index a3a66ad..26ddd77 100644 --- a/src/dahu/__init__.py +++ b/src/dahu/__init__.py @@ -24,7 +24,7 @@ # ###########################################################################*/ """ -Data Analysis Highly tailored for Upbl09a +Data Analysis Highly tailored for Upbl09a """ from __future__ import with_statement, print_function, absolute_import, division @@ -33,18 +33,18 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "03/02/2025" +__date__ = "11/03/2026" __status__ = "production" import os +from . import utils as utils +from .factory import plugin_factory as plugin_factory +from . import plugin as plugin +from . import job as job + project = os.path.basename(os.path.dirname(os.path.abspath(__file__))) try: from .version import __date__ as date # noqa from .version import version, version_info, hexversion, strictversion, citation # noqa except ImportError: raise RuntimeError("Do NOT use %s from its sources: build it and use the built version" % project) - -from . import utils -from .factory import plugin_factory -from . import plugin -from . import job diff --git a/src/dahu/app/reprocess.py b/src/dahu/app/reprocess.py index a7a3d57..e0a3a00 100644 --- a/src/dahu/app/reprocess.py +++ b/src/dahu/app/reprocess.py @@ -4,11 +4,15 @@ Reprocess a job using the current """ -import os, json, logging, time +import os +import json +import logging +import time from argparse import ArgumentParser import dahu.factory -from dahu.job import Job -from dahu.utils import get_workdir, NumpyEncoder +from ..job import Job +from ..utils import get_workdir, NumpyEncoder + logging.basicConfig() STATE_UNINITIALIZED = Job.STATE_UNINITIALIZED @@ -70,7 +74,7 @@ def _run_(plugin, what): def process(args): """Process a set of arguments - + :param args: list of files to process """ working_dir = get_workdir() diff --git a/src/dahu/app/tango_server.py b/src/dahu/app/tango_server.py index 96581ee..f99ee83 100644 --- a/src/dahu/app/tango_server.py +++ b/src/dahu/app/tango_server.py @@ -14,7 +14,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "22/02/2024" +__date__ = "11/03/2026" __status__ = "beta" __docformat__ = 'restructuredtext' @@ -22,6 +22,11 @@ import os import tempfile import logging +from argparse import ArgumentParser +import PyTango +from .. import utils as dahu_utils +from ..server import DahuDSClass, DahuDS + logging.basicConfig(level=logging.INFO) logger = logging.getLogger("dahu_server") try: @@ -31,25 +36,21 @@ logger.debug("No socket opened for debugging. Please install rfoo") -import dahu.utils -from dahu.server import DahuDSClass, DahuDS # set loglevel at least at INFO if logger.getEffectiveLevel() > logging.INFO: logger.setLevel(logging.INFO) -from argparse import ArgumentParser -import PyTango def main(argv=None): if argv is None: argv = sys.argv logger.info("Starting Dahu Tango Device Server") - description = """Data Analysis Tango device server + description = """Data Analysis Tango device server """ - epilog = """ Provided by the Data analysis unit - ESRF + epilog = """ Provided by the Data analysis unit - ESRF """ usage = "dahu_server [-d] tango-options" - parser = ArgumentParser(description=description, epilog=epilog, add_help=True) + parser = ArgumentParser(description=description, epilog=epilog, add_help=True, usage=usage) parser.add_argument("-V", "--version", action='version', version='%(prog)s 0.0') parser.add_argument("-d", "--debug", dest="debug", default=False, action="store_true", help="Switch to debug mode ",) @@ -57,14 +58,14 @@ def main(argv=None): help="tango trace level") parser.add_argument("-f", "--file", dest="tango_file", default=None, help="tango log filename") - parser.add_argument("-l", "--log", dest="dahu_log", + parser.add_argument("-l", "--log", dest="dahu_log", default=os.path.join(os.environ.get("HOME",tempfile.gettempdir()),"log"), help="directory where dahu stores logs ") # parser.add_argument("-n", "--nbcpu", dest="nbcpu", type=int, # help="Maximum bumber of processing threads to be started", default=None) parser.add_argument(dest="tango", nargs="*", help="Tango device server options") options = parser.parse_args() - dahu.utils.get_workdir(options.dahu_log) + dahu_utils.get_workdir(options.dahu_log) tangoParam = ["DahuDS"] + options.tango if options.tango_verbose: tangoParam.append(f"-v{options.tango_verbose}") diff --git a/src/dahu/cache.py b/src/dahu/cache.py index c4a3647..002dc9e 100644 --- a/src/dahu/cache.py +++ b/src/dahu/cache.py @@ -3,16 +3,16 @@ # """ -Data Analysis RPC server over Tango: +Data Analysis RPC server over Tango: -Class Cache for storing the data in a Borg +Class Cache for storing the data in a Borg """ __authors__ = ["Jérôme Kieffer"] __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "22/02/2022" +__date__ = "11/03/2026" __status__ = "production" import os @@ -23,10 +23,10 @@ class DataCache(dict): """ - Class behaves like a dict with a finite size, used to cache some data for some time + Class behaves like a dict with a finite size, used to cache some data for some time then discard them when other things are stored. - - This class can be configured as a borg (singleton-like): + + This class can be configured as a borg (singleton-like): It always returns the same values regardless to the instance of the object """ __shared_state = {} @@ -36,7 +36,7 @@ def __init__(self, max_size=10, borg=True): """ Constructor of DataCache :param max_size: number of element to keep in memory - :param borg: set to false to have the behavour of a normal class. By default, this is a Borg, all instances have the same content. + :param borg: set to false to have the behavour of a normal class. By default, this is a Borg, all instances have the same content. """ if borg: self.__dict__ = self.__shared_state @@ -55,7 +55,7 @@ def __init__(self, max_size=10, borg=True): self.dict = {} self.max_size = max_size self._sem = Semaphore() - + def __repr__(self): """ """ @@ -130,8 +130,8 @@ def pop(self, key): logger.debug("DataCache.pop %s", key) try: index = self.ordered.index(key) - except: - raise KeyError + except Exception as err: + raise KeyError from err self.ordered.pop(index) myData = self.dict.pop(key) return myData diff --git a/src/dahu/factory.py b/src/dahu/factory.py index 489f190..98d8ec5 100644 --- a/src/dahu/factory.py +++ b/src/dahu/factory.py @@ -3,7 +3,7 @@ # """ -Data Analysis RPC server over Tango: +Data Analysis RPC server over Tango: Factory for the loading of plugins """ @@ -12,25 +12,24 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "17/03/2020" +__date__ = "11/03/2026" __status__ = "production" import os import os.path as op import logging from collections import OrderedDict -logger = logging.getLogger("dahu.factory") from threading import Semaphore from .utils import get_workdir, fully_qualified_name - import importlib.util +logger = logging.getLogger("dahu.factory") def load_source(module_name, file_path): "Plugin loader which does not pollute sys.module" spec = importlib.util.spec_from_file_location(module_name, file_path) #module = importlib.util.module_from_spec(spec) - #spec.loader.exec_module(module) + #spec.loader.exec_module(module) module = spec.loader.load_module(spec.name) #Option: remove from sys.modules ... return module @@ -77,7 +76,7 @@ def add_directory(self, directory): python_files.append(i[:-3]) if op.isdir(j) and op.exists(op.join(j, "__init__.py")): python_files.append(i) - + logger.info(f"Available modules in dahu from {directory}:{os.linesep}" + " ".join(python_files)) with self._sem: self.plugin_dirs[abs_dir] = python_files @@ -85,7 +84,7 @@ def add_directory(self, directory): def search_plugin(self, plugin_name): """ Search for a given plugins ... - starting from the FQN package.class, + starting from the FQN package.class, """ if "." not in plugin_name: logger.error("plugin name have to be fully qualified, here: %s" % plugin_name) @@ -110,7 +109,7 @@ def search_plugin(self, plugin_name): def __call__(self, plugin_name): """ create a plugin instance from its name - + @param plugin_name: name of the plugin as a string @return: plugin instance """ @@ -129,13 +128,13 @@ def __call__(self, plugin_name): def register(cls, klass, fqn=None): """ Register a class as a plugin which can be instanciated. - + This can be used as a decorator - - @plugin_factor.register - + + @plugin_factor.register + @param klass: class to be registered as a plugin - @param fqn: fully qualified name + @param fqn: fully qualified name @return klass """ if fqn is None: diff --git a/src/dahu/job.py b/src/dahu/job.py index f39b945..86a2114 100644 --- a/src/dahu/job.py +++ b/src/dahu/job.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # """ -Data Analysis RPC server over Tango: +Data Analysis RPC server over Tango: Contains the Job class which handles jobs. A static part of the class contains statistics of the class @@ -12,7 +12,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "20/02/2025" +__date__ = "11/03/2026" __status__ = "production" from threading import Thread, Semaphore @@ -24,11 +24,11 @@ import json import logging import traceback -logger = logging.getLogger("dahu.job") -# logger.setLevel(logging.DEBUG) from . import utils from .factory import plugin_factory from .utils import NumpyEncoder +logger = logging.getLogger("dahu.job") +# logger.setLevel(logging.DEBUG) # Python 2to3 compatibility StringTypes = (six.binary_type, six.text_type) diff --git a/src/dahu/server.py b/src/dahu/server.py index 3d1ca6f..949c37e 100644 --- a/src/dahu/server.py +++ b/src/dahu/server.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 # coding: utf-8 -from __future__ import with_statement, print_function, absolute_import, division """ -Data Analysis RPC server over Tango: +Data Analysis RPC server over Tango: Tango device server """ @@ -11,32 +10,24 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "09/07/2021" +__date__ = "11/03/2026" __status__ = "production" __docformat__ = 'restructuredtext' import sys import os -import json import threading import logging import time -import types -import multiprocessing -import six -if six.PY2: - from Queue import Queue -else: - from queue import Queue +from queue import Queue +import PyTango +from .job import Job, plugin_factory logger = logging.getLogger("dahu.server") # set loglevel at least at INFO if logger.getEffectiveLevel() > logging.INFO: logger.setLevel(logging.INFO) -import PyTango -from .job import Job, plugin_factory - try: from rfoo.utils import rconsole rconsole.spawn_server() @@ -120,7 +111,14 @@ def listPlugins(self): res = ["List of all plugin currently loaded (use initPlugin to loaded additional plugins):"] plugins = list(plugin_factory.registry.keys()) plugins.sort() - return os.linesep.join(res + [" %s : %s" % (i, plugin_factory.registry[i].__doc__.split("\n")[0]) for i in plugins]) + plugins_doc = {} + for i in plugins: + for j in plugin_factory.registry[i].__doc__.split(os.linesep): + doc = j.strip() + if doc: # Non empty line in docstring + break + plugins_doc[i] = doc + return os.linesep.join(res + [f' {i} : {doc}' for i, j in plugins_doc.items()]) def initPlugin(self, name): """ @@ -134,9 +132,9 @@ def initPlugin(self, name): err = "plugin %s failed to be instanciated: %s" % (name, error) logger.error(err) if plugin is None or err: - return "Plugin not found: %s, err" % (name, err) + return f"Plugin not found: {name}, {err}" else: - return "Plugin loaded: %s%s%s" % (name, os.linesep, plugin.__doc__) + return f"Plugin loaded: {name}{os.linesep}{plugin.__doc__}" def abort(self, jobId): """ @@ -283,7 +281,7 @@ def waitJob(self, jobId): Wait for a job to be finished and returns the status. May cause Tango timeout if too slow to finish .... May do polling to wait the job actually started - + @param jobId: identifier of the job (int) @return: status of the job """ @@ -351,5 +349,5 @@ class DahuDSClass(PyTango.DeviceClass): def __init__(self, name): PyTango.DeviceClass.__init__(self, name) - self.set_type(name); + self.set_type(name) logger.debug("In DahuDSClass constructor") diff --git a/src/dahu/test/__init__.py b/src/dahu/test/__init__.py index d82cf2f..fa810f0 100644 --- a/src/dahu/test/__init__.py +++ b/src/dahu/test/__init__.py @@ -27,21 +27,14 @@ # THE SOFTWARE. -from __future__ import absolute_import, division, print_function - -__doc__ = """Test module for pyFAI""" +"""Test module for pyFAI""" __authors__ = ["Jérôme Kieffer"] __contact__ = "jerome.kieffer@esrf.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "07/02/2020" +__date__ = "11/03/2026" -import sys -import os import unittest - - -from . import utilstest from . import test_all diff --git a/src/dahu/test/test_all.py b/src/dahu/test/test_all.py index 1c550a9..3aa26df 100755 --- a/src/dahu/test/test_all.py +++ b/src/dahu/test/test_all.py @@ -4,8 +4,6 @@ """Test suite for all dahu modules.""" -from __future__ import with_statement, print_function - __authors__ = ["Jérôme Kieffer"] __contact__ = "jerome.kieffer@esrf.eu" __license__ = "MIT" @@ -15,12 +13,12 @@ import sys import unittest from .utilstest import getLogger -logger = getLogger(__file__) - from . import test_job from . import test_plugin from . import test_cache +logger = getLogger(__file__) + def suite(): testSuite = unittest.TestSuite() diff --git a/src/dahu/test/test_cache.py b/src/dahu/test/test_cache.py index 0b334f3..f794a33 100644 --- a/src/dahu/test/test_cache.py +++ b/src/dahu/test/test_cache.py @@ -6,22 +6,21 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "22/02/2022" +__date__ = "11/03/2026" __status__ = "production" import unittest from . import utilstest -logger = utilstest.getLogger(__name__) from ..cache import DataCache - +logger = utilstest.getLogger(__name__) class TestCache(unittest.TestCase): - def test_cache(self): + def test_cache(self): b0 = DataCache(5, borg=True) b1 = DataCache(7, borg=True) self.assertEqual(b0.max_size, 5, "borg initialized") self.assertEqual(b1.max_size, 5, "borg not re-initialized") - + n0 = DataCache(5, borg=False) n1 = DataCache(7, borg=False) self.assertEqual(n0.max_size, 5, "Normal class behaviour") @@ -33,16 +32,16 @@ def test_cache(self): self.assertEqual(b1.get("Iam"), "borg", "borg behavour") self.assertEqual(n0.get("Iam"), "normal", "Normal class`") self.assertEqual(n1.get("Iam"), None, "Normal class`") - + for i in range(6): b0[i] = i n0[i] = i - + self.assertEqual(b0.get("Iam"), None, "object dropped") self.assertEqual(b1.get("Iam"), None, "object dropped, borg") self.assertEqual(n0.get("Iam"), None, "object dropped") self.assertEqual(n1.get("Iam"), None, "Normal class`") - + def suite(): testSuite = unittest.TestSuite() testSuite.addTest(TestCache("test_cache")) diff --git a/src/dahu/test/test_job.py b/src/dahu/test/test_job.py index a117ceb..334699c 100644 --- a/src/dahu/test/test_job.py +++ b/src/dahu/test/test_job.py @@ -6,14 +6,14 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "22/02/2022" +__date__ = "11/03/2026" __status__ = "production" import os import unittest from . import utilstest -logger = utilstest.getLogger(__name__) from .. import job +logger = utilstest.getLogger(__name__) class TestJob(unittest.TestCase): diff --git a/src/dahu/test/test_plugin.py b/src/dahu/test/test_plugin.py index b3c087f..d24c18a 100644 --- a/src/dahu/test/test_plugin.py +++ b/src/dahu/test/test_plugin.py @@ -6,15 +6,15 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "22/02/2022" +__date__ = "11/03/2026" __status__ = "production" import unittest from . import utilstest -logger = utilstest.getLogger("test_plugin") from ..plugin import Plugin, plugin_from_function from ..factory import plugin_factory from ..job import Job +logger = utilstest.getLogger("test_plugin") class TestPlugin(unittest.TestCase): @@ -42,12 +42,12 @@ def test_wait_for(self): print(dir(p)) print(p.__class__.__module__) p.wait_for(42) #this job does not exist, fails with a warning: - + # Test synchonization with finished job j = Job("example.square", {"x": 5}) j.start() p.wait_for(j.id) - + #Test failure when it does not start (timeout) p.TIMEOUT=0.2 j = Job("example.square", {"x": 6}) @@ -57,11 +57,11 @@ def test_wait_for(self): logger.debug("Failed as expected with: %s", err) else: raise RuntimeError("Expected to fail !") - - - - + + + + def suite(): diff --git a/src/dahu/test/utilstest.py b/src/dahu/test/utilstest.py index 656047b..f091fdf 100644 --- a/src/dahu/test/utilstest.py +++ b/src/dahu/test/utilstest.py @@ -1,6 +1,6 @@ # coding: utf-8 # -# Copyright (C) 2012-2016 European Synchrotron Radiation Facility, Grenoble, France +# Copyright (C) 2012-2026 European Synchrotron Radiation Facility, Grenoble, France # # Principal author: Jérôme Kieffer (Jerome.Kieffer@ESRF.eu) # @@ -27,33 +27,26 @@ __contact__ = "jerome.kieffer@esrf.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "04/12/2024" +__date__ = "11/03/2026" -PACKAGE = "dahu" -DATA_KEY = "DAHU_DATA" - -if __name__ == "__main__": - __name__ = "dahu.test" import os import sys import getpass -import subprocess -import threading import unittest import logging -try: # Python3 - from urllib.request import urlopen, ProxyHandler, build_opener, URLError -except ImportError: # Python2 - from urllib2 import urlopen, ProxyHandler, build_opener, URLError -# import urllib2 +from urllib.request import urlopen, ProxyHandler, build_opener, URLError import numpy import shutil import json import tempfile +from argparse import ArgumentParser +import threading + +PACKAGE = "dahu" +DATA_KEY = "DAHU_DATA" logging.basicConfig(level=logging.WARNING) logger = logging.getLogger("%s.utilstest" % PACKAGE) - TEST_HOME = os.path.dirname(os.path.abspath(__file__)) @@ -134,9 +127,9 @@ def timeoutDuringDownload(cls, imagename=None): imagename = "2252/testimages.tar.bz2 unzip it " raise RuntimeError(f"""Could not automatically download test images! If you are behind a firewall, please set both environment variable http_proxy and https_proxy. -This even works under windows ! -Otherwise please try to download the images manually from: -{cls.url_base}/{imagename} +This even works under windows ! +Otherwise please try to download the images manually from: +{cls.url_base}/{imagename} and put it in in test/testimages.""") @classmethod @@ -221,10 +214,7 @@ def get_options(cls): Parse the command line to analyse options ... returns options """ if cls.options is None: - try: - from argparse import ArgumentParser - except: - from pyFAI.third_party.argparse import ArgumentParser + parser = ArgumentParser(usage="Tests for %s" % cls.name) parser.add_argument("-d", "--debug", dest="debug", help="run in debugging mode", @@ -365,9 +355,9 @@ def diff_crv(ref, obt, comment=""): fig = plt.figure() ax1 = fig.add_subplot(1, 2, 1) ax2 = fig.add_subplot(1, 2, 2) - im_ref = ax1.plot(ref, label="%s ref" % comment) - im_obt = ax1.plot(obt, label="%s obt" % comment) - im_delta = ax2.plot(delta, label="delta") + ax1.plot(ref, label="%s ref" % comment) + ax1.plot(obt, label="%s obt" % comment) + ax2.plot(delta, label="delta") fig.show() from pyFAI.utils import input input() @@ -398,3 +388,7 @@ def parameterise(testcase_klass, testcase_method=None, param=None): for name in testnames: suite.addTest(testcase_klass(name, param=param)) return suite + + +if __name__ == "__main__": + __name__ = "dahu.test" diff --git a/version.py b/version.py index 13364f1..29233dc 100755 --- a/version.py +++ b/version.py @@ -64,7 +64,7 @@ "final": 15} MAJOR = 2026 -MINOR = 1 +MINOR = 3 MICRO = 0 RELEV = "dev" # <16 SERIAL = 0 # <16