From 5d70851a38801a610ba9a275607f995ec715410f Mon Sep 17 00:00:00 2001 From: frederick Date: Thu, 23 Jul 2026 14:57:42 +0100 Subject: [PATCH 1/2] fixing errors in sample/timestamp min and max length settings --- e02/templates/Notebook_tmp.yaml | 114 ++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 e02/templates/Notebook_tmp.yaml diff --git a/e02/templates/Notebook_tmp.yaml b/e02/templates/Notebook_tmp.yaml new file mode 100644 index 0000000..517f87a --- /dev/null +++ b/e02/templates/Notebook_tmp.yaml @@ -0,0 +1,114 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: calibrate-ptycho + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.argoproj.io/title: Calibrate ptycho STEM data + workflows.argoproj.io/description: | + A utility which takes in a 4D STEM data set and outs the approximate real space pixel size, measured camera length and scan rotation, based off dpc + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" + workflows.diamond.ac.uk/parameter-schema: | + {{- .Files.Get "schema/notebook_Schema.json" | nindent 6 }} + workflows.diamond.ac.uk/ui-schema: | + {{- .Files.Get "schema/notebook_ui.json" | nindent 6 }} +spec: + entrypoint: calibration-notebook + arguments: + parameters: + - name: visitdir + valueFrom: + configMapKeyRef: + name: sessionspaces + key: data_directory + volumeClaimTemplates: + - metadata: + name: tmp + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + storageClassName: netapp + + templates: + - name: calibration-notebook + steps: + - - name: copy-notebook + template: mount-files + - - name: run-notebook + template: notebook + + - name: mount-files + script: + image: debian:stable-slim + volumeMounts: + - name: tmp + mountPath: /tmp + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + command: [bash] + source: | + OUTPUT="{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/ + mkdir -p $OUTPUT + echo "something!" + echo '{{ .Files.Get "notebooks/Ptycho_calibration.ipynb" | b64enc }}' | base64 -d > $OUTPUT/Ptycho_calibration.ipynb + + - name: notebook + inputs: + parameters: + - name: sample + value: "{{`{{workflow.parameters.sample}}`}}" + - name: timestamp + value: "{{`{{workflow.parameters.timestamp}}`}}" + - name: outpath + value: "{{`{{workflow.parameters.outpath}}`}}" + - name: thresh_lower + value: "{{`{{workflow.parameters.thresh_lower}}`}}" + - name: thresh_upper + value: "{{`{{workflow.parameters.thresh_upper}}`}}" + script: + image: gitlab.diamond.ac.uk:5050/scisoft/ptychography/dimtools/mib2x + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmp + mountPath: /tmp + - name: software + mountPath: /dls_sw/e02/medipix_mask + command: [bash] + source: | + ERROR_TXT="raise error" + ERROR_STR="An error occured within the notebook. Please open the output file to check" + OUTPUT="{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/ + python -m papermill "{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/Ptycho_calibration.ipynb $OUTPUT/Calibrate-notebook.ipynb \ + -p visit "{{`{{ workflow.parameters.visitdir }}`}}" + -p sample "{{`{{ workflow.parameters.sample }}`}}" + -p timestamp "{{`{{workflow.parameters.timestamp}}`}}" + -p maskpath /software/29042024_12bitmask2.h5 + -p outpath "{{`{{workflow.parameters.outpath}}`}}" + -p lower "{{`{{workflow.parameters.thresh_lower}}`}}" + -p upper "{{`{{workflow.parameters.thresh_upper}}`}}" + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/Calibrate-notebook.ipynb \ + --to html + + outputs: + artifacts: + - name: phase-output + path: "{{`{{workflow.parameters.visitdir}}`}}/processing/workflows/Calibrate-notebook.html" + archive: + none: {} + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: 10 + memory: 50Gi + limits: + cpu: 10 + memory: 50Gi \ No newline at end of file From e5b429f9d410de3486be982ce3a550192e840736 Mon Sep 17 00:00:00 2001 From: frederick Date: Thu, 23 Jul 2026 15:04:42 +0100 Subject: [PATCH 2/2] deleting tmp workflow and updating schema of epsic workflow --- e02/schema/notebook_Schema.json | 6 +- e02/templates/Notebook_tmp.yaml | 114 -------------------------------- 2 files changed, 3 insertions(+), 117 deletions(-) delete mode 100644 e02/templates/Notebook_tmp.yaml diff --git a/e02/schema/notebook_Schema.json b/e02/schema/notebook_Schema.json index 5cb0732..baa6af7 100644 --- a/e02/schema/notebook_Schema.json +++ b/e02/schema/notebook_Schema.json @@ -4,15 +4,15 @@ "properties": { "sample": { "default": null, - "maxLength": 128, "minLength": 1, + "maxLength": 128, "title": "Sample name", "type": "string" }, "timestamp": { "default": null, - "maxLength": 1, - "minLength": 128, + "minLength": 1, + "maxLength": 128, "title": "Time stamp", "type": "string" }, diff --git a/e02/templates/Notebook_tmp.yaml b/e02/templates/Notebook_tmp.yaml deleted file mode 100644 index 517f87a..0000000 --- a/e02/templates/Notebook_tmp.yaml +++ /dev/null @@ -1,114 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: calibrate-ptycho - labels: - workflows.diamond.ac.uk/science-group-imaging: "true" - annotations: - workflows.argoproj.io/title: Calibrate ptycho STEM data - workflows.argoproj.io/description: | - A utility which takes in a 4D STEM data set and outs the approximate real space pixel size, measured camera length and scan rotation, based off dpc - workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" - workflows.diamond.ac.uk/parameter-schema: | - {{- .Files.Get "schema/notebook_Schema.json" | nindent 6 }} - workflows.diamond.ac.uk/ui-schema: | - {{- .Files.Get "schema/notebook_ui.json" | nindent 6 }} -spec: - entrypoint: calibration-notebook - arguments: - parameters: - - name: visitdir - valueFrom: - configMapKeyRef: - name: sessionspaces - key: data_directory - volumeClaimTemplates: - - metadata: - name: tmp - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi - storageClassName: netapp - - templates: - - name: calibration-notebook - steps: - - - name: copy-notebook - template: mount-files - - - name: run-notebook - template: notebook - - - name: mount-files - script: - image: debian:stable-slim - volumeMounts: - - name: tmp - mountPath: /tmp - - name: session - mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" - command: [bash] - source: | - OUTPUT="{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/ - mkdir -p $OUTPUT - echo "something!" - echo '{{ .Files.Get "notebooks/Ptycho_calibration.ipynb" | b64enc }}' | base64 -d > $OUTPUT/Ptycho_calibration.ipynb - - - name: notebook - inputs: - parameters: - - name: sample - value: "{{`{{workflow.parameters.sample}}`}}" - - name: timestamp - value: "{{`{{workflow.parameters.timestamp}}`}}" - - name: outpath - value: "{{`{{workflow.parameters.outpath}}`}}" - - name: thresh_lower - value: "{{`{{workflow.parameters.thresh_lower}}`}}" - - name: thresh_upper - value: "{{`{{workflow.parameters.thresh_upper}}`}}" - script: - image: gitlab.diamond.ac.uk:5050/scisoft/ptychography/dimtools/mib2x - volumeMounts: - - name: session - mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" - - name: tmp - mountPath: /tmp - - name: software - mountPath: /dls_sw/e02/medipix_mask - command: [bash] - source: | - ERROR_TXT="raise error" - ERROR_STR="An error occured within the notebook. Please open the output file to check" - OUTPUT="{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/ - python -m papermill "{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/Ptycho_calibration.ipynb $OUTPUT/Calibrate-notebook.ipynb \ - -p visit "{{`{{ workflow.parameters.visitdir }}`}}" - -p sample "{{`{{ workflow.parameters.sample }}`}}" - -p timestamp "{{`{{workflow.parameters.timestamp}}`}}" - -p maskpath /software/29042024_12bitmask2.h5 - -p outpath "{{`{{workflow.parameters.outpath}}`}}" - -p lower "{{`{{workflow.parameters.thresh_lower}}`}}" - -p upper "{{`{{workflow.parameters.thresh_upper}}`}}" - > $OUTPUT/papermill.log 2>&1 - - python -m jupyter nbconvert $OUTPUT/Calibrate-notebook.ipynb \ - --to html - - outputs: - artifacts: - - name: phase-output - path: "{{`{{workflow.parameters.visitdir}}`}}/processing/workflows/Calibrate-notebook.html" - archive: - none: {} - podSpecPatch: | - containers: - - name: main - resources: - requests: - cpu: 10 - memory: 50Gi - limits: - cpu: 10 - memory: 50Gi \ No newline at end of file