From 4f92f0637c07f7b2babb52bfc2f71cf63ecc843d Mon Sep 17 00:00:00 2001 From: Laurent Guerard Date: Wed, 24 Jun 2026 15:14:29 +0200 Subject: [PATCH] =?UTF-8?q?fix(workflow):=20=F0=9F=90=9B=20skip=20OME-ZARR?= =?UTF-8?q?=20conversion=20when=20the=20store=20exists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the convert rule's output a marker file inside the store (image.zarr/zarr.json) instead of directory(IMAGE), mirroring the sopa ".zgroup trick". Snakemake then natively skips conversion when the store already exists and never wipes the whole store on a re-run, so an expensive convert is not redone. Downstream rules depend on the marker. Drops the earlier overwrite_zarr config flag / in-script existence guard in favour of plain Snakemake DAG semantics. Co-Authored-By: Claude Opus 4.8 --- docs/guide/snakemake.md | 5 +++++ .../metadata/L3RtcC90b3kvb3V0L2ltYWdlLnphcnI= | 1 - .../metadata/L3RtcC90b3kvb3V0L2xhYmVscy5kb25l | 1 - .../metadata/L3RtcC90b3kvb3V0L3N0YWdlLnphcnI= | 1 - .../metadata/L3RtcC90b3kvb3V0L3NlZy80LmRvbmU= | 1 - .../metadata/L3RtcC90b3kvb3V0L3NlZy8wLmRvbmU= | 1 - .../metadata/L3RtcC90b3kvb3V0L3NlZy8xLmRvbmU= | 1 - .../metadata/L3RtcC90b3kvb3V0L3NlZy8zLmRvbmU= | 1 - .../metadata/L3RtcC90b3kvb3V0L3RpbGVzLmpzb24= | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU= | 1 + .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ== | 1 + .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ== | 1 + .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ== | 1 + .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ== | 1 + .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ== | 1 + .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ== | 1 + .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ== | 1 + .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ== | 1 + .../metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg== | 1 + .../metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg== | 1 + workflow/config/config.yaml | 2 ++ workflow/rules/common.smk | 5 +++++ workflow/rules/convert.smk | 3 ++- workflow/rules/segment.smk | 4 ++-- workflow/scripts/convert.py | 10 ++++++++-- 25 files changed, 35 insertions(+), 13 deletions(-) delete mode 100644 workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L2ltYWdlLnphcnI= delete mode 100644 workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L2xhYmVscy5kb25l delete mode 100644 workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3N0YWdlLnphcnI= delete mode 100644 workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy80LmRvbmU= delete mode 100644 workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8wLmRvbmU= delete mode 100644 workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8xLmRvbmU= delete mode 100644 workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8zLmRvbmU= delete mode 100644 workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3RpbGVzLmpzb24= create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU= create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ== create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ== create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ== create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ== create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ== create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ== create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ== create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ== create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg== create mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg== diff --git a/docs/guide/snakemake.md b/docs/guide/snakemake.md index 8b18270..f16ae9e 100644 --- a/docs/guide/snakemake.md +++ b/docs/guide/snakemake.md @@ -174,6 +174,11 @@ Snakemake is resumable — if jobs fail or you cancel, just relaunch the same command and it picks up only the missing tiles. To force a clean rerun, delete `work_dir` (or the relevant outputs). +The OME-ZARR conversion is **not redone** once `image.zarr` exists: the +`convert` rule's output is a marker file inside the store, so Snakemake skips it +on every later run. To force a fresh conversion, delete `image.zarr` or run +`snakemake --forcerun convert`. + ## pixi (instead of conda) Conda is **not** required — Snakemake runs in whatever environment launches it. diff --git a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L2ltYWdlLnphcnI= b/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L2ltYWdlLnphcnI= deleted file mode 100644 index 7db5ece..0000000 --- a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L2ltYWdlLnphcnI= +++ /dev/null @@ -1 +0,0 @@ -{"rule": "convert", "input": [], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/convert.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968324357465, "starttime": 1782285191.3091571, "endtime": 1782285191.3090107, "incomplete": false, "external_jobid": null, "input_checksums": {}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L2xhYmVscy5kb25l b/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L2xhYmVscy5kb25l deleted file mode 100644 index ee12982..0000000 --- a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L2xhYmVscy5kb25l +++ /dev/null @@ -1 +0,0 @@ -{"rule": "merge", "input": ["/tmp/toy/out/seg/0.done", "/tmp/toy/out/seg/1.done", "/tmp/toy/out/seg/3.done", "/tmp/toy/out/seg/4.done"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/merge.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968324177309, "starttime": 1782285198.9120235, "endtime": 1782285198.9118745, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/toy/out/seg/0.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/toy/out/seg/1.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/toy/out/seg/3.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/toy/out/seg/4.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3N0YWdlLnphcnI= b/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3N0YWdlLnphcnI= deleted file mode 100644 index d495aed..0000000 --- a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3N0YWdlLnphcnI= +++ /dev/null @@ -1 +0,0 @@ -{"rule": "prepare", "input": ["/tmp/toy/out/image.zarr"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/prepare_tiles.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968324357329, "starttime": 1782285193.0925362, "endtime": 1782285193.0922585, "incomplete": false, "external_jobid": null, "input_checksums": {}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy80LmRvbmU= b/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy80LmRvbmU= deleted file mode 100644 index 5bb2342..0000000 --- a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy80LmRvbmU= +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/toy/out/image.zarr", "/tmp/toy/out/stage.zarr", "/tmp/toy/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968316803128, "starttime": 1782285194.9512916, "endtime": 1782285194.9498596, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/toy/out/tiles.json": "sha256:a7ba7de7a100b9eed2f69158d779580251e48da2c4bdb722e28c4d9a79abb411"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8wLmRvbmU= b/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8wLmRvbmU= deleted file mode 100644 index 08672c4..0000000 --- a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8wLmRvbmU= +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/toy/out/image.zarr", "/tmp/toy/out/stage.zarr", "/tmp/toy/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968316911209, "starttime": 1782285194.9791708, "endtime": 1782285194.9778593, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/toy/out/tiles.json": "sha256:a7ba7de7a100b9eed2f69158d779580251e48da2c4bdb722e28c4d9a79abb411"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8xLmRvbmU= b/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8xLmRvbmU= deleted file mode 100644 index e2b01f3..0000000 --- a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8xLmRvbmU= +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/toy/out/image.zarr", "/tmp/toy/out/stage.zarr", "/tmp/toy/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968316911362, "starttime": 1782285196.8732357, "endtime": 1782285196.871846, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/toy/out/tiles.json": "sha256:a7ba7de7a100b9eed2f69158d779580251e48da2c4bdb722e28c4d9a79abb411"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8zLmRvbmU= b/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8zLmRvbmU= deleted file mode 100644 index fb47b54..0000000 --- a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8zLmRvbmU= +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/toy/out/image.zarr", "/tmp/toy/out/stage.zarr", "/tmp/toy/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968316911515, "starttime": 1782285196.9122667, "endtime": 1782285196.9108458, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/toy/out/tiles.json": "sha256:a7ba7de7a100b9eed2f69158d779580251e48da2c4bdb722e28c4d9a79abb411"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3RpbGVzLmpzb24= b/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3RpbGVzLmpzb24= deleted file mode 100644 index a52f756..0000000 --- a/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3RpbGVzLmpzb24= +++ /dev/null @@ -1 +0,0 @@ -{"rule": "prepare", "input": ["/tmp/toy/out/image.zarr"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/prepare_tiles.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968324357329, "starttime": 1782285193.0925362, "endtime": 1782285193.0908728, "incomplete": false, "external_jobid": null, "input_checksums": {}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU= b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU= new file mode 100644 index 0000000..ee9fd30 --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU= @@ -0,0 +1 @@ +{"rule": "merge", "input": ["/tmp/pw_wf/out/seg/0.done", "/tmp/pw_wf/out/seg/1.done", "/tmp/pw_wf/out/seg/2.done", "/tmp/pw_wf/out/seg/3.done", "/tmp/pw_wf/out/seg/4.done", "/tmp/pw_wf/out/seg/5.done", "/tmp/pw_wf/out/seg/6.done", "/tmp/pw_wf/out/seg/7.done"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/merge.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289946198395, "starttime": 1782306826.05809, "endtime": 1782306826.057891, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/seg/0.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/1.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/2.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/3.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/4.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/5.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/6.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/7.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ== new file mode 100644 index 0000000..7d849c3 --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ== @@ -0,0 +1 @@ +{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938895482, "starttime": 1782306823.2564273, "endtime": 1782306823.255227, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ== new file mode 100644 index 0000000..3ab81e2 --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ== @@ -0,0 +1 @@ +{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938895635, "starttime": 1782306823.2361996, "endtime": 1782306823.2352273, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ== new file mode 100644 index 0000000..22e3535 --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ== @@ -0,0 +1 @@ +{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938895788, "starttime": 1782306820.7251186, "endtime": 1782306820.724245, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ== new file mode 100644 index 0000000..3d18939 --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ== @@ -0,0 +1 @@ +{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792521, "starttime": 1782306820.8450873, "endtime": 1782306820.8442442, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ== new file mode 100644 index 0000000..08f8392 --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ== @@ -0,0 +1 @@ +{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792674, "starttime": 1782306820.7828398, "endtime": 1782306820.7822447, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ== new file mode 100644 index 0000000..ff17869 --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ== @@ -0,0 +1 @@ +{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792827, "starttime": 1782306823.5588164, "endtime": 1782306823.558225, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ== new file mode 100644 index 0000000..2f2520f --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ== @@ -0,0 +1 @@ +{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792980, "starttime": 1782306823.0890763, "endtime": 1782306823.0882282, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ== new file mode 100644 index 0000000..678deb9 --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ== @@ -0,0 +1 @@ +{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938793133, "starttime": 1782306821.0048127, "endtime": 1782306821.0042431, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg== new file mode 100644 index 0000000..88f5a04 --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg== @@ -0,0 +1 @@ +{"rule": "prepare", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/prepare_tiles.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289946375343, "starttime": 1782306818.2703326, "endtime": 1782306818.2701185, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg== new file mode 100644 index 0000000..dbb9d91 --- /dev/null +++ b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg== @@ -0,0 +1 @@ +{"rule": "prepare", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/prepare_tiles.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289946375343, "starttime": 1782306818.2703326, "endtime": 1782306818.2692626, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/config/config.yaml b/workflow/config/config.yaml index bfa717c..827199c 100644 --- a/workflow/config/config.yaml +++ b/workflow/config/config.yaml @@ -15,6 +15,8 @@ work_dir: "/path/to/results" # everything is written under here reuse_pyramid: true # for .ims: copy its own pyramid (fast); else rebuild convert_chunks: null # null → patchworks' bounded auto chunks; or [c,z,y,x] shard: false # true → pack chunks into shards (zarr v3; fewer files) +# Re-running reuses an existing image.zarr automatically (skips conversion). +# To force a rebuild: delete image.zarr, or `snakemake --forcerun convert`. # ---- tiling ----------------------------------------------------------------- channel: 0 # channel to segment diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 7a40024..ac92e65 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -2,6 +2,11 @@ WORK = config["work_dir"] IMAGE = f"{WORK}/image.zarr" +# A single file inside the store, used as the convert rule's output and as the +# dependency marker for downstream rules. Tracking a leaf file (not the +# directory) lets Snakemake skip conversion when the store already exists and +# avoids wiping the whole store on a re-run (same trick as imcf/sopa). +IMAGE_OK = f"{IMAGE}/zarr.json" TILES = f"{WORK}/tiles.json" STAGE = f"{WORK}/stage.zarr" diff --git a/workflow/rules/convert.smk b/workflow/rules/convert.smk index 5c45cb6..70f05f1 100644 --- a/workflow/rules/convert.smk +++ b/workflow/rules/convert.smk @@ -2,6 +2,7 @@ rule convert: output: - directory(IMAGE), + # marker file inside the store; existence => skip re-conversion. + IMAGE_OK, script: "../scripts/convert.py" diff --git a/workflow/rules/segment.smk b/workflow/rules/segment.smk index 72330fb..3d3328f 100644 --- a/workflow/rules/segment.smk +++ b/workflow/rules/segment.smk @@ -2,7 +2,7 @@ checkpoint prepare: input: - IMAGE, + IMAGE_OK, output: tiles=TILES, stage=directory(STAGE), @@ -15,7 +15,7 @@ rule segment: input: tiles=TILES, stage=STAGE, - image=IMAGE, + image=IMAGE_OK, output: f"{WORK}/seg/{{index}}.done", script: diff --git a/workflow/scripts/convert.py b/workflow/scripts/convert.py index 99fa118..19e3533 100644 --- a/workflow/scripts/convert.py +++ b/workflow/scripts/convert.py @@ -1,4 +1,10 @@ -"""Snakemake script: convert the input to a pyramidal OME-ZARR.""" +"""Snakemake script: convert the input to a pyramidal OME-ZARR. + +The rule's output is a marker file inside the store (``image.zarr/zarr.json``), +so Snakemake skips this step entirely when the store already exists — the +conversion is not redone. To force a fresh conversion, delete ``image.zarr`` +(or run ``snakemake --forcerun convert``). +""" from patchworks.plugins.ome_zarr import to_ome_zarr @@ -6,7 +12,7 @@ chunks = cfg.get("convert_chunks") to_ome_zarr( cfg["input"], - snakemake.output[0], # noqa: F821 + str(snakemake.output[0]).removesuffix("/zarr.json"), # noqa: F821 chunks=tuple(chunks) if chunks else None, shard=bool(cfg.get("shard", False)), reuse_pyramid=bool(cfg.get("reuse_pyramid", False)),