diff --git a/docs/guide/snakemake.md b/docs/guide/snakemake.md index f16ae9e..8ad67bf 100644 --- a/docs/guide/snakemake.md +++ b/docs/guide/snakemake.md @@ -56,6 +56,7 @@ shard: false # true → pack chunks into shards (fewer files) channel: 0 # channel to segment (null = keep all) level: 0 # pyramid level (0 = full resolution) tile_shape: "auto" # "auto", or e.g. [16, 1024, 1024] (zyx) +gpu_memory_gb: null # for "auto" on SLURM: your segment GPU's VRAM overlap: 30 # halo ≈ one object diameter skip_empty: true # skip background tiles empty_threshold: null # null → Otsu @@ -81,6 +82,11 @@ sequential_labels: true # renumber labels to a contiguous 1..N more (faster) jobs; very large 3-D tiles are slow. Keep `do_3D: false` (2-D per slice) if your objects segment fine per slice — it is much faster. + Tile planning runs on a **CPU** node, which cannot see the segment GPU, so + it logs `GPU memory query failed … using 8 GiB default` and sizes tiles for + 8 GiB. Harmless, but to size for the real GPU set `gpu_memory_gb:` to its + VRAM (e.g. `24`, `40`, `80`) — or just set `tile_shape` explicitly. + ## 4. Dry-run (always do this first) Check the plan without running anything: diff --git a/workflow/.snakemake/iocache/latest.pkl b/workflow/.snakemake/iocache/latest.pkl new file mode 100644 index 0000000..2923948 Binary files /dev/null and b/workflow/.snakemake/iocache/latest.pkl differ diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU= b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU= deleted file mode 100644 index ee9fd30..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU= +++ /dev/null @@ -1 +0,0 @@ -{"rule": "merge", "input": ["/tmp/pw_wf/out/seg/0.done", "/tmp/pw_wf/out/seg/1.done", "/tmp/pw_wf/out/seg/2.done", "/tmp/pw_wf/out/seg/3.done", "/tmp/pw_wf/out/seg/4.done", "/tmp/pw_wf/out/seg/5.done", "/tmp/pw_wf/out/seg/6.done", "/tmp/pw_wf/out/seg/7.done"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/merge.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289946198395, "starttime": 1782306826.05809, "endtime": 1782306826.057891, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/seg/0.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/1.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/2.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/3.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/4.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/5.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/6.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/7.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ== deleted file mode 100644 index 7d849c3..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938895482, "starttime": 1782306823.2564273, "endtime": 1782306823.255227, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ== deleted file mode 100644 index 3ab81e2..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938895635, "starttime": 1782306823.2361996, "endtime": 1782306823.2352273, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ== deleted file mode 100644 index 22e3535..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938895788, "starttime": 1782306820.7251186, "endtime": 1782306820.724245, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ== deleted file mode 100644 index 3d18939..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792521, "starttime": 1782306820.8450873, "endtime": 1782306820.8442442, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ== deleted file mode 100644 index 08f8392..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792674, "starttime": 1782306820.7828398, "endtime": 1782306820.7822447, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ== deleted file mode 100644 index ff17869..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792827, "starttime": 1782306823.5588164, "endtime": 1782306823.558225, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ== deleted file mode 100644 index 2f2520f..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792980, "starttime": 1782306823.0890763, "endtime": 1782306823.0882282, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ== deleted file mode 100644 index 678deb9..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938793133, "starttime": 1782306821.0048127, "endtime": 1782306821.0042431, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg== deleted file mode 100644 index 88f5a04..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "prepare", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/prepare_tiles.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289946375343, "starttime": 1782306818.2703326, "endtime": 1782306818.2701185, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg== b/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg== deleted file mode 100644 index dbb9d91..0000000 --- a/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "prepare", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json"], "log": [], "shellcmd": null, "params": [], "code": " \"../scripts/prepare_tiles.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289946375343, "starttime": 1782306818.2703326, "endtime": 1782306818.2692626, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}} \ No newline at end of file diff --git a/workflow/config/config.yaml b/workflow/config/config.yaml index 827199c..b5e3e74 100644 --- a/workflow/config/config.yaml +++ b/workflow/config/config.yaml @@ -22,6 +22,8 @@ shard: false # true → pack chunks into shards (zarr v3; fewer files) channel: 0 # channel to segment level: 0 # pyramid level to segment (0 = full res) tile_shape: "auto" # "auto", or e.g. [16, 1024, 1024] (zyx) +gpu_memory_gb: null # for "auto" on a CPU prepare node: your segment GPU's VRAM +# (e.g. 24, 40, 80). null → 8 GiB guess + a harmless warning overlap: 30 # halo (≈ one object diameter) skip_empty: true # skip background tiles empty_threshold: null # null → Otsu; or a number diff --git a/workflow/scripts/prepare_tiles.py b/workflow/scripts/prepare_tiles.py index c178699..ea49062 100644 --- a/workflow/scripts/prepare_tiles.py +++ b/workflow/scripts/prepare_tiles.py @@ -20,12 +20,17 @@ ts = cfg.get("tile_shape", "auto") if ts == "auto": cp = cfg["cellpose"] + # prepare runs on a CPU node, so the segment GPU's VRAM can't be queried + # here; pass gpu_memory_gb from the config to size tiles for it (avoids the + # "GPU memory query failed" fallback). None => the built-in 8 GiB default. + gpu_gb = cfg.get("gpu_memory_gb") tile_shape = tuple( partial( auto_tile_shape_cellpose, do_3D=cp.get("do_3D", False), use_gpu=cp.get("gpu", True), diameter=cp.get("diameter"), + gpu_memory=int(gpu_gb * 1024**3) if gpu_gb else None, )(image.shape, image.dtype) ) else: