From 8eebd5f3dfd09404e8bb8b327324a26c4cd7ad86 Mon Sep 17 00:00:00 2001 From: Laurent Guerard Date: Wed, 24 Jun 2026 15:39:39 +0200 Subject: [PATCH] =?UTF-8?q?fix(workflow):=20=F0=9F=90=9B=20let=20auto=20ti?= =?UTF-8?q?ling=20target=20the=20segment=20GPU's=20VRAM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tile planning (prepare) runs on a CPU node, so it cannot query the segment GPU and falls back to an 8 GiB guess, logging "GPU memory query failed". Add a gpu_memory_gb config option forwarded to auto_tile_shape_cellpose so tiles are sized for the real GPU (e.g. 24, 40, 80) without a query or warning. null keeps the previous behaviour. Co-Authored-By: Claude Opus 4.8 --- docs/guide/snakemake.md | 6 ++++++ workflow/.snakemake/iocache/latest.pkl | Bin 0 -> 485 bytes .../metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU= | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ== | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ== | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ== | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ== | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ== | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ== | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ== | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ== | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg== | 1 - .../metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg== | 1 - workflow/config/config.yaml | 2 ++ workflow/scripts/prepare_tiles.py | 5 +++++ 15 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 workflow/.snakemake/iocache/latest.pkl delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU= delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ== delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ== delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ== delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ== delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ== delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ== delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ== delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ== delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg== delete mode 100644 workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg== diff --git a/docs/guide/snakemake.md b/docs/guide/snakemake.md index f16ae9e..8ad67bf 100644 --- a/docs/guide/snakemake.md +++ b/docs/guide/snakemake.md @@ -56,6 +56,7 @@ shard: false # true → pack chunks into shards (fewer files) channel: 0 # channel to segment (null = keep all) level: 0 # pyramid level (0 = full resolution) tile_shape: "auto" # "auto", or e.g. [16, 1024, 1024] (zyx) +gpu_memory_gb: null # for "auto" on SLURM: your segment GPU's VRAM overlap: 30 # halo ≈ one object diameter skip_empty: true # skip background tiles empty_threshold: null # null → Otsu @@ -81,6 +82,11 @@ sequential_labels: true # renumber labels to a contiguous 1..N more (faster) jobs; very large 3-D tiles are slow. Keep `do_3D: false` (2-D per slice) if your objects segment fine per slice — it is much faster. + Tile planning runs on a **CPU** node, which cannot see the segment GPU, so + it logs `GPU memory query failed … using 8 GiB default` and sizes tiles for + 8 GiB. Harmless, but to size for the real GPU set `gpu_memory_gb:` to its + VRAM (e.g. `24`, `40`, `80`) — or just set `tile_shape` explicitly. + ## 4. Dry-run (always do this first) Check the plan without running anything: diff --git a/workflow/.snakemake/iocache/latest.pkl b/workflow/.snakemake/iocache/latest.pkl new file mode 100644 index 0000000000000000000000000000000000000000..2923948fa2217906c0734d44ed9a7c6996f09d86 GIT binary patch literal 485 zcma)3O-lnY5UteK53C34rCtT8m)(PZKv5|o)}Ij5kZxj|Y}q8;S`e%U>0!`=Blz3; zN4yA{W@*ubU?3sPJl?$b=3(`1wo(cQwv47duKbN;im-NZeM*_&_=<-UyvHU~iLz30 z|A+n9szE%+N#2c9XJe^o!lNQG(6$P&kv! z7A~65_)|DeVf%N}SN`R5>gJLwETkpMP!rU6!$YpU)jNDa*MXgdIH{MqMZE}CG(=hK z1z3d`ISvVL5T8Bk8K>rXS1Ge{$i1R^=2s-O)CtL{v?K)AIzIaPyzy& the built-in 8 GiB default. + gpu_gb = cfg.get("gpu_memory_gb") tile_shape = tuple( partial( auto_tile_shape_cellpose, do_3D=cp.get("do_3D", False), use_gpu=cp.get("gpu", True), diameter=cp.get("diameter"), + gpu_memory=int(gpu_gb * 1024**3) if gpu_gb else None, )(image.shape, image.dtype) ) else: