From 28cf0558d889e07a781aeb1c28e8cb2a850a72f7 Mon Sep 17 00:00:00 2001 From: zachtheyek Date: Sun, 12 Jul 2026 04:39:05 +0800 Subject: [PATCH 1/4] Document NGC pip resolver warning as KNOWN_ISSUES entry 16 The pydot/pyparsing conflict reported during singularity/apptainer builds is pre-existing in the NGC tensorflow:25.02-tf2-py3 base image and harmless (Aetherscan never imports pydot); document it so future builders don't chase it. Co-Authored-By: Claude Fable 5 --- KNOWN_ISSUES.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index 6a404b2..f5b2a6e 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -537,3 +537,41 @@ by `TF_CPP_MIN_LOG_LEVEL`, so it can't be suppressed without redirecting stderr. **Won't fix.** Expected LLVM/PTX version-skew behavior. See also [`docs/GPU_RUNTIME_GUIDE.md`](docs/GPU_RUNTIME_GUIDE.md). + +--- + +## 16. Container-Build pip Resolver Warning (pydot/pyparsing) + +### Symptom + +During `singularity/apptainer build` of `aetherscan.def`, while `%post` runs +`pip install -r requirements-container.txt`, the build log prints: + +``` +ERROR: pip's dependency resolver does not currently take into account all the packages +that are installed. This behaviour is the source of the following dependency conflicts. +pydot 3.0.4 requires pyparsing>=3.0.9, but you have pyparsing 2.4.7 which is incompatible. +``` + +### Cause + +The conflict is **pre-existing inside the NGC `tensorflow:25.02-tf2-py3` base image**, +which ships `pydot 3.0.4` alongside `pyparsing 2.4.7`. Installing our extras makes pip's +resolver re-inspect the environment and report the already-broken pair; nothing in +`requirements-container.txt` installs or touches either package. + +### Impact + +**None.** Aetherscan never imports `pydot` (it is only used by +`keras.utils.plot_model`-style graph plotting, which we don't call). The build completes +and the image is fully functional. + +### Workaround + +None needed; safe to ignore. Do not "fix" by upgrading `pyparsing` in the base image — +that risks disturbing NGC-pinned packages. NGC 25.02 is the final TF container release, +so this won't change upstream. + +### Status + +**Won't fix.** Pre-existing conflict in the upstream NGC base image; documented here. From c7059693b4947f2940c1b5a9f59598f590ee91e8 Mon Sep 17 00:00:00 2001 From: zachtheyek Date: Sun, 12 Jul 2026 04:39:21 +0800 Subject: [PATCH 2/4] Group GPU legend entries by metric in resource utilization plot Matplotlib fills legends column-major, so the combined usage+memory legend with ncol=min(4, n_gpus*2) interleaved metrics across columns (5 GPUs -> 10 entries in 4 columns = 3/3/2/2, with memory entries continuing mid-column under usage). Pad each metric's handle/label lists to a whole number of columns with invisible handles so usage occupies its own column(s) and memory always starts a fresh column (5 GPUs -> columns of 3,2 | 3,2), capping the legend at 3 rows. Co-Authored-By: Claude Fable 5 --- src/aetherscan/monitor/monitor.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/aetherscan/monitor/monitor.py b/src/aetherscan/monitor/monitor.py index 6a34b35..0f05a0e 100644 --- a/src/aetherscan/monitor/monitor.py +++ b/src/aetherscan/monitor/monitor.py @@ -12,6 +12,7 @@ import gc import json import logging +import math import os import subprocess import threading @@ -22,6 +23,7 @@ import numpy as np import psutil import tensorflow as tf +from matplotlib.lines import Line2D matplotlib.use("Agg") # Non-interactive backend for headless environments @@ -582,13 +584,24 @@ def _save_plot(self): ax_gpu.set_ylim(0, 100) ax_gpu_mem.set_ylim(0, 100) - # Combine legends + # Combine legends, grouped by metric. Matplotlib fills legends + # column-major, so pad each metric's entries with invisible + # handles up to a whole number of columns — usage entries then + # occupy their own column(s) and memory entries always start a + # fresh column (e.g. 5 GPUs -> columns of 3,2 | 3,2). lines1, labels1 = ax_gpu.get_legend_handles_labels() lines2, labels2 = ax_gpu_mem.get_legend_handles_labels() + cols_per_metric = math.ceil(len(gpu_data) / 3) # cap the legend at 3 rows + nrows = math.ceil(len(gpu_data) / cols_per_metric) + slots = nrows * cols_per_metric + lines1 += [Line2D([], [], alpha=0)] * (slots - len(lines1)) + labels1 += [""] * (slots - len(labels1)) + lines2 += [Line2D([], [], alpha=0)] * (slots - len(lines2)) + labels2 += [""] * (slots - len(labels2)) ax_gpu.legend( lines1 + lines2, labels1 + labels2, - ncol=min(4, len(gpu_data) * 2), + ncol=2 * cols_per_metric, fontsize=8, loc="upper right", ) From 3f6baa2143340285e87c5b6972b93d28503021f5 Mon Sep 17 00:00:00 2001 From: zachtheyek Date: Sun, 12 Jul 2026 04:39:22 +0800 Subject: [PATCH 3/4] Add AETHERSCAN_EXTRA_BINDS knob to run_container.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comma-separated host paths, each bound 1:1 and appended to the standard bind list, so data outside the standard dirs (e.g. raw .h5 files under /datag for CSV inference on blpc3) no longer requires the runtime's native SINGULARITY_BIND passthrough — which continues to work and remains additive. Documented in the script header, the README raw-.h5 inference example, and the README .env example block. Co-Authored-By: Claude Fable 5 --- README.md | 9 +++++++-- utils/run_container.sh | 29 +++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4a41083..91e7d42 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,10 @@ SLACK_CHANNEL=your-slack-channel AETHERSCAN_DATA_PATH=/path/to/data AETHERSCAN_MODEL_PATH=/path/to/models AETHERSCAN_OUTPUT_PATH=/path/to/outputs + +# Optional: comma-separated extra host paths for run_container.sh to bind 1:1, +# for data outside the standard dirs (e.g. raw .h5 files for CSV inference) +AETHERSCAN_EXTRA_BINDS=/datag ``` > [!TIP] @@ -320,8 +324,9 @@ PYTHONPATH=src python -m aetherscan.main inference \ **Inference from raw `.h5` files (invokes energy detection preprocessing)** ```bash -# Container -./utils/run_container.sh python -m aetherscan.main inference \ +# Container — the raw .h5 paths in the CSV live outside the standard bind +# mounts (e.g. under /datag), so bind them via AETHERSCAN_EXTRA_BINDS +AETHERSCAN_EXTRA_BINDS=/datag ./utils/run_container.sh python -m aetherscan.main inference \ --inference-files complete_cadences_catalog.csv \ --encoder-path /path/to/vae_encoder.keras \ --rf-path /path/to/random_forest.joblib \ diff --git a/utils/run_container.sh b/utils/run_container.sh index 0b20e16..be535c0 100755 --- a/utils/run_container.sh +++ b/utils/run_container.sh @@ -24,9 +24,16 @@ # (default: /datax/scratch/zachy/models/aetherscan) # AETHERSCAN_OUTPUT_PATH Host outputs dir, bound 1:1 # (default: /datax/scratch/zachy/outputs/aetherscan) +# AETHERSCAN_EXTRA_BINDS Comma-separated extra host paths, each bound 1:1 +# and appended to the standard bind list, e.g. +# AETHERSCAN_EXTRA_BINDS=/datag for CSV inference +# (default: none) # SLACK_BOT_TOKEN Slack bot token, forwarded into the container # SLACK_CHANNEL Slack channel, forwarded into the container # +# The runtime's native SINGULARITY_BIND / APPTAINER_BIND env vars still pass +# through untouched and are additive with the binds set up here. +# # /.env is auto-loaded if present, so secrets set by "source .env" in the # user's shell survive the trip into this child process. Anything already in our # exec env (inline VAR=val invocation, real exports) takes precedence over the @@ -90,11 +97,25 @@ OUTPUT_PATH=${AETHERSCAN_OUTPUT_PATH:-/datax/scratch/zachy/outputs/aetherscan} # Each of the three data dirs gets a 1:1 bind (host path == container path) so # absolute paths persisted in the DB / config snapshots stay valid across both # host and container processes. +BIND_ARGS=( + --bind "$REPO:/workspace/aetherscan" + --bind "$DATA_PATH:$DATA_PATH" + --bind "$MODEL_PATH:$MODEL_PATH" + --bind "$OUTPUT_PATH:$OUTPUT_PATH" +) + +# Extra host paths (comma-separated), each bound 1:1, for data that lives +# outside the standard dirs (e.g. raw .h5 files under /datag for CSV inference). +if [[ -n ${AETHERSCAN_EXTRA_BINDS:-} ]]; then + IFS=',' read -ra EXTRA_BINDS <<<"$AETHERSCAN_EXTRA_BINDS" + for extra_path in "${EXTRA_BINDS[@]}"; do + [[ -z $extra_path ]] && continue + BIND_ARGS+=(--bind "$extra_path:$extra_path") + done +fi + exec "$RUNTIME" exec --nv \ - --bind "$REPO:/workspace/aetherscan" \ - --bind "$DATA_PATH:$DATA_PATH" \ - --bind "$MODEL_PATH:$MODEL_PATH" \ - --bind "$OUTPUT_PATH:$OUTPUT_PATH" \ + "${BIND_ARGS[@]}" \ --pwd /workspace/aetherscan \ --env PYTHONPATH=/workspace/aetherscan/src \ --env AETHERSCAN_DATA_PATH="$DATA_PATH" \ From 3daa974b5e182c9c68f58f38ed82cff848362ef6 Mon Sep 17 00:00:00 2001 From: zachtheyek Date: Mon, 13 Jul 2026 23:19:54 +0800 Subject: [PATCH 4/4] Update comments --- README.md | 18 +++++++++--------- utils/run_container.sh | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 91e7d42..3ebd659 100644 --- a/README.md +++ b/README.md @@ -109,19 +109,19 @@ Aetherscan reads secrets and path overrides from a `.env` file at the repo root. ```ini # .env example -# If none specified, Slack integration is automatically disabled -SLACK_BOT_TOKEN=your-slack-bot-token -SLACK_CHANNEL=your-slack-channel - # If none specified, defaults to /datax/scratch/zachy/{data|models|outputs}/aetherscan # Note, CLI flags (--data-path, --model-path, --output-path) override these AETHERSCAN_DATA_PATH=/path/to/data AETHERSCAN_MODEL_PATH=/path/to/models AETHERSCAN_OUTPUT_PATH=/path/to/outputs -# Optional: comma-separated extra host paths for run_container.sh to bind 1:1, -# for data outside the standard dirs (e.g. raw .h5 files for CSV inference) -AETHERSCAN_EXTRA_BINDS=/datag +# Optional: comma-separated extra host paths for run_container.sh to bind 1:1, for +# data outside the standard dirs (e.g. parent dir with raw .h5 files for inference) +AETHERSCAN_EXTRA_BINDS=/extra/host/paths + +# If none specified, Slack integration is automatically disabled +SLACK_BOT_TOKEN=your-slack-bot-token +SLACK_CHANNEL=your-slack-channel ``` > [!TIP] @@ -324,8 +324,8 @@ PYTHONPATH=src python -m aetherscan.main inference \ **Inference from raw `.h5` files (invokes energy detection preprocessing)** ```bash -# Container — the raw .h5 paths in the CSV live outside the standard bind -# mounts (e.g. under /datag), so bind them via AETHERSCAN_EXTRA_BINDS +# Container — if the raw .h5 paths in the CSV live outside the standard bind +# mounts (e.g. under /datag), then we bind them via AETHERSCAN_EXTRA_BINDS AETHERSCAN_EXTRA_BINDS=/datag ./utils/run_container.sh python -m aetherscan.main inference \ --inference-files complete_cadences_catalog.csv \ --encoder-path /path/to/vae_encoder.keras \ diff --git a/utils/run_container.sh b/utils/run_container.sh index be535c0..63e5fa9 100755 --- a/utils/run_container.sh +++ b/utils/run_container.sh @@ -26,13 +26,13 @@ # (default: /datax/scratch/zachy/outputs/aetherscan) # AETHERSCAN_EXTRA_BINDS Comma-separated extra host paths, each bound 1:1 # and appended to the standard bind list, e.g. -# AETHERSCAN_EXTRA_BINDS=/datag for CSV inference +# AETHERSCAN_EXTRA_BINDS=/datag for inference # (default: none) # SLACK_BOT_TOKEN Slack bot token, forwarded into the container # SLACK_CHANNEL Slack channel, forwarded into the container # -# The runtime's native SINGULARITY_BIND / APPTAINER_BIND env vars still pass -# through untouched and are additive with the binds set up here. +# Note, the runtime's native SINGULARITY_BIND / APPTAINER_BIND env vars still pass +# through untouched and are additive with the binds set up from AETHERSCAN_EXTRA_BINDS. # # /.env is auto-loaded if present, so secrets set by "source .env" in the # user's shell survive the trip into this child process. Anything already in our @@ -105,7 +105,7 @@ BIND_ARGS=( ) # Extra host paths (comma-separated), each bound 1:1, for data that lives -# outside the standard dirs (e.g. raw .h5 files under /datag for CSV inference). +# outside the standard dirs (e.g. raw .h5 files in /datag during inference). if [[ -n ${AETHERSCAN_EXTRA_BINDS:-} ]]; then IFS=',' read -ra EXTRA_BINDS <<<"$AETHERSCAN_EXTRA_BINDS" for extra_path in "${EXTRA_BINDS[@]}"; do