diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index 6a404b2..f5b2a6e 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -537,3 +537,41 @@ by `TF_CPP_MIN_LOG_LEVEL`, so it can't be suppressed without redirecting stderr. **Won't fix.** Expected LLVM/PTX version-skew behavior. See also [`docs/GPU_RUNTIME_GUIDE.md`](docs/GPU_RUNTIME_GUIDE.md). + +--- + +## 16. Container-Build pip Resolver Warning (pydot/pyparsing) + +### Symptom + +During `singularity/apptainer build` of `aetherscan.def`, while `%post` runs +`pip install -r requirements-container.txt`, the build log prints: + +``` +ERROR: pip's dependency resolver does not currently take into account all the packages +that are installed. This behaviour is the source of the following dependency conflicts. +pydot 3.0.4 requires pyparsing>=3.0.9, but you have pyparsing 2.4.7 which is incompatible. +``` + +### Cause + +The conflict is **pre-existing inside the NGC `tensorflow:25.02-tf2-py3` base image**, +which ships `pydot 3.0.4` alongside `pyparsing 2.4.7`. Installing our extras makes pip's +resolver re-inspect the environment and report the already-broken pair; nothing in +`requirements-container.txt` installs or touches either package. + +### Impact + +**None.** Aetherscan never imports `pydot` (it is only used by +`keras.utils.plot_model`-style graph plotting, which we don't call). The build completes +and the image is fully functional. + +### Workaround + +None needed; safe to ignore. Do not "fix" by upgrading `pyparsing` in the base image — +that risks disturbing NGC-pinned packages. NGC 25.02 is the final TF container release, +so this won't change upstream. + +### Status + +**Won't fix.** Pre-existing conflict in the upstream NGC base image; documented here. diff --git a/README.md b/README.md index 4a41083..3ebd659 100644 --- a/README.md +++ b/README.md @@ -109,15 +109,19 @@ Aetherscan reads secrets and path overrides from a `.env` file at the repo root. ```ini # .env example -# If none specified, Slack integration is automatically disabled -SLACK_BOT_TOKEN=your-slack-bot-token -SLACK_CHANNEL=your-slack-channel - # If none specified, defaults to /datax/scratch/zachy/{data|models|outputs}/aetherscan # Note, CLI flags (--data-path, --model-path, --output-path) override these AETHERSCAN_DATA_PATH=/path/to/data AETHERSCAN_MODEL_PATH=/path/to/models AETHERSCAN_OUTPUT_PATH=/path/to/outputs + +# Optional: comma-separated extra host paths for run_container.sh to bind 1:1, for +# data outside the standard dirs (e.g. parent dir with raw .h5 files for inference) +AETHERSCAN_EXTRA_BINDS=/extra/host/paths + +# If none specified, Slack integration is automatically disabled +SLACK_BOT_TOKEN=your-slack-bot-token +SLACK_CHANNEL=your-slack-channel ``` > [!TIP] @@ -320,8 +324,9 @@ PYTHONPATH=src python -m aetherscan.main inference \ **Inference from raw `.h5` files (invokes energy detection preprocessing)** ```bash -# Container -./utils/run_container.sh python -m aetherscan.main inference \ +# Container — if the raw .h5 paths in the CSV live outside the standard bind +# mounts (e.g. under /datag), then we bind them via AETHERSCAN_EXTRA_BINDS +AETHERSCAN_EXTRA_BINDS=/datag ./utils/run_container.sh python -m aetherscan.main inference \ --inference-files complete_cadences_catalog.csv \ --encoder-path /path/to/vae_encoder.keras \ --rf-path /path/to/random_forest.joblib \ diff --git a/src/aetherscan/monitor/monitor.py b/src/aetherscan/monitor/monitor.py index 6a34b35..0f05a0e 100644 --- a/src/aetherscan/monitor/monitor.py +++ b/src/aetherscan/monitor/monitor.py @@ -12,6 +12,7 @@ import gc import json import logging +import math import os import subprocess import threading @@ -22,6 +23,7 @@ import numpy as np import psutil import tensorflow as tf +from matplotlib.lines import Line2D matplotlib.use("Agg") # Non-interactive backend for headless environments @@ -582,13 +584,24 @@ def _save_plot(self): ax_gpu.set_ylim(0, 100) ax_gpu_mem.set_ylim(0, 100) - # Combine legends + # Combine legends, grouped by metric. Matplotlib fills legends + # column-major, so pad each metric's entries with invisible + # handles up to a whole number of columns — usage entries then + # occupy their own column(s) and memory entries always start a + # fresh column (e.g. 5 GPUs -> columns of 3,2 | 3,2). lines1, labels1 = ax_gpu.get_legend_handles_labels() lines2, labels2 = ax_gpu_mem.get_legend_handles_labels() + cols_per_metric = math.ceil(len(gpu_data) / 3) # cap the legend at 3 rows + nrows = math.ceil(len(gpu_data) / cols_per_metric) + slots = nrows * cols_per_metric + lines1 += [Line2D([], [], alpha=0)] * (slots - len(lines1)) + labels1 += [""] * (slots - len(labels1)) + lines2 += [Line2D([], [], alpha=0)] * (slots - len(lines2)) + labels2 += [""] * (slots - len(labels2)) ax_gpu.legend( lines1 + lines2, labels1 + labels2, - ncol=min(4, len(gpu_data) * 2), + ncol=2 * cols_per_metric, fontsize=8, loc="upper right", ) diff --git a/utils/run_container.sh b/utils/run_container.sh index 0b20e16..63e5fa9 100755 --- a/utils/run_container.sh +++ b/utils/run_container.sh @@ -24,9 +24,16 @@ # (default: /datax/scratch/zachy/models/aetherscan) # AETHERSCAN_OUTPUT_PATH Host outputs dir, bound 1:1 # (default: /datax/scratch/zachy/outputs/aetherscan) +# AETHERSCAN_EXTRA_BINDS Comma-separated extra host paths, each bound 1:1 +# and appended to the standard bind list, e.g. +# AETHERSCAN_EXTRA_BINDS=/datag for inference +# (default: none) # SLACK_BOT_TOKEN Slack bot token, forwarded into the container # SLACK_CHANNEL Slack channel, forwarded into the container # +# Note, the runtime's native SINGULARITY_BIND / APPTAINER_BIND env vars still pass +# through untouched and are additive with the binds set up from AETHERSCAN_EXTRA_BINDS. +# # /.env is auto-loaded if present, so secrets set by "source .env" in the # user's shell survive the trip into this child process. Anything already in our # exec env (inline VAR=val invocation, real exports) takes precedence over the @@ -90,11 +97,25 @@ OUTPUT_PATH=${AETHERSCAN_OUTPUT_PATH:-/datax/scratch/zachy/outputs/aetherscan} # Each of the three data dirs gets a 1:1 bind (host path == container path) so # absolute paths persisted in the DB / config snapshots stay valid across both # host and container processes. +BIND_ARGS=( + --bind "$REPO:/workspace/aetherscan" + --bind "$DATA_PATH:$DATA_PATH" + --bind "$MODEL_PATH:$MODEL_PATH" + --bind "$OUTPUT_PATH:$OUTPUT_PATH" +) + +# Extra host paths (comma-separated), each bound 1:1, for data that lives +# outside the standard dirs (e.g. raw .h5 files in /datag during inference). +if [[ -n ${AETHERSCAN_EXTRA_BINDS:-} ]]; then + IFS=',' read -ra EXTRA_BINDS <<<"$AETHERSCAN_EXTRA_BINDS" + for extra_path in "${EXTRA_BINDS[@]}"; do + [[ -z $extra_path ]] && continue + BIND_ARGS+=(--bind "$extra_path:$extra_path") + done +fi + exec "$RUNTIME" exec --nv \ - --bind "$REPO:/workspace/aetherscan" \ - --bind "$DATA_PATH:$DATA_PATH" \ - --bind "$MODEL_PATH:$MODEL_PATH" \ - --bind "$OUTPUT_PATH:$OUTPUT_PATH" \ + "${BIND_ARGS[@]}" \ --pwd /workspace/aetherscan \ --env PYTHONPATH=/workspace/aetherscan/src \ --env AETHERSCAN_DATA_PATH="$DATA_PATH" \