Inference: stage-aware retries (run manifest, supersede-on-retry) + visualization suite

README.md

@@ -702,6 +702,9 @@ usage: inference [-h] [--data-path DATA_PATH] [--model-path MODEL_PATH]
                  [--overlap-search | --no-overlap-search]
                  [--overlap-fraction OVERLAP_FRACTION]
                  [--preprocess-output-dir PREPROCESS_OUTPUT_DIR]
+                 [--inference-viz | --no-inference-viz]
+                 [--stamp-gallery-top-k STAMP_GALLERY_TOP_K]
+                 [--max-candidate-plots MAX_CANDIDATE_PLOTS]
                  [--max-retries MAX_RETRIES] [--retry-delay RETRY_DELAY]
                  [--save-tag SAVE_TAG]
 
@@ -823,6 +826,23 @@ options:
                         from existing .npy files, while a new tag starts
                         clean. Pass an old run's directory explicitly to reuse
                         its preprocessing (shared across CSVs)
+  --inference-viz, --no-inference-viz
+                        Render the inference visualization suite (energy
+                        detection distributions, hit spectrum, bandpass
+                        overlay, stamp/candidate galleries, confidence
+                        distribution, latent projection, summary card) at the
+                        end of a CSV inference run, saved under
+                        plots/inference/{save_tag}/ and uploaded to Slack
+                        (default: enabled). Pass --no-inference-viz to
+                        disable.
+  --stamp-gallery-top-k STAMP_GALLERY_TOP_K
+                        Number of top-statistic stamps shown in the stamp
+                        gallery figure, each as a 6-observation waterfall grid
+                        (default: 12)
+  --max-candidate-plots MAX_CANDIDATE_PLOTS
+                        Maximum number of per-candidate figures rendered per
+                        run, highest confidence first (default: 50; the
+                        candidate gallery is unaffected)
   --max-retries MAX_RETRIES
                         Maximum number of retry attempts for inference
                         (including preprocessing) on failure

src/aetherscan/cli.py

@@ -823,6 +823,26 @@ def _add_inference_flags_to(parser):
         default=None,
         help="Directory for per-cadence .npy outputs from preprocessing. Default: a per-CSV, tag-scoped directory {data_path}/inference/preprocessed/<csv_stem>_<save_tag>/ — retrying with the same tag resumes from existing .npy files, while a new tag starts clean. Pass an old run's directory explicitly to reuse its preprocessing (shared across CSVs)",
     )
+
+    # Visualization suite
+    parser.add_argument(
+        "--inference-viz",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        help="Render the inference visualization suite (energy detection distributions, hit spectrum, bandpass overlay, stamp/candidate galleries, confidence distribution, latent projection, summary card) at the end of a CSV inference run, saved under plots/inference/{save_tag}/ and uploaded to Slack (default: enabled). Pass --no-inference-viz to disable.",
+    )
+    parser.add_argument(
+        "--stamp-gallery-top-k",
+        type=int,
+        default=None,
+        help="Number of top-statistic stamps shown in the stamp gallery figure, each as a 6-observation waterfall grid (default: 12)",
+    )
+    parser.add_argument(
+        "--max-candidate-plots",
+        type=int,
+        default=None,
+        help="Maximum number of per-candidate figures rendered per run, highest confidence first (default: 50; the candidate gallery is unaffected)",
+    )
     parser.add_argument(
         "--max-retries",
         type=int,
@@ -864,6 +884,12 @@ def apply_saved_config(config_path: str) -> None:
     forward-compat with newer/older saved configs. Top-level scalar entries
     (`data_path`, `model_path`, `output_path`) are applied directly.
 
+    The `checkpoint` section is skipped entirely: a saved *training* config's
+    checkpoint fields (most damagingly `save_tag`) are never what an inference run
+    wants — layering them would make this run masquerade under the training run's
+    tag, corrupting DB provenance and output paths. The CLI `--save-tag` (or the
+    default import-time timestamp) stays authoritative.
+
     Raises `ValueError` if the file is missing or malformed — caught by main.py's
     wrapper alongside `validate_args` failures.
     """
@@ -880,6 +906,10 @@ def apply_saved_config(config_path: str) -> None:
         raise ValueError("get_config() returned None")
 
     for key, value in saved.items():
+        if key == "checkpoint":
+            # Never layer a saved training run's checkpoint section (save_tag/load_tag/
+            # load_dir/start_round) under CLI flags — see docstring.
+            continue
         target = getattr(config, key, None)
         if target is None:
             continue
@@ -1152,6 +1182,17 @@ def apply_args_to_config(args: argparse.Namespace) -> None:
         config.inference.overlap_fraction = args.overlap_fraction
     if hasattr(args, "preprocess_output_dir") and args.preprocess_output_dir is not None:
         config.inference.preprocess_output_dir = args.preprocess_output_dir
+
+    # Visualization suite
+    # inference_viz uses argparse.BooleanOptionalAction with default=None so that the CLI
+    # can express "leave the config default" (omit), "force on" (--inference-viz), and
+    # "force off" (--no-inference-viz)
+    if hasattr(args, "inference_viz") and args.inference_viz is not None:
+        config.inference.inference_viz_enabled = args.inference_viz
+    if hasattr(args, "stamp_gallery_top_k") and args.stamp_gallery_top_k is not None:
+        config.inference.stamp_gallery_top_k = args.stamp_gallery_top_k
+    if hasattr(args, "max_candidate_plots") and args.max_candidate_plots is not None:
+        config.inference.max_candidate_plots = args.max_candidate_plots
     if (
         hasattr(args, "max_retries")
         and args.max_retries is not None
@@ -2021,6 +2062,34 @@ def collect_validation_errors(
                 )
             )
 
+        # Visualization suite
+        stamp_gallery_top_k = _resolve(
+            args, "stamp_gallery_top_k", config.inference.stamp_gallery_top_k
+        )
+        if stamp_gallery_top_k is not None and stamp_gallery_top_k < 1:
+            errors.append(
+                ValidationError(
+                    field="inference.stamp_gallery_top_k",
+                    current=stamp_gallery_top_k,
+                    message=f"--stamp-gallery-top-k must be >= 1, got {stamp_gallery_top_k}",
+                    fix_kind="clamp_low",
+                    min_val=1,
+                )
+            )
+        max_candidate_plots = _resolve(
+            args, "max_candidate_plots", config.inference.max_candidate_plots
+        )
+        if max_candidate_plots is not None and max_candidate_plots < 0:
+            errors.append(
+                ValidationError(
+                    field="inference.max_candidate_plots",
+                    current=max_candidate_plots,
+                    message=f"--max-candidate-plots must be >= 0, got {max_candidate_plots}",
+                    fix_kind="clamp_low",
+                    min_val=0,
+                )
+            )
+
         # Retries (inference-scoped)
         mr = _resolve(args, "max_retries", config.inference.max_retries)
         if mr is not None and mr < 0:

src/aetherscan/config.py

@@ -341,6 +341,16 @@ class InferenceConfig:
     # explicitly to share/reuse one directory across runs and CSVs.
     preprocess_output_dir: str | None = None
 
+    # Visualization suite (aetherscan.inference_viz): rendered at the end of a streaming
+    # CSV inference run, saved under {output_path}/plots/inference/{save_tag}/ and uploaded
+    # to Slack. Every figure is individually exception-guarded — a plot bug can never kill
+    # a science run.
+    inference_viz_enabled: bool = True
+    # Number of top-statistic stamps shown in the stamp gallery (6-obs waterfall grids).
+    stamp_gallery_top_k: int = 12
+    # Cap on per-candidate figures (candidate_{i}_{tag}.png), highest confidence first.
+    max_candidate_plots: int = 50
+
     # NOTE: come back to this later (is this implemented correctly?)
     # Fault tolerance
     max_retries: int = 3
@@ -616,6 +626,9 @@ def to_dict(self) -> dict:
                 "discard_side_channels": self.inference.discard_side_channels,
                 "side_channel_count": self.inference.side_channel_count,
                 "preprocess_output_dir": self.inference.preprocess_output_dir,
+                "inference_viz_enabled": self.inference.inference_viz_enabled,
+                "stamp_gallery_top_k": self.inference.stamp_gallery_top_k,
+                "max_candidate_plots": self.inference.max_candidate_plots,
                 "max_retries": self.inference.max_retries,
                 "retry_delay": self.inference.retry_delay,
             },