Publish verified REDD T0 leaderboard

.dockerignore

@@ -0,0 +1,13 @@
+.git
+.github
+.venv
+__pycache__
+*.pyc
+.pytest_cache
+.ruff_cache
+data
+results
+slides
+static
+*.h5
+*.hdf5

.github/workflows/benchmark-ci.yml

@@ -0,0 +1,42 @@
+name: Benchmark runner CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  cpu-smoke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.11.28"
+          enable-cache: true
+      - name: Create environment
+        run: uv venv --python 3.11
+      - name: Install test dependencies
+        run: uv pip install -e ".[dev]"
+      - name: Lint
+        run: uv run ruff check src tests
+      - name: Test
+        run: uv run pytest
+      - name: Verify generated leaderboard artifacts
+        run: |
+          uv run nilmbench leaderboard --results results/published --output /tmp/leaderboard.json --csv /tmp/leaderboard.csv
+          diff -u leaderboard.json /tmp/leaderboard.json
+          diff -u leaderboard.csv /tmp/leaderboard.csv
+      - name: Validate CPU smoke specification
+        run: >-
+          uv run nilmbench run
+          --task corrected-t1-redd
+          --model PatchTST
+          --appliance fridge
+          --epochs 1
+          --max-samples 1024
+          --device cpu
+          --dry-run

.github/workflows/publish-images.yml

@@ -0,0 +1,50 @@
+name: Publish benchmark images
+
+on:
+  workflow_dispatch:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: read
+  packages: write
+  id-token: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [cpu, cuda]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/metadata-action@v5
+        id: meta
+        with:
+          images: ghcr.io/${{ github.repository_owner }}/nilmbench
+          tags: |
+            type=raw,value=${{ matrix.target }}
+            type=sha,prefix=${{ matrix.target }}-
+            type=semver,pattern={{version}},suffix=-${{ matrix.target }}
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: docker/Dockerfile.${{ matrix.target }}
+          build-contexts: |
+            contrib=https://github.com/nilmtk/nilmtk-contrib.git#825740b39bcd44b3f4bfaf146f4c0d944843b131
+          build-args: |
+            SOURCE_REVISION=${{ github.sha }}
+            NILMTK_CONTRIB_REVISION=825740b39bcd44b3f4bfaf146f4c0d944843b131
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=${{ matrix.target }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.target }}
+          provenance: mode=max
+          sbom: true

.gitignore

@@ -1,3 +1,11 @@
 __pycache__/
 *.pyc
 .DS_Store
+.pytest_cache/
+.ruff_cache/
+.venv/
+dist/
+results/*
+!results/published/
+!results/published/**
+data/

README.md

@@ -1,7 +1,6 @@
-# NILMBench2026 — project website
+# NILMbench
 
-Source for the **[NILMBench2026](https://sustainability-lab.github.io/nilmbench/)** project page —
-*A Benchmark for Energy Disaggregation* (BuildSys '26, **Best Paper Candidate**).
+Reproducible runner and project website for **[NILMBench2026](https://sustainability-lab.github.io/nilmbench/)** — *A Benchmark for Energy Disaggregation* (BuildSys '26, **Best Paper Candidate**).
 
 > One aggregate power signal in. Appliance-level estimates out. We benchmark **16 NILM models**
 > across **3 datasets** and **2 resolutions** — on accuracy, efficiency, and generalization —
@@ -16,29 +15,188 @@ Source for the **[NILMBench2026](https://sustainability-lab.github.io/nilmbench/
 - 💻 **Code (modernized NILMTK):** https://github.com/nilmtk/nilmtk-contrib
 - 🌐 **Website:** https://sustainability-lab.github.io/nilmbench/
 
-## About this repo
+## What is reproducible here
 
-A single self-contained `index.html` (no build step) deployed to GitHub Pages via GitHub Actions.
+The benchmark is now an installable command-line application rather than a set of order-dependent notebooks. It provides:
 
+- typed TOML definitions for the paper's T1/T2/T3 protocols;
+- explicit `historical-*` profiles reconstructed from the executed notebooks;
+- strict corrected profiles that reject silently truncated data windows;
+- canonical appliance resolution through NILM Metadata, with exact resolved
+  appliance/meter identities and shared-circuit contamination in every result;
+- ordered active/apparent power preferences resolved per dataset window;
+- explicit legacy and paper appliance-threshold policies for F1;
+- deterministic seeds and persistent, resumable Optuna SQLite studies;
+- deterministic Torch algorithms with an explicit cuBLAS workspace policy;
+- structured JSON and CSV results with source, dataset, runtime, parameter/FLOP, and container provenance;
+- separate CPU-smoke and CUDA-benchmark containers.
+- a NILMTK Mean sanity-check baseline alongside the contrib architectures.
+
+The REDD, UK-DALE, and REFIT data are not redistributed. The runner expects user-provided NILMTK HDF5 conversions and verifies them against the recorded file sizes and SHA-256 digests. The exact protocol discrepancies recovered from the old notebooks are documented in the [protocol audit](https://github.com/sustainability-lab/nilmbench/blob/main/docs/protocol-audit.md).
+
+## Install for development
+
+Use Python 3.11, which is the version currently supported by nilmtk-contrib:
+
+```bash
+git clone https://github.com/nilmtk/nilmtk-contrib.git
+git clone https://github.com/sustainability-lab/nilmbench.git
+cd nilmbench
+
+uv venv --python 3.11
+source .venv/bin/activate
+UV_TORCH_BACKEND=cpu uv pip install -e "../nilmtk-contrib[torch]" -e ".[runtime,dev]"
+nilmbench list
+pytest
 ```
-index.html              # the entire site (HTML + CSS + JS inline)
-static/images/          # paper figures (real UK-DALE / REDD / REFIT predictions)
-.github/workflows/      # GitHub Pages deploy workflow
+
+For a non-editable install using the exact PatchTST revision pinned by this
+repository:
+
+```bash
+uv venv --python 3.11
+source .venv/bin/activate
+UV_TORCH_BACKEND=cpu uv pip install ".[benchmark]"
 ```
 
-- **Theme:** dark/light toggle, an "energy / power-spectrum" palette.
-- **Hero:** animated SVG of the disaggregation task (aggregate mains → fridge / washing machine / microwave / kettle).
-- **Results explorer:** the paper's tables, heat-mapped (green = good, red = bad), with best/second highlighting.
-- **Extend & Compete:** how to add a model / metric, and the leaderboard vision — *NILM's ImageNet*.
+## Data
 
-## Develop locally
+Place the three converted datasets in one directory with these names:
+
+```text
+data/
+├── redd.h5
+├── refit.h5
+└── ukdale.h5
+```
 
-Just open `index.html` in a browser, or serve the folder:
+Alternatively, set `NILMBENCH_REDD`, `NILMBENCH_REFIT`, and `NILMBENCH_UKDALE` to their full paths. Check the files before a long run:
 
 ```bash
-python3 -m http.server 8000   # then visit http://localhost:8000
+nilmbench doctor --checksums
+nilmbench validate --task corrected-t1-redd --check-data --max-samples 64
 ```
 
+The dataset mounts in `compose.yaml` are read-only. Results are written to a separate `/results` mount.
+New runs land under `results/candidates`; publication is an explicit review/copy
+into `results/published`, never an automatic side effect of training.
+
+## CPU smoke and CUDA benchmark
+
+Container builds take nilmtk-contrib as a named BuildKit context. The default Compose configuration expects the two repositories to be sibling directories; set `NILMTK_CONTRIB_CONTEXT` to override that location.
+
+Published images pin their nilmtk-contrib build context to the exact reviewed integration commit rather than a moving branch. The current dependency and image pin is [`825740b39bcd44b3f4bfaf146f4c0d944843b131`](https://github.com/nilmtk/nilmtk-contrib/commit/825740b39bcd44b3f4bfaf146f4c0d944843b131). Update that pin deliberately when a reviewed model release is adopted. Both image variants synchronize their runtime, NILMTK, and NILM Metadata dependencies from the checked-in `uv.lock` with `--frozen`; the project and named-context contrib source are then installed with `--no-deps`. The CPU-only Torch wheel is installed with `--no-deps` after its common Python dependencies have been synchronized from the same lock, avoiding the CUDA wheel stack in the CPU image.
+
+Model contributions and benchmark-image releases have separate cadences. A
+model can merge after its contrib contract, CPU, and targeted CUDA checks pass;
+it does not trigger a public image by itself. NILMbench periodically batches
+eligible contrib changes, advances its single immutable contrib pin, builds the
+matching `-cpu` and `-cuda` variants once, and runs the real-data matrix against
+the candidate CUDA digest. The versioned images and leaderboard update are
+promoted together only after that matrix passes. Development builds may use a
+local contrib checkout, but their results cannot become verified leaderboard
+rows unless the source revisions and immutable container digest are recorded.
+For an official run, the orchestrator obtains the candidate image's registry or
+local content digest and supplies it as `NILMBENCH_IMAGE_DIGEST`. The exact
+runner commit, contrib commit, image name/digest, and hardware must then be
+reviewed into `configs/runtimes.toml`; self-asserted environment variables alone
+cannot produce a verified row.
+
+For local Compose builds, pass the two source revisions into the OCI labels and
+runtime result metadata:
+
+```bash
+export NILMBENCH_GIT_SHA="$(git rev-parse HEAD)"
+export NILMTK_CONTRIB_GIT_SHA="$(git -C ../nilmtk-contrib rev-parse HEAD)"
+```
+
+The CPU path is deliberately small and is suitable for CI or a laptop:
+
+```bash
+export NILMBENCH_DATA_DIR=/absolute/path/to/data
+export NILMBENCH_RESULTS_DIR=/absolute/path/to/results
+docker compose run --rm cpu-smoke
+```
+
+The real benchmark path uses the pinned PyTorch 2.6.0 / CUDA 12.4 image and all visible NVIDIA GPUs:
+
+```bash
+docker compose --profile cuda run --rm cuda-benchmark
+```
+
+To run one inspectable A100 smoke before spending on 20 trials:
+
+```bash
+docker compose --profile cuda run --rm cuda-benchmark \
+  run --task corrected-t1-redd --model PatchTST --appliance fridge \
+  --seed 42 --epochs 1 --max-samples 1024 --device cuda --results /results/candidates
+```
+
+For the full paper matrix, repeat each task at 60 and 900 seconds for evaluation seeds 10, 20, and 42. Optuna studies live under `results/optuna/` and resume to the requested total trial count. Model selection uses the fixed tuning seed 42 once per scientific study, then freezes the selected parameters for every evaluation seed; independently tuned seeds are never pooled into one score. Trials are scored only on a blocked 20% holdout from each `task.train` window; `task.test` is loaded only after model selection, during final benchmark evaluation. Fixed epoch and sequence-length overrides apply during every trial. Study identity covers the runner and nilmtk-contrib revisions, container digest, device/runtime, source dataset identity, full task protocol, appliance subset, resolution, and smoke overrides, so an incompatible environment creates a new study instead of resuming an old one. Persistent HPO also fails closed for unknown or dirty source/container provenance.
+
+## Historical versus corrected protocols
+
+The recovered notebooks requested REDD building 1 from 1–30 April 2011, although this converted file starts on 18 April. `historical-t1-redd` retains that request and emits a coverage warning. Two other historical definitions request unavailable appliance/building pairs; validation reports those explicitly. Historical tasks also retain the legacy joint appliance alignment and fixed 10 W F1 threshold.
+
+The eight `corrected-*` tasks form a real-data-validated T1/T2/T3 matrix. They enforce dataset/meter coverage, use the paper's appliance-specific F1 thresholds, select active or apparent power from an explicit preference list, and align each appliance independently. NILM Metadata remains the source of truth for taxonomy and synonyms. Shared physical circuits are warned and recorded; a future clean-meter profile can reject them. Corrected profiles are the basis for new leaderboard claims, while historical profiles are retained for forensic reproduction.
+
+## Repository layout
+
+```text
+configs/                 # dataset manifests and T1/T2/T3 task definitions
+src/nilmbench/           # CLI, data loading, registry, runner, provenance
+docker/                  # separate CPU-only and CUDA 12.4 images
+tests/                   # dependency-light runner/config tests
+index.html               # self-contained project website
+static/images/           # paper figures
+```
+
+The website still has no build step:
+
+```bash
+python3 -m http.server 8000
+```
+
+## Living leaderboard
+
+The live table is generated from immutable `result.json` bundles under
+`results/published`; its numbers are never edited into the website. Regenerate
+both reviewable artifacts after adding a result bundle:
+
+```bash
+nilmbench leaderboard --results results/published \
+  --output leaderboard.json --csv leaderboard.csv
+git diff -- leaderboard.json leaderboard.csv
+```
+
+Those immutable, hashable JSON bundles are the scientific source of truth.
+Each completed HPO trial also has a write-once JSON audit record under its
+`results/optuna/<study>/trials/` directory, and those records are embedded in
+the final result bundle. SQLite is reserved for mutable coordination such as resumable Optuna studies;
+an optional SQLite query index may be generated later, but it must always be
+rebuildable from the result bundles and never replace them.
+The CSV is written before the JSON commit marker; the JSON records the CSV's
+SHA-256 so consumers can reject a partially updated artifact pair.
+
+Every aggregate is separated by task/config revision, model revision, runner
+revision, container digest, hardware, resolution, appliance, target-data access,
+smoke/full scope, and a digest of every protocol override. Context length,
+epochs, and sample limits remain visible in the generated table.
+The website's rank groups use a separate comparison-protocol digest covering
+effective context length and epochs, tuning method/trial budget/selection
+protocol, and the immutable runner, contrib, container, and hardware tuple. It
+intentionally excludes model-specific selected parameters and the study digest,
+so different models remain comparable and evaluation seeds from one frozen
+study can still aggregate. A corrected
+full run becomes `full-verified`, and a smoke run becomes `smoke-verified`, only
+after the required seeds 10, 20, and 42 pass source, container, and dataset
+provenance checks. Incomplete clean smoke matrices are labelled `smoke-partial`.
+CI regenerates the artifacts and rejects hand-edited or stale tables.
+
+## Add a model
+
+Models belong in nilmtk-contrib. Once a model has its own tests and lazy export there, add a small entry and search space in `src/nilmbench/registry.py`; task/data logic should not be copied into model notebooks. PatchTST is the first model using this route.
+
 ## Cite
 
 ```bibtex
@@ -52,4 +210,4 @@ python3 -m http.server 8000   # then visit http://localhost:8000
   publisher = {ACM},
   address   = {Banff, AB, Canada}
 }
-```
+```

compose.yaml

@@ -0,0 +1,61 @@
+services:
+  cpu-smoke:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.cpu
+      additional_contexts:
+        contrib: ${NILMTK_CONTRIB_CONTEXT:-../nilmtk-contrib}
+      args:
+        SOURCE_REVISION: ${NILMBENCH_GIT_SHA:-unknown}
+        NILMTK_CONTRIB_REVISION: ${NILMTK_CONTRIB_GIT_SHA:-unknown}
+    image: nilmbench:cpu
+    command:
+      - run
+      - --task
+      - corrected-t1-redd
+      - --model
+      - PatchTST
+      - --appliance
+      - fridge
+      - --epochs
+      - "1"
+      - --max-samples
+      - "1024"
+      - --device
+      - cpu
+      - --results
+      - /results/candidates
+    volumes:
+      - ${NILMBENCH_DATA_DIR:-./data}:/data:ro
+      - ${NILMBENCH_RESULTS_DIR:-./results}:/results
+
+  cuda-benchmark:
+    profiles: ["cuda"]
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.cuda
+      additional_contexts:
+        contrib: ${NILMTK_CONTRIB_CONTEXT:-../nilmtk-contrib}
+      args:
+        SOURCE_REVISION: ${NILMBENCH_GIT_SHA:-unknown}
+        NILMTK_CONTRIB_REVISION: ${NILMTK_CONTRIB_GIT_SHA:-unknown}
+    image: nilmbench:cuda
+    command:
+      - run
+      - --task
+      - corrected-t1-redd
+      - --model
+      - PatchTST
+      - --seed
+      - "42"
+      - --trials
+      - "20"
+      - --device
+      - cuda
+      - --results
+      - /results/candidates
+    gpus: all
+    shm_size: 8gb
+    volumes:
+      - ${NILMBENCH_DATA_DIR:-./data}:/data:ro
+      - ${NILMBENCH_RESULTS_DIR:-./results}:/results