From 0e9f818032a561ef2e4a851c3dfddc39bd828689 Mon Sep 17 00:00:00 2001
From: almondsun <maramirezes@outlook.com>
Date: Mon, 13 Jul 2026 01:44:48 -0500
Subject: [PATCH] add sealed test evaluation

---
 AGENTS.md                                     |   9 +-
 README.md                                     |   7 +-
 configs/gptiny_alice_bytebpe512_sealed.yaml   |  39 ++++++
 configs/gptiny_alice_char_sealed.yaml         |  37 +++++
 .../gptiny_peterpan_bytebpe512_sealed.yaml    |  39 ++++++
 configs/gptiny_peterpan_char_sealed.yaml      |  37 +++++
 docs/architecture.md                          |  13 +-
 docs/codex/architecture.md                    |   5 +-
 docs/codex/build-and-test.md                  |  21 ++-
 docs/codex/experiments.md                     |   5 +
 docs/experiments.md                           |   9 +-
 docs/training.md                              |  26 ++++
 experiments/026-sealed-test-evaluation.md     | 130 ++++++++++++++++++
 notes/02-data-and-tokenization.md             |  13 +-
 notes/06-reproducibility.md                   |  34 +++++
 scripts/evaluate_baselines.py                 |  10 +-
 scripts/evaluate_test.py                      |  28 ++++
 scripts/prepare_corpus.py                     |   2 +
 scripts/prepare_data.py                       |  11 +-
 scripts/show_run.py                           |   7 +-
 src/smallm/config.py                          |  11 ++
 src/smallm/data/__init__.py                   |  11 +-
 src/smallm/data/corpus.py                     |  18 ++-
 src/smallm/data/dataset.py                    |  18 +++
 src/smallm/evaluation/__init__.py             |   3 +-
 src/smallm/evaluation/sealed.py               | 105 ++++++++++++++
 src/smallm/training/artifacts.py              |  47 ++++++-
 src/smallm/training/trainer.py                |  14 +-
 tests/test_config.py                          |  10 ++
 tests/test_corpus_preparation.py              |  15 ++
 tests/test_dataset.py                         |  13 +-
 tests/test_sealed_test.py                     |  82 +++++++++++
 tests/test_training_artifacts.py              |  46 +++++++
 33 files changed, 829 insertions(+), 46 deletions(-)
 create mode 100644 configs/gptiny_alice_bytebpe512_sealed.yaml
 create mode 100644 configs/gptiny_alice_char_sealed.yaml
 create mode 100644 configs/gptiny_peterpan_bytebpe512_sealed.yaml
 create mode 100644 configs/gptiny_peterpan_char_sealed.yaml
 create mode 100644 experiments/026-sealed-test-evaluation.md
 create mode 100644 scripts/evaluate_test.py
 create mode 100644 src/smallm/evaluation/sealed.py
 create mode 100644 tests/test_sealed_test.py

diff --git a/AGENTS.md b/AGENTS.md
index 97a9915..a3b6e76 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -90,11 +90,10 @@ If a relevant check cannot be run, state why and what remains unverified.
 
 ## Current Technical Status
 
-Milestone 025 is the latest modeling evidence. In a balanced 2-tokenizer × 2-corpus × 3-seed
-matrix, ByteBPE512 beats character in all six paired comparisons. Its mean advantage is `0.0619`
-BPC on Alice and `0.0252` on Peter Pan; the `+0.0367` corpus interaction means effect magnitude is
-not distribution-invariant. The next contract change should add a sealed chronological test segment
-before further model selection.
+Milestone 026 is the latest modeling evidence. Frozen 80/10/10 runs evaluated once on sealed
+terminal segments confirm ByteBPE512 over character by `0.0614` BPC on Alice and `0.0258` on Peter
+Pan. Test BPC is worse than validation for all four models. These test segments are consumed and
+must not guide further tuning; future modeling needs a new untouched evaluation distribution.
 
 ## Safety And Security
 
diff --git a/README.md b/README.md
index e285d42..5d25bb7 100644
--- a/README.md
+++ b/README.md
@@ -17,8 +17,8 @@ For a fast technical review, inspect:
 1. [`docs/architecture.md`](docs/architecture.md) for boundaries and the
    end-to-end pipeline.
 2. [`docs/experiments.md`](docs/experiments.md) for the milestone index.
-3. [`experiments/025-corpus-by-seed-matrix.md`](experiments/025-corpus-by-seed-matrix.md)
-   for the latest balanced robustness evidence and its limitations.
+3. [`experiments/026-sealed-test-evaluation.md`](experiments/026-sealed-test-evaluation.md)
+   for the latest confirmatory held-out evidence and its limitations.
 4. [`src/smallm/model/`](src/smallm/model/) for the from-scratch GPTiny model.
 5. [`src/smallm/training/`](src/smallm/training/) for training and run artifacts.
 6. [`src/smallm/data/`](src/smallm/data/) for corpus and tokenizer contracts.
@@ -50,6 +50,7 @@ chronological 90/10 split unless noted otherwise.
 | [023](experiments/023-multi-seed-robustness.md) | ByteBPE512 across three seeds | Best BPC mean ± population SD | `2.0225 ± 0.0124` | All tested seeds beat the character control; stopping varies from 2500–3000 steps. |
 | [024](experiments/024-cross-corpus-robustness.md) | Peter Pan character vs ByteBPE512 | Full-validation best bits/character | `2.1721` vs `2.1539` | ByteBPE512 replicates the direction on a second book, but only by 0.83%. |
 | [025](experiments/025-corpus-by-seed-matrix.md) | 2 tokenizers × 2 corpora × 3 seeds | Paired ByteBPE512 minus character BPC | `-0.0619` Alice; `-0.0252` Peter Pan | ByteBPE512 wins all six pairs; effect magnitude is corpus-dependent. |
+| [026](experiments/026-sealed-test-evaluation.md) | Frozen decision on terminal 10% test segments | ByteBPE512 minus character test BPC | `-0.0614` Alice; `-0.0258` Peter Pan | The tokenizer decision survives one-shot full-coverage tests on both books. |
 
 Token-level loss and perplexity are not directly comparable between character
 and BPE tokenizers because they predict different units. The tokenizer
@@ -84,7 +85,7 @@ shapes link directly to the implementation and its tests.
 | Corpus preparation | Normalized corpus output, stats, checksums, source metadata, and manifest files. |
 | Tokenization | Character, educational character-BPE, and lossless boundary-aware UTF-8 byte-BPE tokenizers. |
 | Model | Decoder-only GPT-style Transformer with causal self-attention. |
-| Evaluation | Uniform, unigram, and add-one smoothed bigram baselines. |
+| Evaluation | Uniform, unigram, and add-one bigram baselines; optional sealed chronological test evaluation. |
 | Training | Config-driven training with validation loss, optional early stopping, progress logging, checkpoints, metrics, summaries, and samples. |
 | Run records | Preserved run directories with copied dataset manifests and selected provenance fields in `summary.json`. |
 | Generation | `max_new_tokens`, `temperature`, `top_k`, `seed`, and greedy decoding. |
diff --git a/configs/gptiny_alice_bytebpe512_sealed.yaml b/configs/gptiny_alice_bytebpe512_sealed.yaml
new file mode 100644
index 0000000..caec841
--- /dev/null
+++ b/configs/gptiny_alice_bytebpe512_sealed.yaml
@@ -0,0 +1,39 @@
+data:
+  input_path: data/raw/input.txt
+  prepared_path: data/processed/corpus.txt
+  manifest_path: data/processed/corpus_sealed_manifest.json
+  tokenizer_path: data/processed/tokenizer_bytebpe512_sealed.json
+  tokenizer_type: byte_bpe
+  bpe_vocab_size: 512
+  bpe_min_frequency: 2
+  block_size: 37
+  train_split: 0.8
+  validation_split: 0.1
+
+model:
+  vocab_size: 512
+  block_size: 37
+  n_layer: 4
+  n_head: 4
+  n_embd: 128
+  dropout: 0.1
+
+train:
+  run_name: gptiny_alice_bytebpe512_sealed
+  runs_dir: runs
+  batch_size: 27
+  max_steps: 5000
+  learning_rate: 0.001
+  weight_decay: 0.0
+  log_interval: 100
+  eval_interval: 250
+  eval_batches: null
+  early_stopping_patience: 3
+  early_stopping_min_delta: 0.0
+  sample_prompt: Once
+  sample_max_new_tokens: 100
+  sample_temperature: 1.0
+  sample_top_k: null
+  sample_seed: 1337
+  sample_greedy: false
+  seed: 1337
diff --git a/configs/gptiny_alice_char_sealed.yaml b/configs/gptiny_alice_char_sealed.yaml
new file mode 100644
index 0000000..f45a6e2
--- /dev/null
+++ b/configs/gptiny_alice_char_sealed.yaml
@@ -0,0 +1,37 @@
+data:
+  input_path: data/raw/input.txt
+  prepared_path: data/processed/corpus.txt
+  manifest_path: data/processed/corpus_sealed_manifest.json
+  tokenizer_path: data/processed/tokenizer_char_sealed.json
+  tokenizer_type: char
+  block_size: 64
+  train_split: 0.8
+  validation_split: 0.1
+
+model:
+  vocab_size: 256
+  block_size: 64
+  n_layer: 4
+  n_head: 4
+  n_embd: 128
+  dropout: 0.1
+
+train:
+  run_name: gptiny_alice_char_sealed
+  runs_dir: runs
+  batch_size: 16
+  max_steps: 5000
+  learning_rate: 0.001
+  weight_decay: 0.0
+  log_interval: 100
+  eval_interval: 250
+  eval_batches: null
+  early_stopping_patience: 3
+  early_stopping_min_delta: 0.0
+  sample_prompt: Once
+  sample_max_new_tokens: 100
+  sample_temperature: 1.0
+  sample_top_k: null
+  sample_seed: 1337
+  sample_greedy: false
+  seed: 1337
diff --git a/configs/gptiny_peterpan_bytebpe512_sealed.yaml b/configs/gptiny_peterpan_bytebpe512_sealed.yaml
new file mode 100644
index 0000000..35ba534
--- /dev/null
+++ b/configs/gptiny_peterpan_bytebpe512_sealed.yaml
@@ -0,0 +1,39 @@
+data:
+  input_path: data/raw/peter_pan_body.txt
+  prepared_path: data/processed/peter_pan_corpus.txt
+  manifest_path: data/processed/peter_pan_corpus_sealed_manifest.json
+  tokenizer_path: data/processed/peter_pan_tokenizer_bytebpe512_sealed.json
+  tokenizer_type: byte_bpe
+  bpe_vocab_size: 512
+  bpe_min_frequency: 2
+  block_size: 37
+  train_split: 0.8
+  validation_split: 0.1
+
+model:
+  vocab_size: 512
+  block_size: 37
+  n_layer: 4
+  n_head: 4
+  n_embd: 128
+  dropout: 0.1
+
+train:
+  run_name: gptiny_peterpan_bytebpe512_sealed
+  runs_dir: runs
+  batch_size: 27
+  max_steps: 5000
+  learning_rate: 0.001
+  weight_decay: 0.0
+  log_interval: 100
+  eval_interval: 250
+  eval_batches: null
+  early_stopping_patience: 3
+  early_stopping_min_delta: 0.0
+  sample_prompt: Once
+  sample_max_new_tokens: 100
+  sample_temperature: 1.0
+  sample_top_k: null
+  sample_seed: 1337
+  sample_greedy: false
+  seed: 1337
diff --git a/configs/gptiny_peterpan_char_sealed.yaml b/configs/gptiny_peterpan_char_sealed.yaml
new file mode 100644
index 0000000..8fd9a3f
--- /dev/null
+++ b/configs/gptiny_peterpan_char_sealed.yaml
@@ -0,0 +1,37 @@
+data:
+  input_path: data/raw/peter_pan_body.txt
+  prepared_path: data/processed/peter_pan_corpus.txt
+  manifest_path: data/processed/peter_pan_corpus_sealed_manifest.json
+  tokenizer_path: data/processed/peter_pan_tokenizer_char_sealed.json
+  tokenizer_type: char
+  block_size: 64
+  train_split: 0.8
+  validation_split: 0.1
+
+model:
+  vocab_size: 256
+  block_size: 64
+  n_layer: 4
+  n_head: 4
+  n_embd: 128
+  dropout: 0.1
+
+train:
+  run_name: gptiny_peterpan_char_sealed
+  runs_dir: runs
+  batch_size: 16
+  max_steps: 5000
+  learning_rate: 0.001
+  weight_decay: 0.0
+  log_interval: 100
+  eval_interval: 250
+  eval_batches: null
+  early_stopping_patience: 3
+  early_stopping_min_delta: 0.0
+  sample_prompt: Once
+  sample_max_new_tokens: 100
+  sample_temperature: 1.0
+  sample_top_k: null
+  sample_seed: 1337
+  sample_greedy: false
+  seed: 1337
diff --git a/docs/architecture.md b/docs/architecture.md
index d680720..dc3ab77 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -21,6 +21,9 @@ flowchart TD
     H --> I[checkpoint.pt]
     H --> J[best_checkpoint.pt]
     H --> K[metrics.jsonl + summary.json]
+    J --> O[One-shot sealed test evaluator]
+    B -. terminal split .-> O
+    O --> P[test_evaluation_best.json]
     I --> L[Generation + diagnostics]
     J --> L
     K --> M[Experiment reports]
@@ -35,8 +38,8 @@ block construction.
 `model/` owns the GPT network: embeddings, causal self-attention, Transformer
 blocks, final normalization, and language-model head.
 
-`evaluation/` owns simple character-level reference models used to interpret
-validation loss.
+`evaluation/` owns simple reference models, bounded run observations, and shared evaluation
+contracts used to interpret validation and sealed-test loss.
 
 `training/` owns orchestration: dataloaders, optimization, evaluation intervals,
 progress logging, checkpoints, run directories, metrics, summaries, and copied
@@ -82,6 +85,11 @@ blocks and weights NLL by target count; summaries disclose full or sampled cover
 a complete UTF-8 byte fallback, forbids merges across whitespace boundaries, and preserves aligned
 Unicode-character completion counts for exact BPC.
 
+An optional explicit validation fraction activates a chronological train/validation/test split.
+Training does not encode the terminal test text. After checkpoint selection, the sealed evaluator
+verifies corpus and checkpoint identity, evaluates full coverage once, and refuses to overwrite its
+result artifact.
+
 ## Run Artifacts
 
 Each training run writes:
@@ -95,6 +103,7 @@ Each training run writes:
 | `best_checkpoint.pt` | Same checkpoint payload captured at the best validation step, when validation is available. |
 | `sample.txt` | Text generated at the end of training. |
 | `dataset_manifest.json` | Copied corpus manifest for run-local provenance. |
+| `test_evaluation_best.json` | Optional post-training sealed-test result with corpus and checkpoint hashes. |
 
 Run utilities resolve explicit run paths and `latest` by run name.
 
diff --git a/docs/codex/architecture.md b/docs/codex/architecture.md
index e77ff3f..0238774 100644
--- a/docs/codex/architecture.md
+++ b/docs/codex/architecture.md
@@ -35,7 +35,7 @@ Core logic is deterministic or tensor-oriented package code:
   inspectable JSON state.
 - `smallm.data.byte_bpe_tokenizer`: lossless UTF-8 byte fallback, boundary-aware merges, and
   character-aligned evaluation counts.
-- `smallm.data.dataset`: train/validation split and shifted token blocks.
+- `smallm.data.dataset`: backward-compatible train/validation/test split and shifted token blocks.
 - `smallm.model`: causal attention, Transformer blocks, GPT forward/loss.
 - `smallm.evaluation`: baseline losses and perplexity.
 - `smallm.generation`: decoding controls over model logits.
@@ -59,6 +59,8 @@ policy explicit in config and summary fields.
 Scripts under `scripts/` are CLI adapters. They should parse arguments, load
 config/checkpoints, call package code, and print results. Do not put reusable
 domain logic in scripts when it belongs under `src/smallm/`.
+`scripts/evaluate_test.py` is the post-selection edge: it verifies run-local provenance and writes
+the one-shot sealed-test artifact without exposing test data to training orchestration.
 
 ## Module Boundaries
 
@@ -81,6 +83,7 @@ domain logic in scripts when it belongs under `src/smallm/`.
 
 - baseline probability models
 - validation loss and perplexity calculations
+- exact full-coverage sealed-test evaluation results
 
 `training/` owns:
 
diff --git a/docs/codex/build-and-test.md b/docs/codex/build-and-test.md
index 4a52e97..17091df 100644
--- a/docs/codex/build-and-test.md
+++ b/docs/codex/build-and-test.md
@@ -50,7 +50,7 @@ The individual commands remain canonical and are listed below.
 python -m pytest
 ```
 
-Current expected result after milestone 025+: at least 155 tests passing with at least 90% coverage.
+Current expected result after milestone 026+: at least 163 tests passing with at least 90% coverage.
 
 ### Compile Check
 
@@ -136,6 +136,16 @@ python scripts/generate.py --run latest --run-name gptiny --prompt "Once" --gree
 python scripts/generate.py --run latest --run-name gptiny --prompt "Once" --temperature 0.8 --top-k 10 --seed 1337 --max-new-tokens 100
 ```
 
+### Sealed Test Evaluation
+
+For an explicitly configured three-way split, evaluate only after the checkpoint decision is frozen:
+
+```bash
+python scripts/evaluate_test.py --run runs/<name>/<id> --checkpoint-kind best
+```
+
+Do not rerun, delete, or use the resulting test artifact for model selection.
+
 ## Validation Matrix
 
 | Change type | Minimum validation |
@@ -147,6 +157,7 @@ python scripts/generate.py --run latest --run-name gptiny --prompt "Once" --temp
 | Model code | `python -m pytest`, compileall. |
 | Baselines | `python -m pytest`, `evaluate_baselines.py`, compileall. |
 | Training/artifacts/runs | smoke or tiny training path, `show_run.py`, `python -m pytest`, compileall. |
+| Split or sealed-test contract | Legacy smoke plus three-way fixture, one-shot evaluator, `python -m pytest`, compileall. |
 | Generation | greedy and seeded top-k generation commands, `python -m pytest`, compileall. |
 | Experiment report | Commands claimed in the report, plus tests and compileall when code changed. |
 
@@ -165,6 +176,8 @@ current task or explicitly quoted from a prior report.
 
 ## Current Milestone Status
 
-Milestone 019 is the current validation contract: frozen dependencies, Ruff, strict mypy, at
-least 90% coverage, compile and link checks, a smoke pipeline, and corrected full-validation
-character/BPE evidence.
+Milestone 026 is the current evaluation contract: legacy two-way splits remain compatible, while
+explicit validation fractions reserve a chronological test region that training leaves unencoded
+and unscored.
+Best-checkpoint test evaluation verifies corpus/checkpoint identity, requires full coverage, and
+refuses artifact overwrite.
diff --git a/docs/codex/experiments.md b/docs/codex/experiments.md
index ef24225..c5b74c5 100644
--- a/docs/codex/experiments.md
+++ b/docs/codex/experiments.md
@@ -170,3 +170,8 @@ Milestone 025 completes the balanced corpus-by-seed matrix. ByteBPE512 beats cha
 same-seed pairs. Mean candidate-minus-reference BPC is `-0.0619` on Alice and `-0.0252` on Peter
 Pan; the corpus interaction is `+0.0367`, so report a robust direction but corpus-dependent size.
 The next contract change should introduce a sealed test segment before further model selection.
+
+Milestone 026 introduces an optional `validation_split`; explicit 80/10/10 configs keep the final
+region unavailable to tokenizer fitting, training, early stopping, and checkpoint selection.
+One-shot best-checkpoint test evaluation confirms ByteBPE512 by `0.0614` BPC on Alice and `0.0258`
+on Peter Pan. Those test segments are consumed and must not guide further tuning.
diff --git a/docs/experiments.md b/docs/experiments.md
index 86a677b..7153245 100644
--- a/docs/experiments.md
+++ b/docs/experiments.md
@@ -32,6 +32,7 @@ not a replacement for the original reports.
 | [023 Multi-Seed Robustness](../experiments/023-multi-seed-robustness.md) | Robustness | Three preregistered seeds average best BPC `2.0225 ± 0.0124`; every seed beats the corrected character control. |
 | [024 Cross-Corpus Robustness](../experiments/024-cross-corpus-robustness.md) | External validity | On near-size-matched Peter Pan, ByteBPE512 narrowly beats character at `2.1539` versus `2.1721` BPC. |
 | [025 Corpus-by-Seed Matrix](../experiments/025-corpus-by-seed-matrix.md) | Factorial robustness | ByteBPE512 wins all six paired comparisons; mean advantage is `0.0619` BPC on Alice and `0.0252` on Peter Pan. |
+| [026 Sealed Test Evaluation](../experiments/026-sealed-test-evaluation.md) | Confirmatory evaluation | On untouched terminal segments, ByteBPE512 beats character by `0.0614` BPC on Alice and `0.0258` on Peter Pan. |
 
 ## Topic Shortcuts
 
@@ -58,7 +59,8 @@ not a replacement for the original reports.
   [020](../experiments/020-bpe-context-and-learning-rate.md),
   [021](../experiments/021-boundary-aware-byte-bpe.md),
   [024](../experiments/024-cross-corpus-robustness.md),
-  [025](../experiments/025-corpus-by-seed-matrix.md).
+  [025](../experiments/025-corpus-by-seed-matrix.md),
+  [026](../experiments/026-sealed-test-evaluation.md).
 
 ## Current Status
 
@@ -98,3 +100,8 @@ advantage; a corpus-by-seed matrix is the next stronger test.
 Milestone 025 completes that balanced matrix. ByteBPE512 beats character for seeds 1337, 2027, and
 4242 on both Alice and Peter Pan. Its paired mean advantage is `0.0619` BPC on Alice and `0.0252`
 on Peter Pan; the `+0.0367` BPC interaction shows that effect magnitude remains corpus-dependent.
+
+Milestone 026 freezes that decision and evaluates new 80/10/10 runs once on terminal test segments.
+ByteBPE512 reaches test BPC `2.1178` versus `2.1792` on Alice and `2.2484` versus `2.2742` on Peter
+Pan. The direction and approximate margins survive, while every model's test BPC is worse than its
+validation BPC.
diff --git a/docs/training.md b/docs/training.md
index 311c5db..8ea2da6 100644
--- a/docs/training.md
+++ b/docs/training.md
@@ -14,6 +14,7 @@ dataset provenance.
 | Evaluate baselines | `python scripts/evaluate_baselines.py --config configs/<name>.yaml` |
 | Train | `python scripts/train.py --config configs/<name>.yaml` |
 | Inspect run | `python scripts/show_run.py --run latest --run-name <name>` |
+| Evaluate sealed test | `python scripts/evaluate_test.py --run <run-dir> --checkpoint-kind best` |
 | Generate | `python scripts/generate.py --run latest --run-name <name> --prompt "Once"` |
 
 ## Common Make Targets
@@ -47,6 +48,7 @@ compileall, and Markdown links. `make smoke` writes an ignored smoke run, and
 | `configs/gptiny_peterpan_bytebpe512_5k_lr1e-3_ctx37_earlystop.yaml` | Context-matched Peter Pan ByteBPE512 run. |
 | `configs/gptiny_char_5k_lr1e-3_earlystop*.yaml` | Three-seed Alice character early-stopping controls. |
 | `configs/gptiny_peterpan_*_earlystop_seed*.yaml` | Additional Peter Pan matrix seeds. |
+| `configs/gptiny_{alice,peterpan}_{char,bytebpe512}_sealed.yaml` | Frozen 80/10/10 confirmatory runs. |
 
 ## Corpus Preparation
 
@@ -64,6 +66,12 @@ python scripts/prepare_corpus.py \
 Use `--source-note` for fetch URLs, extraction notes, or manual curation notes.
 The manifest is copied into each training run.
 
+Add `--validation-split 0.1` with `--train-split 0.8` to reserve a terminal 10% test segment. When
+`data.validation_split` is absent, existing configs retain the legacy two-way behavior. With an
+explicit validation fraction, tokenizer fitting and training use only the leading train region,
+early stopping uses only the middle validation region, and the final region stays unencoded and
+unscored.
+
 ## Smoke Run
 
 ```bash
@@ -121,6 +129,20 @@ the saved sample.
 inspectable. The final checkpoint is the model at the stop step, while `best_checkpoint.pt` remains
 the lowest observed validation-loss model.
 
+For a run with an explicit sealed split, training records the logical status
+`test_status: sealed_unread` and the test character count but no test tokens or metrics. The label
+means sealed from modeling and selection; the prepared corpus is still loaded to calculate slice
+boundaries. After freezing the decision, evaluate once:
+
+```bash
+python scripts/evaluate_test.py --run runs/<name>/<id> --checkpoint-kind best
+```
+
+The command verifies the run's copied manifest, corpus checksum, selected checkpoint step, and full
+coverage. It writes `test_evaluation_best.json` with checkpoint/corpus hashes, loss, BPC, and exact
+target counts, then refuses to overwrite the artifact. Deleting a local artifact can bypass this
+guardrail, so the scientific one-shot rule remains procedural.
+
 Aggregate completed runs without selecting a winner:
 
 ```bash
@@ -179,6 +201,7 @@ runs; summaries record evaluated targets and coverage.
 | `best_checkpoint.pt` | Same payload at the best validation step; absent when validation is unavailable. |
 | `sample.txt` | End-of-training generated sample. |
 | `dataset_manifest.json` | Copied corpus manifest. |
+| `test_evaluation_best.json` | Optional one-shot sealed-test result; created after training only. |
 
 ## Current Reading Of Results
 
@@ -205,6 +228,9 @@ effect size.
 Experiment 025 completes the 2-tokenizer × 2-corpus × 3-seed matrix. ByteBPE512 wins every paired
 comparison. Its mean advantage is `0.0619` BPC on Alice and `0.0252` on Peter Pan, so the direction
 is robust within the matrix while the effect magnitude remains corpus-dependent.
+Experiment 026 adds a three-way chronological contract and evaluates the frozen decision once.
+ByteBPE512 beats character on sealed test BPC by `0.0614` on Alice and `0.0258` on Peter Pan. All
+four test results are worse than validation, and these terminal segments are now consumed evidence.
 
 ## Artifact Policy
 
diff --git a/experiments/026-sealed-test-evaluation.md b/experiments/026-sealed-test-evaluation.md
new file mode 100644
index 0000000..0481053
--- /dev/null
+++ b/experiments/026-sealed-test-evaluation.md
@@ -0,0 +1,130 @@
+# 026 — Sealed Test Evaluation
+
+## Goal
+
+Harden the evaluation contract with a chronological train/validation/test split, freeze the
+ByteBPE512 decision from milestone 025, and evaluate that decision once on terminal test segments
+that were unavailable to tokenizer fitting, gradient updates, early stopping, and checkpoint
+selection.
+
+## Contract
+
+`data.validation_split` is optional. When absent, legacy configs retain the existing two-way
+train/remainder-validation behavior. When present, smaLLM creates three chronological regions:
+
+1. training fits tokenizer merges and model parameters;
+2. validation drives early stopping and selects `best_checkpoint.pt`;
+3. test is sliced only to record its character count during training; it is not tokenized, scored,
+   or used for selection until `scripts/evaluate_test.py` runs.
+
+The four confirmatory configs use 80% train, 10% validation, and 10% test. Training summaries expose
+only the test character count and the logical status `test_status: sealed_unread`; they contain no
+test tokens, loss, or BPC. The status means unavailable to the modeling and selection workflow, not
+that the containing corpus file was never opened. The evaluator requires a completed run with an explicit test split, verifies the copied
+manifest and prepared-corpus checksum, loads the declared best checkpoint, verifies its step against
+the summary, evaluates full coverage, hashes the checkpoint, writes `test_evaluation_best.json`, and
+refuses to overwrite it.
+
+The refusal is an operational guardrail, not tamper-proof enforcement: local artifacts can be
+deleted. Scientific compliance still requires treating these four test results as consumed after
+this milestone.
+
+## Frozen Setup
+
+The comparison was fixed before any test access: seed 1337, character versus ByteBPE512, 4 layers,
+4 heads, width 128, dropout 0.1, AdamW `lr=1e-3`, zero weight decay, full validation every 250
+steps, patience 3, and a 5,000-step ceiling. Character uses context 64 and batch 16; ByteBPE512 uses
+context 37 and batch 27. No configuration was changed after observing validation or test results.
+
+| corpus | prepared chars | train | validation | sealed test | checksum |
+| --- | ---: | ---: | ---: | ---: | --- |
+| Alice | 144,530 | 115,624 | 14,453 | 14,453 | `a4c81ef23eb9…` |
+| Peter Pan | 144,489 | 115,591 | 14,449 | 14,449 | `16e4f26e7e52…` |
+
+## Validation And Stopping
+
+| corpus | tokenizer | actual steps | best step | best validation BPC |
+| --- | --- | ---: | ---: | ---: |
+| Alice | character | 5,000 | 4,250 | 2.119716 |
+| Alice | ByteBPE512 | 2,250 | 1,500 | **2.076651** |
+| Peter Pan | character | 5,000 | 4,750 | 2.195677 |
+| Peter Pan | ByteBPE512 | 2,750 | 2,000 | **2.187737** |
+
+The validation advantage is 0.043066 BPC on Alice and 0.007940 on Peter Pan. ByteBPE512 again
+selects and stops much earlier, while character continues improving near the ceiling.
+
+## One-Shot Test Results
+
+All evaluations use the best-validation checkpoint and full non-overlapping coverage. One source
+character is consumed as the autoregressive input before the first scored target, so target
+characters equal test characters minus one.
+
+| corpus | tokenizer | test tokens | target chars | test token loss | test BPC |
+| --- | --- | ---: | ---: | ---: | ---: |
+| Alice | character | 14,453 | 14,452 | 1.510502 | 2.179194 |
+| Alice | ByteBPE512 | 8,471 | 14,452 | 2.504684 | **2.117790** |
+| Peter Pan | character | 14,449 | 14,448 | 1.576362 | 2.274210 |
+| Peter Pan | ByteBPE512 | 8,569 | 14,448 | 2.628048 | **2.248431** |
+
+ByteBPE512 wins both sealed comparisons: 0.061404 BPC on Alice and 0.025780 on Peter Pan. These are
+strikingly close to milestone 025's three-seed validation-matrix mean advantages of 0.061863 and
+0.025203. The tokenizer decision therefore survives a genuinely untouched chronological segment on
+both books.
+
+Absolute test BPC is worse than validation BPC for every model: +0.059478 Alice character,
++0.041139 Alice ByteBPE512, +0.078533 Peter Pan character, and +0.060693 Peter Pan ByteBPE512. The
+terminal text is harder than the middle validation region, which demonstrates why validation alone
+was an optimistic description of forward generalization.
+
+## Baselines
+
+These are validation token losses under the new 80/10/10 training fit and are comparable only within
+tokenizer:
+
+| corpus | tokenizer | uniform | unigram | add-one bigram |
+| --- | --- | ---: | ---: | ---: |
+| Alice | character | 4.3307 | 3.2019 | 2.4569 |
+| Alice | ByteBPE512 | 6.2383 | 4.3664 | 3.7471 |
+| Peter Pan | character | 4.3820 | 3.0862 | 2.4118 |
+| Peter Pan | ByteBPE512 | 6.2383 | 4.2748 | 3.7324 |
+
+## Exact Commands
+
+Each corpus was re-manifested without changing its prepared checksum:
+
+```bash
+python scripts/prepare_corpus.py ... --train-split 0.8 --validation-split 0.1
+python scripts/prepare_data.py --config configs/gptiny_alice_char_sealed.yaml
+python scripts/prepare_data.py --config configs/gptiny_alice_bytebpe512_sealed.yaml
+python scripts/prepare_data.py --config configs/gptiny_peterpan_char_sealed.yaml
+python scripts/prepare_data.py --config configs/gptiny_peterpan_bytebpe512_sealed.yaml
+python scripts/evaluate_baselines.py --config configs/<sealed-config>.yaml
+python scripts/train.py --config configs/gptiny_alice_char_sealed.yaml
+python scripts/train.py --config configs/gptiny_alice_bytebpe512_sealed.yaml
+python scripts/train.py --config configs/gptiny_peterpan_char_sealed.yaml
+python scripts/train.py --config configs/gptiny_peterpan_bytebpe512_sealed.yaml
+python scripts/evaluate_test.py --run runs/<sealed-run>/<run-id> --checkpoint-kind best
+make check
+make audit
+```
+
+Run paths are:
+
+- `runs/gptiny_alice_char_sealed/2026-07-13_01-06-53`
+- `runs/gptiny_alice_bytebpe512_sealed/2026-07-13_01-15-47`
+- `runs/gptiny_peterpan_char_sealed/2026-07-13_01-19-42`
+- `runs/gptiny_peterpan_bytebpe512_sealed/2026-07-13_01-28-21`
+
+## Limitations And Next Step
+
+- One seed is confirmatory for the fixed matrix-level decision, not a new seed-distribution study.
+- Alice and Peter Pan are related English literary distributions.
+- Parameter counts remain vocabulary-dependent.
+- The terminal segment may differ in chapter structure and intrinsic difficulty.
+- These test segments are consumed and cannot support further model or hyperparameter selection.
+- The local one-shot guard is procedural rather than cryptographically enforced.
+
+The next modeling experiment must define a new untouched evaluation distribution before training.
+The strongest choice is a stylistically different public-domain corpus family—such as nonfiction,
+speeches, or plays—with its own preregistered 80/10/10 split. The current Alice and Peter Pan test
+results should remain final evidence for the frozen ByteBPE512 decision.
diff --git a/notes/02-data-and-tokenization.md b/notes/02-data-and-tokenization.md
index 51abfbd..36290b4 100644
--- a/notes/02-data-and-tokenization.md
+++ b/notes/02-data-and-tokenization.md
@@ -7,10 +7,12 @@ distribution. smaLLM normalizes line endings, strips trailing whitespace, collap
 lines, and ensures one final newline. A manifest records raw and prepared SHA-256 hashes, counts,
 split policy, and normalization rules so a run identifies bytes rather than a mutable filename.
 
-For split fraction \(\alpha\) and `C` characters, \(s=\lfloor\alpha C\rfloor\). Training receives
-`text[:s]`, validation `text[s:]`. A chronological split can expose distribution shift across the
-source; unlike randomized windows, it does not scatter near-duplicate neighboring contexts across
-both partitions.
+For training fraction \(\alpha\), optional validation fraction \(\beta\), and \(C\) characters,
+the boundaries are \(s=\lfloor\alpha C\rfloor\) and
+\(v=\lfloor(\alpha+\beta)C\rfloor\). Training receives `text[:s]`, validation `text[s:v]`, and
+test `text[v:]`. Without \(\beta\), legacy configs use `text[s:]` entirely for validation. A
+chronological split can expose distribution shift across the source; unlike randomized windows, it
+does not scatter near-duplicate neighboring contexts across partitions.
 
 ## Character tokenizer
 
@@ -74,10 +76,13 @@ flowchart LR
     A[Prepared text] --> B[Character split]
     B --> C[Training text]
     B --> D[Validation text]
+    B --> H[Sealed test text]
     C --> E[Fit tokenizer]
     E --> F[Encode training]
     E --> G[Encode validation]
     D --> G
+    E -.->|after frozen checkpoint only| I[One-shot test encoding]
+    H -.-> I
 ```
 
 If character length is \(C_s\) and token length \(T_s\), compression is \(C_s/T_s\). Shorter
diff --git a/notes/06-reproducibility.md b/notes/06-reproducibility.md
index fbf0fc5..2e482a0 100644
--- a/notes/06-reproducibility.md
+++ b/notes/06-reproducibility.md
@@ -154,3 +154,37 @@ early stopping and configuration choice; a sealed test segment is read once afte
 is frozen. For chronological text, the ordering must remain explicit because a terminal test block
 measures forward generalization, not exchangeable random-split performance. Test metrics must never
 flow back into checkpoint choice, hyperparameter tuning, or seed selection.
+
+### The test set as an information firewall
+
+Let the research process before test access produce a decision
+\(D=f(X_{\mathrm{train}},X_{\mathrm{val}},R)\), where \(R\) includes seeds, code, and declared
+rules. A confirmatory test estimate is interpretable because \(X_{\mathrm{test}}\) is absent from
+\(f\). Once its result \(T(D,X_{\mathrm{test}})\) is observed, any later decision
+\(D'=g(D,T)\) is statistically coupled to the test set. The same examples have become validation
+data in practice, regardless of their filename.
+
+An information firewall therefore requires both a data boundary and a workflow boundary. smaLLM's
+training path computes split indices and records the terminal character count, but does not encode
+the terminal text into tokens. The separate evaluator verifies the copied manifest, prepared-text
+hash, checkpoint hash, and checkpoint step before evaluating full coverage. Recording these
+identities makes accidental substitution detectable.
+
+The no-overwrite artifact rule prevents a common accidental second look, but cannot prove secrecy:
+a user controls the local filesystem and can delete outputs or modify code. Reproducibility tooling
+can make violations conspicuous; it cannot replace research discipline or an external data
+custodian. After milestone 026, the Alice and Peter Pan terminal segments are permanently consumed
+for this research line.
+
+The observed generalization gap is
+
+\[
+G=\operatorname{BPC}_{\mathrm{test}}-\operatorname{BPC}_{\mathrm{val}}.
+\]
+
+A positive \(G\) does not by itself prove overfitting: a chronological terminal segment may have
+higher irreducible entropy or a shifted style. Comparing tokenizer contrasts
+\(\Delta_c=\operatorname{BPC}_{\mathrm{Byte},c}-\operatorname{BPC}_{\mathrm{char},c}\) across
+validation and test asks a different question: whether the frozen decision preserves its direction
+under forward distribution shift. Milestone 026 finds \(G>0\) for every model while
+\(\Delta_c<0\) on both sealed tests.
diff --git a/scripts/evaluate_baselines.py b/scripts/evaluate_baselines.py
index 1631577..c8188e0 100644
--- a/scripts/evaluate_baselines.py
+++ b/scripts/evaluate_baselines.py
@@ -3,7 +3,7 @@
 import argparse
 
 from smallm.config import load_config
-from smallm.data import load_prepared_corpus, train_tokenizer
+from smallm.data import load_prepared_corpus, split_corpus_text, train_tokenizer
 from smallm.evaluation import evaluate_baselines
 from smallm.training.artifacts import load_dataset_manifest, verify_dataset_manifest
 
@@ -20,9 +20,13 @@ def main() -> None:
         prepared_path=config.data.prepared_path,
         prepared_text=text,
         train_split=config.data.train_split,
+        validation_split=config.data.validation_split,
+    )
+    train_text, val_text, _ = split_corpus_text(
+        text,
+        train_split=config.data.train_split,
+        validation_split=config.data.validation_split,
     )
-    split_index = int(len(text) * config.data.train_split)
-    train_text, val_text = text[:split_index], text[split_index:]
     tokenizer = train_tokenizer(config.data, train_text)
     train_tokens = tokenizer.encode(train_text)
     val_tokens = tokenizer.encode(val_text)
diff --git a/scripts/evaluate_test.py b/scripts/evaluate_test.py
new file mode 100644
index 0000000..4f3d828
--- /dev/null
+++ b/scripts/evaluate_test.py
@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import argparse
+
+from smallm.evaluation import evaluate_sealed_test
+from smallm.training.runs import resolve_run_path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run", required=True)
+    parser.add_argument("--run-name")
+    parser.add_argument("--runs-dir", default="runs")
+    parser.add_argument("--checkpoint-kind", choices=("best", "final"), default="best")
+    args = parser.parse_args()
+    try:
+        run_dir = resolve_run_path(args.run, run_name=args.run_name, runs_dir=args.runs_dir)
+        output_path, payload = evaluate_sealed_test(run_dir, checkpoint_kind=args.checkpoint_kind)
+    except (OSError, UnicodeError, TypeError, AttributeError, KeyError, ValueError) as exc:
+        parser.error(str(exc))
+    print(f"sealed test evaluation: {output_path}")
+    print(f"test loss: {payload['test_loss']:.6f}")
+    print(f"test BPC: {payload['test_bits_per_character']:.6f}")
+    print(f"coverage: {payload['test_coverage']:.6f}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/prepare_corpus.py b/scripts/prepare_corpus.py
index f29257e..c6bbb2e 100644
--- a/scripts/prepare_corpus.py
+++ b/scripts/prepare_corpus.py
@@ -17,6 +17,7 @@ def main() -> None:
     parser.add_argument("--source-name")
     parser.add_argument("--source-note")
     parser.add_argument("--train-split", type=float, default=0.9)
+    parser.add_argument("--validation-split", type=float)
     args = parser.parse_args()
 
     input_path = Path(args.input)
@@ -31,6 +32,7 @@ def main() -> None:
         source_name=args.source_name,
         source_note=args.source_note,
         output_path=output_path,
+        validation_split=args.validation_split,
     )
     stats_path = Path(args.stats)
     atomic_write_text(stats_path, json.dumps(stats, indent=2, sort_keys=True) + "\n")
diff --git a/scripts/prepare_data.py b/scripts/prepare_data.py
index 7912e05..6c701f5 100644
--- a/scripts/prepare_data.py
+++ b/scripts/prepare_data.py
@@ -3,7 +3,7 @@
 import argparse
 
 from smallm.config import load_config
-from smallm.data import load_prepared_corpus, train_tokenizer
+from smallm.data import load_prepared_corpus, split_corpus_text, train_tokenizer
 from smallm.training.artifacts import load_dataset_manifest, verify_dataset_manifest
 
 
@@ -19,9 +19,14 @@ def main() -> None:
         prepared_path=config.data.prepared_path,
         prepared_text=text,
         train_split=config.data.train_split,
+        validation_split=config.data.validation_split,
     )
-    split_index = int(len(text) * config.data.train_split)
-    tokenizer = train_tokenizer(config.data, text[:split_index])
+    train_text, _, _ = split_corpus_text(
+        text,
+        train_split=config.data.train_split,
+        validation_split=config.data.validation_split,
+    )
+    tokenizer = train_tokenizer(config.data, train_text)
     tokenizer.save(config.data.tokenizer_path)
     label = {
         "bpe": "BPE",
diff --git a/scripts/show_run.py b/scripts/show_run.py
index 979a6a9..eb06f8d 100644
--- a/scripts/show_run.py
+++ b/scripts/show_run.py
@@ -39,7 +39,12 @@ def main() -> None:
         print(
             "  split_characters: "
             f"train={dataset.get('train_characters')} "
-            f"val={dataset.get('validation_characters')}"
+            f"val={dataset.get('validation_characters')} "
+            f"test={dataset.get('test_characters', 0)}"
+        )
+    if summary.get("test_status") is not None:
+        print(
+            f"test: status={summary.get('test_status')} characters={summary.get('test_characters')}"
         )
     if last_metric is not None:
         print(
diff --git a/src/smallm/config.py b/src/smallm/config.py
index f17a807..91c98b5 100644
--- a/src/smallm/config.py
+++ b/src/smallm/config.py
@@ -31,12 +31,23 @@ class DataConfig:
     bpe_min_frequency: int = 2
     block_size: int = 128
     train_split: float = 0.9
+    validation_split: float | None = None
 
     def __post_init__(self) -> None:
         if self.block_size <= 0:
             raise ValueError("data.block_size must be positive")
         if not 0.0 < self.train_split < 1.0:
             raise ValueError("data.train_split must be between 0 and 1")
+        if self.validation_split is not None and (
+            not isinstance(self.validation_split, int | float)
+            or isinstance(self.validation_split, bool)
+            or not isfinite(float(self.validation_split))
+            or self.validation_split <= 0
+            or self.train_split + self.validation_split >= 1.0
+        ):
+            raise ValueError(
+                "data.validation_split must be positive and leave a non-empty test fraction"
+            )
         if self.tokenizer_type not in {"char", "bpe", "byte_bpe"}:
             raise ValueError("data.tokenizer_type must be 'char', 'bpe', or 'byte_bpe'")
         if self.tokenizer_type in {"bpe", "byte_bpe"}:
diff --git a/src/smallm/data/__init__.py b/src/smallm/data/__init__.py
index 4eff05f..ae0aa3e 100644
--- a/src/smallm/data/__init__.py
+++ b/src/smallm/data/__init__.py
@@ -31,6 +31,7 @@
     "file_sha256",
     "load_prepared_corpus",
     "load_tokenizer",
+    "split_corpus_text",
     "split_tokens",
     "tokenizer_from_state",
     "train_tokenizer",
@@ -38,8 +39,12 @@
 
 
 def __getattr__(name: str) -> Any:
-    if name in {"TokenBlockDataset", "split_tokens"}:
-        from smallm.data.dataset import TokenBlockDataset, split_tokens
+    if name in {"TokenBlockDataset", "split_corpus_text", "split_tokens"}:
+        from smallm.data.dataset import TokenBlockDataset, split_corpus_text, split_tokens
 
-        return {"TokenBlockDataset": TokenBlockDataset, "split_tokens": split_tokens}[name]
+        return {
+            "TokenBlockDataset": TokenBlockDataset,
+            "split_corpus_text": split_corpus_text,
+            "split_tokens": split_tokens,
+        }[name]
     raise AttributeError(f"module 'smallm.data' has no attribute {name!r}")
diff --git a/src/smallm/data/corpus.py b/src/smallm/data/corpus.py
index 37eb383..6eb8114 100644
--- a/src/smallm/data/corpus.py
+++ b/src/smallm/data/corpus.py
@@ -64,13 +64,21 @@ def corpus_stats(
     source_name: str | None,
     source_note: str | None,
     output_path: str | Path,
+    validation_split: float | None = None,
     top_n: int = 20,
 ) -> dict[str, Any]:
     if not 0.0 < train_split < 1.0:
         raise ValueError("train_split must be between 0 and 1")
     counter = Counter(text)
     total_characters = len(text)
-    split_index = int(total_characters * train_split)
+    if validation_split is not None and not 0.0 < validation_split < 1.0 - train_split:
+        raise ValueError("validation_split must be positive and leave a non-empty test fraction")
+    train_end = int(total_characters * train_split)
+    validation_end = (
+        total_characters
+        if validation_split is None
+        else int(total_characters * (train_split + validation_split))
+    )
     lines = text.splitlines()
     return {
         "source_name": source_name,
@@ -85,8 +93,10 @@ def corpus_stats(
             {"character": char, "count": count} for char, count in counter.most_common(top_n)
         ],
         "train_split": train_split,
-        "train_characters": split_index,
-        "validation_characters": total_characters - split_index,
+        "train_characters": train_end,
+        "validation_split": validation_split,
+        "validation_characters": validation_end - train_end,
+        "test_characters": total_characters - validation_end,
     }
 
 
@@ -113,7 +123,9 @@ def corpus_manifest(
         "unique_characters": stats["unique_characters"],
         "train_split": stats["train_split"],
         "train_characters": stats["train_characters"],
+        "validation_split": stats.get("validation_split"),
         "validation_characters": stats["validation_characters"],
+        "test_characters": stats.get("test_characters", 0),
         "normalization_rules": NORMALIZATION_RULES,
         "generated_at": datetime.now(timezone.utc).isoformat(),
     }
diff --git a/src/smallm/data/dataset.py b/src/smallm/data/dataset.py
index 9a0c418..71f983c 100644
--- a/src/smallm/data/dataset.py
+++ b/src/smallm/data/dataset.py
@@ -4,6 +4,24 @@
 from torch.utils.data import Dataset
 
 
+def split_corpus_text(
+    text: str,
+    *,
+    train_split: float,
+    validation_split: float | None = None,
+) -> tuple[str, str, str]:
+    if not 0.0 < train_split < 1.0:
+        raise ValueError("train_split must be between 0 and 1")
+    if validation_split is None:
+        split_index = int(len(text) * train_split)
+        return text[:split_index], text[split_index:], ""
+    if not 0.0 < validation_split < 1.0 - train_split:
+        raise ValueError("validation_split must be positive and leave a non-empty test fraction")
+    train_end = int(len(text) * train_split)
+    validation_end = int(len(text) * (train_split + validation_split))
+    return text[:train_end], text[train_end:validation_end], text[validation_end:]
+
+
 class TokenBlockDataset(Dataset[tuple[torch.Tensor, torch.Tensor]]):
     def __init__(self, tokens: list[int] | torch.Tensor, block_size: int) -> None:
         if block_size < 1:
diff --git a/src/smallm/evaluation/__init__.py b/src/smallm/evaluation/__init__.py
index 5ea37ea..9673032 100644
--- a/src/smallm/evaluation/__init__.py
+++ b/src/smallm/evaluation/__init__.py
@@ -1,3 +1,4 @@
 from smallm.evaluation.baselines import BaselineResult, evaluate_baselines, perplexity
+from smallm.evaluation.sealed import evaluate_sealed_test
 
-__all__ = ["BaselineResult", "evaluate_baselines", "perplexity"]
+__all__ = ["BaselineResult", "evaluate_baselines", "evaluate_sealed_test", "perplexity"]
diff --git a/src/smallm/evaluation/sealed.py b/src/smallm/evaluation/sealed.py
new file mode 100644
index 0000000..cc87501
--- /dev/null
+++ b/src/smallm/evaluation/sealed.py
@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import torch
+
+from smallm.config import load_config
+from smallm.data import load_prepared_corpus, split_corpus_text, tokenizer_from_state
+from smallm.data.corpus import file_sha256
+from smallm.evaluation.run_observation import load_run_summary
+from smallm.model import GPT, GPTConfig
+from smallm.training.artifacts import load_dataset_manifest, verify_dataset_manifest, write_json
+from smallm.training.checkpoints import load_checkpoint
+from smallm.training.runs import resolve_run_checkpoint
+from smallm.training.trainer import evaluate_tokens
+from smallm.utils.device import default_device
+
+
+def evaluate_sealed_test(
+    run_dir: Path, *, checkpoint_kind: str = "best"
+) -> tuple[Path, dict[str, Any]]:
+    config = load_config(run_dir / "config.yaml")
+    if config.data.validation_split is None:
+        raise ValueError("run does not configure a sealed test split")
+    output_path = run_dir / f"test_evaluation_{checkpoint_kind}.json"
+    if output_path.exists() or output_path.is_symlink():
+        raise FileExistsError(f"sealed test evaluation already exists: {output_path}")
+
+    text = load_prepared_corpus(config.data.prepared_path)
+    manifest_path = run_dir / "dataset_manifest.json"
+    manifest = load_dataset_manifest(manifest_path)
+    verify_dataset_manifest(
+        manifest,
+        prepared_path=config.data.prepared_path,
+        prepared_text=text,
+        train_split=config.data.train_split,
+        validation_split=config.data.validation_split,
+    )
+    _, _, test_text = split_corpus_text(
+        text,
+        train_split=config.data.train_split,
+        validation_split=config.data.validation_split,
+    )
+    if len(test_text) <= config.data.block_size:
+        raise ValueError("sealed test split is too short for full evaluation")
+
+    checkpoint_path = resolve_run_checkpoint(run_dir, checkpoint_kind)
+    checkpoint = load_checkpoint(checkpoint_path)
+    summary = load_run_summary(run_dir)
+    if summary.get("test_status") != "sealed_unread" or summary.get("test_characters") != len(
+        test_text
+    ):
+        raise ValueError("run summary does not match the sealed test split")
+    expected_step = (
+        summary.get("best_val_step") if checkpoint_kind == "best" else summary.get("actual_steps")
+    )
+    if checkpoint.get("step") != expected_step:
+        raise ValueError("checkpoint step does not match the completed run summary")
+    tokenizer = tokenizer_from_state(checkpoint["tokenizer"])
+    token_ids, character_counts = tokenizer.encode_with_character_counts(test_text)
+    tokens = torch.tensor(token_ids, dtype=torch.long)
+    counts = torch.tensor(character_counts, dtype=torch.long)
+    device = default_device()
+    model = GPT(GPTConfig(**checkpoint["model_config"])).to(device)
+    model.load_state_dict(checkpoint["model_state"])
+    result = evaluate_tokens(
+        model,
+        tokens,
+        tokenizer=tokenizer,
+        device=device,
+        block_size=config.data.block_size,
+        max_batches=None,
+        character_counts=counts,
+    )
+    if result is None:
+        raise ValueError("sealed test split produced no evaluation targets")
+    if result.mode != "full" or result.coverage != 1.0:
+        raise ValueError("sealed test evaluation did not cover every target")
+
+    payload = {
+        "schema_version": 1,
+        "status": "complete",
+        "evaluated_at": datetime.now(timezone.utc).isoformat(),
+        "run_dir": str(run_dir),
+        "checkpoint_kind": checkpoint_kind,
+        "checkpoint_path": str(checkpoint_path),
+        "checkpoint_sha256": file_sha256(checkpoint_path),
+        "checkpoint_step": checkpoint["step"],
+        "prepared_sha256": manifest["prepared_sha256"],
+        "train_split": config.data.train_split,
+        "validation_split": config.data.validation_split,
+        "test_characters": len(test_text),
+        "test_tokens": len(token_ids),
+        "test_loss": result.loss,
+        "test_bits_per_character": result.bits_per_character,
+        "test_target_tokens": result.target_tokens,
+        "test_total_target_tokens": result.total_target_tokens,
+        "test_target_characters": result.target_characters,
+        "test_coverage": result.coverage,
+        "evaluation_mode": result.mode,
+    }
+    write_json(output_path, payload)
+    return output_path, payload
diff --git a/src/smallm/training/artifacts.py b/src/smallm/training/artifacts.py
index 596f056..c296eae 100644
--- a/src/smallm/training/artifacts.py
+++ b/src/smallm/training/artifacts.py
@@ -7,7 +7,7 @@
 from datetime import datetime
 from pathlib import Path
 from types import TracebackType
-from typing import Any, cast
+from typing import Any
 
 from smallm.config import ExperimentConfig
 from smallm.utils.io import atomic_write_text
@@ -22,10 +22,14 @@
     "unique_characters",
     "train_split",
     "train_characters",
+    "validation_split",
     "validation_characters",
+    "test_characters",
     "normalization_rules",
 ]
 
+MAX_DATASET_MANIFEST_BYTES = 1_000_000
+
 
 def create_run_dir(runs_dir: str | Path, run_name: str) -> Path:
     base = Path(runs_dir) / run_name
@@ -86,21 +90,36 @@ def write_json(path: str | Path, payload: dict[str, Any]) -> None:
 
 
 def load_dataset_manifest(path: str | Path) -> dict[str, Any]:
+    _, manifest = _read_dataset_manifest(path)
+    return manifest
+
+
+def _read_dataset_manifest(path: str | Path) -> tuple[str, dict[str, Any]]:
     manifest_path = Path(path)
     if not manifest_path.exists():
         raise FileNotFoundError(
             f"dataset manifest not found at {manifest_path}. "
             "Run scripts/prepare_corpus.py with --manifest before training."
         )
-    return cast(dict[str, Any], json.loads(manifest_path.read_text(encoding="utf-8")))
+    with manifest_path.open("rb") as handle:
+        raw = handle.read(MAX_DATASET_MANIFEST_BYTES + 1)
+    if len(raw) > MAX_DATASET_MANIFEST_BYTES:
+        raise ValueError(
+            f"dataset manifest exceeds {MAX_DATASET_MANIFEST_BYTES} bytes: {manifest_path}"
+        )
+    text = raw.decode("utf-8")
+    parsed = json.loads(text)
+    if not isinstance(parsed, dict):
+        raise ValueError("dataset manifest must contain a JSON object")
+    return text, parsed
 
 
 def copy_dataset_manifest(
     manifest_path: str | Path, run_dir: str | Path
 ) -> tuple[Path, dict[str, Any]]:
-    manifest = load_dataset_manifest(manifest_path)
+    text, manifest = _read_dataset_manifest(manifest_path)
     destination = Path(run_dir) / "dataset_manifest.json"
-    atomic_write_text(destination, Path(manifest_path).read_text(encoding="utf-8"))
+    atomic_write_text(destination, text)
     return destination, manifest
 
 
@@ -110,6 +129,7 @@ def verify_dataset_manifest(
     prepared_path: str | Path,
     prepared_text: str,
     train_split: float,
+    validation_split: float | None = None,
 ) -> None:
     required = {"prepared_sha256", "prepared_characters", "train_split"}
     missing = sorted(required - manifest.keys())
@@ -122,10 +142,23 @@ def verify_dataset_manifest(
         raise ValueError("prepared corpus character count does not match dataset manifest")
     if float(manifest["train_split"]) != train_split:
         raise ValueError("configured train split does not match dataset manifest")
-    split_index = int(len(prepared_text) * train_split)
+    manifest_validation_split = manifest.get("validation_split")
+    if validation_split is None:
+        if manifest_validation_split is not None:
+            raise ValueError("configured validation split does not match dataset manifest")
+        validation_end = len(prepared_text)
+    else:
+        if (
+            manifest_validation_split is None
+            or float(manifest_validation_split) != validation_split
+        ):
+            raise ValueError("configured validation split does not match dataset manifest")
+        validation_end = int(len(prepared_text) * (train_split + validation_split))
+    train_end = int(len(prepared_text) * train_split)
     expected_counts = {
-        "train_characters": split_index,
-        "validation_characters": len(prepared_text) - split_index,
+        "train_characters": train_end,
+        "validation_characters": validation_end - train_end,
+        "test_characters": len(prepared_text) - validation_end,
     }
     for field, expected in expected_counts.items():
         if field in manifest and int(manifest[field]) != expected:
diff --git a/src/smallm/training/trainer.py b/src/smallm/training/trainer.py
index 9dabd5f..41e3e1c 100644
--- a/src/smallm/training/trainer.py
+++ b/src/smallm/training/trainer.py
@@ -12,7 +12,7 @@
 from torch.utils.data import DataLoader
 
 from smallm.config import ExperimentConfig
-from smallm.data import TokenBlockDataset, load_prepared_corpus, train_tokenizer
+from smallm.data import TokenBlockDataset, load_prepared_corpus, split_corpus_text, train_tokenizer
 from smallm.generation import generate
 from smallm.model import GPT, GPTConfig
 from smallm.training.artifacts import (
@@ -240,10 +240,13 @@ def train(config: ExperimentConfig) -> Path:
         prepared_path=config.data.prepared_path,
         prepared_text=text,
         train_split=config.data.train_split,
+        validation_split=config.data.validation_split,
+    )
+    train_text, val_text, test_text = split_corpus_text(
+        text,
+        train_split=config.data.train_split,
+        validation_split=config.data.validation_split,
     )
-    character_split_index = int(len(text) * config.data.train_split)
-    train_text = text[:character_split_index]
-    val_text = text[character_split_index:]
     tokenizer = train_tokenizer(config.data, train_text)
     tokenizer.save(config.data.tokenizer_path)
     train_token_ids, _ = tokenizer.encode_with_character_counts(train_text)
@@ -253,6 +256,7 @@ def train(config: ExperimentConfig) -> Path:
     val_character_counts_tensor = torch.tensor(val_character_counts, dtype=torch.long)
     train_characters = len(train_text)
     val_characters = len(val_text)
+    test_characters = len(test_text)
     train_dataset = TokenBlockDataset(train_tokens, config.data.block_size)
     train_loader = DataLoader(train_dataset, batch_size=config.train.batch_size, shuffle=True)
     run_dir = create_run_dir(config.train.runs_dir, config.train.run_name)
@@ -477,6 +481,8 @@ def checkpoint_payload(checkpoint_step: int) -> dict[str, object]:
             "val_tokens": int(val_tokens.numel()),
             "train_characters": train_characters,
             "val_characters": val_characters,
+            "test_characters": test_characters,
+            "test_status": "sealed_unread" if test_characters else "not_configured",
             "final_val_bits_per_char": final_evaluation.bits_per_character
             if final_evaluation
             else None,
diff --git a/tests/test_config.py b/tests/test_config.py
index 4868ce7..4bad855 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -26,6 +26,14 @@ def test_data_config_defaults_to_char_tokenizer():
 
     assert config.tokenizer_type == "char"
     assert config.bpe_vocab_size is None
+    assert config.validation_split is None
+
+
+def test_data_config_accepts_explicit_sealed_test_split():
+    config = DataConfig(train_split=0.8, validation_split=0.1)
+
+    assert config.train_split == 0.8
+    assert config.validation_split == 0.1
 
 
 def test_load_config_reads_bpe_tokenizer_settings(tmp_path):
@@ -86,6 +94,8 @@ def test_data_config_rejects_invalid_bpe_vocab_size():
     [
         (lambda: DataConfig(block_size=0), "block_size"),
         (lambda: DataConfig(train_split=1.0), "train_split"),
+        (lambda: DataConfig(train_split=0.9, validation_split=0.1), "validation_split"),
+        (lambda: DataConfig(validation_split=True), "validation_split"),
         (
             lambda: DataConfig(tokenizer_type="bpe", bpe_vocab_size=10, bpe_min_frequency=0),
             "min_frequency",
diff --git a/tests/test_corpus_preparation.py b/tests/test_corpus_preparation.py
index 5843e54..e06b2be 100644
--- a/tests/test_corpus_preparation.py
+++ b/tests/test_corpus_preparation.py
@@ -98,6 +98,21 @@ def test_corpus_stats_json_round_trip(tmp_path):
     assert loaded["unique_characters"] == 2
 
 
+def test_corpus_stats_records_three_way_chronological_split(tmp_path):
+    stats = corpus_stats(
+        "abcdefghij",
+        train_split=0.6,
+        validation_split=0.2,
+        source_name="source",
+        source_note=None,
+        output_path=tmp_path / "corpus.txt",
+    )
+
+    assert stats["train_characters"] == 6
+    assert stats["validation_characters"] == 2
+    assert stats["test_characters"] == 2
+
+
 def test_file_sha256_hashes_file_contents(tmp_path):
     path = tmp_path / "tiny.txt"
     path.write_text("abc\n", encoding="utf-8")
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
index eca2e6d..a8b4a08 100644
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -1,7 +1,7 @@
 import pytest
 import torch
 
-from smallm.data import TokenBlockDataset, split_tokens
+from smallm.data import TokenBlockDataset, split_corpus_text, split_tokens
 
 
 def test_token_block_dataset_returns_shifted_blocks():
@@ -20,6 +20,15 @@ def test_split_tokens_uses_train_fraction():
     assert val.tolist() == [4, 5]
 
 
+def test_split_corpus_text_preserves_legacy_and_sealed_contracts():
+    assert split_corpus_text("abcdefghij", train_split=0.6) == ("abcdef", "ghij", "")
+    assert split_corpus_text("abcdefghij", train_split=0.6, validation_split=0.2) == (
+        "abcdef",
+        "gh",
+        "ij",
+    )
+
+
 def test_dataset_and_split_reject_invalid_boundaries():
     with pytest.raises(ValueError, match="positive"):
         TokenBlockDataset([1, 2], block_size=0)
@@ -27,3 +36,5 @@ def test_dataset_and_split_reject_invalid_boundaries():
         TokenBlockDataset([1, 2], block_size=2)
     with pytest.raises(ValueError, match="between"):
         split_tokens([1, 2], 0.0)
+    with pytest.raises(ValueError, match="test fraction"):
+        split_corpus_text("text", train_split=0.8, validation_split=0.2)
diff --git a/tests/test_sealed_test.py b/tests/test_sealed_test.py
new file mode 100644
index 0000000..d8b9f4d
--- /dev/null
+++ b/tests/test_sealed_test.py
@@ -0,0 +1,82 @@
+import json
+
+import pytest
+
+from smallm.config import DataConfig, ExperimentConfig, ModelConfig, TrainConfig
+from smallm.data.corpus import file_sha256
+from smallm.evaluation import evaluate_sealed_test
+from smallm.training.trainer import train
+
+
+def test_sealed_test_is_unscored_during_training_and_evaluated_once(tmp_path):
+    prepared_path = tmp_path / "corpus.txt"
+    manifest_path = tmp_path / "corpus_manifest.json"
+    text = "Once upon a time.\n" * 20
+    prepared_path.write_text(text, encoding="utf-8")
+    train_end = int(len(text) * 0.6)
+    validation_end = int(len(text) * 0.8)
+    manifest_path.write_text(
+        json.dumps(
+            {
+                "source_name": "sealed fixture",
+                "prepared_sha256": file_sha256(prepared_path),
+                "prepared_characters": len(text),
+                "train_split": 0.6,
+                "validation_split": 0.2,
+                "train_characters": train_end,
+                "validation_characters": validation_end - train_end,
+                "test_characters": len(text) - validation_end,
+            }
+        ),
+        encoding="utf-8",
+    )
+    config = ExperimentConfig(
+        data=DataConfig(
+            prepared_path=str(prepared_path),
+            manifest_path=str(manifest_path),
+            tokenizer_path=str(tmp_path / "tokenizer.json"),
+            block_size=4,
+            train_split=0.6,
+            validation_split=0.2,
+        ),
+        model=ModelConfig(vocab_size=256, block_size=4, n_layer=1, n_head=1, n_embd=8),
+        train=TrainConfig(
+            run_name="sealed",
+            runs_dir=str(tmp_path / "runs"),
+            batch_size=2,
+            max_steps=2,
+            log_interval=1,
+            eval_interval=1,
+            eval_batches=None,
+            sample_max_new_tokens=1,
+            seed=1337,
+            sample_seed=1337,
+        ),
+    )
+
+    run_dir = train(config).parent
+    summary = json.loads((run_dir / "summary.json").read_text(encoding="utf-8"))
+
+    assert summary["test_status"] == "sealed_unread"
+    assert summary["test_characters"] == len(text) - validation_end
+    assert "test_tokens" not in summary
+    assert "test_loss" not in summary
+
+    summary_path = run_dir / "summary.json"
+    invalid_summary = {**summary, "test_characters": summary["test_characters"] + 1}
+    summary_path.write_text(json.dumps(invalid_summary), encoding="utf-8")
+    with pytest.raises(ValueError, match="does not match the sealed test split"):
+        evaluate_sealed_test(run_dir)
+    summary_path.write_text(json.dumps(summary), encoding="utf-8")
+
+    output_path, evaluation = evaluate_sealed_test(run_dir)
+    assert output_path == run_dir / "test_evaluation_best.json"
+    assert evaluation["checkpoint_kind"] == "best"
+    assert evaluation["checkpoint_step"] == summary["best_val_step"]
+    assert evaluation["test_characters"] == len(text) - validation_end
+    assert evaluation["test_coverage"] == 1.0
+    assert evaluation["evaluation_mode"] == "full"
+    assert evaluation["test_bits_per_character"] > 0
+
+    with pytest.raises(FileExistsError, match="already exists"):
+        evaluate_sealed_test(run_dir)
diff --git a/tests/test_training_artifacts.py b/tests/test_training_artifacts.py
index 78bf1b6..4c0a478 100644
--- a/tests/test_training_artifacts.py
+++ b/tests/test_training_artifacts.py
@@ -74,7 +74,9 @@ def test_dataset_manifest_copy_and_summary_fields(tmp_path):
         "unique_characters": 4,
         "train_split": 0.75,
         "train_characters": 6,
+        "validation_split": None,
         "validation_characters": 2,
+        "test_characters": 0,
         "normalization_rules": ["rule"],
         "extra": "ignored",
     }
@@ -95,7 +97,9 @@ def test_dataset_manifest_copy_and_summary_fields(tmp_path):
         "unique_characters": 4,
         "train_split": 0.75,
         "train_characters": 6,
+        "validation_split": None,
         "validation_characters": 2,
+        "test_characters": 0,
         "normalization_rules": ["rule"],
         "manifest_path": str(copied_path),
     }
@@ -106,6 +110,17 @@ def test_missing_dataset_manifest_has_actionable_error(tmp_path):
         load_dataset_manifest(tmp_path / "missing.json")
 
 
+def test_dataset_manifest_must_be_a_bounded_json_object(tmp_path):
+    manifest_path = tmp_path / "manifest.json"
+    manifest_path.write_text("[]", encoding="utf-8")
+    with pytest.raises(ValueError, match="JSON object"):
+        load_dataset_manifest(manifest_path)
+
+    manifest_path.write_bytes(b" " * 1_000_001)
+    with pytest.raises(ValueError, match="exceeds 1000000 bytes"):
+        load_dataset_manifest(manifest_path)
+
+
 def test_verify_dataset_manifest_rejects_stale_metadata(tmp_path):
     prepared = tmp_path / "corpus.txt"
     text = "abcdefghij"
@@ -137,6 +152,37 @@ def test_verify_dataset_manifest_rejects_stale_metadata(tmp_path):
             )
 
 
+def test_verify_dataset_manifest_accepts_and_checks_sealed_split(tmp_path):
+    prepared = tmp_path / "corpus.txt"
+    text = "abcdefghij"
+    prepared.write_text(text, encoding="utf-8")
+    valid = {
+        "prepared_sha256": file_sha256(prepared),
+        "prepared_characters": 10,
+        "train_split": 0.6,
+        "validation_split": 0.2,
+        "train_characters": 6,
+        "validation_characters": 2,
+        "test_characters": 2,
+    }
+
+    verify_dataset_manifest(
+        valid,
+        prepared_path=prepared,
+        prepared_text=text,
+        train_split=0.6,
+        validation_split=0.2,
+    )
+    with pytest.raises(ValueError, match="validation split"):
+        verify_dataset_manifest(
+            valid,
+            prepared_path=prepared,
+            prepared_text=text,
+            train_split=0.6,
+            validation_split=0.1,
+        )
+
+
 def test_checkpoint_round_trip_and_schema_validation(tmp_path):
     path = tmp_path / "checkpoint.pt"
     payload = {