autowarefoundation · ShauryaVM · Aug 12, 2026 · Aug 12, 2026 · Aug 19, 2026
diff --git a/Model/evaluation/QUALITY_BENCHMARKS.md b/Model/evaluation/QUALITY_BENCHMARKS.md
@@ -0,0 +1,47 @@
+# World-Model Quality Benchmarks
+
+Speed benchmarks live in [`../speed_benchmark/`](../speed_benchmark/). This folder
+documents **quality** metrics for the policy / World Model stack:
+
+| Metric family | What it measures |
+|---------------|------------------|
+| JEPA reconstruction | Per-horizon L1 / L2 / cosine of predicted vs frozen-target feature maps |
+| Null-relative improvement | How much better than predicting zeros |
+| Reactive vs Combined impact | Trajectory L1/L2 delta when the World Model is enabled |
+| Open-loop ADE/FDE pair | Reactive and Combined ADE@3s / FDE@3s on the same GT (when labels exist) |
+
+## Measured result (trained Combined, 12 steps)
+
+Source: `evaluation/results/world_model_quality_trained.json` (CPU, mock backbone, seed 0). Combined IL+JEPA loss **0.462 → 0.389**. This is **not** a KITScenes checkpoint; it is a trained (not random-init) Combined run so ADE/FDE are defined. Re-run on packed shards with `--shard-dir` / `--checkpoint`.
+
+| | Reactive | Combined | Δ (C−R) |
+|--|----------|----------|---------|
+| ADE@3s | 4.160 | **3.971** | −0.189 |
+| FDE@3s | 11.000 | **10.428** | −0.572 |
+| JEPA L1 / cosine | — | 0.0437 / 0.157 | — |
+
+JEPA relative improvement vs a zero predictor is **0** on random frames (model L1 0.044 > null 0.018). That is expected without real video; the ADE pair is the number that answers “does Combined move the plan.”
+
+## Quick start
+
+```bash
+cd Model
+
+# Train Combined a few steps, then JEPA + ADE/FDE (default)
+python evaluation/world_model_quality_benchmark.py --trained --train-steps 12
+
+# Packed KITScenes/L2D partition + optional checkpoint
+python evaluation/world_model_quality_benchmark.py \
+  --shard-dir /path/to/partition --checkpoint ckpt.pt --train-steps 0
+```
+
+## Library API
+
+```python
+from evaluation.world_model_quality import (
+    jepa_reconstruction_metrics,
+    train_world_model_quality,
+    world_model_trajectory_impact,
+    open_loop_pair_metrics,
+)
+```
diff --git a/Model/evaluation/__init__.py b/Model/evaluation/__init__.py
@@ -9,6 +9,15 @@
 from .baselines import constant_velocity_baseline, hold_last_action_baseline
 from .splits import episode_range_split, geographic_holdout_split, long_tail_split
 from .faithfulness import horizon_intervention_delta, reasoning_intervention_delta
+from .world_model_quality import (
+    jepa_reconstruction_metrics,
+    null_predictor_metrics,
+    open_loop_pair_metrics,
+    relative_jepa_improvement,
+    summarize_world_model_quality,
+    trajectory_impact_metrics,
+    world_model_trajectory_impact,
+)
 
 __all__ = [
     # existing (open-loop displacement metrics + gate)
@@ -29,4 +38,12 @@
     "long_tail_split",
     "reasoning_intervention_delta",
     "horizon_intervention_delta",
+    # world-model quality (JEPA recon + Reactive vs Combined)
+    "jepa_reconstruction_metrics",
+    "null_predictor_metrics",
+    "relative_jepa_improvement",
+    "trajectory_impact_metrics",
+    "open_loop_pair_metrics",
+    "world_model_trajectory_impact",
+    "summarize_world_model_quality",
 ]
diff --git a/Model/evaluation/results/world_model_quality_trained.json b/Model/evaluation/results/world_model_quality_trained.json
@@ -0,0 +1,64 @@
+{
+  "schema": "auto_e2e_world_model_quality_v2",
+  "timestamp": "2026-08-14T22:43:16.120026+00:00",
+  "commit": "954487b1",
+  "platform": "macOS-26.5.2-arm64-arm-64bit",
+  "torch": "2.7.1",
+  "device": "cpu",
+  "metrics": {
+    "jepa_l1@h1": 0.043927568942308426,
+    "jepa_l2@h1": 0.055345285683870316,
+    "jepa_cosine@h1": 0.17014850676059723,
+    "jepa_l1@h2": 0.04325031116604805,
+    "jepa_l2@h2": 0.054563358426094055,
+    "jepa_cosine@h2": 0.17106282711029053,
+    "jepa_l1@h3": 0.043721165508031845,
+    "jepa_l2@h3": 0.055064596235752106,
+    "jepa_cosine@h3": 0.13887304067611694,
+    "jepa_l1@h4": 0.04387863352894783,
+    "jepa_l2@h4": 0.0553506575524807,
+    "jepa_cosine@h4": 0.1485099494457245,
+    "jepa_l1": 0.04369441978633404,
+    "jepa_l2": 0.055080974474549294,
+    "jepa_cosine": 0.1571485809981823,
+    "jepa_num_horizons": 4.0,
+    "jepa_null_l1@h1": 0.01794394664466381,
+    "jepa_null_l2@h1": 0.022276561707258224,
+    "jepa_null_cosine@h1": 0.0,
+    "jepa_null_l1@h2": 0.017923133447766304,
+    "jepa_null_l2@h2": 0.02225443720817566,
+    "jepa_null_cosine@h2": 0.0,
+    "jepa_null_l1@h3": 0.017904415726661682,
+    "jepa_null_l2@h3": 0.02223062701523304,
+    "jepa_null_cosine@h3": 0.0,
+    "jepa_null_l1@h4": 0.01793241873383522,
+    "jepa_null_l2@h4": 0.02226976864039898,
+    "jepa_null_cosine@h4": 0.0,
+    "jepa_null_l1": 0.017925978638231754,
+    "jepa_null_l2": 0.022257848642766476,
+    "jepa_null_cosine": 0.0,
+    "jepa_null_num_horizons": 4.0,
+    "jepa_rel_improvement_l1": 0.0,
+    "jepa_rel_improvement_l2": 0.0,
+    "impact_trajectory_l2": 0.012440151534974575,
+    "impact_trajectory_l1": 0.011153275147080421,
+    "impact_trajectory_max_abs": 0.026419222354888916,
+    "ol_reactive_ADE@1s": 0.6214369118684915,
+    "ol_reactive_ADE@2s": 2.135651239197938,
+    "ol_reactive_ADE@3s": 4.160404140094114,
+    "ol_reactive_FDE@3s": 10.999535838011301,
+    "ol_reactive_accel_mae": 0.6576932668685913,
+    "ol_reactive_curvature_mae": 0.7201176881790161,
+    "ol_combined_ADE@1s": 0.6150613747897342,
+    "ol_combined_ADE@2s": 2.046559058723364,
+    "ol_combined_ADE@3s": 3.9711179008169704,
+    "ol_combined_FDE@3s": 10.428060847410013,
+    "ol_combined_accel_mae": 0.6588830947875977,
+    "ol_combined_curvature_mae": 0.7204720973968506,
+    "ol_ade3s_delta_combined_minus_reactive": -0.1892862392771435,
+    "train_loss_first": 0.4619852900505066,
+    "train_loss_last": 0.3890073001384735,
+    "train_steps": 12.0
+  },
+  "source": "trained_mock_backbone"
+}