feat(alpasim-integration): add alpasim driver plugin for closed-loop operation

Model/data_parsing/README.md

@@ -8,5 +8,6 @@ Dataset loaders and utilities for AutoE2E training data.
 - **`nvidia_physical_ai/`** — [NVIDIA Autonomous Vehicle dataset](https://huggingface.co/datasets/nvidia/PhysicalAI-Autonomous-Vehicles) loader
 - **`map_rendering/`** — Map tile rendering and GPS-to-map conversions
 - **`kit_scenes/`** — [KITScenes](https://kitscenes.com/multimodal/) data utilities
+- **`alpasim_stream/`** — Real-time observation stream parser for NVIDIA AlpaSim closed-loop simulation (`PredictionInput` parity with `kit_scenes`)
 
 Each module provides dataset classes (`*Dataset`) and helper functions for loading camera frames, extracting egomotion, and handling map data.

Model/data_parsing/alpasim_stream/__init__.py

@@ -0,0 +1,3 @@
+from .parser import AlpasimStreamParser, PredictionInput
+
+__all__ = ["AlpasimStreamParser", "PredictionInput"]

Model/data_parsing/alpasim_stream/parser.py

@@ -0,0 +1,97 @@
+from typing import Any, Dict, TypedDict
+import collections
+import io
+import torch
+import numpy as np
+from torchvision import transforms
+from PIL import Image
+
+_TRANSFORM = transforms.Compose([
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+])
+
+_HISTORY_STEPS = 64
+_HISTORY_SIGNALS = 4
+_VISUAL_HISTORY_DIM = 896
+
+CAMERA_NAMES = [
+    "camera_base_front_center",
+    "camera_ring_front",
+    "camera_ring_front_left",
+    "camera_ring_front_right",
+    "camera_ring_rear",
+    "camera_ring_rear_left",
+    "camera_ring_rear_right",
+]
+
+class PredictionInput(TypedDict):
+    cameras: Dict[str, Any]
+    speed: float
+    acceleration: float
+    command: int
+
+class AlpasimStreamParser:
+    """Parses live AlpaSim frames into the exact tensor format produced by pre_extracted.py."""
+    def __init__(self) -> None:
+        self._egomotion_buffer: collections.deque[list[float]] = collections.deque(maxlen=_HISTORY_STEPS)
+        for _ in range(_HISTORY_STEPS):
+            self._egomotion_buffer.append([0.0, 0.0, 0.0, 0.0])
+
+    def _decode_image(self, data: Any) -> torch.Tensor:
+        """Decode and normalize image exactly as the offline loader."""
+        if isinstance(data, bytes):
+            data = io.BytesIO(data)
+        img = Image.open(data) if isinstance(data, (str, io.BytesIO)) else data
+        if not isinstance(img, Image.Image):
+            img = Image.fromarray(img)
+        img = img.resize((256, 256), resample=Image.Resampling.BILINEAR)
+        return _TRANSFORM(img)
+
+    def parse_observation(self, observation: PredictionInput) -> Dict[str, torch.Tensor]:
+        """Convert a live PredictionInput into the pipeline's expected batch tensors.
+
+        Returns:
+            Dict containing:
+                - visual_tiles: ``[1, 7, 3, 256, 256]``
+                - egomotion_history: ``[1, 256]``
+                - visual_history: ``[1, 896]``
+                - map_context: ``[1, 3, 256, 256]``
+                - route_mask: ``[1, 2, 256, 256]``
+                - map_valid: ``[1]``
+                - route_valid: ``[1]``
+        """
+        frames = []
+        for cam_name in CAMERA_NAMES:
+            frame_data = observation["cameras"].get(cam_name)
+            if frame_data is None:
+                frames.append(torch.zeros(3, 256, 256))
+            else:
+                frames.append(self._decode_image(frame_data))
+        visual_tiles = torch.stack(frames).unsqueeze(0)
+
+        current_ego = [float(observation["speed"]), float(observation["acceleration"]), 0.0, 0.0]
+        self._egomotion_buffer.append(current_ego)
+
+        ego_history_np = np.array(self._egomotion_buffer, dtype=np.float32).flatten()
+        egomotion_history = torch.from_numpy(ego_history_np).unsqueeze(0)
+
+        visual_history = torch.zeros(1, _VISUAL_HISTORY_DIM, dtype=torch.float32)
+
+        map_context = torch.zeros(1, 3, 256, 256, dtype=torch.float32)
+        route_mask = torch.zeros(1, 2, 256, 256, dtype=torch.float32)
+        map_valid = torch.tensor([False], dtype=torch.bool)
+        route_valid = torch.tensor([False], dtype=torch.bool)
+
+        camera_params = torch.eye(4)[:3].unsqueeze(0).repeat(7, 1, 1).unsqueeze(0).to(torch.float32)
+
+        return {
+            "visual_tiles": visual_tiles,
+            "egomotion_history": egomotion_history,
+            "visual_history": visual_history,
+            "map_context": map_context,
+            "route_mask": route_mask,
+            "map_valid": map_valid,
+            "route_valid": route_valid,
+            "camera_params": camera_params,
+        }

Model/plugins/alpasim_driver/README.md

@@ -0,0 +1,143 @@
+# AutoE2E AlpaSim Driver Plugin
+
+This package provides the official **AutoE2E driver plugin** for [NVIDIA AlpaSim](https://github.com/NVlabs/alpasim), enabling real-time closed-loop evaluation and policy rollouts of the AutoE2E VLA driving model on the KitScenes 7-camera sensor topology.
+
+---
+
+## Architecture Overview
+
+The plugin connects AutoE2E directly to AlpaSim's microservices simulation loop without custom networking overhead.
+
+```mermaid
+graph TD
+    AlpaSim[AlpaSim Simulation Runtime] -->|PredictionInput: 7 RGB cams, speed, accel, command| DriverPlugin[AutoE2EDriver Plugin]
+    DriverPlugin --> Parser[AlpasimStreamParser]
+    Parser -->|Normalized Tensors| Model[AutoE2E PyTorch Model]
+    Model -->|Trajectory Waypoints + Headings| DriverPlugin
+    DriverPlugin -->|ModelPrediction: trajectory_xy, headings| AlpaSim
+```
+
+### Key Components
+
+- **`AutoE2EDriver`** ([`plugin.py`](./plugin.py)): Subclass of AlpaSim's `BaseTrajectoryModel`. Implements `from_config()`, `camera_ids`, `context_length`, `output_frequency_hz`, and `predict()`.
+- **`AutoE2EAlpaSimConfig`** ([`config.py`](./config.py)): Dataclass defining model checkpoint paths, 7-camera topology configuration, and trajectory horizon parameters.
+- **Entry Points** ([`pyproject.toml`](./pyproject.toml)): Registers `autoe2e` under entry point groups `alpasim.models` and `alpasim.configs`.
+
+---
+
+## Data Contract & Sensor Topology
+
+### Input Observations (`PredictionInput`)
+- **Visual Topology**: 7 KitScenes camera streams (`camera_base_front_center`, `camera_ring_front`, `camera_ring_front_left`, `camera_ring_front_right`, `camera_ring_rear`, `camera_ring_rear_left`, `camera_ring_rear_right`).
+- **Telemetry**: Scalar ego vehicle speed ($\text{m/s}$), acceleration ($\text{m/s}^2$), and high-level routing `DriveCommand` (LEFT, STRAIGHT, RIGHT).
+
+### Output Predictions (`ModelPrediction`)
+- **`trajectory_xy`**: Waypoint coordinates $[64, 2]$ in rig frame ($X$ forward, $Y$ left).
+- **`headings`**: Vehicle target headings $[64]$ in radians.
+
+---
+
+## Installation & Setup
+
+### 1. Install Driver & Dependencies
+
+Install the driver plugin and dataset parser in editable mode:
+
+```bash
+# 1. Install alpasim_driver plugin package
+pip install -e Model/plugins/alpasim_driver
+
+# 2. Install KITScenes SDK
+pip install -e Model/data_parsing/kit_scenes/kitscenes --no-deps
+
+# 3. Install Lanelet2 (for vector HD map parsing & BEV rasterization)
+pip install lanelet2
+```
+
+### 2. Environment Configuration
+
+Configure root directories for KITScenes dataset files and AlpaSim source repository. You can source them from `.env` or export them manually:
+
+```bash
+# Option A: Load from .env file
+set -a; source .env; set +a
+
+# Option B: Set environment variables manually
+export KITSCENES_ROOT="/path/to/auto_e2e/.KITdata"
+export ALPASIM_ROOT="/path/to/auto_e2e/.alpasim"
+```
+
+### 3. Download KITScenes Data Samples
+
+Download dataset scene archives using the `kitscenes` CLI:
+
+```bash
+python -m kitscenes.download "$KITSCENES_ROOT" --scenes c34c778f-ad8c-0aa9-7e1a-c86a73f887c7
+```
+
+---
+
+## Model Control Parameters
+
+Controls for simulation execution in [`config.py`](./config.py) and [`plugin.py`](./plugin.py):
+
+| Parameter | Type | Default | Description |
+| :--- | :--- | :--- | :--- |
+| `checkpoint_path` | `str` | `"autoe2e_model.ckpt"` | Path to pre-trained AutoE2E PyTorch checkpoint file. |
+| `allow_untrained_model` | `bool` | `False` | When `True`, initializes a fresh `AutoE2E(num_views=7)` PyTorch neural network with random weights if no checkpoint file exists on disk. |
+| `allow_mock` | `bool` | `False` | When `False` (default), strictly requires the actual AlpaSim runtime and real model execution, failing fast if dependencies are missing. |
+
+---
+
+## Plugin Discovery Verification
+
+Confirm that AlpaSim discovers the `autoe2e` plugin entry points:
+
+```python
+import alpasim_driver.plugin
+import alpasim_plugins.plugins as p
+
+print("Registered Models:", p.PluginRegistry("alpasim.models").get_names())
+print("Registered Configs:", p.PluginRegistry("alpasim.configs").get_names())
+```
+
+**Expected Output**:
+```text
+Registered Models: ['autoe2e']
+Registered Configs: ['autoe2e']
+```
+
+---
+
+## Running Closed-Loop Workflows
+
+### Workflow A: Closed-Loop Model Policy Rollouts (`run_closed_loop.py`)
+
+Executes real-time closed-loop rollouts of the `AutoE2E` PyTorch neural network model taking 7 camera streams at 10 Hz:
+
+```bash
+python Model/plugins/alpasim_driver/examples/run_closed_loop.py
+```
+
+### Workflow B: World Renderer Verification (`verify_world_renderer.py`)
+
+Drives closed-loop simulation using ground-truth trajectory predictions to evaluate and compare world renderers (AlpaSim vs NuRec vs KITScenes renderer) without policy prediction noise:
+
+```bash
+python Model/plugins/alpasim_driver/examples/verify_world_renderer.py
+```
+
+### Expected Output Example
+```text
+[INFO] Starting World Renderer Verification (Ground Truth Trajectory Driver)
+[INFO] Discovered AlpaSim Registered Models: ['autoe2e']
+[INFO] Discovered AlpaSim Registered Configs: ['autoe2e']
+[INFO] Initialized Ground Truth Driver: GroundTruthTrajectoryDriver
+[INFO] Subscribed Camera Topology (7 cameras): ['camera_base_front_center', 'camera_ring_front', 'camera_ring_front_left', 'camera_ring_front_right', 'camera_ring_rear', 'camera_ring_rear_left', 'camera_ring_rear_right']
+[INFO] Evaluating World Renderer across 50 simulation steps...
+[INFO] [Renderer Step 00/50] t= 0.0s | Ego Pos: (  0.48m,   0.00m) | Speed:  4.76 m/s | Prediction Step Time:  0.60 ms
+[INFO] [Renderer Step 49/50] t= 4.9s | Ego Pos: (  6.80m,   0.00m) | Speed:  4.76 m/s | Prediction Step Time:  0.17 ms
+[INFO] World Renderer Verification completed successfully!
+[INFO] Final Ground-Truth Position: (6.80m, 0.00m)
+[INFO] Saved visualization GIF: /path/to/verify_world_renderer.gif
+```

Model/plugins/alpasim_driver/__init__.py

@@ -0,0 +1,10 @@
+"""AlpaSim driver plugin package for AutoE2E.
+
+Registers AutoE2E model and configuration entry points with the AlpaSim simulator.
+"""
+
+from .config import AutoE2EAlpaSimConfig
+from .plugin import AutoE2EDriver, AutoE2EAlpaSimModel
+
+__all__ = ["AutoE2EAlpaSimConfig", "AutoE2EDriver", "AutoE2EAlpaSimModel"]
+

Model/plugins/alpasim_driver/alpasim_autoe2e_config.py

@@ -0,0 +1,3 @@
+from .config import AutoE2EAlpaSimConfig
+
+__all__ = ["AutoE2EAlpaSimConfig"]

Model/plugins/alpasim_driver/alpasim_autoe2e_plugin.py

@@ -0,0 +1,3 @@
+from .plugin import AutoE2EDriver, AutoE2EAlpaSimModel
+
+__all__ = ["AutoE2EDriver", "AutoE2EAlpaSimModel"]

Model/plugins/alpasim_driver/config.py

@@ -0,0 +1,48 @@
+"""Configuration dataclasses for the AutoE2E AlpaSim driver plugin.
+
+Defines model checkpoints, camera topology settings, and trajectory planning horizon settings.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Tuple
+
+
+@dataclass
+class AutoE2EAlpaSimConfig:
+    """Configuration options for ``AutoE2EAlpaSimModel`` driver plugin.
+
+    Registered with AlpaSim under entry point ``alpasim.configs``.
+    """
+
+    checkpoint_path: str
+    """Path to trained AutoE2E model checkpoint file."""
+
+    allow_mock: bool = False
+    """Whether to allow mock fallback mode when running without AlpaSim."""
+
+    allow_untrained_model: bool = False
+    """Whether to initialize an untrained AutoE2E model if model checkpoint is missing."""
+
+    image_size: Tuple[int, int] = (256, 256)
+    """Target camera resolution ``(H, W)`` expected by perception backbone."""
+
+    planning_horizon_s: float = 3.0
+    """Total future trajectory planning horizon in seconds."""
+
+    planning_steps: int = 64
+    """Number of output waypoint steps along the planning horizon."""
+
+    camera_names: List[str] = field(
+        default_factory=lambda: [
+            "camera_base_front_center",
+            "camera_ring_front",
+            "camera_ring_front_left",
+            "camera_ring_front_right",
+            "camera_ring_rear",
+            "camera_ring_rear_left",
+            "camera_ring_rear_right",
+        ]
+    )
+    """List of 7 logical camera names matching KitScenes topology."""