Feature/add delta action mode

README.md

@@ -30,6 +30,7 @@ Given any model that produces latent action representations (LAMs or visual enco
 ---
 
 ## News
+- **[2026-06-22]** We release new action mode called delta-action where difference between first and last action is calculated in a physical sense along with new evaluation metrics which also have physical meaning. 
 - **[2026-06-10]** LARYBench now supports V-JEPA 2.1 and simplify the way to add new custom models. We welcome all kinds of models evaluating on LARYBench and contributing to our leaderboards!
 - **[2026-05-01]** LARYBench now supports SigLIP2, relative-action regression evaluation (`target = action_tgt - action_src`), and a fast dataset integrity checker. Happy Labor Day!
 - **[2026-04-27]** We have open-sourced all datasets on [HuggingFace](https://huggingface.co/datasets/meituan-longcat/LARYBench).
@@ -223,11 +224,13 @@ python -m lary.cli extract \
 # Step 2: train the regression probe.
 # Uses CUDA_VISIBLE_DEVICES; one visible GPU means single-card training, multiple visible GPUs use accelerate.
 # Keep --dataset and --stride consistent with the extracted CSV names.
+# Available --action-mode includes absolute | relative | delta
 CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python -m lary.cli regress \
   --model dinov2 \
   --dataset calvin \
   --stride 5 \
   --model-type mlp
+  --action-mode absolute
 ```
 
 The extraction step writes latent-action `.npz` files under `$LARY_LA_DIR` and CSVs such as `data/train_la_calvin_5_dinov2.csv`. Regression logs and metrics are written under `$LARY_LOG_DIR/regression/`.
@@ -272,13 +275,14 @@ python -m lary.cli classify \
 
 Classification outputs are written under `$LARY_LOG_DIR/classification/`.
 
-## Relative-Action Regression
+## Relative/Delta-Action Regression
 
-Absolute regression predicts the absolute action chunk. Relative regression predicts relative motion between two frames. Generate non-overwriting relative-action files first:
+Absolute regression predicts the absolute action chunk. Relative regression predicts relative motion between two frames. Delta regression predicts the difference between the 1st and last frames in physics sense. Generate non-overwriting relative-action/delta-action files first designated by action-mode options:
 
 ```bash
 python utils/prepare_relative_actions.py \
   --dataset calvin \
+  --cation-mode [relative | delta ] \
   --input-root $DATA_DIR \
   --output-root $DATA_DIR \
   --csv data/train_la_calvin_5_dinov2.csv \

classification/evals/video_classification_frozen/eval.py

@@ -27,6 +27,7 @@
 import wandb 
 from utils.model_utils import print_model_params
 
+
 # --- Configuration & Setup ---
 logging.basicConfig()
 logger = logging.getLogger()

get_latent_action/models/base.py

@@ -8,7 +8,8 @@
 import torchvision.transforms.functional as F
 
 from registry import MODEL
-from get_latent_action.utils import print_model_params, freeze_backbone, load_video_frames, read_video_tensor
+from get_latent_action.utils import freeze_backbone, load_video_frames, read_video_tensor
+from utils.model_utils import print_model_params
 model_dir = os.environ.get("MODEL_DIR")
 
 

get_latent_action/utils.py

@@ -3,71 +3,6 @@
 import numpy as np
 from torchvision.io import read_video
 import torchvision.transforms.functional as F
-from prettytable import PrettyTable
-
-
-def format_params(num_params):
-    """Convert parameter count to human-readable format (M or B)"""
-    if num_params >= 1e9:
-        return f"{num_params / 1e9:.2f}B"
-    elif num_params >= 1e6:
-        return f"{num_params / 1e6:.2f}M"
-    else:
-        return f"{int(num_params)}"
-
-
-def print_model_params(model):
-    """Print model parameters in table format
-
-    Args:
-        model: Model instance, automatically iterates through first-level submodules
-    """
-    total_params = sum(p.numel() for p in model.parameters())
-    total_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
-    total_frozen = total_params - total_trainable
-    trainable_ratio = (total_trainable / total_params * 100) if total_params > 0 else 0
-
-    table = PrettyTable()
-    table.title = "Model Parameters Summary"
-    table.field_names = ["Component", "Total", "Trainable", "Frozen", "Trainable %"]
-
-    for name, module in model.named_children():
-        module_params = sum(p.numel() for p in module.parameters())
-        module_trainable = sum(
-            p.numel() for p in module.parameters() if p.requires_grad
-        )
-        module_frozen = module_params - module_trainable
-        module_ratio = (
-            (module_trainable / module_params * 100) if module_params > 0 else 0
-        )
-
-        table.add_row(
-            [
-                name,
-                format_params(module_params),
-                format_params(module_trainable),
-                format_params(module_frozen),
-                f"{module_ratio:.2f}%",
-            ]
-        )
-
-    table.add_row(
-        [
-            "Total",
-            format_params(total_params),
-            format_params(total_trainable),
-            format_params(total_frozen),
-            f"{trainable_ratio:.2f}%",
-        ]
-    )
-
-    table.align["Component"] = "l"
-    table.align["Total"] = "r"
-    table.align["Trainable"] = "r"
-    table.align["Frozen"] = "r"
-    table.align["Trainable %"] = "r"
-
-    print(f"\n{table}\n")
 
 
 def disabled_train(self, mode=True):

lary/cli.py

@@ -100,8 +100,8 @@ def create_parser() -> argparse.ArgumentParser:
                                 help="Stride for frame sampling")
     regress_parser.add_argument("--model-type", type=str, required=True,
                                 help="Model type for regression")
-    regress_parser.add_argument("--action-mode", type=str, default="absolute", choices=["absolute", "relative"],
-                                help="Regression target: full absolute action chunk or relative last-minus-first action.")
+    regress_parser.add_argument("--action-mode", type=str, default="absolute", choices=["absolute", "relative", "delta"],
+                                help="Regression target: full absolute action chunk; relative last-minus-first action; delta last-minus-first action with physical meaning")
     regress_parser.add_argument("--action-data-root", type=str, default=None,
                                 help="Optional LARYBench data root. Relative mode reads regression_relative/<dataset>/ under this root.")
     regress_parser.add_argument("--batch-size", type=int, default=256)
@@ -344,9 +344,9 @@ def find_free_port(start=11325, end=11425):
     # Build devices list
     devices = [f"cuda:{g}" for g in args.gpus.split(",")]
 
-    project_root = Path(__file__).resolve().parents[1]
-    train_csv = str(project_root / "data" / f"train_la_{args.dataset}_{args.model}.csv")
-    val_csv = str(project_root / "data" / f"val_la_{args.dataset}_{args.model}.csv")
+    project_root = str(config.paths.project_root)
+    train_csv = str(config.paths.project_root / "data" / f"train_la_{args.dataset}_{args.model}.csv")
+    val_csv = str(config.paths.project_root / "data" / f"val_la_{args.dataset}_{args.model}.csv")
     if not args.skip_preflight:
         preflight_latent_actions([train_csv, val_csv])
 
@@ -386,13 +386,22 @@ def find_free_port(start=11325, end=11425):
 }
 
 
-def _relative_stats_path(action_data_root: str, dataset: str) -> str:
+def _action_stats_path(action_data_root: str, dataset: str, action_mode: str) -> str:
     root = os.path.normpath(action_data_root)
-    if os.path.basename(root) != "regression_relative":
-        root = os.path.join(root, "regression_relative")
+    leaf = "regression_delta" if action_mode == "delta" else "regression_relative"
+    if os.path.basename(root) != leaf:
+        root = os.path.join(root, leaf)
     dataset_key = dataset.lower()
     stats_dir = os.path.join(root, _REGRESSION_DATA_SUBDIR.get(dataset_key, dataset_key))
-    return os.path.join(stats_dir, f"relative_action_stats_{dataset_key}.json")
+    return os.path.join(stats_dir, f"{action_mode}_action_stats_{dataset_key}.json")
+
+
+def _relative_stats_path(action_data_root: str, dataset: str) -> str:
+    return _action_stats_path(action_data_root, dataset, "relative")
+
+
+def _delta_stats_path(action_data_root: str, dataset: str) -> str:
+    return _action_stats_path(action_data_root, dataset, "delta")
 
 
 def run_regress(args) -> None:
@@ -438,6 +447,10 @@ def run_regress(args) -> None:
         action_root = args.action_data_root or os.environ.get("DATA_DIR")
         if action_root:
             global_stats_json = _relative_stats_path(action_root, dataset)
+    if not global_stats_json and args.action_mode == "delta":
+        action_root = args.action_data_root or os.environ.get("DATA_DIR")
+        if action_root:
+            global_stats_json = _delta_stats_path(action_root, dataset)
     if not global_stats_json and dataset in _STATS_JSON_NAME:
         data_root = get_data_root(dataset, 'seen_train')
         if data_root:

regression/action_chunk.py

@@ -0,0 +1,188 @@
+from dataclasses import dataclass
+from typing import Sequence
+
+import numpy as np
+from numpy.typing import NDArray
+
+from regression.pose import EndPose
+
+
+@dataclass(frozen=True)
+class ArmSpec:
+    position: slice
+    rotation: slice
+    rotation_type: str
+    rotation_order: str | None = None
+
+
+DATASET_ARM_SPECS = {
+    "calvin": [ArmSpec(slice(0, 3), slice(3, 6), "euler", "xyz")],
+    "vlabench": [ArmSpec(slice(0, 3), slice(3, 6), "euler", "xyz")],
+    "vlabench_15": [ArmSpec(slice(0, 3), slice(3, 6), "euler", "xyz")],
+    "vlabench_30": [ArmSpec(slice(0, 3), slice(3, 6), "euler", "xyz")],
+    "agibotbeta": [
+        ArmSpec(slice(0, 3), slice(6, 10), "quat", "xyzw"),
+        ArmSpec(slice(3, 6), slice(10, 14), "quat", "xyzw"),
+    ],
+    "robocoin": [
+        ArmSpec(slice(0, 3), slice(3, 6), "euler", "xyz"),
+        ArmSpec(slice(6, 9), slice(9, 12), "euler", "xyz"),
+    ],
+}
+
+
+DATASET_GRIPPER_SPECS = {
+    "calvin": [slice(6, 7)],
+    "vlabench": [slice(6, 7)],
+    "vlabench_15": [slice(6, 7)],
+    "vlabench_30": [slice(6, 7)],
+    "agibotbeta": [slice(14, 16)],
+}
+
+
+class EndActionChunk:
+    """A chunk of absolute end-effector actions with SE(3) relative conversion."""
+
+    def __init__(
+        self,
+        actions: NDArray[np.float64],
+        arm_specs: Sequence[ArmSpec],
+        action_dim: int,
+    ) -> None:
+        actions = np.asarray(actions, dtype=np.float64)
+        if actions.ndim != 2:
+            raise ValueError(f"actions must have shape (T, D), got {actions.shape}.")
+        if actions.shape[1] != action_dim:
+            raise ValueError(f"actions width must be action_dim={action_dim}, got {actions.shape[1]}.")
+        self.actions = actions
+        self.arm_specs = list(arm_specs)
+        self.action_dim = action_dim
+        self.gripper_specs: list[slice] = []
+
+    @classmethod
+    def from_array(cls, array: NDArray[np.float64], dataset_name: str, action_dim: int) -> "EndActionChunk":
+        flat = np.asarray(array, dtype=np.float64).reshape(-1)
+        if flat.size % action_dim != 0:
+            raise ValueError(f"Action size {flat.size} is not divisible by action_dim={action_dim}.")
+        dataset_key = dataset_name.lower()
+        if dataset_key not in DATASET_ARM_SPECS:
+            raise ValueError(f"Delta action mode does not know the pose layout for dataset: {dataset_name}.")
+        chunk = cls(flat.reshape(-1, action_dim), DATASET_ARM_SPECS[dataset_key], action_dim)
+        chunk.gripper_specs = DATASET_GRIPPER_SPECS.get(dataset_key, [])
+        return chunk
+
+    @property
+    def poses(self) -> list[list[EndPose]]:
+        return [
+            [
+                EndPose(
+                    translation=row[spec.position],
+                    rotation=row[spec.rotation],
+                    rotation_type=spec.rotation_type,
+                    rotation_order=spec.rotation_order,
+                    degrees=False,
+                )
+                for spec in self.arm_specs
+            ]
+            for row in self.actions
+        ]
+
+    @property
+    def relative_action(self) -> NDArray[np.float32]:
+        """Return a full chunk where each end-effector pose is relative to the first timestep."""
+        reference_poses = self.poses[0]
+        relative = self.actions.copy()
+
+        for t, current_poses in enumerate(self.poses):
+            for spec, current_pose, reference_pose in zip(self.arm_specs, current_poses, reference_poses):
+                relative_pose = current_pose - reference_pose
+                relative[t, spec.position] = relative_pose.translation
+                relative[t, spec.rotation] = self._rotation_vector(relative_pose, spec)
+
+        for gripper_slice in self.gripper_specs:
+            relative[:, gripper_slice] = self.actions[:, gripper_slice] - self.actions[0, gripper_slice]
+
+        return relative.astype(np.float32, copy=False).reshape(-1)
+
+    @staticmethod
+    def _rotation_vector(pose: EndPose, spec: ArmSpec) -> NDArray[np.float64]:
+        if spec.rotation_type == "quat":
+            return pose.to_vector("quat", spec.rotation_order)[3:]
+        if spec.rotation_type == "euler":
+            return pose.to_vector("euler", spec.rotation_order)[3:]
+        if spec.rotation_type == "rotvec":
+            return pose.rotvec
+        raise ValueError(f"Unsupported output rotation type: {spec.rotation_type}.")
+
+
+def to_delta_action_target(action: NDArray[np.float64], dataset_name: str, action_dim: int) -> NDArray[np.float32]:
+    return EndActionChunk.from_array(action, dataset_name, action_dim).relative_action.reshape(-1, action_dim)[-1]
+
+
+def delta_to_absolute_last_action(
+    first_action: NDArray[np.float64],
+    delta_action: NDArray[np.float64],
+    dataset_name: str,
+    action_dim: int,
+) -> NDArray[np.float32]:
+    """Compose a predicted final relative end-pose with the first absolute action."""
+    first = np.asarray(first_action, dtype=np.float64).reshape(action_dim)
+    delta = np.asarray(delta_action, dtype=np.float64).reshape(action_dim)
+    dataset_key = dataset_name.lower()
+    if dataset_key not in DATASET_ARM_SPECS:
+        raise ValueError(f"Delta action mode does not know the pose layout for dataset: {dataset_name}.")
+
+    absolute = first.copy()
+    for spec in DATASET_ARM_SPECS[dataset_key]:
+        reference_pose = EndPose(
+            translation=first[spec.position],
+            rotation=first[spec.rotation],
+            rotation_type=spec.rotation_type,
+            rotation_order=spec.rotation_order,
+            degrees=False,
+        )
+        relative_pose = EndPose(
+            translation=delta[spec.position],
+            rotation=delta[spec.rotation],
+            rotation_type=spec.rotation_type,
+            rotation_order=spec.rotation_order,
+            degrees=False,
+        )
+        absolute_pose = EndPose(homogeneous=reference_pose.homogeneous @ relative_pose.homogeneous)
+        absolute[spec.position] = absolute_pose.translation
+        absolute[spec.rotation] = EndActionChunk._rotation_vector(absolute_pose, spec)
+
+    for gripper_slice in DATASET_GRIPPER_SPECS.get(dataset_key, []):
+        absolute[gripper_slice] = first[gripper_slice] + delta[gripper_slice]
+
+    return absolute.astype(np.float32, copy=False)
+
+
+def canonicalize_absolute_action(
+    action: NDArray[np.float64],
+    dataset_name: str,
+    action_dim: int,
+    reference_action: NDArray[np.float64] | None = None,
+) -> NDArray[np.float32]:
+    """Return an absolute action vector in the same pose representation used by delta composition."""
+    absolute = np.asarray(action, dtype=np.float64).reshape(action_dim).copy()
+    dataset_key = dataset_name.lower()
+    if dataset_key not in DATASET_ARM_SPECS:
+        return absolute.astype(np.float32, copy=False)
+
+    reference = None if reference_action is None else np.asarray(reference_action, dtype=np.float64).reshape(action_dim)
+    for spec in DATASET_ARM_SPECS[dataset_key]:
+        pose = EndPose(
+            translation=absolute[spec.position],
+            rotation=absolute[spec.rotation],
+            rotation_type=spec.rotation_type,
+            rotation_order=spec.rotation_order,
+            degrees=False,
+        )
+        absolute[spec.position] = pose.translation
+        absolute[spec.rotation] = EndActionChunk._rotation_vector(pose, spec)
+        if spec.rotation_type == "quat" and reference is not None:
+            if np.dot(absolute[spec.rotation], reference[spec.rotation]) < 0:
+                absolute[spec.rotation] *= -1.0
+
+    return absolute.astype(np.float32, copy=False)