Zero-copy joint access from Python

cmake/GenerateFlatBuffers.cmake

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
 # CMake function to generate C++ headers from FlatBuffer schema files.
@@ -51,12 +51,16 @@ function(generate_flatbuffer_headers OUT_VAR INPUT_DIR OUTPUT_DIR)
               --gen-mutable
               --schema
               --bfbs-gen-embed
+              # Mini-reflect type tables with field names. Supersedes --reflect-types, which
+              # sets the same flatc option to a names-less value.
               --reflect-names
-              --reflect-types
               -I ${INPUT_DIR}
               -o ${OUTPUT_DIR}
               ${INPUT_DIR}/${SCHEMA_FILE}
-      DEPENDS ${INPUT_DIR}/${SCHEMA_FILE} flatc
+      # Depend on this file too: the flatc flags live here, and generators that compare
+      # timestamps rather than command lines would otherwise keep stale headers after a
+      # flag change.
+      DEPENDS ${INPUT_DIR}/${SCHEMA_FILE} flatc ${CMAKE_CURRENT_FUNCTION_LIST_FILE}
       COMMENT "Generating FlatBuffers C++ for ${SCHEMA_FILE}"
       VERBATIM
     )

src/core/deviceio_trackers/python/tracker_bindings.cpp

@@ -19,10 +19,46 @@
 
 #include <array>
 #include <cstring>
+#include <memory>
 #include <stdexcept>
 
 namespace py = pybind11;
 
+namespace
+{
+
+// Hand a tracker's tracked snapshot to Python without cloning its payload.
+//
+// Every Tracked wrapper holds its payload behind a shared_ptr, but flatc gives each generated
+// -T a deep-copying copy constructor, so returning one by value clones the whole payload on
+// every call: for a hand that is two allocations and 936 bytes of joints, twice per frame.
+// Copying the shared_ptr instead costs one small wrapper allocation and a refcount bump, which
+// is also what the .data property on these wrappers has always done.
+//
+// The payload is therefore live tracker storage. Most impls refill it in place on the next
+// session.update(), so the returned object is a view valid until that call, not a snapshot --
+// see the note repeated on each accessor below.
+template <typename TrackedT>
+std::shared_ptr<TrackedT> share_tracked(const TrackedT& tracked)
+{
+    // Only `data` is carried over, so a Tracked wrapper that grows a second field (the parallel
+    // Record wrappers already carry a timestamp) would be silently dropped here with nothing to
+    // flag it. NativeTable is empty, so a one-field wrapper is exactly as wide as its member.
+    static_assert(sizeof(TrackedT) == sizeof(decltype(TrackedT::data)),
+                  "Tracked wrapper has fields beyond .data; share_tracked() would drop them");
+    auto shared = std::make_shared<TrackedT>();
+    shared->data = tracked.data;
+    return shared;
+}
+
+} // namespace
+
+// Appended to the docstring of every accessor that returns a Tracked wrapper. A macro so it
+// concatenates with the surrounding literal at compile time; pybind11 takes a const char*.
+#define TRACKED_LIFETIME_DOC                                                                                           \
+    "\n\nThe returned wrapper shares the tracker's storage rather than copying it: its contents are "                  \
+    "refilled by the next session.update(). Read what you need before that call, or copy it."
+
 PYBIND11_MODULE(_deviceio_trackers, m)
 {
     // Load schema pybind converters (TrackedT / schema types) before exposing tracker accessors.
@@ -38,35 +74,38 @@ PYBIND11_MODULE(_deviceio_trackers, m)
         .def(py::init<>())
         .def(
             "get_left_hand",
-            [](const core::HandTracker& self, const core::ITrackerSession& session) -> core::HandPoseTrackedT
-            { return self.get_left_hand(session); },
-            py::arg("session"))
+            [](const core::HandTracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_left_hand(session)); },
+            // No lifetime note on either hand: like full body, the impls install freshly allocated
+            // joint storage each frame rather than refilling it, so the result is a snapshot of the
+            // frame it was read on. Call again to see newer hand data.
+            py::arg("session"), "Get the left hand tracked state (data is None if inactive)")
         .def(
             "get_right_hand",
-            [](const core::HandTracker& self, const core::ITrackerSession& session) -> core::HandPoseTrackedT
-            { return self.get_right_hand(session); },
-            py::arg("session"));
+            [](const core::HandTracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_right_hand(session)); },
+            py::arg("session"), "Get the right hand tracked state (data is None if inactive)");
 
     py::class_<core::HeadTracker, core::ITracker, std::shared_ptr<core::HeadTracker>>(m, "HeadTracker")
         .def(py::init<>())
         .def(
             "get_head",
-            [](const core::HeadTracker& self, const core::ITrackerSession& session) -> core::HeadPoseTrackedT
-            { return self.get_head(session); },
-            py::arg("session"));
+            [](const core::HeadTracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_head(session)); },
+            py::arg("session"), "Get the head tracked state (data is None if inactive)" TRACKED_LIFETIME_DOC);
 
     py::class_<core::ControllerTracker, core::ITracker, std::shared_ptr<core::ControllerTracker>>(m, "ControllerTracker")
         .def(py::init<>())
         .def(
             "get_left_controller",
-            [](const core::ControllerTracker& self, const core::ITrackerSession& session) -> core::ControllerSnapshotTrackedT
-            { return self.get_left_controller(session); },
-            py::arg("session"), "Get the left controller tracked state (data is None if inactive)")
+            [](const core::ControllerTracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_left_controller(session)); },
+            py::arg("session"), "Get the left controller tracked state (data is None if inactive)" TRACKED_LIFETIME_DOC)
         .def(
             "get_right_controller",
-            [](const core::ControllerTracker& self, const core::ITrackerSession& session) -> core::ControllerSnapshotTrackedT
-            { return self.get_right_controller(session); },
-            py::arg("session"), "Get the right controller tracked state (data is None if inactive)")
+            [](const core::ControllerTracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_right_controller(session)); },
+            py::arg("session"), "Get the right controller tracked state (data is None if inactive)" TRACKED_LIFETIME_DOC)
         .def(
             "apply_left_haptic_feedback",
             [](const core::ControllerTracker& self, const core::ITrackerSession& session, float amplitude,
@@ -111,9 +150,10 @@ PYBIND11_MODULE(_deviceio_trackers, m)
              "Construct a MessageChannelTracker for XR_NV_opaque_data_channel")
         .def(
             "get_messages",
-            [](const core::MessageChannelTracker& self,
-               const core::ITrackerSession& session) -> core::MessageChannelMessagesTrackedT
-            { return self.get_messages(session); },
+            [](const core::MessageChannelTracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_messages(session)); },
+            // No lifetime note here: this payload is a vector, so share_tracked() copies the list
+            // itself, and every drained message is freshly allocated. The result is a snapshot.
             py::arg("session"), "Get all messages drained during the last update (possibly empty)")
         .def(
             "get_status",
@@ -134,11 +174,11 @@ PYBIND11_MODULE(_deviceio_trackers, m)
              "Construct a multi-stream FrameMetadataTrackerOak")
         .def(
             "get_stream_data",
-            [](const core::FrameMetadataTrackerOak& self, const core::ITrackerSession& session,
-               size_t stream_index) -> core::FrameMetadataOakTrackedT
-            { return self.get_stream_data(session, stream_index); },
+            [](const core::FrameMetadataTrackerOak& self, const core::ITrackerSession& session, size_t stream_index)
+            { return share_tracked(self.get_stream_data(session, stream_index)); },
             py::arg("session"), py::arg("stream_index"),
-            "Get FrameMetadataOakTrackedT for a specific stream by index; .data is None until first frame arrives")
+            "Get FrameMetadataOakTrackedT for a specific stream by index; .data is None until first frame "
+            "arrives" TRACKED_LIFETIME_DOC)
         .def_property_readonly("stream_count", &core::FrameMetadataTrackerOak::get_stream_count,
                                "Number of streams this tracker is configured for");
 
@@ -149,10 +189,10 @@ PYBIND11_MODULE(_deviceio_trackers, m)
              "Construct a Generic3AxisPedalTracker for the given tensor collection ID")
         .def(
             "get_pedal_data",
-            [](const core::Generic3AxisPedalTracker& self,
-               const core::ITrackerSession& session) -> core::Generic3AxisPedalOutputTrackedT
-            { return self.get_data(session); },
-            py::arg("session"), "Get the current foot pedal tracked state (data is None when no data available)");
+            [](const core::Generic3AxisPedalTracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_data(session)); },
+            py::arg("session"),
+            "Get the current foot pedal tracked state (data is None when no data available)" TRACKED_LIFETIME_DOC);
 
     py::class_<core::OgloTactileTracker, core::ITracker, std::shared_ptr<core::OgloTactileTracker>>(
         m, "OgloTactileTracker")
@@ -162,9 +202,10 @@ PYBIND11_MODULE(_deviceio_trackers, m)
              "(e.g. 'oglo/left' / 'oglo/right', matching the oglo_tactile plugin's --collection-prefix)")
         .def(
             "get_glove_data",
-            [](const core::OgloTactileTracker& self, const core::ITrackerSession& session) -> core::OgloGloveSampleTrackedT
-            { return self.get_data(session); },
-            py::arg("session"), "Get the current tactile glove tracked state (data is None when no data available)");
+            [](const core::OgloTactileTracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_data(session)); },
+            py::arg("session"),
+            "Get the current tactile glove tracked state (data is None when no data available)" TRACKED_LIFETIME_DOC);
 
     py::class_<core::TensorPushTracker, core::ITracker, std::shared_ptr<core::TensorPushTracker>> tensor_push_tracker(
         m, "TensorPushTracker");
@@ -192,9 +233,10 @@ PYBIND11_MODULE(_deviceio_trackers, m)
              "joint-space device: leader arm, exoskeleton, ...)")
         .def(
             "get_data",
-            [](const core::JointStateTracker& self, const core::ITrackerSession& session) -> core::JointStateOutputTrackedT
-            { return self.get_data(session); },
-            py::arg("session"), "Get the current joint-state tracked snapshot (data is None when no data available)");
+            [](const core::JointStateTracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_data(session)); },
+            py::arg("session"),
+            "Get the current joint-state tracked state (data is None when no data available)" TRACKED_LIFETIME_DOC);
 
     py::class_<core::Se3Tracker, core::ITracker, std::shared_ptr<core::Se3Tracker>>(m, "Se3Tracker")
         .def(py::init<const std::string&, size_t>(), py::arg("collection_id"),
@@ -203,20 +245,23 @@ PYBIND11_MODULE(_deviceio_trackers, m)
              "6-DoF pose source: tracker puck, mocap rigid body, logical tracker, ...)")
         .def(
             "get_data",
-            [](const core::Se3Tracker& self, const core::ITrackerSession& session) -> core::Se3TrackerPoseTrackedT
-            { return self.get_data(session); },
+            [](const core::Se3Tracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_data(session)); },
             py::arg("session"),
-            "Get the current SE3 tracked snapshot (data is None when no data available; gate on "
-            "data.is_valid before consuming the pose)");
+            "Get the current SE3 tracked state (data is None when no data available; gate on "
+            "data.is_valid before consuming the pose)" TRACKED_LIFETIME_DOC);
 
     py::class_<core::FullBodyTracker, core::ITracker, std::shared_ptr<core::FullBodyTracker>>(m, "FullBodyTracker")
         .def(py::init<>(),
              "Construct a vendor-agnostic full body tracker marker. The live session selects the "
              "vendor via VendorConfig (default: native PICO XR_BD_body_tracking); replay is vendor-neutral.")
         .def(
             "get_body_pose",
-            [](const core::FullBodyTracker& self, const core::ITrackerSession& session) -> core::FullBodyPoseTrackedT
-            { return self.get_body_pose(session); },
+            [](const core::FullBodyTracker& self, const core::ITrackerSession& session)
+            { return share_tracked(self.get_body_pose(session)); },
+            // No lifetime note here: the full-body impls install freshly allocated joint storage
+            // each frame rather than refilling it, so the result is a snapshot of the frame it was
+            // read on. Call again to see newer body data.
             py::arg("session"), "Get full body pose tracked state (data is None if inactive)");
 
     m.attr("NUM_JOINTS") = static_cast<int>(core::HandJoint_NUM_JOINTS);

src/core/live_trackers/cpp/live_full_body_tracker_pico_impl.cpp

@@ -130,17 +130,12 @@ void LiveFullBodyTrackerPicoImpl::update(int64_t monotonic_time_ns)
         throw std::runtime_error("[FullBodyTracker] xrLocateBodyJointsBD failed: " + std::to_string(result));
     }
 
-    if (!tracked_.data)
-    {
-        tracked_.data = std::make_shared<FullBodyPoseT>();
-    }
-
-    tracked_.data->all_joint_poses_tracked = locations.allJointPosesTracked;
-
-    if (!tracked_.data->joints)
-    {
-        tracked_.data->joints = std::make_shared<BodyJoints>();
-    }
+    // Publish freshly allocated joint storage each frame instead of refilling the previous
+    // frame's. The query API hands the pose out by reference and callers may still hold an
+    // earlier frame's joints, so an in-place refill would change data already handed out.
+    auto data = std::make_shared<FullBodyPoseT>();
+    data->all_joint_poses_tracked = locations.allJointPosesTracked;
+    data->joints = std::make_shared<BodyJoints>();
 
     for (uint32_t i = 0; i < XR_BODY_JOINT_COUNT_BD; ++i)
     {
@@ -155,9 +150,11 @@ void LiveFullBodyTrackerPicoImpl::update(int64_t monotonic_time_ns)
                         (joint_loc.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT);
 
         BodyJointPose joint_pose(pose, is_valid);
-        tracked_.data->joints->mutable_joints()->Mutate(i, joint_pose);
+        data->joints->mutable_joints()->Mutate(i, joint_pose);
     }
 
+    tracked_.data = std::move(data);
+
     if (mcap_channels_)
     {
         DeviceDataTimestamp timestamp(last_update_time_, last_update_time_, xr_time);

src/core/live_trackers/cpp/live_hand_tracker_impl.cpp

@@ -408,15 +408,11 @@ bool LiveHandTrackerImpl::try_update_hand(XrHandTrackerEXT tracker, XrTime time,
         return false;
     }
 
-    if (!tracked.data)
-    {
-        tracked.data = std::make_shared<HandPoseT>();
-    }
-
-    if (!tracked.data->joints)
-    {
-        tracked.data->joints = std::make_shared<HandJoints>();
-    }
+    // Publish freshly allocated joint storage each frame instead of refilling the previous
+    // frame's. The query API hands the pose out by reference and callers may still hold an
+    // earlier frame's joints, so an in-place refill would change data already handed out.
+    auto data = std::make_shared<HandPoseT>();
+    data->joints = std::make_shared<HandJoints>();
 
     for (uint32_t i = 0; i < XR_HAND_JOINT_COUNT_EXT; ++i)
     {
@@ -431,9 +427,10 @@ bool LiveHandTrackerImpl::try_update_hand(XrHandTrackerEXT tracker, XrTime time,
                         (joint_loc.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT);
 
         HandJointPose joint_pose(pose, is_valid, joint_loc.radius);
-        tracked.data->joints->mutable_poses()->Mutate(i, joint_pose);
+        data->joints->mutable_poses()->Mutate(i, joint_pose);
     }
 
+    tracked.data = std::move(data);
     return true;
 }
 

src/core/schema/python/CMakeLists.txt

@@ -13,6 +13,7 @@ pybind11_add_module(schema_py
     oglo_tactile_bindings.h
     pedals_bindings.h
     pose_bindings.h
+    schema_array_views.h
     se3_tracker_bindings.h
     schema_module.cpp
 )