diff --git a/deps/cloudxr/.env.default b/deps/cloudxr/.env.default index 86ef06030..3480da163 100644 --- a/deps/cloudxr/.env.default +++ b/deps/cloudxr/.env.default @@ -6,7 +6,7 @@ # CloudXR Docker Images and Configs ########################################################### CXR_RUNTIME_SDK_VERSION=6.2.1 -CXR_WEB_SDK_VERSION=6.2.0 +CXR_WEB_SDK_VERSION=6.3.0-rc2 CXR_HOST_VOLUME_PATH=$HOME/.cloudxr ########################################################### diff --git a/deps/cloudxr/webxr_client/helpers/metricsAccumulator.test.ts b/deps/cloudxr/webxr_client/helpers/metricsAccumulator.test.ts new file mode 100644 index 000000000..8b2df4aa4 --- /dev/null +++ b/deps/cloudxr/webxr_client/helpers/metricsAccumulator.test.ts @@ -0,0 +1,146 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @jest-environment jsdom + * + * Jest only reads this pragma from the file's first comment block, hence its placement + * here. The CloudXR SDK bundle touches `window` at import time, and this suite imports it + * (transitively, via metricsAccumulator) to assert against the real MetricsName values + * rather than a mock that could drift from them. + */ + +import * as CloudXR from '@nvidia/cloudxr'; + +import { MetricsAccumulator } from './metricsAccumulator'; + +/** Finds one cadence in a snapshot list. */ +const cadence = (snapshots: ReturnType, name: string) => + snapshots.find(s => s.cadence === name); + +describe('MetricsAccumulator', () => { + it('reports nothing before any metric arrives', () => { + expect(new MetricsAccumulator().takeSnapshot()).toEqual([]); + }); + + it('merges partial updates instead of replacing them', () => { + const acc = new MetricsAccumulator(); + // The send-begin and render-begin paths each emit their own PerRender tick. + acc.recordRender({ framerate: 72 }); + acc.recordRender({ sendFramerate: 60 }); + + const render = cadence(acc.takeSnapshot(), 'render'); + expect(render?.metrics).toEqual({ + [CloudXR.MetricsName.RenderFramerate]: 72, + [CloudXR.MetricsName.PoseSendFramerate]: 60, + }); + }); + + it('overwrites a metric when a later tick carries it again', () => { + const acc = new MetricsAccumulator(); + acc.recordRender({ framerate: 72 }); + acc.recordRender({ framerate: 71 }); + + expect(cadence(acc.takeSnapshot(), 'render')?.metrics).toEqual({ + [CloudXR.MetricsName.RenderFramerate]: 71, + }); + }); + + it('ignores undefined and non-finite values rather than reporting them as zero', () => { + const acc = new MetricsAccumulator(); + acc.recordRender({ framerate: 72, sendFramerate: undefined }); + acc.recordNetwork({ rttMs: NaN, packetLoss: Infinity }); + + expect(cadence(acc.takeSnapshot(), 'render')?.metrics).toEqual({ + [CloudXR.MetricsName.RenderFramerate]: 72, + }); + // An all-invalid update leaves the cadence empty, so it is omitted entirely. + expect(cadence(acc.takeSnapshot(), 'network')).toBeUndefined(); + }); + + it('keeps a zero value, which is meaningful for loss and quality metrics', () => { + const acc = new MetricsAccumulator(); + acc.recordNetwork({ packetLoss: 0, sessionQuality: 0 }); + + expect(cadence(acc.takeSnapshot(), 'network')?.metrics).toEqual({ + [CloudXR.MetricsName.NetworkPacketLoss]: 0, + [CloudXR.MetricsName.SessionQuality]: 0, + }); + }); + + it('separates the three cadences and omits empty ones', () => { + const acc = new MetricsAccumulator(); + acc.recordRender({ framerate: 72 }); + acc.recordNetwork({ sessionQuality: 3 }); + + const snapshots = acc.takeSnapshot(); + expect(snapshots.map(s => s.cadence).sort()).toEqual(['network', 'render']); + }); + + it('reports every field of every cadence under its SDK metric name', () => { + const acc = new MetricsAccumulator(); + acc.recordRender({ framerate: 1, sendFramerate: 2, xrPoseAgeMs: 3 }); + acc.recordFrame({ + framerate: 4, + frameCount: 5, + poseToRenderTimeMs: 6, + poseUploadMs: 7, + poseToFrameReceivedMs: 8, + compositorSkippedPercent: 9, + outOfOrderPercent: 10, + mismatchedPercent: 11, + }); + acc.recordNetwork({ + streamingRateMbps: 12, + availableBandwidthMbps: 13, + rttMs: 14, + packetLoss: 15, + avgDecodeTimeMs: 16, + qualityScore: 17, + bandwidthScore: 18, + lossScore: 19, + latencyScore: 20, + sessionQuality: 21, + }); + + const snapshots = acc.takeSnapshot(); + expect(Object.keys(cadence(snapshots, 'render')!.metrics)).toHaveLength(3); + expect(Object.keys(cadence(snapshots, 'frame')!.metrics)).toHaveLength(8); + expect(Object.keys(cadence(snapshots, 'network')!.metrics)).toHaveLength(10); + // Pin the wire names the OOB hub and its dashboards consume, not just the enum + // reference: renaming a key here is a breaking protocol change, not a refactor. + expect(cadence(snapshots, 'render')!.metrics['render.framerate']).toBe(1); + expect(cadence(snapshots, 'render')!.metrics['pose.send_framerate']).toBe(2); + expect(cadence(snapshots, 'network')!.metrics[CloudXR.MetricsName.SessionQuality]).toBe(21); + }); + + it('snapshots are detached, so later ticks do not mutate an emitted payload', () => { + const acc = new MetricsAccumulator(); + acc.recordRender({ framerate: 72 }); + const first = cadence(acc.takeSnapshot(), 'render')!.metrics; + acc.recordRender({ framerate: 30 }); + + expect(first[CloudXR.MetricsName.RenderFramerate]).toBe(72); + }); + + it('reset drops everything so a new session cannot inherit stale values', () => { + const acc = new MetricsAccumulator(); + acc.recordRender({ framerate: 72 }); + acc.recordNetwork({ sessionQuality: 4 }); + acc.reset(); + + expect(acc.takeSnapshot()).toEqual([]); + }); +}); diff --git a/deps/cloudxr/webxr_client/helpers/metricsAccumulator.ts b/deps/cloudxr/webxr_client/helpers/metricsAccumulator.ts new file mode 100644 index 000000000..ea97d1c12 --- /dev/null +++ b/deps/cloudxr/webxr_client/helpers/metricsAccumulator.ts @@ -0,0 +1,153 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * MetricsAccumulator - last-known value per CloudXR metric, grouped by cadence. + * + * The in-XR HUD only needs a handful of metrics, but the OOB teleop hub wants + * everything the SDK reports. The SDK's `onMetrics` callbacks are *partial*: one + * tick may carry only `RenderFramerate`, the next only `PoseSendFramerate`. + * Replacing the stored record on each tick would make metrics flicker in and out + * of hub snapshots, so this merges instead, and `HeadsetControlChannel` polls the + * merged state on its own timer (default 500 ms). + * + * Keys are the SDK's `CloudXR.MetricsName` string values, so the hub and any + * downstream dashboards see the same names the SDK documents. The hub side + * (`oob_teleop_hub.py`) stores metrics as an arbitrary `{str: float}` map per + * cadence, so new metric names flow through without a hub change. + */ + +import * as CloudXR from '@nvidia/cloudxr'; + +import type { + FrameMetricsUpdate, + NetworkMetricsUpdate, + RenderMetricsUpdate, +} from './metricsUpdates'; + +/** Cadence labels used in `clientMetrics` payloads sent to the OOB hub. */ +export type MetricsCadenceLabel = 'render' | 'frame' | 'network'; + +/** One cadence's worth of metrics, ready to place in a `clientMetrics` payload. */ +export interface MetricsSnapshot { + cadence: string; + metrics: Record; +} + +/** + * Copies the defined entries of `update` into `target` under their SDK metric names. + * + * @param target - Accumulated metrics for one cadence; mutated in place. + * @param update - Partial callback payload; `undefined` fields are left untouched. + * @param names - Maps each update field to the `CloudXR.MetricsName` to store it under. + */ +function mergeDefined( + target: Record, + update: T, + names: { [K in keyof T]-?: string } +): void { + for (const key of Object.keys(names) as Array) { + const value = update[key]; + // Guard on the value, not the key: absent and explicitly-undefined must behave alike. + if (typeof value === 'number' && Number.isFinite(value)) { + target[names[key]] = value; + } + } +} + +/** Field-to-MetricsName maps, one per cadence. Kept beside the interfaces they mirror. */ +const RENDER_NAMES: { [K in keyof Required]: string } = { + framerate: CloudXR.MetricsName.RenderFramerate, + sendFramerate: CloudXR.MetricsName.PoseSendFramerate, + xrPoseAgeMs: CloudXR.MetricsName.LatencyXrPoseAgeMs, +}; + +const FRAME_NAMES: { [K in keyof Required]: string } = { + framerate: CloudXR.MetricsName.StreamingFramerate, + frameCount: CloudXR.MetricsName.StreamingFrameCount, + poseToRenderTimeMs: CloudXR.MetricsName.PoseToRenderTime, + poseUploadMs: CloudXR.MetricsName.LatencyPoseUploadMs, + poseToFrameReceivedMs: CloudXR.MetricsName.LatencyPoseToFrameReceivedMs, + compositorSkippedPercent: CloudXR.MetricsName.FramePipelineCompositorSkippedPercent, + outOfOrderPercent: CloudXR.MetricsName.FramePipelineOutOfOrderPercent, + mismatchedPercent: CloudXR.MetricsName.FramePipelineMismatchedPercent, +}; + +const NETWORK_NAMES: { [K in keyof Required]: string } = { + streamingRateMbps: CloudXR.MetricsName.NetworkStreamingRateMbps, + availableBandwidthMbps: CloudXR.MetricsName.NetworkAvailableBandwidthMbps, + rttMs: CloudXR.MetricsName.NetworkRttMs, + packetLoss: CloudXR.MetricsName.NetworkPacketLoss, + avgDecodeTimeMs: CloudXR.MetricsName.NetworkAvgDecodeTimeMs, + qualityScore: CloudXR.MetricsName.NetworkQualityScore, + bandwidthScore: CloudXR.MetricsName.NetworkBandwidthScore, + lossScore: CloudXR.MetricsName.NetworkLossScore, + latencyScore: CloudXR.MetricsName.NetworkLatencyScore, + sessionQuality: CloudXR.MetricsName.SessionQuality, +}; + +export class MetricsAccumulator { + private readonly byCadence: Record> = { + render: {}, + frame: {}, + network: {}, + }; + + /** Merge a PerRender callback payload. */ + recordRender(update: RenderMetricsUpdate): void { + mergeDefined(this.byCadence.render, update, RENDER_NAMES); + } + + /** Merge a PerFrame callback payload. */ + recordFrame(update: FrameMetricsUpdate): void { + mergeDefined(this.byCadence.frame, update, FRAME_NAMES); + } + + /** Merge a PerNetwork callback payload. */ + recordNetwork(update: NetworkMetricsUpdate): void { + mergeDefined(this.byCadence.network, update, NETWORK_NAMES); + } + + /** + * Drop everything recorded so far. + * + * Called when a session ends so the next session's hub snapshots cannot report + * the previous session's last-known values as if they were live. + */ + reset(): void { + for (const cadence of Object.keys(this.byCadence) as MetricsCadenceLabel[]) { + this.byCadence[cadence] = {}; + } + } + + /** + * Snapshot the accumulated metrics for transmission. + * + * @returns One entry per cadence that has at least one metric; empty cadences + * are omitted so the hub is not sent empty `clientMetrics` frames. + */ + takeSnapshot(): MetricsSnapshot[] { + const snapshots: MetricsSnapshot[] = []; + for (const cadence of Object.keys(this.byCadence) as MetricsCadenceLabel[]) { + const metrics = this.byCadence[cadence]; + if (Object.keys(metrics).length === 0) continue; + // Copy: the caller serializes asynchronously, and callbacks keep mutating. + snapshots.push({ cadence, metrics: { ...metrics } }); + } + return snapshots; + } +} diff --git a/deps/cloudxr/webxr_client/helpers/metricsUpdates.ts b/deps/cloudxr/webxr_client/helpers/metricsUpdates.ts new file mode 100644 index 000000000..7ddd41a24 --- /dev/null +++ b/deps/cloudxr/webxr_client/helpers/metricsUpdates.ts @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Shapes for the metrics CloudXRComponent forwards to the app, one per SDK + * MetricsCadence. + * + * The SDK delivers `onMetrics` as a partial `Record`: a + * single tick carries only the keys that were sampled on that path (the + * send-begin and render-begin paths, for example, each emit their own PerRender + * callback). Every field here is therefore optional, and consumers must merge + * rather than replace — see `MetricsAccumulator` in metricsAccumulator.ts. + */ + +/** Metrics delivered at {@link CloudXR.MetricsCadence.PerRender} (once per client render). */ +export interface RenderMetricsUpdate { + /** Client render rate, rolling average (FPS). */ + framerate?: number; + /** Rate at which poses are sent upstream, rolling average (FPS). */ + sendFramerate?: number; + /** Age of the XR pose at send time (ms). */ + xrPoseAgeMs?: number; +} + +/** Metrics delivered at {@link CloudXR.MetricsCadence.PerFrame} (once per streamed video frame). */ +export interface FrameMetricsUpdate { + /** Streaming rate, rolling average (FPS). */ + framerate?: number; + /** Monotonic count of streamed frames. */ + frameCount?: number; + /** Pose-to-render latency, rolling average (ms). */ + poseToRenderTimeMs?: number; + /** Time spent uploading the pose (ms). */ + poseUploadMs?: number; + /** Time from pose send to frame received (ms). */ + poseToFrameReceivedMs?: number; + /** Share of frames the compositor skipped (%). */ + compositorSkippedPercent?: number; + /** Share of frames that arrived out of order (%). */ + outOfOrderPercent?: number; + /** Share of frames whose pose did not match the rendered pixels (%). */ + mismatchedPercent?: number; +} + +/** Metrics delivered at {@link CloudXR.MetricsCadence.PerNetwork} (network sampling cadence). */ +export interface NetworkMetricsUpdate { + /** Current streaming rate (Mbps). */ + streamingRateMbps?: number; + /** Estimated available bandwidth (Mbps). */ + availableBandwidthMbps?: number; + /** Round-trip time (ms). */ + rttMs?: number; + /** Packet loss ratio. */ + packetLoss?: number; + /** Average video decode time (ms). */ + avgDecodeTimeMs?: number; + /** Composite network quality, a {@link CloudXR.QualityScore}. */ + qualityScore?: number; + /** Bandwidth component of the quality score, a {@link CloudXR.QualityScore}. */ + bandwidthScore?: number; + /** Loss component of the quality score, a {@link CloudXR.QualityScore}. */ + lossScore?: number; + /** Latency component of the quality score, a {@link CloudXR.QualityScore}. */ + latencyScore?: number; + /** Overall session quality 0-4, a {@link CloudXR.QualityScore}; drives the HUD indicator. */ + sessionQuality?: number; +} diff --git a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx index 3db6a2f98..2fcf0f6f5 100644 --- a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx +++ b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx @@ -34,6 +34,11 @@ */ import { MetricsTracker } from '@helpers/Metrics'; +import type { + FrameMetricsUpdate, + NetworkMetricsUpdate, + RenderMetricsUpdate, +} from '@helpers/metricsUpdates'; import { getConnectionConfig, ConnectionConfiguration, CloudXRConfig } from '@helpers/utils'; import { bindGL } from '@helpers/WebGLStateBinding'; import { clearPendingGLErrors } from '@helpers/WebGlUtils'; @@ -72,11 +77,42 @@ interface CloudXRComponentProps { /** Callback fired with the resolved server address after proxy configuration is applied. */ onServerAddress?: (address: string) => void; - /** Callback fired with render performance metrics. Receives rolling average of render FPS. */ - onRenderPerformanceMetrics?: (renderFps: number) => void; + /** + * Callback fired with render performance metrics ({@link CloudXR.MetricsCadence.PerRender}). + * Payloads are partial - only the fields sampled on this tick are present. + */ + onRenderPerformanceMetrics?: (metrics: RenderMetricsUpdate) => void; - /** Callback fired with streaming performance metrics. Receives rolling averages of streaming FPS and pose-to-render latency (ms). */ - onStreamingPerformanceMetrics?: (streamingFps: number, poseToRenderMs: number) => void; + /** + * Callback fired with streaming performance metrics ({@link CloudXR.MetricsCadence.PerFrame}). + * Payloads are partial - the render and decode paths each report their own subset. + */ + onStreamingPerformanceMetrics?: (metrics: FrameMetricsUpdate) => void; + + /** + * Callback fired with network performance metrics ({@link CloudXR.MetricsCadence.PerNetwork}), + * including session quality. Payloads are partial. + */ + onNetworkPerformanceMetrics?: (metrics: NetworkMetricsUpdate) => void; + + /** + * Callback fired with batched SDK log entries, in addition to the browser console. + * Entries are already mirrored to the console; use this only to route them elsewhere. + */ + onLog?: (entries: CloudXR.LogEntry[]) => void; + + /** + * Pre-stream network test. When set, the SDK measures link quality before streaming + * starts. Leave undefined to skip the test entirely (the IsaacTeleop default): a teleop + * operator connecting to a robot should not be held behind a measurement window. + */ + streamTest?: { durationSeconds: number; mode: 'off' | 'warn' | 'block' }; + + /** Callback fired once when the network test's measurement window begins. */ + onStreamTestStarted?: () => void; + + /** Callback fired when the network test finishes, with its result. */ + onStreamTestStopped?: (result: CloudXR.StreamTestResult) => void; /** * Settings for the performance metrics reported via onRenderPerformanceMetrics and onStreamingPerformanceMetrics callbacks. @@ -120,6 +156,11 @@ export default function CloudXRComponent({ onServerAddress, onRenderPerformanceMetrics, onStreamingPerformanceMetrics, + onNetworkPerformanceMetrics, + onLog, + streamTest, + onStreamTestStarted, + onStreamTestStopped, metricsSettings = {}, headless = false, trackingFrameAdapter, @@ -137,6 +178,11 @@ export default function CloudXRComponent({ const renderFpsTrackerRef = useRef( new MetricsTracker(metricsSettings.renderFpsWindow ?? 100) ); + // Pose send rate is sampled on the same PerRender cadence as render FPS, so it uses + // the same window to keep the two HUD cards directly comparable. + const poseSendFpsTrackerRef = useRef( + new MetricsTracker(metricsSettings.renderFpsWindow ?? 100) + ); const streamingFpsTrackerRef = useRef( new MetricsTracker(metricsSettings.streamingFpsWindow ?? 20) ); @@ -266,10 +312,13 @@ export default function CloudXRComponent({ mediaPort: config.mediaPort, iceServers: iceServers, glBinding: glBinding, + // Undefined when the network test is off, which is the IsaacTeleop default. + streamTest, + logLevel: CloudXR.LogLevel.Info, // Deliver Info-and-above SDK logs to onLog telemetry: { enabled: true, appInfo: { - version: '6.2.0', + version: '6.3.0', product: applicationName, }, }, @@ -277,6 +326,29 @@ export default function CloudXRComponent({ // Store the render target and key GL bindings to restore after CloudXR rendering const cloudXRDelegates: CloudXR.SessionDelegates = { + onLog: (entries: CloudXR.LogEntry[]) => { + for (const { level, message } of entries) { + // Pick the console method matching this severity so DevTools filtering works. + const printFn = + level === CloudXR.LogLevel.Error + ? console.error + : level === CloudXR.LogLevel.Warning + ? console.warn + : level === CloudXR.LogLevel.Debug + ? console.debug + : console.info; + printFn(`[CloudXR] ${message}`); + } + onLog?.(entries); + }, + onStreamTestStarted: () => { + onStatusChange?.(false, 'Testing network'); + onStreamTestStarted?.(); + }, + onStreamTestStopped: (result: CloudXR.StreamTestResult) => { + console.debug('Network test complete:', result); + onStreamTestStopped?.(result); + }, onWebGLStateChangeBegin: () => { // Save the current render target before CloudXR changes state trackedGL.save(); @@ -314,23 +386,88 @@ export default function CloudXRComponent({ onSessionReady?.(null); }, onMetrics: (metrics: CloudXR.Metrics, cadence: CloudXR.MetricsCadence) => { - // Handle render performance metrics (PerRender cadence) + // Every cadence delivers a *partial* record: only the metrics sampled on this + // tick are present. Forward just those keys and let the consumer merge, so an + // absent metric is never reported downstream as a zero. if (onRenderPerformanceMetrics && cadence === CloudXR.MetricsCadence.PerRender) { - // Return the averaged metrics to the parent component - const renderFps = metrics[CloudXR.MetricsName.RenderFramerate] ?? 0; - onRenderPerformanceMetrics(renderFpsTrackerRef.current.add(renderFps)); + const update: RenderMetricsUpdate = {}; + const renderFps = metrics[CloudXR.MetricsName.RenderFramerate]; + if (renderFps !== undefined) { + update.framerate = renderFpsTrackerRef.current.add(renderFps); + } + const poseSendFps = metrics[CloudXR.MetricsName.PoseSendFramerate]; + if (poseSendFps !== undefined) { + update.sendFramerate = poseSendFpsTrackerRef.current.add(poseSendFps); + } + const xrPoseAgeMs = metrics[CloudXR.MetricsName.LatencyXrPoseAgeMs]; + if (xrPoseAgeMs !== undefined) { + update.xrPoseAgeMs = xrPoseAgeMs; + } + // The send-begin and render-begin paths each emit their own PerRender tick, + // so skip ticks that carried none of the keys we track. + if (Object.keys(update).length > 0) { + onRenderPerformanceMetrics(update); + } + } + + if (onNetworkPerformanceMetrics && cadence === CloudXR.MetricsCadence.PerNetwork) { + const update: NetworkMetricsUpdate = {}; + const networkFields: Array<[keyof NetworkMetricsUpdate, string]> = [ + ['streamingRateMbps', CloudXR.MetricsName.NetworkStreamingRateMbps], + ['availableBandwidthMbps', CloudXR.MetricsName.NetworkAvailableBandwidthMbps], + ['rttMs', CloudXR.MetricsName.NetworkRttMs], + ['packetLoss', CloudXR.MetricsName.NetworkPacketLoss], + ['avgDecodeTimeMs', CloudXR.MetricsName.NetworkAvgDecodeTimeMs], + ['qualityScore', CloudXR.MetricsName.NetworkQualityScore], + ['bandwidthScore', CloudXR.MetricsName.NetworkBandwidthScore], + ['lossScore', CloudXR.MetricsName.NetworkLossScore], + ['latencyScore', CloudXR.MetricsName.NetworkLatencyScore], + ['sessionQuality', CloudXR.MetricsName.SessionQuality], + ]; + for (const [field, name] of networkFields) { + const value = metrics[name as CloudXR.MetricsName]; + if (value !== undefined) { + update[field] = value; + } + } + if (Object.keys(update).length > 0) { + onNetworkPerformanceMetrics(update); + } } // Handle streaming performance metrics (PerFrame cadence) if (onStreamingPerformanceMetrics && cadence === CloudXR.MetricsCadence.PerFrame) { - const streamingFps = metrics[CloudXR.MetricsName.StreamingFramerate] ?? 0; - const poseToRenderMs = metrics[CloudXR.MetricsName.PoseToRenderTime] ?? 0; - - // Return the averaged metrics to the parent component - onStreamingPerformanceMetrics( - streamingFpsTrackerRef.current.add(streamingFps), - poseToRenderTrackerRef.current.add(poseToRenderMs) - ); + const update: FrameMetricsUpdate = {}; + const streamingFps = metrics[CloudXR.MetricsName.StreamingFramerate]; + if (streamingFps !== undefined) { + update.framerate = streamingFpsTrackerRef.current.add(streamingFps); + } + const poseToRenderMs = metrics[CloudXR.MetricsName.PoseToRenderTime]; + if (poseToRenderMs !== undefined) { + update.poseToRenderTimeMs = poseToRenderTrackerRef.current.add(poseToRenderMs); + } + // Reported raw: these are diagnostic counters for the hub, not HUD values, + // so averaging them would only obscure spikes. + const frameFields: Array<[keyof FrameMetricsUpdate, string]> = [ + ['frameCount', CloudXR.MetricsName.StreamingFrameCount], + ['poseUploadMs', CloudXR.MetricsName.LatencyPoseUploadMs], + ['poseToFrameReceivedMs', CloudXR.MetricsName.LatencyPoseToFrameReceivedMs], + [ + 'compositorSkippedPercent', + CloudXR.MetricsName.FramePipelineCompositorSkippedPercent, + ], + ['outOfOrderPercent', CloudXR.MetricsName.FramePipelineOutOfOrderPercent], + ['mismatchedPercent', CloudXR.MetricsName.FramePipelineMismatchedPercent], + ]; + for (const [field, name] of frameFields) { + const value = metrics[name as CloudXR.MetricsName]; + if (value !== undefined) { + update[field] = value; + } + } + if (Object.keys(update).length > 0) { + onStreamingPerformanceMetrics(update); + } } }, }; @@ -426,6 +563,24 @@ export default function CloudXRComponent({ // Get session from reference. const cxrSession: CloudXR.Session = cxrSessionRef.current; + // Pump tracking during Connecting so the SDK can read XRSession.enabledFeatures from + // these early frames and negotiate optional capabilities - notably body tracking - + // before the stream starts. Without this the SDK never sees a frame pre-Connected and + // full-body teleop silently degrades to upper body. Returns false while Connecting, + // but may throw deferred exceptions from callbacks. + // + // Deliberately the raw xrFrame, not trackingFrameAdapter's: during replay the adapter + // returns a Proxy with a substituted session, and capability detection must reflect + // the real XRSession. The adapter still applies on the Connected path below. + if (cxrSession.state === CloudXR.SessionState.Connecting) { + try { + cxrSession.sendTrackingStateToServer(state.clock.elapsedTime * 1000, xrFrame); + } catch (error) { + console.error('CloudXR render loop error:', error); + return; + } + } + // If the CloudXR session is not connected, skip the frame. if (cxrSession.state !== CloudXR.SessionState.Connected) { console.debug('Skipping frame, session not connected, state:', cxrSession.state); diff --git a/deps/cloudxr/webxr_client/helpers/react/PerformanceCanvasImage.tsx b/deps/cloudxr/webxr_client/helpers/react/PerformanceCanvasImage.tsx index 00068bcd4..1cb2040a1 100644 --- a/deps/cloudxr/webxr_client/helpers/react/PerformanceCanvasImage.tsx +++ b/deps/cloudxr/webxr_client/helpers/react/PerformanceCanvasImage.tsx @@ -40,21 +40,37 @@ import { CanvasTexture } from 'three'; /** Canvas resolution (pixels). High values keep text sharp when the texture is scaled to the display size. */ const CANVAS_WIDTH = 1024; -const CANVAS_HEIGHT = 500; +const CANVAS_HEIGHT = 760; // Layout constants (canvas space) — compact card style with label + value side-by-side. +// SQ card (200px) + 4 metric cards (110px) + 4 gaps (14px) = 696px, centred in 760px canvas (32px margin each side). const LAYOUT = { - fontSize: 56, - cardHeight: 130, - cardGap: 18, - numCards: 3, - margin: 32, + fontSize: 52, + sqCardHeight: 200, // session-quality bars card — must exceed tallest bar (160px) plus 10px padding + cardHeight: 110, // metric text cards + cardGap: 14, + margin: 56, paddingLeft: 40, radius: 18, cardFillStyle: 'rgba(0, 0, 0, 0.5)', labelColor: 'rgba(180, 180, 180, 1)', } as const; +/** RGBA fill colors for each session quality level (index = quality integer 0–4). */ +const SESSION_QUALITY_COLORS = [ + 'rgba(120, 120, 120, 1)', // NoData — grey + 'rgba(220, 50, 50, 1)', // Unsustainable — red + 'rgba(220, 160, 30, 1)', // Degraded — amber + 'rgba(80, 200, 80, 1)', // Good — green + 'rgba(0, 255, 128, 1)', // Excellent — bright green +] as const; + +/** Dim fill used for unlit bars in the session quality indicator. */ +const SESSION_QUALITY_BAR_DIM = 'rgba(60, 60, 60, 0.6)'; + +/** Heights (px, canvas space) of the 4 quality bars, shortest to tallest. */ +const SESSION_QUALITY_BAR_HEIGHTS = [50, 80, 115, 160] as const; + const CARD_WIDTH = CANVAS_WIDTH - LAYOUT.margin * 2; /** Draw a rounded rectangle path; caller must ctx.fill() or ctx.stroke() after. */ @@ -79,6 +95,7 @@ function drawRoundRect( ctx.closePath(); } +/** Props for {@link PerformanceCanvasImage}: sizing and per-frame metric/quality signals. */ export interface PerformanceCanvasImageProps { /** Display width of the metrics image (uikit units). */ width?: number; @@ -86,10 +103,14 @@ export interface PerformanceCanvasImageProps { height?: number; /** Signal for render FPS value (e.g. "72.0"). Label "Render FPS: " is drawn here. */ renderFpsText?: ReadonlySignal; + /** Signal for pose send FPS value (e.g. "72.0"). Label "Pose Send FPS: " is drawn here. */ + poseSendFpsText?: ReadonlySignal; /** Signal for streaming FPS value. Label "Streaming FPS: " is drawn here. */ streamingFpsText?: ReadonlySignal; /** Signal for pose-to-render latency (e.g. "12.3ms"). Label "Pose-to-Render: " is drawn here. */ poseToRenderText?: ReadonlySignal; + /** Signal carrying live session quality (0–4); see {@link CloudXR.MetricsName.SessionQuality}. */ + sessionQuality?: ReadonlySignal; } /** @@ -101,8 +122,10 @@ export function PerformanceCanvasImage({ width = 512, height = 512, renderFpsText, + poseSendFpsText, streamingFpsText, poseToRenderText, + sessionQuality, }: PerformanceCanvasImageProps) { /** Ref for the uikit Image; we set .texture.value on it to use our CanvasTexture. */ const imageRef = useRef<{ texture: { value: CanvasTexture | undefined } } | null>(null); @@ -154,31 +177,59 @@ export function PerformanceCanvasImage({ const { fontSize, + sqCardHeight, cardHeight, cardGap, - numCards, margin, paddingLeft, radius, cardFillStyle, labelColor, } = LAYOUT; - // Vertically center the stack of cards within the canvas. - const totalHeight = numCards * cardHeight + (numCards - 1) * cardGap; - let cardY = (canvas.height - totalHeight) / 2; - // Each tuple: [label, current signal value (or em-dash fallback), value color]. const metrics: [string, string, string][] = [ ['Render FPS', renderFpsText?.value ?? '—', 'rgba(100, 255, 100, 1)'], + ['Pose Send FPS', poseSendFpsText?.value ?? '—', 'rgba(180, 255, 140, 1)'], ['Streaming FPS', streamingFpsText?.value ?? '—', 'rgba(100, 200, 255, 1)'], ['Pose-to-Render', poseToRenderText?.value ?? '—', 'rgba(255, 200, 100, 1)'], ]; + // Vertically center: 1 SQ card + N metric cards + N gaps between each adjacent pair. + const totalHeight = sqCardHeight + metrics.length * cardHeight + metrics.length * cardGap; + let cardY = (canvas.height - totalHeight) / 2; ctx.font = `bold ${fontSize}px system-ui, sans-serif`; ctx.textBaseline = 'middle'; - // Offset to vertically center text within each card. - const centerY = cardHeight / 2; + // Session Quality card first — appears directly under the "Performance" label. + // Clamp against COLORS (length 5, indices 0–4) not HEIGHTS (length 4) so Excellent (4) renders correctly. + const qualityLevel = Math.max( + 0, + Math.min(SESSION_QUALITY_COLORS.length - 1, Math.round(sessionQuality?.value ?? 0)) + ); + const qualityColor = SESSION_QUALITY_COLORS[qualityLevel]; + ctx.fillStyle = cardFillStyle; + drawRoundRect(ctx, margin, cardY, CARD_WIDTH, sqCardHeight, radius); + ctx.fill(); + + // Draw 4 vertical bars of increasing height, bottom-aligned within the SQ card. + const numBars = SESSION_QUALITY_BAR_HEIGHTS.length; + const barWidth = 130; + const barGap = 70; + const barsTotal = numBars * barWidth + (numBars - 1) * barGap; + const barBaseX = margin + (CARD_WIDTH - barsTotal) / 2; + const barBottomY = cardY + sqCardHeight - 10; + for (let i = 0; i < numBars; i++) { + const barH = SESSION_QUALITY_BAR_HEIGHTS[i]; + const barX = barBaseX + i * (barWidth + barGap); + // Bars 0..qualityLevel-1 are lit; the rest are dim. For NoData all are dim. + ctx.fillStyle = qualityLevel > 0 && i < qualityLevel ? qualityColor : SESSION_QUALITY_BAR_DIM; + drawRoundRect(ctx, barX, barBottomY - barH, barWidth, barH, 8); + ctx.fill(); + } + cardY += sqCardHeight + cardGap; + + // Three metric cards below the quality indicator. + const centerY = cardHeight / 2; for (const [label, value, valueColor] of metrics) { // Draw the rounded card background. ctx.fillStyle = cardFillStyle; diff --git a/deps/cloudxr/webxr_client/helpers/react/utils.ts b/deps/cloudxr/webxr_client/helpers/react/utils.ts index 5ebd38489..b2ef062bd 100644 --- a/deps/cloudxr/webxr_client/helpers/react/utils.ts +++ b/deps/cloudxr/webxr_client/helpers/react/utils.ts @@ -90,6 +90,26 @@ export function parseAutoRefreshMode( return fallback; } +/** Pre-stream network test mode: skip it, measure and report, or measure and gate on it. */ +export type StreamTestMode = 'off' | 'warn' | 'block'; + +const STREAM_TEST_MODES: readonly StreamTestMode[] = ['off', 'warn', 'block']; + +/** + * Parses a string into a valid network test mode. + * @param unvalidatedValue - String to validate (e.g. from URL, config, or form). May be invalid or empty. + * @param fallback - Value to return when unvalidatedValue is not valid. + */ +export function parseStreamTestMode( + unvalidatedValue: string, + fallback: StreamTestMode +): StreamTestMode { + if (STREAM_TEST_MODES.includes(unvalidatedValue as StreamTestMode)) { + return unvalidatedValue as StreamTestMode; + } + return fallback; +} + const CONTROL_PANEL_POSITIONS: readonly ControlPanelPosition[] = ['left', 'center', 'right']; /** diff --git a/deps/cloudxr/webxr_client/helpers/utils.ts b/deps/cloudxr/webxr_client/helpers/utils.ts index a19dd1a73..e9f20ab34 100644 --- a/deps/cloudxr/webxr_client/helpers/utils.ts +++ b/deps/cloudxr/webxr_client/helpers/utils.ts @@ -187,6 +187,18 @@ export interface CloudXRConfig { /** Media server port for WebRTC streaming (use 0 for server-provided port) */ mediaPort?: number; + + /** + * Pre-stream network test: 'off' skips it, 'warn' measures and reports, 'block' also + * refuses to stream when the link fails the gate. + * + * Defaults to 'off' for IsaacTeleop. The test holds the session in Connecting for its + * whole window, and an operator connecting to a robot should not be gated on it. + */ + streamTestMode?: 'off' | 'warn' | 'block'; + + /** Length of the network test measurement window in seconds. Ignored when the test is off. */ + streamTestDurationSeconds?: number; } /** diff --git a/deps/cloudxr/webxr_client/jest.setup.js b/deps/cloudxr/webxr_client/jest.setup.js new file mode 100644 index 000000000..ba84e9a53 --- /dev/null +++ b/deps/cloudxr/webxr_client/jest.setup.js @@ -0,0 +1,33 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Jest setup shared by every test environment. + * + * jsdom implements `performance.now()` but not the User Timing entries, which the CloudXR + * SDK bundle calls during module initialization. Suites that import the SDK therefore need + * these stubs to get as far as their first assertion. No-op under the default `node` + * environment, where the real implementations already exist. + */ +if (typeof performance !== 'undefined' && typeof performance.mark !== 'function') { + performance.mark = () => {}; + performance.measure = () => {}; + performance.clearMarks = () => {}; + performance.clearMeasures = () => {}; + performance.getEntriesByName = () => []; + performance.getEntriesByType = () => []; +} diff --git a/deps/cloudxr/webxr_client/package.json b/deps/cloudxr/webxr_client/package.json index 285ec4e3c..3f9a8e142 100644 --- a/deps/cloudxr/webxr_client/package.json +++ b/deps/cloudxr/webxr_client/package.json @@ -1,6 +1,6 @@ { "name": "cloudxr-isaac-lab-teleop", - "version": "6.2.0", + "version": "6.3.0-rc2", "private": true, "description": "NVIDIA Isaac Teleop Web Client", "author": "NVIDIA Corporation", @@ -23,13 +23,16 @@ "jest": { "preset": "ts-jest", "testEnvironment": "node", + "setupFiles": [ + "/jest.setup.js" + ], "roots": [ "/src", "/helpers" ] }, "dependencies": { - "@nvidia/cloudxr": "file:../nvidia-cloudxr-6.2.0.tgz", + "@nvidia/cloudxr": "file:../nvidia-cloudxr-6.3.0-rc2.tgz", "@preact/signals-react": "^3.10.0", "@react-three/drei": "^10.7.7", "@react-three/fiber": "^9.6.0", @@ -55,6 +58,7 @@ "html-webpack-plugin": "^5.6.7", "iwer": "^2.2.1", "jest": "^30", + "jest-environment-jsdom": "^30.4.1", "rimraf": "^5.0.5", "style-loader": "^3.3.3", "ts-jest": "^29", diff --git a/deps/cloudxr/webxr_client/src/App.tsx b/deps/cloudxr/webxr_client/src/App.tsx index 9bd003a11..a1231554d 100644 --- a/deps/cloudxr/webxr_client/src/App.tsx +++ b/deps/cloudxr/webxr_client/src/App.tsx @@ -62,22 +62,41 @@ import { CloudXR2DUI, COUNTDOWN_STORAGE_KEY } from './CloudXR2DUI'; import { readUrlParam } from './config/resolve'; import CloudXR3DUI from './CloudXRUI'; import { HeadsetControlChannel } from '@helpers/controlChannel'; +import { MetricsAccumulator } from '@helpers/metricsAccumulator'; +import type { + FrameMetricsUpdate, + NetworkMetricsUpdate, + RenderMetricsUpdate, +} from '@helpers/metricsUpdates'; import { RecorderProvider, useRecorder } from './RecorderContext'; import { RecorderComponent } from './RecorderComponent'; import { TraceVisualization } from './TraceVisualization'; -// Performance metrics signals - raw numeric data, one per callback cadence. +// Performance metrics signals - raw numeric data backing the in-XR HUD. // Signals update their value without triggering React re-renders. // See: https://pmndrs.github.io/uikit/docs/advanced/performance -const renderMetrics = signal<{ fps: number } | null>(null); +// +// Only the metrics the HUD draws live here. The full set the SDK reports goes to the +// OOB teleop hub via metricsAccumulator, which is a superset of these. +const renderFps = signal(null); +const poseSendFps = signal(null); const streamingMetrics = signal<{ fps: number; latencyMs: number } | null>(null); +// Live session quality 0-4; see CloudXR.QualityScore. 0 is NoData, which is also the +// resting state between sessions. +const sessionQuality = signal(0); + +// Network test status: text plus a traffic-light color for the in-XR panel. Module-scoped +// to match the metric signals above; persists last known value across sessions. +const streamTest = signal<{ text: string; color: string } | null>(null); + // Computed signals derive formatted text from raw data. -// When renderMetrics.value changes, computed() automatically recalculates the text. +// When a source signal changes, computed() automatically recalculates the text. // The @react-three/uikit Text component subscribes to these computed signals // and updates the displayed text directly in Three.js - bypassing React entirely. -const renderFpsText = computed(() => - renderMetrics.value ? renderMetrics.value.fps.toFixed(1) : '-' +const renderFpsText = computed(() => (renderFps.value !== null ? renderFps.value.toFixed(1) : '-')); +const poseSendFpsText = computed(() => + poseSendFps.value !== null ? poseSendFps.value.toFixed(1) : '-' ); const streamingFpsText = computed(() => streamingMetrics.value ? streamingMetrics.value.fps.toFixed(1) : '-' @@ -85,6 +104,26 @@ const streamingFpsText = computed(() => const poseToRenderText = computed(() => streamingMetrics.value ? `${streamingMetrics.value.latencyMs.toFixed(1)}ms` : '-' ); +const streamTestText = computed(() => streamTest.value?.text ?? ''); +const streamTestColor = computed(() => streamTest.value?.color ?? 'white'); + +/** Accumulates every metric the SDK reports, for periodic upload to the OOB teleop hub. */ +const metricsAccumulator = new MetricsAccumulator(); + +/** + * Fallback network-test window when the config omits a duration, and the bounds the SDK + * clamps `SessionOptions.streamTest.durationSeconds` to. Clamping here too keeps the + * on-panel countdown honest when a config or URL param asks for something out of range. + */ +const DEFAULT_STREAM_TEST_SECONDS = 5; +const MIN_STREAM_TEST_SECONDS = 5; +const MAX_STREAM_TEST_SECONDS = 30; + +/** Clamps a configured network-test duration into the range the SDK accepts. */ +function resolveStreamTestSeconds(configured: number | undefined): number { + const seconds = configured ?? DEFAULT_STREAM_TEST_SECONDS; + return Math.min(MAX_STREAM_TEST_SECONDS, Math.max(MIN_STREAM_TEST_SECONDS, seconds)); +} const CONTROL_PANEL_LAYOUT = { distance: 1.8, @@ -164,6 +203,18 @@ function AppContent() { const [countdownRemaining, setCountdownRemaining] = useState(0); const [isTeleopRunning, setIsTeleopRunning] = useState(false); const countdownTimerRef = useRef(null); + /** App-owned countdown for the pre-stream network test window. */ + const streamTestCountdownRef = useRef | null>(null); + // Clear the countdown if the component unmounts mid-test (e.g. the user exits XR). + useEffect( + () => () => { + if (streamTestCountdownRef.current) { + clearInterval(streamTestCountdownRef.current); + streamTestCountdownRef.current = null; + } + }, + [] + ); /** Avoid repeating immersive session dumps on every XR store tick. */ const immersiveSessionDumpLoggedRef = useRef(false); const [countdownDuration, setCountdownDuration] = useState(() => { @@ -490,6 +541,24 @@ function AppContent() { setSessionStatus(status); controlChannelRef.current?.sendStreamStatus(connected && status === 'Connected'); + // Drop the previous session's quality reading rather than leaving a stale green + // indicator on a dead stream. Safe on every non-Connected status, including the + // 'Testing network' transition, since quality is only sampled while streaming. + if (!connected || status !== 'Connected') { + sessionQuality.value = 0; + } + // Reset metrics only on an actual session end. Deliberately not on 'Testing network': + // that status is emitted by onStreamTestStarted, i.e. on the way *into* a session, and + // clearing there would discard any samples taken during the measurement window. + if (status === 'Disconnected' || status === 'Error') { + metricsAccumulator.reset(); + // Clear the HUD cards too, so a dead session cannot leave its last FPS and latency + // readings on the panel looking live. The computed texts fall back to '-' on null. + renderFps.value = null; + poseSendFps.value = null; + streamingMetrics.value = null; + } + // Reload on session end per mode; read live off the stable 2D UI to avoid a stale closure. const autoRefreshMode = cloudXR2DUI?.getConfiguration().autoRefreshMode; if ( @@ -500,14 +569,95 @@ function AppContent() { } }; - // Render performance metrics callback handler - updates raw data signal - const handleRenderPerformanceMetrics = (fps: number) => { - renderMetrics.value = { fps }; + // Metrics callbacks. Each payload is partial, so HUD signals are only overwritten for + // fields actually present, and everything is merged into the accumulator for the hub. + const handleRenderPerformanceMetrics = (metrics: RenderMetricsUpdate) => { + if (metrics.framerate !== undefined) renderFps.value = metrics.framerate; + if (metrics.sendFramerate !== undefined) poseSendFps.value = metrics.sendFramerate; + metricsAccumulator.recordRender(metrics); + }; + + const handleStreamingPerformanceMetrics = (metrics: FrameMetricsUpdate) => { + if (metrics.framerate !== undefined || metrics.poseToRenderTimeMs !== undefined) { + // Carry forward the fields this tick did not carry so the HUD does not flicker. + const prev = streamingMetrics.value ?? { fps: 0, latencyMs: 0 }; + streamingMetrics.value = { + fps: metrics.framerate ?? prev.fps, + latencyMs: metrics.poseToRenderTimeMs ?? prev.latencyMs, + }; + } + metricsAccumulator.recordFrame(metrics); + }; + + const handleNetworkPerformanceMetrics = (metrics: NetworkMetricsUpdate) => { + if (metrics.sessionQuality !== undefined) { + sessionQuality.value = metrics.sessionQuality; + } + metricsAccumulator.recordNetwork(metrics); + }; + + const stopStreamTestCountdown = () => { + if (streamTestCountdownRef.current) { + clearInterval(streamTestCountdownRef.current); + streamTestCountdownRef.current = null; + } }; - // Streaming performance metrics callback handler - updates raw data signal - const handleStreamingPerformanceMetrics = (fps: number, latencyMs: number) => { - streamingMetrics.value = { fps, latencyMs }; + // The SDK only signals the start and end of the measurement window, so the countdown + // is driven here from the configured duration. + const handleStreamTestStarted = () => { + stopStreamTestCountdown(); + let remaining = resolveStreamTestSeconds(config?.streamTestDurationSeconds); + const tick = () => { + // Hold at 0 until the result arrives rather than counting into negatives. + if (remaining <= 0) { + streamTest.value = { text: 'Testing network… 0s', color: '#ffd24d' }; + stopStreamTestCountdown(); + return; + } + streamTest.value = { text: `Testing network… ${remaining}s`, color: '#ffd24d' }; + remaining -= 1; + }; + tick(); + streamTestCountdownRef.current = setInterval(tick, 1000); + }; + + const handleStreamTestStopped = (result: CloudXR.StreamTestResult) => { + stopStreamTestCountdown(); + // Red matches the gate exactly: red iff the test failed. On a pass, green only when + // every measured dimension is Good or better, otherwise yellow as a caution. + const measured = [ + result.latencyScore, + result.jitterScore, + result.bandwidthScore, + result.devicePerformanceScore, + ].filter(score => score !== CloudXR.QualityScore.NoData); + let color: string; + let label: string; + if (!result.passed) { + color = '#ff5d5d'; + label = 'failed'; + } else { + const worst = measured.length ? Math.min(...measured) : CloudXR.QualityScore.NoData; + color = worst >= CloudXR.QualityScore.Good ? '#5dd35d' : '#ffd24d'; + label = 'passed'; + } + // QualityScore is a numeric enum; reverse-map to the name so the panel reads as a + // rating ("Good") rather than an ambiguous number. + const scoreName = (score: CloudXR.QualityScore) => CloudXR.QualityScore[score]; + streamTest.value = { + text: `Network test ${label} — latency: ${scoreName(result.latencyScore)}, jitter: ${scoreName(result.jitterScore)}, ${result.serverFps ?? '-'} fps`, + color, + }; + }; + + // Clear the network-test indicator on both a new session and a teardown. + // onStreamTestStopped does not fire when the user disconnects mid-test (an abort is not + // a result), so this is what stops a countdown that would otherwise keep ticking. + const handleSessionReady = (session: CloudXR.Session | null) => { + stopStreamTestCountdown(); + streamTest.value = null; + setCloudXRSession(session); }; /** @@ -701,27 +851,10 @@ function AppContent() { onConfig: () => { // Config push handling deferred to phase 2. }, - getMetricsSnapshot: () => { - const snapshots: Array<{ cadence: string; metrics: Record }> = []; - const rm = renderMetrics.value; - if (rm) { - snapshots.push({ - cadence: 'render', - metrics: { [CloudXR.MetricsName.RenderFramerate]: rm.fps }, - }); - } - const sm = streamingMetrics.value; - if (sm) { - snapshots.push({ - cadence: 'frame', - metrics: { - [CloudXR.MetricsName.StreamingFramerate]: sm.fps, - [CloudXR.MetricsName.PoseToRenderTime]: sm.latencyMs, - }, - }); - } - return snapshots; - }, + // Reports every metric the SDK emits, keyed by CloudXR.MetricsName, across the + // render / frame / network cadences. The hub stores metrics as an arbitrary + // {name: value} map per cadence, so new SDK metrics need no hub-side change. + getMetricsSnapshot: () => metricsAccumulator.takeSnapshot(), }); channel.connect(); controlChannelRef.current = channel; @@ -796,7 +929,11 @@ function AppContent() { // Set up message receiving from MessageChannel (new API) or legacy callback // Poll for channel availability since channels can be announced at any time useEffect(() => { - if (!cloudXRSession) { + // Wait for Connected, not just a non-null session: opening a channel sends a control + // message that requires a connected session. With the pre-stream network test enabled + // the session sits in Connecting for the whole test window, and opening the teleop + // channel there would fail. + if (!cloudXRSession || !isConnected) { return; } @@ -871,7 +1008,7 @@ function AppContent() { active = false; clearInterval(pollInterval); }; - }, [cloudXRSession]); + }, [cloudXRSession, isConnected]); return ( <> @@ -926,10 +1063,23 @@ function AppContent() { } }} onExitImmersiveXR={handleDisconnect} - onSessionReady={setCloudXRSession} + onSessionReady={handleSessionReady} onServerAddress={setServerAddress} onRenderPerformanceMetrics={handleRenderPerformanceMetrics} onStreamingPerformanceMetrics={handleStreamingPerformanceMetrics} + onNetworkPerformanceMetrics={handleNetworkPerformanceMetrics} + streamTest={ + config.streamTestMode && config.streamTestMode !== 'off' + ? { + durationSeconds: resolveStreamTestSeconds( + config.streamTestDurationSeconds + ), + mode: config.streamTestMode, + } + : undefined + } + onStreamTestStarted={handleStreamTestStarted} + onStreamTestStopped={handleStreamTestStopped} headless={!!config.headless} /> {!config.headless && ( @@ -956,8 +1106,12 @@ function AppContent() { position={controlPanelPositionVector} rotation={[0, 0, 0]} renderFpsText={renderFpsText} + poseSendFpsText={poseSendFpsText} streamingFpsText={streamingFpsText} poseToRenderText={poseToRenderText} + sessionQuality={sessionQuality} + streamTestText={streamTestText} + streamTestColor={streamTestColor} showRecordingControls={config.showRecordingControls} /> )} diff --git a/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx b/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx index aa6306603..42f74fc82 100644 --- a/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx @@ -42,6 +42,7 @@ import { loadPerProject, parseAutoRefreshMode, parseControlPanelPosition, + parseStreamTestMode, ReactUIConfig, savePerProject, } from '@helpers/react/utils'; @@ -169,6 +170,10 @@ export class CloudXR2DUI { private mediaAddressInput!: HTMLInputElement; /** Input field for media server port */ private mediaPortInput!: HTMLInputElement; + /** Dropdown selecting the pre-stream network test mode (off / warn / block) */ + private streamTestModeSelect!: HTMLSelectElement; + /** Input field for the network test window length in seconds */ + private streamTestDurationSecondsInput!: HTMLInputElement; /** Dropdown for controller model visibility (show / hide) */ private controllerModelVisibilitySelect!: HTMLSelectElement; private showTraceInXRSelect!: HTMLSelectElement; @@ -466,6 +471,10 @@ export class CloudXR2DUI { this.certLink = this.getElement('certLink'); this.mediaAddressInput = this.getElement('mediaAddress'); this.mediaPortInput = this.getElement('mediaPort'); + this.streamTestModeSelect = this.getElement('streamTestMode'); + this.streamTestDurationSecondsInput = this.getElement( + 'streamTestDurationSeconds' + ); this.controllerModelVisibilitySelect = this.getElement( 'controllerModelVisibility' ); @@ -532,6 +541,11 @@ export class CloudXR2DUI { showTrace: false, showRecordingControls: false, replayPacing: 'time', + // Off by default: the test holds the session in Connecting for its whole window, + // and a teleop operator connecting to a robot should not be gated on it. Opt in + // via the settings panel or ?streamTestMode=warn when diagnosing a link. + streamTestMode: 'off', + streamTestDurationSeconds: 5, headless: false, autoRefreshMode: 'clean', teleopPath: DEFAULT_TELEOP_PATH, @@ -573,6 +587,8 @@ export class CloudXR2DUI { { el: this.xrOffsetZInput, key: 'xrOffsetZ' }, { el: this.mediaAddressInput, key: 'mediaAddress' }, { el: this.mediaPortInput, key: 'mediaPort' }, + { el: this.streamTestModeSelect, key: 'streamTestMode' }, + { el: this.streamTestDurationSecondsInput, key: 'streamTestDurationSeconds' }, { el: this.controllerModelVisibilitySelect, key: 'controllerModelVisibility' }, { el: this.showTraceInXRSelect, key: 'showTraceInXR' }, { el: this.showRecordingControlsSelect, key: 'showRecordingControls' }, @@ -815,6 +831,9 @@ export class CloudXR2DUI { addListener(this.mediaAddressInput, 'change', updateConfig); addListener(this.mediaPortInput, 'input', updateConfig); addListener(this.mediaPortInput, 'change', updateConfig); + addListener(this.streamTestModeSelect, 'change', updateConfig); + addListener(this.streamTestDurationSecondsInput, 'input', updateConfig); + addListener(this.streamTestDurationSecondsInput, 'change', updateConfig); addListener(this.controllerModelVisibilitySelect, 'change', updateConfig); addListener(this.showTraceInXRSelect, 'change', updateConfig); addListener(this.showRecordingControlsSelect, 'change', updateConfig); @@ -1028,6 +1047,14 @@ export class CloudXR2DUI { const v = parseInt(this.mediaPortInput.value, 10); return !isNaN(v) ? v : undefined; })(), + streamTestMode: parseStreamTestMode( + this.streamTestModeSelect.value, + this.getDefaultConfiguration().streamTestMode ?? 'off' + ), + streamTestDurationSeconds: (() => { + const v = parseInt(this.streamTestDurationSecondsInput.value, 10); + return !isNaN(v) ? v : this.getDefaultConfiguration().streamTestDurationSeconds; + })(), hideControllerModel: this.controllerModelVisibilitySelect.value === 'hide', showTrace: this.showTraceInXRSelect.value === 'true', showRecordingControls: this.showRecordingControlsSelect.value === 'true', diff --git a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx index 4ca4e6f8c..1e582baa9 100644 --- a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx @@ -55,7 +55,8 @@ const FACE_CAMERA_DAMPING = 10; // Higher = faster rotation toward camera /** Display size for the Performance metrics slot (width and height passed to PerformanceCanvasImage and its container). */ const METRIC_SLOT_WIDTH = 512; -const METRIC_SLOT_HEIGHT = 250; +/** Tracks PerformanceCanvasImage's 1024x760 canvas: the session-quality card plus four metric cards. */ +const METRIC_SLOT_HEIGHT = 380; interface CloudXRUIProps { onStartTeleop?: () => void; @@ -73,10 +74,18 @@ interface CloudXRUIProps { rotation?: [number, number, number]; /** Computed signal for render FPS text - updates without React re-render */ renderFpsText?: ReadonlySignal; + /** Computed signal for pose send FPS text - the rate operator intent reaches the robot */ + poseSendFpsText?: ReadonlySignal; /** Computed signal for streaming FPS text - updates without React re-render */ streamingFpsText?: ReadonlySignal; /** Computed signal for pose-to-render latency text - updates without React re-render */ poseToRenderText?: ReadonlySignal; + /** Live session quality 0-4 ({@link CloudXR.QualityScore}); drives the HUD quality bars. */ + sessionQuality?: ReadonlySignal; + /** Network test status line; empty when no test is running or configured. */ + streamTestText?: ReadonlySignal; + /** Traffic-light color for {@link streamTestText}. */ + streamTestColor?: ReadonlySignal; /** From settings: hide control panel when immersive XR begins. */ panelHiddenAtStart?: boolean; /** Immersive XR active; used to apply panelHiddenAtStart on session enter. */ @@ -150,8 +159,12 @@ export default function CloudXR3DUI({ position = [1.8, 1.75, -1.3], rotation = [0, 0, 0], // Note: Y rotation is controlled by face-camera logic renderFpsText, + poseSendFpsText, streamingFpsText, poseToRenderText, + sessionQuality, + streamTestText, + streamTestColor, panelHiddenAtStart = false, isXRMode = false, showRecordingControls = false, @@ -433,8 +446,10 @@ export default function CloudXR3DUI({ width={METRIC_SLOT_WIDTH} height={METRIC_SLOT_HEIGHT} renderFpsText={renderFpsText} + poseSendFpsText={poseSendFpsText} streamingFpsText={streamingFpsText} poseToRenderText={poseToRenderText} + sessionQuality={sessionQuality} /> @@ -533,6 +548,11 @@ export default function CloudXR3DUI({ Status: {sessionStatus} + {/* Network test status. Signals drive the text and traffic-light color; + both are empty when the test is off, which is the default. */} + + {streamTestText} + {/* Countdown Config Row */} diff --git a/deps/cloudxr/webxr_client/src/config/params.ts b/deps/cloudxr/webxr_client/src/config/params.ts index 0b3531cea..6678c077d 100644 --- a/deps/cloudxr/webxr_client/src/config/params.ts +++ b/deps/cloudxr/webxr_client/src/config/params.ts @@ -86,6 +86,8 @@ export const URL_PARAMS: UrlParam[] = [ { key: 'proxyUrl', elementId: 'proxyUrl', description: 'Proxy URL for routing (HTTPS); leave empty for direct WSS.' }, { key: 'mediaAddress', elementId: 'mediaAddress', description: 'WebRTC media server address for NAT traversal (optional).' }, { key: 'mediaPort', elementId: 'mediaPort', isValid: isNumber, description: 'WebRTC media server port (0 = auto).' }, + { key: 'streamTestMode', elementId: 'streamTestMode', isValid: oneOf('off', 'warn', 'block'), description: 'Pre-stream network test: off (default), warn, or block.' }, + { key: 'streamTestDurationSeconds', elementId: 'streamTestDurationSeconds', isValid: isNumber, description: 'Network test measurement window in seconds (ignored when off).' }, // --- Direct params: read straight from the URL by app logic (no control, never stored) --- // TURN/ICE for NAT traversal and OOB hub, typically set in USB-local mode by oob_teleop_env.py. diff --git a/deps/cloudxr/webxr_client/src/config/resolve.test.ts b/deps/cloudxr/webxr_client/src/config/resolve.test.ts index b85e76b08..f4b5ca31f 100644 --- a/deps/cloudxr/webxr_client/src/config/resolve.test.ts +++ b/deps/cloudxr/webxr_client/src/config/resolve.test.ts @@ -124,7 +124,7 @@ function sampleValid(key: string): string { const numeric = new Set([ 'port', 'deviceFrameRate', 'maxStreamingBitrateMbps', 'perEyeWidth', 'perEyeHeight', 'reprojectionGridCols', 'reprojectionGridRows', 'posePredictionFactor', - 'xrOffsetX', 'xrOffsetY', 'xrOffsetZ', 'mediaPort', + 'xrOffsetX', 'xrOffsetY', 'xrOffsetZ', 'mediaPort', 'streamTestDurationSeconds', ]); if (numeric.has(key)) return '1'; const enums: Record = { @@ -143,6 +143,7 @@ function sampleValid(key: string): string { useQuestColorWorkaround: 'true', panelHiddenAtStart: 'true', headless: 'true', + streamTestMode: 'warn', }; return enums[key] ?? 'x'; } diff --git a/deps/cloudxr/webxr_client/src/index.html b/deps/cloudxr/webxr_client/src/index.html index 3f6d3f9ca..97d68b37d 100644 --- a/deps/cloudxr/webxr_client/src/index.html +++ b/deps/cloudxr/webxr_client/src/index.html @@ -1010,6 +1010,21 @@

Advanced settings

Configure WebRTC media server address and port for NAT traversal. Only needed when the streaming server is behind NAT. Leave empty to use server-provided ICE information. + +
+ + + + + +
+ Measures latency, jitter, bandwidth and device performance before streaming starts, reporting the result on the in-XR panel. The session waits in "Connecting" for the whole window, so this is off by default; use Warn when diagnosing a link. Block additionally refuses to stream when the link fails the gate. +
+
diff --git a/docs/source/getting_started/build_from_source/webxr.rst b/docs/source/getting_started/build_from_source/webxr.rst index 651f7e659..93a2e238f 100644 --- a/docs/source/getting_started/build_from_source/webxr.rst +++ b/docs/source/getting_started/build_from_source/webxr.rst @@ -34,7 +34,7 @@ From the project root: source deps/cloudxr/webxr_client/scripts/setup_cloudxr_env.sh deps/cloudxr/webxr_client/scripts/download_cloudxr_sdk.sh -This will automatically download the CloudXR.js SDK and place it in ``deps/cloudxr/nvidia-cloudxr-6.1.0.tgz``. The +This will automatically download the CloudXR.js SDK and place it in ``deps/cloudxr/nvidia-cloudxr-6.3.0-rc2.tgz``. The `package.json` is configured to install the SDK from this local file. 2. Install dependencies @@ -44,7 +44,7 @@ From the ``deps/cloudxr/webxr_client/`` directory: .. code-block:: bash - npm install ../nvidia-cloudxr-6.1.0.tgz + npm install ../nvidia-cloudxr-6.3.0-rc2.tgz 3. Build & Run -------------- diff --git a/docs/source/references/oob_teleop_control.rst b/docs/source/references/oob_teleop_control.rst index 29f220238..3af46c6d0 100644 --- a/docs/source/references/oob_teleop_control.rst +++ b/docs/source/references/oob_teleop_control.rst @@ -241,6 +241,89 @@ Headset → hub: ``clientMetrics`` } } +One message is sent per cadence per tick, carrying the last known value of every metric +reported so far in that cadence. Metric names are the CloudXR.js ``MetricsName`` values, +so they match the SDK documentation. The hub stores them as an arbitrary +``{name: value}`` map per cadence, so metrics added by a future SDK need no hub change. + +.. list-table:: Metrics reported per cadence (CloudXR.js 6.3.0) + :header-rows: 1 + :widths: 12 40 48 + + * - Cadence + - Metric + - Meaning + * - ``render`` + - ``render.framerate`` + - Client render rate (FPS), rolling average. + * - ``render`` + - ``pose.send_framerate`` + - Rate at which poses are sent upstream (FPS), rolling average. For teleop this is + the rate operator intent reaches the robot. + * - ``render`` + - ``latency.xr_pose_age_ms`` + - Age of the XR pose at send time. + * - ``frame`` + - ``streaming.framerate`` + - Streamed video rate (FPS), rolling average. + * - ``frame`` + - ``streaming.frame_count`` + - Monotonic count of streamed frames. + * - ``frame`` + - ``render.pose_to_render_time`` + - Pose-to-render latency (ms), rolling average. + * - ``frame`` + - ``latency.pose_upload_ms`` + - Time spent uploading the pose. + * - ``frame`` + - ``latency.pose_to_frame_received_ms`` + - Time from pose send to frame received. + * - ``frame`` + - ``frame_pipeline.compositor_skipped_percent`` + - Share of frames the compositor skipped. + * - ``frame`` + - ``frame_pipeline.out_of_order_percent`` + - Share of frames that arrived out of order. + * - ``frame`` + - ``frame_pipeline.mismatched_percent`` + - Share of frames whose pose did not match the rendered pixels. + * - ``network`` + - ``network.streaming_rate_mbps`` + - Current streaming rate. + * - ``network`` + - ``network.available_bandwidth_mbps`` + - Estimated available bandwidth. + * - ``network`` + - ``network.rtt_ms`` + - Round-trip time. + * - ``network`` + - ``network.packet_loss`` + - Packet loss ratio. + * - ``network`` + - ``network.avg_decode_time_ms`` + - Average video decode time. + * - ``network`` + - ``network.quality_score`` + - Composite network quality, 0-4. + * - ``network`` + - ``network.bandwidth_score`` + - Bandwidth component of the quality score, 0-4. + * - ``network`` + - ``network.network_loss_score`` + - Loss component of the quality score, 0-4. + * - ``network`` + - ``network.latency_score`` + - Latency component of the quality score, 0-4. + * - ``network`` + - ``session.quality`` + - Overall session quality, 0-4. Also drives the in-XR quality bars. + +Score metrics use the SDK's ``QualityScore`` scale: 0 NoData, 1 Unsustainable, +2 Degraded, 3 Good, 4 Excellent. + +Metrics are cleared when the stream stops, so a new session never reports the previous +session's last known values. + HTTP API -------- diff --git a/scripts/download_cloudxr_sdk.sh b/scripts/download_cloudxr_sdk.sh index ddc4fb06c..dc1792e0d 100755 --- a/scripts/download_cloudxr_sdk.sh +++ b/scripts/download_cloudxr_sdk.sh @@ -9,7 +9,7 @@ # Three ways to obtain the SDK (tried in order): # 1) Local tarball: place cloudxr-web-sdk-${CXR_WEB_SDK_VERSION}.tar.gz in deps/cloudxr/. # The tarball must extract to the same layout as the NGC release: root must contain -# isaac/ and nvidia-cloudxr-${CXR_WEB_SDK_VERSION}.tgz (optionally inside a single top-level directory). +# nvidia-cloudxr-${CXR_WEB_SDK_VERSION}.tgz (optionally inside a single top-level directory). # 2) Public NGC: downloads via curl from the public NGC resource API. # 3) Private NGC: downloads via curl from the private NGC resource API; requires NGC_API_KEY. @@ -55,10 +55,15 @@ is_valid_sdk_bundle() { [[ -f "$dir/$SDK_FILE" ]] } -# Returns 0 if the given directory has valid SDK layout (isaac/ and nvidia-cloudxr-*.tgz) +# Returns 0 if the given directory has valid SDK layout (nvidia-cloudxr-*.tgz) +# +# Only the .tgz is consumed by Dockerfile.web-app; IsaacTeleop builds its web client from +# webxr_client/ in this repo. Earlier bundles also shipped an isaac/ directory, which is no +# longer part of the release, so requiring it here would reject otherwise valid tarballs. +# Kept in sync with is_valid_sdk_bundle(), which the NGC paths use. is_valid_sdk_layout() { local dir="$1" - [[ -d "$dir/isaac" ]] && [[ -f "$dir/$SDK_FILE" ]] + [[ -f "$dir/$SDK_FILE" ]] } # ----------------------------------------------------------------------------- @@ -81,7 +86,7 @@ install_from_local_tarball() { tar -xzf "$SDK_TARBALL" -C "$SDK_RELEASE_DIR" if ! is_valid_sdk_layout "$SDK_RELEASE_DIR"; then - echo -e "${RED}Error: Tarball layout invalid. Root must contain isaac/ and $SDK_FILE${NC}" + echo -e "${RED}Error: Tarball layout invalid. Root must contain $SDK_FILE${NC}" exit 1 fi echo -e "${GREEN}✓ CloudXR Web SDK installed from local tarball${NC}"