Data Flow Metrics

docs.openc3.com/docs/tools/cmd-tlm-server.md

@@ -73,6 +73,12 @@ The Data Flows tab displays all the Interface data flows through the system.
 
 ![Data Flows](/img/cmd_tlm_server/data_flows.png)
 
+Click an Interface or Router node to see its processing metrics below the properties and raw data. Click a Target Processing node on the left to see metrics for that target's services, including decommutation, logging, and database ingestion. Each service instance is listed separately so a slow instance is visible. The same metrics are available in the connection maps opened from the Interfaces and Routers tabs.
+
+Nodes show an amber or red outline and a **Falling behind** indicator when a reported queue delay reaches the same thresholds that trigger the server's falling behind notifications: 5 seconds (amber) or 30 seconds (red) by default, set by `OPENC3_LAG_WARN_SECONDS` and `OPENC3_LAG_CRITICAL_SECONDS`. If you change these, set them on both the openc3-operator and openc3-cosmos-cmd-tlm-api containers so the notifications and the indicators agree. Nodes are also flagged when a processing duration reaches 1 second (amber) or 5 seconds (red), or buffered input reaches 1 MiB (amber) or 10 MiB (red). Click the indicator to see the affected metric and service. These thresholds are diagnostic indicators; normal buffering and processing times depend on your configuration and traffic.
+
+Metrics refresh approximately every 5 seconds. Reports older than 30 seconds are marked **Stale metrics**, and request failures show **Metrics unavailable**. Services with no delay or buffer samples show **No delay samples**. Measurements describe the last processed data, so idle services retain their last measurement; they do not measure the current age of unprocessed data. Counters are cumulative and reset when a service restarts. Use the pause button in either details dialog to freeze the displayed values for inspection.
+
 You can disconnect the interface from the COSMOS Processing by clicking and deleting the lines connecting the Processing to the Interface. This will leave the interface "connected" but no commands are sent out or telemetry processed. Removing the lines from the Interface to the Target will effectively "unmap" the interface and restart it. This may require re-installing the plugin to restore the interface.
 
 From the Details dialog you can also view details about the individual Protocols applied to the Interface. In the EXAMPLE_INT Interface, when you click on the Length protocol you can see the details of the protocol.

openc3-cosmos-init/plugins/packages/openc3-cosmos-tool-cmdtlmserver/package.json

@@ -4,6 +4,7 @@
   "private": true,
   "type": "module",
   "scripts": {
+    "test": "node --test tests/*.test.js",
     "serve": "vite build --watch --mode dev-server",
     "build": "vite build",
     "lint": "eslint src",

openc3-cosmos-init/plugins/packages/openc3-cosmos-tool-cmdtlmserver/src/tools/CmdTlmServer/InterfaceComponents/CosmosNode.vue

@@ -12,7 +12,7 @@
 -->
 
 <template>
-  <div class="cosmos-node">
+  <div class="cosmos-node" :class="`flow-node--${health.state}`">
     <!-- Input handles on the left -->
     <Handle
       id="cosmos__cmd"
@@ -35,35 +35,47 @@
     <v-icon color="#10b981">mdi-rocket-launch</v-icon>
     <div class="node-label">{{ data.label }}</div>
     <div class="node-label">Processing</div>
+    <FlowHealthIndicator
+      :health="health"
+      :name="data.label"
+      @details="$emit('details')"
+    />
   </div>
 </template>
 
 <script>
 import { Handle } from '@vue-flow/core'
+import FlowHealthIndicator from './FlowHealthIndicator.vue'
+import { flowHealth } from './flowMetrics'
 
 export default {
-  name: 'TargetNode',
+  name: 'CosmosNode',
   components: {
     Handle,
+    FlowHealthIndicator,
   },
   props: {
+    health: { type: Object, default: () => flowHealth() },
     data: {
       type: Object,
       required: true,
     },
   },
+  emits: ['details'],
 }
 </script>
 
 <style scoped>
 .cosmos-node {
+  position: relative;
+  cursor: pointer;
   background: #ffffff;
   border: 2px solid #10b981;
   border-radius: 8px;
   padding: 10px;
   text-align: center;
   box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
-  height: 200px;
+  height: 212px;
   width: 200px;
 }
 

openc3-cosmos-init/plugins/packages/openc3-cosmos-tool-cmdtlmserver/src/tools/CmdTlmServer/InterfaceComponents/DetailsTable.vue

@@ -31,49 +31,59 @@
       </v-btn>
     </v-card-title>
     <v-card-text class="overflow-y-auto flex-grow-1 pa-0">
-      <div class="details-table">
-        <v-row>
-          <v-col cols="4">
-            <v-data-table
-              v-if="shownDetails"
-              :headers="headers"
-              :items="detailsItems"
-              :items-per-page="-1"
-              hide-default-footer
-              density="compact"
-              class="details-table"
-            >
-              <template #item.value="{ item }">
-                <span class="text-wrap">{{ item.value }}</span>
-              </template>
-            </v-data-table>
-            <div v-else class="d-flex justify-center align-center pa-4">
-              <v-progress-circular indeterminate color="primary" />
-            </div>
-          </v-col>
-          <v-col cols="8">
-            <div>{{ upperData.label }}: {{ upperData.time }}</div>
-            <raw-buffer v-model="upperData.data" />
-            <div>{{ lowerData.label }}: {{ lowerData.time }}</div>
-            <raw-buffer v-model="lowerData.data" />
-          </v-col>
-        </v-row>
+      <div class="details-table d-flex">
+        <div class="details-properties">
+          <v-data-table
+            v-if="shownDetails"
+            :headers="headers"
+            :items="detailsItems"
+            :items-per-page="-1"
+            hide-default-footer
+            density="compact"
+            class="details-table"
+          >
+            <template #item.value="{ item }">
+              <span class="text-wrap">{{ item.value }}</span>
+            </template>
+          </v-data-table>
+          <div v-else class="d-flex justify-center align-center pa-4">
+            <v-progress-circular indeterminate color="primary" />
+          </div>
+        </div>
+        <div class="details-raw pa-3">
+          <div>{{ upperData.label }}: {{ upperData.time }}</div>
+          <raw-buffer v-model="upperData.data" />
+          <div>{{ lowerData.label }}: {{ lowerData.time }}</div>
+          <raw-buffer v-model="lowerData.data" />
+        </div>
       </div>
+      <FlowMetricsPanel
+        v-if="
+          health && readProtocolIndex === null && writeProtocolIndex === null
+        "
+        :health="paused ? pausedHealth : health"
+      />
     </v-card-text>
   </v-card>
 </template>
 
 <script>
 import Updater from '../Updater'
 import RawBuffer from '../RawBuffer.vue'
+import FlowMetricsPanel from './FlowMetricsPanel.vue'
 
 export default {
   name: 'DetailsTable',
   components: {
     RawBuffer,
+    FlowMetricsPanel,
   },
   mixins: [Updater],
   props: {
+    health: {
+      type: Object,
+      default: null,
+    },
     mode: {
       type: String,
       default: 'Interface',
@@ -100,6 +110,7 @@ export default {
         { title: 'Value', key: 'value', width: '70%' },
       ],
       paused: false,
+      pausedHealth: null,
       updatedDetails: null,
     }
   },
@@ -199,6 +210,7 @@ export default {
   },
   methods: {
     pause: function () {
+      this.pausedHealth = this.health
       this.paused = !this.paused
     },
     closeClick: function () {
@@ -242,4 +254,15 @@ export default {
   overflow-x: hidden;
   overflow-y: hidden;
 }
+/* Properties take whatever width the raw data leaves */
+.details-properties {
+  flex: 1 1 0;
+  min-width: 250px;
+}
+/* RawBuffer picks 16 bytes per line when its textarea is over 770px wide,
+   so hold the raw data at that width rather than stretching it */
+.details-raw {
+  flex: 0 1 820px;
+  min-width: 0;
+}
 </style>

openc3-cosmos-init/plugins/packages/openc3-cosmos-tool-cmdtlmserver/src/tools/CmdTlmServer/InterfaceComponents/FlowHealthIndicator.vue

@@ -0,0 +1,61 @@
+<!-- Copyright 2026 OpenC3, Inc. All Rights Reserved. See LICENSE.md. -->
+
+<template>
+  <button
+    type="button"
+    class="flow-health nodrag nopan"
+    :class="`flow-health--${health.state}`"
+    :title="`${health.label}. Click for processing metrics.`"
+    :aria-label="`${name}: ${health.label}. View processing metrics`"
+    data-test="flow-health"
+    @click.stop="$emit('details')"
+  >
+    <v-icon size="14" :color="health.state === 'ok' ? 'success' : undefined">
+      {{ health.icon }}
+    </v-icon>
+    {{ health.state === 'ok' ? 'View metrics' : health.label }}
+    <v-icon size="14">mdi-chevron-right</v-icon>
+  </button>
+</template>
+
+<script setup>
+defineProps({
+  health: { type: Object, required: true },
+  name: { type: String, required: true },
+})
+defineEmits(['details'])
+</script>
+
+<style scoped>
+.flow-health {
+  position: absolute;
+  bottom: 4px;
+  left: 50%;
+  transform: translateX(-50%);
+  display: flex;
+  align-items: center;
+  gap: 3px;
+  padding: 0 6px;
+  border-radius: 4px;
+  white-space: nowrap;
+  font-size: 11px;
+  line-height: 16px;
+  cursor: pointer;
+  color: #374151;
+  background: #f3f4f6;
+}
+.flow-health--warning,
+.flow-health--stale,
+.flow-health--unavailable {
+  background: #fff3cd;
+  color: #713f12;
+}
+.flow-health--critical {
+  background: #fee2e2;
+  color: #991b1b;
+}
+.flow-health:focus-visible {
+  outline: 2px solid #1d85e1;
+  outline-offset: 2px;
+}
+</style>

openc3-cosmos-init/plugins/packages/openc3-cosmos-tool-cmdtlmserver/src/tools/CmdTlmServer/InterfaceComponents/FlowMetricsPanel.vue

@@ -0,0 +1,89 @@
+<!-- Copyright 2026 OpenC3, Inc. All Rights Reserved. See LICENSE.md. -->
+
+<template>
+  <section class="pa-4" data-test="flow-metrics">
+    <div class="d-flex align-center flex-wrap ga-2 mb-3">
+      <h3 class="text-subtitle-1">Processing metrics</h3>
+      <v-chip :color="health.color" :prepend-icon="health.icon" size="small">
+        {{ health.label }}
+      </v-chip>
+    </div>
+    <v-alert
+      v-if="health.state === 'unavailable'"
+      type="warning"
+      variant="tonal"
+      density="compact"
+      class="mb-3"
+    >
+      Unable to refresh metrics. Any values below are from the last successful
+      request. Retrying automatically.
+    </v-alert>
+    <p class="text-body-2 mb-3">
+      Latest reported measurements; services publish about every 5 seconds.
+      Queue delay warnings start at {{ health.lag.warning }} s (red at
+      {{ health.lag.critical }} s), matching the server's falling behind
+      notifications. Duration warnings start at {{ durationWarning }} s (red at
+      {{ durationCritical }} s). Buffered input warnings start at 1 MiB (red at
+      10 MiB). Reports older than {{ staleSeconds }} seconds are stale. Idle
+      services retain their last measurement. Totals reset on restart.
+    </p>
+    <v-data-table
+      :headers="headers"
+      :items="health.rows"
+      :items-per-page="-1"
+      item-value="id"
+      density="compact"
+      hide-default-footer
+      no-data-text="No metrics reported yet for this node."
+    >
+      <template #item.service="{ item }">
+        <span class="text-break">{{ item.service }}</span>
+      </template>
+      <template #item.label="{ item }">
+        <span :title="`${item.metric}: ${item.help}`">{{ item.label }}</span>
+      </template>
+      <template #item.value="{ item }">
+        <span class="metric-value">{{ item.value }}</span>
+      </template>
+      <template #item.status="{ item }">
+        <v-chip
+          :color="health.state === 'unavailable' ? 'warning' : item.color"
+          size="small"
+          variant="tonal"
+        >
+          {{ health.state === 'unavailable' ? 'Unavailable' : item.status }}
+        </v-chip>
+      </template>
+      <template #item.age="{ item }">
+        {{ item.age === null ? 'Unknown' : `${Math.floor(item.age)} s ago` }}
+      </template>
+    </v-data-table>
+  </section>
+</template>
+
+<script setup>
+import {
+  DURATION_WARNING_SECONDS,
+  DURATION_CRITICAL_SECONDS,
+  STALE_SECONDS,
+} from './flowMetrics'
+
+defineProps({ health: { type: Object, required: true } })
+const durationWarning = DURATION_WARNING_SECONDS
+const durationCritical = DURATION_CRITICAL_SECONDS
+const staleSeconds = STALE_SECONDS
+const headers = [
+  { title: 'Service', key: 'service' },
+  { title: 'Metric', key: 'label' },
+  { title: 'Value', key: 'value', sortable: false },
+  { title: 'Status', key: 'status' },
+  { title: 'Report age', key: 'age' },
+]
+</script>
+
+<style scoped>
+.metric-value {
+  white-space: nowrap;
+  font-variant-numeric: tabular-nums;
+}
+</style>