diff --git a/Makefile b/Makefile
index f34e3a4..e3c8348 100644
--- a/Makefile
+++ b/Makefile
@@ -422,6 +422,30 @@ sim-drive-stop: ## Stop the rover
@$(CONTAINER_ENGINE) exec $(SIM_CONTAINER) bash -c \
'ign topic -t $(GZ_DRIVE_TOPIC) -m ignition.msgs.Twist -p "linear: {x: 0}, angular: {z: 0}"'
+# Teleport the rover back to the origin. Useful when an OTA controller
+# runs the rover off the world or into the boulder. Override
+# TELEPORT_X / TELEPORT_Y / TELEPORT_Z to land somewhere else.
+SIM_GAZEBO_CONTAINER ?= temporal-hack-lab-gazebo-1
+SIM_WORLD_NAME ?= moon
+ROBOT_MODEL ?= perseverance
+TELEPORT_X ?= 0
+TELEPORT_Y ?= 0
+TELEPORT_Z ?= 0.30
+
+.PHONY: sim-teleport
+sim-teleport: ## Teleport the rover back to the origin (override TELEPORT_X/Y/Z)
+ @echo "[sim-teleport] $(ROBOT_MODEL) -> ($(TELEPORT_X), $(TELEPORT_Y), $(TELEPORT_Z))"
+ @# Stop the rover first so its old velocity doesn't get carried over.
+ @$(CONTAINER_ENGINE) exec $(SIM_GAZEBO_CONTAINER) bash -c \
+ 'ign topic -t $(GZ_DRIVE_TOPIC) -m ignition.msgs.Twist -p "linear: {x: 0}, angular: {z: 0}"' >/dev/null 2>&1 || true
+ @# set_pose service from inside the gazebo container. Orientation
+ @# is identity quaternion (w=1) — rover faces +x.
+ @$(CONTAINER_ENGINE) exec $(SIM_GAZEBO_CONTAINER) bash -c \
+ 'ign service -s /world/$(SIM_WORLD_NAME)/set_pose \
+ --reqtype ignition.msgs.Pose --reptype ignition.msgs.Boolean \
+ --timeout 2000 \
+ --req "name: \"$(ROBOT_MODEL)\", position: {x: $(TELEPORT_X), y: $(TELEPORT_Y), z: $(TELEPORT_Z)}, orientation: {w: 1.0}"'
+
# =============================================================================
# CI cluster (smoke / pre-push parity) — alternate ports so it can run
# alongside `make lab-up` on the same host. Used by .git-hooks/installer-smoke.sh
@@ -437,6 +461,44 @@ sim-drive-stop: ## Stop the rover
CI_PROJECT := temporal-hack-ci
CI_FILES := -f docker-compose.yml -f docker-compose.ci.yml
+# =============================================================================
+# Demo reset — wipes all transient demo state and (optionally) brings
+# the stack back up clean.
+#
+# make demo-reset stops everything, wipes volumes + .run/,
+# and BRINGS THE STACK BACK UP fresh
+# make demo-reset NOUP=1 same, but stops short of starting again
+# =============================================================================
+
+.PHONY: demo-reset
+demo-reset: container-check ## Stop everything, wipe demo state, and start fresh (NOUP=1 to skip the bring-up)
+ @echo "[demo-reset] stopping host-side processes"
+ -@$(MAKE) -s controlplane-down
+ -@$(MAKE) -s workers-down
+ -@$(MAKE) -s agent-down
+ @echo "[demo-reset] killing OTA-spawned robot-app containers"
+ -@$(CONTAINER_ENGINE) rm -f robot-app robot-app-new >/dev/null 2>&1 || true
+ @echo "[demo-reset] tearing down sim + lab compose, wiping volumes"
+ -@cd installer/docker-compose && $(COMPOSE) -p $(LAB_PROJECT) \
+ -f docker-compose.yml -f docker-compose.sim.yml down -v >/dev/null 2>&1 || true
+ @echo "[demo-reset] clearing .run/ pid files and logs"
+ @rm -rf .run/
+ @if [ "$${NOUP:-0}" = "1" ]; then \
+ echo "[demo-reset] NOUP=1 — stopping after teardown"; \
+ exit 0; \
+ fi
+ @echo "[demo-reset] bringing the stack back up"
+ @$(MAKE) -s sim-up
+ @$(MAKE) -s agent-up
+ @$(MAKE) -s workers-up
+ @$(MAKE) -s controlplane-up
+ @echo
+ @echo " demo reset complete. Re-run a demo:"
+ @echo " make ota-circle"
+ @echo " make collide"
+ @echo " GUI: http://localhost:14680/vnc.html?autoconnect=1&resize=scale"
+ @echo " Temporal UI: http://localhost:14080"
+
.PHONY: ci-up
ci-up: container-check ## Bring up an isolated CI/smoke cluster on alternate ports
@echo "[$(CONTAINER_ENGINE)] bringing up CI stack on alt ports"
diff --git a/README.md b/README.md
index 42f56ef..7984335 100644
--- a/README.md
+++ b/README.md
@@ -29,39 +29,187 @@ ops/ runbooks
## Service shape
+The runtime splits across **three containers** + **four host-side
+processes**. The agent owns the OTA path: it shells out to the host
+docker/podman CLI to pull, run, swap, and roll back the
+**robot-app** container — which runs alongside the others on the lab
+network and joins the same ROS DDS domain.
+
+```mermaid
+flowchart TB
+ classDef host fill:#e8f0ff,stroke:#5277b8,color:#000
+ classDef cont fill:#f4ecd8,stroke:#a98246,color:#000
+ classDef ota fill:#ffe6cc,stroke:#d79b00,color:#000
+ classDef lab fill:#dae8fc,stroke:#6c8ebf,color:#000
+ classDef ext fill:#f5f5f5,stroke:#666,color:#000
+
+ User([browser /
operator]):::ext
+
+ subgraph LabCluster["lab cluster (compose)"]
+ direction TB
+ Gz["gazebo container
ign gazebo
ros_gz_bridge
Xvfb + x11vnc + noVNC :14680"]:::cont
+ Robot["robot container
bridge_node :50051
sim_battery
collision_publisher
twist_subscriber"]:::cont
+ RobotApp["robot-app container
drive-circle | drive-figure-eight
(OTA-swappable)"]:::ota
+ MQTT[("EMQX MQTT
:14883")]:::lab
+ Tmp[("Temporal :14733
Postgres :14432
UI :14080")]:::lab
+ Reg[("Registry :14050")]:::lab
+ end
+
+ subgraph Host["host-side Go binaries (make-managed)"]
+ direction TB
+ Agent["agent
ROS gRPC client
MQTT pub/sub
OTA executor"]:::host
+ Otaw["ota-worker
Temporal worker +
MQTT bridge"]:::host
+ Colw["collision-worker
Temporal worker +
MQTT bridge"]:::host
+ Cp["controlplane
HTTP :8081"]:::host
+ Eng["docker / podman CLI
host engine"]:::host
+ end
+
+ User -->|noVNC :14680
UI :14080| LabCluster
+ User -->|POST /v1/ota/rollouts| Cp
+
+ Gz <-->|ROS DDS
domain 42| Robot
+ Gz <-->|ROS DDS
/cmd_vel| RobotApp
+ Robot <-->|gRPC| Agent
+ Agent <-->|MQTT| MQTT
+ Otaw <-->|MQTT| MQTT
+ Colw <-->|MQTT| MQTT
+ Otaw <-->|gRPC| Tmp
+ Colw <-->|gRPC| Tmp
+ Cp -->|StartWorkflow| Tmp
+
+ Agent ==>|shells| Eng
+ Eng ==>|pull / run / rename| RobotApp
+ Reg --o RobotApp
+
+ linkStyle 11 stroke:#d79b00,stroke-width:2px
+ linkStyle 12 stroke:#d79b00,stroke-width:2px
```
- ┌────────────────── browser ──────────────────┐
- │ http://localhost:14680 Gazebo GUI (noVNC) │
- └──────────────────┬──────────────────────────┘
- │
- ┌───────────────────────▼───────────────────────┐
- │ gazebo container │
- │ • ign gazebo + ros_gz_bridge │
- │ • Xvfb + x11vnc + noVNC │
- └────┬─────────────────────┬────────────────────┘
- │ ROS DDS (domain 42) │
- ┌─────────▼──────────┐ ┌───────▼────────────────────┐
- │ robot container │ │ robot-app container │
- │ • bridge_node │ │ (drive-circle | -fig-eight) │
- │ • sim_battery │ │ — OTA-swappable │
- │ • collision_pub │ └────────────────────────────┘
- │ • twist_subscriber │
- └─────────┬──────────┘
- │ gRPC (TCP)
- ┌─────────▼──────────────────────────┐
- │ agent (Go, native macOS binary) │
- │ • MQTT pub/sub on lab broker │
- │ • OTA executor (docker/podman CLI) │
- └─────────┬──────────────────────────┘
- │
- ┌─────────▼──────────┐ ┌───────────────────────┐
- │ MQTT (lab :14883) │◀──▶│ ota-worker │
- │ │ │ collision-worker │
- │ │ │ Temporal :14733 │
- └────────────────────┘ └───────────────────────┘
+
+Bold orange edges are the OTA path: the agent shells out to the host
+engine, which pulls from the lab registry and swaps the robot-app
+container in place.
+
+### OTA flow
+
+```mermaid
+sequenceDiagram
+ autonumber
+ actor Op as Operator
+ participant Cp as controlplane
+ participant Ow as ota-worker
+ participant Tmp as Temporal
+ participant MQ as MQTT
+ participant Ag as agent
+ participant Eng as host engine
+ participant App as robot-app
+
+ Op->>Cp: POST /v1/ota/rollouts
+ Cp->>Tmp: StartWorkflow OTARollout
+ Tmp-->>Ow: dispatch task
+ Ow->>MQ: publish cmd/{robot_id}/ota
+ MQ-->>Ag: deliver
+ Ag->>Eng: pull image_ref
+ Ag->>MQ: ack PHASE_PULLED
+ Ag->>Eng: run new under temp name
+ Ag->>Eng: rm old then rename new
+ Ag->>MQ: ack PHASE_SWAPPED
+ Eng->>App: container starts
+ Ag->>Eng: exec smoke_command
+ Ag->>MQ: ack PHASE_HEALTHY
+ MQ-->>Ow: signal workflow per phase
+ Ow->>Tmp: RecordRolloutEnded
+ Cp-->>Op: GET /v1/ota/rollouts shows completed
+```
+
+### Collision flow
+
+```mermaid
+sequenceDiagram
+ autonumber
+ participant Gz as gazebo
+ participant RGB as ros_gz_bridge
+ participant Pub as collision_publisher
+ participant MQ as MQTT
+ participant Cw as collision-worker
+ participant Tmp as Temporal
+ participant Sub as twist_subscriber
+ participant Rover as gz DiffDrive
+
+ Gz->>RGB: ignition.msgs.Contacts
+ RGB->>Pub: ROS /contacts
+ Pub->>MQ: events/{id}/collision after 2s debounce
+ MQ-->>Cw: deliver
+ Cw->>Tmp: StartWorkflow CollisionResponse
+ Tmp-->>Cw: dispatch SendTwist back 3s
+ loop each phase back, settle, turn, forward, stop
+ Cw->>MQ: cmd/{id}/twist at 10 Hz QoS 0
+ MQ-->>Sub: deliver
+ Sub->>Rover: ROS /cmd_vel forwards to gz cmd_vel
+ end
+ Cw->>MQ: cmd/{id}/twist 0,0 QoS 1 final stop
+ Cw->>Tmp: workflow Completed
```
-## Lab quickstart
+## Make-target interaction map
+
+The four bring-up targets in the **baseline** lane are the ones you
+run; everything else either depends on those or operates on them.
+Demo triggers (right column) need the baseline lane up to function.
+
+```mermaid
+flowchart LR
+ classDef baseline fill:#dae8fc,stroke:#6c8ebf,color:#000
+ classDef demo fill:#ffe6cc,stroke:#d79b00,color:#000
+ classDef status fill:#d5e8d4,stroke:#82b366,color:#000
+ classDef cleanup fill:#f8cecc,stroke:#b85450,color:#000
+
+ subgraph Baseline["BASELINE — bring up in any order"]
+ direction TB
+ SimUp["make sim-up
compose up: gazebo + robot + lab cluster
publishes :14050 :14080 :14432 :14680
:14733 :14883 :14900 :50051"]:::baseline
+ AgentUp["make agent-up
./bin/agent (.run/agent.pid)
BROKER_URL=tcp://localhost:14883
BRIDGE_ADDR=localhost:50051"]:::baseline
+ WorkersUp["make workers-up
./bin/ota-worker + ./bin/collision-worker
(.run/*.pid)
TEMPORAL_ADDR=localhost:14733"]:::baseline
+ CpUp["make controlplane-up
./bin/controlplane :8081
(.run/controlplane.pid)"]:::baseline
+ end
+
+ subgraph Demos["DEMO TRIGGERS"]
+ direction TB
+ OtaC["make ota-circle
build + push +
POST /v1/ota/rollouts"]:::demo
+ OtaF["make ota-figure-eight
(same shape)"]:::demo
+ Coll["make collide
publish events/{id}/collision"]:::demo
+ OtaS["make ota-status
GET /v1/ota/rollouts"]:::demo
+ Drive["make sim-drive-fwd LX= /
-back / -left / -right / -stop
(no Temporal in the loop)"]:::demo
+ end
+
+ subgraph Lifecycle["LIFECYCLE"]
+ direction TB
+ Down["make sim-down / agent-down /
workers-down / controlplane-down"]:::cleanup
+ Reset["make demo-reset
(or demo-reset NOUP=1)"]:::cleanup
+ end
+
+ subgraph Obs["STATUS / OBSERVABILITY"]
+ direction TB
+ Stat["make agent-status / workers-status
controlplane-status / lab-status"]:::status
+ Gui["make sim-gui
open noVNC URL"]:::status
+ Logs["make sim-logs
tail sim+robot+agent"]:::status
+ end
+
+ SimUp -- gazebo, robot, lab containers --> AgentUp
+ AgentUp -- shells host docker/podman --> SimUp
+ WorkersUp -- gRPC --> SimUp
+ CpUp -- gRPC --> SimUp
+
+ OtaC -- HTTP --> CpUp
+ OtaC -- via MQTT --> AgentUp
+ OtaF -- HTTP --> CpUp
+ OtaF -- via MQTT --> AgentUp
+ Coll -- MQTT publish --> WorkersUp
+ OtaS -- HTTP GET --> CpUp
+ Drive -- ign topic exec --> SimUp
+
+ Reset -. wipes + restarts .- Baseline
+```
+
+### Quickstart
Requires Go 1.22+, Python 3.10+, and either Docker or Podman with
compose. The Makefile auto-detects the container engine.
@@ -78,6 +226,8 @@ That's the whole baseline. Tear down:
```bash
make controlplane-down && make workers-down && make agent-down && make sim-down
+# OR, full wipe + restart in one shot:
+make demo-reset
```
## Drive demo (no Temporal in the loop)
@@ -106,6 +256,11 @@ What you'll see: a `rollout-…` workflow appears at
1–2 seconds, and the `robot-app` container under `podman ps` flips to
the new image. The rover's behaviour changes immediately.
+The full data path for a rollout — see the **OTA flow** sequence
+diagram in *Service shape* above. The agent is the only process that
+runs `podman pull / run / rename`; workers never touch the host
+engine, they orchestrate via MQTT.
+
## Collision demo (Temporal drives the rover out of an obstacle)
The moon world spawns the rover with a 0.9 m boulder at `x = 8` —
diff --git a/bridge/bridge_node/collision_publisher.py b/bridge/bridge_node/collision_publisher.py
index e294d0d..fb71ae9 100644
--- a/bridge/bridge_node/collision_publisher.py
+++ b/bridge/bridge_node/collision_publisher.py
@@ -26,6 +26,12 @@
DEBOUNCE_SEC = 2.0
+# Collision partners we IGNORE — these are continuous "of course
+# you're touching the ground" contacts that shouldn't trigger an
+# avoidance workflow. Substring match on the contact's collision2
+# name (e.g. "lunar_ground::link::collision").
+IGNORE_PARTNERS = ("lunar_ground", "ground_plane")
+
class CollisionPublisher(Node):
def __init__(self) -> None:
@@ -53,12 +59,23 @@ def _on_contacts(self, msg: Contacts) -> None:
# means "no contacts this step"; only act on non-empty.
if not msg.contacts:
return
+ # Pick a contact whose partner is *not* the ground — the rover
+ # stands on its wheels (or rests on its deck) so the ground is
+ # always touching something. We only care about novel obstacles.
+ partner = None
+ for c in msg.contacts:
+ name = c.collision2.name or ""
+ if any(ig in name for ig in IGNORE_PARTNERS):
+ continue
+ partner = name
+ break
+ if partner is None:
+ return
now = time.monotonic()
if now - self._last_emit < DEBOUNCE_SEC:
return
self._last_emit = now
self._counter += 1
- partner = msg.contacts[0].collision2.name if msg.contacts else "unknown"
body = json.dumps({
"robot_id": self.robot_id,
"at": time.time(),
diff --git a/cloud/cmd/collision-worker/main.go b/cloud/cmd/collision-worker/main.go
index 13f4b76..1534109 100644
--- a/cloud/cmd/collision-worker/main.go
+++ b/cloud/cmd/collision-worker/main.go
@@ -71,7 +71,12 @@ func connectMQTT(url string) (mqtt.Client, error) {
opts := mqtt.NewClientOptions().
AddBroker(url).
SetClientID("collision-worker").
- SetCleanSession(false).
+ // CleanSession=true on purpose: a missed collision event is
+ // fine (the next contact will fire another). With clean=false
+ // EMQX queues every event the bridge missed; after a storm or
+ // restart the worker reconnects to a flood of replays that
+ // blocks the publisher with backpressure.
+ SetCleanSession(true).
SetAutoReconnect(true).
SetMaxReconnectInterval(60 * time.Second).
SetOrderMatters(false)
diff --git a/cloud/internal/collision/activities.go b/cloud/internal/collision/activities.go
index f537951..7c8407b 100644
--- a/cloud/internal/collision/activities.go
+++ b/cloud/internal/collision/activities.go
@@ -34,11 +34,11 @@ func (a *Activities) SendTwist(ctx context.Context, args SendTwistArgs) error {
tick := time.NewTicker(100 * time.Millisecond)
defer tick.Stop()
for {
- // paho handles its own reconnects; don't second-guess via
- // IsConnectionOpen(), which has tight semantics around
- // reconnect windows. The token's WaitTimeout + Error are
- // the authoritative result for a single publish.
- tok := a.MQTT.Publish(topic, 1, false, body)
+ // QoS 0: twist messages are republished at 10 Hz so a dropped
+ // frame doesn't matter, and the QoS-1 PUBACK round-trip
+ // causes inflight-queue saturation against EMQX. Fire-and-
+ // forget is the right choice for high-rate control commands.
+ tok := a.MQTT.Publish(topic, 0, false, body)
if !tok.WaitTimeout(2 * time.Second) {
return fmt.Errorf("mqtt publish timeout for %s", topic)
}
@@ -55,7 +55,8 @@ func (a *Activities) SendTwist(ctx context.Context, args SendTwistArgs) error {
}
}
- // Final explicit stop frame.
+ // Final explicit stop frame (QoS 1 here so the rover always sees the
+ // stop even if the last QoS 0 packet got dropped).
tok := a.MQTT.Publish(topic, 1, false, stop)
tok.WaitTimeout(2 * time.Second)
return tok.Error()
diff --git a/specs/overview.md b/specs/overview.md
index 6d6785f..5867a8d 100644
--- a/specs/overview.md
+++ b/specs/overview.md
@@ -24,39 +24,68 @@ phase artifacts (DSM, threat model, FMEA, project plan).
## System context
+The cloud / robot split as of the demo cut. Production
+("customer DC") and dev-loop sim ("local Mac") share the same
+component shape — only the substrate (Kubernetes vs `make sim-up`)
+and the simulator role differ.
+
```mermaid
-graph LR
- subgraph customer_dc[Customer Data Center]
- cp[Control Plane
Go services]
- tc[Temporal Cluster
self-hosted]
- mq[MQTT Broker
EMQX or VerneMQ]
- cr[Container Registry
OCI images]
- ts[Telemetry Store
TSDB]
- pg[(Postgres)]
+flowchart TB
+ classDef cloud fill:#dae8fc,stroke:#6c8ebf,color:#000
+ classDef host fill:#e8f0ff,stroke:#5277b8,color:#000
+ classDef cont fill:#f4ecd8,stroke:#a98246,color:#000
+ classDef ota fill:#ffe6cc,stroke:#d79b00,color:#000
+
+ Op([operator])
+
+ subgraph CloudDC["customer DC (or local lab cluster)"]
+ direction TB
+ Cp["control plane
Go HTTP API"]:::cloud
+ Tc[("Temporal cluster
+ Postgres")]:::cloud
+ Mq[("MQTT broker
EMQX, persistent sessions")]:::cloud
+ Cr[("Container registry
OCI images")]:::cloud
+ Ts[("Telemetry store
TimescaleDB hypertable")]:::cloud
+ Tlm["telemetry-ingest
(MQTT → TSDB)"]:::cloud
+ Otaw["ota-worker
Temporal + MQTT bridge"]:::cloud
+ Colw["collision-worker
Temporal + MQTT bridge"]:::cloud
end
- subgraph robots[Robot Fleet 10–100 units]
- ag[Robot Agent Go]
- br[ROS 2 Bridge
Python or C++]
- au[Customer Autonomy
ROS 2 nodes]
+ subgraph Robot["robot (or local sim host)"]
+ direction TB
+ Gz["gazebo container
ign gazebo + ros_gz_bridge
(local sim only)"]:::cont
+ RobotInfra["robot container
bridge_node (gRPC) +
sim_battery + collision_publisher +
twist_subscriber"]:::cont
+ RobotApp["robot-app container
OTA-swappable controller
(drive-circle, drive-figure-eight,
customer apps...)"]:::ota
+ Agent["agent (Go)
MQTT pub/sub +
OTA executor"]:::host
+ Eng["docker / podman
engine"]:::host
end
- dev[Developer Workstation
Blender + Phobos + Gazebo]
-
- cp --> tc
- cp --> mq
- cp --> cr
- cp --> ts
- tc --> pg
-
- ag <-. MQTT QoS 1/2 .-> mq
- ag --> br
- br -. DDS .- au
-
- dev -. URDF .- au
- dev -. dev iteration .- au
+ Op -->|POST /v1/ota/rollouts| Cp
+ Cp -->|StartWorkflow| Tc
+ Tc <-->|workflow tasks| Otaw
+ Tc <-->|workflow tasks| Colw
+ Otaw <--> Mq
+ Colw <--> Mq
+ Tlm --> Ts
+ Tlm <--> Mq
+ Cr --o RobotApp
+
+ Agent <-->|gRPC| RobotInfra
+ Agent <-->|MQTT QoS 1| Mq
+ Agent ==>|shells| Eng
+ Eng ==>|pull / run / rename| RobotApp
+
+ Gz <-->|ROS DDS domain 42| RobotInfra
+ Gz <-->|ROS /cmd_vel via ros_gz_bridge| RobotApp
+
+ %% Edges 11 and 12 are the agent->engine->robot-app OTA path.
+ linkStyle 11 stroke:#d79b00,stroke-width:2px
+ linkStyle 12 stroke:#d79b00,stroke-width:2px
```
+In production the gazebo container is absent (real robots, real
+sensors). Everything else — agent, robot-container infra, MQTT path,
+the cloud workers — is identical between dev sim and production.
+
## Component inventory
### Cloud side (in customer DC)
@@ -73,11 +102,21 @@ graph LR
### Robot side (per Ubuntu 22.04 + Docker)
-| Component | Language | Responsibility |
-|-----------|----------|----------------|
-| Robot agent | Go | Maintain MQTT connection, buffer telemetry during disconnect, execute OTA updates, report health |
-| ROS 2 bridge node | Python or C++ (rclpy / rclcpp) | Subscribe to selected DDS topics, republish via gRPC over Unix socket |
-| Customer autonomy stack | C++ / Python (ROS 2 nodes) | Out of scope; customer's responsibility |
+The robot now runs **three** containers per host, not one. The
+agent is a sibling Go binary outside the compose lifecycle (in the
+dev demo it runs natively on the macOS host; in production it
+ships as a systemd unit).
+
+| Component | Language | Container? | Responsibility |
+|-----------|----------|-----------|----------------|
+| **gazebo** (sim only) | C++ / Python | container | ign gazebo + ros_gz_bridge + Xvfb / x11vnc / noVNC. Absent on a real robot. |
+| **robot** (always-on infra) | Python (rclpy) | container | bridge_node (rclpy → gRPC TCP :50051), sim_battery (sim only), collision_publisher (ROS /contacts → MQTT), twist_subscriber (MQTT → ROS /cmd_vel). |
+| **robot-app** (OTA target) | any (ROS 2 image) | container | The replaceable controller. drive-circle, drive-figure-eight in dev; customer autonomy code in production. The agent OTAs this image; everything else is static between releases. |
+| robot agent | Go | host binary | Maintain MQTT connection, buffer telemetry during disconnect, execute OTA updates via the host docker/podman CLI, report health. |
+
+The agent owns the path from MQTT command → host engine → robot-app.
+Workers in the cloud orchestrate via MQTT; only the agent ever
+shells out to the host's container engine.
### Developer toolchain
@@ -90,21 +129,24 @@ graph LR
### Telemetry path (robot → cloud)
-```
-ROS 2 nodes ─DDS topic─▶ ROS 2 bridge ─gRPC UDS─▶ Robot agent
- │
- local SQLite buffer
- (intermittent)
- │
- MQTT QoS 1/2
- │
- ▼
- MQTT broker
- │
- subscriber: telemetry-ingest
- │
- ▼
- Telemetry store
+```mermaid
+sequenceDiagram
+ participant Nodes as ROS 2 nodes
+ participant Bridge as bridge_node
+ participant Agent as agent
+ participant Buf as local SQLite buffer
+ participant MQ as MQTT broker
+ participant Ing as telemetry-ingest
+ participant TS as TimescaleDB
+
+ Nodes->>Bridge: DDS topic
+ Bridge->>Agent: gRPC TCP :50051
+ Agent->>Buf: append (bounded ring)
+ Note over Agent,Buf: drains while connected
+ Buf->>Agent: pop
+ Agent->>MQ: publish QoS 1 (persistent session)
+ MQ->>Ing: deliver
+ Ing->>TS: INSERT into telemetry hypertable
```
**Properties:**
@@ -117,33 +159,90 @@ ROS 2 nodes ─DDS topic─▶ ROS 2 bridge ─gRPC UDS─▶ Robot agent
### OTA path (cloud → robot)
-```
-Operator API ─▶ Control Plane ─▶ Temporal workflow (rollout)
- │
- 1) Resolve target image tag
- 2) For each robot in cohort:
- a) Publish MQTT command (with QoS 1)
- b) Wait for ACK (timer-bounded)
- c) Wait for health check
- d) On failure → rollback child workflow
- e) Record outcome
- │
- ▼
-Robot agent ─▶ pull image from registry (over MQTT-signaled URL)
- ─▶ swap container
- ─▶ run smoke check
- ─▶ ACK / NACK via MQTT
- ─▶ on failure: revert to previous container, signal NACK
+```mermaid
+sequenceDiagram
+ autonumber
+ actor Op as Operator
+ participant Cp as control plane
+ participant Tmp as Temporal
+ participant Ow as ota-worker
+ participant MQ as MQTT
+ participant Ag as agent
+ participant Eng as host engine
+ participant App as robot-app
+
+ Op->>Cp: POST /v1/ota/rollouts
+ Cp->>Tmp: StartWorkflow OTARollout
+ Tmp->>Ow: dispatch task
+ loop per cohort phase canary then 25% then rest
+ Ow->>MQ: publish cmd/{robot_id}/ota
+ MQ->>Ag: deliver
+ Ag->>Eng: pull image_ref from registry
+ Ag->>MQ: ack PHASE_PULLED
+ Ag->>Eng: run new under temp name then verify then rm old then rename
+ Ag->>MQ: ack PHASE_SWAPPED
+ Eng->>App: container starts
+ Ag->>Eng: exec smoke_command
+ Ag->>MQ: ack PHASE_HEALTHY
+ MQ->>Ow: signal workflow per phase
+ end
+ Ow->>Tmp: RecordRolloutEnded
```
**Properties:**
- Rollout cohort policy (canary, batched, full-fleet) lives in the
Temporal workflow definition.
-- Rollback is a child workflow with its own retry semantics.
+- Rollback is a child workflow with its own retry semantics; the
+ agent always emits PHASE_ROLLED_BACK so the rollback workflow
+ terminates instead of timing out.
- Image signature verification is an mTLS-shaped seam (see D-11):
v1 uses TLS to the registry only; production gates require signed
images verified by a customer-controlled key.
+### Collision-response path (robot → cloud → robot)
+
+This is the demo wiring used to show that a Temporal workflow can
+own a recovery sequence in response to a robot-side event.
+
+```mermaid
+sequenceDiagram
+ autonumber
+ participant Sim as gazebo
+ participant RGB as ros_gz_bridge
+ participant Pub as collision_publisher
+ participant MQ as MQTT
+ participant Cw as collision-worker
+ participant Tmp as Temporal
+ participant Sub as twist_subscriber
+ participant Drive as gz DiffDrive
+
+ Sim->>RGB: ignition.msgs.Contacts
+ RGB->>Pub: ROS /contacts
+ Pub->>Pub: filter ground contacts then 2s debounce
+ Pub->>MQ: events/{id}/collision
+ MQ->>Cw: deliver
+ Cw->>Tmp: StartWorkflow CollisionResponse
+ loop each phase back, settle, turn, forward, stop
+ Tmp->>Cw: dispatch SendTwist activity
+ Cw->>MQ: cmd/{id}/twist at 10 Hz QoS 0
+ MQ->>Sub: deliver
+ Sub->>Drive: ROS /cmd_vel forwards to gz cmd_vel
+ end
+ Cw->>MQ: cmd/{id}/twist 0,0 QoS 1 final stop
+ Cw->>Tmp: workflow Completed
+```
+
+**Properties:**
+- Twist commands ride MQTT QoS 0 at 10 Hz; the final stop frame is
+ QoS 1 so the rover always settles even if a tail QoS 0 packet
+ drops.
+- Workers run with `clean_session=true`. A missed collision event
+ is fine (the next contact emits another); persistent-session
+ replay floods the bridge after restart.
+- The collision_publisher filters out ground-plane contacts so the
+ rover's resting weight doesn't trigger a workflow on every sim
+ tick.
+
## Constraints (carried forward from Phase 0)
1. Architecture must function under intermittent connectivity (D-04).
@@ -162,19 +261,18 @@ Robot agent ─▶ pull image from registry (over MQTT-signaled URL)
6. v1 scope is Telemetry + OTA. Mission dispatch and teleop are
non-goals (D-02).
-## Open architecture decisions (ADR placeholders)
-
-To be resolved in Phase 2:
+## Closed architectural decisions (ADRs)
-| ADR | Topic | Constraint |
-|-----|-------|------------|
-| ADR-001 | MQTT broker selection | EMQX vs VerneMQ vs Mosquitto cluster; persistent session capacity, HA model |
-| ADR-002 | Container registry | Harbor / Distribution / Zot; signing / verification path |
-| ADR-003 | Telemetry storage | TimescaleDB vs VictoriaMetrics vs Prometheus + Mimir; retention / cardinality |
-| ADR-004 | Bridge node language | Python (rclpy, faster delivery) vs C++ (rclcpp, more efficient) |
-| ADR-005 | Installer toolchain | Helm-on-k3s vs Ansible vs custom |
-| ADR-006 | Local-buffer durability format on robot | SQLite vs filesystem queue vs embedded NATS |
-| ADR-007 | OTA artifact swap mechanism | Recreate vs blue-green container; health check definition |
+| ADR | Decision | Status |
+|-----|----------|--------|
+| ADR-001 | MQTT broker = EMQX 5.x (lab) | accepted; see `specs/adr/ADR-001-mqtt-broker.md` |
+| ADR-002 | Container registry = Distribution (`registry:2`) | accepted |
+| ADR-003 | Telemetry storage = TimescaleDB hypertable | accepted |
+| ADR-004 | Bridge node = Python rclpy | accepted |
+| ADR-005 | Installer toolchain | open (Sprint 8) |
+| ADR-006 | Local-buffer durability format on robot = SQLite (WAL) | accepted (in code) |
+| ADR-007 | OTA artifact swap mechanism = blue-green | accepted; see `specs/adr/ADR-007-ota-swap-strategy.md` |
+| ADR-008 | OTA command/ack transport = MQTT topic pair | accepted; see `specs/adr/ADR-008-ota-command-transport.md` |
## Risks (from Phase 0)
@@ -196,7 +294,10 @@ See `decisions.md` for the full risk register. Highest-impact items:
## Phase 1 status
-This overview is the Phase 1 deliverable. Next phase (DSM) analyzes
-module boundaries between the Go cloud control plane, the Go robot
-agent, and the bridge node — looking for premature coupling and the
-right place to draw the gRPC contract between agent and bridge.
+This overview is the Phase 1 deliverable, refreshed after the demo
+cut (gazebo+robot container split, OTA-swappable robot-app images,
+CollisionResponse workflow). Next phase (DSM) analyzes module
+boundaries between the Go cloud workers, the agent, and the
+robot-side ROS code — looking for premature coupling and the right
+place to draw the contracts (gRPC between agent and bridge_node;
+MQTT topic schemas between agent and workers).