From 264c4722e3a461e922bce89335d79b94d0e32526 Mon Sep 17 00:00:00 2001
From: Ben Kearns <35475+bkearns@users.noreply.github.com>
Date: Tue, 5 May 2026 16:04:47 -0700
Subject: [PATCH 1/7] =?UTF-8?q?feat(make):=20demo-reset=20=E2=80=94=20sing?=
=?UTF-8?q?le=20command=20to=20wipe=20state=20and=20restart?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Stops the host-side processes (controlplane, workers, agent),
removes any OTA-spawned robot-app containers, tears down the sim+lab
compose project with -v (drops Temporal history, ota_rollouts, the
telemetry hypertable, the registry contents, MQTT persistent
sessions, and the agent SQLite buffer), clears .run/, and brings
the stack back up clean.
make demo-reset # wipe + restart
make demo-reset NOUP=1 # wipe and stop (skip the bring-up)
Each step is best-effort (- prefix) so partial earlier teardown
doesn't block the next one.
---
Makefile | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/Makefile b/Makefile
index f34e3a4..a934147 100644
--- a/Makefile
+++ b/Makefile
@@ -437,6 +437,44 @@ sim-drive-stop: ## Stop the rover
CI_PROJECT := temporal-hack-ci
CI_FILES := -f docker-compose.yml -f docker-compose.ci.yml
+# =============================================================================
+# Demo reset — wipes all transient demo state and (optionally) brings
+# the stack back up clean.
+#
+# make demo-reset stops everything, wipes volumes + .run/,
+# and BRINGS THE STACK BACK UP fresh
+# make demo-reset NOUP=1 same, but stops short of starting again
+# =============================================================================
+
+.PHONY: demo-reset
+demo-reset: container-check ## Stop everything, wipe demo state, and start fresh (NOUP=1 to skip the bring-up)
+ @echo "[demo-reset] stopping host-side processes"
+ -@$(MAKE) -s controlplane-down
+ -@$(MAKE) -s workers-down
+ -@$(MAKE) -s agent-down
+ @echo "[demo-reset] killing OTA-spawned robot-app containers"
+ -@$(CONTAINER_ENGINE) rm -f robot-app robot-app-new >/dev/null 2>&1 || true
+ @echo "[demo-reset] tearing down sim + lab compose, wiping volumes"
+ -@cd installer/docker-compose && $(COMPOSE) -p $(LAB_PROJECT) \
+ -f docker-compose.yml -f docker-compose.sim.yml down -v >/dev/null 2>&1 || true
+ @echo "[demo-reset] clearing .run/ pid files and logs"
+ @rm -rf .run/
+ @if [ "$${NOUP:-0}" = "1" ]; then \
+ echo "[demo-reset] NOUP=1 — stopping after teardown"; \
+ exit 0; \
+ fi
+ @echo "[demo-reset] bringing the stack back up"
+ @$(MAKE) -s sim-up
+ @$(MAKE) -s agent-up
+ @$(MAKE) -s workers-up
+ @$(MAKE) -s controlplane-up
+ @echo
+ @echo " demo reset complete. Re-run a demo:"
+ @echo " make ota-circle"
+ @echo " make collide"
+ @echo " GUI: http://localhost:14680/vnc.html?autoconnect=1&resize=scale"
+ @echo " Temporal UI: http://localhost:14080"
+
.PHONY: ci-up
ci-up: container-check ## Bring up an isolated CI/smoke cluster on alternate ports
@echo "[$(CONTAINER_ENGINE)] bringing up CI stack on alt ports"
From 21fdc1cdcc228c0a1a47abca608cf31caccfefa1 Mon Sep 17 00:00:00 2001
From: Ben Kearns <35475+bkearns@users.noreply.github.com>
Date: Tue, 5 May 2026 16:15:15 -0700
Subject: [PATCH 2/7] fix(collision): ignore ground contacts + QoS 0 twists +
clean session
User reported the collision demo storming Temporal with ~150
concurrent CollisionResponse workflows, every one of them failing.
Three issues compounded:
1. The rover's deck rests on the ground (the wheel-joint physics
isn't carrying the body weight in our simplified SDF), so the
contact sensor fires on every sim step. The 2s debounce in
collision_publisher.py only suppressed to 1 workflow per 2s,
which was still 30 workflows/min.
Fix: ignore contact partners whose collision name contains
'lunar_ground' or 'ground_plane'. Real obstacles (the boulder,
future robots) still trigger. Long-term we should also fix the
wheel-joint physics so the deck doesn't ride the ground; tracked
as a follow-up.
2. SendTwist activity used QoS 1 publishes at 10 Hz. Each PUBACK
round-trip held a slot in the inflight queue; against EMQX with
a flooded session this saturated quickly and Wait timed out.
Fix: QoS 0 for the streaming twists (loss is fine, they're
republished at 10 Hz). The trailing 0,0 stop frame stays QoS 1
so the rover sees a definitive stop even if a tail QoS 0 packet
was dropped.
3. SetCleanSession(false) on the collision-worker meant EMQX held
onto every events/+/collision payload across reconnects. After
the storm + worker restarts, the worker came back to a flood of
replays that occupied the bridge thread; outbound publishes
contended on the connection.
Fix: SetCleanSession(true). A missed collision event is fine
(the next contact fires another); durability across restart is
not a property we want for cloud workers.
Plus: stale collision-worker / ota-worker processes from earlier
debug runs were polling the same task queue alongside the new
managed PIDs in .run/. Workflow activities got picked up by the
zombie worker that had no MQTT connection. The session manifested
as 'mqtt publish timeout' errors attributed to a WorkerID we
weren't running anymore. Cleared by 'pkill -f bin/collision-worker'
once.
Verified end-to-end: post-fix, 'make collide' produces exactly one
CollisionResponse workflow per call; status=Completed in 12s; rover
visibly executes back -> turn -> forward in the noVNC view.
Also adds 'make demo-reset' target which stops everything, wipes
volumes + .run/, and brings the stack back up clean (NOUP=1 to
skip the bring-up).
---
bridge/bridge_node/collision_publisher.py | 19 ++++++++++++++++++-
cloud/cmd/collision-worker/main.go | 7 ++++++-
cloud/internal/collision/activities.go | 13 +++++++------
3 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/bridge/bridge_node/collision_publisher.py b/bridge/bridge_node/collision_publisher.py
index e294d0d..fb71ae9 100644
--- a/bridge/bridge_node/collision_publisher.py
+++ b/bridge/bridge_node/collision_publisher.py
@@ -26,6 +26,12 @@
DEBOUNCE_SEC = 2.0
+# Collision partners we IGNORE — these are continuous "of course
+# you're touching the ground" contacts that shouldn't trigger an
+# avoidance workflow. Substring match on the contact's collision2
+# name (e.g. "lunar_ground::link::collision").
+IGNORE_PARTNERS = ("lunar_ground", "ground_plane")
+
class CollisionPublisher(Node):
def __init__(self) -> None:
@@ -53,12 +59,23 @@ def _on_contacts(self, msg: Contacts) -> None:
# means "no contacts this step"; only act on non-empty.
if not msg.contacts:
return
+ # Pick a contact whose partner is *not* the ground — the rover
+ # stands on its wheels (or rests on its deck) so the ground is
+ # always touching something. We only care about novel obstacles.
+ partner = None
+ for c in msg.contacts:
+ name = c.collision2.name or ""
+ if any(ig in name for ig in IGNORE_PARTNERS):
+ continue
+ partner = name
+ break
+ if partner is None:
+ return
now = time.monotonic()
if now - self._last_emit < DEBOUNCE_SEC:
return
self._last_emit = now
self._counter += 1
- partner = msg.contacts[0].collision2.name if msg.contacts else "unknown"
body = json.dumps({
"robot_id": self.robot_id,
"at": time.time(),
diff --git a/cloud/cmd/collision-worker/main.go b/cloud/cmd/collision-worker/main.go
index 13f4b76..1534109 100644
--- a/cloud/cmd/collision-worker/main.go
+++ b/cloud/cmd/collision-worker/main.go
@@ -71,7 +71,12 @@ func connectMQTT(url string) (mqtt.Client, error) {
opts := mqtt.NewClientOptions().
AddBroker(url).
SetClientID("collision-worker").
- SetCleanSession(false).
+ // CleanSession=true on purpose: a missed collision event is
+ // fine (the next contact will fire another). With clean=false
+ // EMQX queues every event the bridge missed; after a storm or
+ // restart the worker reconnects to a flood of replays that
+ // blocks the publisher with backpressure.
+ SetCleanSession(true).
SetAutoReconnect(true).
SetMaxReconnectInterval(60 * time.Second).
SetOrderMatters(false)
diff --git a/cloud/internal/collision/activities.go b/cloud/internal/collision/activities.go
index f537951..7c8407b 100644
--- a/cloud/internal/collision/activities.go
+++ b/cloud/internal/collision/activities.go
@@ -34,11 +34,11 @@ func (a *Activities) SendTwist(ctx context.Context, args SendTwistArgs) error {
tick := time.NewTicker(100 * time.Millisecond)
defer tick.Stop()
for {
- // paho handles its own reconnects; don't second-guess via
- // IsConnectionOpen(), which has tight semantics around
- // reconnect windows. The token's WaitTimeout + Error are
- // the authoritative result for a single publish.
- tok := a.MQTT.Publish(topic, 1, false, body)
+ // QoS 0: twist messages are republished at 10 Hz so a dropped
+ // frame doesn't matter, and the QoS-1 PUBACK round-trip
+ // causes inflight-queue saturation against EMQX. Fire-and-
+ // forget is the right choice for high-rate control commands.
+ tok := a.MQTT.Publish(topic, 0, false, body)
if !tok.WaitTimeout(2 * time.Second) {
return fmt.Errorf("mqtt publish timeout for %s", topic)
}
@@ -55,7 +55,8 @@ func (a *Activities) SendTwist(ctx context.Context, args SendTwistArgs) error {
}
}
- // Final explicit stop frame.
+ // Final explicit stop frame (QoS 1 here so the rover always sees the
+ // stop even if the last QoS 0 packet got dropped).
tok := a.MQTT.Publish(topic, 1, false, stop)
tok.WaitTimeout(2 * time.Second)
return tok.Error()
From d8c1daed9813d7c2a575fc0474a6f2e375b7d9a6 Mon Sep 17 00:00:00 2001
From: Ben Kearns <35475+bkearns@users.noreply.github.com>
Date: Tue, 5 May 2026 16:43:40 -0700
Subject: [PATCH 3/7] docs(README): real overview diagram + make-target
interaction map
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The old service-shape diagram lumped 'agent' as a passive MQTT
client and didn't show that the agent is what shells out to the host
docker/podman CLI to pull, run, and swap the robot-app container.
Two readers had to ask 'wait, who actually performs the OTA?' so
the diagram clearly wasn't doing its job.
Rewrites:
1. Service shape diagram is now explicit about three containers
(gazebo, robot, robot-app) + four host binaries (agent,
ota-worker, collision-worker, controlplane) and which arrows
are gRPC vs MQTT vs ROS DDS. Calls out that the agent owns
the host-engine path; workers never touch docker/podman.
2. Two flow paragraphs trace OTA and collision-response end to
end so a reader sees how a workflow actually drives the rover.
3. New make-target interaction map: shows the four baseline lanes
(sim-up / agent-up / workers-up / controlplane-up) and which
demo triggers depend on which lanes. Stops people running
make ota-circle without controlplane-up first.
4. The OTA demo section gets an inline data-path diagram showing
build → push → POST → workflow → MQTT cmd → agent → host
podman → robot-app, with the back-channel ack flow.
---
README.md | 190 +++++++++++++++++++++++++++++++++++++++++++++---------
1 file changed, 160 insertions(+), 30 deletions(-)
diff --git a/README.md b/README.md
index 42f56ef..15a1aff 100644
--- a/README.md
+++ b/README.md
@@ -29,39 +29,134 @@ ops/ runbooks
## Service shape
+The runtime splits across **three containers** + **four host-side
+processes**. The agent owns the OTA path: it shells out to the host
+docker/podman CLI to pull, run, swap, and roll back the
+**robot-app** container — which runs alongside the others on the lab
+network and joins the same ROS DDS domain.
+
+```
+ ─── browser ──────────────────────────────────────────────────────
+ http://localhost:14680 Gazebo GUI (noVNC) :14080 Temporal UI
+ ──────────┬────────────────────────────────────────────────┬─────
+ │ │
+ ┌─────────▼────────────────────┐ ┌─────────────────▼────────┐
+ │ gazebo container │ │ Temporal cluster (lab) │
+ │ • ign gazebo │ │ :14733 frontend │
+ │ • ros_gz_bridge │ │ Postgres :14432 │
+ │ • Xvfb + x11vnc + noVNC │ │ EMQX MQTT :14883 │
+ └────┬─────────────────────────┘ │ Registry :14050 │
+ │ ROS DDS (domain 42, cyclonedds) └────┬───────────┬─────────┘
+ │ │ gRPC │ MQTT
+ ┌────▼─────────────────────┐ ┌────────────┐ │ │
+ │ robot container │ │ robot-app │ │ │
+ │ (always-on infra) │ │ container │ │ │
+ │ • bridge_node (gRPC) │ │ drive- │ │ │
+ │ • sim_battery │ │ circle | │ │ │
+ │ • collision_publisher │ │ drive- │ │ │
+ │ • twist_subscriber │ │ fig-eight │ │ │
+ └────┬─────────────────────┘ └─────▲──────┘ │ │
+ │ gRPC (robot:50051) │ │ │
+ │ tunneled to host :50051 │ podman pull/run/swap│
+ │ │ │
+ ┌────▼──────────────────────────────┴──────────────────┐ │
+ │ host-side Go binaries (managed by `make`) │ │
+ │ agent — ROS bridge ↔ MQTT, OTA executor │◀─┤
+ │ ota-worker — Temporal worker + MQTT bridge │◀─┤
+ │ collision-worker— Temporal worker + MQTT bridge │◀─┘
+ │ controlplane — HTTP API on :8081 (POST rollouts) │
+ └──────────────────────────────────────────────────────┘
+ │
+ │ docker / podman CLI on the macOS host
+ ▼
+ ┌────────────────────────┐
+ │ podman engine on host │
+ │ manages robot-app │
+ │ container in the lab │
+ │ network │
+ └────────────────────────┘
+```
+
+Two flows worth tracing:
+
+**OTA rollout.** Operator → `controlplane` POST `/v1/ota/rollouts` →
+`ota-worker` starts an `OTARollout` Temporal workflow → publishes
+`cmd/{robot_id}/ota` on MQTT → `agent` receives, shells out to the
+**host's** podman/docker CLI → `pull localhost:14050/robot-app:tag`
+→ blue-green swap (run new under temp name → verify → rm old →
+rename) → publishes per-phase ACKs back to `ack/{robot_id}/ota` →
+MQTT bridge translates each ACK to a Temporal signal on the
+deterministic workflow ID → workflow proceeds canary → 25% → rest
+→ records terminal status in Postgres.
+
+**Collision response.** Gazebo contact sensor fires →
+`ros_gz_bridge` publishes `/contacts` over ROS DDS →
+`collision_publisher` (in robot container) emits one MQTT event on
+`events/{robot_id}/collision` → `collision-worker` MQTT bridge
+starts a `CollisionResponse` Temporal workflow → workflow runs back
+up → 90° turn-right → forward → stop, each phase a `SendTwist`
+activity that publishes `cmd/{robot_id}/twist` at 10 Hz on MQTT →
+`twist_subscriber` (robot container) republishes onto ROS
+`/cmd_vel` → `ros_gz_bridge` forwards to gz `DiffDrive` plugin →
+rover moves.
+
+## Make-target interaction map
+
+Targets fall in five lanes. The four bring-up targets in the
+**baseline** lane are the ones you run; everything else
+either depends on those or operates on them.
+
```
- ┌────────────────── browser ──────────────────┐
- │ http://localhost:14680 Gazebo GUI (noVNC) │
- └──────────────────┬──────────────────────────┘
- │
- ┌───────────────────────▼───────────────────────┐
- │ gazebo container │
- │ • ign gazebo + ros_gz_bridge │
- │ • Xvfb + x11vnc + noVNC │
- └────┬─────────────────────┬────────────────────┘
- │ ROS DDS (domain 42) │
- ┌─────────▼──────────┐ ┌───────▼────────────────────┐
- │ robot container │ │ robot-app container │
- │ • bridge_node │ │ (drive-circle | -fig-eight) │
- │ • sim_battery │ │ — OTA-swappable │
- │ • collision_pub │ └────────────────────────────┘
- │ • twist_subscriber │
- └─────────┬──────────┘
- │ gRPC (TCP)
- ┌─────────▼──────────────────────────┐
- │ agent (Go, native macOS binary) │
- │ • MQTT pub/sub on lab broker │
- │ • OTA executor (docker/podman CLI) │
- └─────────┬──────────────────────────┘
- │
- ┌─────────▼──────────┐ ┌───────────────────────┐
- │ MQTT (lab :14883) │◀──▶│ ota-worker │
- │ │ │ collision-worker │
- │ │ │ Temporal :14733 │
- └────────────────────┘ └───────────────────────┘
+ baseline
+ (run these in any DEMO TRIGGERS
+ order; each is idempotent) (need baseline up)
+ ───────────────────────── ─────────────────
+
+ ┌─sim-up──────────────────┐ ┌─ota-circle────┐
+ │ podman compose up: ├─────owns containers──▶ │ build push │
+ │ gazebo robot lab │ (gazebo, robot, │ POST /v1/ota │
+ │ cluster │ lab cluster) │ /rollouts │
+ └──────────────┬───────────┘ └──────┬────────┘
+ │ publishes ports 14050 │
+ │ 14080 14432 14680 14733 14883 14900 50051 │
+ ▼ │
+ ┌─agent-up─────────────────┐ │
+ │ ./bin/agent & │◀── shells host docker/podman ──┘
+ │ BROKER_URL=…14883 │ on POST → pull, run, swap robot-app
+ │ BRIDGE_ADDR=…50051 │
+ │ .run/agent.pid │ ┌─ota-figure-
+ └──────────────────────────┘ │ eight ──────┐
+ │ same shape │
+ ┌─workers-up───────────────┐ └──────────────┘
+ │ ./bin/ota-worker │
+ │ ./bin/collision-worker │ ┌─collide──────┐
+ │ TEMPORAL_ADDR=…14733 │◀── start workflow ──────│ publish │
+ │ BROKER_URL=…14883 │ on inbound MQTT │ events/… │
+ │ .run/{ota,collision}- │ event │ /collision │
+ │ worker.pid │ └──────────────┘
+ └──────────────────────────┘
+ ┌─ota-status───┐
+ ┌─controlplane-up──────────┐ │ GET /v1/ota │
+ │ ./bin/controlplane :8081 │◀── HTTP from ───────────│ /rollouts │
+ │ .run/controlplane.pid │ ota-circle / curl └──────────────┘
+ └──────────────────────────┘
+
+ TEAR-DOWN RESET (everything)
+ ───────── ─────────
+ *-down for each lane demo-reset wipe + restart
+ controlplane-down demo-reset NOUP=1 wipe and stop
+ workers-down
+ agent-down DRIVE (manual, no Temporal)
+ sim-down sim-drive-fwd LX= /-back/-left/-right/-stop
+
+ STATUS OBSERVABILITY
+ ───────── ─────────
+ *-status for each lane sim-gui open noVNC
+ ota-status sim-logs tail sim+robot+agent
+ lab-status probe lab ports
```
-## Lab quickstart
+### Quickstart
Requires Go 1.22+, Python 3.10+, and either Docker or Podman with
compose. The Makefile auto-detects the container engine.
@@ -78,6 +173,8 @@ That's the whole baseline. Tear down:
```bash
make controlplane-down && make workers-down && make agent-down && make sim-down
+# OR, full wipe + restart in one shot:
+make demo-reset
```
## Drive demo (no Temporal in the loop)
@@ -106,6 +203,39 @@ What you'll see: a `rollout-…` workflow appears at
1–2 seconds, and the `robot-app` container under `podman ps` flips to
the new image. The rover's behaviour changes immediately.
+The full data path for a rollout:
+
+```
+make ota-circle
+ │
+ │ podman build + podman push → registry :14050
+ │ │
+ │ curl POST /v1/ota/rollouts │
+ ▼ │
+controlplane (host) ──Temporal─▶ ota-worker (host)
+ │
+ │ MQTT publish on cmd/sim-robot-01/ota
+ ▼
+ EMQX (lab :14883)
+ │
+ ▼
+ agent (host) ─── shells ──┐
+ ▲ │ podman pull / run / rename
+ │ MQTT ack on │ on the macOS host
+ │ ack/sim-robot-01 │
+ │ /ota ▼
+ │ robot-app (in lab network,
+ │ ROS_DOMAIN_ID=42 — joins the
+ │ gazebo+robot DDS partition)
+ ▼
+ ota-worker reads acks, advances workflow phase
+ (PHASE_PULLED → PHASE_SWAPPED → PHASE_HEALTHY),
+ writes terminal status to Postgres.
+```
+
+Note the agent is the only thing that runs `podman pull/run/rename`.
+Workers never touch the host engine; they orchestrate via MQTT.
+
## Collision demo (Temporal drives the rover out of an obstacle)
The moon world spawns the rover with a 0.9 m boulder at `x = 8` —
From d1fc42bda318a633e55c3ed513db8a784f6e57a4 Mon Sep 17 00:00:00 2001
From: Ben Kearns <35475+bkearns@users.noreply.github.com>
Date: Tue, 5 May 2026 16:46:41 -0700
Subject: [PATCH 4/7] docs(README): convert overview + interaction diagrams to
Mermaid
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Three Mermaid diagrams replacing the ASCII art:
1. Service shape — flowchart TB. Three subgraphs (lab cluster,
host-side binaries, plus an external operator/browser node).
Bold orange edges trace the OTA path (agent shells the host
engine; engine pulls + swaps the robot-app container). Nodes
colour-coded by role: containers (yellow), host binaries (blue),
lab infra (light blue), OTA-swappable robot-app (orange).
2. OTA flow — sequenceDiagram. operator -> controlplane -> ota-worker
-> Temporal -> MQTT -> agent -> docker/podman -> robot-app, with
PHASE_PULLED / PHASE_SWAPPED / PHASE_HEALTHY ack edges back.
3. Collision flow — sequenceDiagram. gz contact sensor -> ros_gz_bridge
-> collision_publisher -> MQTT -> collision-worker -> Temporal,
with the loop block showing 10 Hz QoS-0 SendTwist publishes and
the trailing QoS-1 stop frame.
4. Make-target interaction map — flowchart LR. Four subgraphs:
baseline (sim-up / agent-up / workers-up / controlplane-up),
demos (ota-*, collide, sim-drive-*), lifecycle (down + reset),
status/observability. Edges show which baseline lane each demo
trigger lands on.
The inline ASCII OTA sub-diagram in the OTA demo section is removed
to avoid duplication; that content now lives as the Mermaid sequence
diagram in the Service shape area.
---
README.md | 315 +++++++++++++++++++++++++++++-------------------------
1 file changed, 170 insertions(+), 145 deletions(-)
diff --git a/README.md b/README.md
index 15a1aff..105f3dc 100644
--- a/README.md
+++ b/README.md
@@ -35,125 +35,178 @@ docker/podman CLI to pull, run, swap, and roll back the
**robot-app** container — which runs alongside the others on the lab
network and joins the same ROS DDS domain.
+```mermaid
+flowchart TB
+ classDef host fill:#e8f0ff,stroke:#5277b8,color:#000
+ classDef cont fill:#f4ecd8,stroke:#a98246,color:#000
+ classDef ota fill:#ffe6cc,stroke:#d79b00,color:#000
+ classDef lab fill:#dae8fc,stroke:#6c8ebf,color:#000
+ classDef ext fill:#f5f5f5,stroke:#666,color:#000
+
+ User([browser /
operator]):::ext
+
+ subgraph LabCluster["lab cluster (compose)"]
+ direction TB
+ Gz["gazebo container
ign gazebo
ros_gz_bridge
Xvfb + x11vnc + noVNC :14680"]:::cont
+ Robot["robot container
bridge_node :50051
sim_battery
collision_publisher
twist_subscriber"]:::cont
+ RobotApp["robot-app container
drive-circle | drive-figure-eight
(OTA-swappable)"]:::ota
+ MQTT[("EMQX MQTT
:14883")]:::lab
+ Tmp[("Temporal :14733
Postgres :14432
UI :14080")]:::lab
+ Reg[("Registry :14050")]:::lab
+ end
+
+ subgraph Host["host-side Go binaries (make-managed)"]
+ direction TB
+ Agent["agent
ROS gRPC client
MQTT pub/sub
OTA executor"]:::host
+ Otaw["ota-worker
Temporal worker +
MQTT bridge"]:::host
+ Colw["collision-worker
Temporal worker +
MQTT bridge"]:::host
+ Cp["controlplane
HTTP :8081"]:::host
+ Eng["docker / podman CLI
host engine"]:::host
+ end
+
+ User -->|noVNC :14680
UI :14080| LabCluster
+ User -->|POST /v1/ota/rollouts| Cp
+
+ Gz <-->|ROS DDS
domain 42| Robot
+ Gz <-->|ROS DDS
/cmd_vel| RobotApp
+ Robot <-->|gRPC| Agent
+ Agent <-->|MQTT| MQTT
+ Otaw <-->|MQTT| MQTT
+ Colw <-->|MQTT| MQTT
+ Otaw <-->|gRPC| Tmp
+ Colw <-->|gRPC| Tmp
+ Cp -->|StartWorkflow| Tmp
+
+ Agent ==>|shells| Eng
+ Eng ==>|pull / run / rename| RobotApp
+ Reg --o RobotApp
+
+ linkStyle 11 stroke:#d79b00,stroke-width:2px
+ linkStyle 12 stroke:#d79b00,stroke-width:2px
```
- ─── browser ──────────────────────────────────────────────────────
- http://localhost:14680 Gazebo GUI (noVNC) :14080 Temporal UI
- ──────────┬────────────────────────────────────────────────┬─────
- │ │
- ┌─────────▼────────────────────┐ ┌─────────────────▼────────┐
- │ gazebo container │ │ Temporal cluster (lab) │
- │ • ign gazebo │ │ :14733 frontend │
- │ • ros_gz_bridge │ │ Postgres :14432 │
- │ • Xvfb + x11vnc + noVNC │ │ EMQX MQTT :14883 │
- └────┬─────────────────────────┘ │ Registry :14050 │
- │ ROS DDS (domain 42, cyclonedds) └────┬───────────┬─────────┘
- │ │ gRPC │ MQTT
- ┌────▼─────────────────────┐ ┌────────────┐ │ │
- │ robot container │ │ robot-app │ │ │
- │ (always-on infra) │ │ container │ │ │
- │ • bridge_node (gRPC) │ │ drive- │ │ │
- │ • sim_battery │ │ circle | │ │ │
- │ • collision_publisher │ │ drive- │ │ │
- │ • twist_subscriber │ │ fig-eight │ │ │
- └────┬─────────────────────┘ └─────▲──────┘ │ │
- │ gRPC (robot:50051) │ │ │
- │ tunneled to host :50051 │ podman pull/run/swap│
- │ │ │
- ┌────▼──────────────────────────────┴──────────────────┐ │
- │ host-side Go binaries (managed by `make`) │ │
- │ agent — ROS bridge ↔ MQTT, OTA executor │◀─┤
- │ ota-worker — Temporal worker + MQTT bridge │◀─┤
- │ collision-worker— Temporal worker + MQTT bridge │◀─┘
- │ controlplane — HTTP API on :8081 (POST rollouts) │
- └──────────────────────────────────────────────────────┘
- │
- │ docker / podman CLI on the macOS host
- ▼
- ┌────────────────────────┐
- │ podman engine on host │
- │ manages robot-app │
- │ container in the lab │
- │ network │
- └────────────────────────┘
+
+Bold orange edges are the OTA path: the agent shells out to the host
+engine, which pulls from the lab registry and swaps the robot-app
+container in place.
+
+### OTA flow
+
+```mermaid
+sequenceDiagram
+ autonumber
+ actor Op as operator (curl / make ota-*)
+ participant Cp as controlplane
+ participant Ow as ota-worker
+ participant Tmp as Temporal
+ participant MQ as MQTT (EMQX)
+ participant Ag as agent (host)
+ participant Eng as docker/podman (host)
+ participant App as robot-app (container)
+
+ Op->>Cp: POST /v1/ota/rollouts
+ Cp->>Tmp: StartWorkflow OTARollout
+ Tmp-->>Ow: dispatch task
+ Ow->>MQ: publish cmd/{robot_id}/ota
+ MQ-->>Ag: deliver
+ Ag->>Eng: pull image_ref
+ Ag->>MQ: ack PHASE_PULLED
+ Ag->>Eng: run new (temp name)
+ Ag->>Eng: rm old; rename new
+ Ag->>MQ: ack PHASE_SWAPPED
+ Eng->>App: container starts
+ Ag->>Eng: exec smoke_command
+ Ag->>MQ: ack PHASE_HEALTHY
+ MQ-->>Ow: signal workflow per phase
+ Ow->>Tmp: RecordRolloutEnded(completed)
+ Cp-->>Op: GET /v1/ota/rollouts → completed
```
-Two flows worth tracing:
-
-**OTA rollout.** Operator → `controlplane` POST `/v1/ota/rollouts` →
-`ota-worker` starts an `OTARollout` Temporal workflow → publishes
-`cmd/{robot_id}/ota` on MQTT → `agent` receives, shells out to the
-**host's** podman/docker CLI → `pull localhost:14050/robot-app:tag`
-→ blue-green swap (run new under temp name → verify → rm old →
-rename) → publishes per-phase ACKs back to `ack/{robot_id}/ota` →
-MQTT bridge translates each ACK to a Temporal signal on the
-deterministic workflow ID → workflow proceeds canary → 25% → rest
-→ records terminal status in Postgres.
-
-**Collision response.** Gazebo contact sensor fires →
-`ros_gz_bridge` publishes `/contacts` over ROS DDS →
-`collision_publisher` (in robot container) emits one MQTT event on
-`events/{robot_id}/collision` → `collision-worker` MQTT bridge
-starts a `CollisionResponse` Temporal workflow → workflow runs back
-up → 90° turn-right → forward → stop, each phase a `SendTwist`
-activity that publishes `cmd/{robot_id}/twist` at 10 Hz on MQTT →
-`twist_subscriber` (robot container) republishes onto ROS
-`/cmd_vel` → `ros_gz_bridge` forwards to gz `DiffDrive` plugin →
-rover moves.
+### Collision flow
+
+```mermaid
+sequenceDiagram
+ autonumber
+ participant Gz as gazebo (contact sensor)
+ participant RGB as ros_gz_bridge
+ participant Cp as collision_publisher (robot)
+ participant MQ as MQTT (EMQX)
+ participant Cw as collision-worker
+ participant Tmp as Temporal
+ participant Ts as twist_subscriber (robot)
+ participant Rover as gz DiffDrive
+
+ Gz->>RGB: ignition.msgs.Contacts
+ RGB->>Cp: ROS /contacts
+ Cp->>MQ: events/{id}/collision (debounced)
+ MQ-->>Cw: deliver
+ Cw->>Tmp: StartWorkflow CollisionResponse
+ Tmp-->>Cw: dispatch SendTwist (back, 3s)
+ loop each phase: back / settle / turn / forward / stop
+ Cw->>MQ: cmd/{id}/twist @ 10 Hz QoS 0
+ MQ-->>Ts: deliver
+ Ts->>Rover: ROS /cmd_vel → gz cmd_vel
+ end
+ Cw->>MQ: cmd/{id}/twist {0,0} QoS 1 (final stop)
+ Cw->>Tmp: workflow Completed
+```
## Make-target interaction map
-Targets fall in five lanes. The four bring-up targets in the
-**baseline** lane are the ones you run; everything else
-either depends on those or operates on them.
-
-```
- baseline
- (run these in any DEMO TRIGGERS
- order; each is idempotent) (need baseline up)
- ───────────────────────── ─────────────────
-
- ┌─sim-up──────────────────┐ ┌─ota-circle────┐
- │ podman compose up: ├─────owns containers──▶ │ build push │
- │ gazebo robot lab │ (gazebo, robot, │ POST /v1/ota │
- │ cluster │ lab cluster) │ /rollouts │
- └──────────────┬───────────┘ └──────┬────────┘
- │ publishes ports 14050 │
- │ 14080 14432 14680 14733 14883 14900 50051 │
- ▼ │
- ┌─agent-up─────────────────┐ │
- │ ./bin/agent & │◀── shells host docker/podman ──┘
- │ BROKER_URL=…14883 │ on POST → pull, run, swap robot-app
- │ BRIDGE_ADDR=…50051 │
- │ .run/agent.pid │ ┌─ota-figure-
- └──────────────────────────┘ │ eight ──────┐
- │ same shape │
- ┌─workers-up───────────────┐ └──────────────┘
- │ ./bin/ota-worker │
- │ ./bin/collision-worker │ ┌─collide──────┐
- │ TEMPORAL_ADDR=…14733 │◀── start workflow ──────│ publish │
- │ BROKER_URL=…14883 │ on inbound MQTT │ events/… │
- │ .run/{ota,collision}- │ event │ /collision │
- │ worker.pid │ └──────────────┘
- └──────────────────────────┘
- ┌─ota-status───┐
- ┌─controlplane-up──────────┐ │ GET /v1/ota │
- │ ./bin/controlplane :8081 │◀── HTTP from ───────────│ /rollouts │
- │ .run/controlplane.pid │ ota-circle / curl └──────────────┘
- └──────────────────────────┘
-
- TEAR-DOWN RESET (everything)
- ───────── ─────────
- *-down for each lane demo-reset wipe + restart
- controlplane-down demo-reset NOUP=1 wipe and stop
- workers-down
- agent-down DRIVE (manual, no Temporal)
- sim-down sim-drive-fwd LX= /-back/-left/-right/-stop
-
- STATUS OBSERVABILITY
- ───────── ─────────
- *-status for each lane sim-gui open noVNC
- ota-status sim-logs tail sim+robot+agent
- lab-status probe lab ports
+The four bring-up targets in the **baseline** lane are the ones you
+run; everything else either depends on those or operates on them.
+Demo triggers (right column) need the baseline lane up to function.
+
+```mermaid
+flowchart LR
+ classDef baseline fill:#dae8fc,stroke:#6c8ebf,color:#000
+ classDef demo fill:#ffe6cc,stroke:#d79b00,color:#000
+ classDef status fill:#d5e8d4,stroke:#82b366,color:#000
+ classDef cleanup fill:#f8cecc,stroke:#b85450,color:#000
+
+ subgraph Baseline["BASELINE — bring up in any order"]
+ direction TB
+ SimUp["make sim-up
compose up: gazebo + robot + lab cluster
publishes :14050 :14080 :14432 :14680
:14733 :14883 :14900 :50051"]:::baseline
+ AgentUp["make agent-up
./bin/agent (.run/agent.pid)
BROKER_URL=tcp://localhost:14883
BRIDGE_ADDR=localhost:50051"]:::baseline
+ WorkersUp["make workers-up
./bin/ota-worker + ./bin/collision-worker
(.run/*.pid)
TEMPORAL_ADDR=localhost:14733"]:::baseline
+ CpUp["make controlplane-up
./bin/controlplane :8081
(.run/controlplane.pid)"]:::baseline
+ end
+
+ subgraph Demos["DEMO TRIGGERS"]
+ direction TB
+ OtaC["make ota-circle
build + push +
POST /v1/ota/rollouts"]:::demo
+ OtaF["make ota-figure-eight
(same shape)"]:::demo
+ Coll["make collide
publish events/{id}/collision"]:::demo
+ OtaS["make ota-status
GET /v1/ota/rollouts"]:::demo
+ Drive["make sim-drive-fwd LX= /
-back / -left / -right / -stop
(no Temporal in the loop)"]:::demo
+ end
+
+ subgraph Lifecycle["LIFECYCLE"]
+ direction TB
+ Down["make sim-down / agent-down /
workers-down / controlplane-down"]:::cleanup
+ Reset["make demo-reset
(or demo-reset NOUP=1)"]:::cleanup
+ end
+
+ subgraph Obs["STATUS / OBSERVABILITY"]
+ direction TB
+ Stat["make agent-status / workers-status
controlplane-status / lab-status"]:::status
+ Gui["make sim-gui
open noVNC URL"]:::status
+ Logs["make sim-logs
tail sim+robot+agent"]:::status
+ end
+
+ SimUp -- gazebo, robot, lab containers --> AgentUp
+ AgentUp -- shells host docker/podman --> SimUp
+ WorkersUp -- gRPC --> SimUp
+ CpUp -- gRPC --> SimUp
+
+ OtaC -- HTTP --> CpUp
+ OtaC -- via MQTT --> AgentUp
+ OtaF -- HTTP --> CpUp
+ OtaF -- via MQTT --> AgentUp
+ Coll -- MQTT publish --> WorkersUp
+ OtaS -- HTTP GET --> CpUp
+ Drive -- ign topic exec --> SimUp
+
+ Reset -. wipes + restarts .- Baseline
```
### Quickstart
@@ -203,38 +256,10 @@ What you'll see: a `rollout-…` workflow appears at
1–2 seconds, and the `robot-app` container under `podman ps` flips to
the new image. The rover's behaviour changes immediately.
-The full data path for a rollout:
-
-```
-make ota-circle
- │
- │ podman build + podman push → registry :14050
- │ │
- │ curl POST /v1/ota/rollouts │
- ▼ │
-controlplane (host) ──Temporal─▶ ota-worker (host)
- │
- │ MQTT publish on cmd/sim-robot-01/ota
- ▼
- EMQX (lab :14883)
- │
- ▼
- agent (host) ─── shells ──┐
- ▲ │ podman pull / run / rename
- │ MQTT ack on │ on the macOS host
- │ ack/sim-robot-01 │
- │ /ota ▼
- │ robot-app (in lab network,
- │ ROS_DOMAIN_ID=42 — joins the
- │ gazebo+robot DDS partition)
- ▼
- ota-worker reads acks, advances workflow phase
- (PHASE_PULLED → PHASE_SWAPPED → PHASE_HEALTHY),
- writes terminal status to Postgres.
-```
-
-Note the agent is the only thing that runs `podman pull/run/rename`.
-Workers never touch the host engine; they orchestrate via MQTT.
+The full data path for a rollout — see the **OTA flow** sequence
+diagram in *Service shape* above. The agent is the only process that
+runs `podman pull / run / rename`; workers never touch the host
+engine, they orchestrate via MQTT.
## Collision demo (Temporal drives the rover out of an obstacle)
From d89fee35d3afedb6763db5fe1b5dd1fd548655d9 Mon Sep 17 00:00:00 2001
From: Ben Kearns <35475+bkearns@users.noreply.github.com>
Date: Tue, 5 May 2026 16:49:51 -0700
Subject: [PATCH 5/7] docs(specs/overview): refresh for gazebo+robot split,
OTA, and collision
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The overview.md system-context Mermaid was from before the demo
cut. It still showed a single 'Robot Agent + ROS Bridge' lump and
omitted the OTA-swappable robot-app, the gazebo container split, and
the collision-response path. Brought it up to current state.
Changes:
1. System context — rewritten as a Mermaid flowchart with
colour-coded nodes for cloud / host / container / OTA-swappable.
Bold orange edges trace the agent-shells-host-engine OTA path.
Notes the gazebo container is sim-only; everything else is
identical between dev and production.
2. Component inventory — robot side now lists the THREE containers
(gazebo / robot / robot-app) plus the host-side agent. Calls
out that the agent is the only thing that touches docker/podman.
3. Telemetry path — converted from ASCII to Mermaid sequence.
4. OTA path — converted from ASCII to Mermaid sequence with
per-cohort loop and the actual phase ack steps
(PHASE_PULLED / SWAPPED / HEALTHY).
5. NEW: collision-response path — Mermaid sequence showing
ros_gz_bridge / collision_publisher / MQTT / collision-worker /
twist_subscriber + the 10 Hz QoS 0 / final QoS 1 stop pattern.
Documents the ground-contact filter and the clean-session
requirement (both load-bearing for the demo to actually work).
6. ADRs — placeholder table replaced with closed-ADR summary
pointing at specs/adr/ files.
7. Phase 1 status updated to reflect post-demo state.
---
specs/overview.md | 260 ++++++++++++++++++++++++++++++++--------------
1 file changed, 180 insertions(+), 80 deletions(-)
diff --git a/specs/overview.md b/specs/overview.md
index 6d6785f..6f5c765 100644
--- a/specs/overview.md
+++ b/specs/overview.md
@@ -24,39 +24,67 @@ phase artifacts (DSM, threat model, FMEA, project plan).
## System context
+The cloud / robot split as of the demo cut. Production
+("customer DC") and dev-loop sim ("local Mac") share the same
+component shape — only the substrate (Kubernetes vs `make sim-up`)
+and the simulator role differ.
+
```mermaid
-graph LR
- subgraph customer_dc[Customer Data Center]
- cp[Control Plane
Go services]
- tc[Temporal Cluster
self-hosted]
- mq[MQTT Broker
EMQX or VerneMQ]
- cr[Container Registry
OCI images]
- ts[Telemetry Store
TSDB]
- pg[(Postgres)]
+flowchart TB
+ classDef cloud fill:#dae8fc,stroke:#6c8ebf,color:#000
+ classDef host fill:#e8f0ff,stroke:#5277b8,color:#000
+ classDef cont fill:#f4ecd8,stroke:#a98246,color:#000
+ classDef ota fill:#ffe6cc,stroke:#d79b00,color:#000
+
+ Op([operator])
+
+ subgraph CloudDC["customer DC (or local lab cluster)"]
+ direction TB
+ Cp["control plane
Go HTTP API"]:::cloud
+ Tc[("Temporal cluster
+ Postgres")]:::cloud
+ Mq[("MQTT broker
EMQX, persistent sessions")]:::cloud
+ Cr[("Container registry
OCI images")]:::cloud
+ Ts[("Telemetry store
TimescaleDB hypertable")]:::cloud
+ Tlm["telemetry-ingest
(MQTT → TSDB)"]:::cloud
+ Otaw["ota-worker
Temporal + MQTT bridge"]:::cloud
+ Colw["collision-worker
Temporal + MQTT bridge"]:::cloud
end
- subgraph robots[Robot Fleet 10–100 units]
- ag[Robot Agent Go]
- br[ROS 2 Bridge
Python or C++]
- au[Customer Autonomy
ROS 2 nodes]
+ subgraph Robot["robot (or local sim host)"]
+ direction TB
+ Gz["gazebo container
ign gazebo + ros_gz_bridge
(local sim only)"]:::cont
+ RobotInfra["robot container
bridge_node (gRPC) +
sim_battery + collision_publisher +
twist_subscriber"]:::cont
+ RobotApp["robot-app container
OTA-swappable controller
(drive-circle, drive-figure-eight,
customer apps...)"]:::ota
+ Agent["agent (Go)
MQTT pub/sub +
OTA executor"]:::host
+ Eng["docker / podman
engine"]:::host
end
- dev[Developer Workstation
Blender + Phobos + Gazebo]
-
- cp --> tc
- cp --> mq
- cp --> cr
- cp --> ts
- tc --> pg
-
- ag <-. MQTT QoS 1/2 .-> mq
- ag --> br
- br -. DDS .- au
-
- dev -. URDF .- au
- dev -. dev iteration .- au
+ Op -->|POST /v1/ota/rollouts| Cp
+ Cp -->|StartWorkflow| Tc
+ Tc <-->|workflow tasks| Otaw
+ Tc <-->|workflow tasks| Colw
+ Otaw <--> Mq
+ Colw <--> Mq
+ Tlm --> Ts
+ Tlm <--> Mq
+ Cr --o RobotApp
+
+ Agent <-->|gRPC| RobotInfra
+ Agent <-->|MQTT QoS 1
persistent sessions| Mq
+ Agent ==>|shells| Eng
+ Eng ==>|pull / run / rename| RobotApp
+
+ Gz <-->|ROS DDS
domain 42| RobotInfra
+ Gz <-->|ROS /cmd_vel
via ros_gz_bridge| RobotApp
+
+ linkStyle 12 stroke:#d79b00,stroke-width:2px
+ linkStyle 13 stroke:#d79b00,stroke-width:2px
```
+In production the gazebo container is absent (real robots, real
+sensors). Everything else — agent, robot-container infra, MQTT path,
+the cloud workers — is identical between dev sim and production.
+
## Component inventory
### Cloud side (in customer DC)
@@ -73,11 +101,21 @@ graph LR
### Robot side (per Ubuntu 22.04 + Docker)
-| Component | Language | Responsibility |
-|-----------|----------|----------------|
-| Robot agent | Go | Maintain MQTT connection, buffer telemetry during disconnect, execute OTA updates, report health |
-| ROS 2 bridge node | Python or C++ (rclpy / rclcpp) | Subscribe to selected DDS topics, republish via gRPC over Unix socket |
-| Customer autonomy stack | C++ / Python (ROS 2 nodes) | Out of scope; customer's responsibility |
+The robot now runs **three** containers per host, not one. The
+agent is a sibling Go binary outside the compose lifecycle (in the
+dev demo it runs natively on the macOS host; in production it
+ships as a systemd unit).
+
+| Component | Language | Container? | Responsibility |
+|-----------|----------|-----------|----------------|
+| **gazebo** (sim only) | C++ / Python | container | ign gazebo + ros_gz_bridge + Xvfb / x11vnc / noVNC. Absent on a real robot. |
+| **robot** (always-on infra) | Python (rclpy) | container | bridge_node (rclpy → gRPC TCP :50051), sim_battery (sim only), collision_publisher (ROS /contacts → MQTT), twist_subscriber (MQTT → ROS /cmd_vel). |
+| **robot-app** (OTA target) | any (ROS 2 image) | container | The replaceable controller. drive-circle, drive-figure-eight in dev; customer autonomy code in production. The agent OTAs this image; everything else is static between releases. |
+| robot agent | Go | host binary | Maintain MQTT connection, buffer telemetry during disconnect, execute OTA updates via the host docker/podman CLI, report health. |
+
+The agent owns the path from MQTT command → host engine → robot-app.
+Workers in the cloud orchestrate via MQTT; only the agent ever
+shells out to the host's container engine.
### Developer toolchain
@@ -90,21 +128,24 @@ graph LR
### Telemetry path (robot → cloud)
-```
-ROS 2 nodes ─DDS topic─▶ ROS 2 bridge ─gRPC UDS─▶ Robot agent
- │
- local SQLite buffer
- (intermittent)
- │
- MQTT QoS 1/2
- │
- ▼
- MQTT broker
- │
- subscriber: telemetry-ingest
- │
- ▼
- Telemetry store
+```mermaid
+sequenceDiagram
+ participant Nodes as ROS 2 nodes
+ participant Bridge as bridge_node (rclpy)
+ participant Agent as agent (Go)
+ participant Buf as local SQLite buffer
+ participant MQ as MQTT broker
+ participant Ing as telemetry-ingest
+ participant TS as TimescaleDB
+
+ Nodes->>Bridge: DDS topic
+ Bridge->>Agent: gRPC TCP :50051
+ Agent->>Buf: append (bounded ring)
+ Note over Agent,Buf: drains while connected
+ Buf->>Agent: pop
+ Agent->>MQ: publish QoS 1 (persistent session)
+ MQ->>Ing: deliver
+ Ing->>TS: INSERT into telemetry hypertable
```
**Properties:**
@@ -117,33 +158,90 @@ ROS 2 nodes ─DDS topic─▶ ROS 2 bridge ─gRPC UDS─▶ Robot agent
### OTA path (cloud → robot)
-```
-Operator API ─▶ Control Plane ─▶ Temporal workflow (rollout)
- │
- 1) Resolve target image tag
- 2) For each robot in cohort:
- a) Publish MQTT command (with QoS 1)
- b) Wait for ACK (timer-bounded)
- c) Wait for health check
- d) On failure → rollback child workflow
- e) Record outcome
- │
- ▼
-Robot agent ─▶ pull image from registry (over MQTT-signaled URL)
- ─▶ swap container
- ─▶ run smoke check
- ─▶ ACK / NACK via MQTT
- ─▶ on failure: revert to previous container, signal NACK
+```mermaid
+sequenceDiagram
+ autonumber
+ actor Op as operator
+ participant Cp as control plane
+ participant Tmp as Temporal
+ participant Ow as ota-worker
+ participant MQ as MQTT
+ participant Ag as agent
+ participant Eng as host docker/podman
+ participant App as robot-app
+
+ Op->>Cp: POST /v1/ota/rollouts
+ Cp->>Tmp: StartWorkflow OTARollout
+ Tmp->>Ow: dispatch task
+ loop per cohort phase (canary → 25% → rest)
+ Ow->>MQ: publish cmd/{robot_id}/ota
+ MQ->>Ag: deliver
+ Ag->>Eng: pull image_ref from registry
+ Ag->>MQ: ack PHASE_PULLED
+ Ag->>Eng: run new (temp name) → verify → rm old → rename
+ Ag->>MQ: ack PHASE_SWAPPED
+ Eng->>App: container starts
+ Ag->>Eng: exec smoke_command
+ Ag->>MQ: ack PHASE_HEALTHY
+ MQ->>Ow: signal workflow per phase
+ end
+ Ow->>Tmp: RecordRolloutEnded(status)
```
**Properties:**
- Rollout cohort policy (canary, batched, full-fleet) lives in the
Temporal workflow definition.
-- Rollback is a child workflow with its own retry semantics.
+- Rollback is a child workflow with its own retry semantics; the
+ agent always emits PHASE_ROLLED_BACK so the rollback workflow
+ terminates instead of timing out.
- Image signature verification is an mTLS-shaped seam (see D-11):
v1 uses TLS to the registry only; production gates require signed
images verified by a customer-controlled key.
+### Collision-response path (robot → cloud → robot)
+
+This is the demo wiring used to show that a Temporal workflow can
+own a recovery sequence in response to a robot-side event.
+
+```mermaid
+sequenceDiagram
+ autonumber
+ participant Sim as gazebo (contact sensor)
+ participant RGB as ros_gz_bridge
+ participant Pub as collision_publisher
+ participant MQ as MQTT
+ participant Cw as collision-worker
+ participant Tmp as Temporal
+ participant Sub as twist_subscriber
+ participant Drive as gz DiffDrive
+
+ Sim->>RGB: ignition.msgs.Contacts
+ RGB->>Pub: ROS /contacts
+ Pub->>Pub: filter ground contacts
+ 2s debounce
+ Pub->>MQ: events/{id}/collision
+ MQ->>Cw: deliver
+ Cw->>Tmp: StartWorkflow CollisionResponse
+ loop each phase: back / settle / turn / forward / stop
+ Tmp->>Cw: dispatch SendTwist activity
+ Cw->>MQ: cmd/{id}/twist @ 10 Hz QoS 0
+ MQ->>Sub: deliver
+ Sub->>Drive: ROS /cmd_vel → gz cmd_vel
+ end
+ Cw->>MQ: cmd/{id}/twist {0,0} QoS 1 (final stop)
+ Cw->>Tmp: workflow Completed
+```
+
+**Properties:**
+- Twist commands ride MQTT QoS 0 at 10 Hz; the final stop frame is
+ QoS 1 so the rover always settles even if a tail QoS 0 packet
+ drops.
+- Workers run with `clean_session=true`. A missed collision event
+ is fine (the next contact emits another); persistent-session
+ replay floods the bridge after restart.
+- The collision_publisher filters out ground-plane contacts so the
+ rover's resting weight doesn't trigger a workflow on every sim
+ tick.
+
## Constraints (carried forward from Phase 0)
1. Architecture must function under intermittent connectivity (D-04).
@@ -162,19 +260,18 @@ Robot agent ─▶ pull image from registry (over MQTT-signaled URL)
6. v1 scope is Telemetry + OTA. Mission dispatch and teleop are
non-goals (D-02).
-## Open architecture decisions (ADR placeholders)
-
-To be resolved in Phase 2:
+## Closed architectural decisions (ADRs)
-| ADR | Topic | Constraint |
-|-----|-------|------------|
-| ADR-001 | MQTT broker selection | EMQX vs VerneMQ vs Mosquitto cluster; persistent session capacity, HA model |
-| ADR-002 | Container registry | Harbor / Distribution / Zot; signing / verification path |
-| ADR-003 | Telemetry storage | TimescaleDB vs VictoriaMetrics vs Prometheus + Mimir; retention / cardinality |
-| ADR-004 | Bridge node language | Python (rclpy, faster delivery) vs C++ (rclcpp, more efficient) |
-| ADR-005 | Installer toolchain | Helm-on-k3s vs Ansible vs custom |
-| ADR-006 | Local-buffer durability format on robot | SQLite vs filesystem queue vs embedded NATS |
-| ADR-007 | OTA artifact swap mechanism | Recreate vs blue-green container; health check definition |
+| ADR | Decision | Status |
+|-----|----------|--------|
+| ADR-001 | MQTT broker = EMQX 5.x (lab) | accepted; see `specs/adr/ADR-001-mqtt-broker.md` |
+| ADR-002 | Container registry = Distribution (`registry:2`) | accepted |
+| ADR-003 | Telemetry storage = TimescaleDB hypertable | accepted |
+| ADR-004 | Bridge node = Python rclpy | accepted |
+| ADR-005 | Installer toolchain | open (Sprint 8) |
+| ADR-006 | Local-buffer durability format on robot = SQLite (WAL) | accepted (in code) |
+| ADR-007 | OTA artifact swap mechanism = blue-green | accepted; see `specs/adr/ADR-007-ota-swap-strategy.md` |
+| ADR-008 | OTA command/ack transport = MQTT topic pair | accepted; see `specs/adr/ADR-008-ota-command-transport.md` |
## Risks (from Phase 0)
@@ -196,7 +293,10 @@ See `decisions.md` for the full risk register. Highest-impact items:
## Phase 1 status
-This overview is the Phase 1 deliverable. Next phase (DSM) analyzes
-module boundaries between the Go cloud control plane, the Go robot
-agent, and the bridge node — looking for premature coupling and the
-right place to draw the gRPC contract between agent and bridge.
+This overview is the Phase 1 deliverable, refreshed after the demo
+cut (gazebo+robot container split, OTA-swappable robot-app images,
+CollisionResponse workflow). Next phase (DSM) analyzes module
+boundaries between the Go cloud workers, the agent, and the
+robot-side ROS code — looking for premature coupling and the right
+place to draw the contracts (gRPC between agent and bridge_node;
+MQTT topic schemas between agent and workers).
From 6a918859040e333d3dce4cf7150a6abe00589b52 Mon Sep 17 00:00:00 2001
From: Ben Kearns <35475+bkearns@users.noreply.github.com>
Date: Tue, 5 May 2026 16:52:28 -0700
Subject: [PATCH 6/7] docs(mermaid): fix sequenceDiagram parser breakage on
alias parens
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
User reported the OTA flow Mermaid diagram in README and overview.md
not rendering. Cause: Mermaid's sequenceDiagram parser doesn't like
unquoted parens, asterisks, or special chars in 'participant X as Y'
aliases. The line:
actor Op as operator (curl / make ota-*)
was breaking parse for the entire diagram. Same issue on
'participant Bridge as bridge_node (rclpy)' and friends.
Fixes:
- Strip parens / slashes / asterisks / arrows from participant
aliases. 'as host docker/podman' becomes 'as host engine';
'as gazebo (contact sensor)' becomes 'as gazebo'; etc.
- Strip parens and unicode arrows from arrow labels too: 'rm old then
rename new' instead of 'rm old; rename new', 'forwards to gz cmd_vel'
instead of 'ROS /cmd_vel → gz cmd_vel'.
- specs/overview.md system-context flowchart: linkStyle indices were
off by one (edges renumbered when I added more relationships).
Fix to 11 and 12 so the bold orange OTA path lands on
agent->engine and engine->robot-app, not the wrong edges.
---
README.md | 42 +++++++++++++++++++++---------------------
specs/overview.md | 35 ++++++++++++++++++-----------------
2 files changed, 39 insertions(+), 38 deletions(-)
diff --git a/README.md b/README.md
index 105f3dc..7984335 100644
--- a/README.md
+++ b/README.md
@@ -94,14 +94,14 @@ container in place.
```mermaid
sequenceDiagram
autonumber
- actor Op as operator (curl / make ota-*)
+ actor Op as Operator
participant Cp as controlplane
participant Ow as ota-worker
participant Tmp as Temporal
- participant MQ as MQTT (EMQX)
- participant Ag as agent (host)
- participant Eng as docker/podman (host)
- participant App as robot-app (container)
+ participant MQ as MQTT
+ participant Ag as agent
+ participant Eng as host engine
+ participant App as robot-app
Op->>Cp: POST /v1/ota/rollouts
Cp->>Tmp: StartWorkflow OTARollout
@@ -110,15 +110,15 @@ sequenceDiagram
MQ-->>Ag: deliver
Ag->>Eng: pull image_ref
Ag->>MQ: ack PHASE_PULLED
- Ag->>Eng: run new (temp name)
- Ag->>Eng: rm old; rename new
+ Ag->>Eng: run new under temp name
+ Ag->>Eng: rm old then rename new
Ag->>MQ: ack PHASE_SWAPPED
Eng->>App: container starts
Ag->>Eng: exec smoke_command
Ag->>MQ: ack PHASE_HEALTHY
MQ-->>Ow: signal workflow per phase
- Ow->>Tmp: RecordRolloutEnded(completed)
- Cp-->>Op: GET /v1/ota/rollouts → completed
+ Ow->>Tmp: RecordRolloutEnded
+ Cp-->>Op: GET /v1/ota/rollouts shows completed
```
### Collision flow
@@ -126,27 +126,27 @@ sequenceDiagram
```mermaid
sequenceDiagram
autonumber
- participant Gz as gazebo (contact sensor)
+ participant Gz as gazebo
participant RGB as ros_gz_bridge
- participant Cp as collision_publisher (robot)
- participant MQ as MQTT (EMQX)
+ participant Pub as collision_publisher
+ participant MQ as MQTT
participant Cw as collision-worker
participant Tmp as Temporal
- participant Ts as twist_subscriber (robot)
+ participant Sub as twist_subscriber
participant Rover as gz DiffDrive
Gz->>RGB: ignition.msgs.Contacts
- RGB->>Cp: ROS /contacts
- Cp->>MQ: events/{id}/collision (debounced)
+ RGB->>Pub: ROS /contacts
+ Pub->>MQ: events/{id}/collision after 2s debounce
MQ-->>Cw: deliver
Cw->>Tmp: StartWorkflow CollisionResponse
- Tmp-->>Cw: dispatch SendTwist (back, 3s)
- loop each phase: back / settle / turn / forward / stop
- Cw->>MQ: cmd/{id}/twist @ 10 Hz QoS 0
- MQ-->>Ts: deliver
- Ts->>Rover: ROS /cmd_vel → gz cmd_vel
+ Tmp-->>Cw: dispatch SendTwist back 3s
+ loop each phase back, settle, turn, forward, stop
+ Cw->>MQ: cmd/{id}/twist at 10 Hz QoS 0
+ MQ-->>Sub: deliver
+ Sub->>Rover: ROS /cmd_vel forwards to gz cmd_vel
end
- Cw->>MQ: cmd/{id}/twist {0,0} QoS 1 (final stop)
+ Cw->>MQ: cmd/{id}/twist 0,0 QoS 1 final stop
Cw->>Tmp: workflow Completed
```
diff --git a/specs/overview.md b/specs/overview.md
index 6f5c765..5867a8d 100644
--- a/specs/overview.md
+++ b/specs/overview.md
@@ -70,15 +70,16 @@ flowchart TB
Cr --o RobotApp
Agent <-->|gRPC| RobotInfra
- Agent <-->|MQTT QoS 1
persistent sessions| Mq
+ Agent <-->|MQTT QoS 1| Mq
Agent ==>|shells| Eng
Eng ==>|pull / run / rename| RobotApp
- Gz <-->|ROS DDS
domain 42| RobotInfra
- Gz <-->|ROS /cmd_vel
via ros_gz_bridge| RobotApp
+ Gz <-->|ROS DDS domain 42| RobotInfra
+ Gz <-->|ROS /cmd_vel via ros_gz_bridge| RobotApp
+ %% Edges 11 and 12 are the agent->engine->robot-app OTA path.
+ linkStyle 11 stroke:#d79b00,stroke-width:2px
linkStyle 12 stroke:#d79b00,stroke-width:2px
- linkStyle 13 stroke:#d79b00,stroke-width:2px
```
In production the gazebo container is absent (real robots, real
@@ -131,8 +132,8 @@ shells out to the host's container engine.
```mermaid
sequenceDiagram
participant Nodes as ROS 2 nodes
- participant Bridge as bridge_node (rclpy)
- participant Agent as agent (Go)
+ participant Bridge as bridge_node
+ participant Agent as agent
participant Buf as local SQLite buffer
participant MQ as MQTT broker
participant Ing as telemetry-ingest
@@ -161,31 +162,31 @@ sequenceDiagram
```mermaid
sequenceDiagram
autonumber
- actor Op as operator
+ actor Op as Operator
participant Cp as control plane
participant Tmp as Temporal
participant Ow as ota-worker
participant MQ as MQTT
participant Ag as agent
- participant Eng as host docker/podman
+ participant Eng as host engine
participant App as robot-app
Op->>Cp: POST /v1/ota/rollouts
Cp->>Tmp: StartWorkflow OTARollout
Tmp->>Ow: dispatch task
- loop per cohort phase (canary → 25% → rest)
+ loop per cohort phase canary then 25% then rest
Ow->>MQ: publish cmd/{robot_id}/ota
MQ->>Ag: deliver
Ag->>Eng: pull image_ref from registry
Ag->>MQ: ack PHASE_PULLED
- Ag->>Eng: run new (temp name) → verify → rm old → rename
+ Ag->>Eng: run new under temp name then verify then rm old then rename
Ag->>MQ: ack PHASE_SWAPPED
Eng->>App: container starts
Ag->>Eng: exec smoke_command
Ag->>MQ: ack PHASE_HEALTHY
MQ->>Ow: signal workflow per phase
end
- Ow->>Tmp: RecordRolloutEnded(status)
+ Ow->>Tmp: RecordRolloutEnded
```
**Properties:**
@@ -206,7 +207,7 @@ own a recovery sequence in response to a robot-side event.
```mermaid
sequenceDiagram
autonumber
- participant Sim as gazebo (contact sensor)
+ participant Sim as gazebo
participant RGB as ros_gz_bridge
participant Pub as collision_publisher
participant MQ as MQTT
@@ -217,17 +218,17 @@ sequenceDiagram
Sim->>RGB: ignition.msgs.Contacts
RGB->>Pub: ROS /contacts
- Pub->>Pub: filter ground contacts
+ 2s debounce
+ Pub->>Pub: filter ground contacts then 2s debounce
Pub->>MQ: events/{id}/collision
MQ->>Cw: deliver
Cw->>Tmp: StartWorkflow CollisionResponse
- loop each phase: back / settle / turn / forward / stop
+ loop each phase back, settle, turn, forward, stop
Tmp->>Cw: dispatch SendTwist activity
- Cw->>MQ: cmd/{id}/twist @ 10 Hz QoS 0
+ Cw->>MQ: cmd/{id}/twist at 10 Hz QoS 0
MQ->>Sub: deliver
- Sub->>Drive: ROS /cmd_vel → gz cmd_vel
+ Sub->>Drive: ROS /cmd_vel forwards to gz cmd_vel
end
- Cw->>MQ: cmd/{id}/twist {0,0} QoS 1 (final stop)
+ Cw->>MQ: cmd/{id}/twist 0,0 QoS 1 final stop
Cw->>Tmp: workflow Completed
```
From 959abbff3cbacb27a40540cd8abc892dd14ea213 Mon Sep 17 00:00:00 2001
From: Ben Kearns <35475+bkearns@users.noreply.github.com>
Date: Tue, 5 May 2026 17:02:43 -0700
Subject: [PATCH 7/7] feat(make): add sim-teleport to reset rover to origin
When an OTA controller drives the rover off the world or wedges it
into the boulder, the only recovery was 'make sim-down && make
sim-up' which rebuilds the world from scratch. This adds a quick
teleport that calls Ignition Fortress's set_pose service from inside
the gazebo container.
Defaults send the rover to (0, 0, 0.30) with identity orientation
(facing +x). Override TELEPORT_X/Y/Z=, ROBOT_MODEL=, or
SIM_WORLD_NAME= to land somewhere else or teleport a different
entity. Sends a zero twist first so any in-flight velocity from a
DiffDrive command doesn't get carried over to the new pose.
---
Makefile | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/Makefile b/Makefile
index a934147..e3c8348 100644
--- a/Makefile
+++ b/Makefile
@@ -422,6 +422,30 @@ sim-drive-stop: ## Stop the rover
@$(CONTAINER_ENGINE) exec $(SIM_CONTAINER) bash -c \
'ign topic -t $(GZ_DRIVE_TOPIC) -m ignition.msgs.Twist -p "linear: {x: 0}, angular: {z: 0}"'
+# Teleport the rover back to the origin. Useful when an OTA controller
+# runs the rover off the world or into the boulder. Override
+# TELEPORT_X / TELEPORT_Y / TELEPORT_Z to land somewhere else.
+SIM_GAZEBO_CONTAINER ?= temporal-hack-lab-gazebo-1
+SIM_WORLD_NAME ?= moon
+ROBOT_MODEL ?= perseverance
+TELEPORT_X ?= 0
+TELEPORT_Y ?= 0
+TELEPORT_Z ?= 0.30
+
+.PHONY: sim-teleport
+sim-teleport: ## Teleport the rover back to the origin (override TELEPORT_X/Y/Z)
+ @echo "[sim-teleport] $(ROBOT_MODEL) -> ($(TELEPORT_X), $(TELEPORT_Y), $(TELEPORT_Z))"
+ @# Stop the rover first so its old velocity doesn't get carried over.
+ @$(CONTAINER_ENGINE) exec $(SIM_GAZEBO_CONTAINER) bash -c \
+ 'ign topic -t $(GZ_DRIVE_TOPIC) -m ignition.msgs.Twist -p "linear: {x: 0}, angular: {z: 0}"' >/dev/null 2>&1 || true
+ @# set_pose service from inside the gazebo container. Orientation
+ @# is identity quaternion (w=1) — rover faces +x.
+ @$(CONTAINER_ENGINE) exec $(SIM_GAZEBO_CONTAINER) bash -c \
+ 'ign service -s /world/$(SIM_WORLD_NAME)/set_pose \
+ --reqtype ignition.msgs.Pose --reptype ignition.msgs.Boolean \
+ --timeout 2000 \
+ --req "name: \"$(ROBOT_MODEL)\", position: {x: $(TELEPORT_X), y: $(TELEPORT_Y), z: $(TELEPORT_Z)}, orientation: {w: 1.0}"'
+
# =============================================================================
# CI cluster (smoke / pre-push parity) — alternate ports so it can run
# alongside `make lab-up` on the same host. Used by .git-hooks/installer-smoke.sh