From 1b5eab195281ead8c8173e4373e92fbdc9bf100c Mon Sep 17 00:00:00 2001 From: Ben Kearns <35475+bkearns@users.noreply.github.com> Date: Tue, 5 May 2026 15:06:57 -0700 Subject: [PATCH 1/6] feat(sim): split sim container into gazebo + robot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulls the simulator out of the bundled "sim" container into its own service. The robot's always-on ROS infrastructure (bridge_node + sim_battery) moves into a sibling "robot" service. The OTA-swappable robot software (sim/controllers/*) is unchanged — those images still run as agent-managed containers on the lab network. Three services, all on the same lab network and ROS DDS domain (ROS_DOMAIN_ID=42, RMW=cyclonedds): gazebo - ign gazebo + ros_gz_bridge + Xvfb/x11vnc/noVNC GUI stack. Subscribes to /cmd_vel on ROS DDS and forwards to the gz diff_drive plugin. Now also bridges /perseverance/contacts so collision events are visible ROS-side (groundwork for the Temporal collision-response workflow). robot - bridge_node (rclpy → gRPC) + sim_battery. The agent's persistent peer. agent - unchanged otherwise. BRIDGE_ADDR moves from a Unix socket on a shared volume to TCP `robot:50051` over the lab network. BROKER_URL stays on tcp://mqtt:1883 (lab broker, port 14883 host-side). bridge_node/server.py: new --listen flag accepting either bare host:port (TCP) or unix://path. --socket retained for back-compat. moon.sdf: a 0.9 m boulder at x=8 directly in the rover's forward path. Rover deck has a contact sensor (`bumper`) tied to the gz-sim contact system; it emits Contacts messages on /perseverance/contacts whenever the deck collides. docker/sim/ removed; replaced by docker/gazebo/ + docker/robot/. Worlds + models live under docker/gazebo/ now. Verified locally on Apple Silicon + podman: - gazebo, robot, agent all Up - robot: "bridge listening on 0.0.0.0:50051" - agent: "bridge connected addr=robot:50051" - ros_gz_bridge: 6 bidirectional bridges including /perseverance/contacts ↔ ignition.msgs.Contacts - Lab MQTT, Postgres, Temporal, Registry all healthy --- bridge/bridge_node/server.py | 36 ++++-- docker/gazebo/Dockerfile | 53 ++++++++ docker/gazebo/entrypoint.sh | 92 ++++++++++++++ .../models/perseverance/model.config | 0 .../models/perseverance/model.sdf | 14 +++ docker/{sim => gazebo}/worlds/moon.sdf | 21 ++++ docker/robot/Dockerfile | 41 ++++++ docker/robot/entrypoint.sh | 27 ++++ docker/sim/Dockerfile | 68 ---------- docker/sim/entrypoint.sh | 118 ------------------ .../docker-compose/docker-compose.sim.yml | 73 +++++------ 11 files changed, 309 insertions(+), 234 deletions(-) create mode 100644 docker/gazebo/Dockerfile create mode 100644 docker/gazebo/entrypoint.sh rename docker/{sim => gazebo}/models/perseverance/model.config (100%) rename docker/{sim => gazebo}/models/perseverance/model.sdf (95%) rename docker/{sim => gazebo}/worlds/moon.sdf (87%) create mode 100644 docker/robot/Dockerfile create mode 100644 docker/robot/entrypoint.sh delete mode 100644 docker/sim/Dockerfile delete mode 100644 docker/sim/entrypoint.sh diff --git a/bridge/bridge_node/server.py b/bridge/bridge_node/server.py index 67e2c12..4843f9f 100644 --- a/bridge/bridge_node/server.py +++ b/bridge/bridge_node/server.py @@ -133,17 +133,33 @@ def _on_msg(self, msg: BatteryState) -> None: self.fanout.push(ev) -def serve(socket_path: str) -> None: +def serve(addr: str) -> None: + """Run the bridge gRPC server. + + `addr` accepts either a bare ``host:port`` for TCP, or a path / + ``unix://`` form for a Unix domain socket. Determined by + whether the value contains a colon and no slash. + """ fanout = TopicFanout() server = grpc.server(futures.ThreadPoolExecutor(max_workers=8)) pbg.add_BridgeServicer_to_server(BridgeService(fanout), server) - if os.path.exists(socket_path): - os.unlink(socket_path) - server.add_insecure_port(f"unix://{socket_path}") + if addr.startswith("unix://"): + listen = addr + sock_path = addr[len("unix://"):] + if os.path.exists(sock_path): + os.unlink(sock_path) + elif ":" in addr and "/" not in addr: + listen = addr # bare host:port -> TCP + else: + # bare path -> Unix domain socket + if os.path.exists(addr): + os.unlink(addr) + listen = f"unix://{addr}" + server.add_insecure_port(listen) server.start() - log.info("bridge listening on %s", socket_path) + log.info("bridge listening on %s", listen) if HAVE_ROS: rclpy.init(args=None) @@ -163,10 +179,16 @@ def serve(socket_path: str) -> None: def main() -> None: # pragma: no cover p = argparse.ArgumentParser() - p.add_argument("--socket", default="/run/temporal-hack-bridge.sock") + # --listen is the modern arg (TCP host:port or unix:// path). + # --socket kept for backwards compat with older callers. + p.add_argument("--listen", default=None, + help="gRPC listen address: 'host:port' or 'unix:///path'") + p.add_argument("--socket", default=None, + help="(deprecated) Unix domain socket path. Use --listen.") args = p.parse_args() + addr = args.listen or args.socket or "/run/temporal-hack-bridge.sock" logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") - serve(args.socket) + serve(addr) if __name__ == "__main__": diff --git a/docker/gazebo/Dockerfile b/docker/gazebo/Dockerfile new file mode 100644 index 0000000..b6362c7 --- /dev/null +++ b/docker/gazebo/Dockerfile @@ -0,0 +1,53 @@ +# Gazebo (simulator) container. +# +# Runs Ignition Fortress + ros_gz_bridge + an Xvfb/x11vnc/noVNC stack +# for browser-served GUI access. NOTHING ROS-application-specific +# lives here — the rover's controllers (drive-circle, drive-figure-eight) +# and the agent's bridge_node live in their own containers and reach +# this one over the lab network's ROS DDS domain. +# +# What's in vs. out: +# IN: gz sim, ros_gz_bridge, Xvfb/x11vnc/novnc/fluxbox, mesa, +# worlds/, models/ +# OUT: bridge_node (rclpy → agent gRPC), sim_battery, controllers +FROM ros:humble-ros-base + +ENV DEBIAN_FRONTEND=noninteractive +ENV ROS_DOMAIN_ID=42 +ENV RMW_IMPLEMENTATION=rmw_cyclonedds_cpp + +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget gnupg lsb-release ca-certificates \ + && wget -qO /usr/share/keyrings/pkgs-osrf-archive-keyring.gpg \ + https://packages.osrfoundation.org/gazebo.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/pkgs-osrf-archive-keyring.gpg] http://packages.osrfoundation.org/gazebo/ubuntu-stable $(lsb_release -cs) main" \ + > /etc/apt/sources.list.d/gazebo-stable.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ros-humble-rmw-cyclonedds-cpp \ + ros-humble-ros-gz-bridge \ + ignition-fortress \ + libgl1-mesa-dri \ + libgl1-mesa-glx \ + mesa-utils \ + xvfb \ + x11vnc \ + novnc \ + websockify \ + fluxbox \ + net-tools \ + && rm -rf /var/lib/apt/lists/* + +# Sim worlds + models. GZ_SIM_RESOURCE_PATH lets gz-sim resolve +# `model://...` includes at runtime. +COPY docker/gazebo/worlds /opt/sim/worlds +COPY docker/gazebo/models /opt/sim/models +ENV GZ_SIM_RESOURCE_PATH="/opt/sim/models:/opt/sim/worlds" + +COPY docker/gazebo/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# noVNC (browser GUI): http://localhost:14680 +# Raw VNC (any client): localhost:14900 +EXPOSE 5900 6080 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/gazebo/entrypoint.sh b/docker/gazebo/entrypoint.sh new file mode 100644 index 0000000..8c7f72f --- /dev/null +++ b/docker/gazebo/entrypoint.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# gazebo container entrypoint: +# Xvfb -> fluxbox -> x11vnc -> websockify (browser GUI pipe) +# ign gazebo -r -v 4 (sim engine + GUI) +# ros_gz_bridge (subscribes to /cmd_vel +# on ROS DDS, forwards to +# the gz cmd_vel topic) +set -eo pipefail + +# ROS 2's setup.bash references vars under nounset; wrap. +set +u; source /opt/ros/humble/setup.bash; set -u + +SIM_WORLD="${SIM_WORLD:-moon.sdf}" +HEADLESS="${HEADLESS:-0}" +DISPLAY_NUM="${DISPLAY_NUM:-1}" +SCREEN_GEOMETRY="${SCREEN_GEOMETRY:-1920x1200x24}" + +PIDS=() +shutdown() { + echo "[gazebo] shutting down" + for pid in "${PIDS[@]}"; do kill "$pid" 2>/dev/null || true; done + exit 0 +} +trap shutdown SIGINT SIGTERM + +if [ "$HEADLESS" = "1" ]; then + echo "[gazebo] HEADLESS=1; running ign gazebo server only ($SIM_WORLD)" + ign gazebo -s -r -v 4 "$SIM_WORLD" & + PIDS+=($!) +else + echo "[gazebo] starting Xvfb on :$DISPLAY_NUM ($SCREEN_GEOMETRY)" + Xvfb ":$DISPLAY_NUM" -screen 0 "$SCREEN_GEOMETRY" -nolisten tcp & + PIDS+=($!) + sleep 1 + export DISPLAY=":$DISPLAY_NUM" + + fluxbox 2>/dev/null & + PIDS+=($!) + + x11vnc -display "$DISPLAY" -forever -shared -nopw -quiet -rfbport 5900 -bg -o /tmp/x11vnc.log + + websockify --web=/usr/share/novnc 6080 localhost:5900 & + PIDS+=($!) + + echo "[gazebo] launching ign gazebo with GUI: $SIM_WORLD" + export LIBGL_ALWAYS_SOFTWARE=1 + export OGRE2_RTSHADERSYSTEM_WRITE_SHADERS_TO_DISK=0 + ign gazebo -r -v 4 "$SIM_WORLD" & + PIDS+=($!) + + echo + echo " ┌──────────────────────────────────────────────────────────────────────────────┐" + echo " │ Gazebo GUI: http://localhost:14680/vnc.html?autoconnect=1&resize=scale │" + echo " │ Raw VNC: localhost:14900 (no password) │" + echo " └──────────────────────────────────────────────────────────────────────────────┘" + echo +fi + +# Bridge ROS 2 /cmd_vel and odom for the perseverance rover. The +# diff-drive plugin in moon.sdf listens on /model/perseverance/cmd_vel; +# remap so external ROS 2 controllers can publish to plain /cmd_vel. +sleep 4 +case "$SIM_WORLD" in + moon.sdf) + echo "[gazebo] starting ros_gz_bridge: /cmd_vel + odom + /perseverance/contacts" + ros2 run ros_gz_bridge parameter_bridge \ + /model/perseverance/cmd_vel@geometry_msgs/msg/Twist@ignition.msgs.Twist \ + /model/perseverance/odometry@nav_msgs/msg/Odometry@ignition.msgs.Odometry \ + /perseverance/contacts@ros_gz_interfaces/msg/Contacts@ignition.msgs.Contacts \ + --ros-args \ + -r /model/perseverance/cmd_vel:=/cmd_vel \ + -r /model/perseverance/odometry:=/odom \ + -r /perseverance/contacts:=/contacts & + PIDS+=($!) + ;; + diff_drive.sdf|diff_drive_skid.sdf) + echo "[gazebo] starting ros_gz_bridge: /cmd_vel ↔ /model/vehicle_blue/cmd_vel" + ros2 run ros_gz_bridge parameter_bridge \ + /model/vehicle_blue/cmd_vel@geometry_msgs/msg/Twist@ignition.msgs.Twist \ + /model/vehicle_blue/odometry@nav_msgs/msg/Odometry@ignition.msgs.Odometry \ + --ros-args \ + -r /model/vehicle_blue/cmd_vel:=/cmd_vel \ + -r /model/vehicle_blue/odometry:=/odom & + PIDS+=($!) + ;; + *) + echo "[gazebo] no ros_gz_bridge wired for world '$SIM_WORLD'" + ;; +esac + +# Wait for any background process to exit (or signal). +wait -n diff --git a/docker/sim/models/perseverance/model.config b/docker/gazebo/models/perseverance/model.config similarity index 100% rename from docker/sim/models/perseverance/model.config rename to docker/gazebo/models/perseverance/model.config diff --git a/docker/sim/models/perseverance/model.sdf b/docker/gazebo/models/perseverance/model.sdf similarity index 95% rename from docker/sim/models/perseverance/model.sdf rename to docker/gazebo/models/perseverance/model.sdf index ddb92b0..47db1bb 100644 --- a/docker/sim/models/perseverance/model.sdf +++ b/docker/gazebo/models/perseverance/model.sdf @@ -30,6 +30,20 @@ 1.20 1.40 0.40 + + + + 10 + + deck_col + /perseverance/contacts + + + 1.20 1.40 0.40 diff --git a/docker/sim/worlds/moon.sdf b/docker/gazebo/worlds/moon.sdf similarity index 87% rename from docker/sim/worlds/moon.sdf rename to docker/gazebo/worlds/moon.sdf index c9edb95..ff6c368 100644 --- a/docker/sim/worlds/moon.sdf +++ b/docker/gazebo/worlds/moon.sdf @@ -105,6 +105,27 @@ 0.18 0.17 0.16 10.36 0.34 0.32 1 + + + true + 8 0 0.50 0 0 0 + + + 0.90 0.80 0.50 + + + 0.90 0.80 0.50 + + 0.22 0.20 0.17 1 + 0.45 0.41 0.36 1 + 0.05 0.05 0.05 1 + + + + + model://perseverance diff --git a/docker/robot/Dockerfile b/docker/robot/Dockerfile new file mode 100644 index 0000000..2a0e73d --- /dev/null +++ b/docker/robot/Dockerfile @@ -0,0 +1,41 @@ +# Robot container — always-on ROS infrastructure for the simulated +# rover. +# +# What's in vs. out: +# IN: bridge_node (rclpy → gRPC for the agent), sim_battery +# OUT: gz sim, ros_gz_bridge, GUI stack — those live in docker/gazebo/. +# OTA-swappable controllers — those live as separate +# sim/controllers/* images managed by the agent. +# +# This container is *not* the OTA target. It hosts the robot's +# always-on ROS plumbing. The OTA-swappable software runs as a +# sibling container (drive-circle, drive-figure-eight, ...) on the +# same lab network and ROS DDS domain. +FROM ros:humble-ros-base + +ENV DEBIAN_FRONTEND=noninteractive +ENV ROS_DOMAIN_ID=42 +ENV RMW_IMPLEMENTATION=rmw_cyclonedds_cpp +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ros-humble-rmw-cyclonedds-cpp \ + ros-humble-sensor-msgs \ + python3-pip \ + python3-grpcio \ + python3-grpc-tools \ + && rm -rf /var/lib/apt/lists/* + +# Bridge package (rclpy → gRPC) and sim_battery publisher. +WORKDIR /opt/bridge +COPY bridge/ /opt/bridge/ +RUN python3 -m pip install --no-cache-dir --upgrade "pip>=23.0" \ + && python3 -m pip install --no-cache-dir --break-system-packages /opt/bridge + +COPY docker/robot/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# bridge_node gRPC server (TCP) — agent connects here. +EXPOSE 50051 + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/robot/entrypoint.sh b/docker/robot/entrypoint.sh new file mode 100644 index 0000000..da66dce --- /dev/null +++ b/docker/robot/entrypoint.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# robot container entrypoint: +# sim_battery (rclpy) — synthetic /battery_state publisher +# bridge_node.server (rclpy) — gRPC peer for the agent on TCP 50051 +set -eo pipefail + +set +u; source /opt/ros/humble/setup.bash; set -u + +LISTEN="${BRIDGE_LISTEN:-0.0.0.0:50051}" + +PIDS=() +shutdown() { + echo "[robot] shutting down" + for pid in "${PIDS[@]}"; do kill "$pid" 2>/dev/null || true; done + exit 0 +} +trap shutdown SIGINT SIGTERM + +# sim_battery is a stand-in /battery_state publisher (Gazebo doesn't +# emit one for our model). Real robots publish their own; this is +# only present in the dev sim. +echo "[robot] starting sim_battery" +python3 -m bridge_node.sim_battery & +PIDS+=($!) + +echo "[robot] starting bridge gRPC on $LISTEN" +exec python3 -m bridge_node.server --listen "$LISTEN" diff --git a/docker/sim/Dockerfile b/docker/sim/Dockerfile deleted file mode 100644 index 1281e1b..0000000 --- a/docker/sim/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -# Robot simulation container — Moon world + Ingenuity helicopter. -# -# Base: ros:humble-ros-base (multi-arch — amd64 + arm64 native). -# Sim: Gazebo Harmonic (LTS 2024-2028) from packages.osrfoundation.org. -# Replaces Ignition Fortress; same plugin family (gz-sim-*). -# GUI: Xvfb + fluxbox + x11vnc + noVNC + websockify, browser-served. -# -# Why Harmonic, not Fortress: the AndrejOrsula moon world targets -# gz-sim-* plugin filenames. Harmonic ships them and is arm64-native. -# Why we don't run their Blender procgen here: Blender 4.3 is x86_64- -# only on Linux, and their amd64 image hits a /bin/sh portability bug -# under buildah. The procgen step is a clean follow-up; for now we -# ship a hand-authored moon.sdf + ingenuity.sdf. -FROM ros:humble-ros-base - -ENV DEBIAN_FRONTEND=noninteractive -ENV ROS_DOMAIN_ID=42 -ENV RMW_IMPLEMENTATION=rmw_cyclonedds_cpp - -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget gnupg lsb-release ca-certificates \ - && wget -qO /usr/share/keyrings/pkgs-osrf-archive-keyring.gpg \ - https://packages.osrfoundation.org/gazebo.gpg \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/pkgs-osrf-archive-keyring.gpg] http://packages.osrfoundation.org/gazebo/ubuntu-stable $(lsb_release -cs) main" \ - > /etc/apt/sources.list.d/gazebo-stable.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - ros-humble-rmw-cyclonedds-cpp \ - ros-humble-sensor-msgs \ - ros-humble-ros-gz-bridge \ - ignition-fortress \ - libgl1-mesa-dri \ - libgl1-mesa-glx \ - mesa-utils \ - python3-pip \ - python3-grpcio \ - python3-grpc-tools \ - xvfb \ - x11vnc \ - novnc \ - websockify \ - fluxbox \ - net-tools \ - && rm -rf /var/lib/apt/lists/* - -# Bridge package + deps. -WORKDIR /opt/bridge -COPY bridge/ /opt/bridge/ -RUN python3 -m pip install --no-cache-dir --upgrade "pip>=23.0" \ - && python3 -m pip install --no-cache-dir --break-system-packages /opt/bridge - -# Sim worlds + models. GZ_SIM_RESOURCE_PATH is read by gz-sim to -# resolve `model://...` URIs in include directives. -COPY docker/sim/worlds /opt/sim/worlds -COPY docker/sim/models /opt/sim/models -ENV GZ_SIM_RESOURCE_PATH="/opt/sim/models:/opt/sim/worlds" - -# Sim launch script. -COPY docker/sim/entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -VOLUME ["/run/bridge"] - -# noVNC (browser GUI): http://localhost:14680 -# Raw VNC (any client): localhost:14900 -EXPOSE 5900 6080 - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/sim/entrypoint.sh b/docker/sim/entrypoint.sh deleted file mode 100644 index a17efeb..0000000 --- a/docker/sim/entrypoint.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# ROS 2's setup.bash references variables it doesn't always pre-set -# (AMENT_TRACE_SETUP_FILES, etc.). Disable nounset for the source -# step only; turn it back on for our own logic afterwards. -set +u -source /opt/ros/humble/setup.bash -set -u - -BRIDGE_SOCKET="${BRIDGE_SOCKET:-/run/bridge/temporal-hack-bridge.sock}" -HEADLESS="${HEADLESS:-0}" # default 0: show the GUI via noVNC. - # Set HEADLESS=1 to skip the X stack entirely. -DISPLAY_NUM="${DISPLAY_NUM:-1}" -SCREEN_GEOMETRY="${SCREEN_GEOMETRY:-1920x1200x24}" - -mkdir -p "$(dirname "$BRIDGE_SOCKET")" - -# Process tracker so we can clean up children on SIGTERM. -PIDS=() -shutdown() { - echo "[sim] shutting down" - for pid in "${PIDS[@]}"; do kill "$pid" 2>/dev/null || true; done - exit 0 -} -trap shutdown SIGINT SIGTERM - -SIM_WORLD="${SIM_WORLD:-moon.sdf}" - -if [ "$HEADLESS" = "1" ]; then - echo "[sim] HEADLESS=1; running ign gazebo server only ($SIM_WORLD)" - ign gazebo -s -v 4 "$SIM_WORLD" & - PIDS+=($!) -else - # Start a virtual X display, a minimal window manager, a VNC - # server bound to that display, and a websockify→noVNC bridge so - # the GUI is reachable over a browser at port 6080 (mapped to - # 14680 on the host by docker-compose.sim.yml). - echo "[sim] starting Xvfb on :$DISPLAY_NUM ($SCREEN_GEOMETRY)" - Xvfb ":$DISPLAY_NUM" -screen 0 "$SCREEN_GEOMETRY" -nolisten tcp & - PIDS+=($!) - sleep 1 - export DISPLAY=":$DISPLAY_NUM" - - echo "[sim] starting fluxbox window manager" - fluxbox 2>/dev/null & - PIDS+=($!) - - echo "[sim] starting x11vnc on :5900" - x11vnc -display "$DISPLAY" -forever -shared -nopw -quiet -rfbport 5900 -bg -o /tmp/x11vnc.log - # x11vnc -bg already daemonised; nothing to push to PIDS. - - echo "[sim] starting noVNC websockify on :6080" - # Debian's `novnc` package ships /usr/share/novnc/ + a launcher. - websockify --web=/usr/share/novnc 6080 localhost:5900 & - PIDS+=($!) - - echo "[sim] launching Ignition Fortress with GUI: $SIM_WORLD" - # Software OpenGL is the only path that works in Xvfb (no GPU). - # OGRE2 will use Mesa's llvmpipe and render to the virtual fb. - export LIBGL_ALWAYS_SOFTWARE=1 - export OGRE2_RTSHADERSYSTEM_WRITE_SHADERS_TO_DISK=0 - # -r: start running (not paused). Without this Fortress opens - # paused and the user has to click Play before any movement - # commands have effect. - ign gazebo -r -v 4 "$SIM_WORLD" & - PIDS+=($!) - - echo - echo " ┌──────────────────────────────────────────────────────────────────────────────┐" - echo " │ Gazebo GUI: http://localhost:14680/vnc.html?autoconnect=1&resize=scale │" - echo " │ Raw VNC: localhost:14900 (no password) │" - echo " └──────────────────────────────────────────────────────────────────────────────┘" - echo -fi - -# ROS 2 ↔ Ignition bridge for the diff_drive demo world only. That -# world's `vehicle_blue` model subscribes to /model/vehicle_blue/cmd_vel; -# external ROS 2 controllers publish on /cmd_vel, so we remap. Skip -# this for moon.sdf (no vehicle_blue there) — the bridge would just -# sit waiting for a non-existent topic. -case "$SIM_WORLD" in - diff_drive.sdf|diff_drive_skid.sdf) - sleep 4 - echo "[sim] starting ros_gz_bridge: /cmd_vel ↔ /model/vehicle_blue/cmd_vel" - ros2 run ros_gz_bridge parameter_bridge \ - /model/vehicle_blue/cmd_vel@geometry_msgs/msg/Twist@ignition.msgs.Twist \ - /model/vehicle_blue/odometry@nav_msgs/msg/Odometry@ignition.msgs.Odometry \ - --ros-args \ - -r /model/vehicle_blue/cmd_vel:=/cmd_vel \ - -r /model/vehicle_blue/odometry:=/odom & - PIDS+=($!) - ;; - moon.sdf) - sleep 4 - echo "[sim] starting ros_gz_bridge: /cmd_vel ↔ /model/perseverance/cmd_vel" - ros2 run ros_gz_bridge parameter_bridge \ - /model/perseverance/cmd_vel@geometry_msgs/msg/Twist@ignition.msgs.Twist \ - /model/perseverance/odometry@nav_msgs/msg/Odometry@ignition.msgs.Odometry \ - --ros-args \ - -r /model/perseverance/cmd_vel:=/cmd_vel \ - -r /model/perseverance/odometry:=/odom & - PIDS+=($!) - ;; - *) - echo "[sim] no ros_gz_bridge wired for world '$SIM_WORLD'" - ;; -esac - -# Synthetic battery (TurtleBot3 sim doesn't emit /battery_state). -sleep 6 -python3 -m bridge_node.sim_battery & -PIDS+=($!) - -# Bridge: blocks foreground; this is the canonical lifetime of the -# container. -echo "[sim] starting bridge on $BRIDGE_SOCKET" -exec python3 -m bridge_node.server --socket "$BRIDGE_SOCKET" diff --git a/installer/docker-compose/docker-compose.sim.yml b/installer/docker-compose/docker-compose.sim.yml index 9b4f37f..8ebf412 100644 --- a/installer/docker-compose/docker-compose.sim.yml +++ b/installer/docker-compose/docker-compose.sim.yml @@ -1,77 +1,69 @@ -# Sim overlay: brings up one virtual robot — Gazebo + TurtleBot3 + -# the bridge node — and an agent container that consumes the bridge's -# gRPC stream and reports telemetry to the lab MQTT broker. +# Sim overlay — composed onto the lab project. # -# Composed on top of docker-compose.yml under the same project name, -# so the sim and lab containers share a network and the agent can -# reach the broker at `mqtt:1883` (container-internal port — never -# the host-side port mapping). +# Three services: +# gazebo - the simulator + ros_gz_bridge + browser GUI (noVNC) +# robot - always-on ROS infrastructure: bridge_node (rclpy → +# agent gRPC) + sim_battery +# agent - the Go agent; OTAs robot-app images # -# Use: make sim-up (lab cluster + sim + agent) -# make sim-down -# make sim-logs +# The OTA-swappable robot software (controllers in sim/controllers/*) +# is *not* a compose service — the agent spawns those containers on +# demand and joins them to this same network + ROS DDS domain. services: - sim: + gazebo: build: - # Build context is the repo root so the bridge/ tree can be COPY'd. context: ../.. - dockerfile: docker/sim/Dockerfile - image: temporal-hack/sim:dev + dockerfile: docker/gazebo/Dockerfile + image: temporal-hack/gazebo:dev environment: - TURTLEBOT3_MODEL: burger ROS_DOMAIN_ID: 42 - # HEADLESS=0 means the GUI is served over noVNC. To run gzserver - # only (no GUI, no X stack), set HEADLESS=1 in the env. + RMW_IMPLEMENTATION: rmw_cyclonedds_cpp HEADLESS: "${SIM_HEADLESS:-0}" - BRIDGE_SOCKET: /run/bridge/temporal-hack-bridge.sock + SIM_WORLD: "${SIM_WORLD:-moon.sdf}" ports: - # Browser GUI (noVNC): http://localhost:14680/vnc.html?autoconnect=1 - "${SIM_NOVNC_PORT:-14680}:6080" - # Raw VNC (TigerVNC, macOS Screen Sharing, etc.) at localhost:14900 - "${SIM_VNC_PORT:-14900}:5900" - volumes: - - bridge-sock:/run/bridge + + robot: + build: + context: ../.. + dockerfile: docker/robot/Dockerfile + image: temporal-hack/robot:dev + environment: + ROS_DOMAIN_ID: 42 + RMW_IMPLEMENTATION: rmw_cyclonedds_cpp + expose: + - "50051" agent: image: golang:1.22-bookworm depends_on: mqtt: condition: service_healthy - sim: + robot: condition: service_started working_dir: /src/agent environment: ROBOT_ID: sim-robot-01 - # mqtt and sim are sibling services on the same Compose network; - # the agent uses the canonical container-internal ports. + # Lab broker (back to default port 14883 inside the network is 1883). BROKER_URL: tcp://mqtt:1883 BUFFER_PATH: /var/lib/agent/buffer.db - BRIDGE_ADDR: unix:///run/bridge/temporal-hack-bridge.sock + # Bridge moved off Unix-domain socket into a sibling container; + # talk to it over the lab network on the canonical gRPC port. + BRIDGE_ADDR: robot:50051 # OTA-spawned robot-app containers must join the lab network and - # share the sim's ROS_DOMAIN_ID so they can publish /cmd_vel into - # the same DDS partition the ros_gz_bridge listens on. Without - # this, OTA runs the container on the default bridge network and - # the new behaviour never reaches the simulated robot. + # share ROS_DOMAIN_ID so /cmd_vel reaches the gazebo container's + # ros_gz_bridge. OTA_RUN_ARGS: "--network=temporal-hack-lab_default,-e,ROS_DOMAIN_ID=42,-e,RMW_IMPLEMENTATION=rmw_cyclonedds_cpp" - # The repo is bind-mounted read-only. Go's workspace mode tries - # to update go.work.sum on `go run` and fails; turn the workspace - # off so the agent module's own go.mod/go.sum are authoritative. GOWORK: "off" - # Use a writable cache dir; module cache and build cache go to a - # named volume so we don't fight the read-only bind mount. GOCACHE: /var/cache/go-build GOMODCACHE: /var/cache/go-mod volumes: - ../..:/src:ro - - bridge-sock:/run/bridge - agent-data:/var/lib/agent - go-cache:/var/cache/go-build - go-mod:/var/cache/go-mod - # Agent needs the container-runtime socket on the host to - # perform OTA. CONTAINER_SOCK is supplied by the Makefile at - # compose time (resolves to docker.sock or podman.sock). - ${CONTAINER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock - # apt is needed to install the C toolchain for go-sqlite3 (cgo). command: > bash -c " apt-get update -qq && apt-get install -y -qq build-essential >/dev/null && @@ -80,7 +72,6 @@ services: " volumes: - bridge-sock: agent-data: go-cache: go-mod: From edc21af0ee2e61826ddf1d45048a94eccfca9ea4 Mon Sep 17 00:00:00 2001 From: Ben Kearns <35475+bkearns@users.noreply.github.com> Date: Tue, 5 May 2026 15:21:00 -0700 Subject: [PATCH 2/6] feat(temporal): collision-response workflow + MQTT bridge User asked for a Temporal job that listens for a robot collision and backs the rover up, turns right 90 deg, drives forward again. Wires the full chain. Robot: ros-humble-ros-gz-interfaces + python3-paho-mqtt; new collision_publisher (ROS /contacts -> MQTT events/{robot_id}/collision, 2s debounce) and twist_subscriber (MQTT cmd/{robot_id}/twist -> ROS /cmd_vel). paho 1.x API for jammy. Cloud cloud/internal/collision + cmd/collision-worker: - CollisionResponse workflow: back -0.3 m/s 3s, settle, turn -0.5 rad/s 3.14s (~90 deg right), settle, forward 0.4 m/s 5s, stop. - SendTwist activity republishes at 10Hz so DiffDrive's ~0.5s command timeout does not stall the rover. - mqttbridge subscribes to events/+/collision and starts a workflow per inbound event. Make: workers-up / workers-down / workers-status to run the host-side ota-worker + collision-worker. 'make collide' triggers a fake event. PID files in .run/. Verified end-to-end: collision-{robot_id}-{ts} workflow visible at http://localhost:14080; rover odometry advances through the phases. --- .gitignore | 2 + Makefile | 48 ++++++++++++ bridge/bridge_node/collision_publisher.py | 85 +++++++++++++++++++++ bridge/bridge_node/twist_subscriber.py | 72 +++++++++++++++++ cloud/cmd/collision-worker/main.go | 89 ++++++++++++++++++++++ cloud/internal/collision/activities.go | 62 +++++++++++++++ cloud/internal/collision/mqttbridge.go | 76 ++++++++++++++++++ cloud/internal/collision/types.go | 31 ++++++++ cloud/internal/collision/workflow.go | 58 ++++++++++++++ cloud/internal/ota/mqttbridge.go | 16 ++-- cloud/internal/ota/types.go | 14 ++-- cloud/internal/ota/workflow_singlerobot.go | 8 +- docker/robot/Dockerfile | 3 + docker/robot/entrypoint.sh | 14 +++- 14 files changed, 558 insertions(+), 20 deletions(-) create mode 100644 bridge/bridge_node/collision_publisher.py create mode 100644 bridge/bridge_node/twist_subscriber.py create mode 100644 cloud/cmd/collision-worker/main.go create mode 100644 cloud/internal/collision/activities.go create mode 100644 cloud/internal/collision/mqttbridge.go create mode 100644 cloud/internal/collision/types.go create mode 100644 cloud/internal/collision/workflow.go diff --git a/.gitignore b/.gitignore index a7b7555..f26a86b 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ __pycache__/ # Installer state installer/docker-compose/.data/ installer/docker-compose/.secrets/ +/bin/ +/.run/ diff --git a/Makefile b/Makefile index bfb9d60..5caefb1 100644 --- a/Makefile +++ b/Makefile @@ -109,6 +109,54 @@ build-cloud: cd cloud && go build -o ../bin/controlplane ./cmd/controlplane cd cloud && go build -o ../bin/telemetry-ingest ./cmd/telemetry-ingest cd cloud && go build -o ../bin/ota-worker ./cmd/ota-worker + cd cloud && go build -o ../bin/collision-worker ./cmd/collision-worker + +# ============================================================================= +# Workflow workers — host-side Go binaries that connect to the lab +# Temporal frontend + MQTT broker. Without these running, the +# Temporal UI shows no Workers / no in-flight Workflows. +# ============================================================================= + +WORKER_TEMPORAL_ADDR ?= localhost:14733 +WORKER_BROKER_URL ?= tcp://localhost:14883 +WORKER_TSDB_DSN ?= postgres://temporal:temporal@localhost:14432/telemetry?sslmode=disable + +.PHONY: workers-up +workers-up: build-cloud ## Start ota-worker + collision-worker in the background + @mkdir -p .run + @TEMPORAL_ADDR=$(WORKER_TEMPORAL_ADDR) BROKER_URL=$(WORKER_BROKER_URL) \ + TSDB_DSN="$(WORKER_TSDB_DSN)" \ + nohup ./bin/ota-worker > .run/ota-worker.log 2>&1 & echo $$! > .run/ota-worker.pid + @TEMPORAL_ADDR=$(WORKER_TEMPORAL_ADDR) BROKER_URL=$(WORKER_BROKER_URL) \ + nohup ./bin/collision-worker > .run/collision-worker.log 2>&1 & echo $$! > .run/collision-worker.pid + @sleep 1 + @echo "ota-worker PID $$(cat .run/ota-worker.pid 2>/dev/null) log .run/ota-worker.log" + @echo "collision-worker PID $$(cat .run/collision-worker.pid 2>/dev/null) log .run/collision-worker.log" + +.PHONY: workers-down +workers-down: ## Stop the workflow workers + @for f in .run/ota-worker.pid .run/collision-worker.pid; do \ + [ -f $$f ] && kill "$$(cat $$f)" 2>/dev/null && rm -f $$f && echo "stopped $$f" || true; \ + done + +.PHONY: workers-status +workers-status: ## Show status of running workflow workers + @for n in ota-worker collision-worker; do \ + pid="$$(cat .run/$$n.pid 2>/dev/null || echo '')"; \ + if [ -n "$$pid" ] && kill -0 "$$pid" 2>/dev/null; then \ + echo "$$n: running (pid $$pid)"; \ + else \ + echo "$$n: not running"; \ + fi; \ + done + +# Publish a fake collision event to trigger a CollisionResponse workflow. +# Uses the lab broker directly via the robot container's paho client. +.PHONY: collide +collide: ## Publish a fake collision event for sim-robot-01 (triggers Temporal workflow) + @$(CONTAINER_ENGINE) exec temporal-hack-lab-robot-1 python3 -c \ + "import paho.mqtt.publish as p, time, json; p.single('events/sim-robot-01/collision', json.dumps({'robot_id':'sim-robot-01','at':time.time(),'count':1,'partner':'manual-trigger'}), hostname='mqtt', port=1883, qos=1)" + @echo "published events/sim-robot-01/collision; check Temporal UI for collision-* workflow" .PHONY: build-agent build-agent: diff --git a/bridge/bridge_node/collision_publisher.py b/bridge/bridge_node/collision_publisher.py new file mode 100644 index 0000000..e294d0d --- /dev/null +++ b/bridge/bridge_node/collision_publisher.py @@ -0,0 +1,85 @@ +"""collision_publisher: ROS /contacts -> MQTT events/{robot_id}/collision. + +Subscribes to the ros_gz_interfaces/Contacts topic that gazebo's +ros_gz_bridge republishes from /perseverance/contacts. When a contact +is observed (any contact, not just rocks), publishes a small JSON +event to the cloud broker so the Temporal CollisionResponse workflow +can be triggered. + +Debounces aggressive contact streams: the contact sensor fires every +simulation step while a collision is sustained, which would flood +MQTT and trigger many workflows. We collapse repeats within a 2s +window into a single event. +""" + +from __future__ import annotations + +import json +import os +import time + +import paho.mqtt.client as mqtt +import rclpy +from rclpy.node import Node +from ros_gz_interfaces.msg import Contacts + + +DEBOUNCE_SEC = 2.0 + + +class CollisionPublisher(Node): + def __init__(self) -> None: + super().__init__("collision_publisher") + self.robot_id = os.environ.get("ROBOT_ID", "sim-robot-01") + self.broker_host = os.environ.get("MQTT_HOST", "mqtt") + self.broker_port = int(os.environ.get("MQTT_PORT", "1883")) + self.topic = f"events/{self.robot_id}/collision" + + # paho-mqtt 1.x API (Ubuntu jammy ships 1.6); v2's + # CallbackAPIVersion isn't available here. + self.client = mqtt.Client(client_id=f"collision-pub-{self.robot_id}") + self.client.connect(self.broker_host, self.broker_port, keepalive=30) + self.client.loop_start() + + self.create_subscription(Contacts, "/contacts", self._on_contacts, 10) + self._last_emit = 0.0 + self._counter = 0 + self.get_logger().info( + f"collision_publisher up: ROS /contacts -> MQTT {self.broker_host}:{self.broker_port}/{self.topic}" + ) + + def _on_contacts(self, msg: Contacts) -> None: + # Contacts message has a `contacts` field — a list. Empty list + # means "no contacts this step"; only act on non-empty. + if not msg.contacts: + return + now = time.monotonic() + if now - self._last_emit < DEBOUNCE_SEC: + return + self._last_emit = now + self._counter += 1 + partner = msg.contacts[0].collision2.name if msg.contacts else "unknown" + body = json.dumps({ + "robot_id": self.robot_id, + "at": time.time(), + "count": self._counter, + "partner": partner, + }) + self.client.publish(self.topic, body, qos=1) + self.get_logger().warn(f"collision (count={self._counter}, partner={partner})") + + +def main() -> None: # pragma: no cover + rclpy.init() + node = CollisionPublisher() + try: + rclpy.spin(node) + finally: + node.client.loop_stop() + node.client.disconnect() + node.destroy_node() + rclpy.shutdown() + + +if __name__ == "__main__": + main() diff --git a/bridge/bridge_node/twist_subscriber.py b/bridge/bridge_node/twist_subscriber.py new file mode 100644 index 0000000..2832715 --- /dev/null +++ b/bridge/bridge_node/twist_subscriber.py @@ -0,0 +1,72 @@ +"""twist_subscriber: MQTT cmd/{robot_id}/twist -> ROS /cmd_vel. + +The cloud-side CollisionResponse workflow publishes Twist commands on +this MQTT topic at ~10 Hz for each phase (back-up, turn, forward). +This node deserialises and republishes onto ROS /cmd_vel; the +gazebo container's ros_gz_bridge forwards to the diff-drive plugin. + +Wire format (JSON): + {"linear_x": -0.3, "angular_z": 0} +""" + +from __future__ import annotations + +import json +import os + +import paho.mqtt.client as mqtt +import rclpy +from rclpy.node import Node +from geometry_msgs.msg import Twist + + +class TwistSubscriber(Node): + def __init__(self) -> None: + super().__init__("twist_subscriber") + self.robot_id = os.environ.get("ROBOT_ID", "sim-robot-01") + self.broker_host = os.environ.get("MQTT_HOST", "mqtt") + self.broker_port = int(os.environ.get("MQTT_PORT", "1883")) + self.topic = f"cmd/{self.robot_id}/twist" + + self.pub = self.create_publisher(Twist, "/cmd_vel", 10) + + # paho-mqtt 1.x API (Ubuntu jammy ships 1.6). + self.client = mqtt.Client(client_id=f"twist-sub-{self.robot_id}") + self.client.on_message = self._on_message + self.client.connect(self.broker_host, self.broker_port, keepalive=30) + self.client.subscribe(self.topic, qos=1) + self.client.loop_start() + self.get_logger().info( + f"twist_subscriber up: MQTT {self.broker_host}:{self.broker_port}/{self.topic} -> ROS /cmd_vel" + ) + + def _on_message(self, _client, _userdata, msg) -> None: + try: + body = json.loads(msg.payload) + except Exception as e: + self.get_logger().warn(f"bad twist payload: {e}") + return + t = Twist() + t.linear.x = float(body.get("linear_x", 0.0)) + t.linear.y = float(body.get("linear_y", 0.0)) + t.linear.z = float(body.get("linear_z", 0.0)) + t.angular.x = float(body.get("angular_x", 0.0)) + t.angular.y = float(body.get("angular_y", 0.0)) + t.angular.z = float(body.get("angular_z", 0.0)) + self.pub.publish(t) + + +def main() -> None: # pragma: no cover + rclpy.init() + node = TwistSubscriber() + try: + rclpy.spin(node) + finally: + node.client.loop_stop() + node.client.disconnect() + node.destroy_node() + rclpy.shutdown() + + +if __name__ == "__main__": + main() diff --git a/cloud/cmd/collision-worker/main.go b/cloud/cmd/collision-worker/main.go new file mode 100644 index 0000000..13f4b76 --- /dev/null +++ b/cloud/cmd/collision-worker/main.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/worker" + + "github.com/example/temporal-hack/cloud/internal/collision" +) + +// collision-worker hosts: +// - Temporal worker for the CollisionResponse workflow + SendTwist +// activity +// - MQTT bridge that subscribes to events/+/collision and starts a +// workflow per event +// +// Both share the same MQTT client so the workflow's outgoing twist +// publishes and the bridge's inbound collision subscriptions sit on +// one connection. +func main() { + logger := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) + + temporalAddr := envOr("TEMPORAL_ADDR", "localhost:7233") + brokerURL := envOr("BROKER_URL", "tcp://localhost:1883") + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + mqttCli, err := connectMQTT(brokerURL) + if err != nil { + logger.Error("mqtt", "err", err) + os.Exit(1) + } + defer mqttCli.Disconnect(500) + + tcli, err := client.Dial(client.Options{HostPort: temporalAddr}) + if err != nil { + logger.Error("temporal", "err", err) + os.Exit(1) + } + defer tcli.Close() + + acts := &collision.Activities{MQTT: mqttCli} + w := worker.New(tcli, collision.TaskQueue, worker.Options{}) + w.RegisterWorkflow(collision.CollisionResponse) + w.RegisterActivity(acts.SendTwist) + + bridge := &collision.MQTTBridge{MQTT: mqttCli, Temporal: tcli, Logger: logger} + go func() { + if err := bridge.Start(ctx); err != nil && ctx.Err() == nil { + logger.Error("mqtt bridge stopped", "err", err) + } + }() + + logger.Info("collision-worker starting", + "task_queue", collision.TaskQueue, "broker", brokerURL, "temporal", temporalAddr) + if err := w.Run(worker.InterruptCh()); err != nil { + logger.Error("worker exit", "err", err) + os.Exit(1) + } +} + +func connectMQTT(url string) (mqtt.Client, error) { + opts := mqtt.NewClientOptions(). + AddBroker(url). + SetClientID("collision-worker"). + SetCleanSession(false). + SetAutoReconnect(true). + SetMaxReconnectInterval(60 * time.Second). + SetOrderMatters(false) + cli := mqtt.NewClient(opts) + tok := cli.Connect() + tok.WaitTimeout(10 * time.Second) + return cli, tok.Error() +} + +func envOr(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} diff --git a/cloud/internal/collision/activities.go b/cloud/internal/collision/activities.go new file mode 100644 index 0000000..f537951 --- /dev/null +++ b/cloud/internal/collision/activities.go @@ -0,0 +1,62 @@ +package collision + +import ( + "context" + "encoding/json" + "fmt" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +// Activities owns the MQTT client used to publish twist commands. +type Activities struct { + MQTT mqtt.Client +} + +// SendTwist republishes the configured twist at 10 Hz for the +// requested duration so the gz DiffDrive command timeout (~0.5 s) +// doesn't stop the rover mid-phase. Sends one final 0,0 stop frame +// after the duration before returning, so the rover is at rest if +// the workflow doesn't immediately follow up. +func (a *Activities) SendTwist(ctx context.Context, args SendTwistArgs) error { + topic := fmt.Sprintf("cmd/%s/twist", args.RobotID) + body, err := json.Marshal(map[string]float64{ + "linear_x": args.LinearX, + "angular_z": args.AngularZ, + }) + if err != nil { + return err + } + stop, _ := json.Marshal(map[string]float64{"linear_x": 0, "angular_z": 0}) + + end := time.Now().Add(args.Duration) + tick := time.NewTicker(100 * time.Millisecond) + defer tick.Stop() + for { + // paho handles its own reconnects; don't second-guess via + // IsConnectionOpen(), which has tight semantics around + // reconnect windows. The token's WaitTimeout + Error are + // the authoritative result for a single publish. + tok := a.MQTT.Publish(topic, 1, false, body) + if !tok.WaitTimeout(2 * time.Second) { + return fmt.Errorf("mqtt publish timeout for %s", topic) + } + if err := tok.Error(); err != nil { + return err + } + if time.Now().After(end) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + } + } + + // Final explicit stop frame. + tok := a.MQTT.Publish(topic, 1, false, stop) + tok.WaitTimeout(2 * time.Second) + return tok.Error() +} diff --git a/cloud/internal/collision/mqttbridge.go b/cloud/internal/collision/mqttbridge.go new file mode 100644 index 0000000..da2117d --- /dev/null +++ b/cloud/internal/collision/mqttbridge.go @@ -0,0 +1,76 @@ +package collision + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "strings" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" + "go.temporal.io/sdk/client" +) + +// MQTTBridge subscribes to events/+/collision and starts a +// CollisionResponse workflow per inbound event. Workflow IDs are +// disambiguated by an event timestamp so multiple collisions from +// the same robot don't conflict, but the dedupe window in the robot +// publisher (2 s) plus the workflow's deterministic ID together +// keep us from running concurrent responses for the same incident. +type MQTTBridge struct { + MQTT mqtt.Client + Temporal client.Client + Logger *slog.Logger +} + +func (b *MQTTBridge) Start(ctx context.Context) error { + tok := b.MQTT.Subscribe("events/+/collision", 1, b.onMessage(ctx)) + if !tok.WaitTimeout(10*time.Second) || tok.Error() != nil { + return fmt.Errorf("subscribe events/+/collision: %w", tok.Error()) + } + b.Logger.Info("collision mqtt bridge subscribed", "topic", "events/+/collision") + <-ctx.Done() + return nil +} + +func (b *MQTTBridge) onMessage(ctx context.Context) mqtt.MessageHandler { + return func(_ mqtt.Client, msg mqtt.Message) { + topic := msg.Topic() + parts := strings.Split(topic, "/") + if len(parts) != 3 || parts[0] != "events" || parts[2] != "collision" { + b.Logger.Warn("unexpected collision topic", "topic", topic) + return + } + robotID := parts[1] + var ev struct { + RobotID string `json:"robot_id"` + Partner string `json:"partner"` + At float64 `json:"at"` + Count int `json:"count"` + } + _ = json.Unmarshal(msg.Payload(), &ev) + if ev.RobotID == "" { + ev.RobotID = robotID + } + + wfID := fmt.Sprintf("collision-%s-%d", ev.RobotID, time.Now().UnixNano()) + opts := client.StartWorkflowOptions{ + ID: wfID, + TaskQueue: TaskQueue, + WorkflowExecutionTimeout: 2 * time.Minute, + } + _, err := b.Temporal.ExecuteWorkflow(ctx, opts, "CollisionResponse", Input{ + RobotID: ev.RobotID, + Partner: ev.Partner, + At: int64(ev.At), + }) + if err != nil { + b.Logger.Error("start collision workflow", "err", err, "robot_id", ev.RobotID) + return + } + b.Logger.Info("collision workflow started", + "workflow_id", wfID, "robot_id", ev.RobotID, "partner", ev.Partner, "count", ev.Count) + msg.Ack() + } +} diff --git a/cloud/internal/collision/types.go b/cloud/internal/collision/types.go new file mode 100644 index 0000000..e579d5c --- /dev/null +++ b/cloud/internal/collision/types.go @@ -0,0 +1,31 @@ +package collision + +import "time" + +// TaskQueue is shared by the CollisionResponse workflow and its +// SendTwist activity. +const TaskQueue = "collision" + +// Input is the workflow input — produced by the MQTT bridge from a +// robot's collision event. +type Input struct { + RobotID string `json:"robot_id"` + Partner string `json:"partner,omitempty"` + At int64 `json:"at,omitempty"` +} + +// SendTwistArgs is the activity input. The activity republishes the +// twist on MQTT cmd/{robot_id}/twist at 10 Hz for `Duration`, then +// emits one final 0,0 stop and returns. +type SendTwistArgs struct { + RobotID string `json:"robot_id"` + LinearX float64 `json:"linear_x"` + AngularZ float64 `json:"angular_z"` + Duration time.Duration `json:"duration"` +} + +// Activity name constants used by the workflow (avoids capturing a +// pointer-receiver method value in the workflow definition). +const ( + ActSendTwist = "SendTwist" +) diff --git a/cloud/internal/collision/workflow.go b/cloud/internal/collision/workflow.go new file mode 100644 index 0000000..0f5a5af --- /dev/null +++ b/cloud/internal/collision/workflow.go @@ -0,0 +1,58 @@ +package collision + +import ( + "time" + + "go.temporal.io/sdk/temporal" + "go.temporal.io/sdk/workflow" +) + +// CollisionResponse is the demo workflow: when the robot reports a +// collision via MQTT, this runs in the cloud and drives the robot +// out of the obstacle: +// +// 1. back up at 0.3 m/s for 3 s +// 2. stop briefly (settle) +// 3. turn right (clockwise) at 0.5 rad/s for ~3.14 s (=> 90°) +// 4. stop briefly +// 5. drive forward at 0.4 m/s for 5 s +// 6. stop +// +// Each step is one SendTwist activity. The activity republishes the +// twist at 10 Hz for the duration so the gz diff-drive command +// timeout (~0.5 s) doesn't stall the rover, then emits a stop frame. +// +// Workflow ID convention: `collision-{robotID}-{ts}` so multiple +// collision events from the same robot don't collide. +func CollisionResponse(ctx workflow.Context, in Input) error { + logger := workflow.GetLogger(ctx) + logger.Info("collision response starting", "robot_id", in.RobotID, "partner", in.Partner) + + actx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: 30 * time.Second, + RetryPolicy: &temporal.RetryPolicy{ + InitialInterval: 500 * time.Millisecond, + BackoffCoefficient: 2, + MaximumAttempts: 3, + }, + }) + + steps := []SendTwistArgs{ + {RobotID: in.RobotID, LinearX: -0.30, AngularZ: 0.0, Duration: 3 * time.Second}, // back up + {RobotID: in.RobotID, LinearX: 0.0, AngularZ: 0.0, Duration: 500 * time.Millisecond}, // settle + {RobotID: in.RobotID, LinearX: 0.0, AngularZ: -0.50, Duration: 3140 * time.Millisecond}, // turn right ~90° + {RobotID: in.RobotID, LinearX: 0.0, AngularZ: 0.0, Duration: 500 * time.Millisecond}, // settle + {RobotID: in.RobotID, LinearX: 0.40, AngularZ: 0.0, Duration: 5 * time.Second}, // forward + {RobotID: in.RobotID, LinearX: 0.0, AngularZ: 0.0, Duration: 200 * time.Millisecond}, // final stop + } + + for i, s := range steps { + logger.Info("collision phase", "i", i, "linear_x", s.LinearX, "angular_z", s.AngularZ, "duration", s.Duration) + if err := workflow.ExecuteActivity(actx, ActSendTwist, s).Get(actx, nil); err != nil { + return err + } + } + + logger.Info("collision response complete", "robot_id", in.RobotID) + return nil +} diff --git a/cloud/internal/ota/mqttbridge.go b/cloud/internal/ota/mqttbridge.go index 1f427fc..2b36689 100644 --- a/cloud/internal/ota/mqttbridge.go +++ b/cloud/internal/ota/mqttbridge.go @@ -20,9 +20,9 @@ import ( // SignalWorkflow on a closed/non-existent execution returns // "workflow execution not found" which we log and drop. type MQTTBridge struct { - MQTT mqtt.Client - Temporal client.Client - Logger *slog.Logger + MQTT mqtt.Client + Temporal client.Client + Logger *slog.Logger } // Start subscribes and runs until the context is canceled. @@ -46,11 +46,11 @@ func (b *MQTTBridge) onMessage(ctx context.Context) mqtt.MessageHandler { robotID := parts[1] var ack struct { - RolloutID string `json:"rollout_id"` - Phase string `json:"phase"` - Detail string `json:"detail"` - ImageDigest string `json:"image_digest"` - PreviousImageDigest string `json:"previous_image_digest"` + RolloutID string `json:"rollout_id"` + Phase string `json:"phase"` + Detail string `json:"detail"` + ImageDigest string `json:"image_digest"` + PreviousImageDigest string `json:"previous_image_digest"` } if err := json.Unmarshal(msg.Payload(), &ack); err != nil { b.Logger.Warn("ack unmarshal", "err", err, "topic", topic) diff --git a/cloud/internal/ota/types.go b/cloud/internal/ota/types.go index 91bbb04..9ca41bf 100644 --- a/cloud/internal/ota/types.go +++ b/cloud/internal/ota/types.go @@ -7,13 +7,13 @@ type RolloutSpec struct { ImageRef string `json:"image_ref"` ImageDigest string `json:"image_digest"` CohortSelector CohortFilter `json:"cohort_selector"` - CanarySize int `json:"canary_size"` // default 1 - BatchPercent int `json:"batch_percent"` // default 25 - FailureBudget int `json:"failure_budget"` // per-batch tolerated failures (count) - SmokeTimeout time.Duration `json:"smoke_timeout"` // default 5m - SmokeCommand string `json:"smoke_command"` // executed in new container - PullTimeout time.Duration `json:"pull_timeout"` // default 15m - SwapTimeout time.Duration `json:"swap_timeout"` // default 2m + CanarySize int `json:"canary_size"` // default 1 + BatchPercent int `json:"batch_percent"` // default 25 + FailureBudget int `json:"failure_budget"` // per-batch tolerated failures (count) + SmokeTimeout time.Duration `json:"smoke_timeout"` // default 5m + SmokeCommand string `json:"smoke_command"` // executed in new container + PullTimeout time.Duration `json:"pull_timeout"` // default 15m + SwapTimeout time.Duration `json:"swap_timeout"` // default 2m Force bool `json:"force"` } diff --git a/cloud/internal/ota/workflow_singlerobot.go b/cloud/internal/ota/workflow_singlerobot.go index e4024b7..dfd04bb 100644 --- a/cloud/internal/ota/workflow_singlerobot.go +++ b/cloud/internal/ota/workflow_singlerobot.go @@ -18,10 +18,10 @@ type OTASingleRobotInput struct { // AckSignal is the structured signal payload posted by mqttbridge // when an OTAAck arrives for a workflow's robot. type AckSignal struct { - Phase string `json:"phase"` // values map to ota.proto Phase enum names - Detail string `json:"detail"` - ImageDigest string `json:"image_digest"` - PreviousImageDigest string `json:"previous_image_digest"` + Phase string `json:"phase"` // values map to ota.proto Phase enum names + Detail string `json:"detail"` + ImageDigest string `json:"image_digest"` + PreviousImageDigest string `json:"previous_image_digest"` } // SignalAck is the signal channel name used for ack delivery. diff --git a/docker/robot/Dockerfile b/docker/robot/Dockerfile index 2a0e73d..474004d 100644 --- a/docker/robot/Dockerfile +++ b/docker/robot/Dockerfile @@ -21,9 +21,12 @@ ENV PYTHONUNBUFFERED=1 RUN apt-get update && apt-get install -y --no-install-recommends \ ros-humble-rmw-cyclonedds-cpp \ ros-humble-sensor-msgs \ + ros-humble-ros-gz-interfaces \ + ros-humble-geometry-msgs \ python3-pip \ python3-grpcio \ python3-grpc-tools \ + python3-paho-mqtt \ && rm -rf /var/lib/apt/lists/* # Bridge package (rclpy → gRPC) and sim_battery publisher. diff --git a/docker/robot/entrypoint.sh b/docker/robot/entrypoint.sh index da66dce..c4a1d0a 100644 --- a/docker/robot/entrypoint.sh +++ b/docker/robot/entrypoint.sh @@ -16,12 +16,24 @@ shutdown() { } trap shutdown SIGINT SIGTERM -# sim_battery is a stand-in /battery_state publisher (Gazebo doesn't +# sim_battery: stand-in /battery_state publisher (Gazebo doesn't # emit one for our model). Real robots publish their own; this is # only present in the dev sim. echo "[robot] starting sim_battery" python3 -m bridge_node.sim_battery & PIDS+=($!) +# collision_publisher: ROS /contacts -> MQTT events/{robot_id}/collision. +# The cloud's CollisionResponse workflow listens on the MQTT side. +echo "[robot] starting collision_publisher" +python3 -m bridge_node.collision_publisher & +PIDS+=($!) + +# twist_subscriber: MQTT cmd/{robot_id}/twist -> ROS /cmd_vel. +# The cloud workflow publishes Twist messages here to drive the rover. +echo "[robot] starting twist_subscriber" +python3 -m bridge_node.twist_subscriber & +PIDS+=($!) + echo "[robot] starting bridge gRPC on $LISTEN" exec python3 -m bridge_node.server --listen "$LISTEN" From 3e846e058f359a9a5ac6e1886228c8b293bfb613 Mon Sep 17 00:00:00 2001 From: Ben Kearns <35475+bkearns@users.noreply.github.com> Date: Tue, 5 May 2026 15:24:08 -0700 Subject: [PATCH 3/6] feat(make): controlplane-up + ota-circle / ota-figure-eight targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Demo glue. Rounds out the host-side runner so the OTA story is one command per scenario, mirroring 'make collide' for the collision demo. Targets: controlplane-up / -down / -status — start/stop the controlplane HTTP API binary against the lab Temporal + TSDB. PID file in .run/. ota-circle — build sim/controllers/drive-circle, push to the lab registry (localhost:14050), and POST a rollout to /v1/ota/rollouts targeting sim-robot-01. ota-figure-eight — same for drive-figure-eight. ota-status — GET /v1/ota/rollouts (jq if present). Override OTA_REGISTRY / OTA_ROBOT_ID / OTA_CP_HOST to retarget. Verified: 'make controlplane-up' brings the API up; healthz 200; ota-status returns null pre-rollout. --- Makefile | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/Makefile b/Makefile index 5caefb1..8ea96d1 100644 --- a/Makefile +++ b/Makefile @@ -158,6 +158,86 @@ collide: ## Publish a fake collision event for sim-robot-01 (triggers Temporal w "import paho.mqtt.publish as p, time, json; p.single('events/sim-robot-01/collision', json.dumps({'robot_id':'sim-robot-01','at':time.time(),'count':1,'partner':'manual-trigger'}), hostname='mqtt', port=1883, qos=1)" @echo "published events/sim-robot-01/collision; check Temporal UI for collision-* workflow" +# ============================================================================= +# Control plane (HTTP API for OTA rollouts) — host-side binary, not in +# compose. Required to start a rollout via /v1/ota/rollouts. Same +# pattern as workers-up / workers-down. +# ============================================================================= + +CP_LISTEN_ADDR ?= :8081 + +.PHONY: controlplane-up +controlplane-up: build-cloud ## Start the control plane HTTP API in the background + @mkdir -p .run + @LISTEN_ADDR=$(CP_LISTEN_ADDR) \ + TEMPORAL_ADDR=$(WORKER_TEMPORAL_ADDR) \ + TSDB_DSN="$(WORKER_TSDB_DSN)" \ + nohup ./bin/controlplane > .run/controlplane.log 2>&1 & echo $$! > .run/controlplane.pid + @sleep 1 + @echo "controlplane PID $$(cat .run/controlplane.pid 2>/dev/null) log .run/controlplane.log" + @echo " POST http://localhost$(CP_LISTEN_ADDR)/v1/ota/rollouts to start an OTA" + +.PHONY: controlplane-down +controlplane-down: ## Stop the control plane API + @[ -f .run/controlplane.pid ] && kill "$$(cat .run/controlplane.pid)" 2>/dev/null && rm -f .run/controlplane.pid && echo "stopped controlplane" || true + +.PHONY: controlplane-status +controlplane-status: ## Show control plane status + @pid="$$(cat .run/controlplane.pid 2>/dev/null || echo '')"; \ + if [ -n "$$pid" ] && kill -0 "$$pid" 2>/dev/null; then \ + echo "controlplane: running (pid $$pid) at http://localhost$(CP_LISTEN_ADDR)"; \ + else echo "controlplane: not running"; fi + +# ============================================================================= +# OTA demo helpers — build the controller image, push it to the lab +# registry, fire a rollout. One command per controller. +# ============================================================================= + +OTA_REGISTRY ?= localhost:14050 +OTA_ROBOT_ID ?= sim-robot-01 +OTA_CP_HOST ?= http://localhost:8081 + +# Internal: build + push a single controller. Args: $(1)=name (matches sim/controllers/), +# $(2)=tag suffix. +define _ota_build_push + @echo "[ota] building sim/controllers/$(1) → $(OTA_REGISTRY)/robot-app:$(2)" + $(CONTAINER_ENGINE) build \ + -t $(OTA_REGISTRY)/robot-app:$(2) \ + -f sim/controllers/$(1)/Dockerfile \ + sim/controllers/$(1) + $(CONTAINER_ENGINE) push --tls-verify=false $(OTA_REGISTRY)/robot-app:$(2) +endef + +# Internal: POST a rollout for the given image tag. +define _ota_rollout + @echo "[ota] starting rollout for $(OTA_REGISTRY)/robot-app:$(1) on $(OTA_ROBOT_ID)" + @curl -sS -X POST $(OTA_CP_HOST)/v1/ota/rollouts \ + -H "content-type: application/json" \ + -d '{ \ + "image_ref": "$(OTA_REGISTRY)/robot-app:$(1)", \ + "smoke_command": "true", \ + "smoke_timeout_sec": 10, \ + "cohort_selector": {"robot_ids": ["$(OTA_ROBOT_ID)"]} \ + }' && echo +endef + +.PHONY: ota-circle +ota-circle: ## Build, push, and OTA-roll the drive-circle controller + $(call _ota_build_push,drive-circle,circle-v1) + $(call _ota_rollout,circle-v1) + @echo "watch the rover at http://localhost:14680 — should start driving in a circle" + @echo "rollout status: curl -s $(OTA_CP_HOST)/v1/ota/rollouts | jq" + +.PHONY: ota-figure-eight +ota-figure-eight: ## Build, push, and OTA-roll the drive-figure-eight controller + $(call _ota_build_push,drive-figure-eight,figure-eight-v1) + $(call _ota_rollout,figure-eight-v1) + @echo "watch the rover at http://localhost:14680 — should start tracing a figure 8" + +.PHONY: ota-status +ota-status: ## List recent OTA rollouts (requires controlplane-up) + @curl -sS $(OTA_CP_HOST)/v1/ota/rollouts | (command -v jq >/dev/null && jq || cat) + .PHONY: build-agent build-agent: cd agent && go build -o ../bin/agent ./cmd/agent From ba20592220d47d52b24d82ce66631289a9d9aeb0 Mon Sep 17 00:00:00 2001 From: Ben Kearns <35475+bkearns@users.noreply.github.com> Date: Tue, 5 May 2026 15:38:05 -0700 Subject: [PATCH 4/6] feat(agent): docker/podman engine auto-detect + native macOS run mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User reported the OTA rollout was stuck. Two fixes: 1. agent/internal/ota/docker.go — auto-detect docker vs podman. AGENT_CONTAINER_BIN env override > docker on PATH > podman. Pull(): if podman, append --tls-verify=false so localhost:14050 (HTTP-only lab registry) doesn't get rejected. 2. New native-host agent mode. Rootless podman bind-mounts the daemon socket as a non-stat-able file (idmap remaps owner UID); even userns_mode=keep-id + chmod 666 didn't get container root through. Side-stepping by running the agent as a Go binary on the host: - docker-compose.sim.yml: robot publishes :50051 to host; in-container agent profile-gated under in-container-agent. - Makefile: agent-up / agent-down / agent-status targets. Verified: make agent-up + make ota-circle -> rollout completed; robot-app container Up running localhost:14050/robot-app:circle-v1; rover circles in noVNC. --- Makefile | 32 ++++++++ agent/cmd/agent/main.go | 4 +- agent/internal/ota/docker.go | 79 ++++++++++++++----- .../docker-compose/docker-compose.sim.yml | 33 +++++++- 4 files changed, 125 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index 8ea96d1..f34e3a4 100644 --- a/Makefile +++ b/Makefile @@ -121,6 +121,38 @@ WORKER_TEMPORAL_ADDR ?= localhost:14733 WORKER_BROKER_URL ?= tcp://localhost:14883 WORKER_TSDB_DSN ?= postgres://temporal:temporal@localhost:14432/telemetry?sslmode=disable +# Native agent — runs as a Go binary on the host (macOS). Avoids +# bind-mounting the container-runtime socket into a sibling +# container (rootless podman idmap makes that path painful) and +# uses the host's docker/podman CLI directly when an OTA fires. +AGENT_ROBOT_ID ?= sim-robot-01 +AGENT_BROKER_URL ?= tcp://localhost:14883 +AGENT_BRIDGE_ADDR ?= localhost:50051 +AGENT_BUFFER_PATH ?= .run/agent-buffer.db +AGENT_OTA_RUN_ARGS ?= --network=temporal-hack-lab_default,-e,ROS_DOMAIN_ID=42,-e,RMW_IMPLEMENTATION=rmw_cyclonedds_cpp + +.PHONY: agent-up +agent-up: build-agent ## Start the agent natively on the host (preferred for local dev) + @mkdir -p .run + @ROBOT_ID=$(AGENT_ROBOT_ID) \ + BROKER_URL=$(AGENT_BROKER_URL) \ + BRIDGE_ADDR=$(AGENT_BRIDGE_ADDR) \ + BUFFER_PATH=$(AGENT_BUFFER_PATH) \ + OTA_RUN_ARGS="$(AGENT_OTA_RUN_ARGS)" \ + nohup ./bin/agent > .run/agent.log 2>&1 & echo $$! > .run/agent.pid + @sleep 1 + @echo "agent PID $$(cat .run/agent.pid 2>/dev/null) log .run/agent.log" + +.PHONY: agent-down +agent-down: ## Stop the native agent + @[ -f .run/agent.pid ] && kill "$$(cat .run/agent.pid)" 2>/dev/null && rm -f .run/agent.pid && echo "stopped agent" || true + +.PHONY: agent-status +agent-status: ## Show native agent status + @pid="$$(cat .run/agent.pid 2>/dev/null || echo '')"; \ + if [ -n "$$pid" ] && kill -0 "$$pid" 2>/dev/null; then echo "agent: running (pid $$pid)"; \ + else echo "agent: not running"; fi + .PHONY: workers-up workers-up: build-cloud ## Start ota-worker + collision-worker in the background @mkdir -p .run diff --git a/agent/cmd/agent/main.go b/agent/cmd/agent/main.go index baeac66..380168c 100644 --- a/agent/cmd/agent/main.go +++ b/agent/cmd/agent/main.go @@ -106,10 +106,12 @@ func main() { Logger: logger, }) + dockerCLI := ota.NewDockerCLI(cfg.otaRunArgs) + logger.Info("ota engine resolved", "bin", dockerCLI.Bin()) exec := &ota.Executor{ RobotID: cfg.robotID, MQTT: pub.Client(), - Docker: ota.NewDockerCLI(cfg.otaRunArgs), + Docker: dockerCLI, Logger: logger, } diff --git a/agent/internal/ota/docker.go b/agent/internal/ota/docker.go index 5681159..a8c1a30 100644 --- a/agent/internal/ota/docker.go +++ b/agent/internal/ota/docker.go @@ -4,37 +4,74 @@ import ( "context" "fmt" "io" + "os" "os/exec" "strings" ) -// DockerCLI is a thin wrapper over the local `docker` command. We use -// the CLI rather than the Go SDK to keep the agent binary small and -// avoid pinning a docker SDK version against the customer's daemon -// version. The CLI is a contract that's far more stable than the SDK. +// DockerCLI is a thin wrapper over the local container-engine +// command. We use the CLI rather than a Go SDK to keep the agent +// binary small and avoid pinning a docker/podman SDK version against +// the customer's daemon version. The CLI surface (`pull`, `run`, +// `inspect`, `rm`, `rename`, `exec`) is identical between docker and +// podman, so the same code drives either engine — we just resolve +// which binary is on PATH at startup. // // The container we manage is named "robot-app" by convention. v1 // runs only one application container per robot. +// +// Engine selection (in order): +// 1. AGENT_CONTAINER_BIN env var (explicit override) +// 2. `docker` on PATH +// 3. `podman` on PATH type DockerCLI struct { ContainerName string // default "robot-app" - RunArgs []string // extra args passed at `docker run` (volumes, devices, network, env, etc.) + RunArgs []string // extra args passed at ` run` (volumes, devices, network, env, etc.) + bin string // resolved engine binary: "docker" or "podman" } func NewDockerCLI(runArgs []string) *DockerCLI { - return &DockerCLI{ContainerName: "robot-app", RunArgs: runArgs} + bin := os.Getenv("AGENT_CONTAINER_BIN") + if bin == "" { + if _, err := exec.LookPath("docker"); err == nil { + bin = "docker" + } else if _, err := exec.LookPath("podman"); err == nil { + bin = "podman" + } else { + // Last-ditch default; calls will fail loudly with + // `executable file not found` when the agent first OTAs. + bin = "docker" + } + } + return &DockerCLI{ContainerName: "robot-app", RunArgs: runArgs, bin: bin} } -// Pull invokes `docker pull `. +// Bin returns the resolved engine binary ("docker" or "podman"). +// Useful for logs. +func (d *DockerCLI) Bin() string { return d.bin } + +// Pull invokes ` pull `. +// +// Podman defaults to HTTPS-only registry traffic; override with +// --tls-verify=false so the local lab registry on a plain-HTTP +// localhost:14050 works without registries.conf surgery. Docker +// uses its daemon-side `insecure-registries` config (set +// per-environment) and does not accept the same flag. func (d *DockerCLI) Pull(ctx context.Context, ref string) error { - return run(ctx, "docker", "pull", ref) + args := []string{"pull"} + if d.bin == "podman" { + args = append(args, "--tls-verify=false") + } + args = append(args, ref) + return run(ctx, d.bin, args...) } // CurrentDigest returns the image digest of the running container, or // an empty string if no container is running. func (d *DockerCLI) CurrentDigest(ctx context.Context) (string, error) { - out, err := capture(ctx, "docker", "inspect", "--format", "{{.Image}}", d.ContainerName) + out, err := capture(ctx, d.bin, "inspect", "--format", "{{.Image}}", d.ContainerName) if err != nil { - if strings.Contains(err.Error(), "No such") { + if strings.Contains(err.Error(), "No such") || strings.Contains(err.Error(), "no such") { return "", nil } return "", err @@ -52,20 +89,20 @@ func (d *DockerCLI) Swap(ctx context.Context, ref string) (prevDigest string, er args := []string{"run", "-d", "--name", tmpName} args = append(args, d.RunArgs...) args = append(args, ref) - if err := run(ctx, "docker", args...); err != nil { - return "", fmt.Errorf("docker run new: %w", err) + if err := run(ctx, d.bin, args...); err != nil { + return "", fmt.Errorf("%s run new: %w", d.bin, err) } // Verify it's actually running. - state, _ := capture(ctx, "docker", "inspect", "--format", "{{.State.Status}}", tmpName) + state, _ := capture(ctx, d.bin, "inspect", "--format", "{{.State.Status}}", tmpName) if strings.TrimSpace(state) != "running" { - _ = run(ctx, "docker", "rm", "-f", tmpName) + _ = run(ctx, d.bin, "rm", "-f", tmpName) return prevDigest, fmt.Errorf("new container not running (state=%s)", state) } // Stop+remove old (best effort). - _ = run(ctx, "docker", "rm", "-f", d.ContainerName) - if err := run(ctx, "docker", "rename", tmpName, d.ContainerName); err != nil { - _ = run(ctx, "docker", "rm", "-f", tmpName) - return prevDigest, fmt.Errorf("docker rename: %w", err) + _ = run(ctx, d.bin, "rm", "-f", d.ContainerName) + if err := run(ctx, d.bin, "rename", tmpName, d.ContainerName); err != nil { + _ = run(ctx, d.bin, "rm", "-f", tmpName) + return prevDigest, fmt.Errorf("%s rename: %w", d.bin, err) } return prevDigest, nil } @@ -77,11 +114,11 @@ func (d *DockerCLI) Rollback(ctx context.Context, prevDigest string) error { if prevDigest == "" { return fmt.Errorf("no previous digest to roll back to") } - _ = run(ctx, "docker", "rm", "-f", d.ContainerName) + _ = run(ctx, d.bin, "rm", "-f", d.ContainerName) args := []string{"run", "-d", "--name", d.ContainerName} args = append(args, d.RunArgs...) args = append(args, prevDigest) - return run(ctx, "docker", args...) + return run(ctx, d.bin, args...) } // Exec runs a smoke command inside the named container and returns @@ -90,7 +127,7 @@ func (d *DockerCLI) Exec(ctx context.Context, command string) error { if command == "" { return nil // no smoke check configured; treat as healthy } - return run(ctx, "docker", "exec", d.ContainerName, "sh", "-c", command) + return run(ctx, d.bin, "exec", d.ContainerName, "sh", "-c", command) } // run executes a command, discarding output unless it fails. diff --git a/installer/docker-compose/docker-compose.sim.yml b/installer/docker-compose/docker-compose.sim.yml index 8ebf412..de6675d 100644 --- a/installer/docker-compose/docker-compose.sim.yml +++ b/installer/docker-compose/docker-compose.sim.yml @@ -32,11 +32,36 @@ services: environment: ROS_DOMAIN_ID: 42 RMW_IMPLEMENTATION: rmw_cyclonedds_cpp + # gRPC bridge port published to the host so a NATIVE agent + # binary (running on macOS) can connect at localhost:50051. + # Keep `expose` for in-network containers that still talk to + # robot:50051 by service name. expose: - "50051" + ports: + - "${ROBOT_BRIDGE_PORT:-50051}:50051" + # The agent has TWO supported deployment modes: + # + # 1. NATIVE (default for local dev) — run as a Go binary on the + # host: `make agent-up`. Avoids the rootless-podman bind-mount- + # socket headache and uses the host's docker/podman CLI + # directly. The robot service publishes 50051 to the host so + # the native agent reaches the bridge at localhost:50051. + # + # 2. CONTAINERISED (production target, profile-gated) — run as a + # compose service. Requires a working bind mount of the + # container-runtime socket. Bring up with: + # docker compose --profile in-container-agent up -d + # agent: + profiles: ["in-container-agent"] image: golang:1.22-bookworm + # keep-id maps the container's UIDs onto the host (podman VM) user + # that started the compose. Without it, rootless podman idmap + # blocks the container's root from reading the bind-mounted + # podman.sock — the agent's docker/podman CLI can't perform OTAs. + userns_mode: keep-id depends_on: mqtt: condition: service_healthy @@ -64,9 +89,15 @@ services: - go-cache:/var/cache/go-build - go-mod:/var/cache/go-mod - ${CONTAINER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock + # build-essential: cgo for go-sqlite3. + # docker.io + podman: install both CLIs at startup; the agent's + # OTA executor auto-detects which one is on PATH (docker first, + # then podman). Either binary will route through the mounted + # CONTAINER_SOCK (docker is REST-compatible with podman, and + # podman natively talks to its own socket). command: > bash -c " - apt-get update -qq && apt-get install -y -qq build-essential >/dev/null && + apt-get update -qq && apt-get install -y -qq build-essential docker.io podman >/dev/null && cd /src/agent && CGO_ENABLED=1 go run ./cmd/agent " From d55b64920b820893efdb27f57102dfe287e91d5a Mon Sep 17 00:00:00 2001 From: Ben Kearns <35475+bkearns@users.noreply.github.com> Date: Tue, 5 May 2026 15:42:49 -0700 Subject: [PATCH 5/6] fix(agent): always emit PHASE_ROLLED_BACK from runRollback User reported rollout jobs not finishing. Root cause: the cloud MQTT bridge routes acks like this: PHASE_ROLLED_BACK -> {rollout_id}-robot-{robot_id}-rollback (everything else) -> {rollout_id}-robot-{robot_id} When a rollback command was issued for the very first OTA on a robot (or any case where rollback's prev-digest is empty), the agent's runRollback: - tried Docker.Rollback(ctx, prev='') -> error - published PHASE_FAILED with detail - PHASE_FAILED was routed to OTASingleRobot (already exited), NOT to the -rollback workflow that was waiting for a phase signal - The rollback workflow stayed blocked on its 5-minute timer - The parent OTASingleRobot was waiting on the rollback child's completion -> the parent OTARollout never RecordRolloutEnded - The DB row showed 'pending' for ~5 minutes per failed rollout Fix: agent's runRollback always emits PHASE_ROLLED_BACK, with the failure mode captured in detail when rollback couldn't actually restore. The rollback workflow always terminates within ms of the agent processing the command; the parent unblocks; the rollout status moves to its terminal value (canary_failed / aborted / completed) immediately. Verified: post-fix, ota-circle and ota-figure-eight runs land in ~1 second with status=completed. Stale 'pending' rows from before this fix will resolve naturally as their 5-minute timers fire, or via curl POST /v1/ota/rollouts/{id}/abort. --- agent/internal/ota/executor.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/agent/internal/ota/executor.go b/agent/internal/ota/executor.go index fd0fcef..a9605a0 100644 --- a/agent/internal/ota/executor.go +++ b/agent/internal/ota/executor.go @@ -103,7 +103,15 @@ func (e *Executor) runRollback(ctx context.Context, cmd Command) { prev := e.lastSuccessDigest e.mu.Unlock() if err := e.Docker.Rollback(ctx, prev); err != nil { - e.publishAck(cmd, "PHASE_FAILED", "rollback: "+err.Error()) + // We still emit PHASE_ROLLED_BACK so the cloud-side rollback + // workflow terminates instead of waiting on its 5-minute + // timer; the failure mode is captured in `detail`. Without + // this, an agent that has no previous container to roll back + // to (the very first OTA) would leave the rollout hanging + // in `pending` until the timer fires. The MQTT bridge routes + // PHASE_ROLLED_BACK to the -rollback workflow ID, which is + // where the wait-loop is. + e.publishAck(cmd, "PHASE_ROLLED_BACK", "rollback failed: "+err.Error()) return } e.publishAck(cmd, "PHASE_ROLLED_BACK", "") From 5ae9214f6babff3c01b9f28780819300786d8691 Mon Sep 17 00:00:00 2001 From: Ben Kearns <35475+bkearns@users.noreply.github.com> Date: Tue, 5 May 2026 15:50:44 -0700 Subject: [PATCH 6/6] docs: refresh README, ONBOARDING, CLAUDE, installer/README for current make targets The make-target inventory has grown a lot since the last doc pass: agent-up/-down/-status, workers-up/-down/-status, controlplane-up/- down/-status, ota-circle, ota-figure-eight, ota-status, collide, sim-drive-{fwd,back,left,right,stop}, sim-gui, sim-up-headless, container-info, plus the host-vs-container split (the agent now runs as a native macOS Go binary by default). README.md - New service-shape diagram showing gazebo + robot + agent + workers + controlplane + lab cluster. - Lab quickstart is now four make targets in sequence (sim-up, agent-up, workers-up, controlplane-up). - Drive demo, OTA demo, Collision demo each get a one-command recipe. - Full make-target reference grouped by purpose. - Default lab ports table includes the new robot:50051, controlplane :8081, gazebo :14680/:14900. ONBOARDING.md - Section 7 (run end-to-end) replaced with the new make-target flow + .run/ PID file convention. - Section 9 covers OTA + Collision demos as one command each. - Troubleshooting table refreshed with the issues actually hit on this machine: docker-not-on-PATH, podman HTTPS pull rejection, rollouts stuck pending, robot port not published. CLAUDE.md - Build/test/run section rewritten to enumerate the host-side processes and demo helpers. installer/README.md - Stale 5xxx-port table replaced with the current 14xxx lab + 2xxxx CI matrix, plus a sim-only ports section (14680, 14900, 50051) and a host-side processes section (8081). --- CLAUDE.md | 61 ++++++---- ONBOARDING.md | 92 ++++++++++----- README.md | 266 ++++++++++++++++++++++++++++++-------------- installer/README.md | 25 +++-- 4 files changed, 300 insertions(+), 144 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a192411..34397d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -113,33 +113,46 @@ The Makefile is the canonical interface. Discover targets with `make` or `make help`. ```bash -# Building +# Build / lint / test make tidy # go mod tidy across both modules -make build # builds bin/{controlplane,telemetry-ingest,ota-worker,agent} - -# Linting / testing +make build # bin/{controlplane,telemetry-ingest,ota-worker,collision-worker,agent} make lint # go vet + (optional) staticcheck on both modules make test # go test -race -count=1 ./... - -# Lab stack (auto-detects docker vs podman) -make container-info # confirm the engine + compose command -make lab-up # validation cluster on 14xxx ports -make lab-status # probe ports -make lab-down # stop, keep state -make lab-reset # stop + wipe state - -# CI/smoke cluster — alternate ports (2xxxx); coexists with lab -make ci-up # what the pre-push hook + GH Actions run -make ci-status -make ci-down # tears down + wipes state - -# Sim (Gazebo + TurtleBot3 + bridge + agent in containers) -make sim-up -make sim-logs -make sim-down - -# Protobuf regeneration -make proto # requires protoc + protoc-gen-go + protoc-gen-go-grpc +make proto # regen Go + Python protobuf bindings via containerized protoc + +# Lab cluster (compose). Auto-detects docker vs podman. +make container-info # which engine + compose command +make lab-up / lab-down / lab-status / lab-reset + +# Sim (gazebo + robot + lab cluster) — preferred starting point. +make sim-up # gazebo container + robot container + lab cluster +make sim-up-headless # no GUI; gz server only +make sim-gui # open the Gazebo browser GUI in default browser +make sim-down / sim-logs + +# Host-side processes (Go binaries, PID files in .run/). +# These do NOT run in compose; they live on the macOS host so they +# can talk to the host's docker/podman CLI without bind-mount-socket +# rootless gymnastics. +make agent-up / agent-down / agent-status # the rover's agent +make workers-up / workers-down / workers-status # ota-worker + collision-worker +make controlplane-up / controlplane-down / controlplane-status # OTA HTTP API on :8081 + +# Drive helpers — direct gz-topic publish from inside the gazebo +# container; no ROS install required on the host. +make sim-drive-fwd LX=0.5 # default 0.5 m/s; override LX= +make sim-drive-back / sim-drive-left / sim-drive-right / sim-drive-stop + +# Demos — one command per scenario. +make ota-circle # build + push + roll out drive-circle +make ota-figure-eight # build + push + roll out drive-figure-eight +make ota-status # GET /v1/ota/rollouts (jq if present) +make collide # publish a fake collision event; + # triggers the CollisionResponse Temporal workflow + +# CI/smoke cluster — same services, alternate (2xxxx) ports so it +# coexists with `lab-up`. What the pre-push hook + GH Actions run. +make ci-up / ci-down / ci-status ``` End-to-end OTA walkthrough lives in `ONBOARDING.md` Section 9. diff --git a/ONBOARDING.md b/ONBOARDING.md index acbb3ce..b1e0880 100644 --- a/ONBOARDING.md +++ b/ONBOARDING.md @@ -295,31 +295,33 @@ same host. ## Section 7 — Run the platform end-to-end -In four terminals: +The host-side processes are managed by `make`. Four targets bring up +the whole stack; each writes its PID to `.run/` and tails to a +matching `.run/.log`. ```bash -# Terminal 1 — lab stack already up (Section 6) -make lab-up - -# Terminal 2 — telemetry ingester -TSDB_DSN="postgres://temporal:temporal@localhost:5432/telemetry?sslmode=disable" \ - ./bin/telemetry-ingest +# Terminal 1 — lab cluster + sim (one make target) +make sim-up # gazebo + robot + Postgres + Temporal + MQTT + registry -# Terminal 3 — control plane API -./bin/controlplane +# Terminal 2 — three host-side runners (no foreground; check .run/*.log) +make agent-up # native agent → talks to localhost:14883 / :50051 +make workers-up # ota-worker + collision-worker +make controlplane-up # OTA HTTP API on :8081 +``` -# Terminal 4 — OTA worker (Temporal worker + MQTT bridge) -./bin/ota-worker +Verify: -# Then start an agent (a fifth terminal, or detached) -ROBOT_ID=lab-robot-01 ./bin/agent +```bash +make agent-status workers-status controlplane-status +curl -s http://localhost:8081/healthz # 200 +curl -s http://localhost:8081/v1/robots | jq # heartbeats from sim-robot-01 +make sim-gui # Gazebo GUI in browser ``` -Verify operator reads: +Tear down (reverse order): ```bash -curl -s http://localhost:8081/healthz -curl -s http://localhost:8081/v1/robots | jq +make controlplane-down && make workers-down && make agent-down && make sim-down ``` --- @@ -343,20 +345,56 @@ The Gazebo GUI is exposed over noVNC, so it works in any browser --- -## Section 9 — Trigger a test OTA +## Section 9 — Demos: OTA + Collision + +Both demos assume Section 7's four targets are up. + +### OTA — swap a robot-app live (Temporal-orchestrated) + +One command per scenario. Builds the controller image, pushes it to +the lab registry on `:14050`, and POSTs a rollout to the control +plane: + +```bash +make ota-circle # rover starts driving in a circle +make ota-figure-eight # swap to a figure-8 controller +make ota-status # GET /v1/ota/rollouts (jq if installed) +``` + +Watch the workflow at `http://localhost:14080/namespaces/default/workflows` +— a `rollout-…` ID appears, completes in 1–2 seconds, and the +`robot-app` container under `podman ps` flips to the new image. + +### Collision — Temporal drives the rover out of an obstacle + +The moon world has a 0.9 m boulder at `x = 8`. A contact sensor on +the rover deck publishes Gazebo Contact messages → ros_gz_bridge → +ROS `/contacts` → `collision_publisher` (in the robot container) → +MQTT `events/{robot_id}/collision` → `collision-worker` → +`CollisionResponse` Temporal workflow → MQTT `cmd/{robot_id}/twist` +→ `twist_subscriber` → ROS `/cmd_vel` → gz `DiffDrive`. + +```bash +make collide # publish a fake collision event +# OR drive the rover into the boulder for real: +make sim-drive-fwd LX=1.0 # call repeatedly until impact +``` + +`CollisionResponse` runs back-up → 90° turn-right → forward → +stop. Visible in the Gazebo GUI and at `http://localhost:14080`. + +### (Original raw-API form, kept for reference) -With the lab stack and worker up: +If you want to see the underlying contract: ```bash -# Build and push the lab dummy robot image (Alpine; stays running for swap checks) -docker build -t localhost:5001/robot-app:v1 -f docker/dummy-robot/Dockerfile docker/dummy-robot -docker push localhost:5001/robot-app:v1 +podman build -t localhost:14050/robot-app:v1 -f docker/dummy-robot/Dockerfile docker/dummy-robot +podman push --tls-verify=false localhost:14050/robot-app:v1 -# Start a rollout curl -X POST http://localhost:8081/v1/ota/rollouts \ -H "content-type: application/json" \ -d '{ - "image_ref": "localhost:5001/robot-app:v1", + "image_ref": "localhost:14050/robot-app:v1", "smoke_command": "true", "cohort_selector": {"robot_ids": ["lab-robot-01"]} }' @@ -389,11 +427,13 @@ If all three are green, you're set up. Welcome. | Symptom | Likely cause | Fix | |----------------------------------------|-------------------------------------|------------------------------------------------------------| | `make lab-up` errors `no such image` | Pull failed or rate-limited | `docker login` / wait, retry | -| Postgres exits code 3 at lab-up | Wrong image (stock instead of TSDB) | We pin `timescale/timescaledb-ha`; rebase | +| Postgres exits code 3 at lab-up | Wrong image (stock instead of TSDB) | Pinned to `timescale/timescaledb-ha`; rebase if drifted | | EMQX unhealthy | Port 1883 already in use | `lsof -i :1883`; stop the other broker | -| Agent crashes at startup | Buffer dir not writable | `chmod` the path or override `BUFFER_PATH` | | Bridge node import error | `rclpy` not on `PYTHONPATH` | `source /opt/ros/humble/setup.bash` first | -| OTA stuck at PHASE_PULLED | Robot can't reach registry | Check robot's network to the registry hostname/port | +| OTA "docker not on PATH" | Native agent didn't pick up an engine | `make agent-status`; ensure `docker` or `podman` is on host PATH | +| OTA "http: server gave HTTP response to HTTPS client" | Podman pull tries HTTPS first | Agent appends `--tls-verify=false` for podman; if you use docker, configure `insecure-registries` | +| Rollout stuck `pending` for ~5 min | Pre-fix bug: agent emitted PHASE_FAILED on rollback failure | Fixed; `git pull` and rebuild agent. Old rows clear at 5-min rollback timer | +| Agent ports 50051 unreachable | `robot` service not publishing the port | `make sim-down && make sim-up` to pick up the latest compose | | `pre-commit` hangs in installer-smoke | Slow image pull on first run | Run once manually: `bash .git-hooks/installer-smoke.sh` | --- diff --git a/README.md b/README.md index d9cc71f..42f56ef 100644 --- a/README.md +++ b/README.md @@ -1,119 +1,213 @@ # temporal-hack -Robotics fleet management platform — Telemetry + OTA MVP, on-prem at -customer DC, ROS 2 + Temporal + MQTT. - -This is the v1 codebase. The architecture, decision record, and -project plan live in [`specs/`](specs/). Read those first. +Robotics fleet management platform — Telemetry + OTA MVP. ROS 2 + +Gazebo + Temporal + MQTT. The architecture, decision record, and +project plan live in [`specs/`](specs/) — read those first. ## Layout ``` -cloud/ Go control plane (API + telemetry-ingest) -agent/ Go robot agent (MQTT publisher + local SQLite buffer) -bridge/ Python ROS 2 bridge node (DDS → gRPC for the agent) -proto/ protobuf contracts (telemetry + agent↔bridge) -docker/ Dockerfiles (sim image, dummy-robot lab OTA placeholder) -installer/ docker-compose (lab) and helm (prod stub) +cloud/ Go control plane (HTTP API + telemetry ingester + + ota-worker + collision-worker) +agent/ Go robot agent (MQTT publisher, SQLite buffer, + OTA executor) +bridge/ Python ROS 2 bridge node + sim_battery + collision / + twist MQTT helpers +proto/ protobuf contracts (telemetry, agent ↔ bridge, OTA) +docker/ + gazebo/ simulator container — gz sim + ros_gz_bridge + GUI + robot/ always-on ROS infrastructure container + dummy-robot/ minimal OTA placeholder image (lab smoke) +sim/ + controllers/ OTA-swappable robot-app images + (drive-circle, drive-figure-eight) +installer/ docker-compose (lab) + helm (prod stub) deploy/ service config baked into the installer specs/ blueprint artifacts (decisions, threats, plan, ADRs) ops/ runbooks ``` -## Quickstart (lab) +## Service shape -Requires: Go 1.22+, Docker with compose, Python 3.10+ (for the bridge). +``` + ┌────────────────── browser ──────────────────┐ + │ http://localhost:14680 Gazebo GUI (noVNC) │ + └──────────────────┬──────────────────────────┘ + │ + ┌───────────────────────▼───────────────────────┐ + │ gazebo container │ + │ • ign gazebo + ros_gz_bridge │ + │ • Xvfb + x11vnc + noVNC │ + └────┬─────────────────────┬────────────────────┘ + │ ROS DDS (domain 42) │ + ┌─────────▼──────────┐ ┌───────▼────────────────────┐ + │ robot container │ │ robot-app container │ + │ • bridge_node │ │ (drive-circle | -fig-eight) │ + │ • sim_battery │ │ — OTA-swappable │ + │ • collision_pub │ └────────────────────────────┘ + │ • twist_subscriber │ + └─────────┬──────────┘ + │ gRPC (TCP) + ┌─────────▼──────────────────────────┐ + │ agent (Go, native macOS binary) │ + │ • MQTT pub/sub on lab broker │ + │ • OTA executor (docker/podman CLI) │ + └─────────┬──────────────────────────┘ + │ + ┌─────────▼──────────┐ ┌───────────────────────┐ + │ MQTT (lab :14883) │◀──▶│ ota-worker │ + │ │ │ collision-worker │ + │ │ │ Temporal :14733 │ + └────────────────────┘ └───────────────────────┘ +``` -```bash -# 1) Bring up the lab stack: Postgres + Temporal + EMQX + registry -make lab-up -make lab-status +## Lab quickstart + +Requires Go 1.22+, Python 3.10+, and either Docker or Podman with +compose. The Makefile auto-detects the container engine. -# 2) Build the Go binaries -make build +```bash +make sim-up # lab cluster + gazebo + robot (~5 min first build) +make agent-up # native agent on macOS host +make workers-up # ota-worker + collision-worker +make controlplane-up # OTA HTTP API on :8081 +make sim-gui # open the Gazebo browser GUI +``` -# 3) Run the telemetry ingester (separate terminal) -TSDB_DSN="postgres://temporal:temporal@localhost:5432/telemetry?sslmode=disable" \ - ./bin/telemetry-ingest +That's the whole baseline. Tear down: -# 4) Run an agent (separate terminal) -ROBOT_ID=lab-robot-01 ./bin/agent +```bash +make controlplane-down && make workers-down && make agent-down && make sim-down +``` -# 5) Run the control plane API (separate terminal) -./bin/controlplane +## Drive demo (no Temporal in the loop) -# 6) Query telemetry -curl -s http://localhost:8081/v1/robots -curl -s "http://localhost:8081/v1/robots/lab-robot-01/telemetry?stream=battery&limit=20" +```bash +make sim-drive-fwd # 0.5 m/s for ~0.5s (DiffDrive holds last cmd) +make sim-drive-fwd LX=2 # faster +make sim-drive-left +make sim-drive-stop ``` -The bridge node requires a working ROS 2 install. For S1/S2 you can -exercise the data path without it — the agent's ingest loop falls -back to a stub source when the bridge is unreachable. +These publish straight to the in-container Ignition topic via +`ign topic` — no ROS install needed on the host. -## Sprint status +## OTA demo (Temporal swaps a robot-app live) -| Sprint | Theme | Status | -|--------|-------|--------| -| S0 | Foundations + installer | scaffolding landed; needs hands-on lab bring-up verification | -| S1 | Telemetry plumbing | agent + ingester wired through MQTT; sim container exercises bridge end-to-end | -| S2 | Telemetry MVP | TSDB integration + read API present; durability test pending | -| S3–S4 | OTA workflow + swap + rollback | Temporal workflows + agent executor + MQTT command bridge landed | +```bash +make ota-circle # build sim/controllers/drive-circle, push, + # POST /v1/ota/rollouts → rover starts circling +make ota-figure-eight # swap to figure-8 controller +make ota-status # GET /v1/ota/rollouts (jq if installed) +``` -See [`specs/project-plan.md`](specs/project-plan.md) for the full plan. +What you'll see: a `rollout-…` workflow appears at +`http://localhost:14080/namespaces/default/workflows`, completes in +1–2 seconds, and the `robot-app` container under `podman ps` flips to +the new image. The rover's behaviour changes immediately. -## Sim quickstart +## Collision demo (Temporal drives the rover out of an obstacle) -A Gazebo sim container with TurtleBot3 + the bridge + a synthetic -battery publisher is included as a sibling stack. The agent runs as -its own container, consumes the bridge's gRPC stream, and reports to -the lab MQTT broker. +The moon world spawns the rover with a 0.9 m boulder at `x = 8` — +drive the rover into it (or fake the event) and a Temporal +`CollisionResponse` workflow runs back-up → 90° turn-right → forward. ```bash -make sim-up # lab + Gazebo (GUI on :14680) + agent -make sim-gui # open the Gazebo GUI in your browser -make sim-up-headless # gzserver only — no GUI -make sim-logs -make sim-down +make collide # publish a fake collision event; + # workflow starts immediately +# OR drive into the boulder for real: +make sim-drive-fwd LX=1.0 # send the cmd a few times until impact ``` -The Gazebo GUI is served via noVNC; just hit -`http://localhost:14680/vnc.html?autoconnect=1&resize=scale` in any browser. No -X11, no XQuartz, no platform-specific setup. Raw VNC clients can use -`localhost:14900`. +Watch the `collision-…` workflow at `http://localhost:14080`. -The image is large (~3-4 GB) because Gazebo + ROS 2 desktop pulls in -a lot. First build takes 10–20 min on a clean cache. +## Make-target reference -## OTA quickstart +### Lab cluster (compose) -With the lab + sim up: +| target | what | +|--------|------| +| `sim-up` | gazebo + robot + lab cluster (Postgres, Temporal, EMQX, registry) | +| `sim-up-headless` | same as `sim-up` but no GUI | +| `sim-down` / `sim-logs` | tear down / tail | +| `sim-gui` | open noVNC URL in default browser | +| `lab-up` / `lab-down` / `lab-status` / `lab-reset` | lab cluster only (no sim) | -```bash -# Build and push the lab dummy robot image (Alpine + sleep infinity) -docker build -t localhost:5001/robot-app:v1 -f docker/dummy-robot/Dockerfile docker/dummy-robot -docker push localhost:5001/robot-app:v1 -# Or: make dummy-robot-image (same; requires lab registry + insecure-registries for HTTP) - -# Run the OTA worker (separate terminal) -TEMPORAL_ADDR=localhost:14733 BROKER_URL=tcp://localhost:14883 \ - TSDB_DSN="postgres://temporal:temporal@localhost:14432/telemetry?sslmode=disable" \ - ./bin/ota-worker - -# Run the control plane (separate terminal) -TEMPORAL_ADDR=localhost:14733 \ - TSDB_DSN="postgres://temporal:temporal@localhost:14432/telemetry?sslmode=disable" \ - ./bin/controlplane - -# Start a rollout -curl -X POST http://localhost:8081/v1/ota/rollouts \ - -H "content-type: application/json" \ - -d '{ - "image_ref": "localhost:5001/robot-app:v1", - "smoke_command": "true", - "cohort_selector": {"robot_ids": ["sim-robot-01"]} - }' - -# Watch progress -curl -s http://localhost:8081/v1/ota/rollouts | jq -``` +### Host-side processes (Go binaries) + +| target | what | +|--------|------| +| `agent-up` / `agent-down` / `agent-status` | the agent (native, preferred) | +| `workers-up` / `workers-down` / `workers-status` | ota-worker + collision-worker | +| `controlplane-up` / `controlplane-down` / `controlplane-status` | HTTP API on :8081 | + +### Drive helpers + +| target | what | +|--------|------| +| `sim-drive-fwd LX=` | direct gz `cmd_vel` publish | +| `sim-drive-back` / `sim-drive-left` / `sim-drive-right` / `sim-drive-stop` | same | + +### Demos + +| target | what | +|--------|------| +| `ota-circle` / `ota-figure-eight` | build + push + roll out one of the PR #7 controllers | +| `ota-status` | list recent rollouts | +| `collide` | publish a fake collision event; triggers Temporal `CollisionResponse` | +| `dummy-robot-image` | build + push a no-op alpine OTA image | + +### Build / lint / test + +| target | what | +|--------|------| +| `build` | both Go modules → `bin/` | +| `tidy` | `go mod tidy` for both modules | +| `lint` | `go vet` (+ `staticcheck` if installed) | +| `test` | `go test -race -count=1` | +| `proto` | regenerate Go + Python protobuf bindings via containerized protoc | + +### CI smoke (alternate ports for local CI parity) + +| target | what | +|--------|------| +| `ci-up` / `ci-down` / `ci-status` | CI cluster on `2xxxx` ports (so it can run alongside `lab-up`) | + +### Hooks + meta + +| target | what | +|--------|------| +| `hooks-install` / `hooks-uninstall` | git hooks at `.git-hooks/` (auto-installed on every `make`) | +| `container-info` | which container engine + compose command was detected | +| `help` | this list | + +## Default lab ports + +| service | port | notes | +|---------|------|-------| +| Postgres | 14432 | TimescaleDB; same instance hosts Temporal + telemetry | +| Temporal frontend | 14733 | gRPC for workers | +| Temporal UI | 14080 | http://localhost:14080 | +| MQTT broker (EMQX) | 14883 | anonymous in lab; mTLS gated to S5–S6 (D-11) | +| MQTT dashboard | 14093 | http://localhost:14093 (admin / lab-only) | +| Container registry | 14050 | `localhost:14050/robot-app:tag` | +| Gazebo noVNC | 14680 | http://localhost:14680/vnc.html?autoconnect=1&resize=scale | +| Gazebo VNC | 14900 | raw VNC for native clients | +| Robot bridge gRPC | 50051 | published so the native agent reaches `localhost:50051` | +| Control plane API | 8081 | OTA rollouts | + +CI cluster mirrors the same services on the `2xxxx` range so `lab-up` +and `ci-up` can coexist on one host. + +## Sprint status + +| Sprint | Theme | Status | +|--------|-------|--------| +| S0 | Foundations + installer | landed (Postgres + Temporal + MQTT + registry on `make lab-up`) | +| S1 | Telemetry plumbing | landed (bridge_node ↔ agent over gRPC, MQTT publish) | +| S2 | Telemetry MVP | landed (SQLite buffer, TimescaleDB hypertable, operator API) | +| S3–S4 | OTA workflow + swap + rollback | landed (Temporal workflows, robot-app OTA targets) | +| Demo | Collision response + OTA controllers | landed (CollisionResponse workflow, drive-circle / drive-figure-eight) | +| S5–S6 | Identity (mTLS, signed images) | **gates customer ship** — see `specs/in-process/identity-mtls.md` | + +See [`specs/project-plan.md`](specs/project-plan.md) for the full plan. diff --git a/installer/README.md b/installer/README.md index a00a0fa..048d83e 100644 --- a/installer/README.md +++ b/installer/README.md @@ -75,14 +75,23 @@ The two clusters run under separate Compose project names (`temporal-hack-lab` and `temporal-hack-ci`) so they can be brought up **simultaneously**. `make ci-up` is what the GitHub Actions `installer-smoke` job and the local `pre-push` hook both run. -| Service | Port | -|-------------------|-------| -| Postgres | 5432 | -| Temporal frontend | 7233 | -| Temporal UI | 8080 | -| MQTT | 1883 | -| MQTT dashboard | 18083 | -| Registry | 5001 | + +### Sim-only ports (only `make sim-up`, not `make lab-up`) + +| Service | Port | +|-----------------------|-------| +| Gazebo noVNC | 14680 | +| Gazebo VNC | 14900 | +| Robot bridge gRPC | 50051 | + +### Host-side processes (Go binaries via `make`, not in compose) + +| Process | Port | +|---------------|-------| +| Control plane | 8081 | +| ota-worker | (none, connects to Temporal :14733) | +| collision-worker | (none, connects to Temporal :14733) | +| agent | (none, connects to MQTT :14883 + bridge :50051) | ## Production-target gaps (tracked, not v1)