[review-mirror] M5 epic (upstream nullislabs/shepherd#21)

.dockerignore

@@ -0,0 +1,55 @@
+# Build context exclusion list for `docker build .`. Keeping the
+# context lean matters: every byte sent to the daemon is hashed for
+# the build's source-changed signal, and the production multi-stage
+# Dockerfile already invalidates the dependency layer cache on any
+# Cargo.lock / Cargo.toml change.
+
+# Cargo build artefacts — re-built inside the build stage anyway.
+/target/
+target/
+**/target/
+
+# Runtime state directory the engine writes the redb file into. Never
+# part of the image.
+/data/
+data/
+
+# Backtest tooling output: large JSON fixtures + Python venv state.
+# Re-collected on demand via `tools/backtest-collect/backtest_collect.py`.
+tools/backtest-collect/fixtures-*.json
+tools/baseline-latency/data/
+tools/**/__pycache__/
+tools/**/*.pyc
+
+# NOTE: `modules/fixtures/*-bomb` are listed in the workspace
+# `Cargo.toml`, so excluding them breaks `cargo build` ("failed to
+# load manifest for workspace member"). They're tiny crates and the
+# Dockerfile doesn't COPY them to the runtime stage, so the
+# image size impact is zero. Keep them in the build context.
+
+# Local-only configs. The production `engine.toml` is supplied at
+# runtime via a bind-mount (`/etc/shepherd/engine.toml`).
+engine.toml
+engine.e2e.toml
+engine.load.toml
+engine.m2.toml
+engine.m3.toml
+
+# Git + GitHub metadata.
+/.git/
+/.github/
+.gitignore
+
+# Editor / OS noise.
+.vscode/
+.idea/
+.DS_Store
+*.swp
+
+# Operator-side docs reports the image doesn't need. Source markdown
+# stays so it's discoverable inside the container if an operator
+# `docker exec`s in for a quick `cat docs/production.md`.
+docs/operations/load-reports/
+docs/operations/e2e-reports/
+docs/operations/backtest-reports/
+docs/operations/baselines/

.env.example

@@ -0,0 +1,25 @@
+# Operator template — copy to `.env` and fill in your paid RPC URLs.
+# `.env` is gitignored; never commit a populated copy.
+#
+# Workflow:
+#   cp .env.example .env
+#   $EDITOR .env
+#   docker compose up -d
+#
+# The engine reads these via `${VAR}` placeholders in
+# `engine.docker.toml` (substitution happens at config-load time,
+# before TOML parse, so a missing variable fails fast).
+#
+# Use `wss://` schemes — `eth_subscribe` is WebSocket-only and the
+# engine emits a boot-time ERROR on http(s):// URLs (see
+# docs/production.md §6 and engine_config::validate_transports).
+
+MAINNET_RPC_URL=wss://eth-mainnet.g.alchemy.com/v2/REPLACE_ME
+GNOSIS_RPC_URL=wss://gnosis-mainnet.g.alchemy.com/v2/REPLACE_ME
+SEPOLIA_RPC_URL=wss://eth-sepolia.g.alchemy.com/v2/REPLACE_ME
+ARBITRUM_RPC_URL=wss://arb-mainnet.g.alchemy.com/v2/REPLACE_ME
+BASE_RPC_URL=wss://base-mainnet.g.alchemy.com/v2/REPLACE_ME
+
+# Optional: override the published image with a locally-built or
+# pinned-by-SHA tag. Leave unset to pull `:latest` from ghcr.io.
+# SHEPHERD_IMAGE=ghcr.io/bleu/nullis-shepherd:sha-abc1234

.github/workflows/docker.yml

@@ -0,0 +1,88 @@
+# Docker image build + publish to ghcr.io.
+#
+# Triggers:
+#   - push to `main`               → publish `latest` + `sha-<sha>`
+#   - tag push `v*`                → publish `v<tag>` + `latest`
+#   - workflow_dispatch (manual)   → publish `manual-<run_id>`
+#   - pull_request to `main`       → build only, no push (CI smoke)
+#
+# Image: ghcr.io/<owner>/nullis-shepherd
+# Auth:  GITHUB_TOKEN (scoped to packages:write below).
+#
+# Pinned action SHAs match the style of `.github/workflows/ci.yml`.
+
+name: docker
+
+on:
+  push:
+    branches: [main]
+    tags: ["v*"]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push:
+    name: build + push (${{ github.event_name }})
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Docker buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Log in to ghcr.io
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Compute image metadata
+        id: meta
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            # `latest` on push to main and on tag.
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=ref,event=tag
+            # `sha-<short>` on every event so a soak run can pin an
+            # exact build.
+            type=sha,prefix=sha-,format=short
+            # manual-<run_id> for workflow_dispatch.
+            type=raw,value=manual-${{ github.run_id }},enable=${{ github.event_name == 'workflow_dispatch' }}
+            # `pr-<n>` on pull-request builds so the smoke artefact
+            # is identifiable. PR builds are NOT pushed (see `push:`).
+            type=ref,event=pr,prefix=pr-
+
+      - name: Build + push
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        with:
+          context: .
+          file: ./Dockerfile
+          # Push on every non-PR event; PR builds are local-only smoke.
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          # Layer cache via the registry: the previous successful
+          # build's intermediate layers are reused so a Cargo.toml-only
+          # change re-compiles only the changed crate.
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max,ignore-error=true
+          # `amd64` is enough for the soak VM. Add `arm64` once an
+          # operator surfaces a real need; multi-arch ~2x the build.
+          platforms: linux/amd64

.gitignore

@@ -21,6 +21,9 @@ Thumbs.db
 # Environment
 .env
 .env.*
+# Exception: the committed template (operator copies it to `.env`,
+# which is then caught by the rule above).
+!.env.example
 
 # Agent skills / AI tooling — installed locally, never committed.
 .agents/
@@ -36,6 +39,11 @@ data/
 scripts/.state
 scripts/.env
 
+# Operator-supplied engine config (carries paid RPC URLs / API keys).
+# The committed siblings `engine.example.toml`, `engine.docker.toml`,
+# and `engine.{m2,m3,e2e,load}.toml` are placeholder templates.
+/engine.toml
+
 # Generated reports under e2e-reports/ (operator commits the filled-in ones
 # manually via `git add -f`).
 docs/operations/e2e-reports/engine-*.log

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
     "crates/nexum-engine",
+    "crates/shepherd-backtest",
     "crates/shepherd-sdk",
     "crates/shepherd-sdk-test",
     "modules/ethflow-watcher",

Dockerfile

@@ -0,0 +1,107 @@
+# syntax=docker/dockerfile:1.6
+#
+# Multi-stage build for `nexum-engine` (Shepherd) — the engine binary
+# plus the five production WASM modules baked into a single image.
+#
+# Stage 1 (`build`): full Rust toolchain + wasm32-wasip2 target, builds
+# the engine in release mode + each module to a Component Model wasm
+# artefact.
+#
+# Stage 2 (`runtime`): minimal Debian slim. Just `ca-certificates`
+# (for HTTPS to cow.fi / paid RPCs), `tini` as PID 1 (forwards SIGINT
+# for graceful shutdown per docs/production.md §2), and a non-root
+# `shepherd` user owning `/var/lib/shepherd`.
+#
+# The runtime entrypoint expects `/etc/shepherd/engine.toml` to be
+# mounted (read-only) — see `docker-compose.yml` and
+# `docs/deployment/docker.md`.
+
+# ----------------------------------------------------------------- build
+
+# Pin the Rust toolchain to a version recent enough for the
+# transitive wasmtime 45.x crates (which require rustc >= 1.93).
+# Bump in lockstep with workspace Cargo.lock minimum-supported
+# rustc — `cargo msrv` if uncertain.
+FROM rust:1.96-slim-bookworm AS build
+
+# Build deps for ring/openssl/cmake-using crates pulled in via alloy
+# and cowprotocol. `clang` is for any inline-C bindings (e.g.
+# pycryptodome-equivalent in the wasm side); cheap enough to bundle.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        pkg-config libssl-dev cmake clang ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN rustup target add wasm32-wasip2
+
+WORKDIR /src
+
+# Copy the whole workspace. `.dockerignore` should keep the build
+# context lean (no `target/`, no `data/`, no large baseline / backtest
+# fixtures).
+COPY . .
+
+# Engine binary in release.
+RUN cargo build -p nexum-engine --release
+
+# Five production modules. The wasm artefacts land under
+# `target/wasm32-wasip2/release/<name_with_underscores>.wasm`.
+RUN cargo build -p twap-monitor     --target wasm32-wasip2 --release \
+ && cargo build -p ethflow-watcher  --target wasm32-wasip2 --release \
+ && cargo build -p price-alert      --target wasm32-wasip2 --release \
+ && cargo build -p balance-tracker  --target wasm32-wasip2 --release \
+ && cargo build -p stop-loss        --target wasm32-wasip2 --release
+
+# ----------------------------------------------------------------- runtime
+
+FROM debian:bookworm-slim AS runtime
+
+# `tini` reaps zombies + forwards SIGINT/SIGTERM to the engine so the
+# COW-1072 graceful-shutdown path actually runs (drain in-flight
+# dispatch, persist `last_dispatched_block:{chain_id}` to local-store).
+# `ca-certificates` is mandatory for HTTPS calls to cow.fi + paid RPC
+# endpoints; the engine has no embedded TLS roots.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates tini \
+    && rm -rf /var/lib/apt/lists/* \
+    && useradd -r -s /usr/sbin/nologin -d /var/lib/shepherd shepherd \
+    && install -d -o shepherd -g shepherd -m 0755 /var/lib/shepherd \
+    && install -d -o root     -g root     -m 0755 /opt/shepherd \
+    && install -d -o root     -g root     -m 0755 /opt/shepherd/modules \
+    && install -d -o root     -g root     -m 0755 /opt/shepherd/manifests \
+    && install -d -o root     -g root     -m 0755 /etc/shepherd
+
+# Engine binary.
+COPY --from=build /src/target/release/nexum-engine /usr/local/bin/nexum-engine
+
+# Module .wasm artefacts. The Component Model wasm files are loaded
+# by the engine at boot via the `[[modules]]` entries in engine.toml.
+COPY --from=build /src/target/wasm32-wasip2/release/*.wasm /opt/shepherd/modules/
+
+# Module manifests (the `module.toml` next to each cdylib crate). The
+# engine resolves capability declarations + chain subscriptions from
+# these at supervisor boot.
+COPY --from=build /src/modules/twap-monitor/module.toml    /opt/shepherd/manifests/twap-monitor.toml
+COPY --from=build /src/modules/ethflow-watcher/module.toml /opt/shepherd/manifests/ethflow-watcher.toml
+COPY --from=build /src/modules/examples/price-alert/module.toml     /opt/shepherd/manifests/price-alert.toml
+COPY --from=build /src/modules/examples/balance-tracker/module.toml /opt/shepherd/manifests/balance-tracker.toml
+COPY --from=build /src/modules/examples/stop-loss/module.toml       /opt/shepherd/manifests/stop-loss.toml
+
+# Drop privileges. The engine never needs root at runtime: it only
+# reads /etc/shepherd/engine.toml, writes to /var/lib/shepherd, and
+# binds 127.0.0.1:9100 inside the container.
+USER shepherd
+WORKDIR /var/lib/shepherd
+
+# Metrics endpoint. The engine binds 127.0.0.1:9100 inside the
+# container by default; docker-compose maps it to the host's
+# loopback so Prometheus scrapes it via the docker network without
+# exposing /metrics to the public internet.
+EXPOSE 9100
+
+# `--engine-config /etc/shepherd/engine.toml` matches the production
+# guide's expected mount point. Operators override via
+# `docker run ... -v /path/to/engine.toml:/etc/shepherd/engine.toml:ro`.
+ENTRYPOINT ["/usr/bin/tini", "--", "nexum-engine"]
+CMD ["--engine-config", "/etc/shepherd/engine.toml"]