From fd32f608e877c4792f141e3e9be7d0724086ecf2 Mon Sep 17 00:00:00 2001 From: Aaron Elijah Mars Date: Mon, 27 Jul 2026 14:36:54 -0400 Subject: [PATCH] fix(security): sandbox read-only runs on every harness; move notify queues out of the repo Three linked defects, fixed together because the first two were what made the third unfixable. 1. .pending-.md was being COMMITTED. .gitignore ignores `.pending-*/` (directories), not the file, so the json-render staging file was tracked. A sandboxed harness that could not overwrite it left the PREVIOUS run's copy in place, and "Capture skill output" published that as the new run's output - a green run reporting another run's work, and feeding it to the health scorer. Measured live: four consecutive runs captured a byte-identical artifact from a fifth. 2. notify's queues (re-delivery, dedup hashes, json-render staging) lived in the workspace, so under the OS sandbox they could not be written at all. They now live in $AEON_PENDING_DIR, exported by the workflow and defaulted in notify.sh so local runs and other entry points keep working. 3. That was the ONLY reason claude and grok ran with --no-sandbox, so read-only skills on those two had no runtime enforcement. On grok it was total: its adapter runs --permission-mode bypassPermissions and never applied an allowlist, so a read-only skill could write anywhere in the repo - verified by asking one to create a file, which it did - with only the post-run guard reverting it afterwards. Both now run under the wrapper sandbox. Bubblewrap installation had to be hoisted out of "Install harness CLI" (which skips claude entirely and skipped grok by name) into its own step, or run-harness would have found no bwrap and silently degraded to "read-only is advisory" with nothing but a stderr notice. Also: the read-only guard now cleans what it reverts. `git checkout` only undoes edits to TRACKED files; the `git clean` list covered 6 of the ~15 CODE_PATHS, so a file a read-only skill CREATED outside those 6 survived and was committed by `git add -A`. Both lists are now the same set. Verified live on a real instance (grok, read-only, MCP enabled): `bwrap: userns OK` -> `read-only: workspace write-locked via bwrap` -> glim MCP still reachable -> chain artifact captured at this run's own 1169 bytes instead of the stale 3436-byte digest that four earlier runs had republished. --- .github/workflows/aeon.yml | 104 +++++++++++++++++++++-------------- .gitignore | 7 +++ AGENTS.md | 2 +- CHANGELOG.md | 27 +++++++++ CLAUDE.md | 2 +- docs/attestation.md | 2 +- scripts/notify.sh | 36 ++++++++---- scripts/tests/test_notify.sh | 62 +++++++++++++++------ 8 files changed, 170 insertions(+), 72 deletions(-) diff --git a/.github/workflows/aeon.yml b/.github/workflows/aeon.yml index 50659cf620..113eea1b15 100644 --- a/.github/workflows/aeon.yml +++ b/.github/workflows/aeon.yml @@ -345,6 +345,25 @@ jobs: # be staged here just like the other adapter harnesses. It reuses # `run-grok.sh setup` — the single source of truth for grok's pin and the # GROK_CREDENTIALS/XAI_API_KEY restore — so nothing is duplicated. + # bubblewrap is what harness-adapter/lib/sandbox.sh uses to make read-only + # mean "the workspace physically cannot be written" on Linux. This used to + # live inside "Install harness CLI", which skips claude entirely and skipped + # grok by name — so those two had no sandbox available even when asked for + # one, and run-harness would have degraded to "read-only is advisory" with + # only a stderr notice. Now every harness gets it, so --mode read-only means + # the same thing on all six. + - name: Install bubblewrap (read-only sandbox) + if: steps.work.outputs.mode != '' + run: | + set -euo pipefail + sudo apt-get update -qq && sudo apt-get install -y -qq bubblewrap + # Ubuntu 24.04 (including GitHub's ubuntu-latest) restricts unprivileged + # user namespaces via AppArmor, so bwrap dies with 'setting up uid map: + # Permission denied' and run-harness fails CLOSED on every read-only + # skill. Measured on this runner 2026-07-21. Re-enable them for the job. + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + bwrap --dev-bind / / --ro-bind /tmp /tmp true && echo 'bwrap: userns OK' || echo '::warning::bwrap still blocked' + - name: Install harness CLI if: steps.harness.outputs.HARNESS != '' && steps.harness.outputs.HARNESS != 'claude' env: @@ -375,21 +394,6 @@ jobs: MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} run: | set -euo pipefail - # bubblewrap is what harness-adapter/lib/sandbox.sh uses to make - # read-only mean "the workspace physically cannot be written" on Linux. - # grok is skipped: like claude it runs through run-harness with - # --no-sandbox (its own bypassPermissions + the post-run stray-write - # revert are its read-only guard), so it never invokes the wrapper. - if [ "$H" != "grok" ]; then - sudo apt-get update -qq && sudo apt-get install -y -qq bubblewrap - # Ubuntu 24.04 (including GitHub's ubuntu-latest) restricts unprivileged - # user namespaces via AppArmor, so bwrap dies with 'setting up uid map: - # Permission denied' and run-harness fails CLOSED on every read-only - # skill. Measured on this runner 2026-07-21. Re-enable them for the job. - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true - bwrap --dev-bind / / --ro-bind /tmp /tmp true && echo 'bwrap: userns OK' || echo '::warning::bwrap still blocked' - fi - # Each harness is configured for the provider AUTH_MODE selected: # native-oauth — restore the captured login (~/.codex, ~/.kimi-code); # the harness then runs on its own default model. @@ -839,12 +843,20 @@ jobs: echo "GATEWAY=${CANDIDATES%% *}" >> "$GITHUB_OUTPUT" # provisional; overwritten with the winner below fi - # Generate notify script in workspace (Claude calls: ./notify "message") - # Messages are saved to .pending-notify/ for reliable post-run delivery - # (sandbox may block outbound curl during Claude's run) - # Pre-create .pending-notify/ so the sandbox doesn't block Claude's first write into it - mkdir -p .pending-notify - : > .notify-sent-hashes + # Generate notify script in workspace (the agent calls: ./notify "message"). + # Undelivered messages queue under $AEON_PENDING_DIR for post-run re-delivery. + # notify's staging area, OUTSIDE the workspace. A read-only skill runs + # under an OS sandbox that mounts the repo read-only, so queues kept in + # the workspace can't be written; and the json-render staging file was + # being committed (.gitignore ignores `.pending-*/` dirs, not + # `.pending-*.md`), which let a sandboxed run republish the PREVIOUS + # run's digest as its own. Exported to $GITHUB_ENV so the post-run + # capture / feed / re-delivery steps read from the same place. + AEON_PENDING_DIR="${RUNNER_TEMP:-/tmp}/aeon-pending" + mkdir -p "$AEON_PENDING_DIR/notify-queue" + : > "$AEON_PENDING_DIR/notify-sent-hashes" + echo "AEON_PENDING_DIR=$AEON_PENDING_DIR" >> "$GITHUB_ENV" + export AEON_PENDING_DIR # notify / notify-jsonrender are the committed scripts/*.sh (testable; see scripts/tests/). cp scripts/notify.sh ./notify @@ -972,14 +984,17 @@ jobs: # There is no longer a native `claude -p` or `run-grok.sh` run path; the # two features those paths carried are preserved AROUND the adapter call: # * claude — wrapped in the multi-provider gateway cascade below, so it - # keeps 8-provider failover + Langfuse tracing. --no-sandbox: - # claude's own --allowedTools + the post-run stray-write - # revert are its read-only guard, and it keeps .pending-notify - # (inside the workspace) writable. - # * grok — GROK_* run-knobs from frontmatter (adapters/grok.sh maps - # them); --no-sandbox for the same reason (bypassPermissions - # + revert are its guard; its own --sandbox is a 0.2.101 no-op). - # * codex/pi/vibe/kimi — the wrapper OS sandbox (bwrap) enforces read-only. + # keeps 8-provider failover + Langfuse tracing. + # * grok — GROK_* run-knobs from frontmatter (adapters/grok.sh maps them). + # ALL six now run read-only under the wrapper OS sandbox (bwrap). claude and + # grok used to pass --no-sandbox because ./notify needed to write its queue + # inside the workspace; that queue moved to $AEON_PENDING_DIR, so the reason + # is gone. It mattered most on grok, whose adapter runs + # --permission-mode bypassPermissions and never applied an allowlist: with + # the sandbox off, a read-only grok skill had NO runtime enforcement at all + # and could write anywhere in the repo (measured — it created a file on + # request). The post-run guard reverted it after the fact; now the write is + # refused outright. CLAUDE_OK=0 RH="${GITHUB_WORKSPACE}/harness-adapter/run-harness" RH_ARGS=(--mode "$SKILL_MODE" --allowed-tools "$ALLOWED" --timeout 900) @@ -1005,7 +1020,7 @@ jobs: MODEL="$BASE_MODEL" source "$GW" >&2 source "$LF" >&2 # optional Langfuse tracing (no-op unless LANGFUSE_* set) - echo "$PROMPT" | bash "$RH" claude --model "$MODEL" --no-sandbox "${RH_ARGS[@]}" 2>/tmp/harness-stderr.txt + echo "$PROMPT" | bash "$RH" claude --model "$MODEL" "${RH_ARGS[@]}" 2>/tmp/harness-stderr.txt ); then USED_GATEWAY="$cand"; CLAUDE_OK=1; break fi @@ -1027,7 +1042,6 @@ jobs: # consumed by adapters/grok.sh — the same frontmatter mapping the # native path used. SKILL_MODEL was already set to grok's model above. eval "$(bash "${GITHUB_WORKSPACE}/scripts/skill_mode.sh" grok-run-env "$SKILL_NAME")" - RH_ARGS+=(--no-sandbox) # BASE_MODEL is grok-4.5 or an explicit grok id (forced above); pass # only a real grok id — the adapter falls back to grok's default # otherwise. (BASE_MODEL, not MODEL: identical here, but MODEL is the @@ -1208,7 +1222,7 @@ jobs: SKILL="${{ steps.skill.outputs.name }}" mkdir -p output/.chains # Save the notification message as a chain artifact - PENDING="apps/dashboard/outputs/.pending-${SKILL}.md" + PENDING="${AEON_PENDING_DIR:-${RUNNER_TEMP:-/tmp}/aeon-pending}/.pending-${SKILL}.md" if [ -f "$PENDING" ]; then cp "$PENDING" "output/.chains/${SKILL}.md" elif [ -f /tmp/skill-result.txt ] && [ -s /tmp/skill-result.txt ]; then @@ -1250,7 +1264,7 @@ jobs: # json-render feed (the trust boundary that actually benefits). PUBLISHED=false if [ "${JSONRENDER_ENABLED}" = "true" ] \ - && [ -f "apps/dashboard/outputs/.pending-${SKILL}.md" ]; then + && [ -f "${AEON_PENDING_DIR:-${RUNNER_TEMP:-/tmp}/aeon-pending}/.pending-${SKILL}.md" ]; then PUBLISHED=true fi @@ -1556,7 +1570,7 @@ jobs: GATEWAY="${{ steps.run.outputs.GATEWAY }}" source "${GITHUB_WORKSPACE}/scripts/llm-gateway.sh" AEON_OTEL_COMPONENT=feed source "${GITHUB_WORKSPACE}/scripts/langfuse-otel.sh" - for pending in apps/dashboard/outputs/.pending-*.md; do + for pending in "${AEON_PENDING_DIR:-${RUNNER_TEMP:-/tmp}/aeon-pending}"/.pending-*.md; do [ -f "$pending" ] || continue SKILL=$(basename "$pending" .md | sed 's/^\.pending-//') echo "json-render: converting $SKILL..." @@ -1592,16 +1606,17 @@ jobs: NOTIFY_EMAIL_SUBJECT_PREFIX: ${{ vars.NOTIFY_EMAIL_SUBJECT_PREFIX || '[Aeon]' }} SKILL_NAME: ${{ steps.skill.outputs.name }} run: | - # Re-deliver any .pending-notify/ messages that failed during sandbox. + # Re-deliver any queued messages that failed to send during the run. + Q="${AEON_PENDING_DIR:-${RUNNER_TEMP:-/tmp}/aeon-pending}/notify-queue" # Skip messages whose hash was already sent by ./notify inline, # and skip short test/trace probes that shouldn't reach channels. - if [ ! -d .pending-notify ] || [ -z "$(ls -A .pending-notify/*.md 2>/dev/null)" ]; then + if [ ! -d "$Q" ] || [ -z "$(ls -A "$Q"/*.md 2>/dev/null)" ]; then echo "No pending notifications" exit 0 fi - HASH_FILE=".notify-sent-hashes" + HASH_FILE="${AEON_PENDING_DIR:-${RUNNER_TEMP:-/tmp}/aeon-pending}/notify-sent-hashes" touch "$HASH_FILE" - for pending in .pending-notify/*.md; do + for pending in "$Q"/*.md; do [ -f "$pending" ] || continue MSG=$(cat "$pending") @@ -1660,7 +1675,7 @@ jobs: '{from:$from, to:[$to], subject:$subject, html:$html, text:$text}')" > /dev/null 2>&1 || true fi done - rm -rf .pending-notify + rm -rf "$Q" - name: Read-only capability guard if: steps.work.outputs.mode != '' && env.SKILL_MODE == 'read-only' @@ -1675,7 +1690,14 @@ jobs: printf '\n## %s (read-only)\nRan %s UTC.\n' "$SKILL" "$(date -u +%FT%TZ)" >> "memory/logs/${TODAY}.md" CODE_PATHS="skills .github apps/mcp-server apps/webhook apps/dashboard/app apps/dashboard/lib apps/dashboard/components scripts docs catalog bin *.md *.json *.yml aeon" git checkout -- $CODE_PATHS 2>/dev/null || true - git clean -fd -- skills scripts docs .github bin catalog 2>/dev/null || true + # Same path set as CODE_PATHS above. `git checkout` only reverts EDITS to + # TRACKED files - it cannot remove a file the run CREATED. The clean list + # used to cover 6 of those paths, so a new untracked file elsewhere (e.g. + # apps/dashboard/lib/x.ts, or a brand-new top-level dir) survived the guard + # and was committed by `git add -A` below. No -x, so gitignored build + # artifacts (node_modules, .next) are untouched; memory/ and output/ are in + # neither list and still commit by design. + git clean -fd -- $CODE_PATHS 2>/dev/null || true echo "read-only guard: logged run + reverted code/config writes for $SKILL" - name: Commit results @@ -1685,7 +1707,7 @@ jobs: run: | git config user.name "aeonframework" git config user.email "aeonframework@proton.me" - rm -f ./notify ./notify-jsonrender .notify-sent-hashes # Remove generated notify scripts + dedup log before committing + rm -f ./notify ./notify-jsonrender # Remove generated notify scripts before committing # OKF: stamp `type:` frontmatter onto any in-scope markdown the run # created bare (e.g. a fresh memory/logs/ day-file appended via bash by diff --git a/.gitignore b/.gitignore index ac83cc42ba..2f17fe07a9 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,13 @@ # .pending-notify (notify re-delivery), .pending-deploy (deploy-prototype build), # plus .pending-notify-temp where skills stage notify bodies for `./notify -f` .pending-*/ +# ...and the FILE form. The trailing slash above matches directories only, so the +# json-render staging file (.pending-.md) was tracked and committed; a later +# read-only run that could not overwrite it had the previous run's digest published +# as its own. It now lives outside the repo ($AEON_PENDING_DIR); this is the guard +# against it coming back. +.pending-*.md +apps/dashboard/outputs/.pending-*.md # Skill provenance lock (per-repo, not committed) skills.lock diff --git a/AGENTS.md b/AGENTS.md index 0c6f802223..aa1cfa865a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -143,7 +143,7 @@ Two exemptions: **reserved `index.md`/`log.md`** stay untyped (OKF §6/§7), and - **`./scripts/skill-runs [--hours N] [--full] [--json] [--failures]`** — Audit recent GitHub Actions skill runs (counts, pass/fail rates, anomalies). Needs `gh` + `jq`. - **WebSearch** / **WebFetch** — built-in Claude tools for search and URL fetching; they bypass the bash sandbox, so prefer them over `curl` for reads. -**json-render feed:** when `JSONRENDER_ENABLED=true` **and** `SKILL_NAME` is set, `./notify` queues your output at `apps/dashboard/outputs/.pending-${SKILL_NAME}.md`; a post-run workflow step then converts it into a rendered spec via `./notify-jsonrender`, which the dashboard feed displays. (`./aeon` itself only launches the dashboard web app — it does not run skills.) +**json-render feed:** when `JSONRENDER_ENABLED=true` **and** `SKILL_NAME` is set, `./notify` queues your output at `$AEON_PENDING_DIR/.pending-${SKILL_NAME}.md` (outside the repo); a post-run workflow step then converts it into a rendered spec via `./notify-jsonrender`, which the dashboard feed displays. (`./aeon` itself only launches the dashboard web app — it does not run skills.) ## Capability mode diff --git a/CHANGELOG.md b/CHANGELOG.md index e7b380eb13..50b63b7e66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,33 @@ from or pin to; the template keeps serving the latest `main` to new forks. ### Fixed +- **Read-only skills are now sandboxed on every harness, and notify's queues moved + out of the repo.** Three linked defects: + - **`.pending-.md` was being committed.** `.gitignore` ignores + `.pending-*/` (directories), not the file, so the json-render staging file was + tracked. A sandboxed harness that could not overwrite it left the previous + run's copy in place and "Capture skill output" published *that* as the new + run's output — a green run reporting another run's work, and feeding it to the + health scorer. Measured on a live instance: four consecutive runs captured a + byte-identical artifact from a fifth. + - **notify's queues lived in the workspace**, so under the OS sandbox they could + not be written at all — costing dedup state, the re-delivery queue and the feed + entry. They now live in `$AEON_PENDING_DIR` (outside the repo, exported by the + workflow), which is also what removed the reason `claude`/`grok` ran with + `--no-sandbox`. + - **`claude` and `grok` therefore ran read-only skills with no runtime + enforcement.** It mattered most on grok, whose adapter uses + `--permission-mode bypassPermissions` and never applied an allowlist: a + read-only grok skill could write anywhere in the repo (verified — it created a + file on request), with only the post-run guard reverting it afterwards. Both + now run under the wrapper sandbox. Bubblewrap installation was hoisted out of + "Install harness CLI" — which skips claude and skipped grok by name — into its + own step, or the sandbox would have silently degraded to advisory. +- **The read-only guard now cleans what it reverts.** `git checkout` only undoes + edits to *tracked* files; the `git clean` list covered 6 of the ~15 `CODE_PATHS`, + so a file a read-only skill *created* outside those 6 (e.g. under + `apps/dashboard/lib/`, or a new top-level directory) survived and was committed + by `git add -A`. Both lists are now the same set. - **`scripts/run-grok.sh` is now setup-only.** With every surface dispatching through `run-harness`, the script's ~260-line run path (model/permission flags, MCP allows, run-shaping knobs, grok invocation, envelope normalization) had no diff --git a/CLAUDE.md b/CLAUDE.md index 5ccc51861d..f8315a3690 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,7 +87,7 @@ Two exemptions: **reserved `index.md`/`log.md`** stay untyped (OKF §6/§7), and - **`./scripts/skill-runs [--hours N] [--full] [--json] [--failures]`** — Audit recent GitHub Actions skill runs (counts, pass/fail rates, anomalies). Needs `gh` + `jq`. - **WebSearch** / **WebFetch** — built-in Claude tools for search and URL fetching; they bypass the bash sandbox, so prefer them over `curl` for reads. -**json-render feed:** when `JSONRENDER_ENABLED=true` **and** `SKILL_NAME` is set, `./notify` queues your output at `apps/dashboard/outputs/.pending-${SKILL_NAME}.md`; a post-run workflow step then converts it into a rendered spec via `./notify-jsonrender`, which the dashboard feed displays. (`./aeon` itself only launches the dashboard web app — it does not run skills.) +**json-render feed:** when `JSONRENDER_ENABLED=true` **and** `SKILL_NAME` is set, `./notify` queues your output at `$AEON_PENDING_DIR/.pending-${SKILL_NAME}.md` (outside the repo); a post-run workflow step then converts it into a rendered spec via `./notify-jsonrender`, which the dashboard feed displays. (`./aeon` itself only launches the dashboard web app — it does not run skills.) ## Capability mode diff --git a/docs/attestation.md b/docs/attestation.md index bab944f504..7f8dbb553a 100644 --- a/docs/attestation.md +++ b/docs/attestation.md @@ -140,7 +140,7 @@ in `aeon.yml` always wins. With the global switch on but no per-skill override, the **default policy attests only runs whose output crossed the trust boundary** — i.e. runs that **published to the json-render feed**. Concretely, the gate checks for -`apps/dashboard/outputs/.pending-${SKILL}.md`, which is written **only when a +`$AEON_PENDING_DIR/.pending-${SKILL}.md` (outside the repo), which is written **only when a skill actually calls `./notify`** (with `JSONRENDER_ENABLED` on, its default). **This is deliberate.** Each attestation is a Sigstore signing plus a permanent, diff --git a/scripts/notify.sh b/scripts/notify.sh index b1047c5a06..9be94f5f9a 100755 --- a/scripts/notify.sh +++ b/scripts/notify.sh @@ -13,7 +13,7 @@ # # Per-channel rendering (via scripts/notify_format.py): Telegram = Markdown # normalized to HTML (parse_mode=HTML, fence-safe chunks, 3900), Discord embeds -# (color by severity), Slack Block Kit. Falls back to .pending-notify/ for +# (color by severity), Slack Block Kit. Falls back to $AEON_PENDING_DIR/notify-queue for # post-run delivery when the sandbox blocks outbound curl. set -euo pipefail @@ -114,10 +114,25 @@ if [ -n "$LINK" ]; then MSG=$(printf '%s\n\n🔗 %s' "$MSG" "$LINK") fi +# --- staging area for notify's queues --------------------------------------- +# These MUST live outside the workspace. Two reasons, both measured: +# 1. A read-only skill runs under an OS sandbox that mounts the repo read-only, +# so a queue inside the workspace simply cannot be written — the run then +# loses its dedup state, its re-delivery queue and its feed entry. +# 2. `.gitignore` ignores `.pending-*/` (directories), not `.pending-*.md`, so +# the json-render staging FILE was getting COMMITTED. A later sandboxed run +# that couldn't overwrite it left the previous run's copy in place, and the +# "Capture skill output" step happily published that stale digest as the new +# run's output — a green run reporting someone else's work. +# The workflow exports AEON_PENDING_DIR; the fallbacks keep local runs and other +# entry points working. +PENDING_DIR="${AEON_PENDING_DIR:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}/aeon-pending}" +mkdir -p "$PENDING_DIR" 2>/dev/null || true + # Dedup within this run — same rendered message never sent twice _sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } HASH=$(printf '%s' "$TITLE|$SEVERITY|$MSG" | _sha | awk '{print $1}') -HASH_FILE=".notify-sent-hashes" +HASH_FILE="$PENDING_DIR/notify-sent-hashes" touch "$HASH_FILE" 2>/dev/null || true if grep -qxF "$HASH" "$HASH_FILE" 2>/dev/null; then echo "notify: duplicate message (hash ${HASH:0:8}), skipping" >&2 @@ -138,7 +153,7 @@ else PLAIN="$MSG" fi -# Try to save to .pending-notify/ for post-run re-delivery. This queue is the +# Try to save to $PENDING_DIR/notify-queue for post-run re-delivery. This queue is the # fallback for the INVERSE sandbox — network blocked, FS writable (the old claude # wrapper case): notify can't reach the wire inline, so the workflow's "Send # pending notifications" step re-delivers post-run from here. @@ -149,8 +164,8 @@ fi # script runs `set -e`, so an unguarded failure here would kill the whole notify # before a single channel is tried). TS=$(date -u +%s) -if ! { mkdir -p .pending-notify && printf '%s' "$PLAIN" > ".pending-notify/${TS}.md"; } 2>/dev/null; then - echo "notify: .pending-notify unwritable (read-only FS) — delivering inline only" >&2 +if ! { mkdir -p "$PENDING_DIR/notify-queue" && printf '%s' "$PLAIN" > "$PENDING_DIR/notify-queue/${TS}.md"; } 2>/dev/null; then + echo "notify: notify-queue unwritable — delivering inline only" >&2 fi DELIVERED=false @@ -219,8 +234,8 @@ if [ -n "${TELEGRAM_BOT_TOKEN:-}" ] && [ -n "${TELEGRAM_CHAT_ID:-}" ]; then # Dry-run (tests): record the payload instead of sending. No network. if [ "${NOTIFY_DRY_RUN:-}" = "1" ]; then - mkdir -p .pending-notify - printf '%s\n' "$TG_PAYLOAD" >> .pending-notify/tg-payload.jsonl + mkdir -p "$PENDING_DIR/notify-queue" + printf '%s\n' "$TG_PAYLOAD" >> "$PENDING_DIR/notify-queue/tg-payload.jsonl" DELIVERED=true continue fi @@ -285,17 +300,16 @@ if [ -n "${RESEND_API_KEY:-}" ] && [ -n "${NOTIFY_EMAIL_TO:-}" ]; then fi # json-render channel — save raw message for post-run conversion. Non-fatal for -# the same reason as the .pending-notify/ queue above: on a read-only-FS harness +# the same reason as the notify-queue above: on a read-only-FS harness # (codex) this write can't land, and it must not abort a run whose channel # delivery already succeeded inline. The feed entry is simply skipped. if [ "${JSONRENDER_ENABLED:-false}" = "true" ] && [ -n "${SKILL_NAME:-}" ]; then - if ! { mkdir -p apps/dashboard/outputs \ - && printf '%s' "$PLAIN" > "apps/dashboard/outputs/.pending-${SKILL_NAME}.md"; } 2>/dev/null; then + if ! printf '%s' "$PLAIN" > "$PENDING_DIR/.pending-${SKILL_NAME}.md" 2>/dev/null; then echo "notify: json-render queue unwritable (read-only FS) — feed entry skipped" >&2 fi fi # Remove pending file if immediate delivery succeeded (prevents double-send) if [ "$DELIVERED" = "true" ]; then - rm -f ".pending-notify/${TS}.md" + rm -f "$PENDING_DIR/notify-queue/${TS}.md" fi diff --git a/scripts/tests/test_notify.sh b/scripts/tests/test_notify.sh index 5eb9211512..aa3545c3e4 100755 --- a/scripts/tests/test_notify.sh +++ b/scripts/tests/test_notify.sh @@ -1,20 +1,26 @@ #!/usr/bin/env bash # Integration test for scripts/notify.sh — exercises arg parsing, probe suppression, -# dedup, severity gate, and the .pending-notify fallback with all channels unset. +# dedup, severity gate, and the notify-queue fallback with all channels unset. +# The queues live under $AEON_PENDING_DIR (outside the workspace), so the test +# points that at a temp dir instead of writing into the repo. # No network, no secrets. Run: bash scripts/tests/test_notify.sh set -uo pipefail cd "$(dirname "$0")/../.." || exit 1 NOTIFY="scripts/notify.sh" +# Queues live outside the repo now; isolate them per-run. +AEON_PENDING_DIR="$(mktemp -d)"; export AEON_PENDING_DIR +trap 'rm -rf "$AEON_PENDING_DIR"' EXIT + # Channels unset → everything falls back to .pending-notify unset TELEGRAM_BOT_TOKEN TELEGRAM_CHAT_ID DISCORD_WEBHOOK_URL SLACK_WEBHOOK_URL \ RESEND_API_KEY NOTIFY_EMAIL_TO JSONRENDER_ENABLED NOTIFY_MIN_SEVERITY 2>/dev/null -WORK=".pending-notify" +WORK="$AEON_PENDING_DIR/notify-queue" fail=0 pass() { echo "ok - $1"; } bad() { echo "FAIL - $1"; fail=1; } -reset() { rm -rf "$WORK" .notify-sent-hashes; } +reset() { rm -rf "$WORK" "$AEON_PENDING_DIR/notify-sent-hashes"; } # 1. structured message lands in pending with title header reset @@ -110,24 +116,24 @@ fi MK="$(mktemp -d)"; mkdir -p "$MK/memory"; cd "$MK" || exit 1 # 9. muted key suppresses the send -rm -rf .pending-notify .notify-sent-hashes; echo "token-movers:BTC" > memory/mutes.log; : > memory/snoozes.log +rm -rf "$WORK" "$AEON_PENDING_DIR/notify-sent-hashes"; echo "token-movers:BTC" > memory/mutes.log; : > memory/snoozes.log TELEGRAM_BOT_TOKEN=x TELEGRAM_CHAT_ID=123 NOTIFY_DRY_RUN=1 \ bash "$ABS_NOTIFY" "BTC alert that should be muted away entirely" --mute-key "token-movers:BTC" >/dev/null 2>&1 -[ ! -f .pending-notify/tg-payload.jsonl ] && pass "--mute-key muted suppresses" || bad "--mute-key muted suppresses" +[ ! -f "$WORK/tg-payload.jsonl" ] && pass "--mute-key muted suppresses" || bad "--mute-key muted suppresses" # 10. future snooze suppresses -rm -rf .pending-notify .notify-sent-hashes; : > memory/mutes.log +rm -rf "$WORK" "$AEON_PENDING_DIR/notify-sent-hashes"; : > memory/mutes.log printf 'token-movers:ETH:%s\n' "$(( $(date -u +%s) + 3600 ))" > memory/snoozes.log TELEGRAM_BOT_TOKEN=x TELEGRAM_CHAT_ID=123 NOTIFY_DRY_RUN=1 \ bash "$ABS_NOTIFY" "ETH alert snoozed for an hour from now" --mute-key "token-movers:ETH" >/dev/null 2>&1 -[ ! -f .pending-notify/tg-payload.jsonl ] && pass "--mute-key future snooze suppresses" || bad "--mute-key future snooze suppresses" +[ ! -f "$WORK/tg-payload.jsonl" ] && pass "--mute-key future snooze suppresses" || bad "--mute-key future snooze suppresses" # 11. expired snooze delivers -rm -rf .pending-notify .notify-sent-hashes +rm -rf "$WORK" "$AEON_PENDING_DIR/notify-sent-hashes" printf 'token-movers:SOL:%s\n' "$(( $(date -u +%s) - 10 ))" > memory/snoozes.log TELEGRAM_BOT_TOKEN=x TELEGRAM_CHAT_ID=123 NOTIFY_DRY_RUN=1 \ bash "$ABS_NOTIFY" "SOL alert should deliver since snooze expired" --mute-key "token-movers:SOL" >/dev/null 2>&1 -[ -f .pending-notify/tg-payload.jsonl ] && pass "--mute-key expired snooze delivers" || bad "--mute-key expired snooze delivers" +[ -f "$WORK/tg-payload.jsonl" ] && pass "--mute-key expired snooze delivers" || bad "--mute-key expired snooze delivers" cd "$ROOT" || exit 1 rm -rf "$MK" @@ -185,31 +191,53 @@ fi reset -# 16. read-only cwd (codex-style FS-blocked / network-open sandbox): notify must -# fall through to inline delivery, NOT abort at the .pending-notify queue write. -# Regression guard for the set -e abort that made codex read-only skills silently -# drop every notification. See scripts/notify.sh "Non-fatal by design" comment. +# 16. read-only cwd (a read-only skill under the OS sandbox): the queue lives +# OUTSIDE the workspace now, so notify must still succeed AND still queue — +# that is what lets the capture/feed steps see this run's output instead of +# republishing the previous run's file. Regression guard for the stale-digest +# bug and for the set -e abort that once dropped codex notifications entirely. reset NOTIFY_ABS="$PWD/scripts/notify.sh" RO="$(mktemp -d)" chmod 555 "$RO" if ( cd "$RO" && : > .wtest ) 2>/dev/null; then rm -f "$RO/.wtest"; chmod 755 "$RO"; rm -rf "$RO" - pass "read-only cwd fallthrough (skipped: cwd still writable, likely root)" + pass "read-only cwd queues outside the workspace (skipped: cwd still writable, likely root)" else ( cd "$RO" && bash "$NOTIFY_ABS" --severity critical \ "A real notification long enough to clear the probe and severity floors here" ) \ >/dev/null 2>"${RO}.err" ec=$? chmod 755 "$RO"; rm -rf "$RO" - if [ "$ec" -eq 0 ] && grep -q "read-only FS" "${RO}.err"; then - pass "read-only cwd -> inline fallthrough (exit 0, no set -e abort)" + if [ "$ec" -eq 0 ] && [ -n "$(ls -A "$WORK"/*.md 2>/dev/null)" ]; then + pass "read-only cwd still queues (queue is outside the workspace)" else - bad "read-only cwd -> inline fallthrough (exit=$ec; err=$(tr '\n' '|' <"${RO}.err"))" + bad "read-only cwd queue (exit=$ec; queued=$(ls -A "$WORK" 2>/dev/null | tr '\n' ' '); err=$(tr '\n' '|' <"${RO}.err"))" fi rm -f "${RO}.err" fi +# 17. …and if the QUEUE ITSELF is unwritable, notify must fall through to inline +# delivery rather than abort under set -e (the original codex regression). +reset +UNWRITABLE="$(mktemp -d)"; chmod 555 "$UNWRITABLE" +if ( : > "$UNWRITABLE/.wtest" ) 2>/dev/null; then + rm -f "$UNWRITABLE/.wtest"; chmod 755 "$UNWRITABLE"; rm -rf "$UNWRITABLE" + pass "unwritable queue -> inline fallthrough (skipped: dir still writable, likely root)" +else + AEON_PENDING_DIR="$UNWRITABLE/nope" bash "$NOTIFY_ABS" --severity critical \ + "Another real notification long enough to clear the probe and severity floors" \ + >/dev/null 2>"${UNWRITABLE}.err" + ec=$? + chmod 755 "$UNWRITABLE"; rm -rf "$UNWRITABLE" + if [ "$ec" -eq 0 ] && grep -q "unwritable" "${UNWRITABLE}.err"; then + pass "unwritable queue -> inline fallthrough (exit 0, no set -e abort)" + else + bad "unwritable queue fallthrough (exit=$ec; err=$(tr '\n' '|' <"${UNWRITABLE}.err"))" + fi + rm -f "${UNWRITABLE}.err" +fi + reset echo "---" [ "$fail" = "0" ] && echo "ALL PASS" || echo "SOME FAILED"