orchestrate: lint the state file from the hooks, script the Now/Done edits and the updated= marker

docs/how-to/use-orchestrate-mode.md

@@ -83,6 +83,33 @@ new session) and an agent listed under `Now` has no report yet, the
 orchestrator is told, so an agent lost to a crash is noticed rather than
 assumed to be running.
 
+## The state file
+
+The goal's `state.md` is written by the orchestrator only, in one format:
+
+- `## Now` lists what is running, one line per agent, and holds
+  `- (nothing running)` when idle.
+- `## Done` is the last section: one line per finished item, `- YYYY-MM-DD HH:MMZ <text>`,
+  appended in time order. Agent returns and review rounds go to the log.
+- `## Decided` holds rulings only, each ending `(<who>, YYYY-MM-DD)`.
+
+The orchestrator makes the most common edits with `goal.sh` rather than by
+hand: `goal.sh now '<entry>'` records a launch, `goal.sh land <label>
+['<done line>']` records a return (and, with a done line, a finished item;
+`land` drops the entry naming `-> <label> ->` or starting with `<label>`),
+and `goal.sh touch` sets the `updated=` time in the first line after any
+other edit. Nobody types that time.
+
+The Stop and SessionStart hooks run `scripts/state-lint.sh` on the file. It
+reports drift as `lint: <code>: <text>` lines: blank or empty `Now`, a `Now`
+agent whose report already exists, items still pending after they merged or
+closed, events or unattributed lines in `Decided`, `Done` out of order or
+too long, a file over 10 KB or a head over the injected cap, a stale
+`updated=`, and pending sections left unchanged while `Done` grew. The Stop
+hook passes a set of findings on once; the SessionStart block always shows
+them. You can run it yourself:
+`bash <plugin>/scripts/state-lint.sh ~/.claude/orchestrate/<slug>/state.md`.
+
 ## Where things are
 
 | Path | Holds |

plugins/orchestrate/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "orchestrate",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Orchestrate mode: the main session holds the goal, a state file and a map of the work, and delegates detailed work to subagents. One goal folder per goal under ~/.claude/orchestrate/, spanning any number of repositories; hooks keep it current across /clear and do nothing until a goal is started.",
   "author": {
     "name": "Diamond Light Source"

plugins/orchestrate/README.md

@@ -8,7 +8,8 @@ Install, usage and file layout:
 | Part | Path |
 |---|---|
 | Entry skill, its verbs and rules | `skills/orchestrate/SKILL.md`, `skills/orchestrate/references/state-file.md` |
-| Goal lifecycle (start, resume, pause, list, close) and the pointer | `scripts/goal.sh` |
+| Goal lifecycle (start, resume, pause, list, close), the pointer, and the scripted state edits (touch, now, land) | `scripts/goal.sh` |
+| State file lint, run by the Stop and SessionStart hooks | `scripts/state-lint.sh` |
 | Hooks (active only for the pointer's goal, in its launch directory, in the session that owns it) | `hooks/hooks.json`, `scripts/session-start.sh`, `scripts/stop.sh`, `scripts/session-end.sh` |
 | Shared helpers, log rotation, thresholds | `scripts/lib.sh` |
 | Transcript to text, for recovery queries | `scripts/transcript-text.sh` |

plugins/orchestrate/scripts/goal.sh

@@ -12,7 +12,16 @@
 #                                             pointer removed
 #     goal.sh list [--all]                    every goal with its status (* = active);
 #                                             --all lists stopped goals too
+#     goal.sh touch                           set the state marker's updated= to now
+#     goal.sh now '<entry>'                   a launch: add the entry to Now, in place
+#                                             of "(nothing running)"
+#     goal.sh land <label> ['<done line>']    a return: drop the Now entry "-> <label> ->",
+#                                             "(nothing running)" when Now empties; with
+#                                             a done line (the item is finished) append
+#                                             "- YYYY-MM-DD HH:MMZ <done line>" to the end of Done;
+#                                             label - when no agent ran
 #
+# touch, now and land act on the active goal's state file and set updated=.
 # The launch directory is the current directory.
 # start and resume pause the goal that is active, if it is another one.
 # Exit 0 done, 1 refused (the message says why), 2 usage, 3 the user must
@@ -22,7 +31,7 @@ set -uo pipefail
 # shellcheck source=lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 
-usage() { sed -n '5,19p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' >&2; exit 2; }
+usage() { sed -n '5,28p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' >&2; exit 2; }
 die() { printf 'goal.sh: %s\n' "$*" >&2; exit 1; }
 
 dir="$(pwd -P)"
@@ -94,7 +103,88 @@ pause_goal() {
     printf 'paused:   %s\n' "$ACTIVE_SLUG"
 }
 
+# Rewrite the active goal's Now section, and append to Done, in one pass.
+# From the environment: NOW_ADD an entry to add, NOW_DROP a label whose
+# entries go, DONE_ADD a line for the end of Done. Now keeps its entries in
+# order with no blank line among them, holds "- (nothing running)" when
+# empty, and ends with one blank line; Done gains its line after its last
+# entry. Then updated= is set and the new Now printed.
+state_edit() {
+    pointer_read || die "no active goal"
+    goal_paths "$ACTIVE_SLUG"
+    [ -f "$STATE_FILE" ] || die "goal '$SLUG' has no state file"
+    grep -qx '## Now' "$STATE_FILE" || die "$STATE_FILE has no '## Now' line"
+    # The copy keeps the file's mode; awk then replaces its content.
+    cp -p "$STATE_FILE" "$STATE_FILE.tmp" || die "cannot write $STATE_FILE.tmp"
+    awk -v end="$HEAD_END" '
+        function heading(l) { return l ~ /^## / || l == end }
+        # An entry names a label as "-> <label> ->" or by starting with it.
+        function named(l, lab,   p, c) {
+            if (index(l, " -> " lab " -> ")) return 1
+            p = "- " lab; c = substr(l, length(p) + 1, 1)
+            return substr(l, 1, length(p)) == p && (c == "" || c == " " || c == ":")
+        }
+        function close_now(   i) {
+            if (ENVIRON["NOW_ADD"] != "") e[++n] = ENVIRON["NOW_ADD"]
+            if (n == 0) e[++n] = "- (nothing running)"
+            for (i = 1; i <= n; i++) print e[i]
+            print ""; innow = 0
+        }
+        function close_done() {
+            if (ENVIRON["DONE_ADD"] != "") print ENVIRON["DONE_ADD"]
+            printf "%s", blanks; blanks = ""; indone = 0; done = 1
+        }
+        innow && heading($0) { close_now() }
+        indone && heading($0) { close_done() }
+        innow {
+            if ($0 ~ /^[[:space:]]*$/ || $0 == "- (nothing running)") next
+            if (ENVIRON["NOW_DROP"] != "" && named($0, ENVIRON["NOW_DROP"])) next
+            e[++n] = $0; next
+        }
+        indone && /^[[:space:]]*$/ { blanks = blanks $0 "\n"; next }
+        indone { printf "%s", blanks; blanks = ""; print; next }
+        { print }
+        $0 == "## Now" { innow = 1 }
+        $0 == "## Done" || index($0, "## Done ") == 1 { indone = 1 }
+        END {
+            if (innow) close_now()
+            if (indone) close_done()
+            else if (!done && ENVIRON["DONE_ADD"] != "") { print "## Done"; print ENVIRON["DONE_ADD"] }
+        }' "$STATE_FILE" > "$STATE_FILE.tmp" && mv -f "$STATE_FILE.tmp" "$STATE_FILE" \
+        || { rm -f "$STATE_FILE.tmp"; die "cannot rewrite $STATE_FILE"; }
+    state_set_field "$STATE_FILE" updated "$(now_iso)"
+    sed -n '/^## Now$/,/^$/p' "$STATE_FILE"
+}
+
+# Entries in the active goal's Now, not counting the placeholder.
+now_count() {
+    pointer_read && goal_paths "$ACTIVE_SLUG" || return 0
+    sed -n '/^## Now$/,/^## /p' "$STATE_FILE" 2>/dev/null | grep '^- ' | grep -vcx -- '- (nothing running)'
+}
+
 case "$cmd" in
+touch)
+    pointer_read || die "no active goal"
+    goal_paths "$ACTIVE_SLUG"
+    [ -f "$STATE_FILE" ] || die "goal '$SLUG' has no state file"
+    state_set_field "$STATE_FILE" updated "$(now_iso)"
+    head -n 1 "$STATE_FILE"
+    ;;
+now)
+    entry="$*"; entry="${entry//$'\n'/ }"; entry="${entry#- }"
+    [ -n "$entry" ] || usage
+    NOW_ADD="- $entry" state_edit
+    ;;
+land)
+    label="${1:-}"; shift || true
+    [ -n "$label" ] || usage
+    done_line="$*"; done_line="${done_line//$'\n'/ }"
+    [ -n "$done_line" ] && done_line="- $(date -u "+%Y-%m-%d %H:%MZ") ${done_line#- }"
+    n="$(now_count)"
+    NOW_DROP="$label" DONE_ADD="$done_line" state_edit
+    [ "$label" = - ] || [ "$(now_count)" != "$n" ] \
+        || printf 'note:     no Now entry names %s; Now is unchanged\n' "$label"
+    ;;
 start)
     slug="${1:-}"; shift || true
     goal="$*"

plugins/orchestrate/scripts/lib.sh

@@ -164,14 +164,36 @@ state_head() {
     fi
 }
 
+# ---- lint ----------------------------------------------------------------
+# The state file's lint findings (scripts/state-lint.sh), one per line. The
+# lint keeps its section hashes in the goal's .lint.
+state_lint() {
+    bash "$PLUGIN_ROOT/scripts/state-lint.sh" "$STATE_FILE" "$GOAL_LOG" "$GOAL_DIR/.lint" 2>/dev/null
+}
+
+# The findings when their set differs from the set last reported, else
+# nothing; the set is kept in the goal's .lint-last. A finding is keyed by
+# its code and subject (the label, section or #<n>), not its counts or
+# times, so one already reported does not nag as its count grows.
+state_lint_changed() {
+    local found key last="$GOAL_DIR/.lint-last"
+    found="$(state_lint)"
+    key="$(sed -E '/^lint: (now-finished|pending-but-done):/b
+                   s/^(lint: section-stale: ## .*) unchanged .*/\1/; t
+                   s/^(lint: [a-z-]+):.*/\1/' <<<"$found")"
+    [ "$key" = "$(cat "$last" 2>/dev/null)" ] && return 0
+    printf '%s\n' "$key" > "$last" 2>/dev/null
+    printf '%s' "$found"
+}
+
 # ---- the context block ---------------------------------------------------
 # What a session that takes the goal over is told: the goal's paths, a
 # crash check against the previous transcript, Now entries with no report,
-# the scratch report and (unless $3 is no) the state file's head. Printed
+# the state file's lint findings, the scratch report and (unless $3 is no) the state file's head. Printed
 # by SessionStart in the owner, and by Stop on the turn that claims the
 # goal. Needs the goal paths and the hook input.
 goal_context() { # WHO MODE_LINE [yes|no: include the head]
-    local scratch_mb total prev crash="" prev_m state_m lost="" line report rest brief label
+    local scratch_mb total prev crash="" prev_m state_m lost="" lint line report rest brief label
     scratch_mb="$(state_scratch_report)"
     total="$(wc -l < "$STATE_FILE")"
 
@@ -218,6 +240,7 @@ Never /resume it."
     printf '%s\n' "$2"
     [ -n "$crash" ] && printf '\n%s\n' "$crash"
     [ -n "$lost" ] && printf '\n%s' "$lost"
+    lint="$(state_lint)" && [ -n "$lint" ] && printf '\nState file lint (fix what is wrong; a finding you judged wrong may stay):\n%s\n' "$lint"
     [ -n "$scratch_mb" ] && printf '\nScratch in %s/*/scratch totals %s MB (report threshold %s MB). Offer the user a cleanup: list each scratch dir with its size and whether its goal is stopped; delete only what they name.\n' "$ROOT" "$scratch_mb" "$SCRATCH_REPORT_MB"
     if [ "${3:-yes}" = yes ]; then
         printf '\n--- head of %s ---\n' "$STATE_FILE"

plugins/orchestrate/scripts/session-start.sh

@@ -4,7 +4,8 @@
 # Only in orchestrate mode (the pointer names a goal, this session runs in
 # its launch directory and owns the goal): re-seeds a cold context with a
 # small block - the goal's paths, a crash check against the previous
-# transcript, and the state file's HEAD only. The body is read on demand.
+# transcript, the state file's lint findings, and its HEAD only. The body
+# is read on demand.
 #
 # A /clear in the owning session starts a new session id; the owner's
 # SessionEnd left session=cleared:<old id>, and this hook claims it when

plugins/orchestrate/scripts/state-lint.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+# Lint a goal's state file for the drift that edits made from memory leave
+# behind. Run by the Stop and SessionStart hooks; also usable by hand.
+#
+#     state-lint.sh <state.md> [<log.md>] [<sidecar>]
+#
+# Prints one line per finding, "lint: <code>: <text>", and nothing when the
+# file is clean. Always exits 0: a hook must never block. Reads only the
+# files given. <log.md> lets done-order place Done entries that carry no
+# YYYY-MM-DD HH:MMZ stamp. <sidecar> (the goal's .lint) keeps each pending section's
+# hash and the Done count when it last changed, for section-stale; it is the
+# only file written, so the orchestrator stays the state file's one writer.
+#
+# The format checked is the one SKILL.md describes and goal.sh now/land
+# write:
+#   ## Now    `- (nothing running)` or one entry per line, no blank lines
+#   ## Done   the last section; one line per finished item, `- YYYY-MM-DD HH:MMZ <text>`,
+#             appended in time order
+#   ## Decided  rulings only, each ending `(<who>, YYYY-MM-DD)`
+# Thresholds may come from the environment (the tests set them).
+
+set -uo pipefail
+# shellcheck source=lib.sh
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+f="${1:?usage: state-lint.sh <state.md> [<log.md>] [<sidecar>]}"; log="${2:-}"; side="${3:-}"
+[ -f "$f" ] || exit 0
+goal_dir="$(cd "$(dirname "$f")" && pwd)"
+MAX_BYTES="${STATE_LINT_MAX_BYTES:-10240}"      # the whole file, read on demand
+MAX_DONE="${STATE_LINT_MAX_DONE:-20}"           # Done entries: one per item
+STALE_DONE="${STATE_LINT_STALE_DONE:-5}"        # Done growth a pending section may ignore
+MARKER_SLACK="${STATE_LINT_MARKER_SLACK:-900}"  # seconds updated= may trail the file
+
+out() { printf 'lint: %s\n' "$*"; }
+
+# The lines of "## NAME" (a heading may carry a suffix) up to the next
+# heading or the end-of-head line.
+section() {
+    awk -v s="## $1" -v end="$HEAD_END" '
+        $0 == s || index($0, s " ") == 1 { on = 1; next }
+        on && (/^## / || $0 == end) { exit }
+        on' "$f"
+}
+
+# ---- Now ---------------------------------------------------------------------
+now="$(section Now)"
+# One blank line before the next heading is the separator; any other is drift.
+blank="$(section Now | awk '/^[[:space:]]*$/ { n++ } END { print (n > 1 ? n - 1 : 0) }')"
+[ "$blank" -gt 0 ] && out "now-blank: $blank blank line(s) inside ## Now; one entry per line, none blank"
+grep -q '^- ' <<<"$now" || out "now-empty: ## Now has no entry; when nothing runs it holds '- (nothing running)'"
+while IFS= read -r line; do
+    case "$line" in "- ["*" -> "*" -> "*" -> "*) ;; *) continue ;; esac
+    report="${line##* -> }"; report="${report%% (*}"; report="${report/#\~/$HOME}"
+    case "$report" in /*) ;; *) report="$goal_dir/$report" ;; esac
+    rest="${line% -> *}"; rest="${rest% -> *}"; label="${rest##* -> }"
+    [ -e "$report" ] && out "now-finished: $label has its report; it has returned: goal.sh land $label"
+done <<<"$now"
+
+# ---- finished numbers still pending ------------------------------------------
+# A #N is finished when a Done or Decided line says it merged or closed. Next,
+# Awaiting user or Queue still naming one is worth a look (a follow-up may
+# legitimately name it; the Stop hook reports a finding once).
+finished="$( { section Done; section Decided; } \
+    | grep -oiE '#[0-9]+[^#]{0,25}(merged|closed)|(merged|closed)[^#]{0,12}#[0-9]+' \
+    | grep -oE '#[0-9]+' | sort -u)"
+for s in Next "Awaiting user" Queue; do
+    for n in $(section "$s" | grep -oE '#[0-9]+' | sort -u); do
+        grep -qxF -- "$n" <<<"$finished" || continue
+        out "pending-but-done: ## $s names $n, which Done or Decided records as merged or closed"
+    done
+done
+
+# ---- Decided: rulings only ---------------------------------------------------
+dec="$(section Decided | grep '^- ')"
+ev="$(grep -cE 'MERGED|CLOSED|APPROVE|CHANGES|^- Opened |- done [0-9]|\((opus|sonnet|haiku|fable)\)[: ]' <<<"$dec")"
+[ "$ev" -gt 0 ] && out "decided-events: $ev ## Decided line(s) read as events (merges, reviews, agent returns); they belong in Done or the log"
+nowho="$(grep -cvE '\([^()]*, *(20[0-9][0-9]|<YYYY-MM-DD>)[^()]*\)' <<<"$dec")"
+[ -n "$dec" ] && [ "$nowho" -gt 0 ] && out "decided-unattributed: $nowho ## Decided line(s) lack '(<who>, YYYY-MM-DD)'; a ruling names who made it, anything else is not a ruling"
+
+# ---- Done: appended in time order --------------------------------------------
+# A stamped entry (- YYYY-MM-DD HH:MMZ) earlier than the one above it was
+# inserted, not appended. An unstamped entry is placed by its first word,
+# when the log timestamps that label.
+inv="$(section Done | awk -v lf="$log" '
+    BEGIN { if (lf != "") while ((getline l < lf) > 0)
+                if (l ~ /^- [0-9]{4}-[0-9][0-9]-[0-9][0-9]T[0-9:]+Z? [a-z0-9-]+:/) {
+                    split(l, a, " "); lab = a[3]; sub(/:$/, "", lab)
+                    if (!(lab in ts)) ts[lab] = a[2] } }
+    /^- [0-9]{4}-[0-9][0-9]-[0-9][0-9] [0-2][0-9]:[0-5][0-9]Z / {
+        m = $2 " " $3
+        if (pm != "" && m < pm) n++
+        pm = m; next }
+    /^- / { if ($2 in ts) { if (pl != "" && ts[$2] < pl) n++; pl = ts[$2] } }
+    END { print n + 0 }')"
+[ "$inv" -gt 0 ] && out "done-order: $inv ## Done entr(y/ies) sit above an older one; append each at the end of Done"
+
+# ---- size ----------------------------------------------------------------------
+bytes="$(wc -c < "$f")"
+[ "$bytes" -gt "$MAX_BYTES" ] && out "size: $bytes bytes (> $MAX_BYTES); collapse each finished item to one Done line and move the detail to the log"
+done_n="$(section Done | grep -c '^- ')"
+[ "$done_n" -gt "$MAX_DONE" ] && out "done-long: $done_n ## Done entries (> $MAX_DONE); one line per finished item, not per agent return"
+hn="$(grep -n -F -x "$HEAD_END" "$f" | head -1 | cut -d: -f1)"
+if [ -n "$hn" ]; then
+    hb="$(head -n "$((hn - 1))" "$f" | wc -c)"
+    { [ "$((hn - 1))" -gt "$STATE_HEAD_CAP_LINES" ] || [ "$hb" -gt "$STATE_HEAD_CAP_BYTES" ]; } \
+        && out "head-long: the head is $((hn - 1)) lines, $hb bytes (cap $STATE_HEAD_CAP_LINES lines, $STATE_HEAD_CAP_BYTES bytes); the injected copy is cut short"
+fi
+
+# ---- marker ------------------------------------------------------------------
+upd="$(state_field "$f" updated)"
+u="$(date -u -d "$upd" +%s 2>/dev/null || echo 0)"
+if [ -n "$upd" ] && [ "$u" -gt 0 ]; then
+    m="$(stat -c %Y "$f")"; t="$(date -u +%s)"
+    if [ "$((m - u))" -gt "$MARKER_SLACK" ] || [ "$((u - t))" -gt 300 ]; then
+        out "marker-stale: updated=$upd does not match the file's last change; run goal.sh touch, never type the time"
+    fi
+fi
+
+# ---- pending sections left alone while work moved ------------------------------
+if [ -n "$side" ]; then
+    touch "$side" 2>/dev/null || exit 0
+    for s in Next "Awaiting user" Queue Map; do
+        # An empty section, or one holding only a placeholder such as
+        # "- (none)", has nothing to go stale.
+        section "$s" | grep '^- ' | grep -qv '^- ([^()]*)$' || continue
+        h="$(section "$s" | md5sum | cut -c1-12)"; key="${s// /_}"
+        prev="$(grep "^$key " "$side" | tail -1)"
+        # A new hash, or a Done that shrank (items collapsed), resets the count.
+        if [ -z "$prev" ] || [ "$(cut -d' ' -f2 <<<"$prev")" != "$h" ] \
+            || [ "$(cut -d' ' -f3 <<<"$prev")" -gt "$done_n" ]; then
+            { grep -v "^$key " "$side"; printf '%s %s %s\n' "$key" "$h" "$done_n"; } > "$side.tmp" \
+                && mv -f "$side.tmp" "$side"
+            continue
+        fi
+        since="$(cut -d' ' -f3 <<<"$prev")"
+        [ "$((done_n - since))" -ge "$STALE_DONE" ] || continue
+        out "section-stale: ## $s unchanged while Done grew by $((done_n - since)); re-read it and fix each line it falsifies"
+        # Reported: count again from here, so a section confirmed as it is
+        # is named again only after another $STALE_DONE items.
+        { grep -v "^$key " "$side"; printf '%s %s %s\n' "$key" "$h" "$done_n"; } > "$side.tmp" \
+            && mv -f "$side.tmp" "$side"
+    done
+fi
+exit 0