diff --git a/cmd/marvel/main.go b/cmd/marvel/main.go index 47b961b..7209145 100644 --- a/cmd/marvel/main.go +++ b/cmd/marvel/main.go @@ -17,6 +17,7 @@ import ( "golang.org/x/term" + "github.com/arcavenae/marvel/internal/admission" "github.com/arcavenae/marvel/internal/api" "github.com/arcavenae/marvel/internal/config" "github.com/arcavenae/marvel/internal/daemon" @@ -406,6 +407,7 @@ Examples: marvel events --kind agent.tool.call # what the agents are doing marvel events --kind agent.session.ended # per-session cost and timing marvel events --kind context.limit-unresolved # why a CTX% cell is blank + marvel events --kind admission.refused # spawns a team budget refused marvel events --warnings # only warning-severity events marvel --cluster desk events # remote daemon via mrvl://`, RunE: func(cmd *cobra.Command, args []string) error { @@ -539,7 +541,7 @@ func getCmd() *cobra.Command { var watchSec string cmd := &cobra.Command{ Use: "get ", - Short: "List resources (sessions, teams, workspaces, endpoints, policies)", + Short: "List resources (sessions, teams, workspaces, endpoints, policies, budgets)", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("watch") { @@ -593,6 +595,8 @@ func getResources(resourceType string) error { return printEndpoints(resp.Result) case "policies", "policy": return printPolicies(resp.Result) + case "budgets", "budget": + return printBudgets(resp.Result) default: fmt.Println(string(resp.Result)) } @@ -1738,6 +1742,66 @@ func printPolicies(data json.RawMessage) error { return w.Flush() } +// printBudgets renders `marvel get budgets`: one row per declared budget +// dimension, with what has been observed against it and that dimension's +// state (ok, at-ceiling, refusing, unmetered). The surface that answers +// "which dimension tripped and by how much" — the event says a refusal +// happened, this says where the team stands. at-ceiling and refusing are +// deliberately separate: a team sized at its ceiling refuses nothing. +func printBudgets(data json.RawMessage) error { + var rows []admission.Row + if err := json.Unmarshal(data, &rows); err != nil { + return err + } + if len(rows) == 0 { + fmt.Println("no teams declare a budget") + return nil + } + fmt.Print(renderBudgetTable(rows)) + return nil +} + +// renderBudgetTable is the pure renderer, split out for the same reason +// renderSessionTable is: the table's absence handling is worth asserting +// without a daemon. +func renderBudgetTable(rows []admission.Row) string { + sort.Slice(rows, func(i, j int) bool { + if rows[i].Workspace != rows[j].Workspace { + return rows[i].Workspace < rows[j].Workspace + } + if rows[i].Team != rows[j].Team { + return rows[i].Team < rows[j].Team + } + return rows[i].Dimension < rows[j].Dimension + }) + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 0, 4, 2, ' ', 0) + _, _ = fmt.Fprintf(w, "WORKSPACE\tTEAM\tDIMENSION\tLIMIT\tOBSERVED\tHEADROOM\tSTATE\tWINDOW\tNOTE\n") + for _, r := range rows { + note := r.Note + if note == "" { + note = "-" + } + _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%d\t%d\t%s\t%s\t%s\n", + r.Workspace, r.Team, r.Dimension, r.Limit, r.Observed, r.Headroom, + r.State, formatWindow(r.Window), note) + } + _ = w.Flush() + return buf.String() +} + +// formatWindow renders how long a cumulative figure has been accumulating. +// A count-shaped dimension has no window and renders a dash, as does a +// dimension nothing has been measured for yet. The accountant is in-memory, +// so this span restarts with the daemon; showing it is what keeps a reset +// visible instead of silent. +func formatWindow(t time.Time) string { + if t.IsZero() { + return "-" + } + return time.Since(t).Round(time.Second).String() +} + // stripComments removes shell-style comments from CLI arguments. // Everything from a bare "#" argument onward is dropped, so that // inline notes work: ./marvel shift test/squad # replace all workers diff --git a/cmd/marvel/render_test.go b/cmd/marvel/render_test.go index 2dec365..c8292d9 100644 --- a/cmd/marvel/render_test.go +++ b/cmd/marvel/render_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/arcavenae/marvel/internal/admission" "github.com/arcavenae/marvel/internal/api" ) @@ -210,3 +211,99 @@ func TestSortSessionsByCPUAndRSS(t *testing.T) { t.Errorf("rss desc put %q first, want a", sessions[0].Name) } } + +// budgetRow returns one rendered row keyed by column name, so a test asserts +// on cells rather than on whitespace. Every cell in this table is non-empty +// (absence renders as a dash), so splitting on fields is unambiguous. +func budgetRow(t *testing.T, table, dimension string) map[string]string { + t.Helper() + lines := strings.Split(strings.TrimRight(table, "\n"), "\n") + header := strings.Fields(lines[0]) + for _, line := range lines[1:] { + fields := strings.Fields(line) + if len(fields) < len(header) || fields[2] != dimension { + continue + } + out := make(map[string]string, len(header)) + for i, h := range header { + if h == "NOTE" { + // The note is prose and holds spaces, so it takes the rest. + out[h] = strings.Join(fields[i:], " ") + break + } + out[h] = fields[i] + } + return out + } + t.Fatalf("no %s row in:\n%s", dimension, table) + return nil +} + +// TestRenderBudgetTable covers `marvel get budgets`, the surface that answers +// which dimension tripped and by how much. Its absence handling matters for +// the same reason CTX%'s does: a token figure nothing has measured must not +// render as headroom the operator does not have. +func TestRenderBudgetTable(t *testing.T) { + rows := []admission.Row{ + { + Workspace: "fanout", Team: "crew", Dimension: api.DimMaxTokens, + Limit: 2000000, Observed: 412118, Headroom: 1587882, + State: admission.RowOK, Window: time.Now().UTC().Add(-14 * time.Minute), + Note: "partial: some sessions unobserved, so this is a floor", + }, + { + Workspace: "fanout", Team: "crew", Dimension: api.DimMaxSessions, + Limit: 6, Observed: 6, Headroom: 0, State: admission.RowAtCeiling, + }, + } + table := renderBudgetTable(rows) + + // Sorted by dimension within a team, so max_sessions comes first. + lines := strings.Split(strings.TrimRight(table, "\n"), "\n") + if len(lines) != 3 { + t.Fatalf("got %d line(s), want a header and two rows:\n%s", len(lines), table) + } + if !strings.Contains(lines[1], string(api.DimMaxSessions)) { + t.Errorf("first row = %q, want max_sessions (rows sort by dimension)", lines[1]) + } + + sessions := budgetRow(t, table, string(api.DimMaxSessions)) + // No headroom is not a refusal: a team whose declared replicas equal its + // ceiling sits here permanently and refuses nothing. + if sessions["STATE"] != admission.RowAtCeiling { + t.Errorf("STATE = %q, want %q with no headroom and nothing refused", sessions["STATE"], admission.RowAtCeiling) + } + if sessions["HEADROOM"] != "0" { + t.Errorf("HEADROOM = %q, want 0", sessions["HEADROOM"]) + } + // A count dimension accumulates over no window, so it must render + // absence rather than invent one. + if sessions["WINDOW"] != "-" { + t.Errorf("WINDOW = %q for a count dimension, want a dash", sessions["WINDOW"]) + } + if sessions["NOTE"] != "-" { + t.Errorf("NOTE = %q with nothing to say, want a dash", sessions["NOTE"]) + } + + tokens := budgetRow(t, table, string(api.DimMaxTokens)) + if tokens["HEADROOM"] != "1587882" { + t.Errorf("HEADROOM = %q, want 1587882", tokens["HEADROOM"]) + } + if tokens["WINDOW"] == "-" { + t.Errorf("WINDOW = %q for a cumulative dimension, want the elapsed span", tokens["WINDOW"]) + } + if !strings.Contains(tokens["NOTE"], "partial") { + t.Errorf("NOTE = %q, want the partial-total notice", tokens["NOTE"]) + } +} + +// TestFormatWindow: a dimension that accumulates over no window renders +// absence, never a zero duration that would read as "just reset". +func TestFormatWindow(t *testing.T) { + if got := formatWindow(time.Time{}); got != "-" { + t.Errorf("formatWindow(zero) = %q, want %q", got, "-") + } + if got := formatWindow(time.Now().UTC().Add(-90 * time.Second)); got == "-" { + t.Errorf("formatWindow(90s ago) = %q, want an elapsed duration", got) + } +} diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 150e2d5..0cbbc63 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -440,3 +440,62 @@ marvel describe session dev/squad-worker-g1-0 Lower the `failure_threshold` or increase the `timeout` if agents need more time to initialize. + +### A spawn was refused + +Two different conditions hold a role back, and they look alike from the +outside. Tell them apart first: + +```bash +marvel get budgets # where each ceiling stands +marvel events --kind admission.refused # a budget refused it +marvel events --kind health.crashloop-backoff # a crash loop is cooling +marvel describe team fanout/crew # the declared budget +``` + +A budget refusal names its arithmetic, so the fix is usually visible in the +message. `marvel get budgets` gives the standing picture: + +``` +WORKSPACE TEAM DIMENSION LIMIT OBSERVED HEADROOM STATE WINDOW NOTE +fanout crew max_sessions 6 6 0 at-ceiling - - +fanout crew max_tokens 2000000 412118 1587882 ok 14m3s partial: some sessions unobserved, so this is a floor +``` + +Read the STATE column carefully, because two of its values look alike and +mean different things: + +| STATE | Meaning | +|---|---| +| `ok` | Headroom left. | +| `at-ceiling` | No headroom for growth, and nothing is being refused. This is the resting state of a healthy team, since declared replicas are allowed to equal the ceiling and replacing a crashed replica is exempt. | +| `refusing` | A refusal is standing right now: the reconciler is holding a role back, and the NOTE column carries the arithmetic. Cross-check with `marvel describe team` (`Admission.held`) and `marvel events --kind admission.refused`. | +| `unmetered` | Nothing has been measured for this dimension yet, so the figure is absence rather than zero. | + +A session row can also read OBSERVED above LIMIT with a `shift` note. That +is a rotation in flight: the new generation runs beside the old, and a +session ceiling exempts the overlap. It resolves itself when draining +finishes. + +Two ways out of a session ceiling: raise `max_sessions` in the manifest and +re-apply, or free headroom with `marvel scale ... --replicas N-1` (a +scale-down is never refused). Either takes effect on the next reconcile +tick, within a couple of seconds. There is no clear command and no resume +verb, because the condition is recomputed from live state every tick. + +A token ceiling has no shedding move: retired spend stays counted, since a +fan-out's cost is mostly in sessions that already exited. Killing sessions +does not un-spend tokens. Raise `max_tokens` and re-apply. + +Two notes in the table matter for trust. `partial` means some contributing +session was never observed, so the figure is a floor rather than a small +number, which is why a refusal against it is still sound while an admission +carries a caveat. `suspect` means the meter caught a cumulation violation +and the figure may be inflated, so check `marvel daemon logs` before raising +a ceiling on its account. + +One limit to know: `max_tokens` counts from when accounting started, and the +meter lives in the daemon's memory. A daemon restart or `marvel daemon +reexec` resets the window, and the daemon says so in its log at startup for +every team that declares one. The `WINDOW` column dropping back near zero is +the visible signal that it happened. diff --git a/docs/user-guide.md b/docs/user-guide.md index fa43041..c450b92 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -170,6 +170,90 @@ adapter (claude, forestage, generic). They coexist in the same team. Claude Code, workers run forestage with personas, and a monitor runs a shell script for health scraping. +### With a team budget + +```yaml +workspace: + name: fanout + +teams: + - name: crew + budget: + max_sessions: 6 + max_tokens: 2000000 + on_unmeasured: admit # optional; this is the default + roles: + - name: crew + replicas: 3 + runtime: + image: claude + command: claude + mode: headless + prompt: "review the diff" +``` + +A budget is a ceiling marvel will refuse to cross. It is the only thing +that turns a measured number into a refusal, so **a team with no budget +block declares no gate** and behaves exactly as it did before budgets +existed. + +Two dimensions are enforced today. + +`max_sessions` caps live sessions across the whole team: every role, plus +ad-hoc `marvel run` sessions attributed to the team. It is counted from +marvel's own records, so it is exact before a spawn and survives a daemon +restart. The sum of your declared replicas has to fit under it, and a +manifest that declares more is refused at parse time; so is a +`marvel scale` that would push the sum over it. + +**A rolling shift is the one exemption.** `marvel shift` starts the new +generation beside the old and drains the old afterwards, so live sessions +can reach twice a role's replicas for the length of the rotation, and the +ceiling does not refuse it: replacing a session is not growth, and refusing +the overlap would mean a team sitting at its ceiling could never rotate. +`marvel get budgets` says so in the NOTE column while a shift is running. +If you are sizing `max_sessions` against a hard external concurrency quota, +size for that overlap, or avoid shifting a team that sits at its ceiling. + +`max_tokens` caps prompt plus output plus reasoning tokens, and it counts +only the sessions marvel can observe: token usage arrives on a harness +stream, and only a stream-capable harness in headless mode publishes one +(claude, codex, opencode today). A team where no role can report is refused +at apply time rather than accepted with a ceiling nothing can ever report +against; a `generic` role declaring `mode: headless` does not count, because +that adapter has no stream to read. **It is a since-accounting-started +budget** — the meter lives in the daemon's memory, so the window restarts +with the daemon and with `marvel daemon reexec`. Every figure marvel prints +for it carries that window. + +`on_unmeasured` decides what a declared clause does when the meter cannot +answer at all, which happens before a headless role's first request lands. +The default admits and says so with an `admission.unmeasured` event, on the +grounds that you declared a ceiling on a dimension rather than "refuse when +unmeasurable". Set it to `refuse` if you want the fail-closed posture. + +A refusal is never silent. The command fails with the arithmetic, the event +ring records it, and `marvel get budgets` shows where the team stands: + +```bash +marvel scale fanout/crew --role crew --replicas 40 +# Error: fanout/crew: refused 37 of 37 spawn(s) for role crew: 3 live + 37 +# requested sessions exceeds max_sessions=6 (trigger=scale). Nothing changed; +# raise the budget in the manifest or free headroom first + +marvel get budgets +marvel events --kind admission.refused +``` + +Scaling down is never refused, and replacing a crashed replica is never +refused: a budget is a ceiling on what you ask for, not a brake on +recovery. + +**When to use:** Any team an agent or a script can scale, where a runaway +fan-out would collide on one API quota. See +`examples/demo-act4-budget.yaml` for a walkthrough that needs no model +auth. + ## Managing sessions ### List sessions diff --git a/examples/demo-act4-budget.toml b/examples/demo-act4-budget.toml new file mode 100644 index 0000000..369937e --- /dev/null +++ b/examples/demo-act4-budget.toml @@ -0,0 +1,87 @@ +# Marvel demo — Act 4, admission against a team-declared budget. +# +# Three crew members under a six-session ceiling. The budget block is the +# operator's declaration that a metered value may refuse a spawn for this +# team: no budget block means no gate, and marvel behaves exactly as it did +# before the feature existed. +# +# The beat is the fan-out. Ask for forty replicas and the verb fails, the +# store keeps the number that is still true, and the event ring carries the +# arithmetic. Nothing is left in a state that can never be satisfied. +# +# `sleep` stands in for any long-running agent process. Every beat below is +# deterministic and needs no model auth, because a session ceiling is counted +# from the store rather than measured from a harness stream. +# +# Requires: `sleep` on PATH (always present). +# +# Setup: +# just build +# ./bin/marvel stop --teardown +# rm -f ~/.marvel/state/marvel.bolt +# ./bin/marvel daemon & +# +# 1. Declared and satisfied. Nothing refused. +# ./bin/marvel work examples/demo-act4-budget.toml +# ./bin/marvel get sessions # three crew, running +# ./bin/marvel get budgets # limit 6, observed 3, headroom 3, ok +# ./bin/marvel events --kind admission.refused # "no events" +# +# 2. The fan-out is refused at the verb. +# ./bin/marvel scale fanout/crew --role crew --replicas 40 +# # Error: fanout/crew: refused 37 of 37 spawn(s) for role crew: 3 live + +# # 37 requested sessions exceeds max_sessions=6 (trigger=scale). Nothing +# # changed; raise the budget in the manifest or free headroom first +# ./bin/marvel get teams # REPLICAS still 3 +# ./bin/marvel get sessions # still three; no retry storm +# ./bin/marvel events --kind admission.refused # the arithmetic, once +# +# 3. Which dimension, and by how much. +# ./bin/marvel get budgets # per-dimension limit/observed/headroom +# ./bin/marvel describe team fanout/crew # the declared budget, verbatim +# +# 4. The declaration clause, before any daemon state is touched. +# # edit this file: replicas = 40, leave max_sessions = 6 +# ./bin/marvel work examples/demo-act4-budget.toml +# # Error: parse manifest: parse manifest: team[0] declares 40 replicas +# # across 1 role(s) but budget.max_sessions is 6 +# # (The prefix is doubled because handleApply re-wraps an error that is +# # already qualified. Pre-existing, and every manifest validation rule +# # prints it that way.) +# +# 5. Recovery is one command. +# # edit this file: max_sessions = 40 +# ./bin/marvel work examples/demo-act4-budget.toml +# ./bin/marvel get sessions # forty crew +# +# 5b (optional, auth-dependent, spends real quota). A token ceiling is +# measured from harness streams, so it needs a headless role on a real model. +# Set a low max_tokens on examples/mixed-adapters.toml and watch +# `marvel get budgets` fill in the OBSERVED and WINDOW columns, then refuse. +# Note that max_tokens counts from when accounting started: the meter is +# in-memory, so the window restarts with the daemon. +# +# Cleanup: +# ./bin/marvel stop --teardown + +[workspace] +name = "fanout" + +[[team]] +name = "crew" + + [team.budget] + # Live sessions across the whole team: every role, plus ad-hoc + # `marvel run` sessions attributed to the team. A rolling shift is exempt, + # so live can read up to twice a role's replicas while the new generation + # runs beside the old. + max_sessions = 6 + + [[team.role]] + name = "crew" + replicas = 3 + + [team.role.runtime] + image = "generic" + command = "sleep" + args = ["3600"] diff --git a/examples/demo-act4-budget.yaml b/examples/demo-act4-budget.yaml new file mode 100644 index 0000000..90895a5 --- /dev/null +++ b/examples/demo-act4-budget.yaml @@ -0,0 +1,84 @@ +# Marvel demo — Act 4, admission against a team-declared budget. +# +# Three crew members under a six-session ceiling. The budget block is the +# operator's declaration that a metered value may refuse a spawn for this +# team: no budget block means no gate, and marvel behaves exactly as it did +# before the feature existed. +# +# The beat is the fan-out. Ask for forty replicas and the verb fails, the +# store keeps the number that is still true, and the event ring carries the +# arithmetic. Nothing is left in a state that can never be satisfied. +# +# `sleep` stands in for any long-running agent process. Every beat below is +# deterministic and needs no model auth, because a session ceiling is counted +# from the store rather than measured from a harness stream. +# +# Requires: `sleep` on PATH (always present). +# +# Setup: +# just build +# ./bin/marvel stop --teardown +# rm -f ~/.marvel/state/marvel.bolt +# ./bin/marvel daemon & +# +# 1. Declared and satisfied. Nothing refused. +# ./bin/marvel work examples/demo-act4-budget.yaml +# ./bin/marvel get sessions # three crew, running +# ./bin/marvel get budgets # limit 6, observed 3, headroom 3, ok +# ./bin/marvel events --kind admission.refused # "no events" +# +# 2. The fan-out is refused at the verb. +# ./bin/marvel scale fanout/crew --role crew --replicas 40 +# # Error: fanout/crew: refused 37 of 37 spawn(s) for role crew: 3 live + +# # 37 requested sessions exceeds max_sessions=6 (trigger=scale). Nothing +# # changed; raise the budget in the manifest or free headroom first +# ./bin/marvel get teams # REPLICAS still 3 +# ./bin/marvel get sessions # still three; no retry storm +# ./bin/marvel events --kind admission.refused # the arithmetic, once +# +# 3. Which dimension, and by how much. +# ./bin/marvel get budgets # per-dimension limit/observed/headroom +# ./bin/marvel describe team fanout/crew # the declared budget, verbatim +# +# 4. The declaration clause, before any daemon state is touched. +# # edit this file: replicas: 40, leave max_sessions: 6 +# ./bin/marvel work examples/demo-act4-budget.yaml +# # Error: parse manifest: parse manifest: team[0] declares 40 replicas +# # across 1 role(s) but budget.max_sessions is 6 +# # (The prefix is doubled because handleApply re-wraps an error that is +# # already qualified. Pre-existing, and every manifest validation rule +# # prints it that way.) +# +# 5. Recovery is one command. +# # edit this file: max_sessions: 40 +# ./bin/marvel work examples/demo-act4-budget.yaml +# ./bin/marvel get sessions # forty crew +# +# 5b (optional, auth-dependent, spends real quota). A token ceiling is +# measured from harness streams, so it needs a headless role on a real model. +# Set a low max_tokens on examples/mixed-adapters.yaml and watch +# `marvel get budgets` fill in the OBSERVED and WINDOW columns, then refuse. +# Note that max_tokens counts from when accounting started: the meter is +# in-memory, so the window restarts with the daemon. +# +# Cleanup: +# ./bin/marvel stop --teardown + +workspace: + name: fanout + +teams: + - name: crew + budget: + # Live sessions across the whole team: every role, plus ad-hoc + # `marvel run` sessions attributed to the team. A rolling shift is + # exempt, so live can read up to twice a role's replicas while the new + # generation runs beside the old. + max_sessions: 6 + roles: + - name: crew + replicas: 3 + runtime: + image: generic + command: sleep + args: ["3600"] diff --git a/internal/admission/admission.go b/internal/admission/admission.go new file mode 100644 index 0000000..267ce8d --- /dev/null +++ b/internal/admission/admission.go @@ -0,0 +1,530 @@ +package admission + +import ( + "fmt" + "strings" + "time" + + "github.com/arcavenae/marvel/internal/api" +) + +// Snapshot is the measured state one Check evaluates. The caller gathers +// it, so the arithmetic reads no store and no meter. +// +// Absence is carried by flags, never by a zero value. internal/usage warns +// in as many words that a gate reading an unresolved measurement as zero +// admits everything, so TokensObserved is meaningless unless TokensMetered +// is true. +type Snapshot struct { + Workspace string + Team string + + // LiveSessions counts sessions whose State.CountsAsAlive(), across the + // whole team. + LiveSessions int + // DeclaredSessions is the sum of Replicas across the team's roles. + DeclaredSessions int + + // TokensObserved is the team's layout-normalized prompt + output + + // reasoning tokens. Valid only when TokensMetered. + TokensObserved int + // TokensMetered is false when no Reader was available at all. + TokensMetered bool + // TokensPartial marks an incomplete total: some contributing session + // was never observed, so the figure is a floor. + TokensPartial bool + // TokensSeen is how many sessions (live plus ended) the meter knows + // about for this team. Zero with a zero Since means nothing has been + // measured yet, which is distinct from a team that spent nothing. + TokensSeen int + // TokensSuspect marks a total the meter itself distrusts (a cumulative + // feed read as levels inflates spend). Surfaced as a note rather than + // changing the decision. + TokensSuspect bool + // Since is when accounting for this team began. Zero means never. + Since time.Time +} + +// Request is the spawn under consideration. +type Request struct { + Role string + // Want is how many sessions this action would add. A request that adds + // nothing is never refused, which is what keeps a scale-down and a + // no-op apply out of the gate. Mid-flight revocation (enforcement locus + // 3) will ask "are we over right now" through this same arithmetic with + // that guard lifted. + Want int + Kind Kind + // Overlap marks sessions that are transient duplicates of sessions + // already counted, as a shift's new generation is beside the old. + // ShapeCount clauses skip an overlap; ShapeCumulative clauses do not. + Overlap bool + // AllowPartial lets the caller take headroom instead of nothing. The + // reconciler sets it, because convergence is best-effort and 2 of 5 is + // better for the operator than 0 of 5. The synchronous verbs do not: a + // declaration is operator intent, and marvel silently applying 6 when + // the operator asked for 40 would be marvel editing intent. + AllowPartial bool +} + +// Kind separates raising what a team declares from converging on what it +// already declared. +type Kind string + +const ( + // Growth raises what the team declares: apply, scale up, ad-hoc run, + // shift. + Growth Kind = "growth" + // Repair converges live sessions toward an already-admitted declared + // count. Count clauses admit repair under the R1 invariant; cumulative + // clauses are never evaluated on this path (R2). + Repair Kind = "repair" +) + +// Trigger names the operator action or loop that asked, for the event and +// the error text. +type Trigger string + +const ( + TriggerApply Trigger = "apply" + TriggerScale Trigger = "scale" + TriggerRun Trigger = "run" + TriggerShift Trigger = "shift" + TriggerReconcile Trigger = "reconcile" +) + +// Decision is the verdict's outcome. +type Decision string + +const ( + Admit Decision = "admit" + Refuse Decision = "refuse" + // Indeterminate means a declared clause could not be evaluated. It is + // never silently collapsed into Admit or Refuse: api.Budget.Unmeasured + // decides what happens, and either way an event names the clause. + Indeterminate Decision = "indeterminate" +) + +// ClauseState is one dimension's reading against its ceiling. +type ClauseState string + +const ( + Within ClauseState = "within" + Exceeded ClauseState = "exceeded" + Unmeasured ClauseState = "unmeasured" +) + +// ClauseResult is one dimension's evaluation. +type ClauseResult struct { + Dimension api.Dimension + // Shape is carried so the renderer reads a clause by shape rather than + // by naming individual dimensions, the same reason overlap and repair + // exemptions are shape properties. + Shape api.Shape + State ClauseState + Used int + Limit int + // Adds is what this request would add to Used. Always 0 for cumulative + // clauses: pricing "what will K more of these cost" needs + // usage.Baseline, which is deliberately unimplemented, and fabricating + // an estimate from no history is the failure that contract forbids. + Adds int + Unit string + // Note is why the clause is Unmeasured, or the partiality caveat on a + // measured one. + Note string +} + +// Verdict is the admission answer. +type Verdict struct { + Decision Decision + // Granted is how many of Request.Want may proceed: 0, Want, or headroom + // when Request.AllowPartial. + Granted int + // Want echoes the request, so Reason can say "3 of 5". + Want int + // Role echoes the request, so Reason can name it. + Role string + Deciding api.Dimension + Clauses []ClauseResult + // Window is when accounting for the team began, carried so every + // cumulative figure marvel prints can say what span it covers. + Window time.Time +} + +// Refused reports a verdict that denies at least part of the request. +func (v Verdict) Refused() bool { return v.Decision == Refuse } + +// Key is a stable composite of the decision, the deciding dimension, and +// that dimension's clause state. +// +// The reconciler latches on it so a standing refusal emits one event per +// transition instead of one per tick. That is arithmetic, not taste: the +// reconcile interval is 2s and the event ring holds 2000, so one event per +// refusal per role flushes the whole ring in about 67 minutes and erases +// every other event class. Digesting the decision rather than comparing +// Reason strings makes the invariant directly testable (identical verdicts +// produce identical keys, a change of deciding dimension re-emits) and +// stops an incidental message-formatting change from re-firing. +func (v Verdict) Key() string { + var state ClauseState + for _, c := range v.Clauses { + if c.Dimension == v.Deciding { + state = c.State + break + } + } + return string(v.Decision) + "|" + string(v.Deciding) + "|" + string(state) +} + +// Reason is the one-line events.Event Message carrying the arithmetic, and +// doubles as the RPC error text. One line because that is the only shape +// events.Event has: it carries no structured payload, and extending it +// would change the JSON the events RPC returns, the CLI renderer, and the +// mrvl:// wire. The format is fixed so a later structured field is a +// mechanical lift. +func (v Verdict) Reason(t Trigger) string { + var b strings.Builder + switch v.Decision { + case Refuse: + fmt.Fprintf(&b, "refused %d of %d spawn(s)", v.Want-v.Granted, v.Want) + if v.Role != "" { + fmt.Fprintf(&b, " for role %s", v.Role) + } + fmt.Fprintf(&b, ": %s", v.decidingDetail()) + if v.Granted > 0 { + fmt.Fprintf(&b, "; granted %d", v.Granted) + } + case Indeterminate: + fmt.Fprintf(&b, "%s; admitted (on_unmeasured=%s)", v.decidingDetail(), api.UnmeasuredAdmit) + default: + fmt.Fprintf(&b, "within budget") + if v.Role != "" { + fmt.Fprintf(&b, " for role %s", v.Role) + } + if d := v.decidingDetail(); d != "" { + fmt.Fprintf(&b, ": %s", d) + } + } + fmt.Fprintf(&b, " (trigger=%s)", t) + return b.String() +} + +// decidingDetail renders the clause that decided, with its numbers. +func (v Verdict) decidingDetail() string { + for _, c := range v.Clauses { + if c.Dimension != v.Deciding { + continue + } + var b strings.Builder + switch { + case c.State == Unmeasured: + fmt.Fprintf(&b, "%s declared but %s", c.Dimension, c.Note) + if v.Decision == Refuse { + fmt.Fprintf(&b, " (on_unmeasured=%s)", api.UnmeasuredRefuse) + } + return b.String() + case c.Shape == api.ShapeCount && c.Adds > 0: + fmt.Fprintf(&b, "%d live + %d requested %s exceeds %s=%d", c.Used, c.Adds, c.Unit, c.Dimension, c.Limit) + case c.Shape == api.ShapeCount: + fmt.Fprintf(&b, "%d live %s against %s=%d", c.Used, c.Unit, c.Dimension, c.Limit) + default: + fmt.Fprintf(&b, "team spent %d %s against %s=%d", c.Used, c.Unit, c.Dimension, c.Limit) + if !v.Window.IsZero() { + fmt.Fprintf(&b, " (since=%s)", v.Window.Format(time.RFC3339)) + } + } + if c.Note != "" { + fmt.Fprintf(&b, " (%s)", c.Note) + } + return b.String() + } + return "" +} + +// Check is the admission verdict. Pure: no store, no meter, no clock, no +// I/O. The single arithmetic shared by every enforcement point, and the one +// mid-flight revocation (locus 3) will extend. +func Check(b api.Budget, s Snapshot, r Request) Verdict { + return check(b, s, r, true) +} + +// CheckSessions is the count-only entry point the reconciler uses. It is +// Check with every cumulative clause skipped, so a caller with no meter +// needs neither a Snapshot's token fields nor a usage import. See R2: the +// reconciler must never evaluate a monotonic clause. +func CheckSessions(b api.Budget, live, declared int, r Request) Verdict { + return check(b, Snapshot{LiveSessions: live, DeclaredSessions: declared}, r, false) +} + +func check(b api.Budget, s Snapshot, r Request, evalCumulative bool) Verdict { + v := Verdict{ + Decision: Admit, + Granted: r.Want, + Want: r.Want, + Role: r.Role, + Window: s.Since, + } + // An undeclared budget is not a gate. Checked first so it costs one + // boolean and nothing else. + if !b.Declared() { + return v + } + // A request that adds nothing is never refused. Keeps a scale-down out + // of the gate: shedding sessions is a recovery move, and refusing it + // would strand an over-budget team. + if r.Want <= 0 { + return v + } + + // Registry order: count clauses before cumulative ones, so a session + // ceiling decides before a spend ceiling. + if b.MaxSessions > 0 { + v.Clauses = append(v.Clauses, countClause(b.MaxSessions, s, r)) + } + if b.MaxTokens > 0 && evalCumulative && r.Kind != Repair { + v.Clauses = append(v.Clauses, tokenClause(b.MaxTokens, s)) + } + + // Refuse outranks Indeterminate: a measured breach is more certain than + // an unevaluable clause. + for _, c := range v.Clauses { + if c.State != Exceeded { + continue + } + // Partial grants are count-shaped only, read by shape rather than by + // naming a dimension for the same reason overlap and repair + // exemptions are: a cumulative clause has no headroom to hand out, + // because pricing "what would K more of these cost" needs a baseline + // that deliberately does not exist (see ClauseResult.Adds). + granted := 0 + if r.AllowPartial && c.Shape == api.ShapeCount { + granted = headroom(c.Limit, c.Used) + } + // Headroom covering the whole ask is not a refusal, and Granted stays + // at Want here rather than rising to the headroom. Two defects lived + // in the single assignment this replaced. Refusing on the clause alone + // logged, emitted admission.refused at warning severity, and latched a + // reconciler hold for a tick that spawned every requested session, + // because Refused() reads the decision and not the count. Assigning + // the raw headroom then handed a whole team's headroom to one role's + // smaller deficit, so the reconciler spawned past role.Replicas and + // deleted the excess on the next tick, printing "refused -4 of 1" + // along the way. Granted is only ever lowered below Want, never + // raised above it. + if granted >= r.Want { + v.Deciding = c.Dimension + continue + } + v.Decision = Refuse + v.Deciding = c.Dimension + v.Granted = granted + return v + } + for _, c := range v.Clauses { + if c.State != Unmeasured { + continue + } + v.Deciding = c.Dimension + if b.Unmeasured() == api.UnmeasuredRefuse { + v.Decision = Refuse + v.Granted = 0 + } else { + v.Decision = Indeterminate + v.Granted = r.Want + } + return v + } + if v.Deciding == "" && len(v.Clauses) > 0 { + v.Deciding = v.Clauses[0].Dimension + } + return v +} + +// countClause evaluates the live-session ceiling. Exact and pre-spawn: it +// reads the store's own count, not a stream, so it has no absence state and +// survives a daemon restart. +func countClause(limit int, s Snapshot, r Request) ClauseResult { + res := ClauseResult{ + Dimension: api.DimMaxSessions, + Shape: api.ShapeCount, + State: Within, + Used: s.LiveSessions, + Limit: limit, + Adds: r.Want, + Unit: "sessions", + } + // R5: a shift's overlapping generation is replacement, not growth. + if r.Overlap { + res.Note = "shift overlap does not count against a session ceiling" + return res + } + // R1: converging on an already-admitted declared count cannot cross the + // cap, because the parser enforces declared <= limit. The else arm is + // the hole that invariant leaves: if the declaration itself is over + // budget, more spawns are the wrong answer and an operator edit is the + // right one. + if r.Kind == Repair { + if s.DeclaredSessions > limit { + res.State = Exceeded + res.Adds = 0 + res.Note = fmt.Sprintf("team declares %d sessions against this ceiling", s.DeclaredSessions) + } + return res + } + if s.LiveSessions+r.Want > limit { + res.State = Exceeded + } + return res +} + +// tokenClause evaluates the cumulative spend ceiling. +// +// Refusal on a partial total is sound because partiality can only +// understate (R3): every unobserved contributor adds zero, so the measured +// figure is a floor and spent >= limit implies true >= limit. Admission on +// a partial total is the ambiguous direction, so that is where the clause +// speaks — Within with a notice, or Unmeasured when nothing was measured at +// all. +func tokenClause(limit int, s Snapshot) ClauseResult { + res := ClauseResult{ + Dimension: api.DimMaxTokens, + Shape: api.ShapeCumulative, + State: Within, + Used: s.TokensObserved, + Limit: limit, + Unit: "tokens", + } + if !s.TokensMetered { + res.State = Unmeasured + res.Used = 0 + res.Note = "no usage meter is wired to this daemon" + return res + } + if s.TokensSeen == 0 && s.Since.IsZero() { + res.State = Unmeasured + res.Used = 0 + res.Note = "no token usage observed yet" + return res + } + var notes []string + if s.TokensPartial { + notes = append(notes, "partial: some sessions unobserved, so this is a floor") + } + if s.TokensSuspect { + notes = append(notes, "suspect: the meter reported a cumulation violation, so this may be inflated") + } + res.Note = strings.Join(notes, "; ") + if s.TokensObserved >= limit { + res.State = Exceeded + } + return res +} + +func headroom(limit, used int) int { + if h := limit - used; h > 0 { + return h + } + return 0 +} + +// Row is one `marvel get budgets` line: a declared dimension, what has +// been observed against it, and that dimension's state. +type Row struct { + Workspace string `json:"workspace"` + Team string `json:"team"` + Dimension api.Dimension `json:"dimension"` + Limit int `json:"limit"` + Observed int `json:"observed"` + Headroom int `json:"headroom"` + State string `json:"state"` + Window time.Time `json:"window,omitempty"` + Note string `json:"note,omitempty"` +} + +// Row states. +const ( + RowOK = "ok" + // RowAtCeiling is a count dimension with no headroom left for growth. + // Distinct from RowRefusing because it is the NORMAL state of a healthy + // team: the declaration clause pushes an operator toward + // sum(replicas) == max_sessions, and repair toward an already-declared + // count is exempt, so such a team sits here forever and refuses nothing. + // Reading it as a refusal made "refusing" the resting state and left the + // only which-dimension-tripped surface unable to tell a real refusal from + // the intended configuration. + RowAtCeiling = "at-ceiling" + // RowRefusing means a refusal is standing right now. Read from + // api.Team.Admission, the condition the reconciler recomputes every tick + // and clears the moment the gate admits, rather than inferred from + // headroom. Only the count clause can produce it: the reconciler + // evaluates count clauses alone (R2), and a synchronous verb's refusal + // changes nothing and so is not a standing condition. + RowRefusing = "refusing" + RowUnmetered = "unmetered" +) + +// Rows assembles the diagnostic view for one team: one row per declared +// dimension, none for a team that declares no budget. Evaluated straight +// from the snapshot rather than from a Verdict, because the operator's +// question ("which dimension tripped and by how much") is per dimension +// and does not depend on any pending request. +// +// "Is it refusing?" is read from the team's standing condition, never +// inferred from zero headroom: a team at its declared ceiling refuses +// nothing, and a shift legitimately reads above it. See RowAtCeiling. +func Rows(t api.Team, s Snapshot) []Row { + if !t.Budget.Declared() { + return nil + } + var out []Row + for _, spec := range api.Specs() { + limit, ok := t.Budget.Limit(spec.Dimension) + if !ok { + continue + } + row := Row{ + Workspace: t.Workspace, + Team: t.Name, + Dimension: spec.Dimension, + Limit: limit, + State: RowOK, + } + switch spec.Dimension { + case api.DimMaxSessions: + row.Observed = s.LiveSessions + row.Headroom = headroom(limit, s.LiveSessions) + var notes []string + switch { + case t.Admission.Held: + row.State = RowRefusing + notes = append(notes, t.Admission.Reason) + case row.Headroom == 0: + row.State = RowAtCeiling + } + // A rotation runs the new generation beside the old and a count + // ceiling exempts that overlap (R5), so Observed can read above + // Limit until draining finishes. Say which mechanism produced the + // overshoot rather than leaving the operator with two numbers that + // cannot both be right. + if t.Shift.Phase != api.ShiftNone { + notes = append(notes, fmt.Sprintf("shift %s: the new generation overlaps the old, which a session ceiling exempts, so live may read above the limit until draining finishes", t.Shift.Phase)) + } + row.Note = strings.Join(notes, "; ") + case api.DimMaxTokens: + c := tokenClause(limit, s) + row.Observed = c.Used + row.Headroom = headroom(limit, c.Used) + row.Note = c.Note + row.Window = s.Since + switch c.State { + case Exceeded: + row.State = RowRefusing + case Unmeasured: + row.State = RowUnmetered + } + } + out = append(out, row) + } + return out +} diff --git a/internal/admission/admission_test.go b/internal/admission/admission_test.go new file mode 100644 index 0000000..6f21660 --- /dev/null +++ b/internal/admission/admission_test.go @@ -0,0 +1,621 @@ +package admission + +import ( + "strings" + "testing" + "time" + + "github.com/arcavenae/marvel/internal/api" +) + +var testSince = time.Date(2026, 8, 1, 14, 8, 0, 0, time.UTC) + +// metered returns a snapshot whose token figures are real, so a test row +// that means "measured and under budget" cannot accidentally read as +// "never measured". +func metered(live, declared, tokens int) Snapshot { + return Snapshot{ + Workspace: "fanout", + Team: "crew", + LiveSessions: live, + DeclaredSessions: declared, + TokensObserved: tokens, + TokensMetered: true, + TokensSeen: live, + Since: testSince, + } +} + +// TestCheck is the whole admission arithmetic (aae-orc-qiay), table-driven +// over the cases that decide the design: the count boundary, the repair +// exemption and its one hole, shift overlap by shape, and every partiality +// state of the token clause. +func TestCheck(t *testing.T) { + t.Parallel() + tests := []struct { + name string + budget api.Budget + snap Snapshot + req Request + want Decision + wantGranted int + wantDeciding api.Dimension + wantClauses int + }{ + // A1. The default-open guarantee, as arithmetic. + { + name: "undeclared budget admits and evaluates nothing", + budget: api.Budget{}, + snap: metered(99, 99, 9_000_000), + req: Request{Role: "crew", Want: 40, Kind: Growth}, + want: Admit, + wantGranted: 40, + wantClauses: 0, + }, + // A2. The count boundary. + { + name: "count at the ceiling admits", + budget: api.Budget{MaxSessions: 6}, + snap: metered(3, 3, 0), + req: Request{Role: "crew", Want: 3, Kind: Growth}, + want: Admit, + wantGranted: 3, + wantDeciding: api.DimMaxSessions, + wantClauses: 1, + }, + { + name: "one over the ceiling refuses", + budget: api.Budget{MaxSessions: 6}, + snap: metered(3, 3, 0), + req: Request{Role: "crew", Want: 4, Kind: Growth}, + want: Refuse, + wantGranted: 0, + wantDeciding: api.DimMaxSessions, + wantClauses: 1, + }, + { + name: "a request that adds nothing is never refused", + budget: api.Budget{MaxSessions: 6}, + snap: metered(40, 40, 0), + req: Request{Role: "crew", Want: 0, Kind: Growth}, + want: Admit, + wantGranted: 0, + wantClauses: 0, + }, + // A3. Partial admission, the reconciler's side of the asymmetry. + { + name: "AllowPartial grants headroom instead of nothing", + budget: api.Budget{MaxSessions: 6}, + snap: metered(5, 5, 0), + req: Request{Role: "crew", Want: 3, Kind: Growth, AllowPartial: true}, + want: Refuse, + wantGranted: 1, + wantDeciding: api.DimMaxSessions, + wantClauses: 1, + }, + // A4. Falsification: without shape-based overlap exemption, a budget + // equal to declared replicas forbids every rolling shift. + { + name: "shift overlap skips the count clause", + budget: api.Budget{MaxSessions: 3}, + snap: metered(3, 3, 0), + req: Request{Want: 3, Kind: Growth, Overlap: true}, + want: Admit, + wantGranted: 3, + wantDeciding: api.DimMaxSessions, + wantClauses: 1, + }, + // A5. Falsification: gating repair on live+want > limit means a + // crashed replica never returns. + { + name: "repair toward declared replicas is admitted", + budget: api.Budget{MaxSessions: 3}, + snap: metered(2, 3, 0), + req: Request{Role: "crew", Want: 1, Kind: Repair, AllowPartial: true}, + want: Admit, + wantGranted: 1, + wantDeciding: api.DimMaxSessions, + wantClauses: 1, + }, + // A6. The hole the R1 invariant leaves: a declaration that is itself + // over budget is an operator edit, not a spawn. + { + name: "repair refused when the declaration exceeds the ceiling", + budget: api.Budget{MaxSessions: 3}, + snap: metered(2, 7, 0), + req: Request{Role: "crew", Want: 3, Kind: Repair, AllowPartial: true}, + want: Refuse, + wantGranted: 1, + wantDeciding: api.DimMaxSessions, + wantClauses: 1, + }, + // A6b. The same over-declaration, asking for what fits. Nothing is + // refused, so the verdict must not say refused: a Refuse here latches + // a reconciler hold and emits a warning event for a tick that + // satisfied the role in full. + { + name: "an over-declaration this request fits inside admits", + budget: api.Budget{MaxSessions: 3}, + snap: metered(2, 7, 0), + req: Request{Role: "crew", Want: 1, Kind: Repair, AllowPartial: true}, + want: Admit, + wantGranted: 1, + wantDeciding: api.DimMaxSessions, + wantClauses: 1, + }, + // A6c. Falsification of the overspawn defect: unclamped, headroom + // larger than the ask granted the whole headroom, so the reconciler + // spawned past role.Replicas and deleted the excess on the next tick. + { + name: "a partial grant never exceeds the request", + budget: api.Budget{MaxSessions: 5}, + snap: metered(0, 6, 0), + req: Request{Role: "sup", Want: 1, Kind: Repair, AllowPartial: true}, + want: Admit, + wantGranted: 1, + wantDeciding: api.DimMaxSessions, + wantClauses: 1, + }, + // A7. The token clause, including the >= boundary. + { + name: "tokens under the ceiling admit", + budget: api.Budget{MaxTokens: 2_000_000}, + snap: metered(2, 2, 412_118), + req: Request{Role: "crew", Want: 1, Kind: Growth}, + want: Admit, + wantGranted: 1, + wantDeciding: api.DimMaxTokens, + wantClauses: 1, + }, + { + name: "tokens exactly at the ceiling refuse", + budget: api.Budget{MaxTokens: 2_000_000}, + snap: metered(2, 2, 2_000_000), + req: Request{Role: "crew", Want: 1, Kind: Growth}, + want: Refuse, + wantGranted: 0, + wantDeciding: api.DimMaxTokens, + wantClauses: 1, + }, + { + name: "tokens over the ceiling refuse", + budget: api.Budget{MaxTokens: 2_000_000}, + snap: metered(2, 2, 2_118_443), + req: Request{Role: "crew", Want: 3, Kind: Growth}, + want: Refuse, + wantGranted: 0, + wantDeciding: api.DimMaxTokens, + wantClauses: 1, + }, + // A8. The governance row. Partiality can only understate, so a + // measured breach on a partial total is still a breach. + { + name: "over budget on a partial total still refuses", + budget: api.Budget{MaxTokens: 2_000_000}, + snap: func() Snapshot { + s := metered(5, 5, 2_118_443) + s.TokensPartial = true + return s + }(), + req: Request{Role: "crew", Want: 3, Kind: Growth}, + want: Refuse, + wantGranted: 0, + wantDeciding: api.DimMaxTokens, + wantClauses: 1, + }, + // A9. Falsification: refusing here would be a refusal caused by + // absent data, which is the failure internal/usage names. + { + name: "under budget on a partial total admits", + budget: api.Budget{MaxTokens: 2_000_000}, + snap: func() Snapshot { + s := metered(5, 5, 412_118) + s.TokensPartial = true + return s + }(), + req: Request{Role: "crew", Want: 3, Kind: Growth}, + want: Admit, + wantGranted: 3, + wantDeciding: api.DimMaxTokens, + wantClauses: 1, + }, + // A10. Nothing measured yet is not the same as spent nothing. + { + name: "no usage observed yet is indeterminate", + budget: api.Budget{MaxTokens: 2_000_000}, + snap: Snapshot{Workspace: "fanout", Team: "crew", LiveSessions: 3, DeclaredSessions: 3, TokensMetered: true}, + req: Request{Role: "crew", Want: 3, Kind: Growth}, + want: Indeterminate, + wantGranted: 3, + wantDeciding: api.DimMaxTokens, + wantClauses: 1, + }, + // A11. No meter wired at all. + { + name: "no meter is indeterminate, never zero spend", + budget: api.Budget{MaxTokens: 2_000_000}, + snap: Snapshot{Workspace: "fanout", Team: "crew", LiveSessions: 3, DeclaredSessions: 3}, + req: Request{Role: "crew", Want: 3, Kind: Growth}, + want: Indeterminate, + wantGranted: 3, + wantDeciding: api.DimMaxTokens, + wantClauses: 1, + }, + // A12. on_unmeasured is the operator's fail-closed instrument. + { + name: "on_unmeasured refuse fails closed", + budget: api.Budget{MaxTokens: 2_000_000, OnUnmeasured: api.UnmeasuredRefuse}, + snap: Snapshot{Workspace: "fanout", Team: "crew", LiveSessions: 3, DeclaredSessions: 3, TokensMetered: true}, + req: Request{Role: "crew", Want: 3, Kind: Growth}, + want: Refuse, + wantGranted: 0, + wantDeciding: api.DimMaxTokens, + wantClauses: 1, + }, + // A13. Overlap is a count-shaped exemption only: a new generation is + // a new spender. + { + name: "shift overlap does not skip a cumulative clause", + budget: api.Budget{MaxSessions: 3, MaxTokens: 2_000_000}, + snap: metered(3, 3, 2_118_443), + req: Request{Want: 3, Kind: Growth, Overlap: true}, + want: Refuse, + wantGranted: 0, + wantDeciding: api.DimMaxTokens, + wantClauses: 2, + }, + // A14. Precedence: a measured breach outranks an unevaluable clause. + { + name: "count exceeded outranks tokens unmeasured", + budget: api.Budget{MaxSessions: 6, MaxTokens: 2_000_000}, + snap: Snapshot{Workspace: "fanout", Team: "crew", LiveSessions: 6, DeclaredSessions: 6, TokensMetered: true}, + req: Request{Role: "crew", Want: 1, Kind: Growth}, + want: Refuse, + wantGranted: 0, + wantDeciding: api.DimMaxSessions, + wantClauses: 2, + }, + // R2, structurally: repair never evaluates a monotonic clause, so an + // exhausted token budget cannot make a team unrepairable. + { + name: "repair never evaluates the token clause", + budget: api.Budget{MaxSessions: 3, MaxTokens: 2_000_000}, + snap: metered(2, 3, 9_000_000), + req: Request{Role: "crew", Want: 1, Kind: Repair, AllowPartial: true}, + want: Admit, + wantGranted: 1, + wantDeciding: api.DimMaxSessions, + wantClauses: 1, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + v := Check(tt.budget, tt.snap, tt.req) + if v.Decision != tt.want { + t.Errorf("Decision = %q, want %q", v.Decision, tt.want) + } + if v.Granted != tt.wantGranted { + t.Errorf("Granted = %d, want %d", v.Granted, tt.wantGranted) + } + if v.Deciding != tt.wantDeciding { + t.Errorf("Deciding = %q, want %q", v.Deciding, tt.wantDeciding) + } + if len(v.Clauses) != tt.wantClauses { + t.Errorf("evaluated %d clause(s), want %d: %+v", len(v.Clauses), tt.wantClauses, v.Clauses) + } + if v.Refused() != (tt.want == Refuse) { + t.Errorf("Refused() = %v, want %v", v.Refused(), tt.want == Refuse) + } + }) + } +} + +// TestUnderBudgetPartialCarriesANotice covers A9's other half: admitting on +// an incomplete total is only honest if the incompleteness is said out loud. +func TestUnderBudgetPartialCarriesANotice(t *testing.T) { + t.Parallel() + s := metered(5, 5, 412_118) + s.TokensPartial = true + v := Check(api.Budget{MaxTokens: 2_000_000}, s, Request{Role: "crew", Want: 1, Kind: Growth}) + if len(v.Clauses) != 1 { + t.Fatalf("expected one clause, got %+v", v.Clauses) + } + if !strings.Contains(v.Clauses[0].Note, "partial") { + t.Errorf("clause note = %q, want it to name the partial total", v.Clauses[0].Note) + } +} + +// TestHeadroomCoveringTheAskIsNotARefusal is the contract the never-silent +// requirement depends on in reverse: the refusal surface must not report +// refusals that did not happen, or a hit on `marvel events --kind +// admission.refused` cannot be trusted. +// +// The defect was structural rather than arithmetic. check set Decision=Refuse +// on an exceeded clause, then overwrote Granted for the partial case without +// revisiting the decision, and Refused() reads the decision alone. A +// reconcile tick that spawned every requested session still logged, emitted a +// warning event, and latched Team.Admission{Held:true}. +func TestHeadroomCoveringTheAskIsNotARefusal(t *testing.T) { + t.Parallel() + // The over-declaration is the only way to reach an exceeded count clause + // on the repair path (R1), so it is also the only way to reach the bug. + v := CheckSessions(api.Budget{MaxSessions: 3}, 0, 4, Request{ + Role: "a", Want: 3, Kind: Repair, AllowPartial: true, + }) + if v.Decision != Admit { + t.Errorf("Decision = %q, want admit: every requested spawn fits", v.Decision) + } + if v.Refused() { + t.Errorf("Refused() = true with Granted %d of Want %d", v.Granted, v.Want) + } + if v.Granted != 3 { + t.Errorf("Granted = %d, want 3", v.Granted) + } + // The over-ceiling declaration is still visible: the fix suppresses the + // false refusal, not the evidence. + if len(v.Clauses) != 1 || v.Clauses[0].State != Exceeded { + t.Fatalf("clauses = %+v, want one exceeded count clause", v.Clauses) + } + if v.Deciding != api.DimMaxSessions { + t.Errorf("Deciding = %q, want max_sessions", v.Deciding) + } + // Granted is clamped to Want, so no caller can spawn past what it asked + // for and no rendered count can go negative. + over := CheckSessions(api.Budget{MaxSessions: 5}, 0, 6, Request{ + Role: "sup", Want: 1, Kind: Repair, AllowPartial: true, + }) + if over.Granted != 1 { + t.Errorf("Granted = %d with headroom 5 and Want 1, want 1", over.Granted) + } + // Unclamped, this rendered "refused -4 of 1 spawn(s) ... granted 5". + if reason := over.Reason(TriggerReconcile); strings.Contains(reason, "refused") { + t.Errorf("Reason = %q, want no refusal claim on an admitted verdict", reason) + } +} + +// TestSuspectTotalIsNotedNotDecided covers the cumulation-violation case: +// the meter's own distrust of a total changes what the operator is told, not +// whether the spawn is refused. +func TestSuspectTotalIsNotedNotDecided(t *testing.T) { + t.Parallel() + s := metered(2, 2, 412_118) + s.TokensSuspect = true + v := Check(api.Budget{MaxTokens: 2_000_000}, s, Request{Role: "crew", Want: 1, Kind: Growth}) + if v.Decision != Admit { + t.Fatalf("Decision = %q, want admit", v.Decision) + } + if !strings.Contains(v.Clauses[0].Note, "suspect") { + t.Errorf("clause note = %q, want it to name the suspect total", v.Clauses[0].Note) + } +} + +// TestReasonIsOneLineWithTheArithmetic covers A15. events.Event carries no +// structured payload, so the Message is the machine-readable surface: it +// must stay one line and it must carry the numbers that decided. +func TestReasonIsOneLineWithTheArithmetic(t *testing.T) { + t.Parallel() + tests := []struct { + name string + budget api.Budget + snap Snapshot + req Request + trigger Trigger + contains []string + }{ + { + name: "count refusal names both numbers and the unit", + budget: api.Budget{MaxSessions: 6}, + snap: metered(6, 6, 0), + req: Request{Role: "crew", Want: 34, Kind: Growth}, + trigger: TriggerScale, + contains: []string{"refused 34 of 34", "role crew", "6 live", "34 requested", "sessions", "max_sessions=6", "trigger=scale"}, + }, + { + name: "partial grant names what was granted", + budget: api.Budget{MaxSessions: 6}, + snap: metered(4, 4, 0), + req: Request{Role: "crew", Want: 5, Kind: Growth, AllowPartial: true}, + trigger: TriggerReconcile, + contains: []string{"refused 3 of 5", "granted 2", "trigger=reconcile"}, + }, + { + name: "token refusal names the window", + budget: api.Budget{MaxTokens: 2_000_000}, + snap: metered(2, 2, 2_118_443), + req: Request{Role: "crew", Want: 3, Kind: Growth}, + trigger: TriggerApply, + contains: []string{"2118443", "tokens", "max_tokens=2000000", "since=", "trigger=apply"}, + }, + { + name: "repair refusal names the disagreeing declaration", + budget: api.Budget{MaxSessions: 3}, + snap: metered(2, 7, 0), + req: Request{Role: "crew", Want: 1, Kind: Repair}, + trigger: TriggerReconcile, + contains: []string{"max_sessions=3", "declares 7 sessions"}, + }, + { + name: "unmeasured admission names the clause and the mode", + budget: api.Budget{MaxTokens: 2_000_000}, + snap: Snapshot{TokensMetered: true}, + req: Request{Role: "crew", Want: 3, Kind: Growth}, + trigger: TriggerApply, + contains: []string{"max_tokens declared", "no token usage observed yet", "on_unmeasured=admit", "trigger=apply"}, + }, + { + name: "unmeasured refusal names the fail-closed mode", + budget: api.Budget{MaxTokens: 2_000_000, OnUnmeasured: api.UnmeasuredRefuse}, + snap: Snapshot{TokensMetered: true}, + req: Request{Role: "crew", Want: 3, Kind: Growth}, + trigger: TriggerRun, + contains: []string{"refused 3 of 3", "on_unmeasured=refuse", "trigger=run"}, + }, + { + name: "admitted verdict still reads as a sentence", + budget: api.Budget{MaxSessions: 6}, + snap: metered(2, 2, 0), + req: Request{Role: "crew", Want: 1, Kind: Growth}, + trigger: TriggerRun, + contains: []string{"within budget", "role crew", "trigger=run"}, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := Check(tt.budget, tt.snap, tt.req).Reason(tt.trigger) + if strings.ContainsAny(got, "\n\r") { + t.Errorf("Reason spans more than one line: %q", got) + } + for _, want := range tt.contains { + if !strings.Contains(got, want) { + t.Errorf("Reason = %q, missing %q", got, want) + } + } + }) + } +} + +// TestKeyLatchesOnTheDecisionNotTheMessage covers A16. The reconciler emits +// only when the key changes, so a key that moved with the message text would +// re-fire on any formatting change and flush the event ring; a key that did +// not move on a change of deciding dimension would hide new information. +func TestKeyLatchesOnTheDecisionNotTheMessage(t *testing.T) { + t.Parallel() + budget := api.Budget{MaxSessions: 6, MaxTokens: 2_000_000} + req := Request{Role: "crew", Want: 4, Kind: Growth} + + overCount := Check(budget, metered(6, 6, 0), req) + overCountAgain := Check(budget, metered(6, 6, 0), req) + if overCount.Key() != overCountAgain.Key() { + t.Errorf("identical verdicts produced different keys: %q vs %q", overCount.Key(), overCountAgain.Key()) + } + + // Same decision, same deciding dimension, different arithmetic in the + // message: the ring must not see a second event. + overCountBigger := Check(budget, metered(9, 9, 0), req) + if overCountBigger.Key() != overCount.Key() { + t.Errorf("a changed message moved the key: %q vs %q", overCountBigger.Key(), overCount.Key()) + } + if overCountBigger.Reason(TriggerReconcile) == overCount.Reason(TriggerReconcile) { + t.Fatal("test is vacuous: the two messages are identical") + } + + // A different dimension deciding IS new information. + overTokens := Check(budget, metered(2, 2, 2_118_443), req) + if overTokens.Key() == overCount.Key() { + t.Errorf("count and token refusals share a key: %q", overTokens.Key()) + } + + // So is stopping refusing. + admitted := Check(budget, metered(1, 1, 10), req) + if admitted.Key() == overCount.Key() { + t.Errorf("admit and refuse share a key: %q", admitted.Key()) + } +} + +// TestCheckSessionsSkipsCumulativeClauses covers A17: the reconciler's +// entry point is Check with every cumulative clause dropped, which is what +// keeps a monotonic meter off the repair path (R2). +func TestCheckSessionsSkipsCumulativeClauses(t *testing.T) { + t.Parallel() + both := api.Budget{MaxSessions: 6, MaxTokens: 2_000_000} + countOnly := api.Budget{MaxSessions: 6} + req := Request{Role: "crew", Want: 2, Kind: Growth, AllowPartial: true} + + for _, live := range []int{0, 3, 5, 6, 9} { + got := CheckSessions(both, live, live, req) + want := Check(countOnly, Snapshot{LiveSessions: live, DeclaredSessions: live}, req) + if got.Decision != want.Decision || got.Granted != want.Granted || got.Deciding != want.Deciding { + t.Errorf("live=%d: CheckSessions = (%s, %d, %s), want (%s, %d, %s)", + live, got.Decision, got.Granted, got.Deciding, want.Decision, want.Granted, want.Deciding) + } + if len(got.Clauses) != 1 { + t.Errorf("live=%d: evaluated %d clause(s), want the count clause alone", live, len(got.Clauses)) + } + } +} + +// TestRowsReportEveryDeclaredDimension covers the `marvel get budgets` +// assembly: a team with no budget contributes nothing, and an unmeasured +// token figure is reported as unmetered rather than as headroom the operator +// does not have. +func TestRowsReportEveryDeclaredDimension(t *testing.T) { + t.Parallel() + if rows := Rows(api.Team{Workspace: "ws", Name: "plain"}, metered(3, 3, 0)); rows != nil { + t.Errorf("undeclared budget produced %d row(s), want none", len(rows)) + } + + team := api.Team{Workspace: "fanout", Name: "crew", Budget: api.Budget{MaxSessions: 6, MaxTokens: 2_000_000}} + rows := Rows(team, metered(6, 6, 412_118)) + if len(rows) != 2 { + t.Fatalf("got %d row(s), want 2: %+v", len(rows), rows) + } + // A healthy team whose declared replicas equal its ceiling refuses + // nothing, so this row must not read as a refusal. See RowAtCeiling. + if rows[0].Dimension != api.DimMaxSessions || rows[0].State != RowAtCeiling || rows[0].Headroom != 0 { + t.Errorf("session row = %+v, want max_sessions at-ceiling with no headroom", rows[0]) + } + if rows[1].Dimension != api.DimMaxTokens || rows[1].State != RowOK || rows[1].Headroom != 1_587_882 { + t.Errorf("token row = %+v, want max_tokens ok with 1587882 headroom", rows[1]) + } + if rows[1].Window != testSince { + t.Errorf("token row window = %s, want %s", rows[1].Window, testSince) + } + + unmetered := Rows(team, Snapshot{LiveSessions: 2, DeclaredSessions: 6}) + if unmetered[1].State != RowUnmetered { + t.Errorf("token row state = %q, want %q when nothing is measured", unmetered[1].State, RowUnmetered) + } +} + +// TestRowsSeparateAtCeilingFromRefusing covers the primary "which dimension +// tripped" surface. Keying refusal on zero headroom made refusing the resting +// state of every team sized at its ceiling, so a real refusal and the intended +// configuration rendered identically. +func TestRowsSeparateAtCeilingFromRefusing(t *testing.T) { + t.Parallel() + budget := api.Budget{MaxSessions: 2} + + held := api.Team{ + Workspace: "fanout", Name: "crew", Budget: budget, + Admission: api.AdmissionState{ + Held: true, Role: "b", + Reason: "refused 1 of 1 spawn(s) for role b: 2 live sessions against max_sessions=2 (trigger=reconcile)", + }, + } + rows := Rows(held, Snapshot{LiveSessions: 2, DeclaredSessions: 3}) + if rows[0].State != RowRefusing { + t.Errorf("state = %q with a standing hold, want %q", rows[0].State, RowRefusing) + } + if !strings.Contains(rows[0].Note, "role b") { + t.Errorf("note = %q, want the held role's arithmetic", rows[0].Note) + } + + // A shift reads above the ceiling by design (R5). Not a refusal, and the + // note has to name the mechanism or the operator is left with two numbers + // that cannot both be right. + shifting := api.Team{ + Workspace: "fanout", Name: "crew", Budget: budget, + Shift: api.ShiftState{Phase: api.ShiftDraining}, + } + rows = Rows(shifting, Snapshot{LiveSessions: 4, DeclaredSessions: 2}) + if rows[0].State == RowRefusing { + t.Errorf("state = %q during a shift, want anything but %q", rows[0].State, RowRefusing) + } + if rows[0].Observed != 4 || rows[0].Headroom != 0 { + t.Errorf("row = %+v, want the overshoot reported verbatim", rows[0]) + } + if !strings.Contains(rows[0].Note, "shift") { + t.Errorf("note = %q, want it to name the shift overlap", rows[0].Note) + } + + // Headroom left, no hold, no shift: plain ok. + plain := api.Team{Workspace: "fanout", Name: "crew", Budget: budget} + rows = Rows(plain, Snapshot{LiveSessions: 1, DeclaredSessions: 2}) + if rows[0].State != RowOK || rows[0].Note != "" { + t.Errorf("row = %+v, want %q with no note", rows[0], RowOK) + } +} diff --git a/internal/admission/doc.go b/internal/admission/doc.go new file mode 100644 index 0000000..4d30741 --- /dev/null +++ b/internal/admission/doc.go @@ -0,0 +1,106 @@ +// Package admission is the arithmetic that refuses a spawn which would +// exceed a team-declared budget. It is the first brick of enforcement +// locus 2 (runtime admission and metering) in the agentic resource matrix; +// see marvel/_kos/nodes/bedrock/elem-agentic-resource-matrix.yaml and +// aae-orc-qiay. +// +// # Meter and gate are separate, on purpose +// +// internal/usage is a METER: it defines no threshold, no policy, and no +// refusal, because a metered value becomes a gate only through a ratified +// written decision (ADR-007 clause 3, SOUL.md section 8). This package is +// the gate, and the thing that ratifies it is the operator's own +// declaration in the manifest. No budget declared means no gate: an +// undeclared team behaves exactly as it did before this package existed, +// with no store read, no meter read, and no event. +// +// The package is pure. It imports internal/api and the standard library, +// and nothing else: no store, no accountant, no clock, no I/O. The caller +// gathers a Snapshot and hands it in, which is why the arithmetic is +// testable with neither a tmux server nor an Accountant. +// +// # The five rulings the design rests on +// +// R1. Repair safety is structural, not a flag. The manifest parser +// enforces sum(role.Replicas) <= max_sessions, so declared <= budget is +// an invariant of every parsed manifest. Converging a role toward its +// declared replicas therefore cannot cross the team cap, so no "is this +// growth?" predicate exists anywhere in the code and repair can never be +// refused. The invariant is only as strong as the doors that hold it: +// `marvel scale` edits a replica count without re-parsing a manifest, so +// it carries the same clause at the verb (daemon.admitDeclaration). +// Without that, scaling while a replica was dead committed a declaration +// the parser refuses. Where the invariant is violated anyway (an +// out-of-band UpdateTeam), papering over it with more spawns would be +// wrong, so the clause refuses even repair and names the two numbers that +// disagree. +// +// R2. A cumulative clause is never evaluated on the repair path. +// usage.TeamSpend is monotonic within a daemon lifetime: retired spend is +// rolled into the team total at Forget and nothing subtracts. Gating +// repair on a monotonic meter is a permanent outage, not a budget. Growth +// carries the cumulative clause; repair does not. +// +// R3. Partiality can only understate, so refusal on a partial total is +// sound. Every source of partiality contributes zero: an unobserved +// session, a nil Reader, a pane adopted from a prior daemon. The measured +// total is therefore a floor, and spent >= limit implies true >= limit. +// Admission on a partial total is the ambiguous direction, which is where +// marvel speaks (Indeterminate plus an event, resolved by +// api.Budget.OnUnmeasured). +// +// R4. A refusal must never touch crash bookkeeping. Both of the team +// controller's crash paths funnel into noteCrashAndBackoff, whose +// MaxRestarts saturation freezes BackoffUntil in the year 9999 and writes +// that through to bolt. A budget refusal routed there would become an +// unrecoverable role kill that survives a restart. A refusal is not a +// crash, and this package never learns about restart counts. +// +// R5. Shift overlap is exempt from count-shaped clauses by shape, not by +// special case. A shift is replacement; its transient double count is a +// mechanism artifact, so Request.Overlap skips ShapeCount clauses and does +// not skip ShapeCumulative ones (a new generation is a new spender). The +// operator-visible consequence is that live sessions can read up to twice +// a role's replicas for the length of a rotation, so the exemption has to +// be stated wherever the ceiling is described: api.Budget.MaxSessions, +// docs/user-guide.md, and the Row note while a shift is in progress. +// +// # Dimensions excluded on evidence +// +// Three registry rows are declared in internal/api and deliberately have +// no evaluator here: +// +// - max_cost_usd. Claude lifts total_cost_usd only on its terminal +// result line, so a running claude team reads CostUSD == 0 with +// CostReported == false; codex publishes no cost at all. Only +// opencode is live per request. A dollar ceiling is therefore +// unenforceable mid-flight on any team containing claude or codex, +// and making it work needs the per-harness capability table that +// lives in internal/usage. +// - max_session_ctx_percent. Context occupancy is a level and never a +// sum (internal/usage/doc.go), so there is no team aggregate to +// compare against a ceiling. How context pressure aggregates across a +// team is an open question owned by the shift-trigger work +// (_kos/nodes/frontier/question-shift-triggers.yaml); admission must +// not answer it first by accident. +// - max_team_rss_bytes. Process metrics are honest about their own +// limits and cannot feed a gate: MetricsAt is zero for a just-spawned +// session, the sampler runs at 5s against a 2s reconcile tick, +// CPUPercent is a subtree rollup with no capacity denominator, IO +// counters are unavailable on darwin, and metrics are deliberately not +// persisted. +// +// Per-provider rate-limit accounting is not excluded pending work: it is +// unachievable. Marvel cannot see a provider, because auth delegates to +// the harness and marvel stores no credentials. The enforceable boundary +// is workspace/team. +// +// # Standing warning +// +// This package is the natural attractor for wiring a health metric into a +// gate. Someone will want a context-percent ceiling next, and it will look +// like a five-line addition. It is not: a vital sign becomes a gate only +// through a ratified written decision, and structural validity is the only +// thing that may gate without one. The behavior trigger for that moment is +// aae-orc's .claude/rules/diagnostic-not-gate.md. +package admission diff --git a/internal/api/bolt_test.go b/internal/api/bolt_test.go index 88e4967..72cdcae 100644 --- a/internal/api/bolt_test.go +++ b/internal/api/bolt_test.go @@ -434,3 +434,83 @@ func TestBoltStore_HeartbeatContextReadingSurvivesRehydrate(t *testing.T) { t.Error("LastHeartbeat dropped") } } + +// TestBoltStore_BudgetRoundTripsAndOldRecordsStayOpen is the +// backward-compatibility proof for aae-orc-qiay, and the reason no +// boltSchemaVersion bump is required. +// +// Records are marshalled whole as JSON, so a declared budget round-trips +// with no bucket or schema work, and a teams record written by a binary that +// never heard of budgets decodes to the zero Budget. Zero means undeclared, +// undeclared means no gate: an operator upgrading marvel under a running +// fleet gets identical behavior until they edit a manifest. A schema bump +// would instead refuse the existing state file outright, because Rehydrate +// rejects a lower on-disk version as well as a higher one. +func TestBoltStore_BudgetRoundTripsAndOldRecordsStayOpen(t *testing.T) { + path := filepath.Join(t.TempDir(), "marvel.bolt") + + s1 := NewStore() + if err := s1.OpenBolt(path); err != nil { + t.Fatalf("OpenBolt #1: %v", err) + } + if err := s1.CreateWorkspace(&Workspace{Name: "fanout", CreatedAt: time.Now().UTC()}); err != nil { + t.Fatalf("create workspace: %v", err) + } + if err := s1.CreateTeam(&Team{ + Name: "crew", + Workspace: "fanout", + Roles: []Role{{Name: "crew", Replicas: 3, Runtime: Runtime{Name: "sleep", Command: "sleep"}}}, + Budget: Budget{MaxSessions: 6, MaxTokens: 2_000_000, OnUnmeasured: UnmeasuredRefuse}, + // A team with no budget, written alongside, stands in for every + // record an older binary wrote: the field is simply absent. + CreatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("create budgeted team: %v", err) + } + if err := s1.CreateTeam(&Team{ + Name: "plain", + Workspace: "fanout", + Roles: []Role{{Name: "worker", Replicas: 1, Runtime: Runtime{Name: "sleep", Command: "sleep"}}}, + CreatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("create plain team: %v", err) + } + if err := s1.CloseBolt(); err != nil { + t.Fatalf("CloseBolt: %v", err) + } + + s2 := NewStore() + if err := s2.OpenBolt(path); err != nil { + t.Fatalf("OpenBolt #2: %v", err) + } + t.Cleanup(func() { _ = s2.CloseBolt() }) + + got, err := s2.GetTeam("fanout/crew") + if err != nil { + t.Fatalf("get budgeted team after rehydrate: %v", err) + } + if got.Budget.MaxSessions != 6 || got.Budget.MaxTokens != 2_000_000 { + t.Errorf("budget after rehydrate = %+v, want max_sessions 6 and max_tokens 2000000", got.Budget) + } + if got.Budget.Unmeasured() != UnmeasuredRefuse { + t.Errorf("on_unmeasured after rehydrate = %q, want %q", got.Budget.Unmeasured(), UnmeasuredRefuse) + } + + plain, err := s2.GetTeam("fanout/plain") + if err != nil { + t.Fatalf("get plain team after rehydrate: %v", err) + } + if plain.Budget.Declared() { + t.Errorf("a record with no budget rehydrated as a gate: %+v", plain.Budget) + } +} + +// TestBoltSchemaVersionUnchangedByBudget states the decision as an +// assertion: adding a JSON field to a record is read-compatible in both +// directions, so this slice does not touch the version. +func TestBoltSchemaVersionUnchangedByBudget(t *testing.T) { + t.Parallel() + if boltSchemaVersion != 1 { + t.Errorf("boltSchemaVersion = %d; adding the Budget field must not bump it, because Rehydrate refuses a lower on-disk version too", boltSchemaVersion) + } +} diff --git a/internal/api/budget.go b/internal/api/budget.go new file mode 100644 index 0000000..908205a --- /dev/null +++ b/internal/api/budget.go @@ -0,0 +1,223 @@ +package api + +import ( + "slices" + "sort" + "strings" + "time" +) + +// Dimension names one row of the agentic resource matrix expressed as a +// budget clause. See aae-orc-qiay. +type Dimension string + +const ( + DimMaxSessions Dimension = "max_sessions" + DimMaxTokens Dimension = "max_tokens" + // The three below are registered, not enforced. Named here so a + // manifest declaring one gets "known dimension, not implemented, + // owner X" rather than silence, and so the matrix rows still awaiting + // an evaluator are visible in one place. + DimMaxCostUSD Dimension = "max_cost_usd" + DimMaxTeamRSSBytes Dimension = "max_team_rss_bytes" + DimMaxSessionCtxPct Dimension = "max_session_ctx_percent" +) + +// Shape decides how a clause reads its meter, whether a shift's +// overlapping generation counts against it, and whether repair is exempt. +// Adopting shape rather than per-clause special cases is what lets a +// third dimension drop in without re-deciding overlap semantics. +type Shape string + +const ( + // ShapeCount is a level over live sessions: it falls when sessions + // exit, a shift's overlapping generation does not count, and repair + // toward already-declared replicas is exempt (the declaration clause + // makes declared <= limit an invariant, so converging on it cannot + // cross the cap). + ShapeCount Shape = "count" + // ShapeCumulative accumulates and never falls within a daemon + // lifetime. A new generation is a new spender, so an overlap counts, + // and it is never evaluated on the repair path: gating repair on a + // monotonic meter is a permanent outage. + ShapeCumulative Shape = "cumulative" + // ShapeLevel is a per-session reading with no team aggregate. No + // dimension of this shape is implemented; context occupancy is a + // level and never a sum (see internal/usage/doc.go), and how it + // aggregates across a team belongs to the shift-trigger work. + ShapeLevel Shape = "level" +) + +// Spec is the registry entry for one dimension. +type Spec struct { + Dimension Dimension + Shape Shape + Unit string + Integral bool + Implemented bool + MatrixRow int + // Owner names the bd ticket or lane that owns an unimplemented row, + // so the validation error points somewhere. + Owner string +} + +// budgetSpecs is the dimension registry, in the order Specs reports and +// clauses evaluate: count-shaped rows before cumulative ones, so a +// session ceiling decides before a spend ceiling. +var budgetSpecs = []Spec{ + {Dimension: DimMaxSessions, Shape: ShapeCount, Unit: "sessions", Integral: true, Implemented: true, MatrixRow: 12, Owner: "aae-orc-qiay"}, + {Dimension: DimMaxTokens, Shape: ShapeCumulative, Unit: "tokens", Integral: true, Implemented: true, MatrixRow: 2, Owner: "aae-orc-qiay"}, + {Dimension: DimMaxCostUSD, Shape: ShapeCumulative, Unit: "usd", Integral: false, Implemented: false, MatrixRow: 2, Owner: "aae-orc-qiay follow-on"}, + {Dimension: DimMaxTeamRSSBytes, Shape: ShapeCount, Unit: "bytes", Integral: true, Implemented: false, MatrixRow: 12, Owner: "aae-orc-hpeu"}, + {Dimension: DimMaxSessionCtxPct, Shape: ShapeLevel, Unit: "percent", Integral: false, Implemented: false, MatrixRow: 1, Owner: "aae-orc-hpeu"}, +} + +// Specs returns the dimension registry in evaluation order. A copy, so a +// caller cannot reorder the registry for everyone else. +func Specs() []Spec { return slices.Clone(budgetSpecs) } + +// LookupDimension returns the registry entry for a dimension. +func LookupDimension(d Dimension) (Spec, bool) { + for _, s := range budgetSpecs { + if s.Dimension == d { + return s, true + } + } + return Spec{}, false +} + +// DimensionList returns every registered dimension sorted and +// comma-joined, for inclusion in validation error messages. Mirrors +// permissionModeList. +func DimensionList() string { + names := make([]string, 0, len(budgetSpecs)) + for _, s := range budgetSpecs { + names = append(names, string(s.Dimension)) + } + sort.Strings(names) + return strings.Join(names, ", ") +} + +// UnmeasuredMode is what a declared clause does when the meter cannot +// answer. +// +// The default is admit: the operator declared a budget on a dimension, +// not "refuse when the meter cannot answer", and failing closed on absent +// data is a gate they did not ratify (ADR-007 clause 3, SOUL.md section +// 8). Declaring refuse IS that ratification, so the fail-closed posture +// stays available to an operator who wants it. Either way the degradation +// is audible: an admitted-but-unmeasured clause emits an event naming the +// clause and the reason. +type UnmeasuredMode string + +const ( + UnmeasuredAdmit UnmeasuredMode = "admit" + UnmeasuredRefuse UnmeasuredMode = "refuse" +) + +// canonicalUnmeasuredModes is the accepted set, checked at parse time so +// a typo does not silently resolve to the default. +var canonicalUnmeasuredModes = map[UnmeasuredMode]bool{ + UnmeasuredAdmit: true, + UnmeasuredRefuse: true, +} + +// unmeasuredModeList returns the valid modes sorted and comma-joined. +func unmeasuredModeList() string { + modes := make([]string, 0, len(canonicalUnmeasuredModes)) + for m := range canonicalUnmeasuredModes { + modes = append(modes, string(m)) + } + sort.Strings(modes) + return strings.Join(modes, ", ") +} + +// Budget is a team's declared resource ceiling. +// +// Every field is zero-means-unset: a team with no budget block declares +// no gate, and marvel's behavior for it is identical to before this field +// existed. A nonzero field is the operator's ratification instrument +// (ADR-007 clause 3, SOUL.md section 8) and the only thing that turns a +// metered value into a refusal. Nothing here is inferred from a metric, +// and no default ceiling exists. See aae-orc-qiay. +type Budget struct { + // MaxSessions caps live sessions across the whole team: every role, + // plus ad-hoc sessions attributed to the team. Counted from the store + // rather than a stream, so it is exact before a spawn and survives a + // daemon restart. + // + // A rolling shift is the one exemption, and it is deliberate: the new + // generation runs beside the old until draining finishes, so live can + // reach twice a role's replicas for the length of the rotation and the + // ceiling does not refuse it (replacement is not growth; see + // admission R5, and Request.Overlap). An operator sizing this against a + // hard external concurrency quota should size for that overlap, or + // avoid shifting a team that sits at its ceiling. + MaxSessions int `toml:"max_sessions,omitempty" json:"max_sessions,omitempty"` + // MaxTokens caps the team's layout-normalized prompt, output, and + // reasoning tokens SINCE ACCOUNTING BEGAN. The accountant is + // in-memory, so this window restarts with the daemon and with + // `marvel daemon reexec`; every figure marvel prints for it carries + // that window. No class weighting: marvel cannot see a provider's + // discount schedule, so any weighting would be invented. + MaxTokens int `toml:"max_tokens,omitempty" json:"max_tokens,omitempty"` + // OnUnmeasured is what a declared clause does when the meter cannot + // answer. Empty resolves to UnmeasuredAdmit. + OnUnmeasured UnmeasuredMode `toml:"on_unmeasured,omitempty" json:"on_unmeasured,omitempty"` +} + +// Declared reports whether any enforceable clause is set. Callers check +// this first so an undeclared budget costs nothing: no store read, no +// meter read, no event. +func (b Budget) Declared() bool { + return b.MaxSessions > 0 || b.MaxTokens > 0 +} + +// Unmeasured resolves the default. +func (b Budget) Unmeasured() UnmeasuredMode { + if b.OnUnmeasured == UnmeasuredRefuse { + return UnmeasuredRefuse + } + return UnmeasuredAdmit +} + +// Limit returns the declared ceiling for an implemented dimension. ok is +// false for an unset clause and for any dimension this slice does not +// enforce. +func (b Budget) Limit(d Dimension) (int, bool) { + switch d { + case DimMaxSessions: + return b.MaxSessions, b.MaxSessions > 0 + case DimMaxTokens: + return b.MaxTokens, b.MaxTokens > 0 + } + return 0, false +} + +// AdmissionState is the status half of Budget: the standing condition the +// reconciler recomputes every tick, surfaced by `marvel describe team`. +// +// Status on a spec record follows the Team.Shift precedent exactly +// (toml:"-", written through UpdateTeam on transitions only, never on +// every tick, so there is no bolt write storm). A copy rehydrated stale +// from bolt is corrected within one reconcile tick, because the condition +// is derived from live state rather than remembered. +type AdmissionState struct { + Held bool `json:"held,omitempty"` + Role string `json:"role,omitempty"` + Reason string `json:"reason,omitempty"` + Since time.Time `json:"since,omitempty"` +} + +// CountAlive returns how many of these sessions count toward replicas. +// Shared by the reconciler and the daemon so both compute "live" +// identically. +func CountAlive(sessions []Session) int { + n := 0 + for i := range sessions { + if sessions[i].State.CountsAsAlive() { + n++ + } + } + return n +} diff --git a/internal/api/budget_test.go b/internal/api/budget_test.go new file mode 100644 index 0000000..330d91e --- /dev/null +++ b/internal/api/budget_test.go @@ -0,0 +1,195 @@ +package api + +import ( + "strings" + "testing" +) + +// TestValidateManifestBudget covers the parse-time rules for a declared +// budget (aae-orc-qiay). The declaration clause is the load-bearing row: +// making sum(replicas) <= max_sessions an invariant of every parsed manifest +// is what lets the reconciler converge on declared replicas without a +// growth predicate, so repair can never be refused. +func TestValidateManifestBudget(t *testing.T) { + t.Parallel() + role := func(replicas int) ManifestRole { + return ManifestRole{Name: "crew", Replicas: replicas, Runtime: ManifestRuntime{Command: "sleep"}} + } + tests := []struct { + name string + team ManifestTeam + wantErr string + }{ + { + name: "no budget block is valid", + team: ManifestTeam{Name: "crew", Roles: []ManifestRole{role(40)}}, + }, + { + name: "an all-zero block declares no gate and is valid", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{}, Roles: []ManifestRole{role(40)}}, + }, + { + name: "declared replicas exactly at the ceiling is valid", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxSessions: 6}, Roles: []ManifestRole{role(6)}}, + }, + { + name: "declared replicas over the ceiling is refused", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxSessions: 6}, Roles: []ManifestRole{role(40)}}, + wantErr: "declares 40 replicas across 1 role(s) but budget.max_sessions is 6", + }, + { + name: "the replica sum spans every role", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxSessions: 6}, Roles: []ManifestRole{ + role(4), + {Name: "reviewer", Replicas: 4, Runtime: ManifestRuntime{Command: "sleep"}}, + }}, + wantErr: "declares 8 replicas across 2 role(s)", + }, + { + name: "a negative session ceiling is refused", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxSessions: -1}, Roles: []ManifestRole{role(1)}}, + wantErr: "budget.max_sessions must be >= 0", + }, + { + name: "a negative token ceiling is refused", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxTokens: -1}, Roles: []ManifestRole{role(1)}}, + wantErr: "budget.max_tokens must be >= 0", + }, + { + name: "an unenforced cost dimension names its owner", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxCostUSD: 5}, Roles: []ManifestRole{role(1)}}, + wantErr: "budget.max_cost_usd is a known dimension (matrix row 2) but is not enforced in this slice", + }, + { + name: "an unenforced memory dimension names its owner", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxTeamRSSBytes: 1 << 30}, Roles: []ManifestRole{role(1)}}, + wantErr: "budget.max_team_rss_bytes is a known dimension", + }, + { + name: "an unenforced occupancy dimension names its owner", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxSessionCtxPct: 80}, Roles: []ManifestRole{role(1)}}, + wantErr: "budget.max_session_ctx_percent is a known dimension", + }, + { + name: "a mistyped on_unmeasured lists the valid values", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxTokens: 10, OnUnmeasured: "deny"}, Roles: []ManifestRole{role(1)}}, + wantErr: `budget.on_unmeasured "deny" is not valid (valid: admit, refuse)`, + }, + { + name: "both valid on_unmeasured values pass", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxTokens: 10, OnUnmeasured: UnmeasuredRefuse}, Roles: []ManifestRole{role(1)}}, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := validateManifestBudget(0, tt.team) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil { + t.Fatalf("expected an error containing %q", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("error = %q, want it to contain %q", err, tt.wantErr) + } + }) + } +} + +// TestBudgetDefaultOpen pins the governance property as code: an undeclared +// budget is not a gate, and a declared one resolves its unmeasured mode +// rather than leaving it empty. +func TestBudgetDefaultOpen(t *testing.T) { + t.Parallel() + var zero Budget + if zero.Declared() { + t.Error("the zero Budget declares a gate; every pre-existing manifest would change behavior") + } + if _, ok := zero.Limit(DimMaxSessions); ok { + t.Error("an unset clause reported a limit") + } + if zero.Unmeasured() != UnmeasuredAdmit { + t.Errorf("default unmeasured mode = %q, want %q", zero.Unmeasured(), UnmeasuredAdmit) + } + + b := Budget{MaxSessions: 6, MaxTokens: 10, OnUnmeasured: UnmeasuredRefuse} + if !b.Declared() { + t.Error("a declared budget reported no gate") + } + if limit, ok := b.Limit(DimMaxSessions); !ok || limit != 6 { + t.Errorf("Limit(max_sessions) = (%d, %v), want (6, true)", limit, ok) + } + if b.Unmeasured() != UnmeasuredRefuse { + t.Errorf("unmeasured mode = %q, want %q", b.Unmeasured(), UnmeasuredRefuse) + } + // An unimplemented dimension must not report a limit even if someone + // later adds a field for it, or a gate would fire with no evaluator. + if _, ok := b.Limit(DimMaxCostUSD); ok { + t.Error("an unimplemented dimension reported a limit") + } +} + +// TestDimensionRegistry covers the registry contract: stable order, working +// lookup, and a caller who cannot reorder it for everyone else. +func TestDimensionRegistry(t *testing.T) { + t.Parallel() + specs := Specs() + if len(specs) != 5 { + t.Fatalf("got %d specs, want 5", len(specs)) + } + // Count-shaped rows evaluate before cumulative ones, so a session + // ceiling decides before a spend ceiling. + if specs[0].Dimension != DimMaxSessions || specs[1].Dimension != DimMaxTokens { + t.Errorf("registry order = %q, %q; want max_sessions then max_tokens", specs[0].Dimension, specs[1].Dimension) + } + specs[0].Dimension = "clobbered" + if Specs()[0].Dimension != DimMaxSessions { + t.Error("Specs returned the live registry: a caller reordered it for everyone") + } + + for _, want := range []Dimension{DimMaxSessions, DimMaxTokens, DimMaxCostUSD, DimMaxTeamRSSBytes, DimMaxSessionCtxPct} { + spec, ok := LookupDimension(want) + if !ok { + t.Errorf("LookupDimension(%q) not found", want) + continue + } + if spec.Dimension != want { + t.Errorf("LookupDimension(%q) returned %q", want, spec.Dimension) + } + if spec.Owner == "" { + t.Errorf("%q has no owner, so its validation error points nowhere", want) + } + } + if _, ok := LookupDimension("max_vibes"); ok { + t.Error("LookupDimension found an unregistered dimension") + } + + if got := DimensionList(); got != "max_cost_usd, max_session_ctx_percent, max_sessions, max_team_rss_bytes, max_tokens" { + t.Errorf("DimensionList() = %q", got) + } +} + +// TestCountAlive pins the shared definition of "live" so the reconciler and +// the daemon cannot drift into counting differently. +func TestCountAlive(t *testing.T) { + t.Parallel() + sessions := []Session{ + {State: SessionPending}, + {State: SessionRunning}, + {State: SessionCrashLoopBackOff}, + {State: SessionSucceeded}, + {State: SessionFailed}, + {State: SessionCrashed}, + } + if got := CountAlive(sessions); got != 3 { + t.Errorf("CountAlive = %d, want 3", got) + } + if got := CountAlive(nil); got != 0 { + t.Errorf("CountAlive(nil) = %d, want 0", got) + } +} diff --git a/internal/api/manifest.go b/internal/api/manifest.go index dd71b88..2dbb9a1 100644 --- a/internal/api/manifest.go +++ b/internal/api/manifest.go @@ -70,8 +70,44 @@ type ManifestWorkspace struct { // ManifestTeam is a team section of a manifest. type ManifestTeam struct { - Name string `toml:"name" yaml:"name"` - Roles []ManifestRole `toml:"role" yaml:"roles"` + Name string `toml:"name" yaml:"name"` + Budget *ManifestBudget `toml:"budget,omitempty" yaml:"budget,omitempty"` + Roles []ManifestRole `toml:"role" yaml:"roles"` +} + +// ManifestBudget is the budget section within a team — the operator's +// declaration that a metered value may refuse a spawn for this team. +// +// A pointer on ManifestTeam so "absent" and "all zeros" stay +// distinguishable in both formats. The three unimplemented dimensions are +// declared here and nowhere else: yaml.v3 and BurntSushi/toml both drop +// undeclared fields silently (the defect the DroppedFields tests exist to +// catch), so declaring them is what lets validation reject a manifest +// naming one instead of accepting it as a no-op. They are never copied to +// api.Budget. +type ManifestBudget struct { + MaxSessions int `toml:"max_sessions,omitempty" yaml:"max_sessions,omitempty"` + MaxTokens int `toml:"max_tokens,omitempty" yaml:"max_tokens,omitempty"` + OnUnmeasured UnmeasuredMode `toml:"on_unmeasured,omitempty" yaml:"on_unmeasured,omitempty"` + + MaxCostUSD float64 `toml:"max_cost_usd,omitempty" yaml:"max_cost_usd,omitempty"` + MaxTeamRSSBytes int64 `toml:"max_team_rss_bytes,omitempty" yaml:"max_team_rss_bytes,omitempty"` + MaxSessionCtxPct float64 `toml:"max_session_ctx_percent,omitempty" yaml:"max_session_ctx_percent,omitempty"` +} + +// Budget converts a declared block into the runtime ceiling. A nil +// receiver (no budget block) yields the zero Budget, which declares no +// gate. Only implemented dimensions cross over; the registered-but- +// unenforced ones are rejected at parse time and never reach here. +func (b *ManifestBudget) Budget() Budget { + if b == nil { + return Budget{} + } + return Budget{ + MaxSessions: b.MaxSessions, + MaxTokens: b.MaxTokens, + OnUnmeasured: b.OnUnmeasured, + } } // ManifestRole is a role section within a team. @@ -133,33 +169,73 @@ func ParseManifest(path string) (*Manifest, error) { } } -// ParseManifestBytes parses manifest content. Tries YAML first (default), -// falls back to TOML if YAML parsing fails. +// ParseManifestBytes parses manifest content whose format is not known +// from a filename: YAML first (the default), TOML otherwise. This is the +// path every `marvel work` takes, because the CLI sends bytes. +// +// Format is settled BEFORE validation runs, and on the required field +// rather than on unmarshal success. Validating inside each attempt made +// every YAML validation failure fall through to the TOML parser, so an +// operator who declared 40 replicas under a 6-session ceiling was told +// "toml: line 72: expected '.' or '='" instead of which clause they broke. +// That masked the declaration clause, the unenforced-dimension rejection, +// and the on_unmeasured typo check alike, on the only apply path there is. +// +// Deciding on Workspace.Name is what keeps TOML working: yaml.Unmarshal +// tolerates some TOML input and yields a manifest with nothing in it, and +// a manifest with no workspace name is not a YAML manifest marvel could +// have applied anyway. func ParseManifestBytes(data []byte) (*Manifest, error) { - // Try YAML first — it's the default format. - m, err := parseManifestYAML(data) - if err == nil { - return m, nil + ym, yerr := unmarshalManifestYAML(data) + if yerr == nil && ym.Workspace.Name != "" { + return validateManifest(ym) } - - // Fall back to TOML. - return parseManifestTOML(data) + tm, terr := unmarshalManifestTOML(data) + if terr == nil { + return validateManifest(tm) + } + if yerr != nil { + // YAML is the documented default, so its error leads; a TOML syntax + // complaint about a YAML file names the wrong language. The TOML + // error rides along for a genuine TOML file that failed to parse. + return nil, fmt.Errorf("%w (also tried TOML: %v)", yerr, terr) + } + // Parsed as YAML but named no workspace, and TOML refused it: report the + // missing required field rather than a syntax error about the other + // format. + return validateManifest(ym) } func parseManifestYAML(data []byte) (*Manifest, error) { + m, err := unmarshalManifestYAML(data) + if err != nil { + return nil, err + } + return validateManifest(m) +} + +func parseManifestTOML(data []byte) (*Manifest, error) { + m, err := unmarshalManifestTOML(data) + if err != nil { + return nil, err + } + return validateManifest(m) +} + +func unmarshalManifestYAML(data []byte) (*Manifest, error) { var m Manifest if err := yaml.Unmarshal(data, &m); err != nil { return nil, fmt.Errorf("parse yaml manifest: %w", err) } - return validateManifest(&m) + return &m, nil } -func parseManifestTOML(data []byte) (*Manifest, error) { +func unmarshalManifestTOML(data []byte) (*Manifest, error) { var m Manifest if err := toml.Unmarshal(data, &m); err != nil { return nil, fmt.Errorf("parse toml manifest: %w", err) } - return validateManifest(&m) + return &m, nil } func validateManifest(m *Manifest) (*Manifest, error) { @@ -209,10 +285,135 @@ func validateManifest(m *Manifest) (*Manifest, error) { return nil, fmt.Errorf("parse manifest: team[%d].role[%d].permissions %q is not a valid permission mode (valid: %s)", i, j, r.Permissions, permissionModeList()) } } + // After the role loop, so the replica sum is available to the + // declaration clause. + if err := validateManifestBudget(i, t); err != nil { + return nil, err + } } return m, nil } +// validateManifestBudget applies the dimension registry's rules plus the +// declaration clause to one team's budget block. +// +// The declaration clause (sum of replicas must fit under max_sessions) is +// the load-bearing one. It makes declared <= limit an invariant of every +// parsed manifest, which is what makes converging a role toward its +// declared replicas provably safe: repair can never cross the team cap, so +// no "is this growth?" predicate exists anywhere in the reconciler and a +// crashed replica can never be refused its replacement. It also catches +// the motivating failure (a declared 40-crew fan-out under a 6-session +// ceiling) before any daemon state is touched. +func validateManifestBudget(i int, t ManifestTeam) error { + b := t.Budget + if b == nil { + return nil + } + // A negative ceiling silently inverts the comparison; 0 means unset. + // Same rationale as runtime.context_window above. + if b.MaxSessions < 0 { + return fmt.Errorf("parse manifest: team[%d].budget.max_sessions must be >= 0", i) + } + if b.MaxTokens < 0 { + return fmt.Errorf("parse manifest: team[%d].budget.max_tokens must be >= 0", i) + } + // Registered-but-unenforced dimensions are rejected rather than + // dropped, driven by the registry so a future row needs no new branch. + unenforced := []struct { + dim Dimension + set bool + }{ + {DimMaxCostUSD, b.MaxCostUSD != 0}, + {DimMaxTeamRSSBytes, b.MaxTeamRSSBytes != 0}, + {DimMaxSessionCtxPct, b.MaxSessionCtxPct != 0}, + } + for _, u := range unenforced { + if !u.set { + continue + } + spec, ok := LookupDimension(u.dim) + if !ok { + return fmt.Errorf("parse manifest: team[%d].budget.%s is not a known dimension (valid: %s)", i, u.dim, DimensionList()) + } + return fmt.Errorf("parse manifest: team[%d].budget.%s is a known dimension (matrix row %d) but is not enforced in this slice; owner %s", i, u.dim, spec.MatrixRow, spec.Owner) + } + if b.OnUnmeasured != "" && !canonicalUnmeasuredModes[b.OnUnmeasured] { + return fmt.Errorf("parse manifest: team[%d].budget.on_unmeasured %q is not valid (valid: %s)", i, b.OnUnmeasured, unmeasuredModeList()) + } + if b.MaxSessions > 0 { + declared := 0 + for _, r := range t.Roles { + declared += r.Replicas + } + if declared > b.MaxSessions { + return fmt.Errorf("parse manifest: team[%d] declares %d replicas across %d role(s) but budget.max_sessions is %d", i, declared, len(t.Roles), b.MaxSessions) + } + } + return nil +} + +// StreamCapableRole reports whether a role's harness can publish the usage +// stream a token ceiling is measured from. +// +// Injected rather than computed here because the answer lives in the +// adapter registry, and internal/runtime imports this package: mode alone +// cannot answer it. A generic role declaring mode: headless satisfies every +// mode check and still can never emit a token, because the generic adapter +// implements no stream path at all. +type StreamCapableRole func(ManifestRole) bool + +// ValidateBudgets is the host-side pre-flight sibling of ValidateRuntimes: +// it reports every team where a declared dimension is enforceable against +// NO role in the team, so a mute gate becomes an apply-time error instead +// of a silent no-op. Same class of fix as ArcavenAE/marvel#9. +// +// Only max_tokens is capability-dependent. Token usage arrives on a harness +// stream, so a team with no stream-capable headless role can never report a +// token to count. The threshold is NO role rather than ANY role: a mixed +// team is allowed, because a partial total and on_unmeasured carry the +// honesty at runtime. max_sessions is counted from the store and depends on +// no harness. +// +// canStream is required. A nil predicate is a wiring error and is reported +// as one, because the alternative (falling back to the mode-only check) is +// the silent hole this function exists to close. +func (m *Manifest) ValidateBudgets(canStream StreamCapableRole) error { + declaresTokens := false + for _, t := range m.Teams { + if t.Budget != nil && t.Budget.MaxTokens > 0 { + declaresTokens = true + break + } + } + if !declaresTokens { + return nil + } + if canStream == nil { + return errors.New("budget pre-flight: no stream-capability predicate supplied, so budget.max_tokens cannot be checked for a role that could report it") + } + var mute []string + for ti, t := range m.Teams { + if t.Budget == nil || t.Budget.MaxTokens <= 0 { + continue + } + reporter := false + for _, r := range t.Roles { + if canStream(r) { + reporter = true + break + } + } + if !reporter { + mute = append(mute, fmt.Sprintf(" team[%d=%s]: budget.max_tokens is declared but no role runs a stream-capable harness in headless mode, so no role can report token usage; marvel would never enforce this ceiling", ti, t.Name)) + } + } + if len(mute) > 0 { + return fmt.Errorf("budget pre-flight failed on %d team(s):\n%s", len(mute), strings.Join(mute, "\n")) + } + return nil +} + // ValidateRuntimes checks that each role's runtime command (and script, // if set) actually resolves on the daemon's host before the manifest // is applied. Returns an aggregated error listing every missing binary @@ -364,18 +565,24 @@ func (m *Manifest) Apply(store *Store) error { roles = append(roles, role) } + budget := mt.Budget.Budget() team := &Team{ Name: mt.Name, Workspace: m.Workspace.Name, Roles: roles, + Budget: budget, Generation: 1, CreatedAt: now, } // Update roles if team already exists; route through the store - // lock so the mutation doesn't race concurrent readers. + // lock so the mutation doesn't race concurrent readers. The budget + // moves with the roles: without that line an edited budget applies + // on create and is silently ignored on every re-apply, while an + // edited role list takes effect — the worst available split. if _, err := store.GetTeam(team.Key()); err == nil { if err := store.UpdateTeam(team.Key(), func(live *Team) error { live.Roles = roles + live.Budget = budget return nil }); err != nil { return fmt.Errorf("apply team %s: %w", mt.Name, err) diff --git a/internal/api/manifest_test.go b/internal/api/manifest_test.go index 567a09b..d6d73a6 100644 --- a/internal/api/manifest_test.go +++ b/internal/api/manifest_test.go @@ -1,6 +1,7 @@ package api import ( + "fmt" "strings" "testing" "time" @@ -261,6 +262,117 @@ func TestParseYAMLManifest(t *testing.T) { } } +// TestParseManifestBytesReportsYAMLValidationErrors covers the only apply +// path there is: `marvel work` sends bytes, so every manifest the daemon sees +// arrives through ParseManifestBytes. +// +// Validating inside each format attempt made ANY validation failure on a YAML +// manifest fall through to the TOML parser, so the operator was told +// "toml: line N: expected '.' or '=', but got ':' instead" instead of which +// rule they broke. It masked every rule equally, which is why the rows below +// include one that predates budgets. +func TestParseManifestBytesReportsYAMLValidationErrors(t *testing.T) { + t.Parallel() + yamlWith := func(budget, replicas string) string { + return ` +workspace: + name: fanout + +teams: + - name: crew + budget: +` + budget + ` + roles: + - name: crew + replicas: ` + replicas + ` + runtime: + image: generic + command: sleep +` + } + tests := []struct { + name string + manifest string + wantErr string + }{ + { + name: "the declaration clause", + manifest: yamlWith(" max_sessions: 6", "40"), + wantErr: "declares 40 replicas across 1 role(s) but budget.max_sessions is 6", + }, + { + name: "a registered but unenforced dimension", + manifest: yamlWith(" max_cost_usd: 5.0", "1"), + wantErr: "is a known dimension (matrix row", + }, + { + name: "an on_unmeasured typo", + manifest: yamlWith(" max_sessions: 6\n on_unmeasured: deny", "1"), + wantErr: `on_unmeasured "deny" is not valid`, + }, + { + name: "a rule that predates budgets", + manifest: yamlWith(" max_sessions: 6", "0"), + wantErr: "replicas must be >= 1", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, err := ParseManifestBytes([]byte(tt.manifest)) + if err == nil { + t.Fatalf("expected an error containing %q", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("error = %q, want it to contain %q", err, tt.wantErr) + } + if strings.Contains(err.Error(), "toml") { + t.Errorf("error = %q, want no TOML complaint about a YAML manifest", err) + } + }) + } +} + +// TestParseManifestBytesStillAcceptsTOML is the other half of the format +// decision above: settling the format on the required field rather than on +// unmarshal success is what keeps TOML working, since yaml.Unmarshal tolerates +// some TOML input and yields a manifest with nothing in it. +func TestParseManifestBytesStillAcceptsTOML(t *testing.T) { + t.Parallel() + m, err := ParseManifestBytes([]byte(validManifest)) + if err != nil { + t.Fatalf("parse TOML through the bytes path: %v", err) + } + if m.Workspace.Name != "test-project" || len(m.Teams) != 1 { + t.Fatalf("manifest = %+v, want the TOML fixture's workspace and team", m) + } + // A TOML validation failure keeps naming the rule, not the format. + _, err = ParseManifestBytes([]byte(` +[workspace] +name = "fanout" + +[[team]] +name = "crew" + + [team.budget] + max_sessions = 6 + + [[team.role]] + name = "crew" + replicas = 40 + + [team.role.runtime] + command = "sleep" +`)) + if err == nil { + t.Fatal("expected the declaration clause to refuse a TOML manifest too") + } + if !strings.Contains(err.Error(), "budget.max_sessions is 6") { + t.Errorf("error = %q, want the declaration clause", err) + } +} + func TestParseYAMLManifestWithHealthcheck(t *testing.T) { t.Parallel() m, err := parseManifestYAML([]byte(` @@ -555,6 +667,283 @@ teams: } } +// TestParseYAMLManifestBudget extends the DroppedFields trio to the budget +// block: yaml.v3 drops fields the struct does not declare, so a budget that +// parses into nothing would be an accepted no-op gate. +func TestParseYAMLManifestBudget(t *testing.T) { + t.Parallel() + m, err := parseManifestYAML([]byte(` +workspace: + name: fanout + +teams: + - name: crew + budget: + max_sessions: 6 + max_tokens: 2000000 + on_unmeasured: refuse + roles: + - name: crew + replicas: 3 + runtime: + command: sleep + args: ["300"] +`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + mb := m.Teams[0].Budget + if mb == nil { + t.Fatal("budget block on ManifestTeam: got nil, want the declared block") + } + if mb.MaxSessions != 6 { + t.Errorf("MaxSessions on ManifestBudget: got %d, want 6", mb.MaxSessions) + } + if mb.MaxTokens != 2000000 { + t.Errorf("MaxTokens on ManifestBudget: got %d, want 2000000", mb.MaxTokens) + } + if mb.OnUnmeasured != UnmeasuredRefuse { + t.Errorf("OnUnmeasured on ManifestBudget: got %q, want %q", mb.OnUnmeasured, UnmeasuredRefuse) + } + + store := NewStore() + if err := m.Apply(store); err != nil { + t.Fatalf("apply: %v", err) + } + team, err := store.GetTeam("fanout/crew") + if err != nil { + t.Fatalf("get team: %v", err) + } + if team.Budget.MaxSessions != 6 || team.Budget.MaxTokens != 2000000 { + t.Errorf("Budget on api.Team after Apply: got %+v", team.Budget) + } + if !team.Budget.Declared() { + t.Error("a declared budget did not survive Apply") + } +} + +// TestParseTOMLManifestBudget is the TOML twin. Every example under +// examples/ ships as a pair, so both formats have to carry the block. +func TestParseTOMLManifestBudget(t *testing.T) { + t.Parallel() + m, err := parseManifestTOML([]byte(` +[workspace] +name = "fanout" + +[[team]] +name = "crew" + + [team.budget] + max_sessions = 6 + max_tokens = 2000000 + + [[team.role]] + name = "crew" + replicas = 3 + + [team.role.runtime] + command = "sleep" + args = ["300"] +`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if m.Teams[0].Budget == nil { + t.Fatal("budget block on ManifestTeam: got nil, want the declared block") + } + if m.Teams[0].Budget.MaxSessions != 6 { + t.Errorf("MaxSessions on ManifestBudget: got %d, want 6", m.Teams[0].Budget.MaxSessions) + } + + store := NewStore() + if err := m.Apply(store); err != nil { + t.Fatalf("apply: %v", err) + } + team, _ := store.GetTeam("fanout/crew") + if team.Budget.MaxTokens != 2000000 { + t.Errorf("MaxTokens on api.Team after Apply: got %d, want 2000000", team.Budget.MaxTokens) + } +} + +// TestParseManifestBudgetDefaults verifies that omitting the block leaves a +// zero Budget, which declares no gate. This is the default-open guarantee +// for every manifest written before the feature existed. +func TestParseManifestBudgetDefaults(t *testing.T) { + t.Parallel() + m, err := ParseManifestBytes([]byte(validManifest)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if m.Teams[0].Budget != nil { + t.Errorf("absent budget block parsed as %+v, want nil", m.Teams[0].Budget) + } + store := NewStore() + if err := m.Apply(store); err != nil { + t.Fatalf("apply: %v", err) + } + team, _ := store.GetTeam("test-project/squad") + if team.Budget.Declared() { + t.Errorf("a manifest with no budget block produced a gate: %+v", team.Budget) + } +} + +// TestReapplyCarriesAnEditedBudget is the fourth member of the quartet and +// the one that catches a real trap. Apply's existing-team branch copies only +// the fields it names: without live.Budget in that closure, an edited budget +// applies on create and is silently ignored on every re-apply, while an +// edited role list takes effect. +func TestReapplyCarriesAnEditedBudget(t *testing.T) { + t.Parallel() + manifest := func(maxSessions, replicas int) []byte { + return []byte(fmt.Sprintf(` +workspace: + name: fanout + +teams: + - name: crew + budget: + max_sessions: %d + roles: + - name: crew + replicas: %d + runtime: + command: sleep + args: ["300"] +`, maxSessions, replicas)) + } + + store := NewStore() + first, err := parseManifestYAML(manifest(6, 3)) + if err != nil { + t.Fatalf("parse #1: %v", err) + } + if err := first.Apply(store); err != nil { + t.Fatalf("apply #1: %v", err) + } + + second, err := parseManifestYAML(manifest(40, 8)) + if err != nil { + t.Fatalf("parse #2: %v", err) + } + if err := second.Apply(store); err != nil { + t.Fatalf("apply #2: %v", err) + } + + team, err := store.GetTeam("fanout/crew") + if err != nil { + t.Fatalf("get team: %v", err) + } + if team.Roles[0].Replicas != 8 { + t.Fatalf("re-apply dropped the edited role list: replicas = %d, want 8", team.Roles[0].Replicas) + } + if team.Budget.MaxSessions != 40 { + t.Errorf("re-apply dropped the edited budget: max_sessions = %d, want 40", team.Budget.MaxSessions) + } +} + +// TestValidateBudgets covers the host-side pre-flight: a token ceiling on a +// team where no role can ever report a token is a mute gate, and a mute gate +// is an apply-time error rather than a silent no-op. Same class of fix as +// ArcavenAE/marvel#9. +// +// The predicate is injected because the answer lives in the adapter registry +// (internal/runtime imports this package). Mode alone is not the question: a +// role whose harness has no stream path satisfies every mode check and can +// still never report a token, which is the exact mute gate this rejects. +func TestValidateBudgets(t *testing.T) { + t.Parallel() + role := func(name string, mode RuntimeMode) ManifestRole { + return ManifestRole{Name: name, Replicas: 1, Runtime: ManifestRuntime{Image: "claude", Command: "claude", Mode: mode}} + } + // Stands in for the registry-backed predicate: headless AND a harness + // with a stream path. + canStream := func(r ManifestRole) bool { + switch r.Runtime.Image { + case "claude", "codex", "opencode": + return r.Runtime.Mode == RuntimeModeHeadless + default: + return false + } + } + tests := []struct { + name string + team ManifestTeam + // noPredicate passes nil where the daemon passes the registry-backed + // predicate, which is a wiring error rather than a manifest problem. + noPredicate bool + wantErr string + wantsTeam bool + }{ + { + name: "a token ceiling with no headless role is refused", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxTokens: 10}, Roles: []ManifestRole{role("crew", RuntimeModeInteractive)}}, + wantErr: "budget.max_tokens is declared but no role runs a stream-capable harness in headless mode", + wantsTeam: true, + }, + { + // The hole a mode-only check left: generic implements no stream + // path, so OBSERVED can never move off zero and the ceiling can + // never refuse. + name: "a headless role whose harness cannot stream is refused", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxTokens: 10}, Roles: []ManifestRole{ + {Name: "crew", Replicas: 1, Runtime: ManifestRuntime{Image: "generic", Command: "sleep", Mode: RuntimeModeHeadless}}, + }}, + wantErr: "no role runs a stream-capable harness in headless mode", + wantsTeam: true, + }, + { + name: "a mixed team is allowed; partiality carries the honesty at runtime", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxTokens: 10}, Roles: []ManifestRole{ + role("crew", RuntimeModeInteractive), + role("reviewer", RuntimeModeHeadless), + }}, + }, + { + name: "a session ceiling depends on no harness", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxSessions: 6}, Roles: []ManifestRole{role("crew", RuntimeModeInteractive)}}, + }, + { + name: "no budget declares nothing to pre-flight", + team: ManifestTeam{Name: "crew", Roles: []ManifestRole{role("crew", RuntimeModeInteractive)}}, + }, + { + // A nil predicate is a wiring error, reported as one. Falling back + // to the mode-only check would restore the silent hole. + name: "a missing predicate is an error, not a fallback", + team: ManifestTeam{Name: "crew", Budget: &ManifestBudget{MaxTokens: 10}, Roles: []ManifestRole{role("crew", RuntimeModeHeadless)}}, + noPredicate: true, + wantErr: "no stream-capability predicate supplied", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + m := &Manifest{Workspace: ManifestWorkspace{Name: "fanout"}, Teams: []ManifestTeam{tt.team}} + pred := canStream + if tt.noPredicate { + pred = nil + } + err := m.ValidateBudgets(pred) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil { + t.Fatalf("expected an error containing %q", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("error = %q, want it to contain %q", err, tt.wantErr) + } + if tt.wantsTeam && !strings.Contains(err.Error(), "team[0=crew]") { + t.Errorf("error = %q, want it to name the team", err) + } + }) + } +} + func TestValidateRuntimesOK(t *testing.T) { t.Parallel() // Any two binaries guaranteed on POSIX test hosts. diff --git a/internal/api/store_test.go b/internal/api/store_test.go index c5c7f4d..9f869e7 100644 --- a/internal/api/store_test.go +++ b/internal/api/store_test.go @@ -384,3 +384,45 @@ func TestUpdateSessionHeartbeatStampsContextAt(t *testing.T) { t.Errorf("a heartbeat invented a window, a token count, or a request count: %+v", got.SessionContext) } } + +// TestCloneTeamCopiesBudget pins the store's snapshot contract for the two +// fields aae-orc-qiay added. Both are flat value structs, so cloneTeam's +// `out := *t` already deep-copies them — this test is what will fail loudly +// if either later grows a slice, map, or pointer and nobody extends the +// clone. See go.md rule 12 and orc finding-032. +func TestCloneTeamCopiesBudget(t *testing.T) { + t.Parallel() + s := NewStore() + if err := s.CreateWorkspace(&Workspace{Name: "fanout", CreatedAt: time.Now().UTC()}); err != nil { + t.Fatalf("create workspace: %v", err) + } + if err := s.CreateTeam(&Team{ + Name: "crew", + Workspace: "fanout", + Roles: []Role{{Name: "crew", Replicas: 3, Runtime: Runtime{Command: "sleep"}}}, + Budget: Budget{MaxSessions: 6, MaxTokens: 2_000_000}, + Admission: AdmissionState{Held: true, Role: "crew", Reason: "refused"}, + CreatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("create team: %v", err) + } + + snap, err := s.GetTeam("fanout/crew") + if err != nil { + t.Fatalf("get team: %v", err) + } + snap.Budget.MaxSessions = 999 + snap.Budget.OnUnmeasured = UnmeasuredRefuse + snap.Admission = AdmissionState{} + + live, err := s.GetTeam("fanout/crew") + if err != nil { + t.Fatalf("get team again: %v", err) + } + if live.Budget.MaxSessions != 6 || live.Budget.OnUnmeasured != "" { + t.Errorf("mutating a snapshot changed store state: %+v", live.Budget) + } + if !live.Admission.Held || live.Admission.Role != "crew" { + t.Errorf("mutating a snapshot cleared the store's admission state: %+v", live.Admission) + } +} diff --git a/internal/api/types.go b/internal/api/types.go index 75c2ea8..35801ef 100644 --- a/internal/api/types.go +++ b/internal/api/types.go @@ -259,12 +259,19 @@ type ShiftState struct { // Team declares desired state: a cohesive unit of agents with heterogeneous roles. type Team struct { - Name string `toml:"name"` - Workspace string `toml:"workspace"` - Roles []Role `toml:"role"` + Name string `toml:"name"` + Workspace string `toml:"workspace"` + Roles []Role `toml:"role"` + // Budget is the team's declared resource ceiling. The zero value + // declares no gate, which is every manifest written before this field + // existed. See budget.go and aae-orc-qiay. + Budget Budget `toml:"budget,omitempty"` Generation int64 `toml:"-"` Shift ShiftState `toml:"-"` - CreatedAt time.Time + // Admission is the standing admission condition the reconciler + // recomputes each tick. Status, not spec — same treatment as Shift. + Admission AdmissionState `toml:"-"` + CreatedAt time.Time } // Endpoint is a stable name for a session role (service equivalent). diff --git a/internal/daemon/admission.go b/internal/daemon/admission.go new file mode 100644 index 0000000..e6dd749 --- /dev/null +++ b/internal/daemon/admission.go @@ -0,0 +1,184 @@ +package daemon + +import ( + "fmt" + "log" + + "github.com/arcavenae/marvel/internal/admission" + "github.com/arcavenae/marvel/internal/api" + "github.com/arcavenae/marvel/internal/events" +) + +// AdmissionSnapshot builds the measured state one admission check +// evaluates for a team: exact store counts plus one TeamSpend call, +// resolving the meter's absence states into flags. +// +// A nil accountant yields TokensMetered false rather than a zero total, +// because "no meter" and "spent nothing" must not read alike — a gate that +// reads an unresolved measurement as zero admits everything (see +// internal/usage/reader.go). Satisfies team.Snapshotter. +func (d *Daemon) AdmissionSnapshot(t api.Team) admission.Snapshot { + s := admission.Snapshot{ + Workspace: t.Workspace, + Team: t.Name, + LiveSessions: api.CountAlive(d.store.ListSessionsByTeam(t.Workspace, t.Name)), + } + for i := range t.Roles { + s.DeclaredSessions += t.Roles[i].Replicas + } + if d.usage == nil { + return s + } + tt := d.usage.TeamSpend(t.Workspace, t.Name) + s.TokensMetered = true + // PromptTokens is the layout-normalized prompt figure; summing the raw + // class fields would double count a subsumptive feed. Output and + // reasoning tokens carry no layout. + s.TokensObserved = tt.PromptTokens + tt.Out + tt.ReasoningOut + s.TokensSeen = tt.LiveSessions + tt.EndedSessions + s.Since = tt.Since + // TeamTotals.Partial does not cover a pane adopted from a prior daemon. + // Bind runs only inside attachInstance, which only Manager.Create calls, + // so an adopted session has no accountant state at all: it contributes + // nothing to the total AND does not set Partial. Comparing the store's + // live count against the meter's catches it. + unobserved := s.LiveSessions - tt.LiveSessions + s.TokensPartial = tt.Partial || unobserved > 0 + s.TokensSuspect = d.usage.Stats().CumulationViolations > 0 + return s +} + +// admitGrowth is the synchronous gate the operator's verbs consult. It +// returns nil when the action is admitted, or a populated error Response +// when refused. +// +// Whole-or-nothing on purpose: a declaration is operator intent, and +// marvel silently applying 6 replicas when the operator asked for 40 would +// be marvel editing intent, which is worse than refusing. The reconciler +// takes the opposite side (partial grants), because convergence is repair +// and some is better than none. +func (d *Daemon) admitGrowth(t api.Team, role string, want int, trig admission.Trigger) *Response { + if !t.Budget.Declared() || want <= 0 { + return nil + } + v := admission.Check(t.Budget, d.AdmissionSnapshot(t), admission.Request{ + Role: role, + Want: want, + Kind: admission.Growth, + }) + reason := v.Reason(trig) + switch v.Decision { + case admission.Refuse: + log.Printf("admission: %s refused: %s", t.Key(), reason) + events.Emit(d.events, events.Event{ + Kind: events.KindAdmissionRefused, + Severity: events.SeverityWarning, + Workspace: t.Workspace, + Team: t.Name, + Role: role, + Message: reason, + }) + return &Response{Error: fmt.Sprintf( + "%s: %s. Nothing changed; raise the budget in the manifest or free headroom first", + t.Key(), reason, + )} + case admission.Indeterminate: + log.Printf("admission: %s admitted unmeasured: %s", t.Key(), reason) + events.Emit(d.events, events.Event{ + Kind: events.KindAdmissionUnmeasured, + Severity: events.SeverityWarning, + Workspace: t.Workspace, + Team: t.Name, + Role: role, + Message: reason, + }) + } + return nil +} + +// admitDeclaration enforces the declaration clause at the one verb that can +// raise a replica count without passing through the manifest parser. +// +// api.validateManifestBudget gives handleApply the invariant everything else +// rests on: sum(role.Replicas) <= max_sessions, which is why converging a +// role toward its declared replicas is provably safe (admission R1). Scale +// had only the spawn gate, which compares LIVE sessions against the ceiling. +// Whenever live < declared, and that is the normal state during a crash loop +// (which is also when an operator reaches for scale), a scale-up was +// admitted while committing a declaration the parser refuses. +// +// The result is exactly what handleApply's gate exists to prevent: a +// permanently unsatisfiable desired state. The last headroom slot goes to +// whichever role sorts earlier in manifest order, the deficient role is held +// forever, and nothing retries, because a refusal never bumps RestartCount +// and so no backoff ever re-fires. +// +// Whole-or-nothing and never a partial edit: a replica count is operator +// intent, and marvel committing a number the operator did not ask for would +// be marvel editing intent. +func (d *Daemon) admitDeclaration(t api.Team, role string, replicas, old int) *Response { + if t.Budget.MaxSessions <= 0 || replicas <= old { + return nil + } + declared := replicas - old + for i := range t.Roles { + declared += t.Roles[i].Replicas + } + if declared <= t.Budget.MaxSessions { + return nil + } + reason := fmt.Sprintf( + "refused role %s at %d replica(s): the team would declare %d sessions across %d role(s) against max_sessions=%d (trigger=%s)", + role, replicas, declared, len(t.Roles), t.Budget.MaxSessions, admission.TriggerScale, + ) + log.Printf("admission: %s refused: %s", t.Key(), reason) + events.Emit(d.events, events.Event{ + Kind: events.KindAdmissionRefused, + Severity: events.SeverityWarning, + Workspace: t.Workspace, + Team: t.Name, + Role: role, + Message: reason, + }) + return &Response{Error: fmt.Sprintf( + "%s: %s. Nothing changed; raise the budget in the manifest or scale another role down first", + t.Key(), reason, + )} +} + +// budgetRows assembles `marvel get budgets`: one row per declared +// dimension per team, none for a team that declares no budget. +// +// This is the only surface in marvel that answers "which dimension tripped +// and by how much". Nothing else exposes a spend or occupancy aggregate, +// and `marvel get teams` is deliberately left alone: a budget column would +// change output for every operator for a feature most teams do not declare. +func (d *Daemon) budgetRows() []admission.Row { + var out []admission.Row + for _, t := range d.store.ListTeams() { + if !t.Budget.Declared() { + continue + } + out = append(out, admission.Rows(t, d.AdmissionSnapshot(t))...) + } + return out +} + +// logTokenBudgetWindows says out loud, once at daemon start, that a +// cumulative token budget counts from now. +// +// The accountant has no bolt bucket, so TeamSpend resets on a daemon +// restart and on `marvel daemon reexec` (which keeps agents alive but not +// accountant state). Persisting stream-derived readings is a separate +// decision this slice does not make, so the limit is stated rather than +// hidden: a silent reset would be the same class of defect as a guessed +// denominator. +func (d *Daemon) logTokenBudgetWindows() { + for _, t := range d.store.ListTeams() { + if t.Budget.MaxTokens <= 0 { + continue + } + log.Printf("admission: %s declares budget.max_tokens=%d, counted since accounting started now (the meter is in-memory, so this window restarts with the daemon)", + t.Key(), t.Budget.MaxTokens) + } +} diff --git a/internal/daemon/admission_test.go b/internal/daemon/admission_test.go new file mode 100644 index 0000000..cb8adb6 --- /dev/null +++ b/internal/daemon/admission_test.go @@ -0,0 +1,528 @@ +package daemon + +import ( + "encoding/json" + "log" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/arcavenae/marvel/internal/admission" + "github.com/arcavenae/marvel/internal/api" + "github.com/arcavenae/marvel/internal/events" +) + +// newHandlerDaemon builds a daemon and calls its RPC handlers directly, with +// no socket and no reconcile goroutine. The admission gates are synchronous +// handler code, so nothing here needs the listener. +func newHandlerDaemon(t *testing.T) *Daemon { + t.Helper() + skipIfNoTmux(t) + d, err := New() + if err != nil { + t.Fatalf("new daemon: %v", err) + } + t.Cleanup(func() { + for _, ws := range d.store.ListWorkspaces() { + _ = d.sessMgr.CleanupWorkspace(ws.Name) + } + }) + return d +} + +func applyManifest(t *testing.T, d *Daemon, manifest string) Response { + t.Helper() + params, err := json.Marshal(map[string]any{"manifest_data": []byte(manifest)}) + if err != nil { + t.Fatalf("marshal apply params: %v", err) + } + return d.handleApply(params) +} + +// budgetedManifest is a team at its declared ceiling: three sleep replicas +// under a six-session budget. Deterministic, and needs no model auth. +const budgetedManifest = ` +workspace: + name: fanout +teams: + - name: crew + budget: + max_sessions: 3 + roles: + - name: crew + replicas: 3 + runtime: + command: sleep + args: ["300"] +` + +// TestHandleScaleRefusesOverBudget is the "refuse the declaration, not the +// spawn" contract as an assertion. +// +// Falsification: gating only the reconciler's spawn would report "scaled", +// leave Replicas at 40, show 3 sessions, and re-decide the same impossible +// deficit every 2s forever. Here the verb fails, and the store never learns +// a number that can never be true. +func TestHandleScaleRefusesOverBudget(t *testing.T) { + d := newHandlerDaemon(t) + if resp := applyManifest(t, d, budgetedManifest); resp.Error != "" { + t.Fatalf("apply: %s", resp.Error) + } + + resp := d.handleScale(mustMarshal(t, scaleParams{TeamKey: "fanout/crew", Role: "crew", Replicas: 40})) + if resp.Error == "" { + t.Fatal("expected an error scaling past the declared ceiling") + } + for _, want := range []string{"3 live", "37 requested", "max_sessions=3", "trigger=scale", "Nothing changed"} { + if !strings.Contains(resp.Error, want) { + t.Errorf("error = %q, missing %q", resp.Error, want) + } + } + + team, err := d.store.GetTeam("fanout/crew") + if err != nil { + t.Fatalf("get team: %v", err) + } + if team.Roles[0].Replicas != 3 { + t.Errorf("replicas = %d after a refused scale, want 3", team.Roles[0].Replicas) + } + if got := len(d.events.Snapshot(events.Filter{Kind: events.KindAdmissionRefused}, 0)); got != 1 { + t.Errorf("admission.refused events = %d, want 1", got) + } +} + +// TestHandleScaleDownIsNeverRefused: shedding sessions is how an operator +// frees headroom, so a request that adds nothing never reaches the gate. +func TestHandleScaleDownIsNeverRefused(t *testing.T) { + d := newHandlerDaemon(t) + if resp := applyManifest(t, d, budgetedManifest); resp.Error != "" { + t.Fatalf("apply: %s", resp.Error) + } + + if resp := d.handleScale(mustMarshal(t, scaleParams{TeamKey: "fanout/crew", Role: "crew", Replicas: 1})); resp.Error != "" { + t.Fatalf("a scale-down was refused: %s", resp.Error) + } + team, _ := d.store.GetTeam("fanout/crew") + if team.Roles[0].Replicas != 1 { + t.Errorf("replicas = %d after scale-down, want 1", team.Roles[0].Replicas) + } +} + +// TestHandleScaleRefusesADeclarationOverTheCeiling covers the second door into +// an over-ceiling declaration. +// +// The spawn gate compares LIVE sessions, so it cannot see this: with a replica +// dead, live sits below declared and live+want still fits under the ceiling +// while the declared sum does not. That is the normal state during a crash +// loop, which is also when an operator reaches for scale. +// +// Falsification: without the declaration clause at the verb, this scale +// returns "scaled", sum(Replicas) goes to 4 under max_sessions=3, and R1 +// (declared <= limit) stops holding. The reconciler then refuses the deficient +// role forever, since a refusal never bumps RestartCount and no backoff +// re-fires. +func TestHandleScaleRefusesADeclarationOverTheCeiling(t *testing.T) { + d := newHandlerDaemon(t) + if resp := applyManifest(t, d, budgetedManifest); resp.Error != "" { + t.Fatalf("apply: %s", resp.Error) + } + + // One replica dies without being reaped: live 2, declared 3, ceiling 3. + sessions := d.store.ListSessionsByTeam("fanout", "crew") + if len(sessions) != 3 { + t.Fatalf("got %d session(s) after apply, want 3", len(sessions)) + } + if err := d.store.UpdateSession(sessions[0].Key(), func(s *api.Session) error { + s.State = api.SessionCrashed + return nil + }); err != nil { + t.Fatalf("mark session crashed: %v", err) + } + if live := api.CountAlive(d.store.ListSessionsByTeam("fanout", "crew")); live != 2 { + t.Fatalf("live = %d, want 2 (the window this gate covers)", live) + } + + resp := d.handleScale(mustMarshal(t, scaleParams{TeamKey: "fanout/crew", Role: "crew", Replicas: 4})) + if resp.Error == "" { + t.Fatal("expected a scale that declares 4 under a 3-session ceiling to be refused") + } + for _, want := range []string{"role crew at 4 replica(s)", "declare 4 sessions", "max_sessions=3", "trigger=scale", "Nothing changed"} { + if !strings.Contains(resp.Error, want) { + t.Errorf("error = %q, missing %q", resp.Error, want) + } + } + team, err := d.store.GetTeam("fanout/crew") + if err != nil { + t.Fatalf("get team: %v", err) + } + if team.Roles[0].Replicas != 3 { + t.Errorf("replicas = %d after a refused scale, want 3", team.Roles[0].Replicas) + } + // Never silent: the refusal carries its arithmetic into the event ring the + // same way the spawn gate's does. + if got := len(d.events.Snapshot(events.Filter{Kind: events.KindAdmissionRefused}, 0)); got != 1 { + t.Errorf("admission.refused events = %d, want 1", got) + } + + // The same live count still admits a scale that keeps the declaration + // under the ceiling, so the new gate refuses declarations rather than + // scales. + if resp := d.handleScale(mustMarshal(t, scaleParams{TeamKey: "fanout/crew", Role: "crew", Replicas: 2})); resp.Error != "" { + t.Fatalf("a within-ceiling scale was refused: %s", resp.Error) + } +} + +// TestHandleScaleUnknownRoleReportsTheRole covers the reorder the budget gate +// forced. +// +// Falsification: the role-existence scan used to run after UpdateTeam. With +// the gate inserted in front of the mutation and the scan left where it was, +// a mistyped role name under a declared budget would report a budget error +// instead of "role not found" — the wrong diagnostic for the actual mistake. +func TestHandleScaleUnknownRoleReportsTheRole(t *testing.T) { + d := newHandlerDaemon(t) + if resp := applyManifest(t, d, budgetedManifest); resp.Error != "" { + t.Fatalf("apply: %s", resp.Error) + } + + resp := d.handleScale(mustMarshal(t, scaleParams{TeamKey: "fanout/crew", Role: "crwe", Replicas: 40})) + if resp.Error == "" { + t.Fatal("expected an error for an unknown role") + } + if !strings.Contains(resp.Error, "role crwe not found") { + t.Errorf("error = %q, want it to name the missing role", resp.Error) + } + if strings.Contains(resp.Error, "max_sessions") { + t.Errorf("error = %q, reported a budget problem for a typo", resp.Error) + } +} + +// TestHandleApplyBudgetRefusals covers both apply-time gates: the parse-time +// declaration clause (which catches the motivating fan-out before any daemon +// state exists) and the host-side pre-flight for a ceiling no role can report +// against. Neither may leave store state behind. +func TestHandleApplyBudgetRefusals(t *testing.T) { + tests := []struct { + name string + manifest string + wantErr string + workspace string + }{ + { + name: "a declared fan-out over the ceiling is refused at parse (toml)", + manifest: ` +[workspace] +name = "overdeclared" + +[[team]] +name = "crew" + + [team.budget] + max_sessions = 6 + + [[team.role]] + name = "crew" + replicas = 40 + + [team.role.runtime] + command = "sleep" + args = ["300"] +`, + wantErr: "declares 40 replicas across 1 role(s) but budget.max_sessions is 6", + workspace: "overdeclared", + }, + { + // The same clause through the format the guide's example uses. + // ParseManifestBytes used to validate inside each format attempt, + // so a YAML validation failure fell through to the TOML parser and + // the operator got "toml: line N: expected '.' or '='" instead of + // the rule. `marvel work` sends bytes, so this was the only apply + // path there is. + name: "a declared fan-out over the ceiling is refused at parse (yaml)", + manifest: ` +workspace: + name: overdeclaredyaml +teams: + - name: crew + budget: + max_sessions: 6 + roles: + - name: crew + replicas: 40 + runtime: + command: sleep + args: ["300"] +`, + wantErr: "declares 40 replicas across 1 role(s) but budget.max_sessions is 6", + workspace: "overdeclaredyaml", + }, + { + name: "a token ceiling no role can report against is refused at pre-flight", + manifest: ` +workspace: + name: mutebudget +teams: + - name: crew + budget: + max_tokens: 2000000 + roles: + - name: crew + replicas: 1 + runtime: + command: sleep + args: ["300"] +`, + wantErr: "no role runs a stream-capable harness in headless mode", + workspace: "mutebudget", + }, + { + // The hole a mode-only pre-flight left. generic implements no + // stream path, so this ceiling could never move off zero however + // long the team ran. + name: "a headless role whose harness cannot stream is still a mute gate", + manifest: ` +workspace: + name: mutegeneric +teams: + - name: crew + budget: + max_tokens: 2000000 + roles: + - name: crew + replicas: 1 + runtime: + image: generic + command: sleep + mode: headless + prompt: "review the diff" +`, + wantErr: "no role runs a stream-capable harness in headless mode", + workspace: "mutegeneric", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + d := newHandlerDaemon(t) + resp := applyManifest(t, d, tt.manifest) + if resp.Error == "" { + t.Fatal("expected apply to be refused") + } + if !strings.Contains(resp.Error, tt.wantErr) { + t.Errorf("error = %q, missing %q", resp.Error, tt.wantErr) + } + if _, err := d.store.GetWorkspace(tt.workspace); err == nil { + t.Errorf("a refused apply created workspace %q", tt.workspace) + } + }) + } +} + +// TestHandleRunRespectsATeamBudget closes the hole a controller-only gate +// would leave: `marvel run` bypasses the reconciler entirely, and its --team +// can name a team that declares a budget. +func TestHandleRunRespectsATeamBudget(t *testing.T) { + d := newHandlerDaemon(t) + if resp := applyManifest(t, d, budgetedManifest); resp.Error != "" { + t.Fatalf("apply: %s", resp.Error) + } + + resp := d.handleRun(mustMarshal(t, runParams{ + Workspace: "fanout", Team: "crew", Role: "extra", + RuntimeCommand: "sleep", RuntimeArgs: []string{"300"}, + })) + if resp.Error == "" { + t.Fatal("expected an ad-hoc run into a full team to be refused") + } + for _, want := range []string{"max_sessions=3", "trigger=run"} { + if !strings.Contains(resp.Error, want) { + t.Errorf("error = %q, missing %q", resp.Error, want) + } + } + if got := api.CountAlive(d.store.ListSessionsByTeam("fanout", "crew")); got != 3 { + t.Errorf("live sessions = %d after a refused run, want 3", got) + } + + // The default ad-hoc team has no Team record, so it declares no budget + // and behaves exactly as it did before admission existed. + resp = d.handleRun(mustMarshal(t, runParams{ + RuntimeCommand: "sleep", RuntimeArgs: []string{"300"}, + })) + if resp.Error != "" { + t.Fatalf("an ad-hoc run with no declared budget was refused: %s", resp.Error) + } +} + +// TestHandleGetBudgets covers the diagnostic surface: rows only for teams +// that declare a budget, and per-dimension numbers an operator can act on. +// Nothing else in marvel exposes a spend or occupancy aggregate. +func TestHandleGetBudgets(t *testing.T) { + d := newHandlerDaemon(t) + if resp := applyManifest(t, d, budgetedManifest); resp.Error != "" { + t.Fatalf("apply: %s", resp.Error) + } + plain := ` +workspace: + name: fanout +teams: + - name: plain + roles: + - name: worker + replicas: 1 + runtime: + command: sleep + args: ["300"] +` + if resp := applyManifest(t, d, plain); resp.Error != "" { + t.Fatalf("apply plain: %s", resp.Error) + } + + resp := d.handleGet(mustMarshal(t, getParams{ResourceType: "budgets"})) + if resp.Error != "" { + t.Fatalf("get budgets: %s", resp.Error) + } + var rows []admission.Row + if err := json.Unmarshal(resp.Result, &rows); err != nil { + t.Fatalf("unmarshal rows: %v", err) + } + if len(rows) != 1 { + t.Fatalf("got %d row(s), want 1 (only the budgeted team): %+v", len(rows), rows) + } + got := rows[0] + if got.Team != "crew" || got.Dimension != api.DimMaxSessions { + t.Fatalf("row = %+v, want the crew team's max_sessions", got) + } + if got.Limit != 3 || got.Observed != 3 || got.Headroom != 0 { + t.Errorf("row = %+v, want limit 3, observed 3, headroom 0", got) + } + // A team whose declared replicas equal its ceiling refuses nothing, so the + // row reads at-ceiling. Keying refusal on zero headroom made "refusing" + // the resting state of a healthy team and left the only + // which-dimension-tripped surface unable to tell one from the other. + if got.State != admission.RowAtCeiling { + t.Errorf("state = %q, want %q at the ceiling with nothing refused", got.State, admission.RowAtCeiling) + } + if len(d.events.Snapshot(events.Filter{Kind: events.KindAdmissionRefused}, 0)) != 0 { + t.Error("a team sitting at its ceiling emitted an admission.refused event") + } + + // And what refusing looks like when a refusal is genuinely standing. The + // out-of-band write is the only remaining door to a declared count above + // the ceiling now that both apply and scale carry the declaration clause. + if err := d.store.UpdateTeam("fanout/crew", func(live *api.Team) error { + live.Roles[0].Replicas = 5 + return nil + }); err != nil { + t.Fatalf("set replicas out of band: %v", err) + } + d.teamCtrl.ReconcileOnce() + + resp = d.handleGet(mustMarshal(t, getParams{ResourceType: "budgets"})) + if resp.Error != "" { + t.Fatalf("get budgets: %s", resp.Error) + } + rows = nil + if err := json.Unmarshal(resp.Result, &rows); err != nil { + t.Fatalf("unmarshal rows: %v", err) + } + if len(rows) != 1 { + t.Fatalf("got %d row(s), want 1: %+v", len(rows), rows) + } + if rows[0].State != admission.RowRefusing { + t.Errorf("state = %q with a role held, want %q", rows[0].State, admission.RowRefusing) + } + if !strings.Contains(rows[0].Note, "max_sessions=3") { + t.Errorf("note = %q, want the held role's arithmetic", rows[0].Note) + } +} + +// TestTokenBudgetWindowReachesTheLogRing covers the one observability +// affordance the max_tokens honesty caveat has: the meter is in-memory, so the +// window restarts with the daemon and with `marvel daemon reexec`, and the +// admin guide promises the daemon says so where `marvel daemon logs` can find +// it. +// +// Falsification: announced from the constructor, the line is written before +// cmd/marvel installs log.SetOutput(ring, --log-file), so it reaches bare +// stderr and neither surface the docs name. Announcing it from Start is what +// puts it in the ring. +func TestTokenBudgetWindowReachesTheLogRing(t *testing.T) { + skipIfNoTmux(t) + d, err := New() + if err != nil { + t.Fatalf("new daemon: %v", err) + } + // A team the daemon already knows about at Start, which is the rehydrated + // case. No roles, so the reconciler has nothing to spawn and the test + // exercises the announcement alone. + if err := d.store.CreateTeam(&api.Team{ + Name: "crew", Workspace: "tokenwindow", + Budget: api.Budget{MaxTokens: 2_000_000}, + }); err != nil { + t.Fatalf("create team: %v", err) + } + + // The ordering under test: log output is installed AFTER construction, + // exactly as cmd/marvel does it. + prevFlags := log.Flags() + log.SetOutput(d.LogBuffer()) + t.Cleanup(func() { + log.SetOutput(os.Stderr) + log.SetFlags(prevFlags) + }) + + // os.TempDir rather than t.TempDir: a Unix socket path has a hard length + // limit (about 104 bytes on darwin) and a per-test temp dir blows past it. + sock := filepath.Join(os.TempDir(), "marvel-test-tokenwindow.sock") + if err := d.Start(sock); err != nil { + t.Fatalf("start daemon: %v", err) + } + t.Cleanup(func() { + d.Stop() + _ = os.Remove(sock) + }) + + // Tail takes a positive count; 0 returns nothing. + lines := d.LogBuffer().Tail(DefaultLogBufferLines) + found := false + for _, line := range lines { + if strings.Contains(line, "declares budget.max_tokens=2000000") { + found = true + break + } + } + if !found { + t.Errorf("the token-budget window was not announced in the log ring; lines: %v", lines) + } +} + +// TestAdmissionSnapshotReportsUnobservedSessions covers the hole +// TeamTotals.Partial does not cover. Bind runs only inside attachInstance, +// which only Manager.Create calls, so a pane adopted from a prior daemon has +// no accountant state: it contributes nothing to the total AND does not set +// Partial. Comparing the store's live count against the meter's is what +// catches it. +func TestAdmissionSnapshotReportsUnobservedSessions(t *testing.T) { + d := newHandlerDaemon(t) + if resp := applyManifest(t, d, budgetedManifest); resp.Error != "" { + t.Fatalf("apply: %s", resp.Error) + } + team, err := d.store.GetTeam("fanout/crew") + if err != nil { + t.Fatalf("get team: %v", err) + } + + snap := d.AdmissionSnapshot(team) + if snap.LiveSessions != 3 || snap.DeclaredSessions != 3 { + t.Errorf("snapshot counts = live %d, declared %d; want 3 and 3", snap.LiveSessions, snap.DeclaredSessions) + } + if !snap.TokensMetered { + t.Error("TokensMetered is false with an accountant wired; absence would read as zero spend") + } + // Interactive sleep sessions publish no stream, so the meter knows none + // of them and the total is a floor rather than a small number. + if !snap.TokensPartial { + t.Error("three unobserved live sessions did not mark the total partial") + } +} diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index ea0e48f..2008a05 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -22,6 +22,7 @@ import ( "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" + "github.com/arcavenae/marvel/internal/admission" "github.com/arcavenae/marvel/internal/api" "github.com/arcavenae/marvel/internal/events" "github.com/arcavenae/marvel/internal/knownhosts" @@ -209,7 +210,7 @@ func NewWithOptions(opts Options) (*Daemon, error) { acct := usage.New(store, usage.NewResolver(limits), usage.WithEvents(evRing)) sessMgr.Usage = acct - return &Daemon{ + d := &Daemon{ store: store, sessMgr: sessMgr, teamCtrl: teamCtrl, @@ -219,7 +220,13 @@ func NewWithOptions(opts Options) (*Daemon, error) { events: evRing, usage: acct, reexec: syscall.Exec, - }, nil + } + // The controller evaluates count-shaped admission clauses on its own + // (store counts, no meter). This seam is what lets InitiateShift also + // evaluate a cumulative clause without internal/team importing + // internal/usage. See aae-orc-qiay. + teamCtrl.Snapshots = d + return d, nil } // Usage returns the daemon's context and token accountant. Exported for @@ -275,6 +282,14 @@ func (d *Daemon) Start(socketPath string) error { log.Printf("AdoptOrKill on startup: %v", err) } + // Announced from Start, not from the constructor. cmd/marvel installs + // log.SetOutput (log ring plus the optional --log-file) only after + // NewWithOptions returns, so a line written during construction reaches + // bare stderr and neither `marvel daemon logs` nor the log file. This is + // the one observability affordance for the in-memory token window, so it + // has to land where the docs say it lands. + d.logTokenBudgetWindows() + ctx, cancel := context.WithCancel(context.Background()) d.cancel = cancel @@ -647,6 +662,39 @@ func (d *Daemon) handleApply(params json.RawMessage) Response { return Response{Error: err.Error()} } + // Pre-flight: refuse a declared dimension no role in the team can ever + // report, so a mute gate is an error rather than a silent no-op. The + // capability predicate comes from the session manager's adapter + // registry: mode alone does not answer it, since a generic role can + // declare headless and still have no stream path. + if err := m.ValidateBudgets(d.sessMgr.CanStreamRole); err != nil { + return Response{Error: err.Error()} + } + + // Admission, before Apply commits anything. Refusing the declaration is + // the whole design: gating only the spawn would leave a permanently + // unsatisfiable desired state, a teams table reporting replicas that + // will never exist, and a reconciler re-deciding the same impossible + // deficit every tick. See aae-orc-qiay. + for _, mt := range m.Teams { + b := mt.Budget.Budget() + if !b.Declared() { + continue + } + t := api.Team{Name: mt.Name, Workspace: m.Workspace.Name, Budget: b} + for _, r := range mt.Roles { + t.Roles = append(t.Roles, api.Role{Name: r.Name, Replicas: r.Replicas}) + } + live := api.CountAlive(d.store.ListSessionsByTeam(t.Workspace, t.Name)) + declared := 0 + for i := range t.Roles { + declared += t.Roles[i].Replicas + } + if resp := d.admitGrowth(t, "", declared-live, admission.TriggerApply); resp != nil { + return *resp + } + } + if err := m.Apply(d.store); err != nil { return Response{Error: fmt.Sprintf("apply manifest: %v", err)} } @@ -693,6 +741,8 @@ func (d *Daemon) handleGet(params json.RawMessage) Response { result = d.store.ListEndpoints() case "policies", "policy": result = d.store.ListPolicies() + case "budgets", "budget": + result = d.budgetRows() default: return Response{Error: fmt.Sprintf("unknown resource type: %s", p.ResourceType)} } @@ -834,16 +884,47 @@ func (d *Daemon) handleScale(params json.RawMessage) Response { return Response{Error: fmt.Sprintf("role is required; available roles: %v", names)} } + // Role existence is checked BEFORE the budget gate and before the + // mutation. It used to be checked after UpdateTeam, which was harmless + // while nothing else could refuse; with a budget gate in front of the + // mutation, a mistyped role name would otherwise report a budget error + // instead of "role not found". The scan reads the snapshot GetTeam + // already returned. + old := -1 + for _, r := range t.Roles { + if r.Name == p.Role { + old = r.Replicas + break + } + } + if old < 0 { + return Response{Error: fmt.Sprintf("role %s not found in team %s", p.Role, p.TeamKey)} + } + + // A scale-down adds nothing and is never refused: shedding sessions is + // how an operator frees headroom. + if resp := d.admitGrowth(t, p.Role, p.Replicas-old, admission.TriggerScale); resp != nil { + return *resp + } + + // Then the declaration clause, which the spawn gate above cannot see: + // it compares LIVE sessions, and live can sit below declared (a crashed + // replica, a role in backoff). Second rather than first because when + // both hold, the spawn gate's message is the more specific one; this + // gate exists for the window where only it can refuse. See + // admitDeclaration. + if resp := d.admitDeclaration(t, p.Role, p.Replicas, old); resp != nil { + return *resp + } + // Commit the replica change to the live team under the store lock. // Pre-fix, this mutated a pointer returned by GetTeam — which used // to alias store state. Now GetTeam returns a snapshot, so scaling // must go through UpdateTeam. See orc finding-032. - var found bool if err := d.store.UpdateTeam(p.TeamKey, func(live *api.Team) error { for i := range live.Roles { if live.Roles[i].Name == p.Role { live.Roles[i].Replicas = p.Replicas - found = true return nil } } @@ -851,9 +932,6 @@ func (d *Daemon) handleScale(params json.RawMessage) Response { }); err != nil { return Response{Error: err.Error()} } - if !found { - return Response{Error: fmt.Sprintf("role %s not found in team %s", p.Role, p.TeamKey)} - } d.teamCtrl.ReconcileOnce() @@ -923,6 +1001,16 @@ func (d *Daemon) handleRun(params json.RawMessage) Response { Script: p.Script, } + // An ad-hoc run bypasses the controller entirely, so a controller-only + // gate would leave a real hole: --team can name a team that declares a + // budget. A run into a team with no Team record (the default + // default/adhoc/adhoc) declares no budget and is admitted unchanged. + if t, gerr := d.store.GetTeam(p.Workspace + "/" + p.Team); gerr == nil { + if resp := d.admitGrowth(t, p.Role, 1, admission.TriggerRun); resp != nil { + return *resp + } + } + sess := &api.Session{ Name: fmt.Sprintf("run-%d", time.Now().UTC().UnixMilli()), Workspace: p.Workspace, diff --git a/internal/events/events.go b/internal/events/events.go index 2b232a2..0e129c6 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -45,6 +45,21 @@ const ( // to "why is that column empty", and the fix is usually one // runtime.context_window line in the manifest. KindContextLimitUnresolved Kind = "context.limit-unresolved" + // KindAdmissionRefused records that marvel refused to spawn against a + // team-declared budget, with the arithmetic in the Message. Fires at + // every refusal point: the operator's verb (apply, scale, run, shift) + // and the reconciler backstop. Edge-triggered on the verdict in the + // reconciler, one per operator action at a verb. See aae-orc-qiay. + KindAdmissionRefused Kind = "admission.refused" + // KindAdmissionCleared records that a standing admission refusal stopped + // applying, so a role held back by a budget may grow again. + KindAdmissionCleared Kind = "admission.cleared" + // KindAdmissionUnmeasured records that a declared clause was admitted + // against a total the meter could not supply. The operator declared a + // ceiling on a dimension, not "refuse when unmeasurable", so the default + // admits — audibly, here — and budget.on_unmeasured = "refuse" is how + // they ratify the fail-closed posture instead. + KindAdmissionUnmeasured Kind = "admission.unmeasured" ) // Agent-stream kinds. These are the runtime adapter vocabulary diff --git a/internal/session/manager.go b/internal/session/manager.go index 86d25df..cec9ff4 100644 --- a/internal/session/manager.go +++ b/internal/session/manager.go @@ -373,6 +373,37 @@ func (m *Manager) planLaunch(sess *api.Session) launchPlan { return plan } +// CanStreamRole reports whether a manifest role would launch a session +// marvel can read a usage stream from. It asks the same registry, and the +// same adapter question, that openSink asks at spawn, so the apply-time +// answer cannot drift from the spawn-time one. +// +// The pre-flight caller (a token budget with no role that could report +// against it) has no session yet, so the LaunchContext carries only the +// resolved runtime. SupportsStream is documented to depend on the runtime +// and the role rather than on the sink, which is what makes that legal. +func (m *Manager) CanStreamRole(r api.ManifestRole) bool { + rt := api.Runtime{ + Name: r.Runtime.Image, + Command: r.Runtime.Command, + Args: r.Runtime.Args, + Script: r.Runtime.Script, + Mode: r.Runtime.Mode, + Prompt: r.Runtime.Prompt, + } + if rt.Name == "" { + rt.Name = rt.Command + } + streamer, ok := m.adapters.Resolve(rt.Name).(runtime.StreamCapable) + if !ok { + return false + } + return streamer.SupportsStream(&runtime.LaunchContext{ + Session: &api.Session{Runtime: rt}, + Role: &api.Role{Name: r.Name, Replicas: r.Replicas, Runtime: rt}, + }) +} + // openSink creates the FIFO for a stream-capable adapter, or returns nil. // A sink that cannot be created is logged and skipped: an unobservable // session is still a working session. diff --git a/internal/team/admission.go b/internal/team/admission.go new file mode 100644 index 0000000..6c67343 --- /dev/null +++ b/internal/team/admission.go @@ -0,0 +1,199 @@ +package team + +import ( + "fmt" + "log" + "strings" + + "github.com/arcavenae/marvel/internal/admission" + "github.com/arcavenae/marvel/internal/api" + "github.com/arcavenae/marvel/internal/events" +) + +// admit evaluates the count clause for one role and returns how many of +// the requested spawns may proceed. Emitting and latching are side effects, +// so the caller only has to read the number. +// +// Never touches RoleHealth in either direction. A refusal is not a crash, +// and routing one through noteCrashAndBackoff would climb RestartCount +// every tick until MaxRestarts saturation froze BackoffUntil in the year +// 9999 — durably, in the bolt role_health bucket, surviving restarts. A +// role held by admission accrues no restart count, so a hold cannot age +// into a saturation freeze. Caller holds c.mu. See aae-orc-qiay. +func (c *Controller) admit(t *api.Team, role *api.Role, want int) int { + live := api.CountAlive(c.store.ListSessionsByTeam(t.Workspace, t.Name)) + // Recomputed per role rather than once per team: spawning for an earlier + // role changes the count a later role must be evaluated against, and + // computing it once would over-admit within a single tick. + v := admission.CheckSessions(t.Budget, live, c.declaredSessions(t), admission.Request{ + Role: role.Name, + Want: want, + // The reconciler only ever converges toward an already-declared + // replica count. The parser guarantees declared <= ceiling, so this + // path is admitted unless the declaration itself is over budget. + Kind: admission.Repair, + // Convergence is best-effort: 2 of 5 is strictly better for the + // operator than 0 of 5. The synchronous verbs do the opposite. + AllowPartial: true, + }) + if !v.Refused() { + c.clearAdmissionHold(t, role.Name) + return v.Granted + } + + roleKey := t.Workspace + "/" + t.Name + "/" + role.Name + key := v.Key() + if c.admissionHolds[roleKey] == key { + return v.Granted + } + c.admissionHolds[roleKey] = key + reason := v.Reason(admission.TriggerReconcile) + // Tee to the log ring as well as the event ring: the event ring is + // bounded and in-memory, and `marvel daemon logs` works over mrvl://. + log.Printf("admission: %s role %s refused: %s", t.Key(), role.Name, reason) + events.Emit(c.Events, events.Event{ + Kind: events.KindAdmissionRefused, + Severity: events.SeverityWarning, + Workspace: t.Workspace, + Team: t.Name, + Role: role.Name, + Message: reason, + }) + c.setAdmissionState(t, api.AdmissionState{ + Held: true, + Role: role.Name, + Reason: reason, + Since: c.nowUTC(), + }) + return v.Granted +} + +// clearAdmissionHold drops a role's latch and says so once. A hold +// describes a refusal that is still happening, so it is dropped the moment +// the gate admits or stops being reached at all. Cheap when no hold exists: +// one map lookup, no store read, no event. Caller holds c.mu. +func (c *Controller) clearAdmissionHold(t *api.Team, role string) { + roleKey := t.Workspace + "/" + t.Name + "/" + role + if _, held := c.admissionHolds[roleKey]; !held { + return + } + delete(c.admissionHolds, roleKey) + live := api.CountAlive(c.store.ListSessionsByTeam(t.Workspace, t.Name)) + msg := fmt.Sprintf("role %s may grow again: %d live", role, live) + if t.Budget.MaxSessions > 0 { + msg = fmt.Sprintf("role %s may grow again: %d live under max_sessions=%d", role, live, t.Budget.MaxSessions) + } + log.Printf("admission: %s %s", t.Key(), msg) + events.Emit(c.Events, events.Event{ + Kind: events.KindAdmissionCleared, + Workspace: t.Workspace, + Team: t.Name, + Role: role, + Message: msg, + }) + if t.Admission.Role == role { + c.setAdmissionState(t, api.AdmissionState{}) + } +} + +// reconcileAdmissionState drops a recorded condition this process never +// refused. +// +// The latch is in-memory because the condition is derived from live state, so +// a durable copy could only outlive its cause: Team.Admission rides the team +// record into bolt, and a daemon restart would otherwise rehydrate a hold +// nothing is holding. Correcting it here, before any role is reconciled, +// means the record is right within one tick whatever else that tick decides +// (including a role sitting in a crash-loop backoff window, which returns +// before admission is reached at all). Silent on purpose: announcing a +// clearing with no matching refusal in the event ring would be noise. +// Caller holds c.mu. +func (c *Controller) reconcileAdmissionState(t *api.Team) { + if !t.Admission.Held { + return + } + if _, held := c.admissionHolds[t.Workspace+"/"+t.Name+"/"+t.Admission.Role]; held { + return + } + c.setAdmissionState(t, api.AdmissionState{}) +} + +// setAdmissionState writes the standing condition through to the store, on +// transitions only. Following the Team.Shift precedent rather than +// inventing a second pattern for status on a spec record; writing every +// tick would be a bolt write storm. Caller holds c.mu. +func (c *Controller) setAdmissionState(t *api.Team, st api.AdmissionState) { + if t.Admission == st { + return + } + t.Admission = st + if err := c.store.UpdateTeam(t.Key(), func(live *api.Team) error { + live.Admission = st + return nil + }); err != nil { + log.Printf("admission: record state for %s: %v", t.Key(), err) + } +} + +// declaredSessions sums Replicas across a team's roles. Caller holds c.mu. +func (c *Controller) declaredSessions(t *api.Team) int { + n := 0 + for i := range t.Roles { + n += t.Roles[i].Replicas + } + return n +} + +// dropAdmissionHolds forgets every latch under a key prefix, paired with +// the crash-loop cascade clears. A hold outliving the team or role it +// describes would suppress the first event of a genuinely new refusal. +// Caller holds c.mu. +func (c *Controller) dropAdmissionHolds(prefix string) { + for k := range c.admissionHolds { + if strings.HasPrefix(k, prefix) { + delete(c.admissionHolds, k) + } + } +} + +// AdmissionHold returns a role's latched verdict key, for tests and for +// operator-facing diagnostics. Returns ("", false) when the role is not +// held. +func (c *Controller) AdmissionHold(workspace, team, role string) (string, bool) { + c.mu.Lock() + defer c.mu.Unlock() + key, ok := c.admissionHolds[workspace+"/"+team+"/"+role] + return key, ok +} + +// admitShift evaluates whether a rotation may start. +// +// The request carries Overlap, so the count clause is skipped: a shift is +// replacement, and its transient double count is a mechanism artifact +// rather than growth (R5). A budget exactly equal to declared replicas +// therefore does not forbid a rolling shift. A cumulative clause is NOT +// skipped, because a new generation is a new spender, so an exhausted token +// budget can refuse a rotation — synchronously, at the verb, where the +// operator can raise the ceiling in one command. +// +// Gating here rather than inside shiftLaunch is deliberate. Refusing at +// launch would return early and leave the team in phase=launching until +// abortStuckShift fires at the shift timeout, so the operator would see a +// shift-timeout warning instead of a budget one, ten minutes late. Caller +// holds c.mu. +func (c *Controller) admitShift(t *api.Team, roles []string) admission.Verdict { + want := 0 + for _, name := range roles { + for i := range t.Roles { + if t.Roles[i].Name == name { + want += t.Roles[i].Replicas + } + } + } + req := admission.Request{Want: want, Kind: admission.Growth, Overlap: true} + if c.Snapshots == nil { + live := api.CountAlive(c.store.ListSessionsByTeam(t.Workspace, t.Name)) + return admission.CheckSessions(t.Budget, live, c.declaredSessions(t), req) + } + return admission.Check(t.Budget, c.Snapshots.AdmissionSnapshot(*t), req) +} diff --git a/internal/team/admission_test.go b/internal/team/admission_test.go new file mode 100644 index 0000000..9f023e3 --- /dev/null +++ b/internal/team/admission_test.go @@ -0,0 +1,685 @@ +package team + +import ( + "path/filepath" + "strings" + "testing" + "time" + + "github.com/arcavenae/marvel/internal/admission" + "github.com/arcavenae/marvel/internal/api" + "github.com/arcavenae/marvel/internal/events" +) + +// sleepRole is the deterministic fixture runtime used throughout this file: +// admission arithmetic is store-based, so no model auth and no real agent are +// needed to exercise any of it. +func sleepRole(name string, replicas int) api.Role { + return api.Role{ + Name: name, + Replicas: replicas, + Runtime: api.Runtime{Name: "sleep", Command: "sleep", Args: []string{"300"}}, + } +} + +// createBudgetTeamFixture is createTeamFixture with a declared budget. +func createBudgetTeamFixture(t *testing.T, store *api.Store, wsName, teamName string, budget api.Budget, roles []api.Role) { + t.Helper() + if err := store.CreateWorkspace(&api.Workspace{Name: wsName, CreatedAt: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + if err := store.CreateTeam(&api.Team{ + Name: teamName, + Workspace: wsName, + Roles: roles, + Budget: budget, + Generation: 1, + CreatedAt: time.Now().UTC(), + }); err != nil { + t.Fatal(err) + } +} + +// setReplicas rewrites a role's replica count out of band, bypassing both +// doors that hold the declaration clause: the manifest parser +// (api.validateManifestBudget) and the scale verb +// (daemon.admitDeclaration). With those two closed, this is the only +// remaining way to reach a declared count above the team ceiling, which is +// precisely the state the reconciler backstop exists to catch. +func setReplicas(t *testing.T, store *api.Store, teamKey, role string, replicas int) { + t.Helper() + if err := store.UpdateTeam(teamKey, func(live *api.Team) error { + for i := range live.Roles { + if live.Roles[i].Name == role { + live.Roles[i].Replicas = replicas + } + } + return nil + }); err != nil { + t.Fatalf("set replicas: %v", err) + } +} + +func setMaxSessions(t *testing.T, store *api.Store, teamKey string, maxSessions int) { + t.Helper() + if err := store.UpdateTeam(teamKey, func(live *api.Team) error { + live.Budget.MaxSessions = maxSessions + return nil + }); err != nil { + t.Fatalf("set max_sessions: %v", err) + } +} + +func admissionEvents(t *testing.T, ring *events.Ring, kind events.Kind) []events.Event { + t.Helper() + return ring.Snapshot(events.Filter{Kind: kind}, 0) +} + +// TestNoBudgetNoAdmissionEvents is the default-open regression. A team that +// declares no budget must reconcile exactly as it did before admission +// existed: same replicas, and not one event of any admission kind. +// +// Falsification: any gate that evaluates before checking Budget.Declared +// would either change replica convergence or leave a trace here. +func TestNoBudgetNoAdmissionEvents(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + + createTeamFixture(t, store, "test-adm-open", "agents", []api.Role{sleepRole("worker", 3)}) + + for i := 0; i < 10; i++ { + ctrl.ReconcileOnce() + } + + if got := len(store.ListSessionsByTeamRole("test-adm-open", "agents", "worker")); got != 3 { + t.Fatalf("expected 3 sessions, got %d", got) + } + for _, kind := range []events.Kind{events.KindAdmissionRefused, events.KindAdmissionCleared, events.KindAdmissionUnmeasured} { + if evs := admissionEvents(t, ring, kind); len(evs) != 0 { + t.Errorf("undeclared budget emitted %d %s event(s): %+v", len(evs), kind, evs) + } + } +} + +// TestAdmissionRefusesOverBudgetFanOut is the backstop firing on the one +// state a manifest declaration cannot see: a declared count above the team +// ceiling, reached out of band. No new sessions, and one event carrying both +// numbers. +func TestAdmissionRefusesOverBudgetFanOut(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + + createBudgetTeamFixture(t, store, "test-adm-refuse", "crew", api.Budget{MaxSessions: 3}, []api.Role{sleepRole("crew", 3)}) + teamKey := "test-adm-refuse/crew" + + ctrl.ReconcileOnce() + if got := len(store.ListSessionsByTeamRole("test-adm-refuse", "crew", "crew")); got != 3 { + t.Fatalf("expected 3 sessions at the ceiling, got %d", got) + } + + setReplicas(t, store, teamKey, "crew", 40) + ctrl.ReconcileOnce() + + if got := len(store.ListSessionsByTeamRole("test-adm-refuse", "crew", "crew")); got != 3 { + t.Fatalf("refused spawn still created sessions: got %d, want 3", got) + } + evs := admissionEvents(t, ring, events.KindAdmissionRefused) + if len(evs) != 1 { + t.Fatalf("expected exactly 1 admission.refused event, got %d", len(evs)) + } + if evs[0].Severity != events.SeverityWarning { + t.Errorf("severity = %q, want warning so refusals land in `marvel events --warnings`", evs[0].Severity) + } + for _, want := range []string{"max_sessions=3", "declares 40 sessions", "trigger=reconcile"} { + if !strings.Contains(evs[0].Message, want) { + t.Errorf("message = %q, missing %q", evs[0].Message, want) + } + } + + team, _ := store.GetTeam(teamKey) + if !team.Admission.Held || team.Admission.Role != "crew" { + t.Errorf("Team.Admission = %+v, want held on role crew", team.Admission) + } +} + +// TestAdmissionEmitsOncePerTransition is the event-ring guard. +// +// Falsification: without the verdict-key latch this records ten events for +// ten refused ticks. At the 2s reconcile interval and a 2000-entry ring, one +// event per tick per role flushes the whole ring in about 67 minutes and +// erases every other event class, so a naive refusal is a denial of service +// on the operator's own observability. +func TestAdmissionEmitsOncePerTransition(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + + createBudgetTeamFixture(t, store, "test-adm-latch", "crew", api.Budget{MaxSessions: 2}, []api.Role{sleepRole("crew", 2)}) + teamKey := "test-adm-latch/crew" + ctrl.ReconcileOnce() + + setReplicas(t, store, teamKey, "crew", 9) + for i := 0; i < 10; i++ { + ctrl.ReconcileOnce() + } + if got := len(admissionEvents(t, ring, events.KindAdmissionRefused)); got != 1 { + t.Fatalf("10 refused ticks emitted %d events, want 1", got) + } + + // Raising the ceiling ends the condition: one clearing, then the latch + // is free to fire again on a genuinely new refusal. + setMaxSessions(t, store, teamKey, 9) + ctrl.ReconcileOnce() + if got := len(admissionEvents(t, ring, events.KindAdmissionCleared)); got != 1 { + t.Fatalf("clearing emitted %d events, want 1", got) + } + + setMaxSessions(t, store, teamKey, 2) + setReplicas(t, store, teamKey, "crew", 20) + for i := 0; i < 5; i++ { + ctrl.ReconcileOnce() + } + if got := len(admissionEvents(t, ring, events.KindAdmissionRefused)); got != 2 { + t.Fatalf("a new refusal after a clearing emitted %d total events, want 2", got) + } +} + +// TestAdmissionRefusalDoesNotTouchRoleHealth is the highest-value test in +// the slice. +// +// Falsification: routing a refusal through noteCrashAndBackoff would climb +// RestartCount on every tick until MaxRestarts saturation froze BackoffUntil +// in the year 9999 — written through to the bolt role_health bucket and +// rehydrated at daemon start. A budget refusal would silently become an +// unrecoverable role kill that survives a restart. +func TestAdmissionRefusalDoesNotTouchRoleHealth(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + + role := sleepRole("crew", 1) + role.MaxRestarts = 2 + createBudgetTeamFixture(t, store, "test-adm-health", "crew", api.Budget{MaxSessions: 1}, []api.Role{role}) + teamKey := "test-adm-health/crew" + ctrl.ReconcileOnce() + + setReplicas(t, store, teamKey, "crew", 8) + for i := 0; i < 20; i++ { + ctrl.ReconcileOnce() + } + + if rh, ok := ctrl.RoleHealthSnapshot("test-adm-health", "crew", "crew"); ok { + t.Fatalf("20 refused ticks recorded crash-loop state: %+v", rh) + } + if _, held := ctrl.AdmissionHold("test-adm-health", "crew", "crew"); !held { + t.Error("expected the role to be held by admission") + } +} + +// TestAdmissionDoesNotClearCrashMarkers pins the placement of the gate. +// +// Falsification: a gate placed after ClearCrashedForRole would delete this +// role's crash history on every wholly-refused tick while never spawning, +// because that call mutates store state. The marker is injected rather than +// produced by a real crash on purpose: a crash frees a live slot, which +// leaves headroom, and any tick with headroom legitimately spawns and +// legitimately clears. The case under test is the one with no headroom at +// all, where the gate must return having changed nothing. +func TestAdmissionDoesNotClearCrashMarkers(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + + createBudgetTeamFixture(t, store, "test-adm-marker", "crew", api.Budget{MaxSessions: 2}, []api.Role{sleepRole("crew", 2)}) + teamKey := "test-adm-marker/crew" + ctrl.ReconcileOnce() + if got := api.CountAlive(store.ListSessionsByTeam("test-adm-marker", "crew")); got != 2 { + t.Fatalf("live sessions = %d, want 2 at the ceiling", got) + } + + // A crash marker from an earlier generation, past its observability job. + if err := store.CreateSession(&api.Session{ + Name: "crew-crew-g1-ghost", Workspace: "test-adm-marker", Team: "crew", Role: "crew", + Generation: 1, Runtime: api.Runtime{Name: "sleep", Command: "sleep"}, + State: api.SessionCrashed, CreatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("inject crash marker: %v", err) + } + + setReplicas(t, store, teamKey, "crew", 9) + for i := 0; i < 5; i++ { + ctrl.ReconcileOnce() + } + + if got := countState(store, "test-adm-marker", "crew", api.SessionCrashed); got != 1 { + t.Errorf("crash marker count = %d after wholly-refused ticks, want 1", got) + } + if got := api.CountAlive(store.ListSessionsByTeam("test-adm-marker", "crew")); got != 2 { + t.Errorf("live sessions = %d, want 2: a refusal with no headroom must spawn nothing", got) + } +} + +func countState(store *api.Store, workspace, team string, state api.SessionState) int { + n := 0 + for _, s := range store.ListSessionsByTeam(workspace, team) { + if s.State == state { + n++ + } + } + return n +} + +// TestAdmissionDoesNotBlockRepair is the R1 invariant end to end: a team +// declared exactly at its ceiling still replaces a crashed replica. +// +// Falsification: gating repair on live + want > limit means a crashed +// replica never returns, and a budget becomes an outage rather than a +// ceiling. +func TestAdmissionDoesNotBlockRepair(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + clock := newTestClock(time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)) + ctrl.now = clock.Now + + createBudgetTeamFixture(t, store, "test-adm-repair", "crew", api.Budget{MaxSessions: 2}, []api.Role{sleepRole("crew", 2)}) + ctrl.ReconcileOnce() + + sessions := store.ListSessionsByTeamRole("test-adm-repair", "crew", "crew") + if len(sessions) != 2 { + t.Fatalf("expected 2 sessions, got %d", len(sessions)) + } + killPaneAndWait(t, sessions[0].PaneID) + ctrl.ReconcileOnce() + + clock.Advance(2 * time.Minute) + ctrl.ReconcileOnce() + + alive := api.CountAlive(store.ListSessionsByTeam("test-adm-repair", "crew")) + if alive != 2 { + t.Errorf("live sessions = %d after repair, want 2: a budget at the declared count blocked a replacement", alive) + } + if evs := admissionEvents(t, ring, events.KindAdmissionRefused); len(evs) != 0 { + t.Errorf("repair produced %d refusal(s): %+v", len(evs), evs) + } +} + +// TestAdmissionBackoffTakesPrecedence pins the ordering by position: a +// cooling role returns before admission is evaluated, so the two conditions +// never race for the operator's attention and no wasted work happens inside +// a backoff window. +func TestAdmissionBackoffTakesPrecedence(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + clock := newTestClock(time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)) + ctrl.now = clock.Now + + createBudgetTeamFixture(t, store, "test-adm-backoff", "crew", api.Budget{MaxSessions: 2}, []api.Role{sleepRole("crew", 2)}) + teamKey := "test-adm-backoff/crew" + ctrl.ReconcileOnce() + + sessions := store.ListSessionsByTeamRole("test-adm-backoff", "crew", "crew") + killPaneAndWait(t, sessions[0].PaneID) + setReplicas(t, store, teamKey, "crew", 9) + ctrl.ReconcileOnce() // reaps, records the crash, enters backoff + + rh, ok := ctrl.RoleHealthSnapshot("test-adm-backoff", "crew", "crew") + if !ok || rh.BackoffUntil.IsZero() { + t.Fatalf("expected a backoff window after the reap, got %+v", rh) + } + for i := 0; i < 5; i++ { + ctrl.ReconcileOnce() + } + if evs := admissionEvents(t, ring, events.KindAdmissionRefused); len(evs) != 0 { + t.Fatalf("admission spoke during a backoff window: %+v", evs) + } + + clock.Advance(2 * time.Minute) + ctrl.ReconcileOnce() + if got := len(admissionEvents(t, ring, events.KindAdmissionRefused)); got != 1 { + t.Errorf("after the window elapsed, admission emitted %d event(s), want 1", got) + } +} + +// TestAdmissionSkippedDuringShift covers R5 at the reconciler. +// +// Falsification: without the shift skip, a launching generation's transient +// double count refuses a non-shifting role's legitimate repair, and a budget +// equal to declared replicas cannot rotate at all. +func TestAdmissionSkippedDuringShift(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + + createBudgetTeamFixture(t, store, "test-adm-shift", "crew", api.Budget{MaxSessions: 2}, []api.Role{sleepRole("crew", 2)}) + teamKey := "test-adm-shift/crew" + ctrl.ReconcileOnce() + + if err := ctrl.InitiateShift(teamKey, ""); err != nil { + t.Fatalf("initiate shift under a budget equal to declared replicas: %v", err) + } + for i := 0; i < 20; i++ { + ctrl.ReconcileOnce() + team, _ := store.GetTeam(teamKey) + if team.Shift.Phase == api.ShiftNone { + break + } + } + team, _ := store.GetTeam(teamKey) + if team.Shift.Phase != api.ShiftNone { + t.Fatalf("shift did not complete under a declared budget, phase: %s", team.Shift.Phase) + } + if evs := admissionEvents(t, ring, events.KindAdmissionRefused); len(evs) != 0 { + t.Errorf("a rolling shift produced %d refusal(s): %+v", len(evs), evs) + } + if got := len(store.ListSessionsByTeamRoleGeneration("test-adm-shift", "crew", "crew", 2)); got != 2 { + t.Errorf("gen-2 sessions = %d, want 2", got) + } +} + +// TestAdmissionPartialGrant is the reconciler's half of the asymmetry: +// convergence takes headroom rather than nothing, and the event names both +// numbers so the operator sees what was and was not granted. +func TestAdmissionPartialGrant(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + + // A differently named team and role, so the latch's keying is exercised + // rather than assumed. + createBudgetTeamFixture(t, store, "test-adm-partial", "squad", api.Budget{MaxSessions: 3}, []api.Role{sleepRole("worker", 1)}) + teamKey := "test-adm-partial/squad" + ctrl.ReconcileOnce() + if got := len(store.ListSessionsByTeamRole("test-adm-partial", "squad", "worker")); got != 1 { + t.Fatalf("expected 1 session, got %d", got) + } + + // Deficit 5, headroom 2. + setReplicas(t, store, teamKey, "worker", 6) + ctrl.ReconcileOnce() + + if got := api.CountAlive(store.ListSessionsByTeam("test-adm-partial", "squad")); got != 3 { + t.Fatalf("live sessions = %d, want 3 (headroom taken, ceiling respected)", got) + } + evs := admissionEvents(t, ring, events.KindAdmissionRefused) + if len(evs) != 1 { + t.Fatalf("expected 1 refusal event, got %d", len(evs)) + } + for _, want := range []string{"refused 3 of 5", "granted 2"} { + if !strings.Contains(evs[0].Message, want) { + t.Errorf("message = %q, missing %q", evs[0].Message, want) + } + } +} + +// TestAdmissionGrantsInFullWithoutClaimingARefusal covers the reconciler half +// of "the refusal surface must not report refusals that did not happen". +// +// An over-ceiling declaration makes the count clause Exceeded for every role +// in the team, including roles the remaining headroom fully satisfies. +// +// Falsification: with the decision keyed on the clause rather than on what was +// granted, role a's tick logged, emitted admission.refused at warning +// severity, and latched Team.Admission{Held:true, Role:"a"} while spawning all +// three of the sessions it asked for. The event message said so in as many +// words: "refused 0 of 3 spawn(s) ... granted 3". +func TestAdmissionGrantsInFullWithoutClaimingARefusal(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + + // Declared 4 against a ceiling of 3: role a fits inside the headroom, role + // b does not. + createBudgetTeamFixture(t, store, "test-adm-full", "crew", api.Budget{MaxSessions: 3}, + []api.Role{sleepRole("a", 3), sleepRole("b", 1)}) + ctrl.ReconcileOnce() + + if got := api.CountAlive(store.ListSessionsByTeamRole("test-adm-full", "crew", "a")); got != 3 { + t.Errorf("role a live = %d, want 3 (the headroom covered its whole ask)", got) + } + evs := admissionEvents(t, ring, events.KindAdmissionRefused) + if len(evs) != 1 { + t.Fatalf("expected exactly 1 refusal (role b), got %d: %+v", len(evs), evs) + } + if evs[0].Role != "b" { + t.Errorf("refusal named role %q, want b: role a was satisfied in full", evs[0].Role) + } + if strings.Contains(evs[0].Message, "refused 0 of") { + t.Errorf("message = %q, which reports a refusal of nothing", evs[0].Message) + } + if _, held := ctrl.AdmissionHold("test-adm-full", "crew", "a"); held { + t.Error("role a is latched as held after a tick that spawned every session it asked for") + } + team, _ := store.GetTeam("test-adm-full/crew") + if team.Admission.Role != "b" { + t.Errorf("Team.Admission names role %q, want b", team.Admission.Role) + } +} + +// TestAdmissionNeverSpawnsPastReplicas is the overspawn regression. +// +// Falsification: with the partial grant unclamped, a team-wide headroom larger +// than one role's deficit handed the whole headroom to that role, so the +// reconciler spawned five sessions for a one-replica role and deleted four of +// them on the next tick. Those are real tmux panes and real harness processes, +// launched and killed seconds apart. +func TestAdmissionNeverSpawnsPastReplicas(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + + // Declared 6 against a ceiling of 5, and the supervisor role is + // reconciled first with the whole headroom available to it. + createBudgetTeamFixture(t, store, "test-adm-clamp", "crew", api.Budget{MaxSessions: 5}, + []api.Role{sleepRole("sup", 1), sleepRole("worker", 5)}) + + // Asserted on the FIRST tick: the excess was deleted on the next one, so a + // steady-state-only check cannot see the panes that were launched and + // killed in between. + ctrl.ReconcileOnce() + if got := len(store.ListSessionsByTeamRole("test-adm-clamp", "crew", "sup")); got != 1 { + t.Errorf("sup sessions = %d after one tick, want 1: a 1-replica role took the team's whole headroom", got) + } + + for i := 0; i < 2; i++ { + ctrl.ReconcileOnce() + } + if got := len(store.ListSessionsByTeamRole("test-adm-clamp", "crew", "sup")); got != 1 { + t.Errorf("sup sessions = %d at steady state, want 1", got) + } + if got := api.CountAlive(store.ListSessionsByTeam("test-adm-clamp", "crew")); got != 5 { + t.Errorf("live sessions = %d, want 5 (the ceiling)", got) + } + for _, ev := range admissionEvents(t, ring, events.KindAdmissionRefused) { + if strings.Contains(ev.Message, "refused -") { + t.Errorf("message = %q, want no negative refusal count", ev.Message) + } + } +} + +// TestAdmissionClearsWhenBudgetRaised is the recovery path: raising the +// ceiling resumes spawning within one tick, with no manual clear command, +// no resume verb, and no daemon restart. +func TestAdmissionClearsWhenBudgetRaised(t *testing.T) { + skipIfNoTmux(t) + store, sessMgr, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + // The manager is the producer of session.created, so this test wires it + // too: resuming spawns is half of what recovery means. + sessMgr.Events = ring + + createBudgetTeamFixture(t, store, "test-adm-clear", "crew", api.Budget{MaxSessions: 3}, []api.Role{sleepRole("crew", 3)}) + teamKey := "test-adm-clear/crew" + ctrl.ReconcileOnce() + + setReplicas(t, store, teamKey, "crew", 7) + ctrl.ReconcileOnce() + if got := len(admissionEvents(t, ring, events.KindAdmissionRefused)); got != 1 { + t.Fatalf("expected the role held, got %d refusal event(s)", got) + } + + setMaxSessions(t, store, teamKey, 7) + ctrl.ReconcileOnce() + + if got := api.CountAlive(store.ListSessionsByTeam("test-adm-clear", "crew")); got != 7 { + t.Errorf("live sessions = %d after raising the ceiling, want 7", got) + } + if got := len(admissionEvents(t, ring, events.KindAdmissionCleared)); got != 1 { + t.Errorf("admission.cleared events = %d, want 1", got) + } + if len(ring.Snapshot(events.Filter{Kind: events.KindSessionCreated}, 0)) == 0 { + t.Error("no session.created event after the ceiling was raised") + } + team, _ := store.GetTeam(teamKey) + if team.Admission.Held { + t.Errorf("Team.Admission still held: %+v", team.Admission) + } + if _, held := ctrl.AdmissionHold("test-adm-clear", "crew", "crew"); held { + t.Error("the latch outlived its condition") + } +} + +// stubSnapshots supplies a fixed measured state, so a cumulative clause can +// be exercised with no accountant and no model auth. +type stubSnapshots struct{ snap admission.Snapshot } + +func (s stubSnapshots) AdmissionSnapshot(api.Team) admission.Snapshot { return s.snap } + +// TestInitiateShiftRefusedOnTokens covers the shift gate's placement. +// +// Falsification: gating inside shiftLaunch instead would return early and +// leave the team in phase=launching until abortStuckShift fires, so the +// operator would see team.shift-timed-out instead of a budget warning, ten +// minutes late. Here the refusal is synchronous, the phase never moves, and +// no timeout event ever appears. +func TestInitiateShiftRefusedOnTokens(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ring := events.NewRing(0) + ctrl.Events = ring + clock := newTestClock(time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)) + ctrl.now = clock.Now + ctrl.ShiftTimeout = 2 * time.Minute + ctrl.Snapshots = stubSnapshots{snap: admission.Snapshot{ + LiveSessions: 2, + DeclaredSessions: 2, + TokensObserved: 2_118_443, + TokensMetered: true, + TokensSeen: 2, + Since: clock.Now().Add(-14 * time.Minute), + }} + + createBudgetTeamFixture(t, store, "test-adm-shift-tok", "crew", + api.Budget{MaxSessions: 2, MaxTokens: 2_000_000}, []api.Role{sleepRole("crew", 2)}) + teamKey := "test-adm-shift-tok/crew" + ctrl.ReconcileOnce() + + err := ctrl.InitiateShift(teamKey, "") + if err == nil { + t.Fatal("expected the rotation refused against an exhausted token budget") + } + for _, want := range []string{"max_tokens=2000000", "trigger=shift"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, missing %q", err, want) + } + } + + team, _ := store.GetTeam(teamKey) + if team.Shift.Phase != api.ShiftNone { + t.Fatalf("refused shift still wrote shift state: phase %s", team.Shift.Phase) + } + if team.Generation != 1 { + t.Errorf("refused shift advanced the generation to %d", team.Generation) + } + if got := len(admissionEvents(t, ring, events.KindAdmissionRefused)); got != 1 { + t.Errorf("admission.refused events = %d, want 1", got) + } + + clock.Advance(5 * time.Minute) + ctrl.ReconcileOnce() + if evs := ring.Snapshot(events.Filter{Kind: events.KindShiftTimedOut}, 0); len(evs) != 0 { + t.Errorf("a refused shift produced a shift timeout: %+v", evs) + } +} + +// TestInitiateShiftAdmittedUnderSessionCeiling is the other side of R5: a +// count ceiling exactly at declared replicas must not forbid a rotation, +// even though the overlap transiently doubles the live count. +func TestInitiateShiftAdmittedUnderSessionCeiling(t *testing.T) { + skipIfNoTmux(t) + store, _, ctrl, cleanup := setup(t) + t.Cleanup(cleanup) + ctrl.Snapshots = stubSnapshots{snap: admission.Snapshot{ + LiveSessions: 2, DeclaredSessions: 2, TokensMetered: true, TokensSeen: 2, + Since: time.Now().UTC(), + }} + + createBudgetTeamFixture(t, store, "test-adm-shift-ok", "crew", api.Budget{MaxSessions: 2}, []api.Role{sleepRole("crew", 2)}) + ctrl.ReconcileOnce() + + if err := ctrl.InitiateShift("test-adm-shift-ok/crew", ""); err != nil { + t.Fatalf("a session ceiling at declared replicas forbade a rotation: %v", err) + } +} + +// TestAdmissionHoldSelfHealsAfterRestart: the latch is in-memory because the +// condition is derived from live state, so a durable copy could only outlive +// its cause. A recorded Team.Admission rehydrated from bolt is corrected +// within the first tick rather than left describing a condition that has +// passed. +func TestAdmissionHoldSelfHealsAfterRestart(t *testing.T) { + skipIfNoTmux(t) + path := filepath.Join(t.TempDir(), "marvel.bolt") + + store1, ctrl1, cleanup1 := setupWithBolt(t, path) + createBudgetTeamFixture(t, store1, "test-adm-restart", "crew", api.Budget{MaxSessions: 2}, []api.Role{sleepRole("crew", 2)}) + teamKey := "test-adm-restart/crew" + ctrl1.ReconcileOnce() + setReplicas(t, store1, teamKey, "crew", 8) + ctrl1.ReconcileOnce() + + team, _ := store1.GetTeam(teamKey) + if !team.Admission.Held { + t.Fatalf("expected the role held before the restart: %+v", team.Admission) + } + cleanup1() + + store2, ctrl2, cleanup2 := setupWithBolt(t, path) + t.Cleanup(cleanup2) + if _, held := ctrl2.AdmissionHold("test-adm-restart", "crew", "crew"); held { + t.Error("an admission hold was rehydrated; it is derived state and must not persist") + } + + // The condition has passed: the ceiling now covers the declaration. + setMaxSessions(t, store2, teamKey, 8) + ctrl2.ReconcileOnce() + + team, _ = store2.GetTeam(teamKey) + if team.Admission.Held { + t.Errorf("a rehydrated admission state outlived its cause: %+v", team.Admission) + } +} diff --git a/internal/team/controller.go b/internal/team/controller.go index f55c3c7..9c337c3 100644 --- a/internal/team/controller.go +++ b/internal/team/controller.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/arcavenae/marvel/internal/admission" "github.com/arcavenae/marvel/internal/api" "github.com/arcavenae/marvel/internal/events" "github.com/arcavenae/marvel/internal/session" @@ -38,15 +39,41 @@ type Controller struct { // RehydrateRoleHealth at daemon start. See aae-orc-qdew. roleHealth map[string]*RoleHealth + // admissionHolds latches the last emitted admission Verdict.Key() per + // role so a standing refusal emits one event per transition rather than + // one per reconcile tick. Keyed like roleHealth ("workspace/team/role"). + // + // In-memory only, deliberately. RoleHealth persists because a restart + // count is history a restart must not erase; an admission hold is + // derived from live state and recomputed within one tick of daemon + // start, so a durable copy could only outlive its cause. Re-emitting + // once after a restart is correct: the operator restarted the daemon and + // the condition is still true. See aae-orc-qiay. + admissionHolds map[string]string + // ShiftTimeout bounds how long a single shift may run before the // reconciler declares it stuck and aborts it. Zero uses // defaultShiftTimeout. See aae-orc-qkfl. ShiftTimeout time.Duration + // Snapshots supplies the measured state a full admission check needs. + // Nil is safe and means count-shaped clauses only, which is all the + // reconciler ever evaluates anyway (R2 in internal/admission: gating + // repair on a monotonic meter would be an outage). It exists for + // InitiateShift, whose cumulative clause needs the daemon's meter, and + // keeps this package free of any usage import. + Snapshots Snapshotter + // now is an injection point for tests; nil means time.Now().UTC(). now func() time.Time } +// Snapshotter supplies the measured state one admission check evaluates. +// The daemon implements it over its usage accountant. +type Snapshotter interface { + AdmissionSnapshot(t api.Team) admission.Snapshot +} + // RoleHealth is the per-role crash-loop tracking state. type RoleHealth struct { RestartCount int @@ -76,9 +103,10 @@ const ( // NewController creates a team controller. func NewController(store *api.Store, sessMgr *session.Manager) *Controller { return &Controller{ - store: store, - sessMgr: sessMgr, - roleHealth: make(map[string]*RoleHealth), + store: store, + sessMgr: sessMgr, + roleHealth: make(map[string]*RoleHealth), + admissionHolds: make(map[string]string), } } @@ -205,6 +233,7 @@ func (c *Controller) ClearRoleHealthForTeam(workspace, team string) { c.forgetRoleHealth(k) } } + c.dropAdmissionHolds(prefix) } // ClearRoleHealthForWorkspace deletes crash-loop state for every role @@ -220,6 +249,7 @@ func (c *Controller) ClearRoleHealthForWorkspace(workspace string) { c.forgetRoleHealth(k) } } + c.dropAdmissionHolds(prefix) } // ReconcileOnce runs one reconciliation pass for all teams. @@ -315,6 +345,10 @@ func (c *Controller) reconcileTeam(t *api.Team) { // See aae-orc-69i2. c.reconcileOrphanedSessions(t) + // Drop a recorded admission condition this process never refused, before + // any role is reconciled. See reconcileAdmissionState. + c.reconcileAdmissionState(t) + if t.Shift.Phase != api.ShiftNone { c.reconcileShift(t) return @@ -351,6 +385,7 @@ func (c *Controller) reconcileOrphanedSessions(t *api.Team) { roleKey := t.Workspace + "/" + t.Name + "/" + role delete(c.roleHealth, roleKey) c.forgetRoleHealth(roleKey) + delete(c.admissionHolds, roleKey) log.Printf("reconcile: role %s removed from team %s, drained %d session(s)", role, t.Key(), n) events.Emit(c.Events, events.Event{ Kind: events.KindRoleRemoved, @@ -376,6 +411,15 @@ func (c *Controller) reconcileRole(t *api.Team, role *api.Role) { } } + // An admission hold describes a refusal that is still happening. Drop it + // as soon as the gate below is not reached at all — the role is + // satisfied, the budget was removed from the manifest, or a shift took + // over — so `admission.cleared` fires and Team.Admission stops naming a + // condition that has passed. + if actual >= desired || !t.Budget.Declared() || t.Shift.Phase != api.ShiftNone { + c.clearAdmissionHold(t, role.Name) + } + if actual < desired { // Respect crash-loop backoff. If the role is cooling down from // a recent restart, hold off on spawning replacements until the @@ -390,6 +434,31 @@ func (c *Controller) reconcileRole(t *api.Team, role *api.Role) { if rh, ok := c.roleHealth[roleKey]; ok && c.nowUTC().Before(rh.BackoffUntil) { return } + // Admission backstop against a team-declared budget (aae-orc-qiay, + // resource-matrix enforcement locus 2). The primary refusal point is + // the operator's verb, where nothing has been committed yet; this + // catches the state a manifest declaration cannot see, chiefly a + // declared count that is itself over the ceiling after an + // out-of-band write or two racing scale calls. + // + // Session-count only: the token clause is monotonic within a daemon + // lifetime, so gating repair on it would make an over-budget team + // permanently unrepairable (R2). Placed AFTER the backoff gate so a + // cooling role emits no admission event (backoff is the older, + // stronger condition), and BEFORE ClearCrashedForRole because that + // call mutates store state: refusing after it would delete this + // role's Crashed markers every tick while never spawning. + // + // Skipped entirely while a shift is in progress, so a launching + // generation's transient double count cannot refuse a non-shifting + // role's legitimate repair (R5). + if t.Budget.Declared() && t.Shift.Phase == api.ShiftNone { + granted := c.admit(t, role, desired-actual) + if granted <= 0 { + return + } + desired = actual + granted + } // Crash markers from the reap path have done their observability // job by now (operators saw them during the backoff window). The // fresh session is the new truth — clear stale Crashed markers @@ -704,6 +773,23 @@ func (c *Controller) InitiateShift(teamKey, role string) error { roles = shiftOrder(t.Roles) } + // Admission, before any shift state is written. See admitShift for why + // the gate is here and not in shiftLaunch. + if t.Budget.Declared() { + if v := c.admitShift(&t, roles); v.Refused() { + reason := v.Reason(admission.TriggerShift) + log.Printf("admission: %s shift refused: %s", teamKey, reason) + events.Emit(c.Events, events.Event{ + Kind: events.KindAdmissionRefused, + Severity: events.SeverityWarning, + Workspace: t.Workspace, + Team: t.Name, + Message: reason, + }) + return fmt.Errorf("team %s: %s", teamKey, reason) + } + } + oldGen := t.Generation newGen := oldGen + 1 if err := c.store.UpdateTeam(teamKey, func(live *api.Team) error { diff --git a/internal/usage/accountant.go b/internal/usage/accountant.go index 27ddaa0..b772447 100644 --- a/internal/usage/accountant.go +++ b/internal/usage/accountant.go @@ -485,6 +485,12 @@ func (a *Accountant) addSpendLocked(st *sessionState, s Sample) { st.spend.CacheReadIn += s.CacheReadIn st.spend.CacheCreationIn += s.CacheCreationIn st.spend.ReasoningOut += s.ReasoningOut + // Sample.Occupancy applies the feed's Layout, so this is the one prompt + // figure that is layout-independent once accumulated. It is defined only + // on a non-terminal sample; every call site here sits after fold's + // terminal early-return, and foldTerminalLocked adds no tokens of its + // own, so nothing is missed and nothing terminal is folded. + st.spend.PromptTokens += s.Occupancy() st.spend.Requests++ if s.CostUSD != nil { st.spend.CostUSD += *s.CostUSD @@ -561,6 +567,7 @@ func (a *Accountant) forgetLocked(agentID string) { ts.retired.CacheReadIn += st.spend.CacheReadIn ts.retired.CacheCreationIn += st.spend.CacheCreationIn ts.retired.ReasoningOut += st.spend.ReasoningOut + ts.retired.PromptTokens += st.spend.PromptTokens ts.retired.CostUSD += st.spend.CostUSD ts.retired.Requests += st.spend.Requests if st.spend.CostReported { @@ -678,6 +685,7 @@ func (a *Accountant) TeamSpend(workspace, team string) TeamTotals { out.CacheReadIn += st.spend.CacheReadIn out.CacheCreationIn += st.spend.CacheCreationIn out.ReasoningOut += st.spend.ReasoningOut + out.PromptTokens += st.spend.PromptTokens out.CostUSD += st.spend.CostUSD out.Requests += st.spend.Requests if st.spend.CostReported { diff --git a/internal/usage/accountant_test.go b/internal/usage/accountant_test.go index 8df0b45..f46ec7e 100644 --- a/internal/usage/accountant_test.go +++ b/internal/usage/accountant_test.go @@ -859,3 +859,138 @@ func TestConcurrentObserveIsSafe(t *testing.T) { t.Errorf("ended sessions = %d, want 8", got.EndedSessions) } } + +// TestSpendPromptTokensAppliesLayout is the measurement aae-orc-qiay's token +// budget rests on. +// +// The raw class fields accumulate exactly as the feed reported them and +// Spend records no layout, so no sum of them is both complete and free of +// double counting: In + CacheReadIn + CacheCreationIn double counts a +// subsumptive feed, while In + Out alone omits most of an additive feed's +// input volume. PromptTokens applies Sample.Occupancy per request, so it is +// the one prompt figure a caller can add up without knowing the harness. +// +// Falsification: with the raw classes summed instead, the codex row here +// reads 63153 against a real 42102, so a declared max_tokens would refuse a +// codex team at roughly two thirds of its ceiling, silently. +func TestSpendPromptTokensAppliesLayout(t *testing.T) { + t.Parallel() + tests := []struct { + name string + harness string + samples []rtevents.RequestUsage + want int + }{ + { + name: "subsumptive counts In alone", + harness: codex.Harness, + samples: []rtevents.RequestUsage{ + {Layout: rtevents.LayoutSubsumptive, In: 13992, CacheReadIn: 6996}, + {Layout: rtevents.LayoutSubsumptive, In: 28110, CacheReadIn: 14055}, + }, + want: 13992 + 28110, + }, + { + name: "additive counts In plus the cache classes", + harness: claudecode.Harness, + samples: []rtevents.RequestUsage{ + additive(11368, 16643, 5366), + additive(2, 22009, 11470), + }, + want: 11368 + 16643 + 5366 + 2 + 22009 + 11470, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + a, _, _ := newTestAccountant(t, Table{}) + for _, s := range tt.samples { + a.Observe(testCoords, turnEvent(tt.harness, s)) + } + + spend, ok := a.SessionSpend(testCoords.AgentID) + if !ok { + t.Fatal("no spend recorded") + } + if spend.PromptTokens != tt.want { + t.Errorf("PromptTokens = %d, want %d", spend.PromptTokens, tt.want) + } + + // The figure must promote through retirement into the team total, + // because a fan-out's spend is mostly in sessions that exited. + team := a.TeamSpend(testCoords.Workspace, testCoords.Team) + if team.PromptTokens != tt.want { + t.Errorf("live TeamSpend PromptTokens = %d, want %d", team.PromptTokens, tt.want) + } + a.Forget(testCoords.AgentID) + retired := a.TeamSpend(testCoords.Workspace, testCoords.Team) + if retired.PromptTokens != tt.want { + t.Errorf("retired TeamSpend PromptTokens = %d, want %d", retired.PromptTokens, tt.want) + } + }) + } +} + +// TestTerminalSampleAddsNoPromptTokens proves the placement is safe. +// Sample.Occupancy is meaningless on a terminal sample, which carries +// session totals rather than a level — folding one in is the defect this +// package exists to prevent. Observe returns to foldTerminalLocked before +// any spend call, so a terminal sample must move the figure by nothing. +func TestTerminalSampleAddsNoPromptTokens(t *testing.T) { + t.Parallel() + a, _, _ := newTestAccountant(t, Table{}) + + a.Observe(testCoords, turnEvent(claudecode.Harness, additive(11368, 16643, 5366))) + before, _ := a.SessionSpend(testCoords.AgentID) + + cost := 0.42 + a.Observe(testCoords, endedEvent( + rtevents.Usage{In: 11368, Out: 900, Cost: &cost}, + &rtevents.Metering{CacheReadIn: 16643, CacheCreationIn: 5366}, + )) + + after, _ := a.SessionSpend(testCoords.AgentID) + if after.PromptTokens != before.PromptTokens { + t.Errorf("a terminal sample moved PromptTokens from %d to %d", before.PromptTokens, after.PromptTokens) + } + if !after.CostReported { + t.Error("test is vacuous: the terminal sample was not folded at all") + } +} + +// TestSubagentAndNonPrimarySpendCountsAsPromptTokens: both classes are real +// money against a DIFFERENT context window, so they stay out of occupancy +// and stay in spend. A budget that dropped them would undercount a team +// running Task tools or routing across models. +func TestSubagentAndNonPrimarySpendCountsAsPromptTokens(t *testing.T) { + t.Parallel() + a, _, _ := newTestAccountant(t, Table{}) + + a.Observe(testCoords, startedEvent("claude-fable-5")) + a.Observe(testCoords, turnEvent(claudecode.Harness, rtevents.RequestUsage{ + Layout: rtevents.LayoutAdditive, Model: "claude-fable-5", In: 1000, CacheReadIn: 200, + })) + primary, _ := a.SessionSpend(testCoords.AgentID) + + // A subagent turn: parentToolUseID marks it, so it never enters the + // occupancy fold. + a.Observe(testCoords, turnEvent(claudecode.Harness, rtevents.RequestUsage{ + Layout: rtevents.LayoutAdditive, Model: "claude-fable-5", ParentToolUseID: "toolu_1", + In: 300, CacheReadIn: 50, + })) + // A non-primary model answering inside the same session. + a.Observe(testCoords, turnEvent(claudecode.Harness, rtevents.RequestUsage{ + Layout: rtevents.LayoutAdditive, Model: "claude-haiku-4-5-20251001", In: 40, + })) + + spend, _ := a.SessionSpend(testCoords.AgentID) + want := primary.PromptTokens + 300 + 50 + 40 + if spend.PromptTokens != want { + t.Errorf("PromptTokens = %d, want %d (subagent and non-primary spend included)", spend.PromptTokens, want) + } + occ, _ := a.SessionOccupancy(testCoords.AgentID) + if occ.Tokens != 1200 { + t.Errorf("occupancy tokens = %d, want 1200 (neither sample entered the level)", occ.Tokens) + } +} diff --git a/internal/usage/reader.go b/internal/usage/reader.go index ac062c2..63aaec3 100644 --- a/internal/usage/reader.go +++ b/internal/usage/reader.go @@ -31,7 +31,22 @@ type Spend struct { CacheReadIn int CacheCreationIn int ReasoningOut int - CostUSD float64 + // PromptTokens is the layout-normalized prompt token count: the sum of + // each request's own prompt size, In alone under a subsumptive layout + // and In + the cache classes under an additive one. + // + // It is the only prompt figure a caller can add up without knowing each + // harness's layout. The raw class fields are accumulated as the feed + // reported them and Spend records no layout, so In + CacheReadIn + + // CacheCreationIn double counts a subsumptive feed (codex) while In + + // Out alone omits most of the input volume of an additive one (claude, + // opencode). A budget summing the raw classes would therefore refuse a + // codex team at roughly half its declared ceiling, silently. + // + // Includes subagent and non-primary-model samples, which are real spend + // against other context windows even though they never enter occupancy. + PromptTokens int + CostUSD float64 // CostReported is false when the harness reports no cost at all // (codex publishes none in its exec stream), which is distinct from // reporting zero.