feat(admission): manifest-declared team budgets refuse over-budget spawns

cmd/marvel/main.go

@@ -17,6 +17,7 @@ import (
 
 	"golang.org/x/term"
 
+	"github.com/arcavenae/marvel/internal/admission"
 	"github.com/arcavenae/marvel/internal/api"
 	"github.com/arcavenae/marvel/internal/config"
 	"github.com/arcavenae/marvel/internal/daemon"
@@ -406,6 +407,7 @@ Examples:
   marvel events --kind agent.tool.call       # what the agents are doing
   marvel events --kind agent.session.ended   # per-session cost and timing
   marvel events --kind context.limit-unresolved  # why a CTX% cell is blank
+  marvel events --kind admission.refused     # spawns a team budget refused
   marvel events --warnings                   # only warning-severity events
   marvel --cluster desk events               # remote daemon via mrvl://`,
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -539,7 +541,7 @@ func getCmd() *cobra.Command {
 	var watchSec string
 	cmd := &cobra.Command{
 		Use:   "get <resource-type>",
-		Short: "List resources (sessions, teams, workspaces, endpoints, policies)",
+		Short: "List resources (sessions, teams, workspaces, endpoints, policies, budgets)",
 		Args:  cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if cmd.Flags().Changed("watch") {
@@ -593,6 +595,8 @@ func getResources(resourceType string) error {
 		return printEndpoints(resp.Result)
 	case "policies", "policy":
 		return printPolicies(resp.Result)
+	case "budgets", "budget":
+		return printBudgets(resp.Result)
 	default:
 		fmt.Println(string(resp.Result))
 	}
@@ -1738,6 +1742,66 @@ func printPolicies(data json.RawMessage) error {
 	return w.Flush()
 }
 
+// printBudgets renders `marvel get budgets`: one row per declared budget
+// dimension, with what has been observed against it and that dimension's
+// state (ok, at-ceiling, refusing, unmetered). The surface that answers
+// "which dimension tripped and by how much" — the event says a refusal
+// happened, this says where the team stands. at-ceiling and refusing are
+// deliberately separate: a team sized at its ceiling refuses nothing.
+func printBudgets(data json.RawMessage) error {
+	var rows []admission.Row
+	if err := json.Unmarshal(data, &rows); err != nil {
+		return err
+	}
+	if len(rows) == 0 {
+		fmt.Println("no teams declare a budget")
+		return nil
+	}
+	fmt.Print(renderBudgetTable(rows))
+	return nil
+}
+
+// renderBudgetTable is the pure renderer, split out for the same reason
+// renderSessionTable is: the table's absence handling is worth asserting
+// without a daemon.
+func renderBudgetTable(rows []admission.Row) string {
+	sort.Slice(rows, func(i, j int) bool {
+		if rows[i].Workspace != rows[j].Workspace {
+			return rows[i].Workspace < rows[j].Workspace
+		}
+		if rows[i].Team != rows[j].Team {
+			return rows[i].Team < rows[j].Team
+		}
+		return rows[i].Dimension < rows[j].Dimension
+	})
+	var buf bytes.Buffer
+	w := tabwriter.NewWriter(&buf, 0, 4, 2, ' ', 0)
+	_, _ = fmt.Fprintf(w, "WORKSPACE\tTEAM\tDIMENSION\tLIMIT\tOBSERVED\tHEADROOM\tSTATE\tWINDOW\tNOTE\n")
+	for _, r := range rows {
+		note := r.Note
+		if note == "" {
+			note = "-"
+		}
+		_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%d\t%d\t%s\t%s\t%s\n",
+			r.Workspace, r.Team, r.Dimension, r.Limit, r.Observed, r.Headroom,
+			r.State, formatWindow(r.Window), note)
+	}
+	_ = w.Flush()
+	return buf.String()
+}
+
+// formatWindow renders how long a cumulative figure has been accumulating.
+// A count-shaped dimension has no window and renders a dash, as does a
+// dimension nothing has been measured for yet. The accountant is in-memory,
+// so this span restarts with the daemon; showing it is what keeps a reset
+// visible instead of silent.
+func formatWindow(t time.Time) string {
+	if t.IsZero() {
+		return "-"
+	}
+	return time.Since(t).Round(time.Second).String()
+}
+
 // stripComments removes shell-style comments from CLI arguments.
 // Everything from a bare "#" argument onward is dropped, so that
 // inline notes work: ./marvel shift test/squad  # replace all workers

cmd/marvel/render_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/arcavenae/marvel/internal/admission"
 	"github.com/arcavenae/marvel/internal/api"
 )
 
@@ -210,3 +211,99 @@ func TestSortSessionsByCPUAndRSS(t *testing.T) {
 		t.Errorf("rss desc put %q first, want a", sessions[0].Name)
 	}
 }
+
+// budgetRow returns one rendered row keyed by column name, so a test asserts
+// on cells rather than on whitespace. Every cell in this table is non-empty
+// (absence renders as a dash), so splitting on fields is unambiguous.
+func budgetRow(t *testing.T, table, dimension string) map[string]string {
+	t.Helper()
+	lines := strings.Split(strings.TrimRight(table, "\n"), "\n")
+	header := strings.Fields(lines[0])
+	for _, line := range lines[1:] {
+		fields := strings.Fields(line)
+		if len(fields) < len(header) || fields[2] != dimension {
+			continue
+		}
+		out := make(map[string]string, len(header))
+		for i, h := range header {
+			if h == "NOTE" {
+				// The note is prose and holds spaces, so it takes the rest.
+				out[h] = strings.Join(fields[i:], " ")
+				break
+			}
+			out[h] = fields[i]
+		}
+		return out
+	}
+	t.Fatalf("no %s row in:\n%s", dimension, table)
+	return nil
+}
+
+// TestRenderBudgetTable covers `marvel get budgets`, the surface that answers
+// which dimension tripped and by how much. Its absence handling matters for
+// the same reason CTX%'s does: a token figure nothing has measured must not
+// render as headroom the operator does not have.
+func TestRenderBudgetTable(t *testing.T) {
+	rows := []admission.Row{
+		{
+			Workspace: "fanout", Team: "crew", Dimension: api.DimMaxTokens,
+			Limit: 2000000, Observed: 412118, Headroom: 1587882,
+			State: admission.RowOK, Window: time.Now().UTC().Add(-14 * time.Minute),
+			Note: "partial: some sessions unobserved, so this is a floor",
+		},
+		{
+			Workspace: "fanout", Team: "crew", Dimension: api.DimMaxSessions,
+			Limit: 6, Observed: 6, Headroom: 0, State: admission.RowAtCeiling,
+		},
+	}
+	table := renderBudgetTable(rows)
+
+	// Sorted by dimension within a team, so max_sessions comes first.
+	lines := strings.Split(strings.TrimRight(table, "\n"), "\n")
+	if len(lines) != 3 {
+		t.Fatalf("got %d line(s), want a header and two rows:\n%s", len(lines), table)
+	}
+	if !strings.Contains(lines[1], string(api.DimMaxSessions)) {
+		t.Errorf("first row = %q, want max_sessions (rows sort by dimension)", lines[1])
+	}
+
+	sessions := budgetRow(t, table, string(api.DimMaxSessions))
+	// No headroom is not a refusal: a team whose declared replicas equal its
+	// ceiling sits here permanently and refuses nothing.
+	if sessions["STATE"] != admission.RowAtCeiling {
+		t.Errorf("STATE = %q, want %q with no headroom and nothing refused", sessions["STATE"], admission.RowAtCeiling)
+	}
+	if sessions["HEADROOM"] != "0" {
+		t.Errorf("HEADROOM = %q, want 0", sessions["HEADROOM"])
+	}
+	// A count dimension accumulates over no window, so it must render
+	// absence rather than invent one.
+	if sessions["WINDOW"] != "-" {
+		t.Errorf("WINDOW = %q for a count dimension, want a dash", sessions["WINDOW"])
+	}
+	if sessions["NOTE"] != "-" {
+		t.Errorf("NOTE = %q with nothing to say, want a dash", sessions["NOTE"])
+	}
+
+	tokens := budgetRow(t, table, string(api.DimMaxTokens))
+	if tokens["HEADROOM"] != "1587882" {
+		t.Errorf("HEADROOM = %q, want 1587882", tokens["HEADROOM"])
+	}
+	if tokens["WINDOW"] == "-" {
+		t.Errorf("WINDOW = %q for a cumulative dimension, want the elapsed span", tokens["WINDOW"])
+	}
+	if !strings.Contains(tokens["NOTE"], "partial") {
+		t.Errorf("NOTE = %q, want the partial-total notice", tokens["NOTE"])
+	}
+}
+
+// TestFormatWindow: a dimension that accumulates over no window renders
+// absence, never a zero duration that would read as "just reset".
+func TestFormatWindow(t *testing.T) {
+	if got := formatWindow(time.Time{}); got != "-" {
+		t.Errorf("formatWindow(zero) = %q, want %q", got, "-")
+	}
+	if got := formatWindow(time.Now().UTC().Add(-90 * time.Second)); got == "-" {
+		t.Errorf("formatWindow(90s ago) = %q, want an elapsed duration", got)
+	}
+}

docs/admin-guide.md

@@ -440,3 +440,62 @@ marvel describe session dev/squad-worker-g1-0
 
 Lower the `failure_threshold` or increase the `timeout` if agents need
 more time to initialize.
+
+### A spawn was refused
+
+Two different conditions hold a role back, and they look alike from the
+outside. Tell them apart first:
+
+```bash
+marvel get budgets                                  # where each ceiling stands
+marvel events --kind admission.refused              # a budget refused it
+marvel events --kind health.crashloop-backoff       # a crash loop is cooling
+marvel describe team fanout/crew                    # the declared budget
+```
+
+A budget refusal names its arithmetic, so the fix is usually visible in the
+message. `marvel get budgets` gives the standing picture:
+
+```
+WORKSPACE  TEAM  DIMENSION     LIMIT    OBSERVED  HEADROOM  STATE       WINDOW  NOTE
+fanout     crew  max_sessions  6        6         0         at-ceiling  -       -
+fanout     crew  max_tokens    2000000  412118    1587882   ok          14m3s   partial: some sessions unobserved, so this is a floor
+```
+
+Read the STATE column carefully, because two of its values look alike and
+mean different things:
+
+| STATE | Meaning |
+|---|---|
+| `ok` | Headroom left. |
+| `at-ceiling` | No headroom for growth, and nothing is being refused. This is the resting state of a healthy team, since declared replicas are allowed to equal the ceiling and replacing a crashed replica is exempt. |
+| `refusing` | A refusal is standing right now: the reconciler is holding a role back, and the NOTE column carries the arithmetic. Cross-check with `marvel describe team` (`Admission.held`) and `marvel events --kind admission.refused`. |
+| `unmetered` | Nothing has been measured for this dimension yet, so the figure is absence rather than zero. |
+
+A session row can also read OBSERVED above LIMIT with a `shift` note. That
+is a rotation in flight: the new generation runs beside the old, and a
+session ceiling exempts the overlap. It resolves itself when draining
+finishes.
+
+Two ways out of a session ceiling: raise `max_sessions` in the manifest and
+re-apply, or free headroom with `marvel scale ... --replicas N-1` (a
+scale-down is never refused). Either takes effect on the next reconcile
+tick, within a couple of seconds. There is no clear command and no resume
+verb, because the condition is recomputed from live state every tick.
+
+A token ceiling has no shedding move: retired spend stays counted, since a
+fan-out's cost is mostly in sessions that already exited. Killing sessions
+does not un-spend tokens. Raise `max_tokens` and re-apply.
+
+Two notes in the table matter for trust. `partial` means some contributing
+session was never observed, so the figure is a floor rather than a small
+number, which is why a refusal against it is still sound while an admission
+carries a caveat. `suspect` means the meter caught a cumulation violation
+and the figure may be inflated, so check `marvel daemon logs` before raising
+a ceiling on its account.
+
+One limit to know: `max_tokens` counts from when accounting started, and the
+meter lives in the daemon's memory. A daemon restart or `marvel daemon
+reexec` resets the window, and the daemon says so in its log at startup for
+every team that declares one. The `WINDOW` column dropping back near zero is
+the visible signal that it happened.

docs/user-guide.md

@@ -170,6 +170,90 @@ adapter (claude, forestage, generic). They coexist in the same team.
 Claude Code, workers run forestage with personas, and a monitor runs
 a shell script for health scraping.
 
+### With a team budget
+
+```yaml
+workspace:
+  name: fanout
+
+teams:
+  - name: crew
+    budget:
+      max_sessions: 6
+      max_tokens: 2000000
+      on_unmeasured: admit   # optional; this is the default
+    roles:
+      - name: crew
+        replicas: 3
+        runtime:
+          image: claude
+          command: claude
+          mode: headless
+          prompt: "review the diff"
+```
+
+A budget is a ceiling marvel will refuse to cross. It is the only thing
+that turns a measured number into a refusal, so **a team with no budget
+block declares no gate** and behaves exactly as it did before budgets
+existed.
+
+Two dimensions are enforced today.
+
+`max_sessions` caps live sessions across the whole team: every role, plus
+ad-hoc `marvel run` sessions attributed to the team. It is counted from
+marvel's own records, so it is exact before a spawn and survives a daemon
+restart. The sum of your declared replicas has to fit under it, and a
+manifest that declares more is refused at parse time; so is a
+`marvel scale` that would push the sum over it.
+
+**A rolling shift is the one exemption.** `marvel shift` starts the new
+generation beside the old and drains the old afterwards, so live sessions
+can reach twice a role's replicas for the length of the rotation, and the
+ceiling does not refuse it: replacing a session is not growth, and refusing
+the overlap would mean a team sitting at its ceiling could never rotate.
+`marvel get budgets` says so in the NOTE column while a shift is running.
+If you are sizing `max_sessions` against a hard external concurrency quota,
+size for that overlap, or avoid shifting a team that sits at its ceiling.
+
+`max_tokens` caps prompt plus output plus reasoning tokens, and it counts
+only the sessions marvel can observe: token usage arrives on a harness
+stream, and only a stream-capable harness in headless mode publishes one
+(claude, codex, opencode today). A team where no role can report is refused
+at apply time rather than accepted with a ceiling nothing can ever report
+against; a `generic` role declaring `mode: headless` does not count, because
+that adapter has no stream to read. **It is a since-accounting-started
+budget** — the meter lives in the daemon's memory, so the window restarts
+with the daemon and with `marvel daemon reexec`. Every figure marvel prints
+for it carries that window.
+
+`on_unmeasured` decides what a declared clause does when the meter cannot
+answer at all, which happens before a headless role's first request lands.
+The default admits and says so with an `admission.unmeasured` event, on the
+grounds that you declared a ceiling on a dimension rather than "refuse when
+unmeasurable". Set it to `refuse` if you want the fail-closed posture.
+
+A refusal is never silent. The command fails with the arithmetic, the event
+ring records it, and `marvel get budgets` shows where the team stands:
+
+```bash
+marvel scale fanout/crew --role crew --replicas 40
+# Error: fanout/crew: refused 37 of 37 spawn(s) for role crew: 3 live + 37
+# requested sessions exceeds max_sessions=6 (trigger=scale). Nothing changed;
+# raise the budget in the manifest or free headroom first
+
+marvel get budgets
+marvel events --kind admission.refused
+```
+
+Scaling down is never refused, and replacing a crashed replica is never
+refused: a budget is a ceiling on what you ask for, not a brake on
+recovery.
+
+**When to use:** Any team an agent or a script can scale, where a runaway
+fan-out would collide on one API quota. See
+`examples/demo-act4-budget.yaml` for a walkthrough that needs no model
+auth.
+
 ## Managing sessions
 
 ### List sessions