From 183af9708d7e34600d52f1dc8016b6d3b406a989 Mon Sep 17 00:00:00 2001 From: bourgois Date: Fri, 17 Jul 2026 03:16:26 +0000 Subject: [PATCH 1/2] feat(observability): unsampled tick heartbeat, interval-relative breach, slow_ticks doctor check, host-load stream (vp-qvqk) Four independent fixes to controller tick-health observability: 1. controller.tick_completed now emits on EVERY completed tick. The old breach-or-every-10th sampling made the stream a biased sample: fast ticks were silently omitted, so period/median arithmetic over the stream was valid only while every tick breached - and would report a phantom regression the moment the controller got healthy. 2. threshold_breach is now computed against 2x the configured [daemon] patrol_interval (legacy absolute 5s only when the interval is unknown or non-positive). The old constant 5s threshold was ON for 100% of ticks in a 30-55s regime - a canary carrying zero bits. 3. New slow_ticks supervisor-doctor check consumes threshold_breach over the doctor window and emits doctor.alert - the consumer that makes the flag load-bearing (previously emitted, never read). 4. New host.load_sample event (load1/5/15, cores, runnable-process count, summed per-process %CPU) at patrol cadence on its own goroutine, so a wedged tick cannot stall the series that attributes the wedge. Runnable + %CPU discriminate CPU oversubscription from blocked-on-I/O - Darwin load averages alone cannot. host.load_sample carries a typed payload struct but stays out of KnownEventTypes and the payload registry (same deferral as provider.health_gate_alert): registering it would sweep the payload into the generated EventPayload union ahead of the SSE-projection follow-up. openapi.json + genclient regenerated for the threshold_breach description change only. Local gates: go vet ./... clean; make dashboard-check green; make test-fast-parallel green on darwin except the pre-existing internal/productmetrics PATH_MAX reds documented in vp-zq8h (reproduced there on unmodified eb743642c). --- CHANGELOG.md | 30 +++ cmd/gc/city_runtime.go | 71 ++++--- cmd/gc/city_runtime_tick_test.go | 117 ++++++++++++ cmd/gc/host_load.go | 166 +++++++++++++++++ cmd/gc/host_load_test.go | 226 +++++++++++++++++++++++ cmd/gc/supervisor_doctor.go | 70 +++++++ cmd/gc/supervisor_doctor_test.go | 96 ++++++++++ docs/reference/schema/openapi.json | 2 +- docs/reference/schema/openapi.txt | 2 +- internal/api/genclient/client_gen.go | 2 +- internal/api/openapi.json | 2 +- internal/events/events.go | 1 + internal/events/hostload_payloads.go | 36 ++++ internal/events/storehealth_payloads.go | 20 +- internal/supervisordoctor/doctor.go | 38 ++++ internal/supervisordoctor/doctor_test.go | 30 +++ 16 files changed, 869 insertions(+), 40 deletions(-) create mode 100644 cmd/gc/city_runtime_tick_test.go create mode 100644 cmd/gc/host_load.go create mode 100644 cmd/gc/host_load_test.go create mode 100644 internal/events/hostload_payloads.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 301d0009d5..203907d64f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 on stderr and as `"path": "storeless-fallback"` in `--json` output. With the flag unset the behavior is unchanged. New file `cmd/gc/cmd_nudge_storeless.go`. +- **Host-load event stream + slow-tick doctor check (vp-qvqk / defects 3+1).** + The controller now emits a periodic `host.load_sample` event (load1/5/15, + logical cores, runnable-process count, summed per-process %CPU) at patrol + cadence from its own goroutine, so a wedged reconcile tick cannot stall the + series that attributes the wedge. Runnable + %CPU ride alongside the load + averages because Darwin's load average also counts uninterruptible waits — + load alone cannot discriminate CPU oversubscription from blocked-on-I/O. + The supervisor doctor gains a `slow_ticks` check that reads the tick + heartbeat's `threshold_breach` flag over the doctor window and emits a + `doctor.alert` when any tick breached — the consumer that makes the flag + load-bearing (it was previously emitted and never read). New event type + `host.load_sample` carries a typed payload struct but is deliberately left + out of `KnownEventTypes` and the payload registry until the SSE projection + follow-up (same deferral as `provider.health_gate_alert`); subscribers + receive it via the custom-event envelope. - **L0 pre-heal in `ensure-project-id`: auto-restore canonical project_id from `city.toml [identity_map]` when the DB confirms it but L1 was wiped (vp-cz7o.21).** @@ -59,6 +74,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 pre-heal step only. New file `cmd/gc/city_identity_map.go`; ~20 lines added to `ensureManagedDoltProjectIDWithRecorder`. +### Changed + +- **Controller tick heartbeat: emit every tick, breach threshold relative to + the patrol interval (vp-qvqk / defects 2+1).** `controller.tick_completed` + now fires once per completed tick instead of on breach-or-every-10th: the + sampled stream was a biased sample (fast ticks silently omitted), so any + period/median arithmetic over it was valid only while every tick breached — + a coincidence that would have flipped into a phantom regression the moment + the controller got healthy. The `threshold_breach` flag is now computed + against 2× the configured `[daemon] patrol_interval` (falling back to the + legacy absolute 5s only when the interval is unknown or non-positive) + instead of a constant 5s that had been ON for 100% of ticks in a 30-55s + regime. Consumers of `threshold_breach` should expect it to mean "lost + cadence for a full interval", not "took more than 5 seconds". + ### Upgrading Notes - **Every graph-owning store scope needs a `Dir`-matched `control-dispatcher` diff --git a/cmd/gc/city_runtime.go b/cmd/gc/city_runtime.go index 77c5f21411..661f3aa767 100644 --- a/cmd/gc/city_runtime.go +++ b/cmd/gc/city_runtime.go @@ -119,13 +119,6 @@ type CityRuntime struct { fsPressureConsecutiveSkips int fsPressureEpisodeLogged bool - // tickCount counts completed reconcile ticks for the controller - // heartbeat. controller.tick_completed is emitted at a patrol multiple - // (every tickHeartbeatEvery ticks) or when a tick's duration breaches - // tickHeartbeatSlowThreshold — never on every tick. Single-goroutine - // (the reconcile loop owns it); no synchronization needed. - tickCount uint64 - convScopes map[string]*convergenceScope // nil until bead store available; keyed by rig name ("" = city/HQ) convScopesMu sync.RWMutex // guards convScopes map pointer convergenceReqCh chan convergenceRequest // receives CLI commands from controller.sock @@ -718,6 +711,10 @@ func (cr *CityRuntime) run(ctx context.Context) { ticker := time.NewTicker(interval) defer ticker.Stop() + // Host-load series runs on its own goroutine at patrol cadence: a + // wedged tick must not stall the stream that attributes the wedge. + go cr.hostLoadSampler(ctx, interval) + // Start the supervisor nudge dispatcher when configured. The wake-socket // listener feeds nudgeWakeCh on every producer enqueue, giving sub-second // dispatch latency. Patrol-tick fallback inside cr.tick() guarantees @@ -848,37 +845,55 @@ func (cr *CityRuntime) safeTick(fn func(), trigger string) (panicked bool) { return false } -// Controller heartbeat cadence. The tick_completed event is the -// supervisor doctor's tick-age signal; emitting it every tick would make -// the event log a hot path, so it fires on a patrol multiple or when a -// tick runs slow. +// Controller heartbeat. Every completed tick emits a +// controller.tick_completed event: the stream is both the supervisor +// doctor's tick-age/slow-tick signal and the fleet's tick-duration +// series, and an unsampled stream is the only shape consumers can do +// period/median arithmetic on without de-biasing (vp-qvqk — the earlier +// breach-or-every-10th sampling silently omitted fast ticks, so the +// series was complete only while every tick breached). One event per +// tick is patrol-cadence volume (~2/min at the 30s default), not a hot +// path. const ( - // tickHeartbeatEvery emits one heartbeat per this many completed ticks. - tickHeartbeatEvery = 10 - // tickHeartbeatSlowThreshold forces an out-of-cadence heartbeat when a - // single tick exceeds it, so a degrading controller surfaces before the - // next scheduled heartbeat. - tickHeartbeatSlowThreshold = 5 * time.Second + // tickSlowIntervalMultiple flags a tick as slow when its duration + // reaches this many patrol intervals. Scaling with the configured + // cadence keeps the canary calibrated to the regime it watches — the + // old absolute 5s threshold was ON for 100% of ticks in a 30-55s + // regime, a constant carrying zero bits. At 2× the loop has lost + // cadence for a full interval, the leading indicator of the doctor's + // 3×-patrol tick-age alert. + tickSlowIntervalMultiple = 2 + // tickSlowFallbackThreshold is the slow-tick threshold when the + // patrol interval is unknown or non-positive, so the breach flag can + // never degenerate to always-true. + tickSlowFallbackThreshold = 5 * time.Second ) -// recordTickHeartbeat emits a controller.tick_completed event at a patrol -// multiple or on a duration-threshold breach. It is the controller -// heartbeat the supervisor-cadence doctor reads to compute tick age -// (plan item 1.9). Best-effort: a nil recorder is a no-op. -func (cr *CityRuntime) recordTickHeartbeat(trigger string, dur time.Duration) { - cr.tickCount++ - breach := dur >= tickHeartbeatSlowThreshold - onMultiple := cr.tickCount%tickHeartbeatEvery == 0 - if !breach && !onMultiple { - return +// slowTickThreshold returns the duration at or above which a completed +// tick is flagged (threshold_breach) in its heartbeat event. +func (cr *CityRuntime) slowTickThreshold() time.Duration { + if cr.cfg == nil { + return tickSlowFallbackThreshold } + if interval := cr.cfg.Daemon.PatrolIntervalDuration(); interval > 0 { + return time.Duration(tickSlowIntervalMultiple) * interval + } + return tickSlowFallbackThreshold +} + +// recordTickHeartbeat emits a controller.tick_completed event for every +// completed tick. It is the controller heartbeat the supervisor-cadence +// doctor reads to compute tick age and surface slow ticks (plan item +// 1.9); ThresholdBreach marks ticks at or past slowTickThreshold. +// Best-effort: a nil recorder is a no-op. +func (cr *CityRuntime) recordTickHeartbeat(trigger string, dur time.Duration) { if cr.rec == nil { return } payload, err := json.Marshal(events.ControllerTickCompletedPayload{ DurationMs: dur.Milliseconds(), Phase: trigger, - ThresholdBreach: breach, + ThresholdBreach: dur >= cr.slowTickThreshold(), }) if err != nil { return diff --git a/cmd/gc/city_runtime_tick_test.go b/cmd/gc/city_runtime_tick_test.go new file mode 100644 index 0000000000..dfcf8897db --- /dev/null +++ b/cmd/gc/city_runtime_tick_test.go @@ -0,0 +1,117 @@ +package main + +import ( + "testing" + "time" + + "github.com/gastownhall/gascity/internal/config" + "github.com/gastownhall/gascity/internal/events" +) + +// decodeTickPayload decodes one controller.tick_completed event's payload. +func decodeTickPayload(t *testing.T, e events.Event) events.ControllerTickCompletedPayload { + t.Helper() + if e.Type != events.ControllerTickCompleted { + t.Fatalf("event type = %q, want %q", e.Type, events.ControllerTickCompleted) + } + decoded, _, err := events.DecodePayload(e.Type, e.Payload) + if err != nil { + t.Fatalf("decode tick payload: %v", err) + } + p, ok := decoded.(events.ControllerTickCompletedPayload) + if !ok { + t.Fatalf("decoded payload type %T, want ControllerTickCompletedPayload", decoded) + } + return p +} + +// TestRecordTickHeartbeatEmitsEveryTick asserts the heartbeat stream is +// unsampled: one event per completed tick, fast or slow (vp-qvqk defect 2 +// — the old breach-or-every-10th gate made the stream a biased sample, so +// period arithmetic over it was valid only while every tick breached). +func TestRecordTickHeartbeatEmitsEveryTick(t *testing.T) { + ep := events.NewFake() + cfg := &config.City{} + cfg.Daemon.PatrolInterval = "10s" + cr := &CityRuntime{cityName: "testcity", cfg: cfg, rec: ep} + + // 25 fast ticks: under the old sampling only ticks 10 and 20 would + // emit; every one of these is far below any breach threshold. + for i := 0; i < 25; i++ { + cr.recordTickHeartbeat("patrol", 10*time.Millisecond) + } + + if got := len(ep.Events); got != 25 { + t.Fatalf("events emitted = %d, want 25 (one per tick, no sampling)", got) + } + for i, e := range ep.Events { + p := decodeTickPayload(t, e) + if p.ThresholdBreach { + t.Fatalf("event %d: ThresholdBreach = true for a 10ms tick at 10s patrol", i) + } + if p.Phase != "patrol" { + t.Errorf("event %d: phase = %q, want patrol", i, p.Phase) + } + } +} + +// TestRecordTickHeartbeatBreachRelativeToPatrolInterval asserts the +// slow-tick flag is calibrated to the configured cadence, not an absolute +// constant (vp-qvqk defect 1 — a fixed 5s threshold in a 30-55s tick +// regime was ON for 100% of ticks and carried zero bits). +func TestRecordTickHeartbeatBreachRelativeToPatrolInterval(t *testing.T) { + tests := []struct { + name string + interval string + dur time.Duration + wantBreach bool + }{ + // 10s patrol → 20s threshold. + {"under threshold", "10s", 19 * time.Second, false}, + {"at threshold", "10s", 20 * time.Second, true}, + {"past threshold", "10s", 443 * time.Second, true}, + // The vc-wz5 regime: 30-55s ticks at a 30s patrol must NOT breach — + // under the old absolute 5s constant every one of them did. + {"steady 55s tick at 30s patrol", "30s", 55 * time.Second, false}, + {"excursion at 30s patrol", "30s", 61 * time.Second, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ep := events.NewFake() + cfg := &config.City{} + cfg.Daemon.PatrolInterval = tc.interval + cr := &CityRuntime{cityName: "testcity", cfg: cfg, rec: ep} + + cr.recordTickHeartbeat("patrol", tc.dur) + + if len(ep.Events) != 1 { + t.Fatalf("events emitted = %d, want 1", len(ep.Events)) + } + p := decodeTickPayload(t, ep.Events[0]) + if p.ThresholdBreach != tc.wantBreach { + t.Fatalf("ThresholdBreach = %v, want %v (dur=%s interval=%s)", p.ThresholdBreach, tc.wantBreach, tc.dur, tc.interval) + } + if p.DurationMs != tc.dur.Milliseconds() { + t.Errorf("DurationMs = %d, want %d", p.DurationMs, tc.dur.Milliseconds()) + } + }) + } +} + +// TestSlowTickThresholdFallback asserts the threshold degrades to the +// legacy absolute rather than to zero (which would make breach +// always-true, the exact defect this replaces). +func TestSlowTickThresholdFallback(t *testing.T) { + nilCfg := &CityRuntime{} + if got := nilCfg.slowTickThreshold(); got != tickSlowFallbackThreshold { + t.Fatalf("nil cfg threshold = %s, want %s", got, tickSlowFallbackThreshold) + } + + cfg := &config.City{} + cfg.Daemon.PatrolInterval = "10s" + cr := &CityRuntime{cfg: cfg} + want := time.Duration(tickSlowIntervalMultiple) * 10 * time.Second + if got := cr.slowTickThreshold(); got != want { + t.Fatalf("threshold = %s, want %s (%d× patrol)", got, want, tickSlowIntervalMultiple) + } +} diff --git a/cmd/gc/host_load.go b/cmd/gc/host_load.go new file mode 100644 index 0000000000..88ed1e10ef --- /dev/null +++ b/cmd/gc/host_load.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + goruntime "runtime" + "strconv" + "strings" + "time" + + "github.com/gastownhall/gascity/internal/events" +) + +// hostLoadSample is one host-load observation. Field semantics match +// events.HostLoadSamplePayload. +type hostLoadSample struct { + load1, load5, load15 float64 + runnableProcs int + totalCPUPercent float64 +} + +// sampleHostLoad probes the live host. Package-level so tests can stub +// the probe without spawning subprocesses. +var sampleHostLoad = sampleHostLoadReal + +// hostLoadSampler emits a host.load_sample event every interval until ctx +// is done. It runs on its own goroutine — never on the reconcile loop — +// so the series keeps flowing while a tick is wedged: attributing a tick +// excursion to host load is needed exactly when the tick loop cannot be +// the emitter (vp-qvqk; without the series every load excursion cost a +// manual ps/uptime forensic and left nothing retrospective). Best-effort: +// a sampler failure warns to stderr once and the loop keeps trying. +func (cr *CityRuntime) hostLoadSampler(ctx context.Context, interval time.Duration) { + if cr.rec == nil || interval <= 0 { + return + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + warned := false + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + s, err := sampleHostLoad() + if err != nil { + if !warned { + warned = true + fmt.Fprintf(cr.stderr, "%s: host-load sampler: %v (suppressing further warnings; sampling continues)\n", //nolint:errcheck // best-effort stderr + cr.logPrefix, err) + } + continue + } + payload, err := json.Marshal(events.HostLoadSamplePayload{ + Load1: s.load1, + Load5: s.load5, + Load15: s.load15, + Cores: goruntime.NumCPU(), + RunnableProcs: s.runnableProcs, + TotalCPUPercent: s.totalCPUPercent, + }) + if err != nil { + continue + } + cr.rec.Record(events.Event{ + Type: events.HostLoadSample, + Actor: eventActor(), + Subject: cr.cityName, + Payload: payload, + }) + } +} + +// sampleHostLoadReal probes the live host: load averages from the +// platform's loadavg source, runnable count and summed %CPU from one ps +// pass. Load average alone cannot answer "oversubscribed or blocked?" — +// on Darwin it also counts uninterruptible waits — which is why runnable +// and %CPU ride in the same sample. +func sampleHostLoadReal() (hostLoadSample, error) { + var s hostLoadSample + l1, l5, l15, err := readLoadAverages() + if err != nil { + return s, err + } + runnable, cpu, err := readProcessTable() + if err != nil { + return s, err + } + return hostLoadSample{ + load1: l1, + load5: l5, + load15: l15, + runnableProcs: runnable, + totalCPUPercent: cpu, + }, nil +} + +// readLoadAverages reads the 1/5/15-minute load averages from +// /proc/loadavg where it exists (Linux) and falls back to +// `sysctl -n vm.loadavg` (Darwin/BSD). +func readLoadAverages() (l1, l5, l15 float64, err error) { + if data, rerr := os.ReadFile("/proc/loadavg"); rerr == nil { + return parseLoadAvgFields(string(data)) + } + out, serr := exec.Command("sysctl", "-n", "vm.loadavg").Output() + if serr != nil { + return 0, 0, 0, fmt.Errorf("no loadavg source: /proc/loadavg unavailable and sysctl -n vm.loadavg failed: %w", serr) + } + return parseLoadAvgFields(string(out)) +} + +// parseLoadAvgFields extracts the first three float fields from a loadavg +// line, tolerating both the Darwin sysctl braces ("{ 1.86 2.02 2.05 }") +// and the /proc/loadavg shape ("1.86 2.02 2.05 2/345 6789" — the +// non-float runnable/total field is skipped before three values land). +func parseLoadAvgFields(raw string) (float64, float64, float64, error) { + fields := strings.Fields(strings.NewReplacer("{", " ", "}", " ").Replace(raw)) + vals := make([]float64, 0, 3) + for _, f := range fields { + v, err := strconv.ParseFloat(f, 64) + if err != nil { + continue + } + vals = append(vals, v) + if len(vals) == 3 { + return vals[0], vals[1], vals[2], nil + } + } + return 0, 0, 0, fmt.Errorf("unparseable loadavg %q", strings.TrimSpace(raw)) +} + +// readProcessTable runs one ps pass over every process and returns the +// count in runnable state plus the summed %CPU. ps is the portable +// (Darwin+Linux) source for per-process state without a /proc walk. +func readProcessTable() (runnable int, totalCPUPercent float64, err error) { + out, err := exec.Command("ps", "-A", "-o", "state=,pcpu=").Output() + if err != nil { + return 0, 0, fmt.Errorf("ps -A -o state=,pcpu=: %w", err) + } + runnable, totalCPUPercent = parseProcessTable(string(out)) + return runnable, totalCPUPercent, nil +} + +// parseProcessTable parses `ps -A -o state=,pcpu=` output: one process +// per line, a state token (leading R = runnable/on-CPU, further modifier +// characters may follow) and a %CPU float. Unparseable lines are skipped +// — a shorter table is a degraded sample, not an error. +func parseProcessTable(out string) (runnable int, totalCPUPercent float64) { + for _, line := range strings.Split(out, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + if strings.HasPrefix(fields[0], "R") { + runnable++ + } + if v, err := strconv.ParseFloat(fields[1], 64); err == nil { + totalCPUPercent += v + } + } + return runnable, totalCPUPercent +} diff --git a/cmd/gc/host_load_test.go b/cmd/gc/host_load_test.go new file mode 100644 index 0000000000..3b481e11a2 --- /dev/null +++ b/cmd/gc/host_load_test.go @@ -0,0 +1,226 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + goruntime "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/gastownhall/gascity/internal/events" +) + +// syncRecorder is a race-safe events.Recorder for goroutine-emitter tests. +type syncRecorder struct { + mu sync.Mutex + events []events.Event + recorded chan struct{} +} + +func newSyncRecorder() *syncRecorder { + return &syncRecorder{recorded: make(chan struct{}, 64)} +} + +func (r *syncRecorder) Record(e events.Event) { + r.mu.Lock() + r.events = append(r.events, e) + r.mu.Unlock() + select { + case r.recorded <- struct{}{}: + default: + } +} + +func (r *syncRecorder) snapshot() []events.Event { + r.mu.Lock() + defer r.mu.Unlock() + return append([]events.Event(nil), r.events...) +} + +// syncBuffer is a race-safe io.Writer for goroutine-emitter tests. +type syncBuffer struct { + mu sync.Mutex + b strings.Builder +} + +func (w *syncBuffer) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.b.Write(p) +} + +func (w *syncBuffer) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.b.String() +} + +func TestParseLoadAvgFields(t *testing.T) { + tests := []struct { + name string + raw string + want [3]float64 + wantErr bool + }{ + {"darwin sysctl braces", "{ 1.86 2.02 2.05 }\n", [3]float64{1.86, 2.02, 2.05}, false}, + // /proc/loadavg's 4th field (runnable/total) is not a float and must + // not be swallowed into the triple. + {"linux proc line", "0.52 0.58 0.59 2/345 6789\n", [3]float64{0.52, 0.58, 0.59}, false}, + {"garbage", "no loads here\n", [3]float64{}, true}, + {"too few fields", "1.5 2.5\n", [3]float64{}, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + l1, l5, l15, err := parseLoadAvgFields(tc.raw) + if (err != nil) != tc.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tc.wantErr) + } + if err != nil { + return + } + if got := [3]float64{l1, l5, l15}; got != tc.want { + t.Fatalf("loads = %v, want %v", got, tc.want) + } + }) + } +} + +func TestParseProcessTable(t *testing.T) { + // Shapes seen from `ps -A -o state=,pcpu=`: bare states, states with + // modifier suffixes (R+, Ss), and whitespace-padded columns. Only a + // leading R counts as runnable; every parseable %CPU sums. + out := strings.Join([]string{ + "S 0.0", + "R 12.5", + "R+ 50.0", + "Ss 0.3", + "U 2.2", + "Z 0.0", + "garbage-line", + "", + }, "\n") + runnable, cpu := parseProcessTable(out) + if runnable != 2 { + t.Fatalf("runnable = %d, want 2 (R and R+)", runnable) + } + if want := 65.0; cpu != want { + t.Fatalf("total %%CPU = %v, want %v", cpu, want) + } +} + +// TestSampleHostLoadRealOnThisHost exercises the live probe end-to-end; +// it asserts shape, not values, so it stays green on any load. +func TestSampleHostLoadRealOnThisHost(t *testing.T) { + s, err := sampleHostLoadReal() + if err != nil { + t.Fatalf("sampleHostLoadReal: %v", err) + } + if s.load1 < 0 || s.load5 < 0 || s.load15 < 0 { + t.Fatalf("negative load average: %+v", s) + } + if s.runnableProcs < 0 || s.totalCPUPercent < 0 { + t.Fatalf("negative process-table values: %+v", s) + } +} + +// TestHostLoadSamplerEmitsTypedEvents asserts the sampler goroutine emits +// host.load_sample events with the sampled values and stops on ctx +// cancellation. +func TestHostLoadSamplerEmitsTypedEvents(t *testing.T) { + prev := sampleHostLoad + sampleHostLoad = func() (hostLoadSample, error) { + return hostLoadSample{load1: 36.5, load5: 20.1, load15: 10.2, runnableProcs: 18, totalCPUPercent: 412.7}, nil + } + defer func() { sampleHostLoad = prev }() + + rec := newSyncRecorder() + cr := &CityRuntime{cityName: "testcity", rec: rec, stderr: &syncBuffer{}, logPrefix: "gc test"} + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { cr.hostLoadSampler(ctx, 2*time.Millisecond); close(done) }() + + select { + case <-rec.recorded: + case <-time.After(5 * time.Second): + t.Fatal("no host.load_sample event within 5s") + } + cancel() + <-done + + evts := rec.snapshot() + if len(evts) == 0 { + t.Fatal("no events recorded") + } + e := evts[0] + if e.Type != events.HostLoadSample { + t.Fatalf("event type = %q, want %q", e.Type, events.HostLoadSample) + } + if e.Subject != "testcity" { + t.Errorf("subject = %q, want testcity", e.Subject) + } + // host.load_sample is deliberately not payload-registered (SSE + // projection deferred — see hostload_payloads.go), so decode the + // typed struct directly rather than via events.DecodePayload. + var p events.HostLoadSamplePayload + if err := json.Unmarshal(e.Payload, &p); err != nil { + t.Fatalf("decode host.load_sample payload: %v", err) + } + if p.Load1 != 36.5 || p.Load5 != 20.1 || p.Load15 != 10.2 { + t.Errorf("loads = %v/%v/%v, want 36.5/20.1/10.2", p.Load1, p.Load5, p.Load15) + } + if p.RunnableProcs != 18 { + t.Errorf("runnable_procs = %d, want 18", p.RunnableProcs) + } + if p.TotalCPUPercent != 412.7 { + t.Errorf("total_cpu_percent = %v, want 412.7", p.TotalCPUPercent) + } + if p.Cores != goruntime.NumCPU() { + t.Errorf("cores = %d, want %d", p.Cores, goruntime.NumCPU()) + } +} + +// TestHostLoadSamplerWarnsOnceOnProbeFailure asserts a failing probe +// warns exactly once (fail-loud, then quiet) and emits no events. +func TestHostLoadSamplerWarnsOnceOnProbeFailure(t *testing.T) { + prev := sampleHostLoad + calls := make(chan struct{}, 64) + sampleHostLoad = func() (hostLoadSample, error) { + select { + case calls <- struct{}{}: + default: + } + return hostLoadSample{}, fmt.Errorf("probe exploded") + } + defer func() { sampleHostLoad = prev }() + + rec := newSyncRecorder() + buf := &syncBuffer{} + cr := &CityRuntime{cityName: "testcity", rec: rec, stderr: buf, logPrefix: "gc test"} + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { cr.hostLoadSampler(ctx, 2*time.Millisecond); close(done) }() + + // Wait for at least three probe attempts so a per-attempt warning + // would have repeated. + for i := 0; i < 3; i++ { + select { + case <-calls: + case <-time.After(5 * time.Second): + t.Fatal("sampler stopped probing after a failure") + } + } + cancel() + <-done + + if evts := rec.snapshot(); len(evts) != 0 { + t.Fatalf("events recorded = %d, want 0 on probe failure", len(evts)) + } + if got := strings.Count(buf.String(), "host-load sampler"); got != 1 { + t.Fatalf("stderr warnings = %d, want exactly 1:\n%s", got, buf.String()) + } +} diff --git a/cmd/gc/supervisor_doctor.go b/cmd/gc/supervisor_doctor.go index b60ae3c497..62f04abb59 100644 --- a/cmd/gc/supervisor_doctor.go +++ b/cmd/gc/supervisor_doctor.go @@ -104,6 +104,25 @@ func evaluateCityDoctorSubset(state api.State, _ io.Writer) { alerts = append(alerts, *a) } + // Slow ticks: consume the heartbeat's threshold_breach flag (vp-qvqk — + // a canary nothing reads carries no signal). The window is this city's + // doctor cadence, so each pass reviews roughly the ticks since the + // previous one. + window := cfg.Doctor.SupervisorIntervalOrDefault() + if window <= 0 { + window = defaultSupervisorDoctorInterval + } + breaches, samples, slowestMs := recentTickBreaches(ep, window) + if a := supervisordoctor.CheckSlowTicksFor(supervisordoctor.SlowTicksInput{ + City: cityName, + BreachCount: breaches, + SampleCount: samples, + SlowestMs: slowestMs, + Window: window, + }); a != nil { + alerts = append(alerts, *a) + } + // S6 connection ceiling: scopes × (pool+1) ≤ 0.8 × max_connections. // collectStage1ScopeRoots dedups colliding rig paths, so the count never // over-estimates the connection ceiling when two rigs resolve to the same @@ -156,6 +175,57 @@ func lastControllerTickAge(ep events.Provider, _ string) (time.Duration, bool) { return age, true } +// maxTickEventsScanned bounds the slow-tick window read. At one heartbeat +// per tick and patrol-cadence ticks, a 10m doctor window holds ~20 +// events; 512 leaves an order of magnitude of headroom without letting a +// pathological log turn the doctor pass into a full scan. +const maxTickEventsScanned = 512 + +// recentTickBreaches inspects the controller.tick_completed events inside +// the trailing window and returns how many carried threshold_breach, how +// many were inspected, and the slowest duration seen. A nil provider or +// read error yields zeros so the slow-ticks check is skipped rather than +// firing a false positive. Events whose payload does not decode to the +// tick payload are not counted as samples. +func recentTickBreaches(ep events.Provider, window time.Duration) (breaches, samples int, slowestMs int64) { + if ep == nil || window <= 0 { + return 0, 0, 0 + } + filter := events.Filter{ + Type: events.ControllerTickCompleted, + Since: supervisorDoctorClock().Add(-window), + } + var list []events.Event + var err error + if tp, ok := ep.(events.TailProvider); ok { + list, err = tp.ListTail(filter, maxTickEventsScanned) + } else { + filter.Limit = maxTickEventsScanned + list, err = ep.List(filter) + } + if err != nil { + return 0, 0, 0 + } + for _, e := range list { + decoded, _, derr := events.DecodePayload(e.Type, e.Payload) + if derr != nil { + continue + } + p, ok := decoded.(events.ControllerTickCompletedPayload) + if !ok { + continue + } + samples++ + if p.DurationMs > slowestMs { + slowestMs = p.DurationMs + } + if p.ThresholdBreach { + breaches++ + } + } + return breaches, samples, slowestMs +} + // agentConfigDirRoots returns the agent config-dir roots the isolation check // should scan for escaping symlinks. It is conservative and read-only: the // city's .gc/agents scaffold tree plus any per-agent config dirs gc diff --git a/cmd/gc/supervisor_doctor_test.go b/cmd/gc/supervisor_doctor_test.go index 464d2e4d8f..cd282ad9b8 100644 --- a/cmd/gc/supervisor_doctor_test.go +++ b/cmd/gc/supervisor_doctor_test.go @@ -1,7 +1,9 @@ package main import ( + "encoding/json" "io" + "strings" "testing" "time" @@ -106,6 +108,100 @@ func TestDoctorSubsetNoHeartbeatNoTickAlert(t *testing.T) { } } +// recordTickEvent records a controller.tick_completed event with a typed +// payload at ts. +func recordTickEvent(t *testing.T, ep *events.Fake, ts time.Time, breach bool, durationMs int64) { + t.Helper() + payload, err := json.Marshal(events.ControllerTickCompletedPayload{ + DurationMs: durationMs, + Phase: "patrol", + ThresholdBreach: breach, + }) + if err != nil { + t.Fatalf("marshal tick payload: %v", err) + } + ep.Record(events.Event{Type: events.ControllerTickCompleted, Ts: ts, Payload: payload}) +} + +// TestDoctorSubsetSlowTicksBreachEmitsAlert asserts breached ticks inside +// the doctor window produce a slow_ticks doctor.alert — the consumer that +// makes the heartbeat's threshold_breach flag load-bearing (vp-qvqk +// defect 1: the flag was emitted and never read). +func TestDoctorSubsetSlowTicksBreachEmitsAlert(t *testing.T) { + cfg := &config.City{} + cfg.Daemon.PatrolInterval = "30s" + cs, ep := newDoctorTestState(t, cfg) + + now := time.Date(2026, 7, 17, 1, 30, 0, 0, time.UTC) + // Two breached ticks and one clean tick inside the default 10m window. + recordTickEvent(t, ep, now.Add(-8*time.Minute), true, 443000) + recordTickEvent(t, ep, now.Add(-5*time.Minute), false, 32000) + recordTickEvent(t, ep, now.Add(-2*time.Minute), true, 91000) + + prev := supervisorDoctorClock + supervisorDoctorClock = func() time.Time { return now } + defer func() { supervisorDoctorClock = prev }() + + evaluateCityDoctorSubset(cs, io.Discard) + + alerts := alertsOfCheck(t, ep, supervisordoctor.CheckNameSlowTicks) + if len(alerts) != 1 { + t.Fatalf("slow_ticks alerts = %d, want 1", len(alerts)) + } + if alerts[0].City != "testcity" { + t.Errorf("alert city = %q, want testcity", alerts[0].City) + } + if !strings.Contains(alerts[0].Detail, "2 of 3") { + t.Errorf("alert detail = %q, want it to count 2 of 3 breached ticks", alerts[0].Detail) + } +} + +// TestDoctorSubsetSlowTicksCleanWindowNoAlert asserts an all-clean window +// stays quiet. +func TestDoctorSubsetSlowTicksCleanWindowNoAlert(t *testing.T) { + cfg := &config.City{} + cfg.Daemon.PatrolInterval = "30s" + cs, ep := newDoctorTestState(t, cfg) + + now := time.Date(2026, 7, 17, 1, 30, 0, 0, time.UTC) + recordTickEvent(t, ep, now.Add(-4*time.Minute), false, 31000) + recordTickEvent(t, ep, now.Add(-2*time.Minute), false, 47000) + + prev := supervisorDoctorClock + supervisorDoctorClock = func() time.Time { return now } + defer func() { supervisorDoctorClock = prev }() + + evaluateCityDoctorSubset(cs, io.Discard) + + if got := len(alertsOfCheck(t, ep, supervisordoctor.CheckNameSlowTicks)); got != 0 { + t.Fatalf("slow_ticks alerts = %d, want 0 (clean window)", got) + } +} + +// TestDoctorSubsetSlowTicksOldBreachOutsideWindowNoAlert asserts a breach +// older than the doctor window does not re-alert forever. +func TestDoctorSubsetSlowTicksOldBreachOutsideWindowNoAlert(t *testing.T) { + cfg := &config.City{} + cfg.Daemon.PatrolInterval = "30s" + cs, ep := newDoctorTestState(t, cfg) + + now := time.Date(2026, 7, 17, 1, 30, 0, 0, time.UTC) + // Breach 2h ago — far outside the default 10m window. A fresh clean + // tick keeps the tick-age check quiet so this isolates slow_ticks. + recordTickEvent(t, ep, now.Add(-2*time.Hour), true, 443000) + recordTickEvent(t, ep, now.Add(-1*time.Minute), false, 30000) + + prev := supervisorDoctorClock + supervisorDoctorClock = func() time.Time { return now } + defer func() { supervisorDoctorClock = prev }() + + evaluateCityDoctorSubset(cs, io.Discard) + + if got := len(alertsOfCheck(t, ep, supervisordoctor.CheckNameSlowTicks)); got != 0 { + t.Fatalf("slow_ticks alerts = %d, want 0 (breach outside window)", got) + } +} + // TestDoctorSubsetS6CeilingBreachEmitsAlert asserts the S6 connection // ceiling fires when scopes × (pool+1) exceeds 0.8 × max_connections. func TestDoctorSubsetS6CeilingBreachEmitsAlert(t *testing.T) { diff --git a/docs/reference/schema/openapi.json b/docs/reference/schema/openapi.json index decc835749..57a051d0ca 100644 --- a/docs/reference/schema/openapi.json +++ b/docs/reference/schema/openapi.json @@ -1781,7 +1781,7 @@ "type": "string" }, "threshold_breach": { - "description": "True when emitted due to a duration-threshold breach rather than the patrol multiple.", + "description": "True when the tick's duration reached the slow-tick threshold (a multiple of the configured patrol interval).", "type": "boolean" } }, diff --git a/docs/reference/schema/openapi.txt b/docs/reference/schema/openapi.txt index decc835749..57a051d0ca 100644 --- a/docs/reference/schema/openapi.txt +++ b/docs/reference/schema/openapi.txt @@ -1781,7 +1781,7 @@ "type": "string" }, "threshold_breach": { - "description": "True when emitted due to a duration-threshold breach rather than the patrol multiple.", + "description": "True when the tick's duration reached the slow-tick threshold (a multiple of the configured patrol interval).", "type": "boolean" } }, diff --git a/internal/api/genclient/client_gen.go b/internal/api/genclient/client_gen.go index deb13ed67d..dec46bbdfc 100644 --- a/internal/api/genclient/client_gen.go +++ b/internal/api/genclient/client_gen.go @@ -1204,7 +1204,7 @@ type ControllerTickCompletedPayload struct { // Phase Tick trigger phase: patrol, poke, control-dispatcher, etc. Phase string `json:"phase"` - // ThresholdBreach True when emitted due to a duration-threshold breach rather than the patrol multiple. + // ThresholdBreach True when the tick's duration reached the slow-tick threshold (a multiple of the configured patrol interval). ThresholdBreach *bool `json:"threshold_breach,omitempty"` } diff --git a/internal/api/openapi.json b/internal/api/openapi.json index decc835749..57a051d0ca 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -1781,7 +1781,7 @@ "type": "string" }, "threshold_breach": { - "description": "True when emitted due to a duration-threshold breach rather than the patrol multiple.", + "description": "True when the tick's duration reached the slow-tick threshold (a multiple of the configured patrol interval).", "type": "boolean" } }, diff --git a/internal/events/events.go b/internal/events/events.go index 6fb7e30669..ddaeea7938 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -312,6 +312,7 @@ var KnownEventTypes = []string{ // yet registered in internal/api (the payload registration lives in a // follow-up that adds the full SSE projection). Until then, subscribers // receive it via the custom-event envelope. + // HostLoadSample is omitted for the same reason (see hostload_payloads.go). } // Event is a single recorded occurrence in the system. diff --git a/internal/events/hostload_payloads.go b/internal/events/hostload_payloads.go new file mode 100644 index 0000000000..b9830f2d06 --- /dev/null +++ b/internal/events/hostload_payloads.go @@ -0,0 +1,36 @@ +package events + +// HostLoadSample is the periodic host-load event type (vp-qvqk defect 3: +// no host-load stream). Emitted by the controller's host-load sampler at +// patrol cadence on its own goroutine, so load excursions stay +// attributable while the reconcile loop itself is wedged — previously +// every excursion cost a manual ps/uptime forensic and left no +// retrospective series. +// +// HostLoadSample is intentionally omitted from KnownEventTypes AND from +// the payload registry (the same deferral as ProviderHealthGateAlert — +// see the KnownEventTypes comment in events.go): RegisterPayload would +// sweep the payload into the generated EventPayload union on the API +// wire, and the typed SSE projection is a follow-up. Until then, +// subscribers receive it via the custom-event envelope; emitters marshal +// HostLoadSamplePayload so the wire shape stays typed at the source. +const HostLoadSample = "host.load_sample" + +// HostLoadSamplePayload is the typed payload for host.load_sample events. +// RunnableProcs and TotalCPUPercent ride alongside the load averages +// because load alone cannot discriminate CPU oversubscription from I/O +// wait: on Darwin the load average also counts uninterruptible/blocked +// threads, so a high load with few runnable processes means +// blocked-on-I/O, not CPU-starved. +type HostLoadSamplePayload struct { + Load1 float64 `json:"load1" doc:"1-minute host load average."` + Load5 float64 `json:"load5" doc:"5-minute host load average."` + Load15 float64 `json:"load15" doc:"15-minute host load average."` + Cores int `json:"cores" doc:"Logical CPU count — the denominator for reading the load averages."` + RunnableProcs int `json:"runnable_procs" doc:"Processes in runnable state (R) at sample time. Discriminates CPU oversubscription (high) from blocked-on-I/O (low) under identical load averages."` + TotalCPUPercent float64 `json:"total_cpu_percent" doc:"Sum of per-process %CPU across the whole process table (100 = one saturated core)."` +} + +// IsEventPayload marks HostLoadSamplePayload as an events.Payload variant +// so the SSE-projection follow-up can register it without reshaping it. +func (HostLoadSamplePayload) IsEventPayload() {} diff --git a/internal/events/storehealth_payloads.go b/internal/events/storehealth_payloads.go index 2c69913d3c..bd83b24553 100644 --- a/internal/events/storehealth_payloads.go +++ b/internal/events/storehealth_payloads.go @@ -50,10 +50,13 @@ const ( // transition (closed/open/half-open), wired from the breaker // registry's state-change callback. BreakerStateChanged = "breaker.state_changed" - // ControllerTickCompleted is the controller heartbeat. It is emitted - // at a patrol multiple or when a tick's duration breaches a threshold - // — never on every tick — so the supervisor doctor can compute tick - // age without the event log itself becoming a hot path. + // ControllerTickCompleted is the controller heartbeat, emitted once + // per completed reconcile tick. The unsampled stream is both the + // supervisor doctor's tick-age/slow-tick signal and a sound basis for + // tick-period arithmetic — the earlier breach-or-every-10th sampling + // made it a biased sample that was complete only by coincidence + // (vp-qvqk). One event per tick is patrol-cadence volume, not a hot + // path. ControllerTickCompleted = "controller.tick_completed" // DoctorAlert fires when the supervisor-cadence doctor evaluates a // cheap check to red. It is the detector that closes the @@ -122,13 +125,14 @@ func (BreakerStateChangedPayload) IsEventPayload() {} // ControllerTickCompletedPayload is the typed payload for // controller.tick_completed events — the controller heartbeat. Duration -// and Phase identify what work the tick did; ThresholdBreach is true when -// the event was emitted because the tick exceeded the duration threshold -// rather than because it landed on the patrol multiple. +// and Phase identify what work the tick did; ThresholdBreach flags a tick +// whose duration reached the slow-tick threshold (a multiple of the +// configured patrol interval — see the controller heartbeat constants), +// which the supervisor doctor's slow_ticks check consumes. type ControllerTickCompletedPayload struct { DurationMs int64 `json:"duration_ms" doc:"Wall-clock duration of the completed tick, in milliseconds."` Phase string `json:"phase" doc:"Tick trigger phase: patrol, poke, control-dispatcher, etc."` - ThresholdBreach bool `json:"threshold_breach,omitempty" doc:"True when emitted due to a duration-threshold breach rather than the patrol multiple."` + ThresholdBreach bool `json:"threshold_breach,omitempty" doc:"True when the tick's duration reached the slow-tick threshold (a multiple of the configured patrol interval)."` } // IsEventPayload marks ControllerTickCompletedPayload as an events.Payload variant. diff --git a/internal/supervisordoctor/doctor.go b/internal/supervisordoctor/doctor.go index 694801120d..6d7e7b0f72 100644 --- a/internal/supervisordoctor/doctor.go +++ b/internal/supervisordoctor/doctor.go @@ -24,6 +24,7 @@ import ( // Check names, used as the doctor.alert "check" field. const ( CheckNameTickAge = "tick_age" + CheckNameSlowTicks = "slow_ticks" CheckNameAgentConfigIsolation = "agent_config_isolation" CheckNameS6ConnectionCeiling = "s6_connection_ceiling" ) @@ -78,6 +79,43 @@ func CheckTickAgeFor(in TickAgeInput) *Alert { } } +// SlowTicksInput holds one city's recent tick-duration facts, gathered +// from the controller.tick_completed events inside the doctor's +// inspection window. +type SlowTicksInput struct { + // City is the city name (alert subject). + City string + // BreachCount is how many inspected ticks carried threshold_breach — + // the controller flags a tick whose duration reached a multiple of + // its own patrol interval. + BreachCount int + // SampleCount is the total tick events inspected in the window. + SampleCount int + // SlowestMs is the largest duration_ms among the inspected events. + SlowestMs int64 + // Window is the trailing inspection window the counts cover. + Window time.Duration +} + +// CheckSlowTicksFor returns an Alert when any inspected tick in the +// window breached the controller's slow-tick threshold. The breach flag +// is computed by the controller against its own patrol interval; this +// check is what makes the flag load-bearing — an emitted-but-never-read +// canary carries no signal (vp-qvqk). Returns nil when nothing was +// sampled or no sampled tick breached. +func CheckSlowTicksFor(in SlowTicksInput) *Alert { + if in.SampleCount <= 0 || in.BreachCount <= 0 { + return nil + } + return &Alert{ + Check: CheckNameSlowTicks, + Subject: in.City, + Detail: fmt.Sprintf("%d of %d controller ticks in the last %s breached the slow-tick threshold (slowest %s); the reconcile loop is degrading", + in.BreachCount, in.SampleCount, in.Window.Round(time.Second), + (time.Duration(in.SlowestMs) * time.Millisecond).Round(time.Millisecond)), + } +} + // S6Input holds the connection-ceiling facts for one city. type S6Input struct { // City is the city name (alert subject). diff --git a/internal/supervisordoctor/doctor_test.go b/internal/supervisordoctor/doctor_test.go index c00caa7fa7..8217d7a8bd 100644 --- a/internal/supervisordoctor/doctor_test.go +++ b/internal/supervisordoctor/doctor_test.go @@ -32,6 +32,36 @@ func TestCheckTickAgeFor(t *testing.T) { } } +func TestCheckSlowTicksFor(t *testing.T) { + tests := []struct { + name string + in SlowTicksInput + wantRed bool + }{ + {"no samples skips", SlowTicksInput{City: "c", BreachCount: 0, SampleCount: 0, Window: 10 * time.Minute}, false}, + {"clean window not red", SlowTicksInput{City: "c", BreachCount: 0, SampleCount: 20, SlowestMs: 4000, Window: 10 * time.Minute}, false}, + {"one breach is red", SlowTicksInput{City: "c", BreachCount: 1, SampleCount: 20, SlowestMs: 443000, Window: 10 * time.Minute}, true}, + {"all breached is red", SlowTicksInput{City: "c", BreachCount: 20, SampleCount: 20, SlowestMs: 55000, Window: 10 * time.Minute}, true}, + // A breach count with zero samples is contradictory gatherer input; + // skip rather than alert on it. + {"breaches without samples skips", SlowTicksInput{City: "c", BreachCount: 3, SampleCount: 0, Window: 10 * time.Minute}, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := CheckSlowTicksFor(tc.in) + if (got != nil) != tc.wantRed { + t.Fatalf("CheckSlowTicksFor red=%v, want %v (alert=%+v)", got != nil, tc.wantRed, got) + } + if got != nil && got.Check != CheckNameSlowTicks { + t.Errorf("alert check = %q, want %q", got.Check, CheckNameSlowTicks) + } + if got != nil && got.Subject != tc.in.City { + t.Errorf("alert subject = %q, want %q", got.Subject, tc.in.City) + } + }) + } +} + func TestCheckS6ConnectionCeiling(t *testing.T) { tests := []struct { name string From 313affdbd6008127c84c223a290ef1630706b060 Mon Sep 17 00:00:00 2001 From: bourgois Date: Fri, 17 Jul 2026 09:39:24 +0000 Subject: [PATCH 2/2] chore(dashboard): regenerate supervisor-client TS for threshold_breach doc-string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tick-heartbeat change updated internal/api/openapi.json (+ Go genclient + schema mirrors) but not the dashboardspa TS client generated from the same spec, tripping Preflight/generated-artifacts. Regenerated via npm run generate:client — 1-line doc-string propagation, no runtime change. --- .../web/shared/src/generated/gc-supervisor-client/types.gen.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/api/dashboardspa/web/shared/src/generated/gc-supervisor-client/types.gen.ts b/internal/api/dashboardspa/web/shared/src/generated/gc-supervisor-client/types.gen.ts index cb8a8af10d..36d3940e7d 100644 --- a/internal/api/dashboardspa/web/shared/src/generated/gc-supervisor-client/types.gen.ts +++ b/internal/api/dashboardspa/web/shared/src/generated/gc-supervisor-client/types.gen.ts @@ -659,7 +659,7 @@ export type ControllerTickCompletedPayload = { */ phase: string; /** - * True when emitted due to a duration-threshold breach rather than the patrol multiple. + * True when the tick's duration reached the slow-tick threshold (a multiple of the configured patrol interval). */ threshold_breach?: boolean; };