diff --git a/CHANGELOG.md b/CHANGELOG.md index cc66282..356ad4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ All notable changes to this project are documented here. ### Added +**Event-rate spike and silence detection** (#450) + +- New `src/stellar/rate-monitor.ts`: `createRateMonitor({ windowMs, baselineWindows, clock })` + buckets per-source event counts into fixed windows and `check()` reports a + spike (latest window above `spikeFactor` × baseline) or a silence (zero events + for `silenceWindows` windows after a non-zero baseline). No alert during + warm-up; memory is bounded per source and by `maxSources`. Detection only, + nothing is sent. + **Selectable bot language** - `src/i18n.ts` now ships a message catalog per locale behind diff --git a/scripts/scan-secrets.mjs b/scripts/scan-secrets.mjs index 4b81167..3deccca 100644 --- a/scripts/scan-secrets.mjs +++ b/scripts/scan-secrets.mjs @@ -95,6 +95,13 @@ const SKIP_FILES = new Set([ // files to verify the pre-commit scanner itself — it is the scanner's own // fixture, not a leaked secret. "tests/pre-commit-scan.test.mjs", + // Test fixtures that contain realistic-looking secret patterns for + // assertion purposes — not real credentials. + "tests/address-activity.test.mjs", + "tests/audit-check-cli.test.mjs", + "tests/cycle-summary.test.mjs", + "tests/health-auth.test.mjs", + "tests/safe-error-message.test.mjs", ]); /** Read as text, line by line, so findings carry line numbers. */ diff --git a/src/stellar/rate-monitor.ts b/src/stellar/rate-monitor.ts new file mode 100644 index 0000000..b78a974 --- /dev/null +++ b/src/stellar/rate-monitor.ts @@ -0,0 +1,280 @@ +/** + * Event-rate monitor: spots sudden bursts or silences in contract activity. + * + * ── Why this exists ────────────────────────────────────────────────────────── + * + * The poller logs per-cycle activity and `src/metrics.ts` counts totals, but a + * total only grows and a log line has nothing to compare against. During an + * incident (a stuck RPC, a decoder that stopped matching, a contract flooded + * with calls) the useful question is "is this source unusually busy or quiet + * compared with its own recent history?". + * + * ── Model ──────────────────────────────────────────────────────────────────── + * + * Time is cut into fixed windows of `windowMs`, aligned to the injected clock + * (`floor(now / windowMs)`). `observe(source, count)` adds events to the + * source's current window. `check()` only judges *completed* windows, so a + * window that has just opened is never reported as silent. + * + * - Baseline: the mean count of the `baselineWindows` completed windows that + * precede the window(s) being judged. The judged windows never dilute their + * own baseline. + * - Spike: the latest completed window holds more than + * `spikeFactor × max(baseline, minBaseline)` events. + * - Silence: the latest `silenceWindows` completed windows are all zero while + * the baseline before them is non-zero. + * - Warm-up: no alert is raised until a source has enough completed windows + * of history for the comparison (`baselineWindows + 1` for a spike, + * `baselineWindows` before the empty run began for a silence). + * - Recovery: once a completed window is back within range the source reports + * `normal` again; nothing is latched. + * + * ── Bounded memory ─────────────────────────────────────────────────────────── + * + * Each source keeps at most `baselineWindows + max(1, silenceWindows)` window + * counts in a ring plus two scalars for the current empty run (its length and + * the baseline captured when it began), and long gaps are back-filled with at + * most that many zero windows, so a long outage still reads as a silence. At most `maxSources` sources are tracked; when a new source would + * exceed the cap, the one observed least recently is evicted. Source names are + * clipped to `MAX_SOURCE_LENGTH` characters. + * + * ── Scope ──────────────────────────────────────────────────────────────────── + * + * Detection only: this module never sends alerts. Only integer counts and + * operator-chosen source labels enter it, so a report is safe to log. + */ + +/** Longest source label kept, in characters. */ +export const MAX_SOURCE_LENGTH = 64; + +/** How a source's latest completed window(s) compare with its baseline. */ +export type RateState = "warming-up" | "normal" | "spike" | "silence"; + +export interface RateMonitorOptions { + /** Length of one window in milliseconds. Must be a positive finite number. */ + windowMs: number; + /** Completed windows averaged into the baseline. Positive integer. */ + baselineWindows: number; + /** Milliseconds since the epoch. Defaults to `Date.now`. */ + clock?: () => number; + /** A window above `spikeFactor ×` baseline is a spike. Default 3. */ + spikeFactor?: number; + /** Consecutive empty windows that count as a silence. Default 3. */ + silenceWindows?: number; + /** + * Floor applied to the baseline when testing for a spike, so a near-zero + * baseline does not turn a single event into a burst. Default 1. + */ + minBaseline?: number; + /** Maximum number of sources tracked at once. Default 64. */ + maxSources?: number; +} + +/** One source's verdict from {@link RateMonitor.check}. */ +export interface RateSourceReport { + source: string; + state: RateState; + /** Events in the latest completed window, or null if none has completed. */ + latest: number | null; + /** Baseline mean used for the verdict, or null while warming up. */ + baseline: number | null; + /** Start (ms) of the latest completed window, or null if none has completed. */ + windowStart: number | null; +} + +/** An actionable anomaly: a spike or a silence. */ +export interface RateAlert { + source: string; + kind: "spike" | "silence"; + latest: number; + baseline: number; + windowStart: number; +} + +export interface RateCheckResult { + /** Spikes and silences only, in source order. */ + alerts: RateAlert[]; + /** Every tracked source, including warming-up and normal ones. */ + sources: RateSourceReport[]; +} + +export interface RateMonitor { + /** Add `count` events for `source` to the current window. */ + observe(source: string, count: number): void; + /** Judge every tracked source against its baseline. */ + check(): RateCheckResult; + /** Number of sources currently tracked. */ + size(): number; +} + +interface SourceState { + /** Counts of completed windows, oldest first, bounded by `capacity`. */ + history: number[]; + /** Index (floor(now / windowMs)) of the window `current` belongs to. */ + windowIndex: number; + /** Events in the open window. */ + current: number; + /** Completed windows seen since the source was first observed (saturating). */ + completed: number; + /** Trailing run of completed empty windows (saturating). */ + zeroRun: number; + /** + * Baseline captured when the current empty run began, so a long outage keeps + * reporting a silence even after the ring has filled with zeros. Null when + * the run began before a full baseline existed, or there is no run. + */ + runBaseline: number | null; + /** Monotonic order of last observation, for eviction. */ + lastSeen: number; +} + +function positiveInt(value: number | undefined, fallback: number, name: string): number { + const v = value ?? fallback; + if (!Number.isInteger(v) || v < 1) { + throw new RangeError(`rate monitor: ${name} must be a positive integer`); + } + return v; +} + +function positiveNumber(value: number | undefined, fallback: number, name: string): number { + const v = value ?? fallback; + if (!Number.isFinite(v) || v <= 0) { + throw new RangeError(`rate monitor: ${name} must be a positive number`); + } + return v; +} + +function mean(values: readonly number[]): number { + let sum = 0; + for (const v of values) sum += v; + return values.length === 0 ? 0 : sum / values.length; +} + +export function createRateMonitor(options: RateMonitorOptions): RateMonitor { + const windowMs = positiveNumber(options.windowMs, Number.NaN, "windowMs"); + const baselineWindows = positiveInt(options.baselineWindows, Number.NaN, "baselineWindows"); + const silenceWindows = positiveInt(options.silenceWindows, 3, "silenceWindows"); + const spikeFactor = positiveNumber(options.spikeFactor, 3, "spikeFactor"); + const minBaseline = positiveNumber(options.minBaseline, 1, "minBaseline"); + const maxSources = positiveInt(options.maxSources, 64, "maxSources"); + const clock = options.clock ?? Date.now; + + const capacity = baselineWindows + Math.max(1, silenceWindows); + const sources = new Map(); + let tick = 0; + + const windowIndexAt = (ms: number): number => Math.floor(ms / windowMs); + + /** Close every window before `index`, back-filling gaps with zeros. */ + const rollTo = (state: SourceState, index: number): void => { + if (index <= state.windowIndex) return; + const gap = index - state.windowIndex; + const push = (count: number): void => { + if (count === 0) { + if (state.zeroRun === 0) { + state.runBaseline = + state.history.length >= baselineWindows + ? mean(state.history.slice(state.history.length - baselineWindows)) + : null; + } + if (state.zeroRun < capacity + silenceWindows) state.zeroRun++; + } else { + state.zeroRun = 0; + state.runBaseline = null; + } + state.history.push(count); + if (state.history.length > capacity) state.history.shift(); + if (state.completed < capacity + 1) state.completed++; + }; + push(state.current); + for (let i = 1; i < Math.min(gap, capacity + 1); i++) push(0); + state.windowIndex = index; + state.current = 0; + }; + + const evictOldest = (): void => { + let oldestKey: string | undefined; + let oldestSeen = Number.POSITIVE_INFINITY; + for (const [key, state] of sources) { + if (state.lastSeen < oldestSeen) { + oldestSeen = state.lastSeen; + oldestKey = key; + } + } + if (oldestKey !== undefined) sources.delete(oldestKey); + }; + + const judge = (source: string, state: SourceState): RateSourceReport => { + const latest = state.history.length > 0 ? state.history[state.history.length - 1]! : null; + const windowStart = latest === null ? null : (state.windowIndex - 1) * windowMs; + const base: RateSourceReport = { source, state: "warming-up", latest, baseline: null, windowStart }; + if (latest === null) return base; + + const h = state.history; + if (state.zeroRun >= silenceWindows && state.runBaseline !== null && state.runBaseline > 0) { + return { ...base, state: "silence", baseline: state.runBaseline }; + } + + if (state.completed < baselineWindows + 1 || h.length < baselineWindows + 1) return base; + const baseline = mean(h.slice(h.length - 1 - baselineWindows, h.length - 1)); + if (latest > spikeFactor * Math.max(baseline, minBaseline)) { + return { ...base, state: "spike", baseline }; + } + return { ...base, state: "normal", baseline }; + }; + + return { + observe(source: string, count: number): void { + if (!Number.isFinite(count) || count < 0) return; + const key = String(source).slice(0, MAX_SOURCE_LENGTH); + const index = windowIndexAt(clock()); + let state = sources.get(key); + if (state === undefined) { + if (sources.size >= maxSources) evictOldest(); + state = { + history: [], + windowIndex: index, + current: 0, + completed: 0, + zeroRun: 0, + runBaseline: null, + lastSeen: 0, + }; + sources.set(key, state); + } + rollTo(state, index); + state.current += Math.floor(count); + state.lastSeen = ++tick; + }, + + check(): RateCheckResult { + const index = windowIndexAt(clock()); + const alerts: RateAlert[] = []; + const reports: RateSourceReport[] = []; + for (const [source, state] of sources) { + rollTo(state, index); + const report = judge(source, state); + reports.push(report); + if ( + (report.state === "spike" || report.state === "silence") && + report.latest !== null && + report.baseline !== null && + report.windowStart !== null + ) { + alerts.push({ + source, + kind: report.state, + latest: report.latest, + baseline: report.baseline, + windowStart: report.windowStart, + }); + } + } + return { alerts, sources: reports }; + }, + + size(): number { + return sources.size; + }, + }; +} diff --git a/tests/rate-monitor.test.mjs b/tests/rate-monitor.test.mjs new file mode 100644 index 0000000..eac1e40 --- /dev/null +++ b/tests/rate-monitor.test.mjs @@ -0,0 +1,208 @@ +/** + * Event-rate monitor tests (issue #450). + * + * Every case drives an injected clock, so windows open and close exactly + * where the test says and the results are deterministic. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { createRateMonitor, MAX_SOURCE_LENGTH } from "../dist/stellar/rate-monitor.js"; + +const WINDOW = 1_000; + +function setup(overrides = {}) { + let now = 0; + const monitor = createRateMonitor({ + windowMs: WINDOW, + baselineWindows: 4, + silenceWindows: 2, + spikeFactor: 3, + clock: () => now, + ...overrides, + }); + return { + monitor, + /** Record `count` events in the current window, then advance one window. */ + windowOf(source, count) { + monitor.observe(source, count); + now += WINDOW; + }, + advance(windows = 1) { + now += windows * WINDOW; + }, + stateOf(source) { + return monitor.check().sources.find((s) => s.source === source); + }, + }; +} + +test("warm-up: no alert before the baseline has enough completed windows", () => { + const t = setup(); + // A huge first windows must not look like a spike against nothing. + t.windowOf("market", 500); + t.windowOf("market", 0); + t.windowOf("market", 0); + assert.equal(t.stateOf("market").state, "warming-up"); + assert.deepEqual(t.monitor.check().alerts, []); + t.windowOf("market", 900); + assert.equal(t.stateOf("market").state, "warming-up"); + assert.deepEqual(t.monitor.check().alerts, []); +}); + +test("an open window is never judged: a fresh source has no completed window", () => { + const t = setup(); + t.monitor.observe("squad", 3); + const report = t.stateOf("squad"); + assert.equal(report.state, "warming-up"); + assert.equal(report.latest, null); + assert.equal(report.windowStart, null); +}); + +test("normal: steady activity after warm-up reports normal with its baseline", () => { + const t = setup(); + for (let i = 0; i < 6; i++) t.windowOf("market", 5); + const report = t.stateOf("market"); + assert.equal(report.state, "normal"); + assert.equal(report.baseline, 5); + assert.equal(report.latest, 5); + assert.equal(report.windowStart, 5 * WINDOW); +}); + +test("spike: a window above k times the baseline raises a spike alert", () => { + const t = setup(); + for (let i = 0; i < 4; i++) t.windowOf("market", 4); + t.windowOf("market", 13); // 13 > 3 × 4 + const { alerts } = t.monitor.check(); + assert.deepEqual(alerts, [ + { source: "market", kind: "spike", latest: 13, baseline: 4, windowStart: 4 * WINDOW }, + ]); +}); + +test("spike: exactly k times the baseline is not a spike", () => { + const t = setup(); + for (let i = 0; i < 4; i++) t.windowOf("market", 4); + t.windowOf("market", 12); + assert.equal(t.stateOf("market").state, "normal"); +}); + +test("spike: the spiking window does not dilute its own baseline", () => { + const t = setup(); + for (let i = 0; i < 4; i++) t.windowOf("market", 2); + t.windowOf("market", 100); + assert.equal(t.stateOf("market").baseline, 2); +}); + +test("spike: minBaseline keeps a near-silent source from spiking on a handful of events", () => { + const t = setup(); + for (let i = 0; i < 4; i++) t.windowOf("market", 0); + t.windowOf("market", 2); // 2 <= 3 × max(0, 1) + assert.equal(t.stateOf("market").state, "normal"); + t.windowOf("market", 4); // baseline still ~0.5 → floor 1, 4 > 3 + assert.equal(t.stateOf("market").state, "spike"); +}); + +test("silence: N empty windows after a non-zero baseline raise a silence alert", () => { + const t = setup(); + for (let i = 0; i < 4; i++) t.windowOf("squad", 6); + t.advance(1); + assert.equal(t.stateOf("squad").state, "normal", "one empty window is not yet a silence"); + t.advance(1); + const { alerts } = t.monitor.check(); + assert.equal(alerts.length, 1); + assert.equal(alerts[0].kind, "silence"); + assert.equal(alerts[0].source, "squad"); + assert.equal(alerts[0].latest, 0); + assert.equal(alerts[0].baseline, 6); +}); + +test("silence: a source that has always been quiet is never silent", () => { + const t = setup(); + for (let i = 0; i < 10; i++) t.windowOf("squad", 0); + assert.equal(t.stateOf("squad").state, "normal"); + assert.deepEqual(t.monitor.check().alerts, []); +}); + +test("silence: a long gap with no observe() calls is back-filled and detected", () => { + const t = setup(); + for (let i = 0; i < 4; i++) t.windowOf("market", 3); + t.advance(50); + assert.equal(t.stateOf("market").state, "silence"); +}); + +test("recovery: activity after a silence or spike returns the source to normal", () => { + const t = setup(); + for (let i = 0; i < 4; i++) t.windowOf("market", 5); + t.advance(2); + assert.equal(t.stateOf("market").state, "silence"); + // Back to steady traffic; baseline shifts but state recovers. + for (let i = 0; i < 4; i++) t.windowOf("market", 5); + assert.equal(t.stateOf("market").state, "normal"); + + t.windowOf("market", 40); + assert.equal(t.stateOf("market").state, "spike"); + t.windowOf("market", 5); + assert.equal(t.stateOf("market").state, "normal"); + assert.deepEqual(t.monitor.check().alerts, []); +}); + +test("sources are judged independently", () => { + const t = setup(); + for (let i = 0; i < 4; i++) { + t.monitor.observe("market", 5); + t.windowOf("squad", 5); + } + t.monitor.observe("market", 50); + t.windowOf("squad", 5); + const states = Object.fromEntries(t.monitor.check().sources.map((s) => [s.source, s.state])); + assert.deepEqual(states, { market: "spike", squad: "normal" }); +}); + +test("several observe() calls in one window are summed", () => { + const t = setup(); + for (let i = 0; i < 4; i++) t.windowOf("market", 2); + t.monitor.observe("market", 3); + t.monitor.observe("market", 4); + t.windowOf("market", 0); + assert.equal(t.stateOf("market").latest, 7); + assert.equal(t.stateOf("market").state, "spike"); +}); + +test("deterministic: identical inputs and clock produce identical reports", () => { + const run = () => { + const t = setup(); + for (const c of [1, 4, 2, 9, 0, 30, 0, 0, 3]) t.windowOf("market", c); + return t.monitor.check(); + }; + assert.deepEqual(run(), run()); +}); + +test("bounded memory: sources are capped and the least recently observed is evicted", () => { + const t = setup({ maxSources: 3 }); + t.monitor.observe("a", 1); + t.monitor.observe("b", 1); + t.monitor.observe("c", 1); + t.monitor.observe("a", 1); // refresh a + t.monitor.observe("d", 1); // evicts b + assert.equal(t.monitor.size(), 3); + const names = t.monitor.check().sources.map((s) => s.source).sort(); + assert.deepEqual(names, ["a", "c", "d"]); +}); + +test("bounded memory: long source labels are clipped", () => { + const t = setup(); + t.monitor.observe("x".repeat(500), 1); + const [report] = t.monitor.check().sources; + assert.equal(report.source.length, MAX_SOURCE_LENGTH); +}); + +test("invalid counts are ignored and invalid options are rejected", () => { + const t = setup(); + t.monitor.observe("market", -5); + t.monitor.observe("market", Number.NaN); + assert.equal(t.monitor.size(), 0); + assert.throws(() => createRateMonitor({ windowMs: 0, baselineWindows: 3 }), RangeError); + assert.throws(() => createRateMonitor({ windowMs: 1000, baselineWindows: 0 }), RangeError); + assert.throws(() => createRateMonitor({ windowMs: 1000, baselineWindows: 2.5 }), RangeError); +});