diff --git a/apps/kortix-sandbox-agent-server/README.md b/apps/kortix-sandbox-agent-server/README.md index 015dd60d808..8c5c944ce5e 100644 --- a/apps/kortix-sandbox-agent-server/README.md +++ b/apps/kortix-sandbox-agent-server/README.md @@ -185,8 +185,21 @@ KORTIX_REPO_URL= KORTIX_BRANCH_NAME= KORTIX_GITHUB_TOKEN= KORTIX_TOKEN= +KORTIX_OPENCODE_WEDGE_KILL=1 # 0|false: never SIGKILL an HTTP-unresponsive opencode (gate-only, pre-2026-09 behaviour) +KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS=60000 # continuous unresponsiveness before the wedge kill; 5 min cooldown between kills ``` +### Wedged OpenCode recovery + +The readiness loop probes the live opencode every 5 s. After 3 failed probes +the daemon marks it `starting`, and the proxy answers 503. When BOTH the +`/session` probe and the Instance-independent `/kortix-liveness-probe` fail +for 60 s (at least 3 probes), the daemon SIGKILLs the opencode process group. +The ordinary exit path then respawns it and finalizes the orphaned turn. The +daemon logs `[opencode] wedge-kill` with `unresponsiveMs`, `failures`, and +`kills`. `GET /kortix/diag` reports `opencode.wedge_kill` +(`kills`, `lastKillAt`, `unresponsiveMs`, `enabled`). + ## Build ``` diff --git a/apps/kortix-sandbox-agent-server/src/__tests__/opencode-wedge-kill-process.e2e.test.ts b/apps/kortix-sandbox-agent-server/src/__tests__/opencode-wedge-kill-process.e2e.test.ts new file mode 100644 index 00000000000..3dd13757622 --- /dev/null +++ b/apps/kortix-sandbox-agent-server/src/__tests__/opencode-wedge-kill-process.e2e.test.ts @@ -0,0 +1,160 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test' +import { chmodSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { OpenCodeConfig as Config } from '../harness/open-code/config' +import { createOpencodeLifecycle } from '../harness/open-code/lifecycle' + +// A wedged opencode: the process is alive and its port stays bound, but no +// request gets an answer. SIGSTOP produces exactly that — the kernel still +// accepts connections into the listen backlog, nothing reads them. The +// lifecycle must SIGKILL the child, respawn it through the ordinary exit +// path, and run onUnplannedRespawn so the orphaned turn is finalized. + +let root: string +let lifecycle: ReturnType | null +const stoppedPids: number[] = [] + +function reservePort(): number { + const server = Bun.serve({ port: 0, fetch: () => new Response('reserved') }) + const port = server.port + server.stop(true) + if (typeof port !== 'number') throw new Error('Bun did not assign a port') + return port +} + +function writeHealthyBinary(path: string) { + writeFileSync( + path, + `#!/usr/bin/env bun +const port = Number(Bun.argv[Bun.argv.indexOf('--port') + 1]) +Bun.serve({ + hostname: '127.0.0.1', + port, + fetch(req) { + const url = new URL(req.url) + if (url.pathname === '/session') return Response.json([]) + return new Response('not found', { status: 404 }) + }, +}) +console.log('opencode server listening on http://127.0.0.1:' + port) +setInterval(() => {}, 60_000) +`, + ) + chmodSync(path, 0o755) +} + +function alive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + +async function until(check: () => boolean, timeoutMs: number, label: string): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (check()) return + await new Promise((r) => setTimeout(r, 50)) + } + throw new Error(`timed out waiting for ${label}`) +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'kortix-wedge-kill-')) + lifecycle = null +}) + +afterEach(async () => { + for (const pid of stoppedPids.splice(0)) { + try { + process.kill(pid, 'SIGKILL') + } catch {} + } + await lifecycle?.stop() + rmSync(root, { recursive: true, force: true }) +}) + +function makeCfg(): Config { + const workspace = join(root, 'workspace') + mkdirSync(workspace) + return { + workspace, + projectTarget: workspace, + opencodeInternalPort: reservePort(), + opencodeStandbyPort: reservePort(), + gitUserName: 'Kortix Agent', + gitUserEmail: 'agent@kortix.ai', + } as Config +} + +function startLifecycle(wedgeKillEnabled: boolean, onRespawn: () => void) { + const configDir = join(root, 'config') + const binary = join(root, 'opencode') + mkdirSync(configDir) + writeHealthyBinary(binary) + return createOpencodeLifecycle(makeCfg(), configDir, undefined, { + binaryPathOverride: binary, + configPathOverride: join(root, 'runtime-config.json'), + readyLivenessMs: 200, + wedgeKill: { enabled: wedgeKillEnabled, thresholdMs: 1_000, minFailures: 2, probeTimeoutMs: 500 }, + onUnplannedRespawn: onRespawn, + }) +} + +describe('OpenCode wedge kill (real child process)', () => { + test('a SIGSTOPped child is SIGKILLed, respawned, and its turn finalized', async () => { + let respawnHookCalls = 0 + lifecycle = startLifecycle(true, () => { + respawnHookCalls += 1 + }) + await lifecycle.start() + await until(() => lifecycle!.getState() === 'ok', 10_000, 'first ready') + const wedgedPid = lifecycle.getPid()! + expect(wedgedPid).toBeGreaterThan(0) + + process.kill(wedgedPid, 'SIGSTOP') + stoppedPids.push(wedgedPid) + + await until(() => !alive(wedgedPid), 20_000, 'wedged child killed') + await until(() => respawnHookCalls === 1, 10_000, 'onUnplannedRespawn after respawn') + + // The hook fires once the replacement answers; the readiness loop flips + // the state on its next tick. + await until(() => lifecycle!.getState() === 'ok', 5_000, 'replacement ready') + const replacementPid = lifecycle.getPid() + expect(replacementPid).not.toBeNull() + expect(replacementPid).not.toBe(wedgedPid) + expect(alive(replacementPid!)).toBe(true) + const stats = lifecycle.getWedgeKillStats!() + expect(stats.kills).toBe(1) + expect(stats.lastKillAt).not.toBeNull() + expect(stats.enabled).toBe(true) + }, 40_000) + + test('kill switch off: the SIGSTOPped child is gated off but left alive', async () => { + let respawnHookCalls = 0 + lifecycle = startLifecycle(false, () => { + respawnHookCalls += 1 + }) + await lifecycle.start() + await until(() => lifecycle!.getState() === 'ok', 10_000, 'first ready') + const wedgedPid = lifecycle.getPid()! + + process.kill(wedgedPid, 'SIGSTOP') + stoppedPids.push(wedgedPid) + + // Long enough for the enabled case above to have killed it (~5-7 s). + await new Promise((r) => setTimeout(r, 9_000)) + expect(alive(wedgedPid)).toBe(true) + expect(lifecycle.getPid()).toBe(wedgedPid) + expect(respawnHookCalls).toBe(0) + const stats = lifecycle.getWedgeKillStats!() + expect(stats.kills).toBe(0) + expect(stats.enabled).toBe(false) + expect(stats.unresponsiveMs).toBeGreaterThan(1_000) + }, 40_000) +}) diff --git a/apps/kortix-sandbox-agent-server/src/__tests__/opencode-wedge-kill.test.ts b/apps/kortix-sandbox-agent-server/src/__tests__/opencode-wedge-kill.test.ts new file mode 100644 index 00000000000..bf322d2e107 --- /dev/null +++ b/apps/kortix-sandbox-agent-server/src/__tests__/opencode-wedge-kill.test.ts @@ -0,0 +1,161 @@ +/** + * Wedge kill: an opencode whose process is alive but whose HTTP server no + * longer answers must be replaced, not only gated off. + * + * The liveness hysteresis (opencode-liveness-hysteresis.test.ts) downgrades + * such a child to `starting` and proxy.ts 503s every request, but respawn ran + * only on process exit, so the session stayed wedged until a manual refresh. + * `nextWedgeAction` decides when the readiness loop SIGKILLs the child; the + * exit handler then respawns it. + */ +import { describe, expect, test } from 'bun:test' +import { + EMPTY_WEDGE_WATCH, + nextWedgeAction, + wedgeKillAfterMs, + wedgeKillEnabled, + type WedgeDecisionInput, + type WedgeWatch, +} from '../harness/open-code/lifecycle' + +const T0 = 1_000_000 +const BASE = { + lastKillAt: null, + enabled: true, + thresholdMs: 60_000, + cooldownMs: 300_000, + minFailures: 3, + maxProbeGapMs: 30_000, +} satisfies Omit + +/** Feed a run of fully-unresponsive probes `stepMs` apart; return every decision. */ +function run(times: number[], overrides: Partial = {}, watch: WedgeWatch = EMPTY_WEDGE_WATCH) { + const out = [] + let w = watch + for (const now of times) { + const d = nextWedgeAction({ ...BASE, ...overrides, watch: w, probeFailed: true, livenessProbeFailed: true, now }) + w = d.watch + out.push(d) + } + return out +} + +/** Probe times every `stepMs` from T0 through T0 + totalMs inclusive. */ +function every(stepMs: number, totalMs: number): number[] { + const times = [] + for (let t = 0; t <= totalMs; t += stepMs) times.push(T0 + t) + return times +} + +describe('nextWedgeAction', () => { + test('no kill before the threshold, even with many failures', () => { + const decisions = run(every(4_000, 56_000)) + expect(decisions.every((d) => !d.kill)).toBe(true) + expect(decisions.at(-1)).toMatchObject({ reason: 'below-threshold', unresponsiveMs: 56_000 }) + expect(decisions.at(-1)?.watch.failures).toBe(15) + }) + + test('kills at the threshold and resets the watch', () => { + const decisions = run(every(4_000, 60_000)) + expect(decisions.slice(0, -1).every((d) => !d.kill)).toBe(true) + expect(decisions.at(-1)).toEqual({ + kill: true, + reason: 'wedged', + unresponsiveMs: 60_000, + watch: { unresponsiveSince: null, failures: 0, lastObservedAt: T0 + 60_000 }, + }) + }) + + test('the threshold alone is not enough: minFailures must also be met', () => { + // Two probes 60 s apart would pass the time test with only 2 failures, + // but a 60 s gap is itself a clock jump; use a wide gap limit to isolate + // the failure-count rule. + const decisions = run([T0, T0 + 60_000], { maxProbeGapMs: 120_000 }) + expect(decisions.at(-1)).toMatchObject({ kill: false, reason: 'below-threshold', unresponsiveMs: 60_000 }) + const third = run([T0, T0 + 60_000, T0 + 61_000], { maxProbeGapMs: 120_000 }) + expect(third.at(-1)).toMatchObject({ kill: true, reason: 'wedged' }) + }) + + test('respects the cooldown after a kill, then kills again once it passes', () => { + const lastKillAt = T0 - 60_000 // 60 s ago; cooldown is 300 s + const inCooldown = run(every(4_000, 60_000), { lastKillAt }) + expect(inCooldown.some((d) => d.kill)).toBe(false) + expect(inCooldown.at(-1)).toMatchObject({ reason: 'cooldown', unresponsiveMs: 60_000 }) + + // Still wedged when the cooldown ends (T0 + 240 s): the run was kept, so + // the next probe kills at once. + let w = inCooldown.at(-1)!.watch + let killedAt: number | null = null + for (let now = T0 + 64_000; now <= T0 + 260_000; now += 4_000) { + const d = nextWedgeAction({ ...BASE, lastKillAt, watch: w, probeFailed: true, livenessProbeFailed: true, now }) + w = d.watch + if (d.kill) { + killedAt = now + break + } + } + expect(killedAt).toBe(T0 + 240_000) + }) + + test('any answer resets the run (recovery)', () => { + const almost = run(every(4_000, 56_000)) + const recovered = nextWedgeAction({ + ...BASE, + watch: almost.at(-1)!.watch, + probeFailed: false, + livenessProbeFailed: false, + now: T0 + 58_000, + }) + expect(recovered).toEqual({ + kill: false, + reason: 'responsive', + unresponsiveMs: 0, + watch: { unresponsiveSince: null, failures: 0, lastObservedAt: T0 + 58_000 }, + }) + // A fresh run needs the full threshold again. + const after = run(every(4_000, 56_000).map((t) => t + 60_000), {}, recovered.watch) + expect(after.some((d) => d.kill)).toBe(false) + }) + + test('a slow /session with a live HTTP server is busy, not wedged', () => { + let w: WedgeWatch = EMPTY_WEDGE_WATCH + for (const now of every(4_000, 120_000)) { + const d = nextWedgeAction({ ...BASE, watch: w, probeFailed: true, livenessProbeFailed: false, now }) + expect(d).toMatchObject({ kill: false, reason: 'responsive' }) + w = d.watch + } + }) + + test('kill switch off: never kills, but still measures the run', () => { + const decisions = run(every(4_000, 120_000), { enabled: false }) + expect(decisions.some((d) => d.kill)).toBe(false) + expect(decisions.at(-1)).toMatchObject({ reason: 'disabled', unresponsiveMs: 120_000 }) + }) + + test('a wall-clock jump (snapshot restore) restarts the run instead of killing', () => { + const before = run([T0, T0 + 4_000]) + // The VM was frozen for an hour; the first probe after resume fails. + const resumed = run([T0 + 3_600_000, T0 + 3_604_000, T0 + 3_608_000], {}, before.at(-1)!.watch) + expect(resumed.some((d) => d.kill)).toBe(false) + expect(resumed.at(-1)).toMatchObject({ reason: 'below-threshold', unresponsiveMs: 8_000 }) + expect(resumed.at(-1)?.watch.failures).toBe(3) + }) +}) + +describe('wedge kill env', () => { + test('KORTIX_OPENCODE_WEDGE_KILL defaults on; 0/false disable it', () => { + expect(wedgeKillEnabled({})).toBe(true) + expect(wedgeKillEnabled({ KORTIX_OPENCODE_WEDGE_KILL: '1' })).toBe(true) + expect(wedgeKillEnabled({ KORTIX_OPENCODE_WEDGE_KILL: 'true' })).toBe(true) + expect(wedgeKillEnabled({ KORTIX_OPENCODE_WEDGE_KILL: '0' })).toBe(false) + expect(wedgeKillEnabled({ KORTIX_OPENCODE_WEDGE_KILL: ' FALSE ' })).toBe(false) + }) + + test('KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS overrides the 60 s threshold; junk falls back', () => { + expect(wedgeKillAfterMs({})).toBe(60_000) + expect(wedgeKillAfterMs({ KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS: '120000' })).toBe(120_000) + expect(wedgeKillAfterMs({ KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS: 'abc' })).toBe(60_000) + expect(wedgeKillAfterMs({ KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS: '0' })).toBe(60_000) + expect(wedgeKillAfterMs({ KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS: '-5' })).toBe(60_000) + }) +}) diff --git a/apps/kortix-sandbox-agent-server/src/__tests__/runtime-env-allowlist-completeness.test.ts b/apps/kortix-sandbox-agent-server/src/__tests__/runtime-env-allowlist-completeness.test.ts index 087d97afeaf..6d7fb9449e7 100644 --- a/apps/kortix-sandbox-agent-server/src/__tests__/runtime-env-allowlist-completeness.test.ts +++ b/apps/kortix-sandbox-agent-server/src/__tests__/runtime-env-allowlist-completeness.test.ts @@ -88,6 +88,10 @@ const BOOT_ONLY_KORTIX_ENV_NAMES = new Set([ // Manual operator debug toggle (checked against `process.env` directly, not // part of the env-sync contract at all). 'KORTIX_OPENCODE_DEBUG', + // Daemon wedge-kill kill switch and threshold (nextWedgeAction). Operator + // tunables for the daemon's own supervision loop; opencode never reads them. + 'KORTIX_OPENCODE_WEDGE_KILL', + 'KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS', // Artifact identity is fixed before PID 1 starts. A live update installs a // new artifact and daemon instead of changing this value through /env. 'KORTIX_COMPILED_RUNTIME_FORMAT', @@ -153,6 +157,8 @@ describe('OPENCODE_RUNTIME_ENV_NAMES — allowlist completeness', () => { 'KORTIX_LLM_PROXY_URL', 'KORTIX_OPENCODE_DEBUG', 'KORTIX_OPENCODE_MODEL', + 'KORTIX_OPENCODE_WEDGE_KILL', + 'KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS', 'KORTIX_PROJECT_ID', 'KORTIX_SECRET_CAPABILITIES', 'KORTIX_TOKEN', diff --git a/apps/kortix-sandbox-agent-server/src/harness/open-code/diagnostics.ts b/apps/kortix-sandbox-agent-server/src/harness/open-code/diagnostics.ts index 2d50f7af4f2..63f4563685d 100644 --- a/apps/kortix-sandbox-agent-server/src/harness/open-code/diagnostics.ts +++ b/apps/kortix-sandbox-agent-server/src/harness/open-code/diagnostics.ts @@ -308,6 +308,9 @@ async function readOpenCodeDiagnosticReport( port_pair: [cfg.opencodeInternalPort, cfg.opencodeStandbyPort], session_id: bootState.initialOpenCodeSessionId ?? null, log_file: opencodeLog, + // Daemon-initiated SIGKILLs of an HTTP-unresponsive opencode (see + // nextWedgeAction). Non-zero means this box recovered from a wedge. + wedge_kill: opencode.getWedgeKillStats?.() ?? null, }, boot: { repo_materialization_error: bootState.repoMaterializationError, diff --git a/apps/kortix-sandbox-agent-server/src/harness/open-code/lifecycle.ts b/apps/kortix-sandbox-agent-server/src/harness/open-code/lifecycle.ts index 06682972151..089978b1f34 100644 --- a/apps/kortix-sandbox-agent-server/src/harness/open-code/lifecycle.ts +++ b/apps/kortix-sandbox-agent-server/src/harness/open-code/lifecycle.ts @@ -96,6 +96,37 @@ const READY_LIVENESS_MS = 5_000 // both recovery and a genuine wedge are detected quickly. const READY_LIVENESS_DOWNGRADE_THRESHOLD = 3 const READY_LIVENESS_RECHECK_MS = 2_000 +// Wedge kill. The downgrade above only GATES a wedged opencode off (proxy.ts +// 503s); it never replaces it, and respawn runs only on process exit. So an +// opencode whose process is alive but whose HTTP server no longer answers +// (event loop stuck, SIGSTOP, deadlock) stayed wedged until a manual +// /kortix/refresh or a box stop. The readiness loop now SIGKILLs such a child +// once BOTH the session probe and the Instance-independent liveness probe have +// failed continuously for WEDGE_KILL_AFTER_MS; the ordinary exit handler then +// respawns it and finalizes the orphaned turn. See nextWedgeAction. +const WEDGE_KILL_AFTER_MS = 60_000 +const WEDGE_KILL_COOLDOWN_MS = 5 * 60_000 +const WEDGE_KILL_MIN_FAILURES = READY_LIVENESS_DOWNGRADE_THRESHOLD +// A gap this long between two probe results means the daemon itself was not +// running (warm-seed snapshot/restore, VM pause): the wall clock jumped, and +// that time is not evidence opencode was unresponsive. +const WEDGE_MAX_PROBE_GAP_MS = 30_000 + +/** + * Kill switch: KORTIX_OPENCODE_WEDGE_KILL=0 (or `false`) disables the wedge + * kill and restores the old gate-only behaviour. Read on every probe, so a + * changed env takes effect without a daemon restart. + */ +export function wedgeKillEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const raw = (env.KORTIX_OPENCODE_WEDGE_KILL ?? '1').trim().toLowerCase() + return raw !== '0' && raw !== 'false' +} + +/** KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS overrides WEDGE_KILL_AFTER_MS. */ +export function wedgeKillAfterMs(env: NodeJS.ProcessEnv = process.env): number { + const parsed = Number.parseInt((env.KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS ?? '').trim(), 10) + return Number.isFinite(parsed) && parsed > 0 ? parsed : WEDGE_KILL_AFTER_MS +} const OPENCODE_DATA_HOME = `${OPENCODE_HOME}/.local/share` const OPENCODE_AUTH_PATH = `${OPENCODE_DATA_HOME}/opencode/auth.json` @@ -1563,6 +1594,98 @@ export function nextLivenessState(input: LivenessDecisionInput): LivenessDecisio return { state: 'starting', consecutiveFailures: 0, downgraded: false } } +/** The wedge watch carried between two readiness-loop probes. */ +export interface WedgeWatch { + /** Epoch ms when the current unbroken run of unresponsive probes began. */ + unresponsiveSince: number | null + /** Unresponsive probes in the current run. */ + failures: number + /** Epoch ms of the previous probe result, to detect a wall-clock jump. */ + lastObservedAt: number | null +} + +export const EMPTY_WEDGE_WATCH: WedgeWatch = { unresponsiveSince: null, failures: 0, lastObservedAt: null } + +export interface WedgeDecisionInput { + watch: WedgeWatch + /** The primary probe (`/session`, or the liveness route while the gate is closed) got no HTTP answer. */ + probeFailed: boolean + /** The Instance-independent `/kortix-liveness-probe` also got no HTTP answer. */ + livenessProbeFailed: boolean + /** Epoch ms of the last wedge kill in this lifecycle, or null. */ + lastKillAt: number | null + now: number + enabled: boolean + thresholdMs: number + cooldownMs: number + minFailures: number + maxProbeGapMs: number +} + +export type WedgeDecisionReason = 'responsive' | 'disabled' | 'below-threshold' | 'cooldown' | 'wedged' + +export interface WedgeDecision { + watch: WedgeWatch + kill: boolean + reason: WedgeDecisionReason + /** Length of the current unresponsive run at `now` (0 when responsive). */ + unresponsiveMs: number +} + +/** + * Decide whether a supervised opencode is wedged and must be killed. Pure + + * deterministic — see opencode-wedge-kill.test.ts. + * + * - Any HTTP answer (either probe) means the process is alive: reset. + * - Unresponsive = BOTH probes failed. A slow `/session` alone is a busy + * Instance, not a wedged process. + * - Kill only when the run is at least `thresholdMs` long AND holds at least + * `minFailures` probes, and no kill happened in the last `cooldownMs`. + * - A gap above `maxProbeGapMs` between two results restarts the run: the + * daemon was frozen with its child (snapshot restore), so the elapsed wall + * time is not evidence. + * - A kill resets the run, so the respawned child gets a full window. + */ +export function nextWedgeAction(input: WedgeDecisionInput): WedgeDecision { + const { watch, now } = input + if (!input.probeFailed || !input.livenessProbeFailed) { + return { + watch: { unresponsiveSince: null, failures: 0, lastObservedAt: now }, + kill: false, + reason: 'responsive', + unresponsiveMs: 0, + } + } + const jumped = watch.lastObservedAt !== null && now - watch.lastObservedAt > input.maxProbeGapMs + const unresponsiveSince = jumped || watch.unresponsiveSince === null ? now : watch.unresponsiveSince + const failures = jumped ? 1 : watch.failures + 1 + const next: WedgeWatch = { unresponsiveSince, failures, lastObservedAt: now } + const unresponsiveMs = now - unresponsiveSince + if (!input.enabled) return { watch: next, kill: false, reason: 'disabled', unresponsiveMs } + if (failures < input.minFailures || unresponsiveMs < input.thresholdMs) { + return { watch: next, kill: false, reason: 'below-threshold', unresponsiveMs } + } + if (input.lastKillAt !== null && now - input.lastKillAt < input.cooldownMs) { + return { watch: next, kill: false, reason: 'cooldown', unresponsiveMs } + } + return { + watch: { unresponsiveSince: null, failures: 0, lastObservedAt: now }, + kill: true, + reason: 'wedged', + unresponsiveMs, + } +} + +export interface WedgeKillStats { + /** Wedge kills since this lifecycle was created. */ + kills: number + /** ISO time of the last wedge kill, or null. */ + lastKillAt: string | null + /** Current unresponsive run length in ms (0 when responsive). */ + unresponsiveMs: number + enabled: boolean +} + export type Opencode = HarnessLifecycleService & { reloadConfig(opts?: { mustRespawn?: boolean }): Promise /** @@ -1614,6 +1737,11 @@ export type Opencode = HarnessLifecycleService & { * then is never answered. Every boot-time request waits for this first. */ waitForCurrentListening(): Promise + /** + * Wedge-kill counters for diagnostics (see nextWedgeAction). Optional so + * partial test doubles of `Opencode` stay valid. + */ + getWedgeKillStats?(): WedgeKillStats } export interface OpencodeLifecycleOptions { @@ -1665,6 +1793,19 @@ export interface OpencodeLifecycleOptions { * and would say it to people whose work completed normally. */ onUnplannedRespawn?: () => void | Promise + /** Test override for READY_LIVENESS_MS (the healthy liveness interval). */ + readyLivenessMs?: number + /** + * Test overrides for the wedge kill. Production reads `enabled` and + * `thresholdMs` from KORTIX_OPENCODE_WEDGE_KILL(_AFTER_MS). + */ + wedgeKill?: { + enabled?: boolean + thresholdMs?: number + cooldownMs?: number + minFailures?: number + probeTimeoutMs?: number + } } export function createOpencodeLifecycle( @@ -1693,6 +1834,12 @@ export function createOpencodeLifecycle( let state: OpencodeState = 'starting' /** Consecutive failed liveness probes while `ok` (see nextLivenessState). */ let livenessFailures = 0 + // Wedge watch (see nextWedgeAction), scoped to the child it observed. + let wedgeWatch: WedgeWatch = EMPTY_WEDGE_WATCH + let wedgeWatchedChild: ChildProcess | null = null + let wedgeKills = 0 + let lastWedgeKillAt: number | null = null + let wedgeCooldownLogged = false let readinessTimer: ReturnType | null = null let firstReadyResponseReported = false let firstListeningResponseReported = false @@ -2320,7 +2467,7 @@ export function createOpencodeLifecycle( state === 'ok' ? livenessFailures > 0 ? READY_LIVENESS_RECHECK_MS - : READY_LIVENESS_MS + : (options.readyLivenessMs ?? READY_LIVENESS_MS) : READY_POLL_MS readinessTimer = setTimeout(async () => { if (stopping) return @@ -2378,10 +2525,92 @@ export function createOpencodeLifecycle( } state = decision.state } + if (probedChild && (await observeWedge(probedChild, probedPort, probe === 'down'))) return scheduleReadinessProbe() }, interval) } + /** + * Feed one readiness-loop result into the wedge watch, and SIGKILL the child + * when nextWedgeAction says it is wedged. Returns true when the loop must + * stop this tick (the lifecycle stopped, or the child changed under the + * extra probe) — the caller then schedules nothing; this function already + * rescheduled. + * + * Only a child that has already been heard from (`listeningProcess`) is + * watched: a process that never answered is a slow or failed boot, which the + * boot path owns. Verification candidates are never `child`, so they are + * never watched — the same exclusion `mayProbe` + the `probedChild !== + * child` guards give the readiness loop. + */ + async function observeWedge(proc: ChildProcess, port: number, probeFailed: boolean): Promise { + if (proc !== wedgeWatchedChild) { + wedgeWatch = EMPTY_WEDGE_WATCH + wedgeWatchedChild = proc + } + if (listeningProcess !== proc) return false + let livenessProbeFailed = probeFailed + if (probeFailed && directoryProbeOpen) { + // `/session` failing alone can be a busy Instance. Ask the route that + // creates no Instance whether the HTTP server itself still answers. + livenessProbeFailed = !(await probeOpencodeListening( + `http://127.0.0.1:${port}`, + options.wedgeKill?.probeTimeoutMs ?? 2_000, + )) + if (stopping) return true + if (proc !== child || port !== livePort()) { + scheduleReadinessProbe() + return true + } + } + const enabled = options.wedgeKill?.enabled ?? wedgeKillEnabled() + const decision = nextWedgeAction({ + watch: wedgeWatch, + probeFailed, + livenessProbeFailed, + lastKillAt: lastWedgeKillAt, + now: Date.now(), + enabled, + thresholdMs: options.wedgeKill?.thresholdMs ?? wedgeKillAfterMs(), + cooldownMs: options.wedgeKill?.cooldownMs ?? WEDGE_KILL_COOLDOWN_MS, + minFailures: options.wedgeKill?.minFailures ?? WEDGE_KILL_MIN_FAILURES, + maxProbeGapMs: WEDGE_MAX_PROBE_GAP_MS, + }) + // A kill resets the watch, so read the run's length before replacing it. + const failures = decision.kill ? wedgeWatch.failures + 1 : decision.watch.failures + wedgeWatch = decision.watch + if (decision.reason !== 'cooldown') wedgeCooldownLogged = false + else if (!wedgeCooldownLogged) { + wedgeCooldownLogged = true + logger.warn('[opencode] wedge-kill suppressed by cooldown', { + pid: proc.pid, + port, + unresponsiveMs: decision.unresponsiveMs, + lastKillAt: lastWedgeKillAt ? new Date(lastWedgeKillAt).toISOString() : null, + }) + } + if (!decision.kill) return false + wedgeKills += 1 + lastWedgeKillAt = Date.now() + // Structured, greppable: `[opencode] wedge-kill` is the one line that says + // the daemon replaced an opencode on its own. + logger.warn('[opencode] wedge-kill', { + reason: 'http-unresponsive', + pid: proc.pid, + port, + unresponsiveMs: decision.unresponsiveMs, + failures, + kills: wedgeKills, + }) + // SIGKILL straight away: a wedged event loop cannot act on SIGTERM, and a + // SIGSTOPped process cannot receive it. Group kill, as stop() does, so no + // grandchild outlives it. The exit handler (superviseChild) sees + // `child === proc` with `stopping` false and runs scheduleUnplannedRespawn, + // which respawns and then calls onUnplannedRespawn to finalize the turn. + void killProcessGroup(proc, 'SIGKILL') + return false + } + return { async start() { @@ -2638,6 +2867,15 @@ export function createOpencodeLifecycle( return child?.pid ?? null }, + getWedgeKillStats(): WedgeKillStats { + return { + kills: wedgeKills, + lastKillAt: lastWedgeKillAt ? new Date(lastWedgeKillAt).toISOString() : null, + unresponsiveMs: wedgeWatch.unresponsiveSince === null ? 0 : Date.now() - wedgeWatch.unresponsiveSince, + enabled: options.wedgeKill?.enabled ?? wedgeKillEnabled(), + } + }, + getActivePort() { return livePort() }, diff --git a/apps/mobile/lib/opencode/hooks/use-opencode-data.ts b/apps/mobile/lib/opencode/hooks/use-opencode-data.ts index 2a60ebd17fc..18f6080879c 100644 --- a/apps/mobile/lib/opencode/hooks/use-opencode-data.ts +++ b/apps/mobile/lib/opencode/hooks/use-opencode-data.ts @@ -380,39 +380,6 @@ export function useUpdateOpenCodeConfig(sandboxUrl: string | undefined) { // ─── MCP Mutations ────────────────────────────────────────────────────────── -export interface AddMcpServerParams { - name: string; - type: 'local' | 'remote'; - command?: string[]; - env?: Record; - url?: string; - headers?: Record; -} - -export function useAddMcpServer(sandboxUrl: string | undefined) { - const queryClient = useQueryClient(); - return useMutation({ - mutationFn: async (params: AddMcpServerParams) => { - if (!sandboxUrl) throw new Error('No sandbox URL'); - const config: Record = { type: params.type }; - if (params.type === 'local') { - config.command = params.command; - if (params.env && Object.keys(params.env).length > 0) config.environment = params.env; - } else { - config.url = params.url; - if (params.headers && Object.keys(params.headers).length > 0) config.headers = params.headers; - } - return opencodeFetch>(sandboxUrl, '/mcp', { - method: 'POST', - body: JSON.stringify({ name: params.name, config }), - }); - }, - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: opencodeKeys.mcpStatus(sandboxUrl || '') }); - }, - }); -} - export function useConnectMcpServer(sandboxUrl: string | undefined) { const queryClient = useQueryClient(); return useMutation({ diff --git a/docs/runbooks/prod-us-east-2-supabase-migration.md b/docs/runbooks/prod-us-east-2-supabase-migration.md index 7b439764c31..b570016dacc 100644 --- a/docs/runbooks/prod-us-east-2-supabase-migration.md +++ b/docs/runbooks/prod-us-east-2-supabase-migration.md @@ -577,7 +577,14 @@ replication and migration administration. ## Target preparation -1. Run the fresh-project bootstrap. +1. Run the fresh-project bootstrap. `bootstrap` refuses every host except + loopback and `supabase-db` unless you pass `--allow-remote`: + + ```bash + DATABASE_URL='?uselibpqcompat=true' \ + bun packages/db/scripts/migrate.ts bootstrap --allow-remote + ``` + 2. Apply the repository migration ledger with `pnpm migrate`. 3. Verify every migration on PostgreSQL 17.6. 4. Verify all required extensions, functions, triggers, RLS policies, grants, diff --git a/packages/db/scripts/migrate.ts b/packages/db/scripts/migrate.ts index 2144bd6c467..8f754eecd6a 100644 --- a/packages/db/scripts/migrate.ts +++ b/packages/db/scripts/migrate.ts @@ -16,6 +16,8 @@ import { join } from 'node:path'; * bun scripts/migrate.ts down [--count=N] roll back N (default 1) * bun scripts/migrate.ts fake mark pending as applied without running (baseline) * bun scripts/migrate.ts bootstrap fresh-DB: install non-kortix prereqs, then `up` + * (loopback or supabase-db only; add --allow-remote + * for a deliberate remote target) * bun scripts/migrate.ts local-up loopback-only; tolerate cross-worktree ledger order * bun scripts/migrate.ts preview-up preview-only; tolerate persistent branch ledger order * @@ -34,7 +36,7 @@ import { import { repairLocalWarmSessionIndex } from './local-warm-session-index-repair'; import { withMigrationDeadlockRetry } from './migration-retry'; import { materializeMigrationRuntimeDirectory } from './migration-runtime-overrides'; -import { migrationBootstrapsPrerequisites, migrationCheckOrder } from './migration-target'; +import { assertBootstrapTarget, migrationBootstrapsPrerequisites, migrationCheckOrder } from './migration-target'; const MIGRATIONS_DIR = join(import.meta.dir, '..', 'migrations'); const BOOTSTRAP_SQL = join(import.meta.dir, '..', 'drizzle', '0000_bootstrap.sql'); @@ -202,6 +204,7 @@ async function selfHostBootstrapIfFresh(databaseUrl: string): Promise { async function main() { const [cmd = 'up', ...rest] = process.argv.slice(2); const databaseUrl = resolveUrl(rest); + assertBootstrapTarget(cmd, databaseUrl, rest.includes('--allow-remote')); const checkOrder = migrationCheckOrder(cmd, databaseUrl, process.env.KORTIX_PREVIEW_MIGRATION); const countArg = rest.find((a) => a.startsWith('--count='))?.slice('--count='.length); const runtimeMigrations = materializeMigrationRuntimeDirectory(MIGRATIONS_DIR); diff --git a/packages/db/scripts/migration-target.test.ts b/packages/db/scripts/migration-target.test.ts index 5b223ce4000..410ee221452 100644 --- a/packages/db/scripts/migration-target.test.ts +++ b/packages/db/scripts/migration-target.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'bun:test'; -import { migrationCheckOrder, migrationBootstrapsPrerequisites } from './migration-target'; +import { assertBootstrapTarget, migrationCheckOrder, migrationBootstrapsPrerequisites } from './migration-target'; describe('migration target mode', () => { test('keeps migration ordering strict for normal commands', () => { @@ -35,4 +35,34 @@ describe('migration target mode', () => { expect(migrationBootstrapsPrerequisites('up')).toBe(false); expect(migrationBootstrapsPrerequisites('status')).toBe(false); }); + + test('allows bootstrap on loopback and the self-host supabase-db host', () => { + for (const url of [ + 'postgresql://user:pass@127.0.0.1:5432/app', + 'postgresql://user:pass@localhost:5432/app', + 'postgresql://user:pass@[::1]:5432/app', + 'postgresql://postgres:pass@supabase-db:5432/postgres', + ]) { + expect(() => assertBootstrapTarget('bootstrap', url, false)).not.toThrow(); + } + }); + + test('refuses bootstrap against a remote database without --allow-remote', () => { + expect(() => assertBootstrapTarget('bootstrap', 'postgresql://user:pass@db.example.com:5432/app', false)).toThrow( + 'bootstrap refuses non-local database host: db.example.com (pass --allow-remote to run it anyway)', + ); + expect(() => assertBootstrapTarget('bootstrap', 'not-a-url', false)).toThrow( + 'bootstrap requires a valid DATABASE_URL', + ); + }); + + test('--allow-remote permits bootstrap against a remote database', () => { + expect(() => assertBootstrapTarget('bootstrap', 'postgresql://user:pass@db.example.com:5432/app', true)).not.toThrow(); + }); + + test('the bootstrap guard ignores every other command', () => { + for (const cmd of ['up', 'status', 'down', 'fake']) { + expect(() => assertBootstrapTarget(cmd, 'postgresql://user:pass@db.example.com:5432/app', false)).not.toThrow(); + } + }); }); diff --git a/packages/db/scripts/migration-target.ts b/packages/db/scripts/migration-target.ts index 8251df1afad..ac1f681857b 100644 --- a/packages/db/scripts/migration-target.ts +++ b/packages/db/scripts/migration-target.ts @@ -24,3 +24,24 @@ export function migrationCheckOrder(command: string, databaseUrl: string, previe export function migrationBootstrapsPrerequisites(command: string): boolean { return command === 'bootstrap' || command === 'local-up' || command === 'preview-up'; } + +const LOCAL_BOOTSTRAP_HOSTS = new Set(['127.0.0.1', 'localhost', '[::1]', 'supabase-db']); + +/** + * `bootstrap` installs the non-kortix prerequisites (credit RPCs, a trigger on + * auth.users, a basejump stub) into a fresh database. That is right for the + * self-host compose stack (`supabase-db`) and a local database, and wrong for + * anything else reached by accident. Refuse any other host unless the operator + * passes `--allow-remote` (e.g. a deliberate fresh Supabase Cloud project). + */ +export function assertBootstrapTarget(command: string, databaseUrl: string, allowRemote: boolean): void { + if (command !== 'bootstrap') return; + let hostname: string; + try { + hostname = new URL(databaseUrl).hostname; + } catch { + throw new Error('bootstrap requires a valid DATABASE_URL'); + } + if (LOCAL_BOOTSTRAP_HOSTS.has(hostname) || allowRemote) return; + throw new Error(`bootstrap refuses non-local database host: ${hostname} (pass --allow-remote to run it anyway)`); +} diff --git a/packages/sdk/API-MAP.md b/packages/sdk/API-MAP.md index 09facd05913..f6ec3696122 100644 --- a/packages/sdk/API-MAP.md +++ b/packages/sdk/API-MAP.md @@ -116,7 +116,7 @@ try/catching every call. | tools ids / list | `client.tool.{ids,list}` | ✅ | | skills **list** | `client.app.skills` → `/skill` | ✅ | | skills **create/update/delete** | daemon `/file/upload`,`/file/mkdir`,`DELETE /file` + `instance.dispose` | ❌ web-local (`features/skills`) | -| MCP status/add/connect/disconnect/oauth | `client.mcp.*` | ✅ | +| MCP status/add/connect/disconnect/oauth | `client.mcp.*` | ✅ (`add` is in-memory only; persist with `createConnector` + `provider: 'mcp'`) | ### 9. Terminal (PTY) ✅ Kortix-native (`opencode/pty.ts`), independent of the agent runtime — daemon diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index aa234f38acc..d38f54366bc 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -37,6 +37,11 @@ follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). `FEATURE_FLAG_KEYS` and `KortixProject.experimental`, and `updateFeatureFlag(id, 'review_center', …)` answers `400`. Removed in the next major. +- `useAddMcpServer` (`@kortix/sdk/react`). OpenCode's `POST /mcp` keeps the + server in process memory only, so it is lost on the next OpenCode or sandbox + restart. Use `createConnector(projectId, { provider: 'mcp', … })` (or + `kortix.project(id).connectors.create`), with `discoverConnectorAuth` for + OAuth detection. Removed in the next major. ### Fixed - `getPlatformUrl()` no longer reads a bare `process.env`, which threw a diff --git a/packages/sdk/src/react/use-opencode-mcp.test.ts b/packages/sdk/src/react/use-opencode-mcp.test.ts new file mode 100644 index 00000000000..cee1c94fe9b --- /dev/null +++ b/packages/sdk/src/react/use-opencode-mcp.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from 'bun:test'; + +import * as root from '../index'; +import * as react from './index'; + +/** The JSDoc block that sits directly on `export function `. */ +async function jsdocFor(name: string): Promise { + const source = await Bun.file(new URL('./use-opencode-mcp.ts', import.meta.url).pathname).text(); + const at = source.indexOf(`export function ${name}(`); + expect(at).toBeGreaterThan(-1); + const before = source.slice(0, at).trimEnd(); + expect(before.endsWith('*/')).toBe(true); + return before.slice(before.lastIndexOf('/**')); +} + +describe('useAddMcpServer deprecation', () => { + test('stays exported from ./react (removing it is a breaking change)', () => { + expect(typeof (react as Record).useAddMcpServer).toBe('function'); + }); + + test('is marked @deprecated, says why, and names the persistent replacement', async () => { + const doc = await jsdocFor('useAddMcpServer'); + expect(doc).toContain('@deprecated'); + // The reason: OpenCode's POST /mcp is process memory only. + expect(doc).toContain('in-memory'); + expect(doc).toContain('restart'); + // The replacement, by its real exported names. + expect(doc).toContain('createConnector'); + expect(doc).toContain("provider: 'mcp'"); + expect(doc).toContain('discoverConnectorAuth'); + }); + + test('the names the deprecation points to are real root exports', () => { + const exported = root as Record; + expect(typeof exported.createConnector).toBe('function'); + expect(typeof exported.discoverConnectorAuth).toBe('function'); + }); +}); diff --git a/packages/sdk/src/react/use-opencode-mcp.ts b/packages/sdk/src/react/use-opencode-mcp.ts index 8cdfb59b21f..309a202a84f 100644 --- a/packages/sdk/src/react/use-opencode-mcp.ts +++ b/packages/sdk/src/react/use-opencode-mcp.ts @@ -5,7 +5,7 @@ * * Wraps the SDK's `client.mcp.*` namespace: * - status() — list all servers + their statuses - * - add() — register a new MCP server + * - add() — register a new MCP server (in-memory only; deprecated, see useAddMcpServer) * - connect() — connect a server by name * - disconnect() — disconnect a server by name * - auth.start() — start OAuth flow (returns authorization URL) @@ -67,6 +67,18 @@ export interface AddMcpServerParams { headers?: Record; } +/** + * Register an MCP server on the session's running OpenCode (`POST /mcp`). + * + * @deprecated The registration is in-memory only. OpenCode keeps it in process + * memory and never writes it to config, so it is lost on the next OpenCode + * restart, sandbox restart, or new session. Add a persistent project connector + * instead: `createConnector(projectId, { slug, provider: 'mcp', transport: + * 'http', url })` from `@kortix/sdk` (or `kortix.project(projectId) + * .connectors.create(...)`). Call `discoverConnectorAuth(projectId, draft)` + * first to learn whether the server needs OAuth. Kept for compatibility; it + * is removed in the next major. + */ export function useAddMcpServer() { const queryClient = useQueryClient();