fix: restart a wedged OpenCode, guard migrate bootstrap, deprecate in-memory MCP add

apps/kortix-sandbox-agent-server/README.md

@@ -185,8 +185,21 @@ KORTIX_REPO_URL=
 KORTIX_BRANCH_NAME=
 KORTIX_GITHUB_TOKEN=
 KORTIX_TOKEN=
+KORTIX_OPENCODE_WEDGE_KILL=1              # 0|false: never SIGKILL an HTTP-unresponsive opencode (gate-only, pre-2026-09 behaviour)
+KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS=60000 # continuous unresponsiveness before the wedge kill; 5 min cooldown between kills
 ```
 
+### Wedged OpenCode recovery
+
+The readiness loop probes the live opencode every 5 s. After 3 failed probes
+the daemon marks it `starting`, and the proxy answers 503. When BOTH the
+`/session` probe and the Instance-independent `/kortix-liveness-probe` fail
+for 60 s (at least 3 probes), the daemon SIGKILLs the opencode process group.
+The ordinary exit path then respawns it and finalizes the orphaned turn. The
+daemon logs `[opencode] wedge-kill` with `unresponsiveMs`, `failures`, and
+`kills`. `GET /kortix/diag` reports `opencode.wedge_kill`
+(`kills`, `lastKillAt`, `unresponsiveMs`, `enabled`).
+
 ## Build
 
 ```

apps/kortix-sandbox-agent-server/src/__tests__/opencode-wedge-kill-process.e2e.test.ts

@@ -0,0 +1,160 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
+import { chmodSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import type { OpenCodeConfig as Config } from '../harness/open-code/config'
+import { createOpencodeLifecycle } from '../harness/open-code/lifecycle'
+
+// A wedged opencode: the process is alive and its port stays bound, but no
+// request gets an answer. SIGSTOP produces exactly that — the kernel still
+// accepts connections into the listen backlog, nothing reads them. The
+// lifecycle must SIGKILL the child, respawn it through the ordinary exit
+// path, and run onUnplannedRespawn so the orphaned turn is finalized.
+
+let root: string
+let lifecycle: ReturnType<typeof createOpencodeLifecycle> | null
+const stoppedPids: number[] = []
+
+function reservePort(): number {
+  const server = Bun.serve({ port: 0, fetch: () => new Response('reserved') })
+  const port = server.port
+  server.stop(true)
+  if (typeof port !== 'number') throw new Error('Bun did not assign a port')
+  return port
+}
+
+function writeHealthyBinary(path: string) {
+  writeFileSync(
+    path,
+    `#!/usr/bin/env bun
+const port = Number(Bun.argv[Bun.argv.indexOf('--port') + 1])
+Bun.serve({
+  hostname: '127.0.0.1',
+  port,
+  fetch(req) {
+    const url = new URL(req.url)
+    if (url.pathname === '/session') return Response.json([])
+    return new Response('not found', { status: 404 })
+  },
+})
+console.log('opencode server listening on http://127.0.0.1:' + port)
+setInterval(() => {}, 60_000)
+`,
+  )
+  chmodSync(path, 0o755)
+}
+
+function alive(pid: number): boolean {
+  try {
+    process.kill(pid, 0)
+    return true
+  } catch {
+    return false
+  }
+}
+
+async function until(check: () => boolean, timeoutMs: number, label: string): Promise<void> {
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    if (check()) return
+    await new Promise((r) => setTimeout(r, 50))
+  }
+  throw new Error(`timed out waiting for ${label}`)
+}
+
+beforeEach(() => {
+  root = mkdtempSync(join(tmpdir(), 'kortix-wedge-kill-'))
+  lifecycle = null
+})
+
+afterEach(async () => {
+  for (const pid of stoppedPids.splice(0)) {
+    try {
+      process.kill(pid, 'SIGKILL')
+    } catch {}
+  }
+  await lifecycle?.stop()
+  rmSync(root, { recursive: true, force: true })
+})
+
+function makeCfg(): Config {
+  const workspace = join(root, 'workspace')
+  mkdirSync(workspace)
+  return {
+    workspace,
+    projectTarget: workspace,
+    opencodeInternalPort: reservePort(),
+    opencodeStandbyPort: reservePort(),
+    gitUserName: 'Kortix Agent',
+    gitUserEmail: 'agent@kortix.ai',
+  } as Config
+}
+
+function startLifecycle(wedgeKillEnabled: boolean, onRespawn: () => void) {
+  const configDir = join(root, 'config')
+  const binary = join(root, 'opencode')
+  mkdirSync(configDir)
+  writeHealthyBinary(binary)
+  return createOpencodeLifecycle(makeCfg(), configDir, undefined, {
+    binaryPathOverride: binary,
+    configPathOverride: join(root, 'runtime-config.json'),
+    readyLivenessMs: 200,
+    wedgeKill: { enabled: wedgeKillEnabled, thresholdMs: 1_000, minFailures: 2, probeTimeoutMs: 500 },
+    onUnplannedRespawn: onRespawn,
+  })
+}
+
+describe('OpenCode wedge kill (real child process)', () => {
+  test('a SIGSTOPped child is SIGKILLed, respawned, and its turn finalized', async () => {
+    let respawnHookCalls = 0
+    lifecycle = startLifecycle(true, () => {
+      respawnHookCalls += 1
+    })
+    await lifecycle.start()
+    await until(() => lifecycle!.getState() === 'ok', 10_000, 'first ready')
+    const wedgedPid = lifecycle.getPid()!
+    expect(wedgedPid).toBeGreaterThan(0)
+
+    process.kill(wedgedPid, 'SIGSTOP')
+    stoppedPids.push(wedgedPid)
+
+    await until(() => !alive(wedgedPid), 20_000, 'wedged child killed')
+    await until(() => respawnHookCalls === 1, 10_000, 'onUnplannedRespawn after respawn')
+
+    // The hook fires once the replacement answers; the readiness loop flips
+    // the state on its next tick.
+    await until(() => lifecycle!.getState() === 'ok', 5_000, 'replacement ready')
+    const replacementPid = lifecycle.getPid()
+    expect(replacementPid).not.toBeNull()
+    expect(replacementPid).not.toBe(wedgedPid)
+    expect(alive(replacementPid!)).toBe(true)
+    const stats = lifecycle.getWedgeKillStats!()
+    expect(stats.kills).toBe(1)
+    expect(stats.lastKillAt).not.toBeNull()
+    expect(stats.enabled).toBe(true)
+  }, 40_000)
+
+  test('kill switch off: the SIGSTOPped child is gated off but left alive', async () => {
+    let respawnHookCalls = 0
+    lifecycle = startLifecycle(false, () => {
+      respawnHookCalls += 1
+    })
+    await lifecycle.start()
+    await until(() => lifecycle!.getState() === 'ok', 10_000, 'first ready')
+    const wedgedPid = lifecycle.getPid()!
+
+    process.kill(wedgedPid, 'SIGSTOP')
+    stoppedPids.push(wedgedPid)
+
+    // Long enough for the enabled case above to have killed it (~5-7 s).
+    await new Promise((r) => setTimeout(r, 9_000))
+    expect(alive(wedgedPid)).toBe(true)
+    expect(lifecycle.getPid()).toBe(wedgedPid)
+    expect(respawnHookCalls).toBe(0)
+    const stats = lifecycle.getWedgeKillStats!()
+    expect(stats.kills).toBe(0)
+    expect(stats.enabled).toBe(false)
+    expect(stats.unresponsiveMs).toBeGreaterThan(1_000)
+  }, 40_000)
+})

apps/kortix-sandbox-agent-server/src/__tests__/opencode-wedge-kill.test.ts

@@ -0,0 +1,161 @@
+/**
+ * Wedge kill: an opencode whose process is alive but whose HTTP server no
+ * longer answers must be replaced, not only gated off.
+ *
+ * The liveness hysteresis (opencode-liveness-hysteresis.test.ts) downgrades
+ * such a child to `starting` and proxy.ts 503s every request, but respawn ran
+ * only on process exit, so the session stayed wedged until a manual refresh.
+ * `nextWedgeAction` decides when the readiness loop SIGKILLs the child; the
+ * exit handler then respawns it.
+ */
+import { describe, expect, test } from 'bun:test'
+import {
+  EMPTY_WEDGE_WATCH,
+  nextWedgeAction,
+  wedgeKillAfterMs,
+  wedgeKillEnabled,
+  type WedgeDecisionInput,
+  type WedgeWatch,
+} from '../harness/open-code/lifecycle'
+
+const T0 = 1_000_000
+const BASE = {
+  lastKillAt: null,
+  enabled: true,
+  thresholdMs: 60_000,
+  cooldownMs: 300_000,
+  minFailures: 3,
+  maxProbeGapMs: 30_000,
+} satisfies Omit<WedgeDecisionInput, 'watch' | 'probeFailed' | 'livenessProbeFailed' | 'now'>
+
+/** Feed a run of fully-unresponsive probes `stepMs` apart; return every decision. */
+function run(times: number[], overrides: Partial<WedgeDecisionInput> = {}, watch: WedgeWatch = EMPTY_WEDGE_WATCH) {
+  const out = []
+  let w = watch
+  for (const now of times) {
+    const d = nextWedgeAction({ ...BASE, ...overrides, watch: w, probeFailed: true, livenessProbeFailed: true, now })
+    w = d.watch
+    out.push(d)
+  }
+  return out
+}
+
+/** Probe times every `stepMs` from T0 through T0 + totalMs inclusive. */
+function every(stepMs: number, totalMs: number): number[] {
+  const times = []
+  for (let t = 0; t <= totalMs; t += stepMs) times.push(T0 + t)
+  return times
+}
+
+describe('nextWedgeAction', () => {
+  test('no kill before the threshold, even with many failures', () => {
+    const decisions = run(every(4_000, 56_000))
+    expect(decisions.every((d) => !d.kill)).toBe(true)
+    expect(decisions.at(-1)).toMatchObject({ reason: 'below-threshold', unresponsiveMs: 56_000 })
+    expect(decisions.at(-1)?.watch.failures).toBe(15)
+  })
+
+  test('kills at the threshold and resets the watch', () => {
+    const decisions = run(every(4_000, 60_000))
+    expect(decisions.slice(0, -1).every((d) => !d.kill)).toBe(true)
+    expect(decisions.at(-1)).toEqual({
+      kill: true,
+      reason: 'wedged',
+      unresponsiveMs: 60_000,
+      watch: { unresponsiveSince: null, failures: 0, lastObservedAt: T0 + 60_000 },
+    })
+  })
+
+  test('the threshold alone is not enough: minFailures must also be met', () => {
+    // Two probes 60 s apart would pass the time test with only 2 failures,
+    // but a 60 s gap is itself a clock jump; use a wide gap limit to isolate
+    // the failure-count rule.
+    const decisions = run([T0, T0 + 60_000], { maxProbeGapMs: 120_000 })
+    expect(decisions.at(-1)).toMatchObject({ kill: false, reason: 'below-threshold', unresponsiveMs: 60_000 })
+    const third = run([T0, T0 + 60_000, T0 + 61_000], { maxProbeGapMs: 120_000 })
+    expect(third.at(-1)).toMatchObject({ kill: true, reason: 'wedged' })
+  })
+
+  test('respects the cooldown after a kill, then kills again once it passes', () => {
+    const lastKillAt = T0 - 60_000 // 60 s ago; cooldown is 300 s
+    const inCooldown = run(every(4_000, 60_000), { lastKillAt })
+    expect(inCooldown.some((d) => d.kill)).toBe(false)
+    expect(inCooldown.at(-1)).toMatchObject({ reason: 'cooldown', unresponsiveMs: 60_000 })
+
+    // Still wedged when the cooldown ends (T0 + 240 s): the run was kept, so
+    // the next probe kills at once.
+    let w = inCooldown.at(-1)!.watch
+    let killedAt: number | null = null
+    for (let now = T0 + 64_000; now <= T0 + 260_000; now += 4_000) {
+      const d = nextWedgeAction({ ...BASE, lastKillAt, watch: w, probeFailed: true, livenessProbeFailed: true, now })
+      w = d.watch
+      if (d.kill) {
+        killedAt = now
+        break
+      }
+    }
+    expect(killedAt).toBe(T0 + 240_000)
+  })
+
+  test('any answer resets the run (recovery)', () => {
+    const almost = run(every(4_000, 56_000))
+    const recovered = nextWedgeAction({
+      ...BASE,
+      watch: almost.at(-1)!.watch,
+      probeFailed: false,
+      livenessProbeFailed: false,
+      now: T0 + 58_000,
+    })
+    expect(recovered).toEqual({
+      kill: false,
+      reason: 'responsive',
+      unresponsiveMs: 0,
+      watch: { unresponsiveSince: null, failures: 0, lastObservedAt: T0 + 58_000 },
+    })
+    // A fresh run needs the full threshold again.
+    const after = run(every(4_000, 56_000).map((t) => t + 60_000), {}, recovered.watch)
+    expect(after.some((d) => d.kill)).toBe(false)
+  })
+
+  test('a slow /session with a live HTTP server is busy, not wedged', () => {
+    let w: WedgeWatch = EMPTY_WEDGE_WATCH
+    for (const now of every(4_000, 120_000)) {
+      const d = nextWedgeAction({ ...BASE, watch: w, probeFailed: true, livenessProbeFailed: false, now })
+      expect(d).toMatchObject({ kill: false, reason: 'responsive' })
+      w = d.watch
+    }
+  })
+
+  test('kill switch off: never kills, but still measures the run', () => {
+    const decisions = run(every(4_000, 120_000), { enabled: false })
+    expect(decisions.some((d) => d.kill)).toBe(false)
+    expect(decisions.at(-1)).toMatchObject({ reason: 'disabled', unresponsiveMs: 120_000 })
+  })
+
+  test('a wall-clock jump (snapshot restore) restarts the run instead of killing', () => {
+    const before = run([T0, T0 + 4_000])
+    // The VM was frozen for an hour; the first probe after resume fails.
+    const resumed = run([T0 + 3_600_000, T0 + 3_604_000, T0 + 3_608_000], {}, before.at(-1)!.watch)
+    expect(resumed.some((d) => d.kill)).toBe(false)
+    expect(resumed.at(-1)).toMatchObject({ reason: 'below-threshold', unresponsiveMs: 8_000 })
+    expect(resumed.at(-1)?.watch.failures).toBe(3)
+  })
+})
+
+describe('wedge kill env', () => {
+  test('KORTIX_OPENCODE_WEDGE_KILL defaults on; 0/false disable it', () => {
+    expect(wedgeKillEnabled({})).toBe(true)
+    expect(wedgeKillEnabled({ KORTIX_OPENCODE_WEDGE_KILL: '1' })).toBe(true)
+    expect(wedgeKillEnabled({ KORTIX_OPENCODE_WEDGE_KILL: 'true' })).toBe(true)
+    expect(wedgeKillEnabled({ KORTIX_OPENCODE_WEDGE_KILL: '0' })).toBe(false)
+    expect(wedgeKillEnabled({ KORTIX_OPENCODE_WEDGE_KILL: ' FALSE ' })).toBe(false)
+  })
+
+  test('KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS overrides the 60 s threshold; junk falls back', () => {
+    expect(wedgeKillAfterMs({})).toBe(60_000)
+    expect(wedgeKillAfterMs({ KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS: '120000' })).toBe(120_000)
+    expect(wedgeKillAfterMs({ KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS: 'abc' })).toBe(60_000)
+    expect(wedgeKillAfterMs({ KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS: '0' })).toBe(60_000)
+    expect(wedgeKillAfterMs({ KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS: '-5' })).toBe(60_000)
+  })
+})

apps/kortix-sandbox-agent-server/src/__tests__/runtime-env-allowlist-completeness.test.ts

@@ -88,6 +88,10 @@ const BOOT_ONLY_KORTIX_ENV_NAMES = new Set([
   // Manual operator debug toggle (checked against `process.env` directly, not
   // part of the env-sync contract at all).
   'KORTIX_OPENCODE_DEBUG',
+  // Daemon wedge-kill kill switch and threshold (nextWedgeAction). Operator
+  // tunables for the daemon's own supervision loop; opencode never reads them.
+  'KORTIX_OPENCODE_WEDGE_KILL',
+  'KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS',
   // Artifact identity is fixed before PID 1 starts. A live update installs a
   // new artifact and daemon instead of changing this value through /env.
   'KORTIX_COMPILED_RUNTIME_FORMAT',
@@ -153,6 +157,8 @@ describe('OPENCODE_RUNTIME_ENV_NAMES — allowlist completeness', () => {
       'KORTIX_LLM_PROXY_URL',
       'KORTIX_OPENCODE_DEBUG',
       'KORTIX_OPENCODE_MODEL',
+      'KORTIX_OPENCODE_WEDGE_KILL',
+      'KORTIX_OPENCODE_WEDGE_KILL_AFTER_MS',
       'KORTIX_PROJECT_ID',
       'KORTIX_SECRET_CAPABILITIES',
       'KORTIX_TOKEN',

apps/kortix-sandbox-agent-server/src/harness/open-code/diagnostics.ts

@@ -308,6 +308,9 @@ async function readOpenCodeDiagnosticReport(
       port_pair: [cfg.opencodeInternalPort, cfg.opencodeStandbyPort],
       session_id: bootState.initialOpenCodeSessionId ?? null,
       log_file: opencodeLog,
+      // Daemon-initiated SIGKILLs of an HTTP-unresponsive opencode (see
+      // nextWedgeAction). Non-zero means this box recovered from a wedge.
+      wedge_kill: opencode.getWedgeKillStats?.() ?? null,
     },
     boot: {
       repo_materialization_error: bootState.repoMaterializationError,