diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 34f3f46f..d96b2876 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -54,7 +54,9 @@ jobs: key: bun-${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('bun.lock') }} - run: bun install --frozen-lockfile # Fake-runtime parser fixtures use local Pillow/Poppler; no image is built and no Podman is run. - - run: sudo apt-get update -q && sudo apt-get install -y -q --no-install-recommends ripgrep python3-pil poppler-utils + # The helper retries apt with backoff so a short runner DNS outage does not fail the shard (#456). + # Each attempt is killed after 70 s; the worst case (~6m48s) leaves room for the tests in 15 min. + - run: bun ops/ci/apt-install.ts ripgrep python3-pil poppler-utils # A shared runner under parallel load can exceed Bun's default 5 s per test. - run: bun test --timeout 30000 $(bun ops/ci/test-shard.ts ${{ matrix.shard }} 6) env: diff --git a/ops/ci/apt-install.ts b/ops/ci/apt-install.ts new file mode 100644 index 00000000..f40d64c2 --- /dev/null +++ b/ops/ci/apt-install.ts @@ -0,0 +1,89 @@ +/** + * Installs the CI apt packages with bounded retries. A short runner DNS or mirror outage + * ("Temporary failure resolving archive.ubuntu.com", #456) then does not fail a shard before + * its tests start. Each attempt has a wall-clock deadline, so a resolver-bound apt walk cannot + * use up the job timeout. Once the retries run out the step still fails loudly; it never skips packages. + */ + +/** Runs one apt-get invocation, killed after `seconds`, and returns its exit code (124/137 on timeout). */ +export type AptRun = (args: string[], seconds: number) => Promise; + +/** Waits between attempts: 10 s, 30 s, then 60 s, so four attempts in total. */ +export const RETRY_DELAYS_MS = [10_000, 30_000, 60_000]; + +/** Wall-clock budget for one attempt (`update` plus `install`). A healthy runner needs about 10 s. */ +export const ATTEMPT_DEADLINE_MS = 70_000; + +/** Seconds `timeout` waits after SIGTERM before it sends SIGKILL to apt-get. */ +export const KILL_AFTER_S = 5; + +/** Exit code a timed-out attempt reports when its budget is already spent before `install` starts. */ +export const TIMED_OUT = 124; + +/** + * No apt-level fetch retries (the outer loop is the retry) and a fetch timeout for both schemes. + * The timeout only helps a stalled connection; the per-attempt deadline is the real cap. + */ +export const APT_OPTIONS = ["-q", "-o", "Acquire::Retries=0", "-o", "Acquire::http::Timeout=30", "-o", "Acquire::https::Timeout=30"]; + +/** Wraps a command in GNU `timeout`: SIGTERM after `seconds`, SIGKILL `KILL_AFTER_S` later. */ +export function timeoutCommand(seconds: number, argv: string[]): string[] { + return ["timeout", `--kill-after=${KILL_AFTER_S}`, String(seconds), ...argv]; +} + +/** `timeout` runs under sudo, so its signals reach apt-get itself and not only sudo. */ +export function aptCommand(args: string[], seconds: number): string[] { + return ["sudo", ...timeoutCommand(seconds, ["apt-get", ...args])]; +} + +/** + * Upper bound on how long `aptInstall` can run with the given delays: every attempt hits its + * deadline, pays the SIGKILL grace and up to 2 s of whole-second rounding, then every wait follows. + */ +export function worstCaseMs(delays = RETRY_DELAYS_MS, deadlineMs = ATTEMPT_DEADLINE_MS): number { + const attempts = delays.length + 1; + return attempts * (deadlineMs + KILL_AFTER_S * 1000 + 2000) + delays.reduce((sum, ms) => sum + ms, 0); +} + +export interface AptInstallOptions { + run: AptRun; + sleep: (ms: number) => Promise; + now?: () => number; + delays?: number[]; + deadlineMs?: number; + log?: (line: string) => void; +} + +/** + * Installs every requested package, running `update` before each `install` attempt. Both share one + * attempt deadline: `install` only gets what `update` left. Defaults to four attempts with 10 s, + * 30 s and 60 s waits after failures; a failed or timed-out install restarts from `update` on the + * next attempt. Rejects with `apt_install_no_packages` for an empty list, with + * `apt_install_failed after N attempts (exit C)` once every attempt fails, or with whatever `run` + * or `sleep` rejects with. + */ +export async function aptInstall(packages: string[], options: AptInstallOptions): Promise { + const { run, sleep, now = Date.now, delays = RETRY_DELAYS_MS, deadlineMs = ATTEMPT_DEADLINE_MS, log = line => console.error(line) } = options; + if (packages.length === 0) throw new Error("apt_install_no_packages"); + for (let attempt = 1; ; attempt++) { + const deadline = now() + deadlineMs; + const left = () => Math.ceil((deadline - now()) / 1000); + // `update` can exit 0 when it only warns about unreachable mirrors, so install failures retry too. + const update = await run([...APT_OPTIONS, "update"], left()); + let code = update; + if (update === 0) code = left() > 0 ? await run([...APT_OPTIONS, "install", "-y", "--no-install-recommends", ...packages], left()) : TIMED_OUT; + if (code === 0) return; + const delay = delays[attempt - 1]; + if (delay === undefined) throw new Error(`apt_install_failed after ${attempt} attempts (exit ${code})`); + log(`apt-install: attempt ${attempt} failed with exit ${code}; retrying in ${delay / 1000}s`); + await sleep(delay); + } +} + +if (import.meta.main) { + const run: AptRun = (args, seconds) => Bun.spawn(aptCommand(args, seconds), { stdout: "inherit", stderr: "inherit" }).exited; + await aptInstall(process.argv.slice(2), { run, sleep: ms => Bun.sleep(ms) }).catch(error => { + console.error(`apt-install: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + }); +} diff --git a/tests/ci-apt-install.test.ts b/tests/ci-apt-install.test.ts new file mode 100644 index 00000000..c0c2c2e4 --- /dev/null +++ b/tests/ci-apt-install.test.ts @@ -0,0 +1,127 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { APT_OPTIONS, ATTEMPT_DEADLINE_MS, aptCommand, aptInstall, KILL_AFTER_S, RETRY_DELAYS_MS, timeoutCommand, worstCaseMs } from "../ops/ci/apt-install"; + +const workflow = readFileSync(join(import.meta.dir, "../.github/workflows/checks.yml"), "utf8"); + +/** Fake apt-get that fails the given verb with `code` a set number of times, then succeeds. */ +function fakeApt(failVerb: "update" | "install", failures: number, code = 100) { + const calls: string[] = []; + let left = failures; + const run = async (args: string[]) => { + const verb = args.includes("update") ? "update" : "install"; + calls.push(verb); + if (verb === failVerb && left > 0) { left--; return code; } + return 0; + }; + return { calls, run }; +} + +function recordSleeps() { + const slept: number[] = []; + return { slept, sleep: async (ms: number) => { slept.push(ms); } }; +} + +const quiet = () => {}; + +test("a failing apt-get update is retried with backoff and then installs", async () => { + const apt = fakeApt("update", 2), clock = recordSleeps(); + await aptInstall(["ripgrep"], { run: apt.run, sleep: clock.sleep, log: quiet }); + expect(apt.calls).toEqual(["update", "update", "update", "install"]); + expect(clock.slept).toEqual([10_000, 30_000]); +}); + +test("an install that fails after a warning-only update is retried from update", async () => { + const apt = fakeApt("install", 1), clock = recordSleeps(); + await aptInstall(["ripgrep"], { run: apt.run, sleep: clock.sleep, log: quiet }); + expect(apt.calls).toEqual(["update", "install", "update", "install"]); + expect(clock.slept).toEqual([10_000]); +}); + +test("an install killed by its deadline (exit 124 or 137) is retried from update", async () => { + for (const code of [124, 137]) { + const apt = fakeApt("install", 1, code), clock = recordSleeps(), lines: string[] = []; + await aptInstall(["ripgrep"], { run: apt.run, sleep: clock.sleep, log: line => lines.push(line) }); + expect(apt.calls).toEqual(["update", "install", "update", "install"]); + expect(lines).toEqual([`apt-install: attempt 1 failed with exit ${code}; retrying in 10s`]); + } +}); + +test("a hanging update and install share one attempt deadline, and a spent budget skips install", async () => { + let t = 0; + const seen: Array<[string, number]> = []; + // Attempt 1: update hangs past the deadline and is killed. Attempt 2: update takes 50 s, install + // hangs and is killed when the attempt's 70 s run out. Attempt 3: update uses the whole budget, so + // install never starts. Attempt 4 succeeds quickly. + const plan = [ + { verb: "update", took: ATTEMPT_DEADLINE_MS + KILL_AFTER_S * 1000, code: 137 }, + { verb: "update", took: 50_000, code: 0 }, + { verb: "install", took: 20_000, code: 124 }, + { verb: "update", took: ATTEMPT_DEADLINE_MS, code: 0 }, + { verb: "update", took: 3_000, code: 0 }, + { verb: "install", took: 5_000, code: 0 }, + ]; + const run = async (args: string[], seconds: number) => { + const step = plan.shift()!; + expect(args.includes("update") ? "update" : "install").toBe(step.verb); + seen.push([step.verb, seconds]); + t += step.took; + return step.code; + }; + const lines: string[] = []; + await aptInstall(["ripgrep"], { run, sleep: async ms => { t += ms; }, now: () => t, log: line => lines.push(line) }); + expect(seen).toEqual([["update", 70], ["update", 70], ["install", 20], ["update", 70], ["update", 70], ["install", 67]]); + expect(lines.map(line => line.match(/exit (\d+)/)?.[1])).toEqual(["137", "124", "124"]); + expect(plan).toEqual([]); +}); + +test("the step still fails loudly once every retry is used", async () => { + const apt = fakeApt("update", 99), clock = recordSleeps(); + await expect(aptInstall(["ripgrep"], { run: apt.run, sleep: clock.sleep, log: quiet })).rejects.toThrow("apt_install_failed after 4 attempts (exit 100)"); + expect(apt.calls).toEqual(["update", "update", "update", "update"]); + expect(clock.slept).toEqual([10_000, 30_000, 60_000]); +}); + +test("an empty package list is refused instead of passing silently", async () => { + const apt = fakeApt("update", 0), clock = recordSleeps(); + await expect(aptInstall([], { run: apt.run, sleep: clock.sleep })).rejects.toThrow("apt_install_no_packages"); + expect(apt.calls).toEqual([]); +}); + +test("apt-get has no fetch retries of its own and installs exactly the requested packages", async () => { + const seen: string[][] = []; + await aptInstall(["ripgrep", "poppler-utils"], { run: async args => { seen.push(args); return 0; }, sleep: async () => {} }); + expect(seen).toEqual([ + [...APT_OPTIONS, "update"], + [...APT_OPTIONS, "install", "-y", "--no-install-recommends", "ripgrep", "poppler-utils"], + ]); + expect(APT_OPTIONS).toContain("Acquire::Retries=0"); + expect(APT_OPTIONS).toContain("Acquire::https::Timeout=30"); +}); + +test("timeout runs under sudo so its signals reach apt-get, not only sudo", () => { + expect(aptCommand(["update"], 70)).toEqual(["sudo", "timeout", `--kill-after=${KILL_AFTER_S}`, "70", "apt-get", "update"]); +}); + +test("the timeout wrapper kills a child that ignores SIGTERM", async () => { + const started = performance.now(); + const code = await Bun.spawn(timeoutCommand(1, ["sh", "-c", "trap '' TERM; sleep 30"]), { stdout: "ignore", stderr: "ignore" }).exited; + expect(code).toBe(137); + expect(performance.now() - started).toBeLessThan((1 + KILL_AFTER_S + 3) * 1000); +}, 15_000); + +test("the worst-case apt budget plus 6 min of setup and tests fits the tests job timeout", () => { + const job = workflow.slice(workflow.indexOf("\n tests:\n"), workflow.indexOf("\n deterministic:\n")); + const minutes = Number(job.match(/\n timeout-minutes: (\d+)\n/)?.[1]); + expect(minutes).toBe(15); + expect(worstCaseMs()).toBe(4 * (70_000 + 5_000 + 2_000) + 100_000); + expect(worstCaseMs()).toBeLessThanOrEqual(7 * 60_000); + expect(worstCaseMs() + 6 * 60_000).toBeLessThanOrEqual(minutes * 60_000); +}); + +test("the CI tests job installs its packages through the retrying helper", () => { + expect(workflow).toContain("- run: bun ops/ci/apt-install.ts ripgrep python3-pil poppler-utils"); + expect(workflow).not.toContain("sudo apt-get"); + expect(RETRY_DELAYS_MS).toHaveLength(3); +});