From c21dc9679d9dfe2a7cb62c185e4388e7930c89c9 Mon Sep 17 00:00:00 2001 From: chayapats Date: Thu, 12 Nov 2026 01:03:04 +0700 Subject: [PATCH 1/3] ci: retry the tests job apt install with backoff (#456) --- .github/workflows/checks.yml | 3 +- ops/ci/apt-install.ts | 36 ++++++++++++++++++++ tests/ci-apt-install.test.ts | 65 ++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 ops/ci/apt-install.ts create mode 100644 tests/ci-apt-install.test.ts diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 34f3f46f..c4a7c9b9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -54,7 +54,8 @@ jobs: key: bun-${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('bun.lock') }} - run: bun install --frozen-lockfile # Fake-runtime parser fixtures use local Pillow/Poppler; no image is built and no Podman is run. - - run: sudo apt-get update -q && sudo apt-get install -y -q --no-install-recommends ripgrep python3-pil poppler-utils + # The helper retries apt with backoff so a short runner DNS outage does not fail the shard (#456). + - run: bun ops/ci/apt-install.ts ripgrep python3-pil poppler-utils # A shared runner under parallel load can exceed Bun's default 5 s per test. - run: bun test --timeout 30000 $(bun ops/ci/test-shard.ts ${{ matrix.shard }} 6) env: diff --git a/ops/ci/apt-install.ts b/ops/ci/apt-install.ts new file mode 100644 index 00000000..32f85626 --- /dev/null +++ b/ops/ci/apt-install.ts @@ -0,0 +1,36 @@ +/** + * Installs the CI apt packages with bounded retries. A short runner DNS or mirror outage + * ("Temporary failure resolving archive.ubuntu.com", #456) then does not fail a shard before + * its tests start. Once the retries run out the step still fails loudly; it never skips packages. + */ + +/** Runs one apt-get invocation and returns its exit code. */ +export type AptRun = (args: string[]) => Promise; + +/** Waits between attempts: 10 s, 30 s, then 60 s, so four attempts in total. */ +export const RETRY_DELAYS_MS = [10_000, 30_000, 60_000]; + +/** apt's own per-download retries and a timeout, so one slow mirror cannot use up the job timeout. */ +export const APT_OPTIONS = ["-q", "-o", "Acquire::Retries=5", "-o", "Acquire::http::Timeout=30"]; + +export async function aptInstall(packages: string[], run: AptRun, sleep: (ms: number) => Promise, delays = RETRY_DELAYS_MS, log: (line: string) => void = line => console.error(line)): Promise { + if (packages.length === 0) throw new Error("apt_install_no_packages"); + for (let attempt = 1; ; attempt++) { + // `update` can exit 0 when it only warns about unreachable mirrors, so install failures retry too. + const update = await run([...APT_OPTIONS, "update"]); + const code = update === 0 ? await run([...APT_OPTIONS, "install", "-y", "--no-install-recommends", ...packages]) : update; + if (code === 0) return; + const delay = delays[attempt - 1]; + if (delay === undefined) throw new Error(`apt_install_failed after ${attempt} attempts (exit ${code})`); + log(`apt-install: attempt ${attempt} failed with exit ${code}; retrying in ${delay / 1000}s`); + await sleep(delay); + } +} + +if (import.meta.main) { + const run: AptRun = args => Bun.spawn(["sudo", "apt-get", ...args], { stdout: "inherit", stderr: "inherit" }).exited; + await aptInstall(process.argv.slice(2), run, ms => Bun.sleep(ms)).catch(error => { + console.error(`apt-install: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + }); +} diff --git a/tests/ci-apt-install.test.ts b/tests/ci-apt-install.test.ts new file mode 100644 index 00000000..acdbee16 --- /dev/null +++ b/tests/ci-apt-install.test.ts @@ -0,0 +1,65 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { APT_OPTIONS, aptInstall, RETRY_DELAYS_MS } from "../ops/ci/apt-install"; + +/** Fake apt-get that fails the given verb a set number of times, then succeeds. */ +function fakeApt(failVerb: "update" | "install", failures: number) { + const calls: string[] = []; + let left = failures; + const run = async (args: string[]) => { + const verb = args.includes("update") ? "update" : "install"; + calls.push(verb); + if (verb === failVerb && left > 0) { left--; return 100; } + return 0; + }; + return { calls, run }; +} + +function recordSleeps() { + const slept: number[] = []; + return { slept, sleep: async (ms: number) => { slept.push(ms); } }; +} + +test("a failing apt-get update is retried with backoff and then installs", async () => { + const apt = fakeApt("update", 2), clock = recordSleeps(); + await aptInstall(["ripgrep"], apt.run, clock.sleep, RETRY_DELAYS_MS, () => {}); + expect(apt.calls).toEqual(["update", "update", "update", "install"]); + expect(clock.slept).toEqual([10_000, 30_000]); +}); + +test("an install that fails after a warning-only update is retried from update", async () => { + const apt = fakeApt("install", 1), clock = recordSleeps(); + await aptInstall(["ripgrep"], apt.run, clock.sleep, RETRY_DELAYS_MS, () => {}); + expect(apt.calls).toEqual(["update", "install", "update", "install"]); + expect(clock.slept).toEqual([10_000]); +}); + +test("the step still fails loudly once every retry is used", async () => { + const apt = fakeApt("update", 99), clock = recordSleeps(); + await expect(aptInstall(["ripgrep"], apt.run, clock.sleep, RETRY_DELAYS_MS, () => {})).rejects.toThrow("apt_install_failed after 4 attempts (exit 100)"); + expect(apt.calls).toEqual(["update", "update", "update", "update"]); + expect(clock.slept).toEqual([10_000, 30_000, 60_000]); +}); + +test("an empty package list is refused instead of passing silently", async () => { + const apt = fakeApt("update", 0), clock = recordSleeps(); + await expect(aptInstall([], apt.run, clock.sleep)).rejects.toThrow("apt_install_no_packages"); + expect(apt.calls).toEqual([]); +}); + +test("apt-get gets its own fetch retries and installs exactly the requested packages", async () => { + const seen: string[][] = []; + await aptInstall(["ripgrep", "poppler-utils"], async args => { seen.push(args); return 0; }, async () => {}); + expect(seen).toEqual([ + [...APT_OPTIONS, "update"], + [...APT_OPTIONS, "install", "-y", "--no-install-recommends", "ripgrep", "poppler-utils"], + ]); + expect(APT_OPTIONS).toContain("Acquire::Retries=5"); +}); + +test("the CI tests job installs its packages through the retrying helper", () => { + const workflow = readFileSync(join(import.meta.dir, "../.github/workflows/checks.yml"), "utf8"); + expect(workflow).toContain("- run: bun ops/ci/apt-install.ts ripgrep python3-pil poppler-utils"); + expect(workflow).not.toContain("sudo apt-get"); +}); From 13640227871d11aa917c706412b29ea12305fd78 Mon Sep 17 00:00:00 2001 From: chayapats Date: Thu, 12 Nov 2026 01:10:26 +0700 Subject: [PATCH 2/3] ci: document the aptInstall retry contract (#456) --- ops/ci/apt-install.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ops/ci/apt-install.ts b/ops/ci/apt-install.ts index 32f85626..f9bcdbbe 100644 --- a/ops/ci/apt-install.ts +++ b/ops/ci/apt-install.ts @@ -13,6 +13,13 @@ export const RETRY_DELAYS_MS = [10_000, 30_000, 60_000]; /** apt's own per-download retries and a timeout, so one slow mirror cannot use up the job timeout. */ export const APT_OPTIONS = ["-q", "-o", "Acquire::Retries=5", "-o", "Acquire::http::Timeout=30"]; +/** + * Installs every requested package, running `update` before each `install` attempt. + * Defaults to four attempts with 10 s, 30 s and 60 s waits after failures; a failed install + * restarts from `update` on the next attempt. Rejects with `apt_install_no_packages` for an empty + * list, with `apt_install_failed after N attempts (exit C)` once every attempt fails, or with + * whatever `run` or `sleep` rejects with. + */ export async function aptInstall(packages: string[], run: AptRun, sleep: (ms: number) => Promise, delays = RETRY_DELAYS_MS, log: (line: string) => void = line => console.error(line)): Promise { if (packages.length === 0) throw new Error("apt_install_no_packages"); for (let attempt = 1; ; attempt++) { From 639feabd9d31a8cfa1be5766b4062022aa5c7b08 Mon Sep 17 00:00:00 2001 From: chayapats Date: Thu, 12 Nov 2026 01:28:20 +0700 Subject: [PATCH 3/3] ci: cap each apt attempt with a wall-clock deadline (#456) Each attempt (update + install) now runs under sudo timeout --kill-after=5, sharing a 70 s budget, so a resolver-bound apt walk cannot fill the 15-minute job. Drop Acquire::Retries (outer loop is the retry), add the https fetch timeout, and test the worst case against checks.yml timeout-minutes. --- .github/workflows/checks.yml | 1 + ops/ci/apt-install.ts | 76 ++++++++++++++++++++++++------- tests/ci-apt-install.test.ts | 86 +++++++++++++++++++++++++++++++----- 3 files changed, 136 insertions(+), 27 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c4a7c9b9..d96b2876 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -55,6 +55,7 @@ jobs: - run: bun install --frozen-lockfile # Fake-runtime parser fixtures use local Pillow/Poppler; no image is built and no Podman is run. # The helper retries apt with backoff so a short runner DNS outage does not fail the shard (#456). + # Each attempt is killed after 70 s; the worst case (~6m48s) leaves room for the tests in 15 min. - run: bun ops/ci/apt-install.ts ripgrep python3-pil poppler-utils # A shared runner under parallel load can exceed Bun's default 5 s per test. - run: bun test --timeout 30000 $(bun ops/ci/test-shard.ts ${{ matrix.shard }} 6) diff --git a/ops/ci/apt-install.ts b/ops/ci/apt-install.ts index f9bcdbbe..f40d64c2 100644 --- a/ops/ci/apt-install.ts +++ b/ops/ci/apt-install.ts @@ -1,31 +1,77 @@ /** * Installs the CI apt packages with bounded retries. A short runner DNS or mirror outage * ("Temporary failure resolving archive.ubuntu.com", #456) then does not fail a shard before - * its tests start. Once the retries run out the step still fails loudly; it never skips packages. + * its tests start. Each attempt has a wall-clock deadline, so a resolver-bound apt walk cannot + * use up the job timeout. Once the retries run out the step still fails loudly; it never skips packages. */ -/** Runs one apt-get invocation and returns its exit code. */ -export type AptRun = (args: string[]) => Promise; +/** Runs one apt-get invocation, killed after `seconds`, and returns its exit code (124/137 on timeout). */ +export type AptRun = (args: string[], seconds: number) => Promise; /** Waits between attempts: 10 s, 30 s, then 60 s, so four attempts in total. */ export const RETRY_DELAYS_MS = [10_000, 30_000, 60_000]; -/** apt's own per-download retries and a timeout, so one slow mirror cannot use up the job timeout. */ -export const APT_OPTIONS = ["-q", "-o", "Acquire::Retries=5", "-o", "Acquire::http::Timeout=30"]; +/** Wall-clock budget for one attempt (`update` plus `install`). A healthy runner needs about 10 s. */ +export const ATTEMPT_DEADLINE_MS = 70_000; + +/** Seconds `timeout` waits after SIGTERM before it sends SIGKILL to apt-get. */ +export const KILL_AFTER_S = 5; + +/** Exit code a timed-out attempt reports when its budget is already spent before `install` starts. */ +export const TIMED_OUT = 124; + +/** + * No apt-level fetch retries (the outer loop is the retry) and a fetch timeout for both schemes. + * The timeout only helps a stalled connection; the per-attempt deadline is the real cap. + */ +export const APT_OPTIONS = ["-q", "-o", "Acquire::Retries=0", "-o", "Acquire::http::Timeout=30", "-o", "Acquire::https::Timeout=30"]; + +/** Wraps a command in GNU `timeout`: SIGTERM after `seconds`, SIGKILL `KILL_AFTER_S` later. */ +export function timeoutCommand(seconds: number, argv: string[]): string[] { + return ["timeout", `--kill-after=${KILL_AFTER_S}`, String(seconds), ...argv]; +} + +/** `timeout` runs under sudo, so its signals reach apt-get itself and not only sudo. */ +export function aptCommand(args: string[], seconds: number): string[] { + return ["sudo", ...timeoutCommand(seconds, ["apt-get", ...args])]; +} + +/** + * Upper bound on how long `aptInstall` can run with the given delays: every attempt hits its + * deadline, pays the SIGKILL grace and up to 2 s of whole-second rounding, then every wait follows. + */ +export function worstCaseMs(delays = RETRY_DELAYS_MS, deadlineMs = ATTEMPT_DEADLINE_MS): number { + const attempts = delays.length + 1; + return attempts * (deadlineMs + KILL_AFTER_S * 1000 + 2000) + delays.reduce((sum, ms) => sum + ms, 0); +} + +export interface AptInstallOptions { + run: AptRun; + sleep: (ms: number) => Promise; + now?: () => number; + delays?: number[]; + deadlineMs?: number; + log?: (line: string) => void; +} /** - * Installs every requested package, running `update` before each `install` attempt. - * Defaults to four attempts with 10 s, 30 s and 60 s waits after failures; a failed install - * restarts from `update` on the next attempt. Rejects with `apt_install_no_packages` for an empty - * list, with `apt_install_failed after N attempts (exit C)` once every attempt fails, or with - * whatever `run` or `sleep` rejects with. + * Installs every requested package, running `update` before each `install` attempt. Both share one + * attempt deadline: `install` only gets what `update` left. Defaults to four attempts with 10 s, + * 30 s and 60 s waits after failures; a failed or timed-out install restarts from `update` on the + * next attempt. Rejects with `apt_install_no_packages` for an empty list, with + * `apt_install_failed after N attempts (exit C)` once every attempt fails, or with whatever `run` + * or `sleep` rejects with. */ -export async function aptInstall(packages: string[], run: AptRun, sleep: (ms: number) => Promise, delays = RETRY_DELAYS_MS, log: (line: string) => void = line => console.error(line)): Promise { +export async function aptInstall(packages: string[], options: AptInstallOptions): Promise { + const { run, sleep, now = Date.now, delays = RETRY_DELAYS_MS, deadlineMs = ATTEMPT_DEADLINE_MS, log = line => console.error(line) } = options; if (packages.length === 0) throw new Error("apt_install_no_packages"); for (let attempt = 1; ; attempt++) { + const deadline = now() + deadlineMs; + const left = () => Math.ceil((deadline - now()) / 1000); // `update` can exit 0 when it only warns about unreachable mirrors, so install failures retry too. - const update = await run([...APT_OPTIONS, "update"]); - const code = update === 0 ? await run([...APT_OPTIONS, "install", "-y", "--no-install-recommends", ...packages]) : update; + const update = await run([...APT_OPTIONS, "update"], left()); + let code = update; + if (update === 0) code = left() > 0 ? await run([...APT_OPTIONS, "install", "-y", "--no-install-recommends", ...packages], left()) : TIMED_OUT; if (code === 0) return; const delay = delays[attempt - 1]; if (delay === undefined) throw new Error(`apt_install_failed after ${attempt} attempts (exit ${code})`); @@ -35,8 +81,8 @@ export async function aptInstall(packages: string[], run: AptRun, sleep: (ms: nu } if (import.meta.main) { - const run: AptRun = args => Bun.spawn(["sudo", "apt-get", ...args], { stdout: "inherit", stderr: "inherit" }).exited; - await aptInstall(process.argv.slice(2), run, ms => Bun.sleep(ms)).catch(error => { + const run: AptRun = (args, seconds) => Bun.spawn(aptCommand(args, seconds), { stdout: "inherit", stderr: "inherit" }).exited; + await aptInstall(process.argv.slice(2), { run, sleep: ms => Bun.sleep(ms) }).catch(error => { console.error(`apt-install: ${error instanceof Error ? error.message : String(error)}`); process.exit(1); }); diff --git a/tests/ci-apt-install.test.ts b/tests/ci-apt-install.test.ts index acdbee16..c0c2c2e4 100644 --- a/tests/ci-apt-install.test.ts +++ b/tests/ci-apt-install.test.ts @@ -1,16 +1,18 @@ import { expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { APT_OPTIONS, aptInstall, RETRY_DELAYS_MS } from "../ops/ci/apt-install"; +import { APT_OPTIONS, ATTEMPT_DEADLINE_MS, aptCommand, aptInstall, KILL_AFTER_S, RETRY_DELAYS_MS, timeoutCommand, worstCaseMs } from "../ops/ci/apt-install"; -/** Fake apt-get that fails the given verb a set number of times, then succeeds. */ -function fakeApt(failVerb: "update" | "install", failures: number) { +const workflow = readFileSync(join(import.meta.dir, "../.github/workflows/checks.yml"), "utf8"); + +/** Fake apt-get that fails the given verb with `code` a set number of times, then succeeds. */ +function fakeApt(failVerb: "update" | "install", failures: number, code = 100) { const calls: string[] = []; let left = failures; const run = async (args: string[]) => { const verb = args.includes("update") ? "update" : "install"; calls.push(verb); - if (verb === failVerb && left > 0) { left--; return 100; } + if (verb === failVerb && left > 0) { left--; return code; } return 0; }; return { calls, run }; @@ -21,45 +23,105 @@ function recordSleeps() { return { slept, sleep: async (ms: number) => { slept.push(ms); } }; } +const quiet = () => {}; + test("a failing apt-get update is retried with backoff and then installs", async () => { const apt = fakeApt("update", 2), clock = recordSleeps(); - await aptInstall(["ripgrep"], apt.run, clock.sleep, RETRY_DELAYS_MS, () => {}); + await aptInstall(["ripgrep"], { run: apt.run, sleep: clock.sleep, log: quiet }); expect(apt.calls).toEqual(["update", "update", "update", "install"]); expect(clock.slept).toEqual([10_000, 30_000]); }); test("an install that fails after a warning-only update is retried from update", async () => { const apt = fakeApt("install", 1), clock = recordSleeps(); - await aptInstall(["ripgrep"], apt.run, clock.sleep, RETRY_DELAYS_MS, () => {}); + await aptInstall(["ripgrep"], { run: apt.run, sleep: clock.sleep, log: quiet }); expect(apt.calls).toEqual(["update", "install", "update", "install"]); expect(clock.slept).toEqual([10_000]); }); +test("an install killed by its deadline (exit 124 or 137) is retried from update", async () => { + for (const code of [124, 137]) { + const apt = fakeApt("install", 1, code), clock = recordSleeps(), lines: string[] = []; + await aptInstall(["ripgrep"], { run: apt.run, sleep: clock.sleep, log: line => lines.push(line) }); + expect(apt.calls).toEqual(["update", "install", "update", "install"]); + expect(lines).toEqual([`apt-install: attempt 1 failed with exit ${code}; retrying in 10s`]); + } +}); + +test("a hanging update and install share one attempt deadline, and a spent budget skips install", async () => { + let t = 0; + const seen: Array<[string, number]> = []; + // Attempt 1: update hangs past the deadline and is killed. Attempt 2: update takes 50 s, install + // hangs and is killed when the attempt's 70 s run out. Attempt 3: update uses the whole budget, so + // install never starts. Attempt 4 succeeds quickly. + const plan = [ + { verb: "update", took: ATTEMPT_DEADLINE_MS + KILL_AFTER_S * 1000, code: 137 }, + { verb: "update", took: 50_000, code: 0 }, + { verb: "install", took: 20_000, code: 124 }, + { verb: "update", took: ATTEMPT_DEADLINE_MS, code: 0 }, + { verb: "update", took: 3_000, code: 0 }, + { verb: "install", took: 5_000, code: 0 }, + ]; + const run = async (args: string[], seconds: number) => { + const step = plan.shift()!; + expect(args.includes("update") ? "update" : "install").toBe(step.verb); + seen.push([step.verb, seconds]); + t += step.took; + return step.code; + }; + const lines: string[] = []; + await aptInstall(["ripgrep"], { run, sleep: async ms => { t += ms; }, now: () => t, log: line => lines.push(line) }); + expect(seen).toEqual([["update", 70], ["update", 70], ["install", 20], ["update", 70], ["update", 70], ["install", 67]]); + expect(lines.map(line => line.match(/exit (\d+)/)?.[1])).toEqual(["137", "124", "124"]); + expect(plan).toEqual([]); +}); + test("the step still fails loudly once every retry is used", async () => { const apt = fakeApt("update", 99), clock = recordSleeps(); - await expect(aptInstall(["ripgrep"], apt.run, clock.sleep, RETRY_DELAYS_MS, () => {})).rejects.toThrow("apt_install_failed after 4 attempts (exit 100)"); + await expect(aptInstall(["ripgrep"], { run: apt.run, sleep: clock.sleep, log: quiet })).rejects.toThrow("apt_install_failed after 4 attempts (exit 100)"); expect(apt.calls).toEqual(["update", "update", "update", "update"]); expect(clock.slept).toEqual([10_000, 30_000, 60_000]); }); test("an empty package list is refused instead of passing silently", async () => { const apt = fakeApt("update", 0), clock = recordSleeps(); - await expect(aptInstall([], apt.run, clock.sleep)).rejects.toThrow("apt_install_no_packages"); + await expect(aptInstall([], { run: apt.run, sleep: clock.sleep })).rejects.toThrow("apt_install_no_packages"); expect(apt.calls).toEqual([]); }); -test("apt-get gets its own fetch retries and installs exactly the requested packages", async () => { +test("apt-get has no fetch retries of its own and installs exactly the requested packages", async () => { const seen: string[][] = []; - await aptInstall(["ripgrep", "poppler-utils"], async args => { seen.push(args); return 0; }, async () => {}); + await aptInstall(["ripgrep", "poppler-utils"], { run: async args => { seen.push(args); return 0; }, sleep: async () => {} }); expect(seen).toEqual([ [...APT_OPTIONS, "update"], [...APT_OPTIONS, "install", "-y", "--no-install-recommends", "ripgrep", "poppler-utils"], ]); - expect(APT_OPTIONS).toContain("Acquire::Retries=5"); + expect(APT_OPTIONS).toContain("Acquire::Retries=0"); + expect(APT_OPTIONS).toContain("Acquire::https::Timeout=30"); +}); + +test("timeout runs under sudo so its signals reach apt-get, not only sudo", () => { + expect(aptCommand(["update"], 70)).toEqual(["sudo", "timeout", `--kill-after=${KILL_AFTER_S}`, "70", "apt-get", "update"]); +}); + +test("the timeout wrapper kills a child that ignores SIGTERM", async () => { + const started = performance.now(); + const code = await Bun.spawn(timeoutCommand(1, ["sh", "-c", "trap '' TERM; sleep 30"]), { stdout: "ignore", stderr: "ignore" }).exited; + expect(code).toBe(137); + expect(performance.now() - started).toBeLessThan((1 + KILL_AFTER_S + 3) * 1000); +}, 15_000); + +test("the worst-case apt budget plus 6 min of setup and tests fits the tests job timeout", () => { + const job = workflow.slice(workflow.indexOf("\n tests:\n"), workflow.indexOf("\n deterministic:\n")); + const minutes = Number(job.match(/\n timeout-minutes: (\d+)\n/)?.[1]); + expect(minutes).toBe(15); + expect(worstCaseMs()).toBe(4 * (70_000 + 5_000 + 2_000) + 100_000); + expect(worstCaseMs()).toBeLessThanOrEqual(7 * 60_000); + expect(worstCaseMs() + 6 * 60_000).toBeLessThanOrEqual(minutes * 60_000); }); test("the CI tests job installs its packages through the retrying helper", () => { - const workflow = readFileSync(join(import.meta.dir, "../.github/workflows/checks.yml"), "utf8"); expect(workflow).toContain("- run: bun ops/ci/apt-install.ts ripgrep python3-pil poppler-utils"); expect(workflow).not.toContain("sudo apt-get"); + expect(RETRY_DELAYS_MS).toHaveLength(3); });