fix(runner): harden disk GC to hourly + age-gated orphan reap

internal/templates/runner.go

@@ -195,22 +195,23 @@ profile rootlesskit /usr/local/bin/rootlesskit flags=(unconfined) {
 	// the runner disk fills to 100% and the actions-runner service crash-loops
 	// (it cannot even copy run-helper.sh), so GitHub marks the runner offline.
 	// vee-runner-gc.sh reclaims that space. It runs as the runner user with the
-	// rootless socket env sourced from runner.env, so it talks to the same
+	// rootless socket env derived from the runner's UID, so it talks to the same
 	// containerd/BuildKit the jobs use.
 	//
-	// It is gated on no job being in progress: the actions-runner worker writes
-	// .runner files under _diag and holds a Runner.Worker process while a job
-	// executes, so GC checks for a live Runner.Worker and skips if one is found.
-	// That keeps GC from deleting an in-flight job's images mid-build.
+	// Correctness against an in-flight job is enforced by AGE-GATING every
+	// destructive action, NOT by skipping the whole run when a job is present.
+	// The earlier "skip if any Runner.Worker is alive" guard was a foot-gun: on a
+	// busy runner the timer keeps firing while a job runs, GC keeps skipping, and
+	// disk usage grows unbounded until the runner falls over (observed: a runner
+	// went offline with the disk at 100% after GC had effectively not run for
+	// days). Instead the reap below only removes containers older than a generous
+	// ceiling — a real job's containers are always younger than that — so stale
+	// stacks that a canceled or crashed job left behind (nerdctl's own
+	// `system prune` never reaps them because they are still "Up") are collected
+	// even while an unrelated job runs, and the live job is never touched.
 	runnerGC := `#!/usr/bin/env bash
 set -uo pipefail
 
-# Skip if a job is currently running, so we never prune an in-flight build.
-if pgrep -u runner -f 'Runner.Worker' >/dev/null 2>&1; then
-  echo "vee-runner-gc: job in progress, skipping"
-  exit 0
-fi
-
 # Derive the rootless socket env from the runner's own UID rather than reading
 # /etc/actions-runner/runner.env: that file is root-owned 0600 and unreadable by
 # the runner user this script runs as. The UID-keyed paths are deterministic.
@@ -221,6 +222,37 @@ export BUILDKIT_HOST="unix:///run/user/${RUNNER_UID}/buildkit/buildkitd.sock"
 export CONTAINERD_NAMESPACE=default
 export PATH=/usr/local/bin:/usr/bin:/bin
 
+# Any container older than this is treated as an orphan and force-removed even if
+# still "Up". A real CI job's containers live far under this; the ceiling only
+# needs to exceed the slowest legitimate job. 90 minutes is generous headroom.
+ORPHAN_MAX_AGE_SEC=$((90 * 60))
+NOW=$(date +%s)
+
+# Force-reap stale containers even if "Up". nerdctl's own 'system prune' only
+# removes STOPPED containers, so a stack a canceled/crashed job left running is
+# never collected — that is the primary disk-growth source. Age-gating by each
+# container's StartedAt guarantees the current job's young containers survive, so
+# this is safe to run with a job in progress and needs no name allow-list.
+echo "vee-runner-gc: reaping containers older than ${ORPHAN_MAX_AGE_SEC}s"
+for id in $(nerdctl ps -aq 2>/dev/null); do
+  name=$(nerdctl inspect --format '{{.Name}}' "${id}" 2>/dev/null) || continue
+  started=$(nerdctl inspect --format '{{.State.StartedAt}}' "${id}" 2>/dev/null)
+  if [ -n "${started}" ] && [ "${started}" != "0001-01-01T00:00:00Z" ]; then
+    started_epoch=$(date -d "${started}" +%s 2>/dev/null || echo 0)
+  else
+    # Created but never started — always safe to drop; treat as maximally old.
+    started_epoch=0
+  fi
+  age=$((NOW - started_epoch))
+  if [ "${started_epoch}" -eq 0 ] || [ "${age}" -ge "${ORPHAN_MAX_AGE_SEC}" ]; then
+    echo "vee-runner-gc: reaping ${name} (${id}, age ${age}s)"
+    nerdctl rm -f "${id}" >/dev/null 2>&1 || true
+  fi
+done
+
+# Standard prune of whatever is now stopped/dangling/unreferenced. Safe with a
+# job in progress: prune only touches stopped containers, dangling images, and
+# unused volumes/networks — an active build's resources are in use and skipped.
 echo "vee-runner-gc: pruning containerd/nerdctl"
 nerdctl system prune -af  || true
 nerdctl volume prune -af  || true
@@ -232,9 +264,17 @@ echo "vee-runner-gc: pruning BuildKit cache to 2GB ceiling"
 buildctl prune --keep-storage 2048 || true
 
 # Trim a runaway Go build cache (cgo/native CI builds), bounded not cleared.
-GOCACHE_DIR="$HOME/.cache/go-build"
-if [ -d "$GOCACHE_DIR" ]; then
-  GOCACHE=$GOCACHE_DIR go clean -cache 2>/dev/null || rm -rf "${GOCACHE_DIR:?}/"* || true
+# Only when no job is running: a live build writes here and clearing it mid-build
+# is a pointless cache-cold penalty (not a correctness problem — the age-gated
+# reap above is what protects an in-flight job). This is the one place a live
+# Runner.Worker is worth checking for.
+if pgrep -u runner -f 'Runner.Worker' >/dev/null 2>&1; then
+  echo "vee-runner-gc: job in progress, leaving go-build cache warm"
+else
+  GOCACHE_DIR="$HOME/.cache/go-build"
+  if [ -d "$GOCACHE_DIR" ]; then
+    GOCACHE=$GOCACHE_DIR go clean -cache 2>/dev/null || rm -rf "${GOCACHE_DIR:?}/"* || true
+  fi
 fi
 
 # Old runner diagnostics and stale per-job temp dirs.
@@ -255,15 +295,17 @@ User=runner
 ExecStart=/usr/local/bin/vee-runner-gc.sh
 `
 
+	// Hourly, not daily: a canceled CI job can leak a still-running stack mid-day,
+	// and a daily cadence lets orphans accumulate for up to 24h between runs.
 	// Persistent=true so a GC missed while the VM was powered off runs at the
-	// next boot rather than silently skipping a day.
+	// next boot rather than silently skipping an interval.
 	runnerGCTimer := `[Unit]
-Description=Run vee runner disk garbage collection daily
+Description=Run vee runner disk garbage collection hourly
 
 [Timer]
-OnCalendar=daily
+OnCalendar=hourly
 Persistent=true
-RandomizedDelaySec=600
+RandomizedDelaySec=120
 
 [Install]
 WantedBy=timers.target
@@ -421,7 +463,7 @@ WantedBy=timers.target
 		"ufw --force enable",
 		"systemctl enable --now vee-ssh-agent",
 		"systemctl enable --now qemu-guest-agent",
-		// Daily disk GC so CI build leftovers can't fill the disk and knock the
+		// Hourly disk GC so CI build leftovers can't fill the disk and knock the
 		// runner offline (see vee-runner-gc.sh above).
 		"systemctl enable --now vee-runner-gc.timer",
 	)

internal/templates/runner_test.go

@@ -94,7 +94,7 @@ func TestGitHubRunnerCloudInit(t *testing.T) {
 	if !strings.Contains(joined, "systemctl enable --now vee-runner-gc.timer") {
 		t.Error("runcmd missing vee-runner-gc.timer enable")
 	}
-	var gcScript string
+	var gcScript, gcTimer string
 	haveGCService, haveGCTimer := false, false
 	for _, f := range files {
 		switch f.Path {
@@ -106,6 +106,7 @@ func TestGitHubRunnerCloudInit(t *testing.T) {
 		case "/etc/systemd/system/vee-runner-gc.service":
 			haveGCService = true
 		case "/etc/systemd/system/vee-runner-gc.timer":
+			gcTimer = f.Content
 			haveGCTimer = true
 		}
 	}
@@ -118,12 +119,8 @@ func TestGitHubRunnerCloudInit(t *testing.T) {
 	if !haveGCTimer {
 		t.Error("no vee-runner-gc.timer write_file")
 	}
-	// GC must derive the rootless env from its own UID (runner.env is
-	// root-owned 0600 and unreadable by the runner user GC runs as) and refuse
-	// to run during a job.
-	if !strings.Contains(gcScript, "Runner.Worker") {
-		t.Error("GC script does not guard against in-progress jobs (Runner.Worker check)")
-	}
+	// GC must derive the rootless env from its own UID: runner.env is root-owned
+	// 0600 and unreadable by the runner user GC runs as.
 	if strings.Contains(gcScript, ". /etc/actions-runner/runner.env") {
 		t.Error("GC script sources root-owned runner.env — must derive env from id -u instead")
 	}
@@ -133,6 +130,35 @@ func TestGitHubRunnerCloudInit(t *testing.T) {
 	if !strings.Contains(gcScript, "nerdctl system prune") {
 		t.Error("GC script does not prune nerdctl")
 	}
+	// The whole GC run must NOT be skipped when a job is in progress — that guard
+	// let orphaned stacks accumulate until the disk filled. The Runner.Worker
+	// check may only survive to gate the go-build cache clear, never as an early
+	// `exit 0` for the entire script.
+	if strings.Contains(gcScript, "job in progress, skipping") {
+		t.Error("GC script still skips the whole run when a job is in progress — reap must be age-gated instead")
+	}
+	// Stale containers left "Up" by a canceled/crashed job are the primary
+	// disk-growth source and nerdctl's own prune never reaps them; GC must
+	// force-remove containers past an age ceiling even while running.
+	if !strings.Contains(gcScript, "ORPHAN_MAX_AGE_SEC") {
+		t.Error("GC script does not age-reap orphaned containers (no ORPHAN_MAX_AGE_SEC)")
+	}
+	if !strings.Contains(gcScript, "nerdctl rm -f") {
+		t.Error("GC script does not force-remove stale containers")
+	}
+	// The Runner.Worker check must remain, but only to keep the go-build cache
+	// warm during a job — not to skip the whole run.
+	if !strings.Contains(gcScript, "Runner.Worker") {
+		t.Error("GC script lost the Runner.Worker check that keeps the go-build cache warm during a job")
+	}
+	// Cadence must be hourly, not daily: a daily timer lets orphans accumulate
+	// for up to 24h between runs on a busy runner.
+	if !strings.Contains(gcTimer, "OnCalendar=hourly") {
+		t.Errorf("GC timer must run hourly, got:\n%s", gcTimer)
+	}
+	if strings.Contains(gcTimer, "OnCalendar=daily") {
+		t.Error("GC timer still set to daily — must be hourly")
+	}
 }
 
 // TestGitHubRunnerRestore verifies that supplying restored credential files