fix(runner): cap the journal and bound the go-build cache by size

internal/templates/runner.go

@@ -263,20 +263,66 @@ nerdctl builder prune -af || true
 echo "vee-runner-gc: pruning BuildKit cache to 2GB ceiling"
 buildctl prune --keep-storage 2048 || true
 
-# Trim a runaway Go build cache (cgo/native CI builds), bounded not cleared.
-# Only when no job is running: a live build writes here and clearing it mid-build
-# is a pointless cache-cold penalty (not a correctness problem — the age-gated
-# reap above is what protects an in-flight job). This is the one place a live
-# Runner.Worker is worth checking for.
-if pgrep -u runner -f 'Runner.Worker' >/dev/null 2>&1; then
-  echo "vee-runner-gc: job in progress, leaving go-build cache warm"
-else
-  GOCACHE_DIR="$HOME/.cache/go-build"
-  if [ -d "$GOCACHE_DIR" ]; then
-    GOCACHE=$GOCACHE_DIR go clean -cache 2>/dev/null || rm -rf "${GOCACHE_DIR:?}/"* || true
+# Bound the Go build cache by SIZE rather than clearing it wholesale. The old
+# behaviour was all-or-nothing — skip entirely while a job ran, otherwise wipe
+# every entry — which meant the cache either grew unchecked on a busy runner or
+# went fully cold on an idle one. GOCACHEPROG-free trimming is not exposed by
+# the go tool, so evict least-recently-used entries directly: the cache is
+# content-addressed under two-hex-digit subdirectories, and its files carry
+# atime-like mtimes that the go tool refreshes on use, so dropping the oldest
+# files first approximates Go's own LRU trimming closely enough for a CI box.
+#
+# Runs even with a job in progress: evicting a cold entry only costs that entry
+# a recompile, and the cache is designed to tolerate arbitrary eviction (a miss
+# is a rebuild, never a failure). Only entries untouched for GOCACHE_MIN_AGE_MIN
+# are eligible, so a live build's working set is never evicted.
+#
+# The age floor is in MINUTES, not days. A day-granularity guard (-mtime +1) is
+# useless here: a busy runner rewrites the whole cache within a single day, so
+# nothing is ever a day old and the ceiling can never be enforced — observed on
+# this runner with 16584 files, all stamped today, and 0 eligible for eviction.
+# 120 minutes comfortably exceeds the longest CI job while still leaving the
+# bulk of a day's accumulation collectable.
+GOCACHE_CEILING_MB=1536
+GOCACHE_MIN_AGE_MIN=120
+GOCACHE_DIR="$HOME/.cache/go-build"
+if [ -d "$GOCACHE_DIR" ]; then
+  used_mb=$(du -xsm "$GOCACHE_DIR" 2>/dev/null | cut -f1)
+  used_mb=${used_mb:-0}
+  if [ "$used_mb" -gt "$GOCACHE_CEILING_MB" ]; then
+    echo "vee-runner-gc: go-build cache ${used_mb}MB over ${GOCACHE_CEILING_MB}MB ceiling, evicting LRU entries"
+    # Track the running total by subtracting each file's own size rather than
+    # re-running du after every unlink: the cache holds tens of thousands of
+    # files, so a du per deletion would mean thousands of full-tree stat walks.
+    freed_kb=0
+    target_kb=$(( (used_mb - GOCACHE_CEILING_MB) * 1024 ))
+    evicted=0
+    # Oldest first; stop as soon as enough has been freed.
+    while IFS= read -r line; do
+      size_kb=${line%% *}
+      f=${line#* }
+      [ -f "$f" ] || continue
+      rm -f "$f" 2>/dev/null || continue
+      freed_kb=$(( freed_kb + size_kb ))
+      evicted=$(( evicted + 1 ))
+      [ "$freed_kb" -ge "$target_kb" ] && break
+    done < <(find "$GOCACHE_DIR" -type f -mmin "+${GOCACHE_MIN_AGE_MIN}" -printf '%T@ %k %p\n' 2>/dev/null \
+               | sort -n | cut -d' ' -f2-)
+    used_mb=$(du -xsm "$GOCACHE_DIR" 2>/dev/null | cut -f1)
+    used_mb=${used_mb:-0}
+    echo "vee-runner-gc: go-build cache now ${used_mb}MB (evicted ${evicted} entries)"
+  else
+    echo "vee-runner-gc: go-build cache ${used_mb}MB within ${GOCACHE_CEILING_MB}MB ceiling"
   fi
 fi
 
+# Bound the journal too. The drop-in config caps steady-state growth, but a
+# burst between rotations can still overshoot, so vacuum explicitly. Needs sudo:
+# the runner user cannot unlink files under /var/log/journal, and an
+# unprivileged vacuum fails per-file and frees nothing.
+echo "vee-runner-gc: vacuuming journal"
+sudo -n journalctl --vacuum-size=200M >/dev/null 2>&1 || true
+
 # Old runner diagnostics and stale per-job temp dirs.
 find /opt/actions-runner/_diag -type f -name '*.log' -mtime +3 -delete 2>/dev/null || true
 rm -rf /opt/actions-runner/_work/_temp/* 2>/dev/null || true
@@ -285,6 +331,25 @@ echo "vee-runner-gc: done"
 df -h / | tail -1
 `
 
+	// systemd's default journal ceiling is 10% of the filesystem, which scales
+	// with the disk instead of with anything the runner actually needs: growing
+	// a 20G runner disk to 60G silently raises the journal's allowance from
+	// ~1.9G to ~5.8G. CI logs are verbose and nothing here reads journal history
+	// older than a few days, so pin an absolute cap instead of a proportional
+	// one. SystemMaxUse bounds the total; MaxRetentionSec drops anything older
+	// than a week even if the size cap is never reached.
+	journaldConf := `[Journal]
+SystemMaxUse=200M
+SystemKeepFree=1G
+MaxRetentionSec=1week
+`
+
+	// The GC service runs as User=runner, which cannot delete files under
+	// /var/log/journal — an unprivileged `journalctl --vacuum-size` reports
+	// "Permission denied" per file and frees 0B, silently. Grant exactly that
+	// one command via sudo rather than running the whole GC as root.
+	journalVacuumSudoers := "runner ALL=(root) NOPASSWD: /usr/bin/journalctl --vacuum-size=200M\n"
+
 	runnerGCService := `[Unit]
 Description=vee runner disk garbage collection
 After=actions-runner.service
@@ -354,6 +419,18 @@ WantedBy=timers.target
 			Content:     runnerGCTimer,
 			Permissions: "0644",
 		},
+		{
+			Path:        "/etc/systemd/journald.conf.d/vee-runner.conf",
+			Content:     journaldConf,
+			Permissions: "0644",
+		},
+		{
+			// 0440 is required: sudo ignores any sudoers file that is
+			// group- or world-writable.
+			Path:        "/etc/sudoers.d/vee-runner-gc",
+			Content:     journalVacuumSudoers,
+			Permissions: "0440",
+		},
 	}
 
 	nerdctlFullURL := fmt.Sprintf(
@@ -466,6 +543,10 @@ WantedBy=timers.target
 		// Hourly disk GC so CI build leftovers can't fill the disk and knock the
 		// runner offline (see vee-runner-gc.sh above).
 		"systemctl enable --now vee-runner-gc.timer",
+		// Apply the journal size cap. Without a restart journald keeps running
+		// with the compiled-in default (10% of the filesystem), which scales
+		// with disk size instead of need.
+		"systemctl restart systemd-journald",
 	)
 
 	return writeFiles, runCmds

internal/templates/runner_test.go

@@ -94,7 +94,7 @@ func TestGitHubRunnerCloudInit(t *testing.T) {
 	if !strings.Contains(joined, "systemctl enable --now vee-runner-gc.timer") {
 		t.Error("runcmd missing vee-runner-gc.timer enable")
 	}
-	var gcScript, gcTimer string
+	var gcScript, gcTimer, journaldConf, sudoersConf string
 	haveGCService, haveGCTimer := false, false
 	for _, f := range files {
 		switch f.Path {
@@ -108,6 +108,15 @@ func TestGitHubRunnerCloudInit(t *testing.T) {
 		case "/etc/systemd/system/vee-runner-gc.timer":
 			gcTimer = f.Content
 			haveGCTimer = true
+		case "/etc/systemd/journald.conf.d/vee-runner.conf":
+			journaldConf = f.Content
+		case "/etc/sudoers.d/vee-runner-gc":
+			sudoersConf = f.Content
+			// sudo silently ignores any sudoers file that is group- or
+			// world-writable, which would make the journal vacuum fail.
+			if f.Permissions != "0440" {
+				t.Errorf("sudoers drop-in must be 0440, got %q", f.Permissions)
+			}
 		}
 	}
 	if gcScript == "" {
@@ -131,9 +140,8 @@ func TestGitHubRunnerCloudInit(t *testing.T) {
 		t.Error("GC script does not prune nerdctl")
 	}
 	// The whole GC run must NOT be skipped when a job is in progress — that guard
-	// let orphaned stacks accumulate until the disk filled. The Runner.Worker
-	// check may only survive to gate the go-build cache clear, never as an early
-	// `exit 0` for the entire script.
+	// let orphaned stacks accumulate until the disk filled. Every destructive
+	// action is age-gated instead, so nothing needs an early `exit 0`.
 	if strings.Contains(gcScript, "job in progress, skipping") {
 		t.Error("GC script still skips the whole run when a job is in progress — reap must be age-gated instead")
 	}
@@ -146,11 +154,59 @@ func TestGitHubRunnerCloudInit(t *testing.T) {
 	if !strings.Contains(gcScript, "nerdctl rm -f") {
 		t.Error("GC script does not force-remove stale containers")
 	}
-	// The Runner.Worker check must remain, but only to keep the go-build cache
-	// warm during a job — not to skip the whole run.
-	if !strings.Contains(gcScript, "Runner.Worker") {
-		t.Error("GC script lost the Runner.Worker check that keeps the go-build cache warm during a job")
+	// The go-build cache must be bounded by SIZE with an age floor, not cleared
+	// wholesale. The previous design gated a full `go clean -cache` on a live
+	// Runner.Worker, so the cache either grew unchecked on a busy runner or went
+	// completely cold on an idle one. Age-gated LRU eviction protects a live
+	// build's working set without ever discarding the whole cache, which also
+	// makes the Runner.Worker check unnecessary.
+	if !strings.Contains(gcScript, "GOCACHE_CEILING_MB") {
+		t.Error("GC script does not bound the go-build cache by size (no GOCACHE_CEILING_MB)")
+	}
+	if !strings.Contains(gcScript, "GOCACHE_MIN_AGE_MIN") {
+		t.Error("GC script does not age-gate go-build eviction (no GOCACHE_MIN_AGE_MIN)")
+	}
+	if strings.Contains(gcScript, "go clean -cache") {
+		t.Error("GC script still clears the whole go-build cache instead of trimming to a ceiling")
+	}
+	// The age floor must be in MINUTES. A day-granularity guard is useless here:
+	// a busy runner rewrites its entire cache within a single day, so nothing is
+	// ever a day old and the ceiling can never be enforced (observed live: 16584
+	// cache files, all stamped the same day, 0 eligible for eviction).
+	if !strings.Contains(gcScript, "-mmin") {
+		t.Error("go-build eviction must select candidates by minutes (-mmin), not days")
+	}
+	if strings.Contains(gcScript, "-mtime +1 -printf") {
+		t.Error("go-build eviction uses day-granularity -mtime; a busy runner never has day-old entries")
+	}
+	// The journal is the one genuinely unbounded path: systemd's default ceiling
+	// is 10% of the filesystem, so growing the runner disk silently raises it.
+	// Vacuuming needs sudo — the runner user cannot unlink files under
+	// /var/log/journal, and an unprivileged vacuum fails per-file while still
+	// exiting 0, so the failure is silent.
+	if !strings.Contains(gcScript, "sudo -n journalctl --vacuum-size") {
+		t.Error("GC script does not vacuum the journal via sudo (unprivileged vacuum frees nothing)")
+	}
+	// systemd's default journal ceiling is 10% of the filesystem — a proportional
+	// cap that silently grows when the runner disk is resized (20G -> 60G raised
+	// it from ~1.9G to ~5.8G). Pin an absolute cap instead.
+	if !strings.Contains(journaldConf, "SystemMaxUse=") {
+		t.Error("journald drop-in does not cap journal size (no SystemMaxUse)")
+	}
+	// The cap only takes effect once journald reloads; without the restart the
+	// daemon keeps running with the compiled-in 10%-of-filesystem default.
+	if !strings.Contains(strings.Join(runs, "\n"), "systemctl restart systemd-journald") {
+		t.Error("runcmd does not restart journald, so the size cap never takes effect")
+	}
+	// The vacuum in the GC script runs as the runner user and needs exactly this
+	// grant; without it the vacuum fails per-file and frees nothing.
+	if !strings.Contains(sudoersConf, "journalctl --vacuum-size") {
+		t.Error("sudoers drop-in does not grant the journal vacuum the GC script runs")
+	}
+	if !strings.Contains(sudoersConf, "NOPASSWD:") {
+		t.Error("sudoers grant must be NOPASSWD — the GC service runs non-interactively")
 	}
+
 	// Cadence must be hourly, not daily: a daily timer lets orphans accumulate
 	// for up to 24h between runs on a busy runner.
 	if !strings.Contains(gcTimer, "OnCalendar=hourly") {