diff --git a/internal/templates/runner.go b/internal/templates/runner.go index c2f97b7..16190ca 100644 --- a/internal/templates/runner.go +++ b/internal/templates/runner.go @@ -263,20 +263,66 @@ nerdctl builder prune -af || true echo "vee-runner-gc: pruning BuildKit cache to 2GB ceiling" buildctl prune --keep-storage 2048 || true -# Trim a runaway Go build cache (cgo/native CI builds), bounded not cleared. -# Only when no job is running: a live build writes here and clearing it mid-build -# is a pointless cache-cold penalty (not a correctness problem — the age-gated -# reap above is what protects an in-flight job). This is the one place a live -# Runner.Worker is worth checking for. -if pgrep -u runner -f 'Runner.Worker' >/dev/null 2>&1; then - echo "vee-runner-gc: job in progress, leaving go-build cache warm" -else - GOCACHE_DIR="$HOME/.cache/go-build" - if [ -d "$GOCACHE_DIR" ]; then - GOCACHE=$GOCACHE_DIR go clean -cache 2>/dev/null || rm -rf "${GOCACHE_DIR:?}/"* || true +# Bound the Go build cache by SIZE rather than clearing it wholesale. The old +# behaviour was all-or-nothing — skip entirely while a job ran, otherwise wipe +# every entry — which meant the cache either grew unchecked on a busy runner or +# went fully cold on an idle one. GOCACHEPROG-free trimming is not exposed by +# the go tool, so evict least-recently-used entries directly: the cache is +# content-addressed under two-hex-digit subdirectories, and its files carry +# atime-like mtimes that the go tool refreshes on use, so dropping the oldest +# files first approximates Go's own LRU trimming closely enough for a CI box. +# +# Runs even with a job in progress: evicting a cold entry only costs that entry +# a recompile, and the cache is designed to tolerate arbitrary eviction (a miss +# is a rebuild, never a failure). Only entries untouched for GOCACHE_MIN_AGE_MIN +# are eligible, so a live build's working set is never evicted. +# +# The age floor is in MINUTES, not days. A day-granularity guard (-mtime +1) is +# useless here: a busy runner rewrites the whole cache within a single day, so +# nothing is ever a day old and the ceiling can never be enforced — observed on +# this runner with 16584 files, all stamped today, and 0 eligible for eviction. +# 120 minutes comfortably exceeds the longest CI job while still leaving the +# bulk of a day's accumulation collectable. +GOCACHE_CEILING_MB=1536 +GOCACHE_MIN_AGE_MIN=120 +GOCACHE_DIR="$HOME/.cache/go-build" +if [ -d "$GOCACHE_DIR" ]; then + used_mb=$(du -xsm "$GOCACHE_DIR" 2>/dev/null | cut -f1) + used_mb=${used_mb:-0} + if [ "$used_mb" -gt "$GOCACHE_CEILING_MB" ]; then + echo "vee-runner-gc: go-build cache ${used_mb}MB over ${GOCACHE_CEILING_MB}MB ceiling, evicting LRU entries" + # Track the running total by subtracting each file's own size rather than + # re-running du after every unlink: the cache holds tens of thousands of + # files, so a du per deletion would mean thousands of full-tree stat walks. + freed_kb=0 + target_kb=$(( (used_mb - GOCACHE_CEILING_MB) * 1024 )) + evicted=0 + # Oldest first; stop as soon as enough has been freed. + while IFS= read -r line; do + size_kb=${line%% *} + f=${line#* } + [ -f "$f" ] || continue + rm -f "$f" 2>/dev/null || continue + freed_kb=$(( freed_kb + size_kb )) + evicted=$(( evicted + 1 )) + [ "$freed_kb" -ge "$target_kb" ] && break + done < <(find "$GOCACHE_DIR" -type f -mmin "+${GOCACHE_MIN_AGE_MIN}" -printf '%T@ %k %p\n' 2>/dev/null \ + | sort -n | cut -d' ' -f2-) + used_mb=$(du -xsm "$GOCACHE_DIR" 2>/dev/null | cut -f1) + used_mb=${used_mb:-0} + echo "vee-runner-gc: go-build cache now ${used_mb}MB (evicted ${evicted} entries)" + else + echo "vee-runner-gc: go-build cache ${used_mb}MB within ${GOCACHE_CEILING_MB}MB ceiling" fi fi +# Bound the journal too. The drop-in config caps steady-state growth, but a +# burst between rotations can still overshoot, so vacuum explicitly. Needs sudo: +# the runner user cannot unlink files under /var/log/journal, and an +# unprivileged vacuum fails per-file and frees nothing. +echo "vee-runner-gc: vacuuming journal" +sudo -n journalctl --vacuum-size=200M >/dev/null 2>&1 || true + # Old runner diagnostics and stale per-job temp dirs. find /opt/actions-runner/_diag -type f -name '*.log' -mtime +3 -delete 2>/dev/null || true rm -rf /opt/actions-runner/_work/_temp/* 2>/dev/null || true @@ -285,6 +331,25 @@ echo "vee-runner-gc: done" df -h / | tail -1 ` + // systemd's default journal ceiling is 10% of the filesystem, which scales + // with the disk instead of with anything the runner actually needs: growing + // a 20G runner disk to 60G silently raises the journal's allowance from + // ~1.9G to ~5.8G. CI logs are verbose and nothing here reads journal history + // older than a few days, so pin an absolute cap instead of a proportional + // one. SystemMaxUse bounds the total; MaxRetentionSec drops anything older + // than a week even if the size cap is never reached. + journaldConf := `[Journal] +SystemMaxUse=200M +SystemKeepFree=1G +MaxRetentionSec=1week +` + + // The GC service runs as User=runner, which cannot delete files under + // /var/log/journal — an unprivileged `journalctl --vacuum-size` reports + // "Permission denied" per file and frees 0B, silently. Grant exactly that + // one command via sudo rather than running the whole GC as root. + journalVacuumSudoers := "runner ALL=(root) NOPASSWD: /usr/bin/journalctl --vacuum-size=200M\n" + runnerGCService := `[Unit] Description=vee runner disk garbage collection After=actions-runner.service @@ -354,6 +419,18 @@ WantedBy=timers.target Content: runnerGCTimer, Permissions: "0644", }, + { + Path: "/etc/systemd/journald.conf.d/vee-runner.conf", + Content: journaldConf, + Permissions: "0644", + }, + { + // 0440 is required: sudo ignores any sudoers file that is + // group- or world-writable. + Path: "/etc/sudoers.d/vee-runner-gc", + Content: journalVacuumSudoers, + Permissions: "0440", + }, } nerdctlFullURL := fmt.Sprintf( @@ -466,6 +543,10 @@ WantedBy=timers.target // Hourly disk GC so CI build leftovers can't fill the disk and knock the // runner offline (see vee-runner-gc.sh above). "systemctl enable --now vee-runner-gc.timer", + // Apply the journal size cap. Without a restart journald keeps running + // with the compiled-in default (10% of the filesystem), which scales + // with disk size instead of need. + "systemctl restart systemd-journald", ) return writeFiles, runCmds diff --git a/internal/templates/runner_test.go b/internal/templates/runner_test.go index d06b8ba..279b58c 100644 --- a/internal/templates/runner_test.go +++ b/internal/templates/runner_test.go @@ -94,7 +94,7 @@ func TestGitHubRunnerCloudInit(t *testing.T) { if !strings.Contains(joined, "systemctl enable --now vee-runner-gc.timer") { t.Error("runcmd missing vee-runner-gc.timer enable") } - var gcScript, gcTimer string + var gcScript, gcTimer, journaldConf, sudoersConf string haveGCService, haveGCTimer := false, false for _, f := range files { switch f.Path { @@ -108,6 +108,15 @@ func TestGitHubRunnerCloudInit(t *testing.T) { case "/etc/systemd/system/vee-runner-gc.timer": gcTimer = f.Content haveGCTimer = true + case "/etc/systemd/journald.conf.d/vee-runner.conf": + journaldConf = f.Content + case "/etc/sudoers.d/vee-runner-gc": + sudoersConf = f.Content + // sudo silently ignores any sudoers file that is group- or + // world-writable, which would make the journal vacuum fail. + if f.Permissions != "0440" { + t.Errorf("sudoers drop-in must be 0440, got %q", f.Permissions) + } } } if gcScript == "" { @@ -131,9 +140,8 @@ func TestGitHubRunnerCloudInit(t *testing.T) { t.Error("GC script does not prune nerdctl") } // The whole GC run must NOT be skipped when a job is in progress — that guard - // let orphaned stacks accumulate until the disk filled. The Runner.Worker - // check may only survive to gate the go-build cache clear, never as an early - // `exit 0` for the entire script. + // let orphaned stacks accumulate until the disk filled. Every destructive + // action is age-gated instead, so nothing needs an early `exit 0`. if strings.Contains(gcScript, "job in progress, skipping") { t.Error("GC script still skips the whole run when a job is in progress — reap must be age-gated instead") } @@ -146,11 +154,59 @@ func TestGitHubRunnerCloudInit(t *testing.T) { if !strings.Contains(gcScript, "nerdctl rm -f") { t.Error("GC script does not force-remove stale containers") } - // The Runner.Worker check must remain, but only to keep the go-build cache - // warm during a job — not to skip the whole run. - if !strings.Contains(gcScript, "Runner.Worker") { - t.Error("GC script lost the Runner.Worker check that keeps the go-build cache warm during a job") + // The go-build cache must be bounded by SIZE with an age floor, not cleared + // wholesale. The previous design gated a full `go clean -cache` on a live + // Runner.Worker, so the cache either grew unchecked on a busy runner or went + // completely cold on an idle one. Age-gated LRU eviction protects a live + // build's working set without ever discarding the whole cache, which also + // makes the Runner.Worker check unnecessary. + if !strings.Contains(gcScript, "GOCACHE_CEILING_MB") { + t.Error("GC script does not bound the go-build cache by size (no GOCACHE_CEILING_MB)") + } + if !strings.Contains(gcScript, "GOCACHE_MIN_AGE_MIN") { + t.Error("GC script does not age-gate go-build eviction (no GOCACHE_MIN_AGE_MIN)") + } + if strings.Contains(gcScript, "go clean -cache") { + t.Error("GC script still clears the whole go-build cache instead of trimming to a ceiling") + } + // The age floor must be in MINUTES. A day-granularity guard is useless here: + // a busy runner rewrites its entire cache within a single day, so nothing is + // ever a day old and the ceiling can never be enforced (observed live: 16584 + // cache files, all stamped the same day, 0 eligible for eviction). + if !strings.Contains(gcScript, "-mmin") { + t.Error("go-build eviction must select candidates by minutes (-mmin), not days") + } + if strings.Contains(gcScript, "-mtime +1 -printf") { + t.Error("go-build eviction uses day-granularity -mtime; a busy runner never has day-old entries") + } + // The journal is the one genuinely unbounded path: systemd's default ceiling + // is 10% of the filesystem, so growing the runner disk silently raises it. + // Vacuuming needs sudo — the runner user cannot unlink files under + // /var/log/journal, and an unprivileged vacuum fails per-file while still + // exiting 0, so the failure is silent. + if !strings.Contains(gcScript, "sudo -n journalctl --vacuum-size") { + t.Error("GC script does not vacuum the journal via sudo (unprivileged vacuum frees nothing)") + } + // systemd's default journal ceiling is 10% of the filesystem — a proportional + // cap that silently grows when the runner disk is resized (20G -> 60G raised + // it from ~1.9G to ~5.8G). Pin an absolute cap instead. + if !strings.Contains(journaldConf, "SystemMaxUse=") { + t.Error("journald drop-in does not cap journal size (no SystemMaxUse)") + } + // The cap only takes effect once journald reloads; without the restart the + // daemon keeps running with the compiled-in 10%-of-filesystem default. + if !strings.Contains(strings.Join(runs, "\n"), "systemctl restart systemd-journald") { + t.Error("runcmd does not restart journald, so the size cap never takes effect") + } + // The vacuum in the GC script runs as the runner user and needs exactly this + // grant; without it the vacuum fails per-file and frees nothing. + if !strings.Contains(sudoersConf, "journalctl --vacuum-size") { + t.Error("sudoers drop-in does not grant the journal vacuum the GC script runs") + } + if !strings.Contains(sudoersConf, "NOPASSWD:") { + t.Error("sudoers grant must be NOPASSWD — the GC service runs non-interactively") } + // Cadence must be hourly, not daily: a daily timer lets orphans accumulate // for up to 24h between runs on a busy runner. if !strings.Contains(gcTimer, "OnCalendar=hourly") {