From 8399c4ee8cfe5dd8c66d923dd915dbdb9069c47b Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Mon, 5 Oct 2026 12:48:47 -0700 Subject: [PATCH 1/2] fix(vm): give macOS VM jobs the cache proxies Container jobs get GOPROXY, RUSTUP_DIST_SERVER, GHREL_PROXY and the rest from the container runtime (pkg/runtime). macOS VM jobs have no container: the runner is started over SSH in a fresh macOS VM, and nothing passed it any proxy env. So every crate, module, toolchain and release asset a macOS job fetched went straight to the internet. The fleet's Mac runs almost nothing but macOS VM jobs, which is why its caches stayed at 0 bytes after #208 fixed the advertised address, while the node kept pulling ~130 GB/day. The env now reaches the job through the runner's .env file, which the runner loads into every job's environment: - MacOSVMConfig.JobEnv carries the cache-proxy env, already rewritten to the host's Vz NAT address by resolveJobProxyEnv. - Before the runner starts, a bounded SSH command probes every proxy address from INSIDE the guest and stages the env only if all of them answer (all-or-nothing: cargo and rustup do not fall back on an unreachable proxy). The outcome is logged per VM. - The runner setup script appends the staged env to the runner's .env. Any failure leaves nothing staged and the job runs exactly as before, uncached. The script builder lives in an untagged file so its tests, which execute the generated script with real sh and curl, run on every platform. --- cmd/ephemerd/main.go | 6 +- pkg/vm/macos_proxyenv.go | 113 +++++++++++++++++++++++ pkg/vm/macos_proxyenv_test.go | 167 ++++++++++++++++++++++++++++++++++ pkg/vm/macosvm_darwin.go | 32 +++++++ pkg/vm/vm.go | 5 + 5 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 pkg/vm/macos_proxyenv.go create mode 100644 pkg/vm/macos_proxyenv_test.go diff --git a/cmd/ephemerd/main.go b/cmd/ephemerd/main.go index e9ae51c..6997429 100644 --- a/cmd/ephemerd/main.go +++ b/cmd/ephemerd/main.go @@ -928,7 +928,11 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP SSHPubKey: sshPubKey, CPUs: cfg.VM.MacOS.CPUs, MemoryMB: cfg.VM.MacOS.MemoryMB, - Log: log, + // Container jobs get this env from the runtime; a macOS VM + // job has no container, so without it every macOS job + // bypassed the caches entirely. + JobEnv: cacheProxyEnvVars, + Log: log, }) log.Info("macOS VM support ready", "disk_image", files.DiskImage) }() diff --git a/pkg/vm/macos_proxyenv.go b/pkg/vm/macos_proxyenv.go new file mode 100644 index 0000000..9addc36 --- /dev/null +++ b/pkg/vm/macos_proxyenv.go @@ -0,0 +1,113 @@ +package vm + +import ( + "regexp" + "strings" + + "github.com/ephpm/ephemerd/pkg/proxies" +) + +// Cache-proxy env for macOS VM jobs. +// +// Container jobs get the cache-proxy env (GOPROXY, RUSTUP_DIST_SERVER, +// GHREL_PROXY, ...) from the container runtime. A macOS VM job has no +// container: the GitHub runner is started over SSH inside a fresh macOS VM, and +// until this existed nothing put that env anywhere the runner could see it. So +// every crate, module, toolchain and release asset a macOS job fetched went +// straight to the internet. On this fleet's Mac — which runs almost nothing but +// macOS VM jobs — the caches stayed at 0 bytes after the proxies had been fixed +// to advertise a reachable address, and the node kept pulling ~130 GB/day. +// +// The env reaches the job through the runner's own .env file, which the runner +// loads into every job's environment at startup. It is staged in two steps so +// that the reachability check runs where it matters — inside the guest: +// +// 1. macOSProxyEnvScript runs as its own bounded SSH command BEFORE the runner +// starts. It writes the env to a staging file only if every proxy address +// answers from inside the VM, and reports which way it went. +// 2. The runner setup script appends the staged file to the runner's .env, +// if it exists, right before starting the runner. +// +// All-or-nothing on purpose. Go falls back to direct on an unreachable GOPROXY +// ("|direct"), but cargo and rustup do not — a dead RUSTUP_DIST_SERVER fails the +// build. A missing proxy costs bandwidth; a black-holed one costs a build. + +// macOSProxyEnvStage is where the verified env is staged inside the guest. A +// per-job VM boots from a fresh clone, so nothing stale can be sitting there. +const macOSProxyEnvStage = "/tmp/ephemerd-runner.env" + +// safeHostPort limits what may be interpolated into the probe loop. The values +// come from ephemerd's own config, but they are pasted into a shell script, so +// anything that is not plainly a host:port is refused rather than quoted. +var safeHostPort = regexp.MustCompile(`^[A-Za-z0-9.\-\[\]:]+$`) + +// heredocDelim terminates the quoted heredoc that carries the env. A quoted +// delimiter disables all expansion, so values are written byte-for-byte. +const heredocDelim = "EPHEMERD_PROXY_ENV" + +// macOSProxyEnvScript returns the guest shell script that stages env at stage +// (macOSProxyEnvStage in production), or "" when there is nothing to stage. It +// prints exactly one status line: "staged " or "unreachable ". +func macOSProxyEnvScript(env []string, stage string) string { + var lines []string + for _, e := range env { + k, _, ok := strings.Cut(e, "=") + if !ok || k == "" || strings.ContainsAny(e, "\r\n") || e == heredocDelim { + continue + } + lines = append(lines, e) + } + if len(lines) == 0 { + return "" + } + + // Every URL-looking element must yield a probeable address. EnvHostPorts + // silently skips what it cannot parse, which would let an unparseable URL + // be staged without ever being checked — so check element by element. + var probes []string + seen := map[string]bool{} + for _, l := range lines { + _, v, _ := strings.Cut(l, "=") + for _, el := range strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == '|' }) { + if !strings.Contains(el, "://") { + continue // "direct", "off", "sparse": not an address + } + hps := proxies.EnvHostPorts([]string{"X=" + el}) + if len(hps) != 1 || !safeHostPort.MatchString(hps[0]) { + // Cannot verify it, so cannot advertise it — or anything else. + return "" + } + if !seen[hps[0]] { + seen[hps[0]] = true + probes = append(probes, "'"+hps[0]+"'") + } + } + } + + var b strings.Builder + b.WriteString("STAGE='" + stage + "'\n") + b.WriteString(`rm -f "$STAGE" "$STAGE.new"` + "\n") + b.WriteString(`cat > "$STAGE.new" <<'` + heredocDelim + "'\n") + for _, l := range lines { + b.WriteString(l + "\n") + } + b.WriteString(heredocDelim + "\n") + if len(probes) > 0 { + b.WriteString("for hp in " + strings.Join(probes, " ") + "; do\n") + // Any HTTP answer at all proves the address is reachable; only a + // connection failure or timeout makes curl exit non-zero without -f. + b.WriteString(` if ! curl -s -o /dev/null --connect-timeout 3 --max-time 5 "http://$hp/"; then` + "\n") + b.WriteString(` rm -f "$STAGE.new"; echo "unreachable $hp"; exit 0` + "\n") + b.WriteString(" fi\n") + b.WriteString("done\n") + } + b.WriteString(`mv "$STAGE.new" "$STAGE"` + "\n") + b.WriteString(`echo "staged $(wc -l < "$STAGE" | tr -d ' ')"` + "\n") + return b.String() +} + +// macOSProxyEnvStaged interprets the script's status line: true only when the +// env was staged. +func macOSProxyEnvStaged(out string) bool { + return strings.HasPrefix(strings.TrimSpace(out), "staged ") +} diff --git a/pkg/vm/macos_proxyenv_test.go b/pkg/vm/macos_proxyenv_test.go new file mode 100644 index 0000000..e4487a6 --- /dev/null +++ b/pkg/vm/macos_proxyenv_test.go @@ -0,0 +1,167 @@ +package vm + +import ( + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// These tests EXECUTE the generated script with a real sh and curl, because the +// script is the whole mechanism: a string that merely looks right proves +// nothing about what a shell does with it. The darwin VM code that sends it +// cannot be compiled off macOS, which is why the logic lives here. + +func requireShellTools(t *testing.T) { + t.Helper() + for _, tool := range []string{"sh", "curl"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s not on PATH: %v", tool, err) + } + } +} + +// runStageScript runs the script and returns its status line and the staged +// file's contents ("" if nothing was staged). +func runStageScript(t *testing.T, script, stage string) (status, staged string) { + t.Helper() + out, err := exec.Command("sh", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("script failed: %v\n%s\n--- script ---\n%s", err, out, script) + } + b, err := os.ReadFile(stage) + if err == nil { + staged = string(b) + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + return strings.TrimSpace(string(out)), staged +} + +func stagePath(t *testing.T) string { + return filepath.ToSlash(filepath.Join(t.TempDir(), "runner.env")) +} + +// The env the fleet's proxies emit (copied off a live node), pointed at addr. +func proxyEnv(addr string) []string { + return []string{ + "GOPROXY=http://" + addr + "|direct", + "GOSUMDB=off", + "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse", + "RUSTUP_DIST_SERVER=http://" + addr + "/rustup", + "GHREL_PROXY=http://" + addr, + } +} + +func TestMacOSProxyEnvScript_StagesWhenReachable(t *testing.T) { + requireShellTools(t) + // 404 on purpose: any HTTP answer proves reachability. + srv := httptest.NewServer(http.NotFoundHandler()) + defer srv.Close() + addr := srv.Listener.Addr().String() + stage := stagePath(t) + env := proxyEnv(addr) + + status, staged := runStageScript(t, macOSProxyEnvScript(env, stage), stage) + + if !macOSProxyEnvStaged(status) || status != "staged 5" { + t.Fatalf("status = %q, want \"staged 5\"", status) + } + if want := strings.Join(env, "\n") + "\n"; staged != want { + t.Errorf("staged env differs\n got: %q\nwant: %q", staged, want) + } +} + +// One dead address stages NOTHING — not the reachable ones, and not the +// plain-value vars like GOSUMDB=off that only make sense alongside a proxy. +// cargo and rustup do not fall back on an unreachable proxy, so a partial env +// can fail a build that would have succeeded uncached. +func TestMacOSProxyEnvScript_AllOrNothing(t *testing.T) { + requireShellTools(t) + srv := httptest.NewServer(http.NotFoundHandler()) + defer srv.Close() + + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + dead := l.Addr().String() + _ = l.Close() + + env := append(proxyEnv(srv.Listener.Addr().String()), "COMPOSER_REPO_PACKAGIST=http://"+dead) + stage := stagePath(t) + + status, staged := runStageScript(t, macOSProxyEnvScript(env, stage), stage) + + if macOSProxyEnvStaged(status) { + t.Fatalf("status = %q: staged although %s is unreachable", status, dead) + } + if status != "unreachable "+dead { + t.Errorf("status = %q, want %q", status, "unreachable "+dead) + } + if staged != "" { + t.Errorf("something was staged despite a dead proxy: %q", staged) + } + if _, err := os.Stat(stage + ".new"); !os.IsNotExist(err) { + t.Errorf("temp file left behind: %v", err) + } +} + +// Values are written byte-for-byte: the quoted heredoc must not expand $, `, +// or quotes. A GOFLAGS or URL with any of these would otherwise be corrupted +// silently, or execute. +func TestMacOSProxyEnvScript_NoExpansion(t *testing.T) { + requireShellTools(t) + stage := stagePath(t) + env := []string{`WEIRD=$HOME "q" 'sq' ` + "`id`" + ` \n $(echo pwned)`} + + status, staged := runStageScript(t, macOSProxyEnvScript(env, stage), stage) + + if status != "staged 1" { + t.Fatalf("status = %q, want \"staged 1\"", status) + } + if staged != env[0]+"\n" { + t.Errorf("value was altered by the shell\n got: %q\nwant: %q", staged, env[0]+"\n") + } +} + +func TestMacOSProxyEnvScript_NothingToDo(t *testing.T) { + for name, env := range map[string][]string{ + "nil": nil, + "empty": {}, + "malformed only": {"NOEQUALS", "=novalue"}, + "newline smuggling": {"A=b\nrm -rf /"}, + "heredoc terminator smuggled": {heredocDelim}, + } { + if got := macOSProxyEnvScript(env, "/tmp/x"); got != "" { + t.Errorf("%s: got a script, want none:\n%s", name, got) + } + } +} + +// An address that is not plainly host:port is never pasted into the probe +// loop. Since unverified means unadvertised, the whole env is dropped. +func TestMacOSProxyEnvScript_RefusesUnsafeAddress(t *testing.T) { + env := []string{"GOPROXY=http://evil$(id):8082|direct"} + if got := macOSProxyEnvScript(env, "/tmp/x"); got != "" { + t.Errorf("script generated for an unsafe address:\n%s", got) + } +} + +func TestMacOSProxyEnvStaged(t *testing.T) { + for out, want := range map[string]bool{ + "staged 5\n": true, + " staged 1": true, + "unreachable 192.168.64.1:8082": false, + "": false, + "stagedX": false, + } { + if got := macOSProxyEnvStaged(out); got != want { + t.Errorf("macOSProxyEnvStaged(%q) = %v, want %v", out, got, want) + } + } +} diff --git a/pkg/vm/macosvm_darwin.go b/pkg/vm/macosvm_darwin.go index 78cb199..4e3f888 100644 --- a/pkg/vm/macosvm_darwin.go +++ b/pkg/vm/macosvm_darwin.go @@ -615,6 +615,12 @@ if [ -f "$RUNNER_SRC/run.sh" ]; then cp -R "$RUNNER_SRC" "$RUNNER_DIR" chown -R admin:staff "$RUNNER_DIR" fi +# Cache-proxy env, staged by ephemerd only after every proxy answered from +# inside this VM (see macos_proxyenv.go). The runner loads .env into each job. +if [ -f /tmp/ephemerd-runner.env ]; then + cat /tmp/ephemerd-runner.env >> "$RUNNER_DIR/.env" + chown admin:staff "$RUNNER_DIR/.env" +fi cd "$RUNNER_DIR" ./run.sh --jitconfig '%s' /tmp/runner.log 2>&1 & RUNNER_PID=$! @@ -793,6 +799,8 @@ func (m *darwinMacOSVM) setupRunnerViaSSH(ctx context.Context, ip string) error // The runner is copied fresh into the VM before this runs; see // macOSRunnerSetupScript for why we always refresh it. Only the JIT // config is per-job, passed inline. + m.stageProxyEnv(ctx, client) + setupScript := fmt.Sprintf(macOSRunnerSetupScript, strings.TrimSpace(string(jitData))) session, err := client.NewSession() @@ -821,6 +829,30 @@ func (m *darwinMacOSVM) setupRunnerViaSSH(ctx context.Context, ip string) error return nil } +// stageProxyEnv verifies the cache proxies from inside the guest and stages +// their env for the runner setup script to pick up. Every failure leaves +// nothing staged and the job runs exactly as it did before — uncached, never +// broken — so errors are logged, not returned. +func (m *darwinMacOSVM) stageProxyEnv(ctx context.Context, client *ssh.Client) { + script := macOSProxyEnvScript(m.cfg.JobEnv, macOSProxyEnvStage) + if script == "" { + return + } + out, err := runSSHCommand(ctx, client, script, sshCommandTimeout) + status := strings.TrimSpace(string(out)) + switch { + case err != nil: + m.cfg.Log.Warn("could not stage cache-proxy env in the macOS VM; job runs without cache proxies", + "id", m.id, "error", err) + case macOSProxyEnvStaged(status): + m.cfg.Log.Info("cache proxies reachable from the macOS VM; advertised to the job", + "id", m.id, "status", status) + default: + m.cfg.Log.Warn("cache proxies not reachable from inside the macOS VM; job runs without them", + "id", m.id, "status", status) + } +} + func (m *darwinMacOSVM) Stop() { m.stopOnce.Do(m.stop) } diff --git a/pkg/vm/vm.go b/pkg/vm/vm.go index 91b220e..cbac103 100644 --- a/pkg/vm/vm.go +++ b/pkg/vm/vm.go @@ -120,6 +120,11 @@ type MacOSVMConfig struct { // MemoryMB per macOS VM. Defaults to 8192. MemoryMB uint64 + // JobEnv is extra KEY=VALUE env for every job's runner: the cache-proxy + // env, already rewritten to the host's address on the VM NAT. It is + // verified reachable from inside each VM before use; see macos_proxyenv.go. + JobEnv []string + Log *slog.Logger } From 72c17b84d9b2210d7cb2577ede2f47f9e83044b0 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Mon, 5 Oct 2026 13:14:09 -0700 Subject: [PATCH 2/2] fix(vm): keep the staging-path const with its only (darwin) user golangci-lint on Linux flagged macOSProxyEnvStage as unused: the untagged file declared it, but only the darwin VM code references it. --- pkg/vm/macos_proxyenv.go | 4 ---- pkg/vm/macosvm_darwin.go | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/vm/macos_proxyenv.go b/pkg/vm/macos_proxyenv.go index 9addc36..5b43ff5 100644 --- a/pkg/vm/macos_proxyenv.go +++ b/pkg/vm/macos_proxyenv.go @@ -32,10 +32,6 @@ import ( // ("|direct"), but cargo and rustup do not — a dead RUSTUP_DIST_SERVER fails the // build. A missing proxy costs bandwidth; a black-holed one costs a build. -// macOSProxyEnvStage is where the verified env is staged inside the guest. A -// per-job VM boots from a fresh clone, so nothing stale can be sitting there. -const macOSProxyEnvStage = "/tmp/ephemerd-runner.env" - // safeHostPort limits what may be interpolated into the probe loop. The values // come from ephemerd's own config, but they are pasted into a shell script, so // anything that is not plainly a host:port is refused rather than quoted. diff --git a/pkg/vm/macosvm_darwin.go b/pkg/vm/macosvm_darwin.go index 4e3f888..d41fa38 100644 --- a/pkg/vm/macosvm_darwin.go +++ b/pkg/vm/macosvm_darwin.go @@ -641,6 +641,11 @@ echo "runner started (pid=$RUNNER_PID)" // session on deadline unblocks the pending call. const sshCommandTimeout = 90 * time.Second +// macOSProxyEnvStage is where the verified cache-proxy env is staged inside +// the guest (see macos_proxyenv.go); macOSRunnerSetupScript reads the same +// path. A per-job VM boots from a fresh clone, so nothing stale is there. +const macOSProxyEnvStage = "/tmp/ephemerd-runner.env" + // runSSHCommand runs cmd on client with a hard deadline, returning its combined // output. On timeout or ctx cancellation it closes the session — which unblocks // the underlying CombinedOutput — and returns an error. The result channel is