fix(vm): give macOS VM jobs the cache proxies

cmd/ephemerd/main.go

@@ -928,7 +928,11 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP
 				SSHPubKey: sshPubKey,
 				CPUs:      cfg.VM.MacOS.CPUs,
 				MemoryMB:  cfg.VM.MacOS.MemoryMB,
-				Log:       log,
+				// Container jobs get this env from the runtime; a macOS VM
+				// job has no container, so without it every macOS job
+				// bypassed the caches entirely.
+				JobEnv: cacheProxyEnvVars,
+				Log:    log,
 			})
 			log.Info("macOS VM support ready", "disk_image", files.DiskImage)
 		}()

pkg/vm/macos_proxyenv.go

@@ -0,0 +1,109 @@
+package vm
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/ephpm/ephemerd/pkg/proxies"
+)
+
+// Cache-proxy env for macOS VM jobs.
+//
+// Container jobs get the cache-proxy env (GOPROXY, RUSTUP_DIST_SERVER,
+// GHREL_PROXY, ...) from the container runtime. A macOS VM job has no
+// container: the GitHub runner is started over SSH inside a fresh macOS VM, and
+// until this existed nothing put that env anywhere the runner could see it. So
+// every crate, module, toolchain and release asset a macOS job fetched went
+// straight to the internet. On this fleet's Mac — which runs almost nothing but
+// macOS VM jobs — the caches stayed at 0 bytes after the proxies had been fixed
+// to advertise a reachable address, and the node kept pulling ~130 GB/day.
+//
+// The env reaches the job through the runner's own .env file, which the runner
+// loads into every job's environment at startup. It is staged in two steps so
+// that the reachability check runs where it matters — inside the guest:
+//
+//  1. macOSProxyEnvScript runs as its own bounded SSH command BEFORE the runner
+//     starts. It writes the env to a staging file only if every proxy address
+//     answers from inside the VM, and reports which way it went.
+//  2. The runner setup script appends the staged file to the runner's .env,
+//     if it exists, right before starting the runner.
+//
+// All-or-nothing on purpose. Go falls back to direct on an unreachable GOPROXY
+// ("|direct"), but cargo and rustup do not — a dead RUSTUP_DIST_SERVER fails the
+// build. A missing proxy costs bandwidth; a black-holed one costs a build.
+
+// safeHostPort limits what may be interpolated into the probe loop. The values
+// come from ephemerd's own config, but they are pasted into a shell script, so
+// anything that is not plainly a host:port is refused rather than quoted.
+var safeHostPort = regexp.MustCompile(`^[A-Za-z0-9.\-\[\]:]+$`)
+
+// heredocDelim terminates the quoted heredoc that carries the env. A quoted
+// delimiter disables all expansion, so values are written byte-for-byte.
+const heredocDelim = "EPHEMERD_PROXY_ENV"
+
+// macOSProxyEnvScript returns the guest shell script that stages env at stage
+// (macOSProxyEnvStage in production), or "" when there is nothing to stage. It
+// prints exactly one status line: "staged <n>" or "unreachable <host:port>".
+func macOSProxyEnvScript(env []string, stage string) string {
+	var lines []string
+	for _, e := range env {
+		k, _, ok := strings.Cut(e, "=")
+		if !ok || k == "" || strings.ContainsAny(e, "\r\n") || e == heredocDelim {
+			continue
+		}
+		lines = append(lines, e)
+	}
+	if len(lines) == 0 {
+		return ""
+	}
+
+	// Every URL-looking element must yield a probeable address. EnvHostPorts
+	// silently skips what it cannot parse, which would let an unparseable URL
+	// be staged without ever being checked — so check element by element.
+	var probes []string
+	seen := map[string]bool{}
+	for _, l := range lines {
+		_, v, _ := strings.Cut(l, "=")
+		for _, el := range strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == '|' }) {
+			if !strings.Contains(el, "://") {
+				continue // "direct", "off", "sparse": not an address
+			}
+			hps := proxies.EnvHostPorts([]string{"X=" + el})
+			if len(hps) != 1 || !safeHostPort.MatchString(hps[0]) {
+				// Cannot verify it, so cannot advertise it — or anything else.
+				return ""
+			}
+			if !seen[hps[0]] {
+				seen[hps[0]] = true
+				probes = append(probes, "'"+hps[0]+"'")
+			}
+		}
+	}
+
+	var b strings.Builder
+	b.WriteString("STAGE='" + stage + "'\n")
+	b.WriteString(`rm -f "$STAGE" "$STAGE.new"` + "\n")
+	b.WriteString(`cat > "$STAGE.new" <<'` + heredocDelim + "'\n")
+	for _, l := range lines {
+		b.WriteString(l + "\n")
+	}
+	b.WriteString(heredocDelim + "\n")
+	if len(probes) > 0 {
+		b.WriteString("for hp in " + strings.Join(probes, " ") + "; do\n")
+		// Any HTTP answer at all proves the address is reachable; only a
+		// connection failure or timeout makes curl exit non-zero without -f.
+		b.WriteString(`  if ! curl -s -o /dev/null --connect-timeout 3 --max-time 5 "http://$hp/"; then` + "\n")
+		b.WriteString(`    rm -f "$STAGE.new"; echo "unreachable $hp"; exit 0` + "\n")
+		b.WriteString("  fi\n")
+		b.WriteString("done\n")
+	}
+	b.WriteString(`mv "$STAGE.new" "$STAGE"` + "\n")
+	b.WriteString(`echo "staged $(wc -l < "$STAGE" | tr -d ' ')"` + "\n")
+	return b.String()
+}
+
+// macOSProxyEnvStaged interprets the script's status line: true only when the
+// env was staged.
+func macOSProxyEnvStaged(out string) bool {
+	return strings.HasPrefix(strings.TrimSpace(out), "staged ")
+}

pkg/vm/macos_proxyenv_test.go

@@ -0,0 +1,167 @@
+package vm
+
+import (
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// These tests EXECUTE the generated script with a real sh and curl, because the
+// script is the whole mechanism: a string that merely looks right proves
+// nothing about what a shell does with it. The darwin VM code that sends it
+// cannot be compiled off macOS, which is why the logic lives here.
+
+func requireShellTools(t *testing.T) {
+	t.Helper()
+	for _, tool := range []string{"sh", "curl"} {
+		if _, err := exec.LookPath(tool); err != nil {
+			t.Skipf("%s not on PATH: %v", tool, err)
+		}
+	}
+}
+
+// runStageScript runs the script and returns its status line and the staged
+// file's contents ("" if nothing was staged).
+func runStageScript(t *testing.T, script, stage string) (status, staged string) {
+	t.Helper()
+	out, err := exec.Command("sh", "-c", script).CombinedOutput()
+	if err != nil {
+		t.Fatalf("script failed: %v\n%s\n--- script ---\n%s", err, out, script)
+	}
+	b, err := os.ReadFile(stage)
+	if err == nil {
+		staged = string(b)
+	} else if !os.IsNotExist(err) {
+		t.Fatal(err)
+	}
+	return strings.TrimSpace(string(out)), staged
+}
+
+func stagePath(t *testing.T) string {
+	return filepath.ToSlash(filepath.Join(t.TempDir(), "runner.env"))
+}
+
+// The env the fleet's proxies emit (copied off a live node), pointed at addr.
+func proxyEnv(addr string) []string {
+	return []string{
+		"GOPROXY=http://" + addr + "|direct",
+		"GOSUMDB=off",
+		"CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse",
+		"RUSTUP_DIST_SERVER=http://" + addr + "/rustup",
+		"GHREL_PROXY=http://" + addr,
+	}
+}
+
+func TestMacOSProxyEnvScript_StagesWhenReachable(t *testing.T) {
+	requireShellTools(t)
+	// 404 on purpose: any HTTP answer proves reachability.
+	srv := httptest.NewServer(http.NotFoundHandler())
+	defer srv.Close()
+	addr := srv.Listener.Addr().String()
+	stage := stagePath(t)
+	env := proxyEnv(addr)
+
+	status, staged := runStageScript(t, macOSProxyEnvScript(env, stage), stage)
+
+	if !macOSProxyEnvStaged(status) || status != "staged 5" {
+		t.Fatalf("status = %q, want \"staged 5\"", status)
+	}
+	if want := strings.Join(env, "\n") + "\n"; staged != want {
+		t.Errorf("staged env differs\n got: %q\nwant: %q", staged, want)
+	}
+}
+
+// One dead address stages NOTHING — not the reachable ones, and not the
+// plain-value vars like GOSUMDB=off that only make sense alongside a proxy.
+// cargo and rustup do not fall back on an unreachable proxy, so a partial env
+// can fail a build that would have succeeded uncached.
+func TestMacOSProxyEnvScript_AllOrNothing(t *testing.T) {
+	requireShellTools(t)
+	srv := httptest.NewServer(http.NotFoundHandler())
+	defer srv.Close()
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	dead := l.Addr().String()
+	_ = l.Close()
+
+	env := append(proxyEnv(srv.Listener.Addr().String()), "COMPOSER_REPO_PACKAGIST=http://"+dead)
+	stage := stagePath(t)
+
+	status, staged := runStageScript(t, macOSProxyEnvScript(env, stage), stage)
+
+	if macOSProxyEnvStaged(status) {
+		t.Fatalf("status = %q: staged although %s is unreachable", status, dead)
+	}
+	if status != "unreachable "+dead {
+		t.Errorf("status = %q, want %q", status, "unreachable "+dead)
+	}
+	if staged != "" {
+		t.Errorf("something was staged despite a dead proxy: %q", staged)
+	}
+	if _, err := os.Stat(stage + ".new"); !os.IsNotExist(err) {
+		t.Errorf("temp file left behind: %v", err)
+	}
+}
+
+// Values are written byte-for-byte: the quoted heredoc must not expand $, `,
+// or quotes. A GOFLAGS or URL with any of these would otherwise be corrupted
+// silently, or execute.
+func TestMacOSProxyEnvScript_NoExpansion(t *testing.T) {
+	requireShellTools(t)
+	stage := stagePath(t)
+	env := []string{`WEIRD=$HOME "q" 'sq' ` + "`id`" + ` \n $(echo pwned)`}
+
+	status, staged := runStageScript(t, macOSProxyEnvScript(env, stage), stage)
+
+	if status != "staged 1" {
+		t.Fatalf("status = %q, want \"staged 1\"", status)
+	}
+	if staged != env[0]+"\n" {
+		t.Errorf("value was altered by the shell\n got: %q\nwant: %q", staged, env[0]+"\n")
+	}
+}
+
+func TestMacOSProxyEnvScript_NothingToDo(t *testing.T) {
+	for name, env := range map[string][]string{
+		"nil":                         nil,
+		"empty":                       {},
+		"malformed only":              {"NOEQUALS", "=novalue"},
+		"newline smuggling":           {"A=b\nrm -rf /"},
+		"heredoc terminator smuggled": {heredocDelim},
+	} {
+		if got := macOSProxyEnvScript(env, "/tmp/x"); got != "" {
+			t.Errorf("%s: got a script, want none:\n%s", name, got)
+		}
+	}
+}
+
+// An address that is not plainly host:port is never pasted into the probe
+// loop. Since unverified means unadvertised, the whole env is dropped.
+func TestMacOSProxyEnvScript_RefusesUnsafeAddress(t *testing.T) {
+	env := []string{"GOPROXY=http://evil$(id):8082|direct"}
+	if got := macOSProxyEnvScript(env, "/tmp/x"); got != "" {
+		t.Errorf("script generated for an unsafe address:\n%s", got)
+	}
+}
+
+func TestMacOSProxyEnvStaged(t *testing.T) {
+	for out, want := range map[string]bool{
+		"staged 5\n":                    true,
+		"  staged 1":                    true,
+		"unreachable 192.168.64.1:8082": false,
+		"":                              false,
+		"stagedX":                       false,
+	} {
+		if got := macOSProxyEnvStaged(out); got != want {
+			t.Errorf("macOSProxyEnvStaged(%q) = %v, want %v", out, got, want)
+		}
+	}
+}

pkg/vm/macosvm_darwin.go

@@ -615,6 +615,12 @@ if [ -f "$RUNNER_SRC/run.sh" ]; then
   cp -R "$RUNNER_SRC" "$RUNNER_DIR"
   chown -R admin:staff "$RUNNER_DIR"
 fi
+# Cache-proxy env, staged by ephemerd only after every proxy answered from
+# inside this VM (see macos_proxyenv.go). The runner loads .env into each job.
+if [ -f /tmp/ephemerd-runner.env ]; then
+  cat /tmp/ephemerd-runner.env >> "$RUNNER_DIR/.env"
+  chown admin:staff "$RUNNER_DIR/.env"
+fi
 cd "$RUNNER_DIR"
 ./run.sh --jitconfig '%s' </dev/null >/tmp/runner.log 2>&1 &
 RUNNER_PID=$!
@@ -635,6 +641,11 @@ echo "runner started (pid=$RUNNER_PID)"
 // session on deadline unblocks the pending call.
 const sshCommandTimeout = 90 * time.Second
 
+// macOSProxyEnvStage is where the verified cache-proxy env is staged inside
+// the guest (see macos_proxyenv.go); macOSRunnerSetupScript reads the same
+// path. A per-job VM boots from a fresh clone, so nothing stale is there.
+const macOSProxyEnvStage = "/tmp/ephemerd-runner.env"
+
 // runSSHCommand runs cmd on client with a hard deadline, returning its combined
 // output. On timeout or ctx cancellation it closes the session — which unblocks
 // the underlying CombinedOutput — and returns an error. The result channel is
@@ -793,6 +804,8 @@ func (m *darwinMacOSVM) setupRunnerViaSSH(ctx context.Context, ip string) error
 	// The runner is copied fresh into the VM before this runs; see
 	// macOSRunnerSetupScript for why we always refresh it. Only the JIT
 	// config is per-job, passed inline.
+	m.stageProxyEnv(ctx, client)
+
 	setupScript := fmt.Sprintf(macOSRunnerSetupScript, strings.TrimSpace(string(jitData)))
 
 	session, err := client.NewSession()
@@ -821,6 +834,30 @@ func (m *darwinMacOSVM) setupRunnerViaSSH(ctx context.Context, ip string) error
 	return nil
 }
 
+// stageProxyEnv verifies the cache proxies from inside the guest and stages
+// their env for the runner setup script to pick up. Every failure leaves
+// nothing staged and the job runs exactly as it did before — uncached, never
+// broken — so errors are logged, not returned.
+func (m *darwinMacOSVM) stageProxyEnv(ctx context.Context, client *ssh.Client) {
+	script := macOSProxyEnvScript(m.cfg.JobEnv, macOSProxyEnvStage)
+	if script == "" {
+		return
+	}
+	out, err := runSSHCommand(ctx, client, script, sshCommandTimeout)
+	status := strings.TrimSpace(string(out))
+	switch {
+	case err != nil:
+		m.cfg.Log.Warn("could not stage cache-proxy env in the macOS VM; job runs without cache proxies",
+			"id", m.id, "error", err)
+	case macOSProxyEnvStaged(status):
+		m.cfg.Log.Info("cache proxies reachable from the macOS VM; advertised to the job",
+			"id", m.id, "status", status)
+	default:
+		m.cfg.Log.Warn("cache proxies not reachable from inside the macOS VM; job runs without them",
+			"id", m.id, "status", status)
+	}
+}
+
 func (m *darwinMacOSVM) Stop() {
 	m.stopOnce.Do(m.stop)
 }

pkg/vm/vm.go

@@ -120,6 +120,11 @@ type MacOSVMConfig struct {
 	// MemoryMB per macOS VM. Defaults to 8192.
 	MemoryMB uint64
 
+	// JobEnv is extra KEY=VALUE env for every job's runner: the cache-proxy
+	// env, already rewritten to the host's address on the VM NAT. It is
+	// verified reachable from inside each VM before use; see macos_proxyenv.go.
+	JobEnv []string
+
 	Log *slog.Logger
 }