diff --git a/cmd/ephemerd/main.go b/cmd/ephemerd/main.go index e9ae51c..6997429 100644 --- a/cmd/ephemerd/main.go +++ b/cmd/ephemerd/main.go @@ -928,7 +928,11 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP SSHPubKey: sshPubKey, CPUs: cfg.VM.MacOS.CPUs, MemoryMB: cfg.VM.MacOS.MemoryMB, - Log: log, + // Container jobs get this env from the runtime; a macOS VM + // job has no container, so without it every macOS job + // bypassed the caches entirely. + JobEnv: cacheProxyEnvVars, + Log: log, }) log.Info("macOS VM support ready", "disk_image", files.DiskImage) }() diff --git a/pkg/vm/macos_proxyenv.go b/pkg/vm/macos_proxyenv.go new file mode 100644 index 0000000..5b43ff5 --- /dev/null +++ b/pkg/vm/macos_proxyenv.go @@ -0,0 +1,109 @@ +package vm + +import ( + "regexp" + "strings" + + "github.com/ephpm/ephemerd/pkg/proxies" +) + +// Cache-proxy env for macOS VM jobs. +// +// Container jobs get the cache-proxy env (GOPROXY, RUSTUP_DIST_SERVER, +// GHREL_PROXY, ...) from the container runtime. A macOS VM job has no +// container: the GitHub runner is started over SSH inside a fresh macOS VM, and +// until this existed nothing put that env anywhere the runner could see it. So +// every crate, module, toolchain and release asset a macOS job fetched went +// straight to the internet. On this fleet's Mac — which runs almost nothing but +// macOS VM jobs — the caches stayed at 0 bytes after the proxies had been fixed +// to advertise a reachable address, and the node kept pulling ~130 GB/day. +// +// The env reaches the job through the runner's own .env file, which the runner +// loads into every job's environment at startup. It is staged in two steps so +// that the reachability check runs where it matters — inside the guest: +// +// 1. macOSProxyEnvScript runs as its own bounded SSH command BEFORE the runner +// starts. It writes the env to a staging file only if every proxy address +// answers from inside the VM, and reports which way it went. +// 2. The runner setup script appends the staged file to the runner's .env, +// if it exists, right before starting the runner. +// +// All-or-nothing on purpose. Go falls back to direct on an unreachable GOPROXY +// ("|direct"), but cargo and rustup do not — a dead RUSTUP_DIST_SERVER fails the +// build. A missing proxy costs bandwidth; a black-holed one costs a build. + +// safeHostPort limits what may be interpolated into the probe loop. The values +// come from ephemerd's own config, but they are pasted into a shell script, so +// anything that is not plainly a host:port is refused rather than quoted. +var safeHostPort = regexp.MustCompile(`^[A-Za-z0-9.\-\[\]:]+$`) + +// heredocDelim terminates the quoted heredoc that carries the env. A quoted +// delimiter disables all expansion, so values are written byte-for-byte. +const heredocDelim = "EPHEMERD_PROXY_ENV" + +// macOSProxyEnvScript returns the guest shell script that stages env at stage +// (macOSProxyEnvStage in production), or "" when there is nothing to stage. It +// prints exactly one status line: "staged " or "unreachable ". +func macOSProxyEnvScript(env []string, stage string) string { + var lines []string + for _, e := range env { + k, _, ok := strings.Cut(e, "=") + if !ok || k == "" || strings.ContainsAny(e, "\r\n") || e == heredocDelim { + continue + } + lines = append(lines, e) + } + if len(lines) == 0 { + return "" + } + + // Every URL-looking element must yield a probeable address. EnvHostPorts + // silently skips what it cannot parse, which would let an unparseable URL + // be staged without ever being checked — so check element by element. + var probes []string + seen := map[string]bool{} + for _, l := range lines { + _, v, _ := strings.Cut(l, "=") + for _, el := range strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == '|' }) { + if !strings.Contains(el, "://") { + continue // "direct", "off", "sparse": not an address + } + hps := proxies.EnvHostPorts([]string{"X=" + el}) + if len(hps) != 1 || !safeHostPort.MatchString(hps[0]) { + // Cannot verify it, so cannot advertise it — or anything else. + return "" + } + if !seen[hps[0]] { + seen[hps[0]] = true + probes = append(probes, "'"+hps[0]+"'") + } + } + } + + var b strings.Builder + b.WriteString("STAGE='" + stage + "'\n") + b.WriteString(`rm -f "$STAGE" "$STAGE.new"` + "\n") + b.WriteString(`cat > "$STAGE.new" <<'` + heredocDelim + "'\n") + for _, l := range lines { + b.WriteString(l + "\n") + } + b.WriteString(heredocDelim + "\n") + if len(probes) > 0 { + b.WriteString("for hp in " + strings.Join(probes, " ") + "; do\n") + // Any HTTP answer at all proves the address is reachable; only a + // connection failure or timeout makes curl exit non-zero without -f. + b.WriteString(` if ! curl -s -o /dev/null --connect-timeout 3 --max-time 5 "http://$hp/"; then` + "\n") + b.WriteString(` rm -f "$STAGE.new"; echo "unreachable $hp"; exit 0` + "\n") + b.WriteString(" fi\n") + b.WriteString("done\n") + } + b.WriteString(`mv "$STAGE.new" "$STAGE"` + "\n") + b.WriteString(`echo "staged $(wc -l < "$STAGE" | tr -d ' ')"` + "\n") + return b.String() +} + +// macOSProxyEnvStaged interprets the script's status line: true only when the +// env was staged. +func macOSProxyEnvStaged(out string) bool { + return strings.HasPrefix(strings.TrimSpace(out), "staged ") +} diff --git a/pkg/vm/macos_proxyenv_test.go b/pkg/vm/macos_proxyenv_test.go new file mode 100644 index 0000000..e4487a6 --- /dev/null +++ b/pkg/vm/macos_proxyenv_test.go @@ -0,0 +1,167 @@ +package vm + +import ( + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// These tests EXECUTE the generated script with a real sh and curl, because the +// script is the whole mechanism: a string that merely looks right proves +// nothing about what a shell does with it. The darwin VM code that sends it +// cannot be compiled off macOS, which is why the logic lives here. + +func requireShellTools(t *testing.T) { + t.Helper() + for _, tool := range []string{"sh", "curl"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s not on PATH: %v", tool, err) + } + } +} + +// runStageScript runs the script and returns its status line and the staged +// file's contents ("" if nothing was staged). +func runStageScript(t *testing.T, script, stage string) (status, staged string) { + t.Helper() + out, err := exec.Command("sh", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("script failed: %v\n%s\n--- script ---\n%s", err, out, script) + } + b, err := os.ReadFile(stage) + if err == nil { + staged = string(b) + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + return strings.TrimSpace(string(out)), staged +} + +func stagePath(t *testing.T) string { + return filepath.ToSlash(filepath.Join(t.TempDir(), "runner.env")) +} + +// The env the fleet's proxies emit (copied off a live node), pointed at addr. +func proxyEnv(addr string) []string { + return []string{ + "GOPROXY=http://" + addr + "|direct", + "GOSUMDB=off", + "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse", + "RUSTUP_DIST_SERVER=http://" + addr + "/rustup", + "GHREL_PROXY=http://" + addr, + } +} + +func TestMacOSProxyEnvScript_StagesWhenReachable(t *testing.T) { + requireShellTools(t) + // 404 on purpose: any HTTP answer proves reachability. + srv := httptest.NewServer(http.NotFoundHandler()) + defer srv.Close() + addr := srv.Listener.Addr().String() + stage := stagePath(t) + env := proxyEnv(addr) + + status, staged := runStageScript(t, macOSProxyEnvScript(env, stage), stage) + + if !macOSProxyEnvStaged(status) || status != "staged 5" { + t.Fatalf("status = %q, want \"staged 5\"", status) + } + if want := strings.Join(env, "\n") + "\n"; staged != want { + t.Errorf("staged env differs\n got: %q\nwant: %q", staged, want) + } +} + +// One dead address stages NOTHING — not the reachable ones, and not the +// plain-value vars like GOSUMDB=off that only make sense alongside a proxy. +// cargo and rustup do not fall back on an unreachable proxy, so a partial env +// can fail a build that would have succeeded uncached. +func TestMacOSProxyEnvScript_AllOrNothing(t *testing.T) { + requireShellTools(t) + srv := httptest.NewServer(http.NotFoundHandler()) + defer srv.Close() + + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + dead := l.Addr().String() + _ = l.Close() + + env := append(proxyEnv(srv.Listener.Addr().String()), "COMPOSER_REPO_PACKAGIST=http://"+dead) + stage := stagePath(t) + + status, staged := runStageScript(t, macOSProxyEnvScript(env, stage), stage) + + if macOSProxyEnvStaged(status) { + t.Fatalf("status = %q: staged although %s is unreachable", status, dead) + } + if status != "unreachable "+dead { + t.Errorf("status = %q, want %q", status, "unreachable "+dead) + } + if staged != "" { + t.Errorf("something was staged despite a dead proxy: %q", staged) + } + if _, err := os.Stat(stage + ".new"); !os.IsNotExist(err) { + t.Errorf("temp file left behind: %v", err) + } +} + +// Values are written byte-for-byte: the quoted heredoc must not expand $, `, +// or quotes. A GOFLAGS or URL with any of these would otherwise be corrupted +// silently, or execute. +func TestMacOSProxyEnvScript_NoExpansion(t *testing.T) { + requireShellTools(t) + stage := stagePath(t) + env := []string{`WEIRD=$HOME "q" 'sq' ` + "`id`" + ` \n $(echo pwned)`} + + status, staged := runStageScript(t, macOSProxyEnvScript(env, stage), stage) + + if status != "staged 1" { + t.Fatalf("status = %q, want \"staged 1\"", status) + } + if staged != env[0]+"\n" { + t.Errorf("value was altered by the shell\n got: %q\nwant: %q", staged, env[0]+"\n") + } +} + +func TestMacOSProxyEnvScript_NothingToDo(t *testing.T) { + for name, env := range map[string][]string{ + "nil": nil, + "empty": {}, + "malformed only": {"NOEQUALS", "=novalue"}, + "newline smuggling": {"A=b\nrm -rf /"}, + "heredoc terminator smuggled": {heredocDelim}, + } { + if got := macOSProxyEnvScript(env, "/tmp/x"); got != "" { + t.Errorf("%s: got a script, want none:\n%s", name, got) + } + } +} + +// An address that is not plainly host:port is never pasted into the probe +// loop. Since unverified means unadvertised, the whole env is dropped. +func TestMacOSProxyEnvScript_RefusesUnsafeAddress(t *testing.T) { + env := []string{"GOPROXY=http://evil$(id):8082|direct"} + if got := macOSProxyEnvScript(env, "/tmp/x"); got != "" { + t.Errorf("script generated for an unsafe address:\n%s", got) + } +} + +func TestMacOSProxyEnvStaged(t *testing.T) { + for out, want := range map[string]bool{ + "staged 5\n": true, + " staged 1": true, + "unreachable 192.168.64.1:8082": false, + "": false, + "stagedX": false, + } { + if got := macOSProxyEnvStaged(out); got != want { + t.Errorf("macOSProxyEnvStaged(%q) = %v, want %v", out, got, want) + } + } +} diff --git a/pkg/vm/macosvm_darwin.go b/pkg/vm/macosvm_darwin.go index 78cb199..d41fa38 100644 --- a/pkg/vm/macosvm_darwin.go +++ b/pkg/vm/macosvm_darwin.go @@ -615,6 +615,12 @@ if [ -f "$RUNNER_SRC/run.sh" ]; then cp -R "$RUNNER_SRC" "$RUNNER_DIR" chown -R admin:staff "$RUNNER_DIR" fi +# Cache-proxy env, staged by ephemerd only after every proxy answered from +# inside this VM (see macos_proxyenv.go). The runner loads .env into each job. +if [ -f /tmp/ephemerd-runner.env ]; then + cat /tmp/ephemerd-runner.env >> "$RUNNER_DIR/.env" + chown admin:staff "$RUNNER_DIR/.env" +fi cd "$RUNNER_DIR" ./run.sh --jitconfig '%s' /tmp/runner.log 2>&1 & RUNNER_PID=$! @@ -635,6 +641,11 @@ echo "runner started (pid=$RUNNER_PID)" // session on deadline unblocks the pending call. const sshCommandTimeout = 90 * time.Second +// macOSProxyEnvStage is where the verified cache-proxy env is staged inside +// the guest (see macos_proxyenv.go); macOSRunnerSetupScript reads the same +// path. A per-job VM boots from a fresh clone, so nothing stale is there. +const macOSProxyEnvStage = "/tmp/ephemerd-runner.env" + // runSSHCommand runs cmd on client with a hard deadline, returning its combined // output. On timeout or ctx cancellation it closes the session — which unblocks // the underlying CombinedOutput — and returns an error. The result channel is @@ -793,6 +804,8 @@ func (m *darwinMacOSVM) setupRunnerViaSSH(ctx context.Context, ip string) error // The runner is copied fresh into the VM before this runs; see // macOSRunnerSetupScript for why we always refresh it. Only the JIT // config is per-job, passed inline. + m.stageProxyEnv(ctx, client) + setupScript := fmt.Sprintf(macOSRunnerSetupScript, strings.TrimSpace(string(jitData))) session, err := client.NewSession() @@ -821,6 +834,30 @@ func (m *darwinMacOSVM) setupRunnerViaSSH(ctx context.Context, ip string) error return nil } +// stageProxyEnv verifies the cache proxies from inside the guest and stages +// their env for the runner setup script to pick up. Every failure leaves +// nothing staged and the job runs exactly as it did before — uncached, never +// broken — so errors are logged, not returned. +func (m *darwinMacOSVM) stageProxyEnv(ctx context.Context, client *ssh.Client) { + script := macOSProxyEnvScript(m.cfg.JobEnv, macOSProxyEnvStage) + if script == "" { + return + } + out, err := runSSHCommand(ctx, client, script, sshCommandTimeout) + status := strings.TrimSpace(string(out)) + switch { + case err != nil: + m.cfg.Log.Warn("could not stage cache-proxy env in the macOS VM; job runs without cache proxies", + "id", m.id, "error", err) + case macOSProxyEnvStaged(status): + m.cfg.Log.Info("cache proxies reachable from the macOS VM; advertised to the job", + "id", m.id, "status", status) + default: + m.cfg.Log.Warn("cache proxies not reachable from inside the macOS VM; job runs without them", + "id", m.id, "status", status) + } +} + func (m *darwinMacOSVM) Stop() { m.stopOnce.Do(m.stop) } diff --git a/pkg/vm/vm.go b/pkg/vm/vm.go index 91b220e..cbac103 100644 --- a/pkg/vm/vm.go +++ b/pkg/vm/vm.go @@ -120,6 +120,11 @@ type MacOSVMConfig struct { // MemoryMB per macOS VM. Defaults to 8192. MemoryMB uint64 + // JobEnv is extra KEY=VALUE env for every job's runner: the cache-proxy + // env, already rewritten to the host's address on the VM NAT. It is + // verified reachable from inside each VM before use; see macos_proxyenv.go. + JobEnv []string + Log *slog.Logger }