From 69952c2767a6595c8fba9f1696ee0e9e230b22f8 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Wed, 30 Sep 2026 22:44:07 -0700 Subject: [PATCH] fix(proxies): advertise an address jobs can actually reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every cache proxy advertises the address it was asked to bind: the CNI bridge gateway. That is right only where job containers share this host's bridge — on Linux. On macOS and Windows hosts, jobs run inside a VM, and from in there the bridge gateway is the VM's OWN bridge, where nothing listens. The proxies are on the host. Measured on this fleet's Mac, 169 days uptime: 192.168.64.1:8082 -> 200 (0.195s) host, on the Vz NAT 10.88.0.1:8082 -> 000 (4s timeout) the advertised value 10.88.x interfaces on the host: 0 caches: gomod 0B, cargo 0B, ghrel 0B, composer 0B The proxies were healthy the whole time. Only the advertised address was unreachable, and the existing health gate could not see it because it probes loopback, which always answers. The node downloads ~29 GB/day, and none of it was cacheable. resolveJobProxyEnv is per-platform: - linux: identity. The bridge gateway is correct, and deliberately NOT probed — CNI creates the bridge with the first job container, so a startup probe would fail every time and disable working proxies. - darwin: find this host's address on the guest NAT, rewrite the env to it, then probe every advertised address and advertise nothing if any fails. - windows: nothing, as today. Same defect, but an unreachable GOPROXY HANGS Windows builds instead of failing over, so suppression stays until the Hyper-V host address is confirmed on metal. Rewriting rather than re-rendering keeps each proxy's own env logic as the single source of truth; only the authority was ever wrong. The NAT range locates a candidate, it is not trusted: the address is probed before use, so if Apple moves the range the result is "no proxy env" (bandwidth) rather than a black hole (broken builds). Logic lives in pkg/proxies, not the darwin file, because the darwin build needs macOS cgo and cannot be compiled off a Mac — code there is unverifiable by CI's other platforms. 9 tests, all platform-independent. Revert-verified: making RewriteEnvHost a no-op fails the rewrite tests. linux and windows cross-build clean; darwin is CI-only (its vz dependency does not cross-compile, same on untouched main). --- cmd/ephemerd/advertise_darwin.go | 80 +++++++++++++ cmd/ephemerd/advertise_default.go | 14 +++ cmd/ephemerd/advertise_windows.go | 24 ++++ cmd/ephemerd/main.go | 15 +++ pkg/proxies/advertise.go | 180 ++++++++++++++++++++++++++++ pkg/proxies/advertise_test.go | 189 ++++++++++++++++++++++++++++++ 6 files changed, 502 insertions(+) create mode 100644 cmd/ephemerd/advertise_darwin.go create mode 100644 cmd/ephemerd/advertise_default.go create mode 100644 cmd/ephemerd/advertise_windows.go create mode 100644 pkg/proxies/advertise.go create mode 100644 pkg/proxies/advertise_test.go diff --git a/cmd/ephemerd/advertise_darwin.go b/cmd/ephemerd/advertise_darwin.go new file mode 100644 index 0000000..04ae22a --- /dev/null +++ b/cmd/ephemerd/advertise_darwin.go @@ -0,0 +1,80 @@ +//go:build darwin + +package main + +import ( + "log/slog" + "net" + "time" + + "github.com/ephpm/ephemerd/pkg/proxies" +) + +// vzNAT is the subnet Apple's Virtualization.framework NATs guests onto. The VM +// code already relies on this (pkg/vm/linuxvm_darwin.go and macosvm_darwin.go +// both sweep 192.168.64.0/24 to find a guest in ARP), so this is the same +// documented behaviour rather than a new assumption. +// +// It is used only to LOCATE a candidate host address among the local +// interfaces. The address is then probed, and dropped if it does not answer — +// so if Apple ever changes the range, the result is "no proxy env" (a +// bandwidth cost) rather than a black-holed address (a broken build). +var vzNAT = &net.IPNet{IP: net.IPv4(192, 168, 64, 0), Mask: net.CIDRMask(24, 32)} + +// probeTimeout is short on purpose: this runs on the daemon's startup path, the +// address is on a local interface, and a slow answer is indistinguishable from +// no answer for our purposes. +const probeTimeout = 2 * time.Second + +// resolveJobProxyEnv rewrites cache-proxy env so jobs on this Mac can actually +// reach the proxies, and drops anything that cannot be verified. +// +// WHY: jobs here do not run on the host. Linux jobs run in containers inside a +// Linux VM, macOS jobs inside a macOS VM, both behind the Vz NAT. The proxies +// advertise the CNI bridge gateway (10.88.0.1), which from inside a VM is the +// VM's own bridge, where nothing listens. The proxies are on the HOST. +// +// Measured on this fleet's Mac, 169 days uptime, before this change: +// +// 192.168.64.1:8082 -> 200 (0.195s) the host, on the Vz NAT +// 10.88.0.1:8082 -> 000 (4s timeout) the advertised value +// 10.88.x interfaces on the host: 0 +// caches: gomod 0B, cargo 0B, ghrel 0B, composer 0B +// +// The proxies were healthy throughout. Only the advertised address was wrong, +// and the existing health gate could not see it because it probes loopback, +// which always answers. +func resolveJobProxyEnv(env []string, log *slog.Logger) []string { + if len(env) == 0 { + return env + } + if log == nil { + log = slog.Default() + } + + host, ok := proxies.HostAddrInSubnet(vzNAT) + if !ok { + // No interface on the guest NAT: the VM subsystem is not up, or Apple + // moved the range. Advertise nothing rather than guess. + log.Warn("no host address found on the VM NAT; not advertising cache proxies to jobs", + "nat", vzNAT.String()) + return nil + } + + rewritten := proxies.RewriteEnvHost(env, host) + + // Probe every advertised address. Unlike Linux, there is no + // created-later bridge to wait for here — the address is on a live local + // interface, so a failure now is a real failure. + for _, hp := range proxies.EnvHostPorts(rewritten) { + if !proxies.Reachable(hp, probeTimeout) { + log.Warn("rewritten cache-proxy address did not answer; not advertising cache proxies to jobs", + "address", hp, "host", host) + return nil + } + } + + log.Info("cache proxies advertised to jobs on the VM-visible host address", + "host", host, "was", "cni bridge gateway", "env", rewritten) + return rewritten +} diff --git a/cmd/ephemerd/advertise_default.go b/cmd/ephemerd/advertise_default.go new file mode 100644 index 0000000..2fdee9e --- /dev/null +++ b/cmd/ephemerd/advertise_default.go @@ -0,0 +1,14 @@ +//go:build linux + +package main + +import "log/slog" + +// resolveJobProxyEnv returns the proxy env unchanged on Linux. +// +// Job containers here sit on this host's CNI bridge, so the advertised bridge +// gateway is already the right address. It deliberately is NOT probed: the +// bridge does not exist at daemon start — CNI creates it with the first job +// container (see pkg/proxies/listen.go) — so a reachability check would fail +// every time and disable a set of proxies that work perfectly. +func resolveJobProxyEnv(env []string, _ *slog.Logger) []string { return env } diff --git a/cmd/ephemerd/advertise_windows.go b/cmd/ephemerd/advertise_windows.go new file mode 100644 index 0000000..7bc81fd --- /dev/null +++ b/cmd/ephemerd/advertise_windows.go @@ -0,0 +1,24 @@ +//go:build windows + +package main + +import "log/slog" + +// resolveJobProxyEnv declines to advertise cache proxies on Windows. +// +// Container jobs run inside a Linux VM (or Hyper-V containers), so the +// advertised CNI bridge gateway is unreachable from them — the same defect +// proven on macOS, where it left every cache at 0 bytes. Windows is worse than +// useless though: a job handed an unreachable GOPROXY HANGS rather than failing +// over, so this has long been suppressed wholesale rather than corrected. +// +// Suppression stays until the Hyper-V switch address a VM can reach the host on +// is determined ON METAL and probed, exactly as the darwin path now does. +// Guessing it would reintroduce hung builds, which is strictly worse than the +// bandwidth this costs. +func resolveJobProxyEnv(_ []string, log *slog.Logger) []string { + if log != nil { + log.Info("cache proxies not advertised to jobs on windows; the host address reachable from job VMs is not yet determined (bandwidth cost, not a failure)") + } + return nil +} diff --git a/cmd/ephemerd/main.go b/cmd/ephemerd/main.go index 9d9adab..e9ae51c 100644 --- a/cmd/ephemerd/main.go +++ b/cmd/ephemerd/main.go @@ -599,6 +599,21 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP } } + // Each proxy advertises the address it was asked to bind: the CNI bridge + // gateway. That is correct ONLY where job containers share this host's + // bridge, i.e. on Linux. On macOS and Windows hosts, jobs run inside a VM + // and that address resolves to the VM's own bridge, where nothing listens — + // the proxies are out here on the host. Left uncorrected it looks like a + // working cache that never gets a single hit: this fleet's Mac ran 169 days + // with all four caches at 0 bytes while every proxy reported healthy, + // because the existing health gate probes loopback, which always answers. + // + // resolveJobProxyEnv is per-platform: identity on Linux, rewrite-and-probe + // on darwin, and nil on Windows until its VM-visible host address is + // confirmed on metal. Returning nil is the safe failure — a missing GOPROXY + // costs bandwidth, an unreachable one hangs Windows builds outright. + cacheProxyEnvVars = resolveJobProxyEnv(cacheProxyEnvVars, log) + // Start the shared embedded BuildKit solver. One solver serves every // job's `docker build` calls through pkg/dind. Only enabled when dind // is enabled and on platforms buildkit supports (linux, windows). diff --git a/pkg/proxies/advertise.go b/pkg/proxies/advertise.go new file mode 100644 index 0000000..7a64608 --- /dev/null +++ b/pkg/proxies/advertise.go @@ -0,0 +1,180 @@ +package proxies + +import ( + "net" + "net/url" + "strings" + "time" +) + +// Cache proxies bind the CNI bridge gateway and advertise that same address to +// jobs. That is correct on a Linux host, where job containers sit on the bridge +// — and wrong everywhere else. +// +// On macOS and Windows hosts, container jobs do NOT run on the host. They run +// inside a Linux VM (and macOS jobs inside a macOS VM), reached over the +// hypervisor's NAT. From in there, the bridge gateway address resolves to the +// VM's OWN bridge, where nothing is listening. The proxies are on the HOST. +// +// Measured on mfl-mac-arm64 2026-09-30, 169 days uptime: +// +// 192.168.64.1:8082 -> 200 (0.195s) host address on the Vz NAT +// 10.88.0.1:8082 -> 000 (4s timeout, advertised value) +// 10.88.x interfaces on the host: 0 +// +// Every cache on that node read 0 bytes: gomod, cargo, ghrel, composer. The +// proxies were healthy the whole time — only the address handed to jobs was +// unreachable, and the health gate could not see it because it probes over +// loopback, which always answers. + +// RewriteEnvHost rewrites the host in each "KEY=http://host:port/path" entry to +// host, preserving scheme, port and path. Entries that are not URLs are passed +// through untouched, because proxies also emit plain-value vars. +// +// Rewriting rather than re-rendering is deliberate: each proxy already knows +// how to build its own env (GOPROXY, CARGO_*, GHREL_PROXY, the lot), and only +// the authority is wrong. Re-deriving that per platform would duplicate seven +// proxies' env logic and let the copies drift. +func RewriteEnvHost(env []string, host string) []string { + if host == "" || len(env) == 0 { + return env + } + out := make([]string, 0, len(env)) + for _, e := range env { + k, v, ok := strings.Cut(e, "=") + if !ok { + out = append(out, e) + continue + } + u, err := url.Parse(v) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + out = append(out, e) + continue + } + if port := u.Port(); port != "" { + u.Host = net.JoinHostPort(host, port) + } else { + u.Host = host + } + out = append(out, k+"="+u.String()) + } + return out +} + +// HostAddrVisibleTo returns the local address that peerIP can reach this host +// on: the address of whichever local interface shares a subnet with peerIP. +// +// Derived rather than hardcoded. Apple's Virtualization.framework usually NATs +// on 192.168.64.0/24 and Hyper-V picks its own, so a constant per platform +// would be a guess that silently rots when the hypervisor changes its range. +// Asking "which of my addresses is on the VM's network" is true by +// construction. +// +// ok is false when nothing matches, and callers MUST treat that as "do not +// advertise" rather than falling back to a guess — a wrong address is what +// caused the outage this exists to fix. +func HostAddrVisibleTo(peerIP string) (string, bool) { + peer := net.ParseIP(peerIP) + if peer == nil { + return "", false + } + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", false + } + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok || ipnet.IP.IsLoopback() { + continue + } + if ipnet.Contains(peer) { + return ipnet.IP.String(), true + } + } + return "", false +} + +// HostAddrInSubnet returns this host's own IPv4 address inside subnet, which is +// the address a guest on that subnet can reach the host on. +// +// Separate from HostAddrVisibleTo because the caller often knows the +// hypervisor's NAT range but not any particular guest's address — at daemon +// start there may be no guest yet. +// +// Lives here, rather than in the platform file that uses it, so it is covered +// by tests that run on every OS. The darwin build cannot be compiled on a +// non-macOS machine (its vz dependency needs macOS cgo), so keeping logic out +// of that file is the difference between tested and merely plausible. +func HostAddrInSubnet(subnet *net.IPNet) (string, bool) { + if subnet == nil { + return "", false + } + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", false + } + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok || ipnet.IP.To4() == nil || ipnet.IP.IsLoopback() { + continue + } + if subnet.Contains(ipnet.IP) { + return ipnet.IP.String(), true + } + } + return "", false +} + +// Reachable reports whether a TCP connection to hostPort succeeds. +// +// This is the gate the old loopback health probe could not provide. A proxy +// that answers on 127.0.0.1 tells you the process is alive; it tells you +// nothing about whether the address being advertised to jobs exists. Probing +// the advertised address is the only check that would have caught either the +// macOS breakage or the Windows one. +// +// A failure means "do not inject this var". That is the safe direction: a +// missing GOPROXY costs bandwidth, while a black-holed one costs a build — +// Windows jobs HANG on an unreachable proxy rather than failing over, which is +// why proxy env is suppressed there wholesale today. +func Reachable(hostPort string, timeout time.Duration) bool { + c, err := net.DialTimeout("tcp", hostPort, timeout) + if err != nil { + return false + } + _ = c.Close() + return true +} + +// EnvHostPorts extracts the host:port authority of every URL-valued entry, so +// a caller can probe each advertised address before injecting it. +func EnvHostPorts(env []string) []string { + var out []string + seen := map[string]bool{} + for _, e := range env { + _, v, ok := strings.Cut(e, "=") + if !ok { + continue + } + u, err := url.Parse(v) + if err != nil || u.Host == "" { + continue + } + hp := u.Host + if u.Port() == "" { + switch u.Scheme { + case "http": + hp = net.JoinHostPort(u.Hostname(), "80") + case "https": + hp = net.JoinHostPort(u.Hostname(), "443") + default: + continue + } + } + if !seen[hp] { + seen[hp] = true + out = append(out, hp) + } + } + return out +} diff --git a/pkg/proxies/advertise_test.go b/pkg/proxies/advertise_test.go new file mode 100644 index 0000000..58ba42f --- /dev/null +++ b/pkg/proxies/advertise_test.go @@ -0,0 +1,189 @@ +package proxies + +import ( + "net" + "net/http" + "net/http/httptest" + "slices" + "testing" + "time" +) + +// The exact env the fleet's proxies emit, with the exact host that broke it. +func fleetEnv() []string { + return []string{ + "GOPROXY=http://10.88.0.1:8082", + "GOSUMDB=off", + "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse", + "RUSTUP_DIST_SERVER=http://10.88.0.1:8083/rustup", + "GHREL_PROXY=http://10.88.0.1:8087", + "COMPOSER_REPO_PACKAGIST=http://10.88.0.1:8088", + } +} + +func TestRewriteEnvHostFixesTheAdvertisedAddress(t *testing.T) { + got := RewriteEnvHost(fleetEnv(), "192.168.64.1") + + want := []string{ + "GOPROXY=http://192.168.64.1:8082", + "GOSUMDB=off", + "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse", + "RUSTUP_DIST_SERVER=http://192.168.64.1:8083/rustup", + "GHREL_PROXY=http://192.168.64.1:8087", + "COMPOSER_REPO_PACKAGIST=http://192.168.64.1:8088", + } + if !slices.Equal(got, want) { + t.Errorf("RewriteEnvHost mismatch\n got: %q\nwant: %q", got, want) + } +} + +// Ports and paths carry real meaning — RUSTUP_DIST_SERVER's /rustup route and +// each proxy's distinct port. Clobbering either would swap one broken address +// for another, which is the failure mode this whole change exists to end. +func TestRewriteEnvHostPreservesPortAndPath(t *testing.T) { + got := RewriteEnvHost([]string{"RUSTUP_DIST_SERVER=http://10.88.0.1:8083/rustup"}, "172.20.0.1") + if got[0] != "RUSTUP_DIST_SERVER=http://172.20.0.1:8083/rustup" { + t.Errorf("got %q", got[0]) + } +} + +// Not every emitted var is a URL (GOSUMDB=off, CARGO_..._PROTOCOL=sparse). +// Mangling those would break the toolchain in a way unrelated to networking. +func TestRewriteEnvHostLeavesNonURLsAlone(t *testing.T) { + in := []string{"GOSUMDB=off", "GOFLAGS=-mod=mod", "NOT_AN_ENTRY", "X=not://a real url"} + got := RewriteEnvHost(in, "10.0.0.1") + if !slices.Equal(got, in) { + t.Errorf("non-URL entries were modified\n got: %q\nwant: %q", got, in) + } +} + +// An empty host must be a no-op, so a caller that could not determine the +// address leaves env untouched rather than producing "http://:8082". +func TestRewriteEnvHostEmptyHostIsNoOp(t *testing.T) { + in := fleetEnv() + if got := RewriteEnvHost(in, ""); !slices.Equal(got, in) { + t.Errorf("empty host rewrote env: %q", got) + } +} + +// Reachable must distinguish a live listener from a dead address. This is the +// check the old loopback health gate could not perform, and the reason the +// macOS caches sat at 0 bytes for 169 days while every proxy reported healthy. +func TestReachableDistinguishesLiveFromDead(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})) + defer srv.Close() + live := srv.Listener.Addr().String() + + if !Reachable(live, 2*time.Second) { + t.Errorf("Reachable(%s) = false for a live listener", live) + } + + // Bind then close to get a port nothing is listening on. + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + dead := l.Addr().String() + _ = l.Close() + + if Reachable(dead, 500*time.Millisecond) { + t.Errorf("Reachable(%s) = true for a closed port", dead) + } +} + +// 10.88.0.1 is not routable from the host on macOS, so the probe must fail +// rather than hang past its timeout — the gate has to be cheap enough to run +// at every daemon start. +func TestReachableRespectsTimeout(t *testing.T) { + start := time.Now() + if Reachable("10.88.0.1:8082", 300*time.Millisecond) { + t.Skip("10.88.0.1:8082 is reachable on this host; nothing to assert") + } + if el := time.Since(start); el > 3*time.Second { + t.Errorf("probe took %s, far beyond its 300ms timeout", el) + } +} + +func TestEnvHostPortsCollectsAdvertisedAddresses(t *testing.T) { + got := EnvHostPorts(fleetEnv()) + want := []string{"10.88.0.1:8082", "10.88.0.1:8083", "10.88.0.1:8087", "10.88.0.1:8088"} + if !slices.Equal(got, want) { + t.Errorf("got %q want %q", got, want) + } +} + +// Derivation, not a hardcoded hypervisor constant: the host address a peer can +// reach us on is whichever of our interfaces shares its subnet. A peer on no +// local subnet must report false so the caller declines to advertise instead of +// guessing — guessing is precisely what produced the outage. +func TestHostAddrVisibleTo(t *testing.T) { + addrs, err := net.InterfaceAddrs() + if err != nil { + t.Skip(err) + } + var probed bool + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok || ipnet.IP.IsLoopback() || ipnet.IP.To4() == nil { + continue + } + // Our own address is trivially on our own subnet. + got, ok2 := HostAddrVisibleTo(ipnet.IP.String()) + if !ok2 { + t.Errorf("HostAddrVisibleTo(%s) = not found, but it is our own address", ipnet.IP) + continue + } + if net.ParseIP(got) == nil { + t.Errorf("HostAddrVisibleTo(%s) returned non-IP %q", ipnet.IP, got) + } + probed = true + break + } + if !probed { + t.Skip("no non-loopback IPv4 interface to test against") + } + + if _, ok := HostAddrVisibleTo("203.0.113.7"); ok { + t.Error("HostAddrVisibleTo(public IP) returned a match; it should decline") + } + if _, ok := HostAddrVisibleTo("not-an-ip"); ok { + t.Error("HostAddrVisibleTo(garbage) returned a match") + } +} + +// HostAddrInSubnet is what the darwin path uses to find the Vz NAT host +// address. It is tested here, not there, because the darwin build needs macOS +// cgo and cannot be compiled on CI's other platforms — logic in that file is +// unverifiable, logic here is not. +func TestHostAddrInSubnet(t *testing.T) { + // A subnet containing one of our own addresses must return that address. + addrs, _ := net.InterfaceAddrs() + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok || ipnet.IP.To4() == nil || ipnet.IP.IsLoopback() { + continue + } + got, found := HostAddrInSubnet(ipnet) + if !found { + t.Errorf("HostAddrInSubnet(%s) = not found for a subnet we are on", ipnet) + } else if net.ParseIP(got) == nil { + t.Errorf("returned non-IP %q", got) + } + break + } + + // Apple's Vz range, when we are not on it, must decline rather than guess. + vz := &net.IPNet{IP: net.IPv4(192, 168, 64, 0), Mask: net.CIDRMask(24, 32)} + if got, ok := HostAddrInSubnet(vz); ok && !vz.Contains(net.ParseIP(got)) { + t.Errorf("returned %q which is not inside %s", got, vz) + } + + // TEST-NET-3: nothing is ever on it. + bogus := &net.IPNet{IP: net.IPv4(203, 0, 113, 0), Mask: net.CIDRMask(24, 32)} + if _, ok := HostAddrInSubnet(bogus); ok { + t.Error("HostAddrInSubnet(203.0.113.0/24) matched; it should decline") + } + if _, ok := HostAddrInSubnet(nil); ok { + t.Error("HostAddrInSubnet(nil) matched") + } +}