diff --git a/cmd/ephemerd/advertise_darwin.go b/cmd/ephemerd/advertise_darwin.go new file mode 100644 index 0000000..04ae22a --- /dev/null +++ b/cmd/ephemerd/advertise_darwin.go @@ -0,0 +1,80 @@ +//go:build darwin + +package main + +import ( + "log/slog" + "net" + "time" + + "github.com/ephpm/ephemerd/pkg/proxies" +) + +// vzNAT is the subnet Apple's Virtualization.framework NATs guests onto. The VM +// code already relies on this (pkg/vm/linuxvm_darwin.go and macosvm_darwin.go +// both sweep 192.168.64.0/24 to find a guest in ARP), so this is the same +// documented behaviour rather than a new assumption. +// +// It is used only to LOCATE a candidate host address among the local +// interfaces. The address is then probed, and dropped if it does not answer — +// so if Apple ever changes the range, the result is "no proxy env" (a +// bandwidth cost) rather than a black-holed address (a broken build). +var vzNAT = &net.IPNet{IP: net.IPv4(192, 168, 64, 0), Mask: net.CIDRMask(24, 32)} + +// probeTimeout is short on purpose: this runs on the daemon's startup path, the +// address is on a local interface, and a slow answer is indistinguishable from +// no answer for our purposes. +const probeTimeout = 2 * time.Second + +// resolveJobProxyEnv rewrites cache-proxy env so jobs on this Mac can actually +// reach the proxies, and drops anything that cannot be verified. +// +// WHY: jobs here do not run on the host. Linux jobs run in containers inside a +// Linux VM, macOS jobs inside a macOS VM, both behind the Vz NAT. The proxies +// advertise the CNI bridge gateway (10.88.0.1), which from inside a VM is the +// VM's own bridge, where nothing listens. The proxies are on the HOST. +// +// Measured on this fleet's Mac, 169 days uptime, before this change: +// +// 192.168.64.1:8082 -> 200 (0.195s) the host, on the Vz NAT +// 10.88.0.1:8082 -> 000 (4s timeout) the advertised value +// 10.88.x interfaces on the host: 0 +// caches: gomod 0B, cargo 0B, ghrel 0B, composer 0B +// +// The proxies were healthy throughout. Only the advertised address was wrong, +// and the existing health gate could not see it because it probes loopback, +// which always answers. +func resolveJobProxyEnv(env []string, log *slog.Logger) []string { + if len(env) == 0 { + return env + } + if log == nil { + log = slog.Default() + } + + host, ok := proxies.HostAddrInSubnet(vzNAT) + if !ok { + // No interface on the guest NAT: the VM subsystem is not up, or Apple + // moved the range. Advertise nothing rather than guess. + log.Warn("no host address found on the VM NAT; not advertising cache proxies to jobs", + "nat", vzNAT.String()) + return nil + } + + rewritten := proxies.RewriteEnvHost(env, host) + + // Probe every advertised address. Unlike Linux, there is no + // created-later bridge to wait for here — the address is on a live local + // interface, so a failure now is a real failure. + for _, hp := range proxies.EnvHostPorts(rewritten) { + if !proxies.Reachable(hp, probeTimeout) { + log.Warn("rewritten cache-proxy address did not answer; not advertising cache proxies to jobs", + "address", hp, "host", host) + return nil + } + } + + log.Info("cache proxies advertised to jobs on the VM-visible host address", + "host", host, "was", "cni bridge gateway", "env", rewritten) + return rewritten +} diff --git a/cmd/ephemerd/advertise_default.go b/cmd/ephemerd/advertise_default.go new file mode 100644 index 0000000..2fdee9e --- /dev/null +++ b/cmd/ephemerd/advertise_default.go @@ -0,0 +1,14 @@ +//go:build linux + +package main + +import "log/slog" + +// resolveJobProxyEnv returns the proxy env unchanged on Linux. +// +// Job containers here sit on this host's CNI bridge, so the advertised bridge +// gateway is already the right address. It deliberately is NOT probed: the +// bridge does not exist at daemon start — CNI creates it with the first job +// container (see pkg/proxies/listen.go) — so a reachability check would fail +// every time and disable a set of proxies that work perfectly. +func resolveJobProxyEnv(env []string, _ *slog.Logger) []string { return env } diff --git a/cmd/ephemerd/advertise_windows.go b/cmd/ephemerd/advertise_windows.go new file mode 100644 index 0000000..7bc81fd --- /dev/null +++ b/cmd/ephemerd/advertise_windows.go @@ -0,0 +1,24 @@ +//go:build windows + +package main + +import "log/slog" + +// resolveJobProxyEnv declines to advertise cache proxies on Windows. +// +// Container jobs run inside a Linux VM (or Hyper-V containers), so the +// advertised CNI bridge gateway is unreachable from them — the same defect +// proven on macOS, where it left every cache at 0 bytes. Windows is worse than +// useless though: a job handed an unreachable GOPROXY HANGS rather than failing +// over, so this has long been suppressed wholesale rather than corrected. +// +// Suppression stays until the Hyper-V switch address a VM can reach the host on +// is determined ON METAL and probed, exactly as the darwin path now does. +// Guessing it would reintroduce hung builds, which is strictly worse than the +// bandwidth this costs. +func resolveJobProxyEnv(_ []string, log *slog.Logger) []string { + if log != nil { + log.Info("cache proxies not advertised to jobs on windows; the host address reachable from job VMs is not yet determined (bandwidth cost, not a failure)") + } + return nil +} diff --git a/cmd/ephemerd/main.go b/cmd/ephemerd/main.go index 9d9adab..e9ae51c 100644 --- a/cmd/ephemerd/main.go +++ b/cmd/ephemerd/main.go @@ -599,6 +599,21 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP } } + // Each proxy advertises the address it was asked to bind: the CNI bridge + // gateway. That is correct ONLY where job containers share this host's + // bridge, i.e. on Linux. On macOS and Windows hosts, jobs run inside a VM + // and that address resolves to the VM's own bridge, where nothing listens — + // the proxies are out here on the host. Left uncorrected it looks like a + // working cache that never gets a single hit: this fleet's Mac ran 169 days + // with all four caches at 0 bytes while every proxy reported healthy, + // because the existing health gate probes loopback, which always answers. + // + // resolveJobProxyEnv is per-platform: identity on Linux, rewrite-and-probe + // on darwin, and nil on Windows until its VM-visible host address is + // confirmed on metal. Returning nil is the safe failure — a missing GOPROXY + // costs bandwidth, an unreachable one hangs Windows builds outright. + cacheProxyEnvVars = resolveJobProxyEnv(cacheProxyEnvVars, log) + // Start the shared embedded BuildKit solver. One solver serves every // job's `docker build` calls through pkg/dind. Only enabled when dind // is enabled and on platforms buildkit supports (linux, windows). diff --git a/pkg/proxies/advertise.go b/pkg/proxies/advertise.go new file mode 100644 index 0000000..7a64608 --- /dev/null +++ b/pkg/proxies/advertise.go @@ -0,0 +1,180 @@ +package proxies + +import ( + "net" + "net/url" + "strings" + "time" +) + +// Cache proxies bind the CNI bridge gateway and advertise that same address to +// jobs. That is correct on a Linux host, where job containers sit on the bridge +// — and wrong everywhere else. +// +// On macOS and Windows hosts, container jobs do NOT run on the host. They run +// inside a Linux VM (and macOS jobs inside a macOS VM), reached over the +// hypervisor's NAT. From in there, the bridge gateway address resolves to the +// VM's OWN bridge, where nothing is listening. The proxies are on the HOST. +// +// Measured on mfl-mac-arm64 2026-09-30, 169 days uptime: +// +// 192.168.64.1:8082 -> 200 (0.195s) host address on the Vz NAT +// 10.88.0.1:8082 -> 000 (4s timeout, advertised value) +// 10.88.x interfaces on the host: 0 +// +// Every cache on that node read 0 bytes: gomod, cargo, ghrel, composer. The +// proxies were healthy the whole time — only the address handed to jobs was +// unreachable, and the health gate could not see it because it probes over +// loopback, which always answers. + +// RewriteEnvHost rewrites the host in each "KEY=http://host:port/path" entry to +// host, preserving scheme, port and path. Entries that are not URLs are passed +// through untouched, because proxies also emit plain-value vars. +// +// Rewriting rather than re-rendering is deliberate: each proxy already knows +// how to build its own env (GOPROXY, CARGO_*, GHREL_PROXY, the lot), and only +// the authority is wrong. Re-deriving that per platform would duplicate seven +// proxies' env logic and let the copies drift. +func RewriteEnvHost(env []string, host string) []string { + if host == "" || len(env) == 0 { + return env + } + out := make([]string, 0, len(env)) + for _, e := range env { + k, v, ok := strings.Cut(e, "=") + if !ok { + out = append(out, e) + continue + } + u, err := url.Parse(v) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + out = append(out, e) + continue + } + if port := u.Port(); port != "" { + u.Host = net.JoinHostPort(host, port) + } else { + u.Host = host + } + out = append(out, k+"="+u.String()) + } + return out +} + +// HostAddrVisibleTo returns the local address that peerIP can reach this host +// on: the address of whichever local interface shares a subnet with peerIP. +// +// Derived rather than hardcoded. Apple's Virtualization.framework usually NATs +// on 192.168.64.0/24 and Hyper-V picks its own, so a constant per platform +// would be a guess that silently rots when the hypervisor changes its range. +// Asking "which of my addresses is on the VM's network" is true by +// construction. +// +// ok is false when nothing matches, and callers MUST treat that as "do not +// advertise" rather than falling back to a guess — a wrong address is what +// caused the outage this exists to fix. +func HostAddrVisibleTo(peerIP string) (string, bool) { + peer := net.ParseIP(peerIP) + if peer == nil { + return "", false + } + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", false + } + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok || ipnet.IP.IsLoopback() { + continue + } + if ipnet.Contains(peer) { + return ipnet.IP.String(), true + } + } + return "", false +} + +// HostAddrInSubnet returns this host's own IPv4 address inside subnet, which is +// the address a guest on that subnet can reach the host on. +// +// Separate from HostAddrVisibleTo because the caller often knows the +// hypervisor's NAT range but not any particular guest's address — at daemon +// start there may be no guest yet. +// +// Lives here, rather than in the platform file that uses it, so it is covered +// by tests that run on every OS. The darwin build cannot be compiled on a +// non-macOS machine (its vz dependency needs macOS cgo), so keeping logic out +// of that file is the difference between tested and merely plausible. +func HostAddrInSubnet(subnet *net.IPNet) (string, bool) { + if subnet == nil { + return "", false + } + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", false + } + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok || ipnet.IP.To4() == nil || ipnet.IP.IsLoopback() { + continue + } + if subnet.Contains(ipnet.IP) { + return ipnet.IP.String(), true + } + } + return "", false +} + +// Reachable reports whether a TCP connection to hostPort succeeds. +// +// This is the gate the old loopback health probe could not provide. A proxy +// that answers on 127.0.0.1 tells you the process is alive; it tells you +// nothing about whether the address being advertised to jobs exists. Probing +// the advertised address is the only check that would have caught either the +// macOS breakage or the Windows one. +// +// A failure means "do not inject this var". That is the safe direction: a +// missing GOPROXY costs bandwidth, while a black-holed one costs a build — +// Windows jobs HANG on an unreachable proxy rather than failing over, which is +// why proxy env is suppressed there wholesale today. +func Reachable(hostPort string, timeout time.Duration) bool { + c, err := net.DialTimeout("tcp", hostPort, timeout) + if err != nil { + return false + } + _ = c.Close() + return true +} + +// EnvHostPorts extracts the host:port authority of every URL-valued entry, so +// a caller can probe each advertised address before injecting it. +func EnvHostPorts(env []string) []string { + var out []string + seen := map[string]bool{} + for _, e := range env { + _, v, ok := strings.Cut(e, "=") + if !ok { + continue + } + u, err := url.Parse(v) + if err != nil || u.Host == "" { + continue + } + hp := u.Host + if u.Port() == "" { + switch u.Scheme { + case "http": + hp = net.JoinHostPort(u.Hostname(), "80") + case "https": + hp = net.JoinHostPort(u.Hostname(), "443") + default: + continue + } + } + if !seen[hp] { + seen[hp] = true + out = append(out, hp) + } + } + return out +} diff --git a/pkg/proxies/advertise_test.go b/pkg/proxies/advertise_test.go new file mode 100644 index 0000000..58ba42f --- /dev/null +++ b/pkg/proxies/advertise_test.go @@ -0,0 +1,189 @@ +package proxies + +import ( + "net" + "net/http" + "net/http/httptest" + "slices" + "testing" + "time" +) + +// The exact env the fleet's proxies emit, with the exact host that broke it. +func fleetEnv() []string { + return []string{ + "GOPROXY=http://10.88.0.1:8082", + "GOSUMDB=off", + "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse", + "RUSTUP_DIST_SERVER=http://10.88.0.1:8083/rustup", + "GHREL_PROXY=http://10.88.0.1:8087", + "COMPOSER_REPO_PACKAGIST=http://10.88.0.1:8088", + } +} + +func TestRewriteEnvHostFixesTheAdvertisedAddress(t *testing.T) { + got := RewriteEnvHost(fleetEnv(), "192.168.64.1") + + want := []string{ + "GOPROXY=http://192.168.64.1:8082", + "GOSUMDB=off", + "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse", + "RUSTUP_DIST_SERVER=http://192.168.64.1:8083/rustup", + "GHREL_PROXY=http://192.168.64.1:8087", + "COMPOSER_REPO_PACKAGIST=http://192.168.64.1:8088", + } + if !slices.Equal(got, want) { + t.Errorf("RewriteEnvHost mismatch\n got: %q\nwant: %q", got, want) + } +} + +// Ports and paths carry real meaning — RUSTUP_DIST_SERVER's /rustup route and +// each proxy's distinct port. Clobbering either would swap one broken address +// for another, which is the failure mode this whole change exists to end. +func TestRewriteEnvHostPreservesPortAndPath(t *testing.T) { + got := RewriteEnvHost([]string{"RUSTUP_DIST_SERVER=http://10.88.0.1:8083/rustup"}, "172.20.0.1") + if got[0] != "RUSTUP_DIST_SERVER=http://172.20.0.1:8083/rustup" { + t.Errorf("got %q", got[0]) + } +} + +// Not every emitted var is a URL (GOSUMDB=off, CARGO_..._PROTOCOL=sparse). +// Mangling those would break the toolchain in a way unrelated to networking. +func TestRewriteEnvHostLeavesNonURLsAlone(t *testing.T) { + in := []string{"GOSUMDB=off", "GOFLAGS=-mod=mod", "NOT_AN_ENTRY", "X=not://a real url"} + got := RewriteEnvHost(in, "10.0.0.1") + if !slices.Equal(got, in) { + t.Errorf("non-URL entries were modified\n got: %q\nwant: %q", got, in) + } +} + +// An empty host must be a no-op, so a caller that could not determine the +// address leaves env untouched rather than producing "http://:8082". +func TestRewriteEnvHostEmptyHostIsNoOp(t *testing.T) { + in := fleetEnv() + if got := RewriteEnvHost(in, ""); !slices.Equal(got, in) { + t.Errorf("empty host rewrote env: %q", got) + } +} + +// Reachable must distinguish a live listener from a dead address. This is the +// check the old loopback health gate could not perform, and the reason the +// macOS caches sat at 0 bytes for 169 days while every proxy reported healthy. +func TestReachableDistinguishesLiveFromDead(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})) + defer srv.Close() + live := srv.Listener.Addr().String() + + if !Reachable(live, 2*time.Second) { + t.Errorf("Reachable(%s) = false for a live listener", live) + } + + // Bind then close to get a port nothing is listening on. + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + dead := l.Addr().String() + _ = l.Close() + + if Reachable(dead, 500*time.Millisecond) { + t.Errorf("Reachable(%s) = true for a closed port", dead) + } +} + +// 10.88.0.1 is not routable from the host on macOS, so the probe must fail +// rather than hang past its timeout — the gate has to be cheap enough to run +// at every daemon start. +func TestReachableRespectsTimeout(t *testing.T) { + start := time.Now() + if Reachable("10.88.0.1:8082", 300*time.Millisecond) { + t.Skip("10.88.0.1:8082 is reachable on this host; nothing to assert") + } + if el := time.Since(start); el > 3*time.Second { + t.Errorf("probe took %s, far beyond its 300ms timeout", el) + } +} + +func TestEnvHostPortsCollectsAdvertisedAddresses(t *testing.T) { + got := EnvHostPorts(fleetEnv()) + want := []string{"10.88.0.1:8082", "10.88.0.1:8083", "10.88.0.1:8087", "10.88.0.1:8088"} + if !slices.Equal(got, want) { + t.Errorf("got %q want %q", got, want) + } +} + +// Derivation, not a hardcoded hypervisor constant: the host address a peer can +// reach us on is whichever of our interfaces shares its subnet. A peer on no +// local subnet must report false so the caller declines to advertise instead of +// guessing — guessing is precisely what produced the outage. +func TestHostAddrVisibleTo(t *testing.T) { + addrs, err := net.InterfaceAddrs() + if err != nil { + t.Skip(err) + } + var probed bool + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok || ipnet.IP.IsLoopback() || ipnet.IP.To4() == nil { + continue + } + // Our own address is trivially on our own subnet. + got, ok2 := HostAddrVisibleTo(ipnet.IP.String()) + if !ok2 { + t.Errorf("HostAddrVisibleTo(%s) = not found, but it is our own address", ipnet.IP) + continue + } + if net.ParseIP(got) == nil { + t.Errorf("HostAddrVisibleTo(%s) returned non-IP %q", ipnet.IP, got) + } + probed = true + break + } + if !probed { + t.Skip("no non-loopback IPv4 interface to test against") + } + + if _, ok := HostAddrVisibleTo("203.0.113.7"); ok { + t.Error("HostAddrVisibleTo(public IP) returned a match; it should decline") + } + if _, ok := HostAddrVisibleTo("not-an-ip"); ok { + t.Error("HostAddrVisibleTo(garbage) returned a match") + } +} + +// HostAddrInSubnet is what the darwin path uses to find the Vz NAT host +// address. It is tested here, not there, because the darwin build needs macOS +// cgo and cannot be compiled on CI's other platforms — logic in that file is +// unverifiable, logic here is not. +func TestHostAddrInSubnet(t *testing.T) { + // A subnet containing one of our own addresses must return that address. + addrs, _ := net.InterfaceAddrs() + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok || ipnet.IP.To4() == nil || ipnet.IP.IsLoopback() { + continue + } + got, found := HostAddrInSubnet(ipnet) + if !found { + t.Errorf("HostAddrInSubnet(%s) = not found for a subnet we are on", ipnet) + } else if net.ParseIP(got) == nil { + t.Errorf("returned non-IP %q", got) + } + break + } + + // Apple's Vz range, when we are not on it, must decline rather than guess. + vz := &net.IPNet{IP: net.IPv4(192, 168, 64, 0), Mask: net.CIDRMask(24, 32)} + if got, ok := HostAddrInSubnet(vz); ok && !vz.Contains(net.ParseIP(got)) { + t.Errorf("returned %q which is not inside %s", got, vz) + } + + // TEST-NET-3: nothing is ever on it. + bogus := &net.IPNet{IP: net.IPv4(203, 0, 113, 0), Mask: net.CIDRMask(24, 32)} + if _, ok := HostAddrInSubnet(bogus); ok { + t.Error("HostAddrInSubnet(203.0.113.0/24) matched; it should decline") + } + if _, ok := HostAddrInSubnet(nil); ok { + t.Error("HostAddrInSubnet(nil) matched") + } +}