fix(proxies): advertise an address jobs can actually reach

cmd/ephemerd/advertise_darwin.go

@@ -0,0 +1,80 @@
+//go:build darwin
+
+package main
+
+import (
+	"log/slog"
+	"net"
+	"time"
+
+	"github.com/ephpm/ephemerd/pkg/proxies"
+)
+
+// vzNAT is the subnet Apple's Virtualization.framework NATs guests onto. The VM
+// code already relies on this (pkg/vm/linuxvm_darwin.go and macosvm_darwin.go
+// both sweep 192.168.64.0/24 to find a guest in ARP), so this is the same
+// documented behaviour rather than a new assumption.
+//
+// It is used only to LOCATE a candidate host address among the local
+// interfaces. The address is then probed, and dropped if it does not answer —
+// so if Apple ever changes the range, the result is "no proxy env" (a
+// bandwidth cost) rather than a black-holed address (a broken build).
+var vzNAT = &net.IPNet{IP: net.IPv4(192, 168, 64, 0), Mask: net.CIDRMask(24, 32)}
+
+// probeTimeout is short on purpose: this runs on the daemon's startup path, the
+// address is on a local interface, and a slow answer is indistinguishable from
+// no answer for our purposes.
+const probeTimeout = 2 * time.Second
+
+// resolveJobProxyEnv rewrites cache-proxy env so jobs on this Mac can actually
+// reach the proxies, and drops anything that cannot be verified.
+//
+// WHY: jobs here do not run on the host. Linux jobs run in containers inside a
+// Linux VM, macOS jobs inside a macOS VM, both behind the Vz NAT. The proxies
+// advertise the CNI bridge gateway (10.88.0.1), which from inside a VM is the
+// VM's own bridge, where nothing listens. The proxies are on the HOST.
+//
+// Measured on this fleet's Mac, 169 days uptime, before this change:
+//
+//	192.168.64.1:8082 -> 200 (0.195s)     the host, on the Vz NAT
+//	10.88.0.1:8082     -> 000 (4s timeout) the advertised value
+//	10.88.x interfaces on the host: 0
+//	caches: gomod 0B, cargo 0B, ghrel 0B, composer 0B
+//
+// The proxies were healthy throughout. Only the advertised address was wrong,
+// and the existing health gate could not see it because it probes loopback,
+// which always answers.
+func resolveJobProxyEnv(env []string, log *slog.Logger) []string {
+	if len(env) == 0 {
+		return env
+	}
+	if log == nil {
+		log = slog.Default()
+	}
+
+	host, ok := proxies.HostAddrInSubnet(vzNAT)
+	if !ok {
+		// No interface on the guest NAT: the VM subsystem is not up, or Apple
+		// moved the range. Advertise nothing rather than guess.
+		log.Warn("no host address found on the VM NAT; not advertising cache proxies to jobs",
+			"nat", vzNAT.String())
+		return nil
+	}
+
+	rewritten := proxies.RewriteEnvHost(env, host)
+
+	// Probe every advertised address. Unlike Linux, there is no
+	// created-later bridge to wait for here — the address is on a live local
+	// interface, so a failure now is a real failure.
+	for _, hp := range proxies.EnvHostPorts(rewritten) {
+		if !proxies.Reachable(hp, probeTimeout) {
+			log.Warn("rewritten cache-proxy address did not answer; not advertising cache proxies to jobs",
+				"address", hp, "host", host)
+			return nil
+		}
+	}
+
+	log.Info("cache proxies advertised to jobs on the VM-visible host address",
+		"host", host, "was", "cni bridge gateway", "env", rewritten)
+	return rewritten
+}

cmd/ephemerd/advertise_default.go

@@ -0,0 +1,14 @@
+//go:build linux
+
+package main
+
+import "log/slog"
+
+// resolveJobProxyEnv returns the proxy env unchanged on Linux.
+//
+// Job containers here sit on this host's CNI bridge, so the advertised bridge
+// gateway is already the right address. It deliberately is NOT probed: the
+// bridge does not exist at daemon start — CNI creates it with the first job
+// container (see pkg/proxies/listen.go) — so a reachability check would fail
+// every time and disable a set of proxies that work perfectly.
+func resolveJobProxyEnv(env []string, _ *slog.Logger) []string { return env }

cmd/ephemerd/advertise_windows.go

@@ -0,0 +1,24 @@
+//go:build windows
+
+package main
+
+import "log/slog"
+
+// resolveJobProxyEnv declines to advertise cache proxies on Windows.
+//
+// Container jobs run inside a Linux VM (or Hyper-V containers), so the
+// advertised CNI bridge gateway is unreachable from them — the same defect
+// proven on macOS, where it left every cache at 0 bytes. Windows is worse than
+// useless though: a job handed an unreachable GOPROXY HANGS rather than failing
+// over, so this has long been suppressed wholesale rather than corrected.
+//
+// Suppression stays until the Hyper-V switch address a VM can reach the host on
+// is determined ON METAL and probed, exactly as the darwin path now does.
+// Guessing it would reintroduce hung builds, which is strictly worse than the
+// bandwidth this costs.
+func resolveJobProxyEnv(_ []string, log *slog.Logger) []string {
+	if log != nil {
+		log.Info("cache proxies not advertised to jobs on windows; the host address reachable from job VMs is not yet determined (bandwidth cost, not a failure)")
+	}
+	return nil
+}

cmd/ephemerd/main.go

@@ -599,6 +599,21 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP
 		}
 	}
 
+	// Each proxy advertises the address it was asked to bind: the CNI bridge
+	// gateway. That is correct ONLY where job containers share this host's
+	// bridge, i.e. on Linux. On macOS and Windows hosts, jobs run inside a VM
+	// and that address resolves to the VM's own bridge, where nothing listens —
+	// the proxies are out here on the host. Left uncorrected it looks like a
+	// working cache that never gets a single hit: this fleet's Mac ran 169 days
+	// with all four caches at 0 bytes while every proxy reported healthy,
+	// because the existing health gate probes loopback, which always answers.
+	//
+	// resolveJobProxyEnv is per-platform: identity on Linux, rewrite-and-probe
+	// on darwin, and nil on Windows until its VM-visible host address is
+	// confirmed on metal. Returning nil is the safe failure — a missing GOPROXY
+	// costs bandwidth, an unreachable one hangs Windows builds outright.
+	cacheProxyEnvVars = resolveJobProxyEnv(cacheProxyEnvVars, log)
+
 	// Start the shared embedded BuildKit solver. One solver serves every
 	// job's `docker build` calls through pkg/dind. Only enabled when dind
 	// is enabled and on platforms buildkit supports (linux, windows).

pkg/proxies/advertise.go

@@ -0,0 +1,180 @@
+package proxies
+
+import (
+	"net"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// Cache proxies bind the CNI bridge gateway and advertise that same address to
+// jobs. That is correct on a Linux host, where job containers sit on the bridge
+// — and wrong everywhere else.
+//
+// On macOS and Windows hosts, container jobs do NOT run on the host. They run
+// inside a Linux VM (and macOS jobs inside a macOS VM), reached over the
+// hypervisor's NAT. From in there, the bridge gateway address resolves to the
+// VM's OWN bridge, where nothing is listening. The proxies are on the HOST.
+//
+// Measured on mfl-mac-arm64 2026-09-30, 169 days uptime:
+//
+//	192.168.64.1:8082  -> 200  (0.195s)   host address on the Vz NAT
+//	10.88.0.1:8082      -> 000  (4s timeout, advertised value)
+//	10.88.x interfaces on the host: 0
+//
+// Every cache on that node read 0 bytes: gomod, cargo, ghrel, composer. The
+// proxies were healthy the whole time — only the address handed to jobs was
+// unreachable, and the health gate could not see it because it probes over
+// loopback, which always answers.
+
+// RewriteEnvHost rewrites the host in each "KEY=http://host:port/path" entry to
+// host, preserving scheme, port and path. Entries that are not URLs are passed
+// through untouched, because proxies also emit plain-value vars.
+//
+// Rewriting rather than re-rendering is deliberate: each proxy already knows
+// how to build its own env (GOPROXY, CARGO_*, GHREL_PROXY, the lot), and only
+// the authority is wrong. Re-deriving that per platform would duplicate seven
+// proxies' env logic and let the copies drift.
+func RewriteEnvHost(env []string, host string) []string {
+	if host == "" || len(env) == 0 {
+		return env
+	}
+	out := make([]string, 0, len(env))
+	for _, e := range env {
+		k, v, ok := strings.Cut(e, "=")
+		if !ok {
+			out = append(out, e)
+			continue
+		}
+		u, err := url.Parse(v)
+		if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
+			out = append(out, e)
+			continue
+		}
+		if port := u.Port(); port != "" {
+			u.Host = net.JoinHostPort(host, port)
+		} else {
+			u.Host = host
+		}
+		out = append(out, k+"="+u.String())
+	}
+	return out
+}
+
+// HostAddrVisibleTo returns the local address that peerIP can reach this host
+// on: the address of whichever local interface shares a subnet with peerIP.
+//
+// Derived rather than hardcoded. Apple's Virtualization.framework usually NATs
+// on 192.168.64.0/24 and Hyper-V picks its own, so a constant per platform
+// would be a guess that silently rots when the hypervisor changes its range.
+// Asking "which of my addresses is on the VM's network" is true by
+// construction.
+//
+// ok is false when nothing matches, and callers MUST treat that as "do not
+// advertise" rather than falling back to a guess — a wrong address is what
+// caused the outage this exists to fix.
+func HostAddrVisibleTo(peerIP string) (string, bool) {
+	peer := net.ParseIP(peerIP)
+	if peer == nil {
+		return "", false
+	}
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return "", false
+	}
+	for _, a := range addrs {
+		ipnet, ok := a.(*net.IPNet)
+		if !ok || ipnet.IP.IsLoopback() {
+			continue
+		}
+		if ipnet.Contains(peer) {
+			return ipnet.IP.String(), true
+		}
+	}
+	return "", false
+}
+
+// HostAddrInSubnet returns this host's own IPv4 address inside subnet, which is
+// the address a guest on that subnet can reach the host on.
+//
+// Separate from HostAddrVisibleTo because the caller often knows the
+// hypervisor's NAT range but not any particular guest's address — at daemon
+// start there may be no guest yet.
+//
+// Lives here, rather than in the platform file that uses it, so it is covered
+// by tests that run on every OS. The darwin build cannot be compiled on a
+// non-macOS machine (its vz dependency needs macOS cgo), so keeping logic out
+// of that file is the difference between tested and merely plausible.
+func HostAddrInSubnet(subnet *net.IPNet) (string, bool) {
+	if subnet == nil {
+		return "", false
+	}
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return "", false
+	}
+	for _, a := range addrs {
+		ipnet, ok := a.(*net.IPNet)
+		if !ok || ipnet.IP.To4() == nil || ipnet.IP.IsLoopback() {
+			continue
+		}
+		if subnet.Contains(ipnet.IP) {
+			return ipnet.IP.String(), true
+		}
+	}
+	return "", false
+}
+
+// Reachable reports whether a TCP connection to hostPort succeeds.
+//
+// This is the gate the old loopback health probe could not provide. A proxy
+// that answers on 127.0.0.1 tells you the process is alive; it tells you
+// nothing about whether the address being advertised to jobs exists. Probing
+// the advertised address is the only check that would have caught either the
+// macOS breakage or the Windows one.
+//
+// A failure means "do not inject this var". That is the safe direction: a
+// missing GOPROXY costs bandwidth, while a black-holed one costs a build —
+// Windows jobs HANG on an unreachable proxy rather than failing over, which is
+// why proxy env is suppressed there wholesale today.
+func Reachable(hostPort string, timeout time.Duration) bool {
+	c, err := net.DialTimeout("tcp", hostPort, timeout)
+	if err != nil {
+		return false
+	}
+	_ = c.Close()
+	return true
+}
+
+// EnvHostPorts extracts the host:port authority of every URL-valued entry, so
+// a caller can probe each advertised address before injecting it.
+func EnvHostPorts(env []string) []string {
+	var out []string
+	seen := map[string]bool{}
+	for _, e := range env {
+		_, v, ok := strings.Cut(e, "=")
+		if !ok {
+			continue
+		}
+		u, err := url.Parse(v)
+		if err != nil || u.Host == "" {
+			continue
+		}
+		hp := u.Host
+		if u.Port() == "" {
+			switch u.Scheme {
+			case "http":
+				hp = net.JoinHostPort(u.Hostname(), "80")
+			case "https":
+				hp = net.JoinHostPort(u.Hostname(), "443")
+			default:
+				continue
+			}
+		}
+		if !seen[hp] {
+			seen[hp] = true
+			out = append(out, hp)
+		}
+	}
+	return out
+}