feat(proxies): cache GitHub release artifacts and Packagist

cmd/ephemerd/cache.go

@@ -104,6 +104,18 @@ func managedCaches() []cacheEntry {
 			Description: "pub.dev proxy cache (version listings + package archives) served to job containers",
 			LiveSafe:    true,
 		},
+		{
+			Name:        "ghrel",
+			Rel:         filepath.Join("cache", "ghrel"),
+			Description: "GitHub release proxy cache (release metadata + asset bytes) served to job containers",
+			LiveSafe:    true,
+		},
+		{
+			Name:        "composer",
+			Rel:         filepath.Join("cache", "composer"),
+			Description: "Packagist proxy cache (metadata + distribution archives) served to job containers",
+			LiveSafe:    true,
+		},
 		{
 			Name:        "buildkit",
 			Rel:         "buildkit",

cmd/ephemerd/pkgproxies.go

@@ -7,6 +7,8 @@ import (
 	"github.com/ephpm/ephemerd/pkg/config"
 	"github.com/ephpm/ephemerd/pkg/networking"
 	"github.com/ephpm/ephemerd/pkg/proxies"
+	composerproxy "github.com/ephpm/ephemerd/pkg/proxies/composer"
+	ghrelproxy "github.com/ephpm/ephemerd/pkg/proxies/ghrel"
 	npmproxy "github.com/ephpm/ephemerd/pkg/proxies/npm"
 	pipproxy "github.com/ephpm/ephemerd/pkg/proxies/pip"
 	pubproxy "github.com/ephpm/ephemerd/pkg/proxies/pub"
@@ -15,19 +17,22 @@ import (
 // Default listen ports for the language package caches, on the bridge
 // gateway alongside the Go module proxy (8082).
 const (
-	defaultNpmProxyPort = 8084
-	defaultPipProxyPort = 8085
-	defaultPubProxyPort = 8086
+	defaultNpmProxyPort      = 8084
+	defaultPipProxyPort      = 8085
+	defaultPubProxyPort      = 8086
+	defaultGhrelProxyPort    = 8087
+	defaultComposerProxyPort = 8088
 )
 
 // healthGated is the interface a cache proxy implements when its env var
 // cannot fail over on its own.
 //
 // GOPROXY has "|direct" and so degrades by itself. npm_config_registry,
-// PIP_INDEX_URL and PUB_HOSTED_URL have no such escape hatch: whatever they
-// name IS the registry. The next best thing is to not name a proxy that is
-// not answering, so these are probed after Start and only injected if the
-// probe passes.
+// PIP_INDEX_URL, PUB_HOSTED_URL, COMPOSER_REPO_PACKAGIST and GHREL_PROXY
+// have no such escape hatch: whatever they name IS the registry, and a
+// consumer only falls back to upstream when the variable is ABSENT. The next
+// best thing is to not name a proxy that is not answering, so these are
+// probed after Start and only injected if the probe passes.
 type healthGated interface {
 	Healthy() bool
 }
@@ -45,11 +50,18 @@ func pkgProxyPorts(cfg *config.Config) []int {
 	if cfg.PubProxy.Enabled {
 		ports = append(ports, cfg.PubProxy.ProxyPort(defaultPubProxyPort))
 	}
+	if cfg.GhrelProxy.Enabled {
+		ports = append(ports, cfg.GhrelProxy.ProxyPort(defaultGhrelProxyPort))
+	}
+	if cfg.ComposerProxy.Enabled {
+		ports = append(ports, cfg.ComposerProxy.ProxyPort(defaultComposerProxyPort))
+	}
 	return ports
 }
 
-// startPkgProxies starts whichever of the npm, pip and pub caches are
-// enabled and returns those that came up healthy, plus a teardown func.
+// startPkgProxies starts whichever of the npm, pip, pub, ghrel and composer
+// caches are enabled and returns those that came up healthy, plus a teardown
+// func.
 //
 // FAIL-OPEN AT STARTUP: a proxy that fails to start, or that starts but does
 // not answer its own health probe, is simply left out of the returned slice.
@@ -111,6 +123,46 @@ func startPkgProxies(cfg *config.Config, dataDir string, net *networking.Manager
 				})
 			},
 		},
+		{
+			enabled: cfg.GhrelProxy.Enabled,
+			port:    cfg.GhrelProxy.ProxyPort(defaultGhrelProxyPort),
+			build: func(addr string) (proxies.CacheProxy, error) {
+				return ghrelproxy.New(ghrelproxy.Config{
+					CacheDir: joinPath(dataDir, "cache", "ghrel"),
+					// [ghrel_proxy].upstream is the REST API origin; the
+					// asset origin tracks it rather than being a second
+					// knob, so pointing at GitHub Enterprise moves both.
+					APIUpstream: cfg.GhrelProxy.Upstream,
+					ListenAddr:  addr,
+					MetadataTTL: cfg.GhrelProxy.ProxyIndexTTL(),
+					// AssetTTL is deliberately left at the package default
+					// (24h). It is a backstop against an asset being
+					// replaced in place, not a knob an operator tunes —
+					// the operator-facing choice is immutable_assets.
+					ImmutableAssets: cfg.GhrelProxy.ImmutableAssets,
+					MaxBytes:        cfg.GhrelProxy.ProxyMaxBytes(),
+					AllowedHosts:    cfg.GhrelProxy.AllowedHosts,
+					Cleanup:         cfg.GhrelProxy.Cleanup,
+					Log:             log.With("component", "ghrel-proxy"),
+				})
+			},
+		},
+		{
+			enabled: cfg.ComposerProxy.Enabled,
+			port:    cfg.ComposerProxy.ProxyPort(defaultComposerProxyPort),
+			build: func(addr string) (proxies.CacheProxy, error) {
+				return composerproxy.New(composerproxy.Config{
+					CacheDir:     joinPath(dataDir, "cache", "composer"),
+					Upstream:     cfg.ComposerProxy.Upstream,
+					ListenAddr:   addr,
+					MetadataTTL:  cfg.ComposerProxy.ProxyIndexTTL(),
+					MaxBytes:     cfg.ComposerProxy.ProxyMaxBytes(),
+					AllowedHosts: cfg.ComposerProxy.AllowedHosts,
+					Cleanup:      cfg.ComposerProxy.Cleanup,
+					Log:          log.With("component", "composer-proxy"),
+				})
+			},
+		},
 	}
 
 	var (

cmd/ephemerd/pkgproxies_test.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"path/filepath"
+	"slices"
+	"testing"
+
+	"github.com/ephpm/ephemerd/pkg/config"
+)
+
+// TestPkgProxyPorts is the firewall half of the wiring. These ports are fed
+// into networking.Config.GatewayPorts; a proxy that is enabled but missing
+// here binds fine, passes its health probe over loopback, advertises its env
+// var — and is then unreachable from every job container, which looks like a
+// hung download rather than a misconfiguration.
+func TestPkgProxyPorts(t *testing.T) {
+	t.Parallel()
+
+	if got := pkgProxyPorts(&config.Config{}); len(got) != 0 {
+		t.Errorf("pkgProxyPorts with nothing enabled = %v, want none opened", got)
+	}
+
+	var cfg config.Config
+	cfg.NpmProxy.Enabled = true
+	cfg.PipProxy.Enabled = true
+	cfg.PubProxy.Enabled = true
+	cfg.GhrelProxy.Enabled = true
+	cfg.ComposerProxy.Enabled = true
+
+	got := pkgProxyPorts(&cfg)
+	for _, want := range []int{
+		defaultNpmProxyPort,
+		defaultPipProxyPort,
+		defaultPubProxyPort,
+		defaultGhrelProxyPort,
+		defaultComposerProxyPort,
+	} {
+		if !slices.Contains(got, want) {
+			t.Errorf("port %d is not opened to job containers; got %v", want, got)
+		}
+	}
+
+	// The defaults must stay distinct, or two proxies fight over a listener
+	// and the second one silently fails to start.
+	seen := map[int]bool{}
+	for _, p := range got {
+		if seen[p] {
+			t.Errorf("port %d is claimed by two proxies: %v", p, got)
+		}
+		seen[p] = true
+	}
+
+	// An explicit port overrides the default.
+	cfg.GhrelProxy.Port = 9999
+	if got := pkgProxyPorts(&cfg); !slices.Contains(got, 9999) || slices.Contains(got, defaultGhrelProxyPort) {
+		t.Errorf("pkgProxyPorts = %v, want 9999 instead of %d", got, defaultGhrelProxyPort)
+	}
+}
+
+// TestNewCachesAreClearable: every cache these proxies write to must be
+// known to `ephemerd cache`, or it grows until it fills the node's disk with
+// nothing to clear it. The directory names must match what startPkgProxies
+// actually passes as CacheDir.
+func TestNewCachesAreClearable(t *testing.T) {
+	t.Parallel()
+	for _, name := range []string{"ghrel", "composer"} {
+		c, ok := cacheByName(name)
+		if !ok {
+			t.Errorf("cache %q is not managed by `ephemerd cache`", name)
+			continue
+		}
+		if want := filepath.Join("cache", name); c.Rel != want {
+			t.Errorf("cache %q lives at %q, want %q", name, c.Rel, want)
+		}
+		if !c.LiveSafe {
+			t.Errorf("cache %q is not live-safe; every entry is a pull-through copy that a miss refetches", name)
+		}
+	}
+}

config.example.toml

@@ -509,6 +509,44 @@ shutdown_timeout = "5m"
 # max_size_gb = 5
 # cleanup     = false
 
+# GitHub releases. Caches release metadata and the asset bytes it points at —
+# prebuilt toolchains, SDK archives, vendored tarballs, the CI inputs that have
+# no registry to cache them in.
+#
+# Unlike the others this one ADVERTISES rather than intercepts: a GitHub
+# download is a plain https://github.com/... URL with no redirect knob, so
+# ephemerd sets GHREL_PROXY and a tool that understands it uses it. A tool that
+# does not is unaffected, and a build stays correct off-fleet because the
+# variable is simply absent. `upstream` is the REST API origin, so a GitHub
+# Enterprise host moves both halves.
+#
+# immutable_assets is the one key the other caches do not have. GitHub is the
+# only ecosystem here that lets an asset be deleted and replaced under the same
+# tag and URL, so assets are revalidated on a long TTL by default. Set this
+# true only if your publishing discipline guarantees an asset is never
+# overwritten in place — it is a promise the proxy cannot verify, and getting
+# it wrong is silent.
+# [ghrel_proxy]
+# enabled          = false
+# port             = 8087
+# upstream         = "https://api.github.com"
+# index_ttl        = "5m"
+# max_size_gb      = 5
+# immutable_assets = false
+# cleanup          = false
+
+# PHP Composer. Injects COMPOSER_REPO_PACKAGIST, so both the Packagist metadata
+# and the distribution archives it points at come from the LAN. A repository
+# declared explicitly in composer.json still wins, and private/authenticated
+# repositories are not proxied — this cache forwards no credentials.
+# [composer_proxy]
+# enabled     = false
+# port        = 8088
+# upstream    = "https://repo.packagist.org"
+# index_ttl   = "5m"
+# max_size_gb = 5
+# cleanup     = false
+
 [log]
 # Log level: debug, info, warn, error
 level = "info"

pkg/config/config.go

@@ -36,6 +36,12 @@ type Config struct {
 	NpmProxy    PkgProxyConfig    `toml:"npm_proxy"`
 	PipProxy    PkgProxyConfig    `toml:"pip_proxy"`
 	PubProxy    PkgProxyConfig    `toml:"pub_proxy"`
+	// ComposerProxy caches Packagist metadata and the PHP distribution
+	// archives it points at. Same shape as the other language caches.
+	ComposerProxy PkgProxyConfig `toml:"composer_proxy"`
+	// GhrelProxy caches GitHub release metadata and asset bytes. It carries
+	// one field the others do not — see GhrelProxyConfig.
+	GhrelProxy GhrelProxyConfig `toml:"ghrel_proxy"`
 	// RegistryMirror routes container image pulls through a LAN pull-through
 	// cache instead of the origin registry. See RegistryMirrorConfig.
 	RegistryMirror RegistryMirrorConfig `toml:"registry_mirror"`
@@ -1103,7 +1109,8 @@ func validateMirrorURL(key, raw string) error {
 }
 
 // PkgProxyConfig configures one language package caching proxy. The
-// same shape serves [npm_proxy], [pip_proxy] and [pub_proxy]: all three are
+// same shape serves [npm_proxy], [pip_proxy], [pub_proxy], [composer_proxy]
+// and (embedded in GhrelProxyConfig) [ghrel_proxy]: all of them are
 // pull-through HTTP caches with an immutable-artifact half and a mutable-
 // metadata half, and differ only in their upstream and their defaults.
 //
@@ -1117,13 +1124,16 @@ type PkgProxyConfig struct {
 	Enabled bool `toml:"enabled"`
 
 	// Port is the listen port on the bridge gateway. Zero takes the
-	// per-ecosystem default (npm 8084, pip 8085, pub 8086).
+	// per-ecosystem default (npm 8084, pip 8085, pub 8086, ghrel 8087,
+	// composer 8088).
 	Port int `toml:"port"`
 
 	// Upstream overrides the registry to pull through to (npm:
 	// https://registry.npmjs.org, pip: https://pypi.org, pub:
-	// https://pub.dev). The upstream's own host is always permitted to
-	// serve artifacts, so an override needs no matching allowed_hosts entry.
+	// https://pub.dev, composer: https://repo.packagist.org, ghrel:
+	// https://api.github.com). The upstream's own host is always permitted
+	// to serve artifacts, so an override needs no matching allowed_hosts
+	// entry.
 	Upstream string `toml:"upstream"`
 
 	// IndexTTL is how long cached MUTABLE metadata (an npm packument, a PEP
@@ -1133,6 +1143,9 @@ type PkgProxyConfig struct {
 	//
 	// Immutable artifacts — tarballs, wheels, sdists, archives — ignore
 	// this entirely: they are cached permanently and never revalidated.
+	//
+	// [ghrel_proxy] is the one exception to that second paragraph, because
+	// GitHub lets an asset be replaced in place. See GhrelProxyConfig.
 	IndexTTL time.Duration `toml:"index_ttl"`
 
 	// MaxSizeGB is the cache's disk budget in GiB. When it is exceeded, the
@@ -1187,6 +1200,36 @@ func (p *PkgProxyConfig) ProxyMaxBytes() int64 {
 	return p.MaxSizeGB << 30
 }
 
+// GhrelProxyConfig configures [ghrel_proxy], the GitHub release cache.
+//
+// It is a PkgProxyConfig plus one flag, rather than its own type, because
+// every other knob means the same thing here as it does for [npm_proxy]:
+// `upstream` is the REST API origin (https://api.github.com by default, or a
+// GitHub Enterprise host), `index_ttl` is how long release METADATA is served
+// before a conditional GET, and `port` defaults to 8087 — continuing npm
+// 8084, pip 8085, pub 8086, with composer at 8088.
+//
+// The extra flag exists because GitHub is the one ecosystem here that does
+// not promise immutable artifacts. PyPI refuses to let a file be re-uploaded
+// even after deletion, and npm and pub are similar, which is why those caches
+// keep an artifact forever and never look at it again. GitHub enforces
+// nothing of the kind: a release asset can be deleted and replaced under the
+// same tag, keeping its name and URL. So asset bytes are revalidated on a
+// long TTL by default — one conditional request per asset per day, answered
+// with a 304 for unchanged bytes, which is cheap next to re-downloading a
+// 200 MB toolchain and stays correct if someone republishes.
+type GhrelProxyConfig struct {
+	PkgProxyConfig
+
+	// ImmutableAssets caches release asset bytes permanently and never
+	// revalidates them, trading the daily conditional request for the
+	// promise that an asset is never overwritten in place (a new build gets
+	// a new tag). Default false: it is a promise the proxy cannot verify,
+	// and getting it wrong is silent — jobs keep building, against bytes
+	// that upstream has replaced.
+	ImmutableAssets bool `toml:"immutable_assets"`
+}
+
 // VMConfig configures virtual machines for cross-OS job execution.
 type VMConfig struct {
 	// CrossPlatform enables macOS and Windows VM support. Default true.