diff --git a/cmd/tag/main.go b/cmd/tag/main.go index 5f2d9475..66b0d15d 100644 --- a/cmd/tag/main.go +++ b/cmd/tag/main.go @@ -191,10 +191,11 @@ func main() { log.Warn().Err(err).Msg("Failed to load credentials from environment") } - // Initialize proxy signer unless running in signing mode. Transparent and - // tiered modes both forward with proxy headers and need TAG's own creds. + // Initialize the proxy signer for transparent forwarding (proxy headers, + // TAG's own creds). Whether tiered forwards transparently or by signing + // is decided by its endpoint — see Config.ForwardsTransparently. var proxySigner *auth.ProxySigner - if cfg.ResolvedMode() != config.ModeSigning { + if cfg.ForwardsTransparently() { accessKey := os.Getenv("AWS_ACCESS_KEY_ID") secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") if accessKey == "" || secretKey == "" { @@ -203,14 +204,22 @@ func main() { proxySigner = auth.NewProxySigner(accessKey, secretKey) log.Info().Str("mode", cfg.ResolvedMode()).Msg("Proxy mode enabled") } else { - log.Info().Str("endpoint", cfg.Upstream.Endpoint).Msg("Signing mode enabled") + log.Info().Str("mode", cfg.ResolvedMode()).Str("endpoint", cfg.Upstream.Endpoint).Msg("Signing forwarding enabled") if !config.IsTigrisEndpoint(cfg.Upstream.Endpoint) { log.Warn().Str("endpoint", cfg.Upstream.Endpoint). Msg("Running against a non-Tigris S3 endpoint; transparent-proxy features are unavailable and third-party backends are community-supported") + if cfg.IsTiered() { + // The cross-tier cleanup DELETE is safe only where If-Match is + // enforced (verified on Tigris), so it is disabled here by + // construction; say so once so the accumulating displaced + // copies are not a surprise. + log.Info().Str("endpoint", cfg.Upstream.Endpoint). + Msg("Tiered mode on a non-Tigris endpoint: cross-tier cleanup is disabled (If-Match on DELETE is verified on Tigris only); displaced upstream copies age out by bucket expiry") + } } } - if credStore.Count() == 0 && cfg.ResolvedMode() == config.ModeSigning { + if credStore.Count() == 0 && !cfg.ForwardsTransparently() { log.Warn().Msg("No credentials loaded - TAG will reject all requests") } @@ -315,9 +324,11 @@ func main() { objectCache = cache.NewDisabledCache() } - // 3. Initialize local auth (transparent and tiered modes) + // 3. Initialize local auth (transparent forwarding: derived signing keys + // learned from Tigris). Signing forwarding validates against the + // credential store inside the forwarder instead. var localAuth *proxy.LocalAuthConfig - if cfg.ResolvedMode() != config.ModeSigning { + if cfg.ForwardsTransparently() { secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") derivedKeyStore := auth.NewDerivedKeyStore(auth.DefaultDerivedKeyTTL) keyUnwrapper, err := auth.NewKeyUnwrapper(secretKey) diff --git a/config/config.go b/config/config.go index 2f381960..bdace42c 100644 --- a/config/config.go +++ b/config/config.go @@ -175,6 +175,32 @@ func (c *Config) ResolvedMode() string { // IsTiered returns whether TAG runs in tiered store mode. func (c *Config) IsTiered() bool { return c.ResolvedMode() == ModeTiered } +// ForwardsTransparently reports whether upstream requests are forwarded the +// transparent way — client signature preserved, X-Tigris-Proxy-* identity +// headers added, credentials learned from Tigris — as opposed to the signing +// way (TAG validates against its credential store and re-signs). Transparent +// mode always does; signing mode never does; TIERED derives it from the +// endpoint: the proxy-header path is meaningful only to Tigris, so a tiered +// deployment fronting a Tigris endpoint forwards transparently and one +// fronting any other S3-compatible endpoint forwards by signing. Tiering is a +// caching topology, not a forwarding flavor — this is the one place the two +// are joined, so every transparent-vs-signing fork must key on it rather than +// on the mode name. +func (c *Config) ForwardsTransparently() bool { + switch c.ResolvedMode() { + case ModeTransparent: + return true + case ModeTiered: + // Strict Tigris-domain check, NOT the transparent allowlist: the + // transparent flavor re-enables the cross-tier cleanup DELETE, whose + // safety needs a backend verified to enforce If-Match. A localhost + // stand-in (a local MinIO) earns no such trust and forwards by signing. + return IsTigrisEndpoint(c.Upstream.Endpoint) + default: + return false + } +} + // UpstreamConfig holds Tigris endpoint configuration. type UpstreamConfig struct { Endpoint string `yaml:"endpoint"` // Tigris S3 endpoint (e.g., https://fly.storage.tigris.dev) @@ -801,7 +827,10 @@ func validate(cfg *Config) error { if err := validateMode(cfg); err != nil { return err } - if err := validateUpstreamEndpoint(cfg.Upstream.Endpoint, cfg.ResolvedMode() != ModeSigning); err != nil { + // Only TRANSPARENT mode is pinned to Tigris (its proxy headers are + // Tigris-specific). Tiered is allowed on any S3-compatible endpoint — it + // derives its forwarding flavor from the endpoint (ForwardsTransparently). + if err := validateUpstreamEndpoint(cfg.Upstream.Endpoint, cfg.ResolvedMode() == ModeTransparent); err != nil { return err } if err := validateTLS(&cfg.Server); err != nil { @@ -884,21 +913,37 @@ func validateTLS(server *ServerConfig) error { return nil } -// IsTigrisEndpoint reports whether the endpoint host is localhost or a Tigris -// domain (*.tigris.dev, *.storage.dev). Transparent proxy mode requires a Tigris -// endpoint; signing mode works with any S3-compatible endpoint. Returns false if -// the endpoint cannot be parsed. +// IsTigrisEndpoint reports whether the endpoint host is a Tigris domain +// (*.tigris.dev, *.storage.dev) — the backends whose behavior TAG has +// verified: X-Tigris-Proxy-* headers understood, keys learnable, If-Match on +// DELETE enforced. It selects tiered mode's forwarding flavor and therefore +// whether the cross-tier cleanup may run, so it is deliberately STRICT: +// localhost is not Tigris (a local MinIO is not), see IsTransparentEndpoint +// for the transparent-mode allowlist. Hostnames are case-insensitive +// (RFC 4343). Returns false if the endpoint cannot be parsed. func IsTigrisEndpoint(endpoint string) bool { u, err := url.Parse(endpoint) if err != nil { return false } + host := strings.ToLower(u.Hostname()) // strips port if present + return strings.HasSuffix(host, ".tigris.dev") || strings.HasSuffix(host, ".storage.dev") +} - host := u.Hostname() // strips port if present - if host == "localhost" { +// IsTransparentEndpoint reports whether transparent proxy mode may target the +// endpoint: a Tigris domain, or localhost for local testing against a Tigris +// stand-in. This is an ALLOWLIST for the transparent-mode validator only — it +// carries no trust claim about the backend, which is why tiered mode's flavor +// selection uses IsTigrisEndpoint instead. +func IsTransparentEndpoint(endpoint string) bool { + if IsTigrisEndpoint(endpoint) { return true } - return strings.HasSuffix(host, ".tigris.dev") || strings.HasSuffix(host, ".storage.dev") + u, err := url.Parse(endpoint) + if err != nil { + return false + } + return strings.ToLower(u.Hostname()) == "localhost" } // validateUpstreamEndpoint ensures the upstream endpoint is a well-formed @@ -918,7 +963,7 @@ func validateUpstreamEndpoint(endpoint string, transparent bool) error { if u.Hostname() == "" { return fmt.Errorf("upstream endpoint %q must include a host", endpoint) } - if transparent && !IsTigrisEndpoint(endpoint) { + if transparent && !IsTransparentEndpoint(endpoint) { return fmt.Errorf("upstream endpoint %q is not allowed in transparent proxy mode: host must be localhost, *.tigris.dev, or *.storage.dev (use signing mode for other S3-compatible services)", endpoint) } return nil diff --git a/config/config_test.go b/config/config_test.go index a543cb32..a00f5d42 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -888,8 +888,9 @@ func TestIsTigrisEndpoint(t *testing.T) { }{ {"tigris.dev domain", "https://fly.storage.tigris.dev", true}, {"storage.dev domain", "https://t3.storage.dev", true}, - {"localhost", "http://localhost:8080", true}, - {"localhost no port", "http://localhost", true}, + {"mixed-case tigris host", "https://T3.Storage.Dev", true}, + {"localhost is not Tigris", "http://localhost:8080", false}, + {"localhost no port is not Tigris", "http://localhost", false}, {"third-party s3", "https://s3.amazonaws.com", false}, {"minio", "http://minio.internal:9000", false}, {"not a suffix match", "https://nottrigris.dev", false}, @@ -905,6 +906,27 @@ func TestIsTigrisEndpoint(t *testing.T) { } } +// The transparent-mode allowlist admits Tigris domains and localhost (any +// case) for local testing, and nothing else. +func TestIsTransparentEndpoint(t *testing.T) { + tests := []struct { + endpoint string + want bool + }{ + {"https://t3.storage.dev", true}, + {"http://localhost:8080", true}, + {"http://Localhost", true}, + {"http://LOCALHOST:9000", true}, + {"http://minio.internal:9000", false}, + {"https://s3.amazonaws.com", false}, + } + for _, tt := range tests { + if got := IsTransparentEndpoint(tt.endpoint); got != tt.want { + t.Errorf("IsTransparentEndpoint(%q) = %v, want %v", tt.endpoint, got, tt.want) + } + } +} + // Signing mode re-signs with standard SigV4 and works against any S3-compatible // service, so the Tigris endpoint allowlist must not be enforced there. func TestLoad_SigningModeAllowsNonTigrisEndpoint(t *testing.T) { diff --git a/config/mode_test.go b/config/mode_test.go index c12614f2..81e3d0b3 100644 --- a/config/mode_test.go +++ b/config/mode_test.go @@ -181,3 +181,75 @@ upstream: t.Fatal("block caching enabled in tiered mode") } } + +// Tiering is a caching topology, not a forwarding flavor: a tiered deployment +// may front any S3-compatible endpoint, forwarding by signing there and +// transparently on Tigris. Only transparent mode itself is pinned to Tigris. +func TestMode_TieredAllowsNonTigrisEndpoint(t *testing.T) { + var cfg Config + applyDefaults(&cfg) + cfg.Mode = ModeTiered + cfg.Upstream.Endpoint = "https://ns.compat.objectstorage.us-ashburn-1.oraclecloud.com" + if err := validate(&cfg); err != nil { + t.Fatalf("validate rejected tiered mode on a non-Tigris endpoint: %v", err) + } + if cfg.ForwardsTransparently() { + t.Fatal("tiered on a non-Tigris endpoint must forward by signing") + } +} + +func TestMode_TransparentRejectsNonTigrisEndpoint(t *testing.T) { + var cfg Config + applyDefaults(&cfg) + cfg.Mode = ModeTransparent + cfg.Upstream.Endpoint = "https://ns.compat.objectstorage.us-ashburn-1.oraclecloud.com" + if err := validate(&cfg); err == nil { + t.Fatal("validate accepted transparent mode on a non-Tigris endpoint") + } +} + +func TestMode_ForwardsTransparently(t *testing.T) { + tigris, oci := "https://t3.storage.dev", "https://ns.compat.objectstorage.us-ashburn-1.oraclecloud.com" + cases := []struct { + name string + mode string + endpoint string + want bool + }{ + {"transparent/tigris", ModeTransparent, tigris, true}, + {"signing/tigris", ModeSigning, tigris, false}, + {"signing/oci", ModeSigning, oci, false}, + {"tiered/tigris", ModeTiered, tigris, true}, + {"tiered/oci", ModeTiered, oci, false}, + {"default(transparent)/tigris", "", tigris, true}, + {"tiered/tigris mixed-case host", ModeTiered, "https://T3.Storage.Dev", true}, + {"tiered/localhost is NOT trusted → signing", ModeTiered, "http://localhost:9000", false}, + {"tiered/Localhost mixed-case → signing", ModeTiered, "http://Localhost:9000", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var cfg Config + applyDefaults(&cfg) + cfg.Mode = tc.mode + cfg.Upstream.Endpoint = tc.endpoint + if got := cfg.ForwardsTransparently(); got != tc.want { + t.Fatalf("ForwardsTransparently() = %v, want %v", got, tc.want) + } + }) + } +} + +// Transparent mode keeps its localhost allowance for local testing; tiered +// treats the same endpoint as untrusted (signing flavor, cleanup disabled). +func TestMode_TransparentAllowsLocalhost(t *testing.T) { + var cfg Config + applyDefaults(&cfg) + cfg.Mode = ModeTransparent + cfg.Upstream.Endpoint = "http://Localhost:9000" + if err := validate(&cfg); err != nil { + t.Fatalf("validate rejected transparent mode on localhost: %v", err) + } + if !cfg.ForwardsTransparently() { + t.Fatal("transparent mode on localhost must forward transparently") + } +} diff --git a/docs/metrics.md b/docs/metrics.md index 7b2b1da8..3d563f7a 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -469,7 +469,7 @@ signal. | Label | Description | | --------- | ------------------------------------------------------------------------ | -| `outcome` | `deleted`, `already_gone` (404), `replaced` (412 — a newer version took the key, left alone), `rejected` (other upstream refusal), `error` (request failed), `no_etag` (no displaced ETag to bind to; skipped), `live_marker` (skipped: the key's current metadata is a live marker for this ETag — the body is authoritative again), `marker_repaired` (a same-ETag marker raced in during the delete; the repair removed it, converging on an authoritative miss the caller re-populates), `repair_failed` (the repair could not run or could not remove the raced-in marker — it may stay authoritative over a deleted body until TTL) | +| `outcome` | `deleted`, `already_gone` (404), `replaced` (412 — a newer version took the key, left alone), `rejected` (other upstream refusal), `error` (request failed), `no_etag` (no displaced ETag to bind to; skipped), `unverified_backend` (skipped: non-Tigris endpoint, where If-Match enforcement on DELETE is unverified — cleanup is disabled by construction), `live_marker` (skipped: the key's current metadata is a live marker for this ETag — the body is authoritative again), `marker_repaired` (a same-ETag marker raced in during the delete; the repair removed it, converging on an authoritative miss the caller re-populates), `repair_failed` (the repair could not run or could not remove the raced-in marker — it may stay authoritative over a deleted body until TTL) | ```promql # Orphan-producing cleanup rate (should be ~0) diff --git a/docs/tiered-mode.md b/docs/tiered-mode.md index 749ee9dd..b2d2304c 100644 --- a/docs/tiered-mode.md +++ b/docs/tiered-mode.md @@ -55,13 +55,41 @@ marker, so they stay readable) — the first such write's 2xx is itself what teaches the keys, and re-tier-on-read moves those objects into the local tier on their first validated read. -**Credential requirement**: unlike proxy mode's read-only guidance (which -targets customer buckets), tiered mode's upstream is the operator's own cache -bucket, and TAG's `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` must have -**delete** permission there — the cross-tier cleanup DELETE and the re-tier -fetch are signed with TAG's own credentials. With read-only credentials every -cleanup is rejected (visible as `tag_tiered_cleanup_total{outcome="rejected"}`) -and displaced upstream copies accumulate until bucket expiry. +**Forwarding flavor is decided by the endpoint.** Tiering is a caching +topology, not a forwarding flavor: a tiered deployment fronting a Tigris +domain (`*.tigris.dev`, `*.storage.dev`) forwards the transparent way (client +signature preserved, `X-Tigris-Proxy-*` identity headers, keys learned from +Tigris); one fronting any other endpoint — including `localhost`, which +transparent mode allows for local testing but which earns no trust claim +about the backend behind it — forwards the signing way (TAG validates the +caller against its credential store and re-signs). The startup log states +which flavor was selected. + +**Credential requirement** (Tigris-backed tiered deployments): unlike proxy +mode's read-only guidance (which targets customer buckets), tiered mode's +upstream is the operator's own cache bucket, and the validated caller's key +must have **delete** permission there — the cross-tier cleanup DELETE is +signed with it. With read-only credentials every cleanup is rejected (visible +as `tag_tiered_cleanup_total{outcome="rejected"}`) and displaced upstream +copies accumulate until bucket expiry. On any other endpoint the cleanup is +never issued (next paragraph), so TAG's *background* operations need only +**read** on the key (the re-tier fetch is a GET) and `rejected` cannot occur. +That is not the key's overall requirement: on the signing flavor the same +credential-store key re-signs every forwarded client operation — large-object +PUTs, upstream-tier DELETEs, multipart, copies — so it must carry whatever +those operations need. + +**Non-Tigris endpoints: cross-tier cleanup is disabled.** The cleanup DELETE +carries `If-Match` so a racing replacement is never deleted, and that safety +rests on the backend *enforcing* the precondition — verified on Tigris only. +A backend that accepts but ignores `If-Match` would let the delete remove a +replacement that landed after the pre-check, a durable loss no metadata repair +can undo. So on any other endpoint TAG does not issue the delete at all: the +displaced upstream copy ages out by bucket expiry (the same outcome as +read-only credentials), counted as +`tag_tiered_cleanup_total{outcome="unverified_backend"}`, and the startup log +says so. Deployments whose keys are never overwritten (content- or +version-addressed) never trigger cleanup in the first place. ## Configuration diff --git a/proxy/forwarder.go b/proxy/forwarder.go index 169a8dfa..076f79fa 100644 --- a/proxy/forwarder.go +++ b/proxy/forwarder.go @@ -384,8 +384,9 @@ func (b *baseForwarder) DoAnonymousFullObjectRequest(ctx context.Context, bucket // SigV4 signing, like every TAG-initiated request. A non-empty etag becomes an // If-Match precondition: a 412 means the object was already replaced and must // be left alone. Tigris enforces If-Match on DELETE (conditional deletes are -// ordered against the object's current version) — tiered mode is endpoint- -// locked to Tigris, so this guard is contractual, not best-effort. +// ordered against the object's current version); because other backends may +// silently ignore it, tiered mode issues this delete on Tigris only (the +// cleanup is disabled elsewhere — see deleteUpstreamObjectAsync). func (b *baseForwarder) DoObjectDeleteRequest(ctx context.Context, bucket, key, etag, accessKey, secretKey string) (*http.Response, error) { var extraHeaders http.Header if etag != "" { diff --git a/proxy/tiered.go b/proxy/tiered.go index d5591e45..66d54c16 100644 --- a/proxy/tiered.go +++ b/proxy/tiered.go @@ -839,6 +839,17 @@ func (s *Service) deleteUpstreamObjectAsync(bucket, key, etag, accessKey, secret metrics.RecordTieredCleanupSkipped("no_etag") return } + if !s.config.ForwardsTransparently() { + // The delete's safety rests on the backend ENFORCING If-Match: a + // backend that accepts but ignores it would let this delete remove a + // replacement that landed after the pre-check — durable loss a + // metadata repair cannot undo. That enforcement is verified on + // Tigris only, so on any other endpoint the cleanup is disabled by + // construction (not merely warned about): the displaced copy ages + // out by bucket expiry, the same outcome as read-only credentials. + metrics.RecordTieredCleanupSkipped("unverified_backend") + return + } go func() { ctx, cancel := context.WithTimeout(context.Background(), tieredCleanupTimeout) defer cancel() @@ -851,9 +862,10 @@ func (s *Service) deleteUpstreamObjectAsync(bucket, key, etag, accessKey, secret // exactly this ETag — that body is authoritative again, not an // orphan. (A racer between this check and the DELETE narrows to the // same-ETag re-establishment landing inside one round trip; the - // If-Match still guards every different-ETag interleaving — Tigris + // If-Match still guards every different-ETag interleaving: Tigris // enforces conditional DELETEs against the object's current version, - // so a different-ETag racer's body is provably left intact.) + // and this path runs only on Tigris — see the unverified_backend gate + // above.) if cur, found, gerr := s.cache.GetMeta(ctx, bucket, key); gerr == nil && found && cur != nil && cur.BodyUpstream && cur.ETag == etag { metrics.RecordTieredCleanupSkipped("live_marker") return diff --git a/proxy/tiered_test.go b/proxy/tiered_test.go index 04251f0a..c3d5e8ae 100644 --- a/proxy/tiered_test.go +++ b/proxy/tiered_test.go @@ -1636,3 +1636,48 @@ func TestTieredConvergeRetriesOnceUnderSameVersion(t *testing.T) { t.Fatal("metadata vanished despite every converge attempt failing") } } + +// On a non-Tigris endpoint the cross-tier cleanup DELETE is disabled by +// construction: If-Match enforcement on DELETE is verified on Tigris only, and +// a backend that ignored it could delete a racing replacement. The displaced +// upstream copy is left for bucket expiry. (The gate returns synchronously +// before any goroutine is spawned, so a delete that was going to happen would +// already be in flight by the time the PUT returns.) +func TestTieredCleanupDisabledOnNonTigrisEndpoint(t *testing.T) { + mock, _, _ := tieredMock() + var deletes atomic.Int32 + mock.doObjectDeleteFunc = func(ctx context.Context, bucket, key, etag, accessKey, secretKey string) (*http.Response, error) { + deletes.Add(1) + return &http.Response{StatusCode: http.StatusNoContent, Body: http.NoBody}, nil + } + cfg := config.NewDefault() + cfg.Mode = config.ModeTiered + cfg.Upstream.Endpoint = "https://ns.compat.objectstorage.us-ashburn-1.oraclecloud.com" + cfg.Cache.SetBlockCachingEnabled(false) + cfg.Cache.SizeThreshold = 8 + if err := cfg.Validate(); err != nil { + t.Fatalf("tiered on a non-Tigris endpoint must validate: %v", err) + } + if cfg.ForwardsTransparently() { + t.Fatal("fixture: expected signing flavor") + } + c := cache.NewCacheWithClient(cacheclient.NewMemoryCache(), &cfg.Cache) + svc := NewService(mock, c, cfg) + + // Large PUT stamps an upstream-tier marker; the small overwrite displaces it — + // the exact trigger for cross-tier cleanup on Tigris. + if w := tieredDo(t, svc, http.MethodPut, "/b/obj", "way past threshold", nil); w.Code != http.StatusOK { + t.Fatalf("large PUT status = %d", w.Code) + } + if w := tieredDo(t, svc, http.MethodPut, "/b/obj", "tiny", nil); w.Code != http.StatusOK { + t.Fatalf("small PUT status = %d", w.Code) + } + time.Sleep(300 * time.Millisecond) + if n := deletes.Load(); n != 0 { + t.Fatalf("cleanup DELETE issued %d time(s) on a non-Tigris endpoint, want 0", n) + } + // The small object is the live, servable local-tier entry regardless. + if g := tieredDo(t, svc, http.MethodGet, "/b/obj", "", nil); g.Code != http.StatusOK || g.Body.String() != "tiny" { + t.Fatalf("GET = %d %q, want the local-tier object", g.Code, g.Body.String()) + } +}