Release

cmd/tag/main.go

@@ -191,10 +191,11 @@ func main() {
 		log.Warn().Err(err).Msg("Failed to load credentials from environment")
 	}
 
-	// Initialize proxy signer unless running in signing mode. Transparent and
-	// tiered modes both forward with proxy headers and need TAG's own creds.
+	// Initialize the proxy signer for transparent forwarding (proxy headers,
+	// TAG's own creds). Whether tiered forwards transparently or by signing
+	// is decided by its endpoint — see Config.ForwardsTransparently.
 	var proxySigner *auth.ProxySigner
-	if cfg.ResolvedMode() != config.ModeSigning {
+	if cfg.ForwardsTransparently() {
 		accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
 		secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
 		if accessKey == "" || secretKey == "" {
@@ -203,14 +204,22 @@ func main() {
 		proxySigner = auth.NewProxySigner(accessKey, secretKey)
 		log.Info().Str("mode", cfg.ResolvedMode()).Msg("Proxy mode enabled")
 	} else {
-		log.Info().Str("endpoint", cfg.Upstream.Endpoint).Msg("Signing mode enabled")
+		log.Info().Str("mode", cfg.ResolvedMode()).Str("endpoint", cfg.Upstream.Endpoint).Msg("Signing forwarding enabled")
 		if !config.IsTigrisEndpoint(cfg.Upstream.Endpoint) {
 			log.Warn().Str("endpoint", cfg.Upstream.Endpoint).
 				Msg("Running against a non-Tigris S3 endpoint; transparent-proxy features are unavailable and third-party backends are community-supported")
+			if cfg.IsTiered() {
+				// The cross-tier cleanup DELETE is safe only where If-Match is
+				// enforced (verified on Tigris), so it is disabled here by
+				// construction; say so once so the accumulating displaced
+				// copies are not a surprise.
+				log.Info().Str("endpoint", cfg.Upstream.Endpoint).
+					Msg("Tiered mode on a non-Tigris endpoint: cross-tier cleanup is disabled (If-Match on DELETE is verified on Tigris only); displaced upstream copies age out by bucket expiry")
+			}
 		}
 	}
 
-	if credStore.Count() == 0 && cfg.ResolvedMode() == config.ModeSigning {
+	if credStore.Count() == 0 && !cfg.ForwardsTransparently() {
 		log.Warn().Msg("No credentials loaded - TAG will reject all requests")
 	}
 
@@ -315,9 +324,11 @@ func main() {
 		objectCache = cache.NewDisabledCache()
 	}
 
-	// 3. Initialize local auth (transparent and tiered modes)
+	// 3. Initialize local auth (transparent forwarding: derived signing keys
+	// learned from Tigris). Signing forwarding validates against the
+	// credential store inside the forwarder instead.
 	var localAuth *proxy.LocalAuthConfig
-	if cfg.ResolvedMode() != config.ModeSigning {
+	if cfg.ForwardsTransparently() {
 		secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
 		derivedKeyStore := auth.NewDerivedKeyStore(auth.DefaultDerivedKeyTTL)
 		keyUnwrapper, err := auth.NewKeyUnwrapper(secretKey)

config/config.go

@@ -175,6 +175,32 @@ func (c *Config) ResolvedMode() string {
 // IsTiered returns whether TAG runs in tiered store mode.
 func (c *Config) IsTiered() bool { return c.ResolvedMode() == ModeTiered }
 
+// ForwardsTransparently reports whether upstream requests are forwarded the
+// transparent way — client signature preserved, X-Tigris-Proxy-* identity
+// headers added, credentials learned from Tigris — as opposed to the signing
+// way (TAG validates against its credential store and re-signs). Transparent
+// mode always does; signing mode never does; TIERED derives it from the
+// endpoint: the proxy-header path is meaningful only to Tigris, so a tiered
+// deployment fronting a Tigris endpoint forwards transparently and one
+// fronting any other S3-compatible endpoint forwards by signing. Tiering is a
+// caching topology, not a forwarding flavor — this is the one place the two
+// are joined, so every transparent-vs-signing fork must key on it rather than
+// on the mode name.
+func (c *Config) ForwardsTransparently() bool {
+	switch c.ResolvedMode() {
+	case ModeTransparent:
+		return true
+	case ModeTiered:
+		// Strict Tigris-domain check, NOT the transparent allowlist: the
+		// transparent flavor re-enables the cross-tier cleanup DELETE, whose
+		// safety needs a backend verified to enforce If-Match. A localhost
+		// stand-in (a local MinIO) earns no such trust and forwards by signing.
+		return IsTigrisEndpoint(c.Upstream.Endpoint)
+	default:
+		return false
+	}
+}
+
 // UpstreamConfig holds Tigris endpoint configuration.
 type UpstreamConfig struct {
 	Endpoint            string `yaml:"endpoint"`                // Tigris S3 endpoint (e.g., https://fly.storage.tigris.dev)
@@ -801,7 +827,10 @@ func validate(cfg *Config) error {
 	if err := validateMode(cfg); err != nil {
 		return err
 	}
-	if err := validateUpstreamEndpoint(cfg.Upstream.Endpoint, cfg.ResolvedMode() != ModeSigning); err != nil {
+	// Only TRANSPARENT mode is pinned to Tigris (its proxy headers are
+	// Tigris-specific). Tiered is allowed on any S3-compatible endpoint — it
+	// derives its forwarding flavor from the endpoint (ForwardsTransparently).
+	if err := validateUpstreamEndpoint(cfg.Upstream.Endpoint, cfg.ResolvedMode() == ModeTransparent); err != nil {
 		return err
 	}
 	if err := validateTLS(&cfg.Server); err != nil {
@@ -884,21 +913,37 @@ func validateTLS(server *ServerConfig) error {
 	return nil
 }
 
-// IsTigrisEndpoint reports whether the endpoint host is localhost or a Tigris
-// domain (*.tigris.dev, *.storage.dev). Transparent proxy mode requires a Tigris
-// endpoint; signing mode works with any S3-compatible endpoint. Returns false if
-// the endpoint cannot be parsed.
+// IsTigrisEndpoint reports whether the endpoint host is a Tigris domain
+// (*.tigris.dev, *.storage.dev) — the backends whose behavior TAG has
+// verified: X-Tigris-Proxy-* headers understood, keys learnable, If-Match on
+// DELETE enforced. It selects tiered mode's forwarding flavor and therefore
+// whether the cross-tier cleanup may run, so it is deliberately STRICT:
+// localhost is not Tigris (a local MinIO is not), see IsTransparentEndpoint
+// for the transparent-mode allowlist. Hostnames are case-insensitive
+// (RFC 4343). Returns false if the endpoint cannot be parsed.
 func IsTigrisEndpoint(endpoint string) bool {
 	u, err := url.Parse(endpoint)
 	if err != nil {
 		return false
 	}
+	host := strings.ToLower(u.Hostname()) // strips port if present
+	return strings.HasSuffix(host, ".tigris.dev") || strings.HasSuffix(host, ".storage.dev")
+}
 
-	host := u.Hostname() // strips port if present
-	if host == "localhost" {
+// IsTransparentEndpoint reports whether transparent proxy mode may target the
+// endpoint: a Tigris domain, or localhost for local testing against a Tigris
+// stand-in. This is an ALLOWLIST for the transparent-mode validator only — it
+// carries no trust claim about the backend, which is why tiered mode's flavor
+// selection uses IsTigrisEndpoint instead.
+func IsTransparentEndpoint(endpoint string) bool {
+	if IsTigrisEndpoint(endpoint) {
 		return true
 	}
-	return strings.HasSuffix(host, ".tigris.dev") || strings.HasSuffix(host, ".storage.dev")
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return false
+	}
+	return strings.ToLower(u.Hostname()) == "localhost"
 }
 
 // validateUpstreamEndpoint ensures the upstream endpoint is a well-formed
@@ -918,7 +963,7 @@ func validateUpstreamEndpoint(endpoint string, transparent bool) error {
 	if u.Hostname() == "" {
 		return fmt.Errorf("upstream endpoint %q must include a host", endpoint)
 	}
-	if transparent && !IsTigrisEndpoint(endpoint) {
+	if transparent && !IsTransparentEndpoint(endpoint) {
 		return fmt.Errorf("upstream endpoint %q is not allowed in transparent proxy mode: host must be localhost, *.tigris.dev, or *.storage.dev (use signing mode for other S3-compatible services)", endpoint)
 	}
 	return nil

config/config_test.go

@@ -888,8 +888,9 @@ func TestIsTigrisEndpoint(t *testing.T) {
 	}{
 		{"tigris.dev domain", "https://fly.storage.tigris.dev", true},
 		{"storage.dev domain", "https://t3.storage.dev", true},
-		{"localhost", "http://localhost:8080", true},
-		{"localhost no port", "http://localhost", true},
+		{"mixed-case tigris host", "https://T3.Storage.Dev", true},
+		{"localhost is not Tigris", "http://localhost:8080", false},
+		{"localhost no port is not Tigris", "http://localhost", false},
 		{"third-party s3", "https://s3.amazonaws.com", false},
 		{"minio", "http://minio.internal:9000", false},
 		{"not a suffix match", "https://nottrigris.dev", false},
@@ -905,6 +906,27 @@ func TestIsTigrisEndpoint(t *testing.T) {
 	}
 }
 
+// The transparent-mode allowlist admits Tigris domains and localhost (any
+// case) for local testing, and nothing else.
+func TestIsTransparentEndpoint(t *testing.T) {
+	tests := []struct {
+		endpoint string
+		want     bool
+	}{
+		{"https://t3.storage.dev", true},
+		{"http://localhost:8080", true},
+		{"http://Localhost", true},
+		{"http://LOCALHOST:9000", true},
+		{"http://minio.internal:9000", false},
+		{"https://s3.amazonaws.com", false},
+	}
+	for _, tt := range tests {
+		if got := IsTransparentEndpoint(tt.endpoint); got != tt.want {
+			t.Errorf("IsTransparentEndpoint(%q) = %v, want %v", tt.endpoint, got, tt.want)
+		}
+	}
+}
+
 // Signing mode re-signs with standard SigV4 and works against any S3-compatible
 // service, so the Tigris endpoint allowlist must not be enforced there.
 func TestLoad_SigningModeAllowsNonTigrisEndpoint(t *testing.T) {

config/mode_test.go

@@ -181,3 +181,75 @@ upstream:
 		t.Fatal("block caching enabled in tiered mode")
 	}
 }
+
+// Tiering is a caching topology, not a forwarding flavor: a tiered deployment
+// may front any S3-compatible endpoint, forwarding by signing there and
+// transparently on Tigris. Only transparent mode itself is pinned to Tigris.
+func TestMode_TieredAllowsNonTigrisEndpoint(t *testing.T) {
+	var cfg Config
+	applyDefaults(&cfg)
+	cfg.Mode = ModeTiered
+	cfg.Upstream.Endpoint = "https://ns.compat.objectstorage.us-ashburn-1.oraclecloud.com"
+	if err := validate(&cfg); err != nil {
+		t.Fatalf("validate rejected tiered mode on a non-Tigris endpoint: %v", err)
+	}
+	if cfg.ForwardsTransparently() {
+		t.Fatal("tiered on a non-Tigris endpoint must forward by signing")
+	}
+}
+
+func TestMode_TransparentRejectsNonTigrisEndpoint(t *testing.T) {
+	var cfg Config
+	applyDefaults(&cfg)
+	cfg.Mode = ModeTransparent
+	cfg.Upstream.Endpoint = "https://ns.compat.objectstorage.us-ashburn-1.oraclecloud.com"
+	if err := validate(&cfg); err == nil {
+		t.Fatal("validate accepted transparent mode on a non-Tigris endpoint")
+	}
+}
+
+func TestMode_ForwardsTransparently(t *testing.T) {
+	tigris, oci := "https://t3.storage.dev", "https://ns.compat.objectstorage.us-ashburn-1.oraclecloud.com"
+	cases := []struct {
+		name     string
+		mode     string
+		endpoint string
+		want     bool
+	}{
+		{"transparent/tigris", ModeTransparent, tigris, true},
+		{"signing/tigris", ModeSigning, tigris, false},
+		{"signing/oci", ModeSigning, oci, false},
+		{"tiered/tigris", ModeTiered, tigris, true},
+		{"tiered/oci", ModeTiered, oci, false},
+		{"default(transparent)/tigris", "", tigris, true},
+		{"tiered/tigris mixed-case host", ModeTiered, "https://T3.Storage.Dev", true},
+		{"tiered/localhost is NOT trusted → signing", ModeTiered, "http://localhost:9000", false},
+		{"tiered/Localhost mixed-case → signing", ModeTiered, "http://Localhost:9000", false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			var cfg Config
+			applyDefaults(&cfg)
+			cfg.Mode = tc.mode
+			cfg.Upstream.Endpoint = tc.endpoint
+			if got := cfg.ForwardsTransparently(); got != tc.want {
+				t.Fatalf("ForwardsTransparently() = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// Transparent mode keeps its localhost allowance for local testing; tiered
+// treats the same endpoint as untrusted (signing flavor, cleanup disabled).
+func TestMode_TransparentAllowsLocalhost(t *testing.T) {
+	var cfg Config
+	applyDefaults(&cfg)
+	cfg.Mode = ModeTransparent
+	cfg.Upstream.Endpoint = "http://Localhost:9000"
+	if err := validate(&cfg); err != nil {
+		t.Fatalf("validate rejected transparent mode on localhost: %v", err)
+	}
+	if !cfg.ForwardsTransparently() {
+		t.Fatal("transparent mode on localhost must forward transparently")
+	}
+}

docs/metrics.md

@@ -469,7 +469,7 @@ signal.
 
 | Label     | Description                                                              |
 | --------- | ------------------------------------------------------------------------ |
-| `outcome` | `deleted`, `already_gone` (404), `replaced` (412 — a newer version took the key, left alone), `rejected` (other upstream refusal), `error` (request failed), `no_etag` (no displaced ETag to bind to; skipped), `live_marker` (skipped: the key's current metadata is a live marker for this ETag — the body is authoritative again), `marker_repaired` (a same-ETag marker raced in during the delete; the repair removed it, converging on an authoritative miss the caller re-populates), `repair_failed` (the repair could not run or could not remove the raced-in marker — it may stay authoritative over a deleted body until TTL) |
+| `outcome` | `deleted`, `already_gone` (404), `replaced` (412 — a newer version took the key, left alone), `rejected` (other upstream refusal), `error` (request failed), `no_etag` (no displaced ETag to bind to; skipped), `unverified_backend` (skipped: non-Tigris endpoint, where If-Match enforcement on DELETE is unverified — cleanup is disabled by construction), `live_marker` (skipped: the key's current metadata is a live marker for this ETag — the body is authoritative again), `marker_repaired` (a same-ETag marker raced in during the delete; the repair removed it, converging on an authoritative miss the caller re-populates), `repair_failed` (the repair could not run or could not remove the raced-in marker — it may stay authoritative over a deleted body until TTL) |
 
 ```promql
 # Orphan-producing cleanup rate (should be ~0)

docs/tiered-mode.md

@@ -55,13 +55,41 @@ marker, so they stay readable) — the first such write's 2xx is itself what
 teaches the keys, and re-tier-on-read moves those objects into the local tier
 on their first validated read.
 
-**Credential requirement**: unlike proxy mode's read-only guidance (which
-targets customer buckets), tiered mode's upstream is the operator's own cache
-bucket, and TAG's `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` must have
-**delete** permission there — the cross-tier cleanup DELETE and the re-tier
-fetch are signed with TAG's own credentials. With read-only credentials every
-cleanup is rejected (visible as `tag_tiered_cleanup_total{outcome="rejected"}`)
-and displaced upstream copies accumulate until bucket expiry.
+**Forwarding flavor is decided by the endpoint.** Tiering is a caching
+topology, not a forwarding flavor: a tiered deployment fronting a Tigris
+domain (`*.tigris.dev`, `*.storage.dev`) forwards the transparent way (client
+signature preserved, `X-Tigris-Proxy-*` identity headers, keys learned from
+Tigris); one fronting any other endpoint — including `localhost`, which
+transparent mode allows for local testing but which earns no trust claim
+about the backend behind it — forwards the signing way (TAG validates the
+caller against its credential store and re-signs). The startup log states
+which flavor was selected.
+
+**Credential requirement** (Tigris-backed tiered deployments): unlike proxy
+mode's read-only guidance (which targets customer buckets), tiered mode's
+upstream is the operator's own cache bucket, and the validated caller's key
+must have **delete** permission there — the cross-tier cleanup DELETE is
+signed with it. With read-only credentials every cleanup is rejected (visible
+as `tag_tiered_cleanup_total{outcome="rejected"}`) and displaced upstream
+copies accumulate until bucket expiry. On any other endpoint the cleanup is
+never issued (next paragraph), so TAG's *background* operations need only
+**read** on the key (the re-tier fetch is a GET) and `rejected` cannot occur.
+That is not the key's overall requirement: on the signing flavor the same
+credential-store key re-signs every forwarded client operation — large-object
+PUTs, upstream-tier DELETEs, multipart, copies — so it must carry whatever
+those operations need.
+
+**Non-Tigris endpoints: cross-tier cleanup is disabled.** The cleanup DELETE
+carries `If-Match` so a racing replacement is never deleted, and that safety
+rests on the backend *enforcing* the precondition — verified on Tigris only.
+A backend that accepts but ignores `If-Match` would let the delete remove a
+replacement that landed after the pre-check, a durable loss no metadata repair
+can undo. So on any other endpoint TAG does not issue the delete at all: the
+displaced upstream copy ages out by bucket expiry (the same outcome as
+read-only credentials), counted as
+`tag_tiered_cleanup_total{outcome="unverified_backend"}`, and the startup log
+says so. Deployments whose keys are never overwritten (content- or
+version-addressed) never trigger cleanup in the first place.
 
 ## Configuration
 

proxy/forwarder.go

@@ -384,8 +384,9 @@ func (b *baseForwarder) DoAnonymousFullObjectRequest(ctx context.Context, bucket
 // SigV4 signing, like every TAG-initiated request. A non-empty etag becomes an
 // If-Match precondition: a 412 means the object was already replaced and must
 // be left alone. Tigris enforces If-Match on DELETE (conditional deletes are
-// ordered against the object's current version) — tiered mode is endpoint-
-// locked to Tigris, so this guard is contractual, not best-effort.
+// ordered against the object's current version); because other backends may
+// silently ignore it, tiered mode issues this delete on Tigris only (the
+// cleanup is disabled elsewhere — see deleteUpstreamObjectAsync).
 func (b *baseForwarder) DoObjectDeleteRequest(ctx context.Context, bucket, key, etag, accessKey, secretKey string) (*http.Response, error) {
 	var extraHeaders http.Header
 	if etag != "" {

proxy/tiered.go

@@ -839,6 +839,17 @@ func (s *Service) deleteUpstreamObjectAsync(bucket, key, etag, accessKey, secret
 		metrics.RecordTieredCleanupSkipped("no_etag")
 		return
 	}
+	if !s.config.ForwardsTransparently() {
+		// The delete's safety rests on the backend ENFORCING If-Match: a
+		// backend that accepts but ignores it would let this delete remove a
+		// replacement that landed after the pre-check — durable loss a
+		// metadata repair cannot undo. That enforcement is verified on
+		// Tigris only, so on any other endpoint the cleanup is disabled by
+		// construction (not merely warned about): the displaced copy ages
+		// out by bucket expiry, the same outcome as read-only credentials.
+		metrics.RecordTieredCleanupSkipped("unverified_backend")
+		return
+	}
 	go func() {
 		ctx, cancel := context.WithTimeout(context.Background(), tieredCleanupTimeout)
 		defer cancel()
@@ -851,9 +862,10 @@ func (s *Service) deleteUpstreamObjectAsync(bucket, key, etag, accessKey, secret
 		// exactly this ETag — that body is authoritative again, not an
 		// orphan. (A racer between this check and the DELETE narrows to the
 		// same-ETag re-establishment landing inside one round trip; the
-		// If-Match still guards every different-ETag interleaving — Tigris
+		// If-Match still guards every different-ETag interleaving: Tigris
 		// enforces conditional DELETEs against the object's current version,
-		// so a different-ETag racer's body is provably left intact.)
+		// and this path runs only on Tigris — see the unverified_backend gate
+		// above.)
 		if cur, found, gerr := s.cache.GetMeta(ctx, bucket, key); gerr == nil && found && cur != nil && cur.BodyUpstream && cur.ETag == etag {
 			metrics.RecordTieredCleanupSkipped("live_marker")
 			return