From 6531b18065f8d69cad58342a512e56e17ff9353f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 21 Oct 2026 18:33:27 +0100 Subject: [PATCH 01/17] feat(iam): forward a DeleteObjects body to Hilt and decide the fast path on it Hilt now reads a DeleteObjects body and needs s3:DeleteObjectVersion when an entry names a version. The body travels only for the operations Hilt's classifier names, so a streaming write's body is never read here; the fast path checks the version permission from the same classification and leaves a body the signature does not bind to Hilt, which refuses it as UnboundBody. Pins libforge to fil-forge/libforge#87 and hilt to fil-forge/hilt#106. Co-Authored-By: Claude Fable 5.1 --- docs/diagrams.md | 11 ++-- go.mod | 4 +- go.sum | 8 +-- iam/service.go | 16 ++++-- iam/service_internal_test.go | 102 +++++++++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 13 deletions(-) diff --git a/docs/diagrams.md b/docs/diagrams.md index 255e2808..c8a5bfc8 100644 --- a/docs/diagrams.md +++ b/docs/diagrams.md @@ -913,10 +913,13 @@ flowchart TB chain probe alone would let a put-only key read and abort on the bucket, or read a copy's source. - A copy's source and the headers that add a permission to a write (a lock - or tags on the version it creates) are decided from headers. Hilt's - `RequirementsFor` refuses a header the signature does not cover, as hilt - itself does, so the fast path decides nothing from such a request: it - goes to hilt, which refuses it as `UnsignedHeader`. + or tags on the version it creates) are decided from headers, and a + DeleteObjects on its body, which iam attaches for the operations hilt + names (`Operation.ReadsBody`) and no other. Hilt's `RequirementsFor` + refuses a header the signature does not cover or a body it does not bind, + as hilt itself does, so the fast path decides nothing from such a + request: it goes to hilt, which refuses it as `UnsignedHeader` or + `UnboundBody`. Cross-references: [`architecture.md` §9](./architecture.md#9-the-system-contract-piri--sprue--indexer). diff --git a/go.mod b/go.mod index e9a26180..1ef9e8c2 100644 --- a/go.mod +++ b/go.mod @@ -12,9 +12,9 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261021163409-4fd8d7b920cf + github.com/fil-forge/hilt v0.0.1-0.20261021173042-01deb7cf097f github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e - github.com/fil-forge/libforge v0.0.0-20261020103720-b6aeeaf7c926 + github.com/fil-forge/libforge v0.0.0-20261021172342-2c6f74f5d004 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 github.com/fil-forge/swarf v0.0.1-0.20260922133653-0b2e5b2e1c6c github.com/fil-forge/ucantone v0.0.0-20260924160040-c31dec73d9b3 diff --git a/go.sum b/go.sum index 36819864..de6b16d2 100644 --- a/go.sum +++ b/go.sum @@ -183,12 +183,12 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261021163409-4fd8d7b920cf h1:jpKCNaF5iiDIxfNW71O6ZBlcF+cdiPJCgeFPwVj10oc= -github.com/fil-forge/hilt v0.0.1-0.20261021163409-4fd8d7b920cf/go.mod h1:E75kjv9Yg0I6VPf0ghWQ8ziYwnY5UM7iSCRm2Hgt8cY= +github.com/fil-forge/hilt v0.0.1-0.20261021173042-01deb7cf097f h1:d3vd6gEzmS9WnsccgJF0A8oZDLrG03+yadbRC7puc28= +github.com/fil-forge/hilt v0.0.1-0.20261021173042-01deb7cf097f/go.mod h1:WVrsW+W/Z0t0//fyZGxgb6lRKyfwi2udmJDjUhMjRbs= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= -github.com/fil-forge/libforge v0.0.0-20261020103720-b6aeeaf7c926 h1:i6Zr11ukpOvJx80T06okJidomSv7DJCj5mSGdOy8ltk= -github.com/fil-forge/libforge v0.0.0-20261020103720-b6aeeaf7c926/go.mod h1:ZtPwUgXm7gqQSmjmIcfMy5AwWpFIR02WMZvbgNOwi/4= +github.com/fil-forge/libforge v0.0.0-20261021172342-2c6f74f5d004 h1:WZWzEoPLwmYyG7eFPO7uwvYSjngtc7HZxvKbZssskQU= +github.com/fil-forge/libforge v0.0.0-20261021172342-2c6f74f5d004/go.mod h1:ZtPwUgXm7gqQSmjmIcfMy5AwWpFIR02WMZvbgNOwi/4= github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 h1:q8V6kT/wfDTvnVK5znv0zWKDXrWrIk91xzcYrEgXv9g= github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0/go.mod h1:D2ILnC2lBNYav4+y4701sGDClyfsWAfFy19D++6rYP8= github.com/fil-forge/swarf v0.0.1-0.20260922133653-0b2e5b2e1c6c h1:uzLK4I8rD2SXBTeb/qOMbnfD/oAr2RoFVP730abzzyo= diff --git a/iam/service.go b/iam/service.go index d1ff343b..832981a2 100644 --- a/iam/service.go +++ b/iam/service.go @@ -51,6 +51,7 @@ package iam import ( + "bytes" "context" "errors" "fmt" @@ -187,6 +188,14 @@ func (s *Service) GetUserAccountForRequest(ctx fiber.Ctx, accessKeyStr string) ( // so the Hilt round-trip is bounded by the request's lifetime. reqCtx := ctx.RequestCtx() req := fasthttputil.RequestFromHTTPContext(reqCtx) + // The body travels to Hilt only for the operations whose body decides a + // permission (a DeleteObjects' entries may name versions), which Hilt's + // classifier names; every other request goes without one, so a streaming + // write's body is never read here. Hilt binds it to the signature before + // reading it, and so does the fast path. + if op, err := hiltauth.OperationFor(req); err == nil && op.ReadsBody() { + req.Body = bytes.Clone(reqCtx.Request.Body()) + } ctx.Locals(reqscope.RequestKey(), req) // A query naming two subresources addresses no operation. S3 refuses it @@ -383,9 +392,10 @@ func mapAuthError(err error) (error, bool) { // Another tenant's bucket is AccessDenied, as S3 answers for another // account's bucket: bucket names are global, so existence is no secret. hiltauth.ForeignBucketErrorName, - // A copy whose source header the signature does not cover cannot be - // authorized; the SDKs always sign it. - hiltauth.UnsignedHeaderErrorName: + // A header the signature does not cover, or a body it does not bind, + // cannot be authorized on; the SDKs sign both. + hiltauth.UnsignedHeaderErrorName, + hiltauth.UnboundBodyErrorName: return s3err.GetAPIError(s3err.ErrAccessDenied), true case hiltauth.TemporarilyUnavailableErrorName: // An authorize that waited out Hilt's lock timeout behind a policy diff --git a/iam/service_internal_test.go b/iam/service_internal_test.go index 0dc6d7a3..0f33aa9e 100644 --- a/iam/service_internal_test.go +++ b/iam/service_internal_test.go @@ -2,6 +2,8 @@ package iam import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "net/http" @@ -476,6 +478,105 @@ func signedRequest(t *testing.T, method, host, rawPath, accessKeyID, secret stri return s3.Request{Method: method, Headers: headers, URL: req.URL.RequestURI()} } +// signedBody builds a real SigV4-signed POST of body to path with the payload +// hash signed, as the SDKs sign a DeleteObjects, and attaches the body. +func signedBody(t *testing.T, host, path, accessKeyID, secret, body string) s3.Request { + t.Helper() + sum := sha256.Sum256([]byte(body)) + payloadHash := hex.EncodeToString(sum[:]) + req, err := http.NewRequest(http.MethodPost, "http://"+host+path, strings.NewReader(body)) + require.NoError(t, err) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + _, err = v4.NewSigner().SignHTTP(context.Background(), + awsv4.Credentials{AccessKeyID: accessKeyID, SecretAccessKey: secret}, + req, payloadHash, "s3", "us-east-1", time.Now(), + []string{"host", "x-amz-content-sha256", "x-amz-date"}, + func(o *v4.SignerOptions) { o.DisableURIPathEscaping = true }) + require.NoError(t, err) + headers := map[string]string{"Host": host} + for k := range req.Header { + headers[k] = req.Header.Get(k) + } + return s3.Request{Method: http.MethodPost, Headers: headers, URL: req.URL.RequestURI(), Body: []byte(body)} +} + +// TestDeleteOnlyKeyIsDeniedOnVersions covers the body the fast path decides +// on: a DeleteObjects entry naming a version needs s3:DeleteObjectVersion, +// which Hilt's classifier reads from the body once the signature binds it. +// A key whose cached set holds s3:DeleteObject alone is refused locally, +// and a tampered body is left to Hilt. +func TestDeleteOnlyKeyIsDeniedOnVersions(t *testing.T) { + accessKey, err := ed25519.GenerateIssuer() + require.NoError(t, err) + spaceIssuer, err := ed25519.GenerateIssuer() + require.NoError(t, err) + agent, err := ed25519.GenerateIssuer() + require.NoError(t, err) + + accessKeyID := accessKey.DID().Identifier() + const secret = "test-secret-access-key" + space := spaceIssuer.DID() + resolver := fixedResolver{name: "bkt", state: ®istry.State{Name: "bkt", Space: space}} + + service := func(t *testing.T, req s3.Request, actions ...string) *Service { + t.Helper() + sr, err := sigv4.Parse(sigv4.Request{Method: req.Method, Headers: req.Headers, URL: req.URL}) + require.NoError(t, err) + key, err := sigv4.DeriveKey(sr, secret) + require.NoError(t, err) + s := localService(agent.DID(), resolver) + s.keys.Put(accessKeyID, time.Hour, s3.VerificationKey{Kind: s3.KeyKindSigV4, Data: key}) + s.proofs.For(accessKey.DID()).PutPermissions(space, time.Hour, actions) + for _, a := range actions { + s.proofs.Deposit(accessKey.DID(), commandChains(t, spaceIssuer, accessKey, agent, s3perm.CommandsFor(a))...) + } + return s + } + const versions = `abv1` + const plain = `a` + + t.Run("a version entry is refused on the cached set", func(t *testing.T) { + req := signedBody(t, "s3.example", "/bkt?delete", accessKeyID, secret, versions) + op, reqs, err := hiltauth.RequirementsFor(req) + require.NoError(t, err) + require.Equal(t, hiltauth.OpDeleteObjects, op) + require.Len(t, reqs, 2) + + s := service(t, req, "s3:DeleteObject") + _, ok, err := s.authorizeLocal(context.Background(), req, accessKeyID, s.proofs.For(accessKey.DID())) + require.False(t, ok) + require.Equal(t, s3err.GetAPIError(s3err.ErrAccessDenied), err) + require.Zero(t, s.authorizer.(*refusingAuthorizer).calls) + }) + + t.Run("plain entries are still authorized", func(t *testing.T) { + req := signedBody(t, "s3.example", "/bkt?delete", accessKeyID, secret, plain) + s := service(t, req, "s3:DeleteObject") + _, ok, err := s.authorizeLocal(context.Background(), req, accessKeyID, s.proofs.For(accessKey.DID())) + require.NoError(t, err) + require.True(t, ok) + }) + + t.Run("the version permission authorizes the batch", func(t *testing.T) { + req := signedBody(t, "s3.example", "/bkt?delete", accessKeyID, secret, versions) + s := service(t, req, "s3:DeleteObject", "s3:DeleteObjectVersion") + _, ok, err := s.authorizeLocal(context.Background(), req, accessKeyID, s.proofs.For(accessKey.DID())) + require.NoError(t, err) + require.True(t, ok) + }) + + t.Run("a body the signature does not bind is left to Hilt", func(t *testing.T) { + req := signedBody(t, "s3.example", "/bkt?delete", accessKeyID, secret, plain) + req.Body = []byte(versions) + _, _, err := hiltauth.RequirementsFor(req) + require.ErrorIs(t, err, hiltauth.ErrUnboundBody) + s := service(t, req, "s3:DeleteObject", "s3:DeleteObjectVersion") + _, ok, err := s.authorizeLocal(context.Background(), req, accessKeyID, s.proofs.For(accessKey.DID())) + require.False(t, ok) + require.NoError(t, err) + }) +} + // TestPutOnlyKeyIsDeniedOnReads pins the defect the action set closes: a key // granted only s3:PutObject holds chains for every command a read, a listing // or a multipart abort needs (PutObject's command set is a superset of each), @@ -681,6 +782,7 @@ func TestMapAuthError(t *testing.T) { hiltauth.BucketNotPermittedErrorName: {"AccessDenied", 403}, hiltauth.ForeignBucketErrorName: {"AccessDenied", 403}, hiltauth.UnsignedHeaderErrorName: {"AccessDenied", 403}, + hiltauth.UnboundBodyErrorName: {"AccessDenied", 403}, hiltauth.TemporarilyUnavailableErrorName: {"ServiceUnavailable", 503}, s3bkt.InvalidPolicyErrorName: {"MalformedPolicy", 400}, } From 22cf746699bce2e3023c09899b62851f5fe904e9 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 09:30:55 +0100 Subject: [PATCH 02/17] chore: move the libforge and hilt pins to the 8 MiB body limit Co-Authored-By: Claude Fable 5.1 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 1ef9e8c2..b4d63b44 100644 --- a/go.mod +++ b/go.mod @@ -12,9 +12,9 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261021173042-01deb7cf097f + github.com/fil-forge/hilt v0.0.1-0.20261021180611-4633230e5472 github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e - github.com/fil-forge/libforge v0.0.0-20261021172342-2c6f74f5d004 + github.com/fil-forge/libforge v0.0.0-20261021180520-6f109fba74b7 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 github.com/fil-forge/swarf v0.0.1-0.20260922133653-0b2e5b2e1c6c github.com/fil-forge/ucantone v0.0.0-20260924160040-c31dec73d9b3 diff --git a/go.sum b/go.sum index de6b16d2..fc10a52c 100644 --- a/go.sum +++ b/go.sum @@ -183,12 +183,12 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261021173042-01deb7cf097f h1:d3vd6gEzmS9WnsccgJF0A8oZDLrG03+yadbRC7puc28= -github.com/fil-forge/hilt v0.0.1-0.20261021173042-01deb7cf097f/go.mod h1:WVrsW+W/Z0t0//fyZGxgb6lRKyfwi2udmJDjUhMjRbs= +github.com/fil-forge/hilt v0.0.1-0.20261021180611-4633230e5472 h1:0BPve4+rT8yH1v8lmk0l5CM/G94OKXEQRxmwseN4H4I= +github.com/fil-forge/hilt v0.0.1-0.20261021180611-4633230e5472/go.mod h1:L2/jT/Qsvb9PGouRYIchTIY14c6gFQ3wMkDAa8EpQ8Y= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= -github.com/fil-forge/libforge v0.0.0-20261021172342-2c6f74f5d004 h1:WZWzEoPLwmYyG7eFPO7uwvYSjngtc7HZxvKbZssskQU= -github.com/fil-forge/libforge v0.0.0-20261021172342-2c6f74f5d004/go.mod h1:ZtPwUgXm7gqQSmjmIcfMy5AwWpFIR02WMZvbgNOwi/4= +github.com/fil-forge/libforge v0.0.0-20261021180520-6f109fba74b7 h1:z/sU1w76G1AsI59Jca3xpAzVLsYef1zVzMDpU85M/Tk= +github.com/fil-forge/libforge v0.0.0-20261021180520-6f109fba74b7/go.mod h1:ZtPwUgXm7gqQSmjmIcfMy5AwWpFIR02WMZvbgNOwi/4= github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 h1:q8V6kT/wfDTvnVK5znv0zWKDXrWrIk91xzcYrEgXv9g= github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0/go.mod h1:D2ILnC2lBNYav4+y4701sGDClyfsWAfFy19D++6rYP8= github.com/fil-forge/swarf v0.0.1-0.20260922133653-0b2e5b2e1c6c h1:uzLK4I8rD2SXBTeb/qOMbnfD/oAr2RoFVP730abzzyo= From e871d5228fa3fd580d393c70003e1fac5dbb5912 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 09:36:54 +0100 Subject: [PATCH 03/17] chore: move the libforge pin to main and the hilt pin along fil-forge/hilt#106 Co-Authored-By: Claude Fable 5.1 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b4d63b44..75191258 100644 --- a/go.mod +++ b/go.mod @@ -12,9 +12,9 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261021180611-4633230e5472 + github.com/fil-forge/hilt v0.0.1-0.20261022083552-3d1859603c28 github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e - github.com/fil-forge/libforge v0.0.0-20261021180520-6f109fba74b7 + github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 github.com/fil-forge/swarf v0.0.1-0.20260922133653-0b2e5b2e1c6c github.com/fil-forge/ucantone v0.0.0-20260924160040-c31dec73d9b3 diff --git a/go.sum b/go.sum index fc10a52c..8fd55230 100644 --- a/go.sum +++ b/go.sum @@ -183,12 +183,12 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261021180611-4633230e5472 h1:0BPve4+rT8yH1v8lmk0l5CM/G94OKXEQRxmwseN4H4I= -github.com/fil-forge/hilt v0.0.1-0.20261021180611-4633230e5472/go.mod h1:L2/jT/Qsvb9PGouRYIchTIY14c6gFQ3wMkDAa8EpQ8Y= +github.com/fil-forge/hilt v0.0.1-0.20261022083552-3d1859603c28 h1:NgEPJJ9vv9CTjRlw/z2qzlTMpS/hbZu63/n0C9bhVWQ= +github.com/fil-forge/hilt v0.0.1-0.20261022083552-3d1859603c28/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= -github.com/fil-forge/libforge v0.0.0-20261021180520-6f109fba74b7 h1:z/sU1w76G1AsI59Jca3xpAzVLsYef1zVzMDpU85M/Tk= -github.com/fil-forge/libforge v0.0.0-20261021180520-6f109fba74b7/go.mod h1:ZtPwUgXm7gqQSmjmIcfMy5AwWpFIR02WMZvbgNOwi/4= +github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 h1:lmjXiMsVFHsln4hmSjtuJxsHhLP49WOp48Ot5KPctMQ= +github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889/go.mod h1:ZtPwUgXm7gqQSmjmIcfMy5AwWpFIR02WMZvbgNOwi/4= github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 h1:q8V6kT/wfDTvnVK5znv0zWKDXrWrIk91xzcYrEgXv9g= github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0/go.mod h1:D2ILnC2lBNYav4+y4701sGDClyfsWAfFy19D++6rYP8= github.com/fil-forge/swarf v0.0.1-0.20260922133653-0b2e5b2e1c6c h1:uzLK4I8rD2SXBTeb/qOMbnfD/oAr2RoFVP730abzzyo= From c5298c2dd9dc2d886e0edb6ab075dfcc5eac7b46 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 10:06:08 +0100 Subject: [PATCH 04/17] chore: move the hilt pin along fil-forge/hilt#106 Co-Authored-By: Claude Fable 5.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 75191258..d3943337 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261022083552-3d1859603c28 + github.com/fil-forge/hilt v0.0.1-0.20261022090533-71875c9f140c github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 diff --git a/go.sum b/go.sum index 8fd55230..63833b98 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261022083552-3d1859603c28 h1:NgEPJJ9vv9CTjRlw/z2qzlTMpS/hbZu63/n0C9bhVWQ= -github.com/fil-forge/hilt v0.0.1-0.20261022083552-3d1859603c28/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= +github.com/fil-forge/hilt v0.0.1-0.20261022090533-71875c9f140c h1:jYZpy8vbsiW4awxm8piDKeSP4Il4Huhbj5WSlZdcn48= +github.com/fil-forge/hilt v0.0.1-0.20261022090533-71875c9f140c/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 h1:lmjXiMsVFHsln4hmSjtuJxsHhLP49WOp48Ot5KPctMQ= From db004f1540967b722d23cad4e058d441a7f692fd Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 10:12:23 +0100 Subject: [PATCH 05/17] chore: move the hilt pin along fil-forge/hilt#106 Co-Authored-By: Claude Fable 5.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d3943337..1fd6309f 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261022090533-71875c9f140c + github.com/fil-forge/hilt v0.0.1-0.20261022091138-562cea9312e2 github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 diff --git a/go.sum b/go.sum index 63833b98..d7ac3e14 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261022090533-71875c9f140c h1:jYZpy8vbsiW4awxm8piDKeSP4Il4Huhbj5WSlZdcn48= -github.com/fil-forge/hilt v0.0.1-0.20261022090533-71875c9f140c/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= +github.com/fil-forge/hilt v0.0.1-0.20261022091138-562cea9312e2 h1:4zu3juOZkYnJYnXbgmjQl7Q81MEG6iiRot3+v6ySRBo= +github.com/fil-forge/hilt v0.0.1-0.20261022091138-562cea9312e2/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 h1:lmjXiMsVFHsln4hmSjtuJxsHhLP49WOp48Ot5KPctMQ= From 9fcbcc80a33c2b655598ae37247c84418b1ce135 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 10:50:28 +0100 Subject: [PATCH 06/17] chore: move the hilt pin along fil-forge/hilt#106 Co-Authored-By: Claude Fable 5.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1fd6309f..c9af5ba4 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261022091138-562cea9312e2 + github.com/fil-forge/hilt v0.0.1-0.20261022094949-a50ad591d85e github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 diff --git a/go.sum b/go.sum index d7ac3e14..34b083ba 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261022091138-562cea9312e2 h1:4zu3juOZkYnJYnXbgmjQl7Q81MEG6iiRot3+v6ySRBo= -github.com/fil-forge/hilt v0.0.1-0.20261022091138-562cea9312e2/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= +github.com/fil-forge/hilt v0.0.1-0.20261022094949-a50ad591d85e h1:t9lAja0rrnUOiviEtOPon7ZyGEpnk72OtoVbNXg1LOk= +github.com/fil-forge/hilt v0.0.1-0.20261022094949-a50ad591d85e/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 h1:lmjXiMsVFHsln4hmSjtuJxsHhLP49WOp48Ot5KPctMQ= From c674e2a7933db4f08740b823e69906a37f079902 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 11:06:50 +0100 Subject: [PATCH 07/17] fix(iam): capture a DeleteObjects body up to Hilt's limit and no further The body was drained from the stream and cloned before anything about the request was verified, so an unauthenticated POST ?delete could cost an unbounded allocation. It is now read only to one byte past Hilt's limit, refused as EntityTooLarge beyond it, and set back on the request within it so the handler reads the same bytes. Co-Authored-By: Claude Fable 5.1 --- iam/service.go | 18 ++++++-- internal/fasthttputil/request.go | 35 +++++++++++++++ internal/fasthttputil/request_test.go | 65 +++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 4 deletions(-) diff --git a/iam/service.go b/iam/service.go index 832981a2..3fefed32 100644 --- a/iam/service.go +++ b/iam/service.go @@ -51,7 +51,6 @@ package iam import ( - "bytes" "context" "errors" "fmt" @@ -191,10 +190,21 @@ func (s *Service) GetUserAccountForRequest(ctx fiber.Ctx, accessKeyStr string) ( // The body travels to Hilt only for the operations whose body decides a // permission (a DeleteObjects' entries may name versions), which Hilt's // classifier names; every other request goes without one, so a streaming - // write's body is never read here. Hilt binds it to the signature before - // reading it, and so does the fast path. + // write's body is never read here. It is captured up to Hilt's limit and + // no further, since nothing about the request is verified yet, and left + // in place for the handler. Hilt binds it to the signature before reading + // it, and so does the fast path. if op, err := hiltauth.OperationFor(req); err == nil && op.ReadsBody() { - req.Body = bytes.Clone(reqCtx.Request.Body()) + body, err := fasthttputil.BoundedBody(reqCtx, hiltauth.MaxBodySize) + if errors.Is(err, fasthttputil.ErrBodyTooLarge) { + path = "local" + s.logger.Debug("hilt/iam: request body over the limit", zap.String("access", accessKeyStr), zap.Stringer("operation", op)) + return auth.Account{}, s3err.GetAPIError(s3err.ErrEntityTooLarge) + } + if err != nil { + return auth.Account{}, fmt.Errorf("hilt/iam: %w", err) + } + req.Body = body } ctx.Locals(reqscope.RequestKey(), req) diff --git a/internal/fasthttputil/request.go b/internal/fasthttputil/request.go index 36d7bd84..14d41b84 100644 --- a/internal/fasthttputil/request.go +++ b/internal/fasthttputil/request.go @@ -1,6 +1,11 @@ package fasthttputil import ( + "bytes" + "errors" + "fmt" + "io" + "github.com/fil-forge/libforge/commands/s3" "github.com/valyala/fasthttp" ) @@ -35,3 +40,33 @@ func RequestFromHTTPContext(rc *fasthttp.RequestCtx) s3.Request { Headers: headers, } } + +// ErrBodyTooLarge is returned by BoundedBody for a body over its limit. +var ErrBodyTooLarge = errors.New("request body exceeds the limit") + +// BoundedBody captures the request's body, at most limit bytes, as the +// caller's own copy, and leaves it in place for the handler. A streamed body +// (the gateway streams every request body) is read from the stream only up +// to one byte past the limit, so an oversized body costs the limit and no +// more before it is refused; within the limit it is set back on the +// request, so the handler's Body() serves it. An in-memory body over the +// limit is refused too. +func BoundedBody(rc *fasthttp.RequestCtx, limit int) ([]byte, error) { + req := &rc.Request + if req.IsBodyStream() { + buf, err := io.ReadAll(io.LimitReader(req.BodyStream(), int64(limit)+1)) + if err != nil { + return nil, fmt.Errorf("reading request body: %w", err) + } + if len(buf) > limit { + return nil, ErrBodyTooLarge + } + req.SetBody(buf) + return bytes.Clone(buf), nil + } + body := req.Body() + if len(body) > limit { + return nil, ErrBodyTooLarge + } + return bytes.Clone(body), nil +} diff --git a/internal/fasthttputil/request_test.go b/internal/fasthttputil/request_test.go index 1a5732b8..205651ee 100644 --- a/internal/fasthttputil/request_test.go +++ b/internal/fasthttputil/request_test.go @@ -1,6 +1,9 @@ package fasthttputil import ( + "errors" + "io" + "strings" "testing" "github.com/valyala/fasthttp" @@ -41,3 +44,65 @@ func TestRequestFromHTTPContext_SingleHeader(t *testing.T) { t.Errorf("Content-Encoding = %q, want %q", v, "gzip") } } + +// TestBoundedBody covers the capture of a body hilt decides on: a streamed +// body within the limit is returned and left for the handler, one over it is +// refused after reading one byte past the limit, and an in-memory body is +// bounded the same way. +func TestBoundedBody(t *testing.T) { + t.Run("streamed within the limit", func(t *testing.T) { + var rc fasthttp.RequestCtx + rc.Request.Header.SetMethod("POST") + rc.Request.SetBodyStream(strings.NewReader(""), -1) + got, err := BoundedBody(&rc, 16) + if err != nil { + t.Fatal(err) + } + if string(got) != "" { + t.Errorf("body = %q", got) + } + if rc.Request.IsBodyStream() || string(rc.Request.Body()) != "" { + t.Errorf("handler body = %q, stream=%v", rc.Request.Body(), rc.Request.IsBodyStream()) + } + got[0] = 'x' + if string(rc.Request.Body()) != "" { + t.Error("the returned body must be the caller's own copy") + } + }) + t.Run("streamed over the limit", func(t *testing.T) { + var rc fasthttp.RequestCtx + rc.Request.Header.SetMethod("POST") + src := &countingReader{Reader: strings.NewReader(strings.Repeat("x", 100))} + rc.Request.SetBodyStream(src, -1) + _, err := BoundedBody(&rc, 16) + if !errors.Is(err, ErrBodyTooLarge) { + t.Fatalf("err = %v, want ErrBodyTooLarge", err) + } + if src.n > 17 { + t.Errorf("read %d bytes of a refused body, want at most 17", src.n) + } + }) + t.Run("in memory", func(t *testing.T) { + var rc fasthttp.RequestCtx + rc.Request.SetBody([]byte("")) + got, err := BoundedBody(&rc, 16) + if err != nil || string(got) != "" { + t.Fatalf("got %q, %v", got, err) + } + rc.Request.SetBody([]byte(strings.Repeat("x", 17))) + if _, err := BoundedBody(&rc, 16); !errors.Is(err, ErrBodyTooLarge) { + t.Fatalf("err = %v, want ErrBodyTooLarge", err) + } + }) +} + +type countingReader struct { + io.Reader + n int +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.Reader.Read(p) + c.n += n + return n, err +} From 39fd02776cbaf65aec27f0a6d557f543b623eaef Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 11:10:27 +0100 Subject: [PATCH 08/17] chore: move the hilt pin along fil-forge/hilt#106 Co-Authored-By: Claude Fable 5.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c9af5ba4..e358a28a 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261022094949-a50ad591d85e + github.com/fil-forge/hilt v0.0.1-0.20261022100944-2ca0b8cb5d4b github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 diff --git a/go.sum b/go.sum index 34b083ba..22623406 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261022094949-a50ad591d85e h1:t9lAja0rrnUOiviEtOPon7ZyGEpnk72OtoVbNXg1LOk= -github.com/fil-forge/hilt v0.0.1-0.20261022094949-a50ad591d85e/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= +github.com/fil-forge/hilt v0.0.1-0.20261022100944-2ca0b8cb5d4b h1:y4XRgIHw5rg0K7EGj+u5L9j4beGNw4mWH0VI/2Tjw+0= +github.com/fil-forge/hilt v0.0.1-0.20261022100944-2ca0b8cb5d4b/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 h1:lmjXiMsVFHsln4hmSjtuJxsHhLP49WOp48Ot5KPctMQ= From 2fc389bb28a3b619829146babccc743560f80d8b Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 11:11:17 +0100 Subject: [PATCH 09/17] refactor(fasthttputil): LimitedBody Co-Authored-By: Claude Fable 5.1 --- iam/service.go | 2 +- internal/fasthttputil/request.go | 6 +++--- internal/fasthttputil/request_test.go | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/iam/service.go b/iam/service.go index 3fefed32..0efa9ef3 100644 --- a/iam/service.go +++ b/iam/service.go @@ -195,7 +195,7 @@ func (s *Service) GetUserAccountForRequest(ctx fiber.Ctx, accessKeyStr string) ( // in place for the handler. Hilt binds it to the signature before reading // it, and so does the fast path. if op, err := hiltauth.OperationFor(req); err == nil && op.ReadsBody() { - body, err := fasthttputil.BoundedBody(reqCtx, hiltauth.MaxBodySize) + body, err := fasthttputil.LimitedBody(reqCtx, hiltauth.MaxBodySize) if errors.Is(err, fasthttputil.ErrBodyTooLarge) { path = "local" s.logger.Debug("hilt/iam: request body over the limit", zap.String("access", accessKeyStr), zap.Stringer("operation", op)) diff --git a/internal/fasthttputil/request.go b/internal/fasthttputil/request.go index 14d41b84..67ae5f13 100644 --- a/internal/fasthttputil/request.go +++ b/internal/fasthttputil/request.go @@ -41,17 +41,17 @@ func RequestFromHTTPContext(rc *fasthttp.RequestCtx) s3.Request { } } -// ErrBodyTooLarge is returned by BoundedBody for a body over its limit. +// ErrBodyTooLarge is returned by LimitedBody for a body over its limit. var ErrBodyTooLarge = errors.New("request body exceeds the limit") -// BoundedBody captures the request's body, at most limit bytes, as the +// LimitedBody captures the request's body, at most limit bytes, as the // caller's own copy, and leaves it in place for the handler. A streamed body // (the gateway streams every request body) is read from the stream only up // to one byte past the limit, so an oversized body costs the limit and no // more before it is refused; within the limit it is set back on the // request, so the handler's Body() serves it. An in-memory body over the // limit is refused too. -func BoundedBody(rc *fasthttp.RequestCtx, limit int) ([]byte, error) { +func LimitedBody(rc *fasthttp.RequestCtx, limit int) ([]byte, error) { req := &rc.Request if req.IsBodyStream() { buf, err := io.ReadAll(io.LimitReader(req.BodyStream(), int64(limit)+1)) diff --git a/internal/fasthttputil/request_test.go b/internal/fasthttputil/request_test.go index 205651ee..e7b25a3b 100644 --- a/internal/fasthttputil/request_test.go +++ b/internal/fasthttputil/request_test.go @@ -45,16 +45,16 @@ func TestRequestFromHTTPContext_SingleHeader(t *testing.T) { } } -// TestBoundedBody covers the capture of a body hilt decides on: a streamed +// TestLimitedBody covers the capture of a body hilt decides on: a streamed // body within the limit is returned and left for the handler, one over it is // refused after reading one byte past the limit, and an in-memory body is // bounded the same way. -func TestBoundedBody(t *testing.T) { +func TestLimitedBody(t *testing.T) { t.Run("streamed within the limit", func(t *testing.T) { var rc fasthttp.RequestCtx rc.Request.Header.SetMethod("POST") rc.Request.SetBodyStream(strings.NewReader(""), -1) - got, err := BoundedBody(&rc, 16) + got, err := LimitedBody(&rc, 16) if err != nil { t.Fatal(err) } @@ -74,7 +74,7 @@ func TestBoundedBody(t *testing.T) { rc.Request.Header.SetMethod("POST") src := &countingReader{Reader: strings.NewReader(strings.Repeat("x", 100))} rc.Request.SetBodyStream(src, -1) - _, err := BoundedBody(&rc, 16) + _, err := LimitedBody(&rc, 16) if !errors.Is(err, ErrBodyTooLarge) { t.Fatalf("err = %v, want ErrBodyTooLarge", err) } @@ -85,12 +85,12 @@ func TestBoundedBody(t *testing.T) { t.Run("in memory", func(t *testing.T) { var rc fasthttp.RequestCtx rc.Request.SetBody([]byte("")) - got, err := BoundedBody(&rc, 16) + got, err := LimitedBody(&rc, 16) if err != nil || string(got) != "" { t.Fatalf("got %q, %v", got, err) } rc.Request.SetBody([]byte(strings.Repeat("x", 17))) - if _, err := BoundedBody(&rc, 16); !errors.Is(err, ErrBodyTooLarge) { + if _, err := LimitedBody(&rc, 16); !errors.Is(err, ErrBodyTooLarge) { t.Fatalf("err = %v, want ErrBodyTooLarge", err) } }) From 8a650f57d1d22b173f613ddfc2f49d4f8e6bf1c3 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 11:18:43 +0100 Subject: [PATCH 10/17] chore: follow hilt's ErrInvalidBody and move the pin along fil-forge/hilt#106 Co-Authored-By: Claude Fable 5.1 --- docs/diagrams.md | 2 +- go.mod | 2 +- go.sum | 4 ++-- iam/service.go | 2 +- iam/service_internal_test.go | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/diagrams.md b/docs/diagrams.md index c8a5bfc8..a7ccf03c 100644 --- a/docs/diagrams.md +++ b/docs/diagrams.md @@ -919,7 +919,7 @@ flowchart TB refuses a header the signature does not cover or a body it does not bind, as hilt itself does, so the fast path decides nothing from such a request: it goes to hilt, which refuses it as `UnsignedHeader` or - `UnboundBody`. + `InvalidBody`. Cross-references: [`architecture.md` §9](./architecture.md#9-the-system-contract-piri--sprue--indexer). diff --git a/go.mod b/go.mod index e358a28a..8c24ff9b 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261022100944-2ca0b8cb5d4b + github.com/fil-forge/hilt v0.0.1-0.20261022101809-a0b7b686540e github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 diff --git a/go.sum b/go.sum index 22623406..0376b6ed 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261022100944-2ca0b8cb5d4b h1:y4XRgIHw5rg0K7EGj+u5L9j4beGNw4mWH0VI/2Tjw+0= -github.com/fil-forge/hilt v0.0.1-0.20261022100944-2ca0b8cb5d4b/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= +github.com/fil-forge/hilt v0.0.1-0.20261022101809-a0b7b686540e h1:cbEyj3g+aPGw8K3vniNZedu/UdGaQ2WdVAwHZuPKcic= +github.com/fil-forge/hilt v0.0.1-0.20261022101809-a0b7b686540e/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 h1:lmjXiMsVFHsln4hmSjtuJxsHhLP49WOp48Ot5KPctMQ= diff --git a/iam/service.go b/iam/service.go index 0efa9ef3..2076c30a 100644 --- a/iam/service.go +++ b/iam/service.go @@ -405,7 +405,7 @@ func mapAuthError(err error) (error, bool) { // A header the signature does not cover, or a body it does not bind, // cannot be authorized on; the SDKs sign both. hiltauth.UnsignedHeaderErrorName, - hiltauth.UnboundBodyErrorName: + hiltauth.InvalidBodyErrorName: return s3err.GetAPIError(s3err.ErrAccessDenied), true case hiltauth.TemporarilyUnavailableErrorName: // An authorize that waited out Hilt's lock timeout behind a policy diff --git a/iam/service_internal_test.go b/iam/service_internal_test.go index 0f33aa9e..be9666df 100644 --- a/iam/service_internal_test.go +++ b/iam/service_internal_test.go @@ -569,7 +569,7 @@ func TestDeleteOnlyKeyIsDeniedOnVersions(t *testing.T) { req := signedBody(t, "s3.example", "/bkt?delete", accessKeyID, secret, plain) req.Body = []byte(versions) _, _, err := hiltauth.RequirementsFor(req) - require.ErrorIs(t, err, hiltauth.ErrUnboundBody) + require.ErrorIs(t, err, hiltauth.ErrInvalidBody) s := service(t, req, "s3:DeleteObject", "s3:DeleteObjectVersion") _, ok, err := s.authorizeLocal(context.Background(), req, accessKeyID, s.proofs.For(accessKey.DID())) require.False(t, ok) @@ -782,7 +782,7 @@ func TestMapAuthError(t *testing.T) { hiltauth.BucketNotPermittedErrorName: {"AccessDenied", 403}, hiltauth.ForeignBucketErrorName: {"AccessDenied", 403}, hiltauth.UnsignedHeaderErrorName: {"AccessDenied", 403}, - hiltauth.UnboundBodyErrorName: {"AccessDenied", 403}, + hiltauth.InvalidBodyErrorName: {"AccessDenied", 403}, hiltauth.TemporarilyUnavailableErrorName: {"ServiceUnavailable", 503}, s3bkt.InvalidPolicyErrorName: {"MalformedPolicy", 400}, } From 955c707a94a37bef4c330812c0643f0659f915ec Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 11:24:58 +0100 Subject: [PATCH 11/17] chore: move the hilt pin along fil-forge/hilt#106 Co-Authored-By: Claude Fable 5.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8c24ff9b..16458484 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261022101809-a0b7b686540e + github.com/fil-forge/hilt v0.0.1-0.20261022102422-71040968d1cf github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 diff --git a/go.sum b/go.sum index 0376b6ed..cd702e92 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261022101809-a0b7b686540e h1:cbEyj3g+aPGw8K3vniNZedu/UdGaQ2WdVAwHZuPKcic= -github.com/fil-forge/hilt v0.0.1-0.20261022101809-a0b7b686540e/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= +github.com/fil-forge/hilt v0.0.1-0.20261022102422-71040968d1cf h1:XO0P6ajHoxnh6OterLkbi5R0QkMWe/GB8D5kqI28wDg= +github.com/fil-forge/hilt v0.0.1-0.20261022102422-71040968d1cf/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 h1:lmjXiMsVFHsln4hmSjtuJxsHhLP49WOp48Ot5KPctMQ= From 0be73a6417c5b5668b71b0ce78beaeba42b745c5 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 11:29:58 +0100 Subject: [PATCH 12/17] chore: move the hilt pin along fil-forge/hilt#106 Co-Authored-By: Claude Fable 5.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 16458484..952d93de 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261022102422-71040968d1cf + github.com/fil-forge/hilt v0.0.1-0.20261022102929-747a709f4241 github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 diff --git a/go.sum b/go.sum index cd702e92..bb16910e 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261022102422-71040968d1cf h1:XO0P6ajHoxnh6OterLkbi5R0QkMWe/GB8D5kqI28wDg= -github.com/fil-forge/hilt v0.0.1-0.20261022102422-71040968d1cf/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= +github.com/fil-forge/hilt v0.0.1-0.20261022102929-747a709f4241 h1:/xhknEEJvjzwitk/9OJ/vzzczJJt/2Cyo8lZioXGJN4= +github.com/fil-forge/hilt v0.0.1-0.20261022102929-747a709f4241/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 h1:lmjXiMsVFHsln4hmSjtuJxsHhLP49WOp48Ot5KPctMQ= From a3618d405a16ceaeea7b534f1dc3cb0d7473a458 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 11:32:38 +0100 Subject: [PATCH 13/17] chore: move the hilt pin along fil-forge/hilt#106 Co-Authored-By: Claude Fable 5.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 952d93de..ae0831ce 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261022102929-747a709f4241 + github.com/fil-forge/hilt v0.0.1-0.20261022103155-47171bd5a7e5 github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 diff --git a/go.sum b/go.sum index bb16910e..0942f844 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261022102929-747a709f4241 h1:/xhknEEJvjzwitk/9OJ/vzzczJJt/2Cyo8lZioXGJN4= -github.com/fil-forge/hilt v0.0.1-0.20261022102929-747a709f4241/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= +github.com/fil-forge/hilt v0.0.1-0.20261022103155-47171bd5a7e5 h1:oFqTJDgEtPWR1tolGTZ1pZfRxURa9bLYdjukBtAQsr0= +github.com/fil-forge/hilt v0.0.1-0.20261022103155-47171bd5a7e5/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 h1:lmjXiMsVFHsln4hmSjtuJxsHhLP49WOp48Ot5KPctMQ= From a54171844bbad1042d528696860785226f4dcdcb Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 11:42:29 +0100 Subject: [PATCH 14/17] chore: move the hilt pin to main, carrying fil-forge/hilt#106 Co-Authored-By: Claude Fable 5.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ae0831ce..d834bf08 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 github.com/cloud-portable/s3tests/packages/go v0.0.0-20260916120817-ce77c26e7f6d github.com/exaring/otelpgx v0.12.0 - github.com/fil-forge/hilt v0.0.1-0.20261022103155-47171bd5a7e5 + github.com/fil-forge/hilt v0.0.1-0.20261022104055-6d996f93ee9e github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 github.com/fil-forge/smelt v0.0.0-20261012172342-6eda2d3266f0 diff --git a/go.sum b/go.sum index 0942f844..d6330da9 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/fil-forge/automobile v0.0.1 h1:9xB3yc4l5b9EdRJSJcNwudgBFNHoMPEAdcb7Gf github.com/fil-forge/automobile v0.0.1/go.mod h1:TsO7jlO8ykJZY5tF8j4GsUcu3F02lEzxO7ULoB61hRA= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d h1:iEXJC5aJgRHq2iM4X6usfow436OQGauLjDuga1Hiq4o= github.com/fil-forge/go-ipni-tools v0.0.0-20260820150418-78ca0c16834d/go.mod h1:fwLjqz34DvfoWcuzyH8Wr7dTJdr3mzYjxZRweysHOH4= -github.com/fil-forge/hilt v0.0.1-0.20261022103155-47171bd5a7e5 h1:oFqTJDgEtPWR1tolGTZ1pZfRxURa9bLYdjukBtAQsr0= -github.com/fil-forge/hilt v0.0.1-0.20261022103155-47171bd5a7e5/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= +github.com/fil-forge/hilt v0.0.1-0.20261022104055-6d996f93ee9e h1:EsJ7N8SI/XLMvnpqdVZuj/7/s1PrUVGW4oACb8Lntgg= +github.com/fil-forge/hilt v0.0.1-0.20261022104055-6d996f93ee9e/go.mod h1:O0ULbedtoH4Zixa13p4/3FbWyZUArbpOtWgEgs/IclU= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e h1:+A26/La8EgF1bsGhSGicYV97ckec1SJwNYLWee3Xo8g= github.com/fil-forge/indexing-service v1.13.5-0.20260820151722-9eb620409a4e/go.mod h1:hrry50c9bNciNj9Z63O2QFrzidnTjkYYG2UgH4/rmVo= github.com/fil-forge/libforge v0.0.0-20261022083349-59d8bde5f889 h1:lmjXiMsVFHsln4hmSjtuJxsHhLP49WOp48Ot5KPctMQ= From 668bfdfa78fba371faad6ceb994e5e89b3618350 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 11:45:57 +0100 Subject: [PATCH 15/17] test(iam): cover body forwarding end to end and the whole-request denial A streamed DeleteObjects body reaches the authorizer exactly and is left for the handler, another operation's body is neither forwarded nor consumed, and an oversized body is refused before the authorizer is asked. The route-permission scenario expects a mixed batch naming a version to be refused as a whole, as Hilt now decides it, with nothing deleted, and the plain batch to go through. Co-Authored-By: Claude Fable 5.1 --- iam/service_test.go | 63 +++++++++++++++++++++++++++++++-- itest/forge_route_perms_test.go | 39 ++++++++++++-------- 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/iam/service_test.go b/iam/service_test.go index 6754e5b6..ace8268f 100644 --- a/iam/service_test.go +++ b/iam/service_test.go @@ -1,10 +1,12 @@ package iam_test import ( + "bytes" "context" "errors" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -24,6 +26,7 @@ import ( "github.com/stretchr/testify/require" hiltclient "github.com/fil-forge/hilt/pkg/client" + hiltauth "github.com/fil-forge/hilt/pkg/rpc/service/auth" "github.com/fil-forge/ingot/iam" "github.com/fil-forge/ingot/internal/reqscope" ) @@ -108,17 +111,73 @@ func signedCopyRequest(t *testing.T, dst, src string) *http.Request { func resolveForRequest(t *testing.T, svc *iam.Service, access string, req *http.Request) (auth.Account, error) { t.Helper() - app := fiber.New() + acct, err, _ := resolveStreamed(t, svc, access, req, false) + return acct, err +} + +// resolveStreamed runs the request through the gateway's auth step as +// resolveForRequest does, streaming the request body as the production +// server does when streamed is set, and returns the body the handler then +// reads. +func resolveStreamed(t *testing.T, svc *iam.Service, access string, req *http.Request, streamed bool) (auth.Account, error, []byte) { + t.Helper() + app := fiber.New(fiber.Config{StreamRequestBody: streamed}) var acct auth.Account var authErr error + var seen []byte app.Use(func(c fiber.Ctx) error { acct, authErr = svc.GetUserAccountForRequest(c, access) + seen = bytes.Clone(c.Body()) return nil }) resp, err := app.Test(req) require.NoError(t, err) require.NoError(t, resp.Body.Close()) - return acct, authErr + return acct, authErr, seen +} + +// TestRequestBodyForwarding covers the body hilt decides on, end to end +// through the gateway's auth step with a streamed request: a DeleteObjects +// body reaches the authorizer exactly and is left for the handler, any +// other operation's body is neither forwarded nor consumed, and an +// oversized DeleteObjects body is refused before the authorizer is asked. +func TestRequestBodyForwarding(t *testing.T) { + access, keyDID := newAccessKey(t) + key := s3.VerificationKey{Kind: s3.KeyKindSigV4, Data: []byte("dk")} + const body = `kv` + + t.Run("a DeleteObjects body is forwarded and left for the handler", func(t *testing.T) { + fake := &fakeAuthorizer{res: authorizeOK(t, keyDID, key)} + svc := iam.New(fake, iam.NewKeyProofs(), iam.NewVerificationKeyCache(), iam.NewTenantCache()) + req := httptest.NewRequest(http.MethodPost, "http://example.com/bkt?delete", strings.NewReader(body)) + _, err, seen := resolveStreamed(t, svc, access, req, true) + require.NoError(t, err) + require.Equal(t, 1, fake.calls) + require.Equal(t, []byte(body), fake.got.Body, "the authorizer must see the exact body") + require.Equal(t, []byte(body), seen, "the handler must read the same body") + }) + + t.Run("another operation's body is neither forwarded nor consumed", func(t *testing.T) { + fake := &fakeAuthorizer{res: authorizeOK(t, keyDID, key)} + svc := iam.New(fake, iam.NewKeyProofs(), iam.NewVerificationKeyCache(), iam.NewTenantCache()) + req := httptest.NewRequest(http.MethodPut, "http://example.com/bkt/k", strings.NewReader("object bytes")) + _, err, seen := resolveStreamed(t, svc, access, req, true) + require.NoError(t, err) + require.Nil(t, fake.got.Body) + require.Equal(t, []byte("object bytes"), seen) + }) + + t.Run("an oversized DeleteObjects body is refused before the authorizer is asked", func(t *testing.T) { + fake := &fakeAuthorizer{res: authorizeOK(t, keyDID, key)} + svc := iam.New(fake, iam.NewKeyProofs(), iam.NewVerificationKeyCache(), iam.NewTenantCache()) + huge := strings.Repeat("x", hiltauth.MaxBodySize+1) + req := httptest.NewRequest(http.MethodPost, "http://example.com/bkt?delete", strings.NewReader(huge)) + _, err, _ := resolveStreamed(t, svc, access, req, true) + var s3e s3err.APIError + require.ErrorAs(t, err, &s3e) + require.Equal(t, s3err.GetAPIError(s3err.ErrEntityTooLarge).Code, s3e.Code) + require.Zero(t, fake.calls) + }) } func TestGetUserAccountForRequest(t *testing.T) { diff --git a/itest/forge_route_perms_test.go b/itest/forge_route_perms_test.go index 977080a0..56d60017 100644 --- a/itest/forge_route_perms_test.go +++ b/itest/forge_route_perms_test.go @@ -20,9 +20,10 @@ import ( // permissions is refused PutBucketVersioning and DeleteObjects on an // existing bucket, since each is authorized on its own permission and // against that bucket's scope; -// - DeleteObjects is authorized on s3:DeleteObject, and an entry naming a -// version is refused, and the version kept, unless the key also holds -// s3:DeleteObjectVersion. +// - DeleteObjects is authorized on s3:DeleteObject, and a batch with an +// entry naming a version is refused as a whole, nothing in it deleted, +// unless the key also holds s3:DeleteObjectVersion: hilt decides the +// request on its body, where S3 would refuse the entry alone. // // Each denied key first makes an allowed request, so the second attempt at a // denial runs against ingot's caches and the local fast path must refuse @@ -92,21 +93,29 @@ func TestForgeRoutePermissions(t *testing.T) { v1 := put("kept") put("marked") - out, err := deleteObjects(deleter, - types.ObjectIdentifier{Key: aws.String("kept"), VersionId: aws.String(v1)}, - types.ObjectIdentifier{Key: aws.String("marked")}, - ) - if err != nil { - t.Fatalf("DeleteObjects with s3:DeleteObject: %v", err) + // The mixed batch is refused as a whole, once through hilt and once + // against the caches, with nothing in it deleted. + denied(t, "delete-only key naming a version", func() error { + _, err := deleteObjects(deleter, + types.ObjectIdentifier{Key: aws.String("kept"), VersionId: aws.String(v1)}, + types.ObjectIdentifier{Key: aws.String("marked")}, + ) + return err + }) + if _, err := full.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String("kept"), VersionId: aws.String(v1)}); err != nil { + t.Fatalf("refused version is gone: %v", err) } - if len(out.Errors) != 1 || aws.ToString(out.Errors[0].Key) != "kept" || aws.ToString(out.Errors[0].Code) != "AccessDenied" { - t.Fatalf("errors = %+v, want AccessDenied for \"kept\" alone", out.Errors) + if _, err := full.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String("marked")}); err != nil { + t.Fatalf("\"marked\" was deleted by a refused batch: %v", err) } - if len(out.Deleted) != 1 || aws.ToString(out.Deleted[0].Key) != "marked" || !aws.ToBool(out.Deleted[0].DeleteMarker) { - t.Fatalf("deleted = %+v, want a delete marker for \"marked\"", out.Deleted) + + // The plain batch is authorized on s3:DeleteObject alone. + out, err := deleteObjects(deleter, types.ObjectIdentifier{Key: aws.String("marked")}) + if err != nil { + t.Fatalf("DeleteObjects with s3:DeleteObject: %v", err) } - if _, err := full.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String("kept"), VersionId: aws.String(v1)}); err != nil { - t.Fatalf("refused version is gone: %v", err) + if len(out.Errors) != 0 || len(out.Deleted) != 1 || aws.ToString(out.Deleted[0].Key) != "marked" || !aws.ToBool(out.Deleted[0].DeleteMarker) { + t.Fatalf("result = deleted %+v, errors %+v; want a delete marker for \"marked\"", out.Deleted, out.Errors) } versionDeleter := client("version-deleter", []string{"s3:DeleteObject", "s3:DeleteObjectVersion"}) From d4d66599a5acd7a6e346c11187367dc38bd42a44 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 11:55:59 +0100 Subject: [PATCH 16/17] test(iam): return the error last Co-Authored-By: Claude Fable 5.1 --- iam/service_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/iam/service_test.go b/iam/service_test.go index ace8268f..50b90d41 100644 --- a/iam/service_test.go +++ b/iam/service_test.go @@ -111,7 +111,7 @@ func signedCopyRequest(t *testing.T, dst, src string) *http.Request { func resolveForRequest(t *testing.T, svc *iam.Service, access string, req *http.Request) (auth.Account, error) { t.Helper() - acct, err, _ := resolveStreamed(t, svc, access, req, false) + acct, _, err := resolveStreamed(t, svc, access, req, false) return acct, err } @@ -119,7 +119,7 @@ func resolveForRequest(t *testing.T, svc *iam.Service, access string, req *http. // resolveForRequest does, streaming the request body as the production // server does when streamed is set, and returns the body the handler then // reads. -func resolveStreamed(t *testing.T, svc *iam.Service, access string, req *http.Request, streamed bool) (auth.Account, error, []byte) { +func resolveStreamed(t *testing.T, svc *iam.Service, access string, req *http.Request, streamed bool) (auth.Account, []byte, error) { t.Helper() app := fiber.New(fiber.Config{StreamRequestBody: streamed}) var acct auth.Account @@ -133,7 +133,7 @@ func resolveStreamed(t *testing.T, svc *iam.Service, access string, req *http.Re resp, err := app.Test(req) require.NoError(t, err) require.NoError(t, resp.Body.Close()) - return acct, authErr, seen + return acct, seen, authErr } // TestRequestBodyForwarding covers the body hilt decides on, end to end @@ -150,7 +150,7 @@ func TestRequestBodyForwarding(t *testing.T) { fake := &fakeAuthorizer{res: authorizeOK(t, keyDID, key)} svc := iam.New(fake, iam.NewKeyProofs(), iam.NewVerificationKeyCache(), iam.NewTenantCache()) req := httptest.NewRequest(http.MethodPost, "http://example.com/bkt?delete", strings.NewReader(body)) - _, err, seen := resolveStreamed(t, svc, access, req, true) + _, seen, err := resolveStreamed(t, svc, access, req, true) require.NoError(t, err) require.Equal(t, 1, fake.calls) require.Equal(t, []byte(body), fake.got.Body, "the authorizer must see the exact body") @@ -161,7 +161,7 @@ func TestRequestBodyForwarding(t *testing.T) { fake := &fakeAuthorizer{res: authorizeOK(t, keyDID, key)} svc := iam.New(fake, iam.NewKeyProofs(), iam.NewVerificationKeyCache(), iam.NewTenantCache()) req := httptest.NewRequest(http.MethodPut, "http://example.com/bkt/k", strings.NewReader("object bytes")) - _, err, seen := resolveStreamed(t, svc, access, req, true) + _, seen, err := resolveStreamed(t, svc, access, req, true) require.NoError(t, err) require.Nil(t, fake.got.Body) require.Equal(t, []byte("object bytes"), seen) @@ -172,7 +172,7 @@ func TestRequestBodyForwarding(t *testing.T) { svc := iam.New(fake, iam.NewKeyProofs(), iam.NewVerificationKeyCache(), iam.NewTenantCache()) huge := strings.Repeat("x", hiltauth.MaxBodySize+1) req := httptest.NewRequest(http.MethodPost, "http://example.com/bkt?delete", strings.NewReader(huge)) - _, err, _ := resolveStreamed(t, svc, access, req, true) + _, _, err := resolveStreamed(t, svc, access, req, true) var s3e s3err.APIError require.ErrorAs(t, err, &s3e) require.Equal(t, s3err.GetAPIError(s3err.ErrEntityTooLarge).Code, s3e.Code) From b1055bf089a35947f04614b09936feaf510248b8 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 22 Oct 2026 12:01:16 +0100 Subject: [PATCH 17/17] test(itest): prime the deleter's caches before the mixed-batch denial Co-Authored-By: Claude Fable 5.1 --- itest/forge_route_perms_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/itest/forge_route_perms_test.go b/itest/forge_route_perms_test.go index 56d60017..1e01db85 100644 --- a/itest/forge_route_perms_test.go +++ b/itest/forge_route_perms_test.go @@ -92,6 +92,11 @@ func TestForgeRoutePermissions(t *testing.T) { deleter := client("deleter", withoutVersion) v1 := put("kept") put("marked") + // An allowed request first, so the key's action set and chains are + // cached and the second denial below runs against them. + if _, err := deleteObjects(deleter, types.ObjectIdentifier{Key: aws.String("probe")}); err != nil { + t.Fatalf("priming DeleteObjects with s3:DeleteObject: %v", err) + } // The mixed batch is refused as a whole, once through hilt and once // against the caches, with nothing in it deleted.