From df2ef43d314c9888e72f692e10a42f9cbc488d51 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Thu, 17 Sep 2026 12:56:16 +0200 Subject: [PATCH 1/2] feat: add the export session store Hilt's record of a credible-exit ceremony (fil-one/RFC #24, Tier 1): a customer exporting one bucket to a key they hold. Hilt owns the session and the customer-facing steps (opened, pop_verified, acked, released); the gateway serving the bucket reports the steps it performs (pinned, manifest_built, car_ready, validated) and keeps its own pin keyed by the same id. States only move forward, with abort from any open state; Pin records the bucket root the export is built from. --- AGENTS.md | 4 +- pkg/fx/store/memory/provider.go | 3 + pkg/fx/store/postgres/provider.go | 7 + pkg/migrations/sql/00009_export_session.sql | 41 +++ pkg/store/exportsession/exportsession.go | 141 ++++++++ pkg/store/exportsession/exportsession_test.go | 304 ++++++++++++++++++ pkg/store/exportsession/memory/store.go | 178 ++++++++++ pkg/store/exportsession/postgres/store.go | 194 +++++++++++ 8 files changed, 870 insertions(+), 2 deletions(-) create mode 100644 pkg/migrations/sql/00009_export_session.sql create mode 100644 pkg/store/exportsession/exportsession.go create mode 100644 pkg/store/exportsession/exportsession_test.go create mode 100644 pkg/store/exportsession/memory/store.go create mode 100644 pkg/store/exportsession/postgres/store.go diff --git a/AGENTS.md b/AGENTS.md index 57524210..dbeadf7d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,8 +105,8 @@ and `sprue` (the upload service; mirror its patterns where relevant). - `pkg/grant` — an access key's stored delegations: `Issue` builds the tenant→key set both key kinds hold, `Rotator` rewrites a principal-bound key's set as its policies change. See the package doc. -- `pkg/store/{tenant,accesskey,bucket,delegation,provider,principal,bucketpolicy}` — - each an interface with `memory` and `postgres` backends. +- `pkg/store/{tenant,accesskey,bucket,delegation,provider,principal,bucketpolicy,exportsession}` + — each an interface with `memory` and `postgres` backends. - `pkg/store/pglock` — the Postgres locking helpers the stores share: bounded lock waits, advisory locks, and the mapping to `store.ErrLockTimeout`. - `pkg/vault` (`memory`, `openbao`) — private-key storage; `paths.go` has the diff --git a/pkg/fx/store/memory/provider.go b/pkg/fx/store/memory/provider.go index f585ffbd..27aab45b 100644 --- a/pkg/fx/store/memory/provider.go +++ b/pkg/fx/store/memory/provider.go @@ -11,6 +11,8 @@ import ( membucketpolicy "github.com/fil-forge/hilt/pkg/store/bucketpolicy/memory" "github.com/fil-forge/hilt/pkg/store/delegation" memdelegation "github.com/fil-forge/hilt/pkg/store/delegation/memory" + "github.com/fil-forge/hilt/pkg/store/exportsession" + memexportsession "github.com/fil-forge/hilt/pkg/store/exportsession/memory" "github.com/fil-forge/hilt/pkg/store/principal" memprincipal "github.com/fil-forge/hilt/pkg/store/principal/memory" "github.com/fil-forge/hilt/pkg/store/provider" @@ -32,6 +34,7 @@ var Module = fx.Module("memory-store", }, fx.As(new(accesskey.Store))), fx.Annotate(membucket.New, fx.As(new(bucket.Store)), fx.As(fx.Self())), fx.Annotate(memdelegation.New, fx.As(new(delegation.Store))), + fx.Annotate(memexportsession.New, fx.As(new(exportsession.Store))), // The policy store refuses a bucket the bucket store does not hold, // as the Postgres backend's bucket lock does. fx.Annotate(func(buckets *membucket.Store) *membucketpolicy.Store { diff --git a/pkg/fx/store/postgres/provider.go b/pkg/fx/store/postgres/provider.go index 883ace70..c9002ca8 100644 --- a/pkg/fx/store/postgres/provider.go +++ b/pkg/fx/store/postgres/provider.go @@ -18,6 +18,8 @@ import ( pgbucketpolicy "github.com/fil-forge/hilt/pkg/store/bucketpolicy/postgres" "github.com/fil-forge/hilt/pkg/store/delegation" pgdelegation "github.com/fil-forge/hilt/pkg/store/delegation/postgres" + "github.com/fil-forge/hilt/pkg/store/exportsession" + pgexportsession "github.com/fil-forge/hilt/pkg/store/exportsession/postgres" "github.com/fil-forge/hilt/pkg/store/principal" pgprincipal "github.com/fil-forge/hilt/pkg/store/principal/postgres" "github.com/fil-forge/hilt/pkg/store/provider" @@ -39,6 +41,7 @@ var Module = fx.Module("postgres-store", NewAccessKeyStore, NewBucketStore, NewDelegationStore, + NewExportSessionStore, NewPolicyStore, NewPrincipalStore, NewProviderStore, @@ -126,6 +129,10 @@ func NewDelegationStore(mdb *MigratedPool) delegation.Store { return pgdelegation.New(mdb.Pool) } +func NewExportSessionStore(mdb *MigratedPool) exportsession.Store { + return pgexportsession.New(mdb.Pool) +} + func NewPolicyStore(mdb *MigratedPool) bucketpolicy.Store { return pgbucketpolicy.New(mdb.Pool) } diff --git a/pkg/migrations/sql/00009_export_session.sql b/pkg/migrations/sql/00009_export_session.sql new file mode 100644 index 00000000..fbc01002 --- /dev/null +++ b/pkg/migrations/sql/00009_export_session.sql @@ -0,0 +1,41 @@ +-- +goose Up +-- +goose StatementBegin +-- export_session is Hilt's record of a credible-exit ceremony (fil-one/RFC #24, +-- Tier 1): a customer exporting one bucket to a key they hold. Hilt owns the +-- session and the customer-facing steps (opened, pop_verified, acked, +-- released); the gateway serving the bucket reports the steps it performs +-- (pinned, manifest_built, car_ready, validated) and keeps its own pin keyed +-- by id. While a session is open (any state but released/aborted) neither the +-- tenant nor the bucket may be deleted. +-- +-- States only move forward, with aborted reachable from any open state. +-- root_cid and pinned_at are set together by the pinned step: the root the +-- customer acknowledges and the export is built from. +CREATE TABLE export_session ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL REFERENCES tenant(id) ON DELETE RESTRICT, -- DID (did:plc) + bucket_id TEXT NOT NULL, -- DID; no FK, so the record outlives the bucket it exported + customer_key TEXT NOT NULL CHECK (customer_key <> ''), -- receiving X25519 public key, multikey + state TEXT NOT NULL + CHECK (state IN ('opened', 'pop_verified', 'pinned', 'acked', 'manifest_built', + 'car_ready', 'validated', 'released', 'aborted')), + root_cid TEXT, -- pinned bucket root; NULL before pinned + pinned_at TIMESTAMPTZ, -- NULL before pinned + audit JSONB NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(audit) = 'array'), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL, -- an open session past this is an abort candidate + CHECK ((root_cid IS NULL) = (pinned_at IS NULL)) +); + +-- The deletion gates ask whether a tenant or a bucket has an open session. +CREATE INDEX export_session_open_tenant ON export_session (tenant_id) + WHERE state NOT IN ('released', 'aborted'); +CREATE INDEX export_session_open_bucket ON export_session (bucket_id) + WHERE state NOT IN ('released', 'aborted'); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS export_session; +-- +goose StatementEnd diff --git a/pkg/store/exportsession/exportsession.go b/pkg/store/exportsession/exportsession.go new file mode 100644 index 00000000..313c6203 --- /dev/null +++ b/pkg/store/exportsession/exportsession.go @@ -0,0 +1,141 @@ +// Package exportsession defines the credible-exit export session: Hilt's record +// of one customer exporting one bucket to a key they hold (fil-one/RFC #24, +// Tier 1). +// +// Hilt owns the session and the customer-facing steps; the gateway serving the +// bucket performs the export and reports its own steps back. The record carries +// no key material: the customer key is a public key, and the pinned root is a +// CID. +package exportsession + +import ( + "context" + "time" + + "github.com/fil-forge/ucantone/did" + "github.com/ipfs/go-cid" +) + +// State is how far an export session has got. States only move forward, and +// [Aborted] is reachable from any open state. +type State string + +const ( + // Opened is a new session: nothing about the customer key is verified yet. + Opened State = "opened" + // PoPVerified means the customer proved possession of the receiving key. + PoPVerified State = "pop_verified" + // Pinned means the gateway pinned a bucket root for the export. + Pinned State = "pinned" + // Acked means the customer acknowledged the (bucket, root, pinned-at) triple. + Acked State = "acked" + // ManifestBuilt means the gateway built and checked the key manifest. + ManifestBuilt State = "manifest_built" + // CarReady means the gateway materialized the exit CAR. + CarReady State = "car_ready" + // Validated means the customer validated the delivered artifact. + Validated State = "validated" + // Released is terminal: the export completed and the pin is lifted. + Released State = "released" + // Aborted is terminal: the session ended without an artifact. + Aborted State = "aborted" +) + +// next is the forward edge out of each open state. +var next = map[State]State{ + Opened: PoPVerified, + PoPVerified: Pinned, + Pinned: Acked, + Acked: ManifestBuilt, + ManifestBuilt: CarReady, + CarReady: Validated, + Validated: Released, +} + +// Open reports whether s is a non-terminal state of the machine. +func (s State) Open() bool { + _, ok := next[s] + return ok +} + +// ValidTransition reports whether from → to is an edge of the state machine: the +// single forward step out of an open state, or an abort from one. A state +// outside the machine has no edges. +func ValidTransition(from, to State) bool { + n, ok := next[from] + if !ok { + return false + } + return to == Aborted || to == n +} + +// Record is one export session. +type Record struct { + // ID identifies the session. The gateway keys its own pin by the same ID. + ID string + // Tenant is the tenant DID (did:plc) the exported bucket belongs to. + Tenant did.DID + // Bucket is the DID of the bucket (space) being exported. + Bucket did.DID + // CustomerKey is the customer's X25519 receiving key, as a multikey string. + CustomerKey string + // State is how far the session has got. + State State + // Root is the pinned bucket root; undefined before [Pinned]. + Root cid.Cid + // PinnedAt is when the root was pinned; zero before [Pinned]. + PinnedAt time.Time + // Audit is the session's append-only event log, a JSON array. + Audit []byte + // CreatedAt is when the session was opened. + CreatedAt time.Time + // UpdatedAt is when the session last changed. + UpdatedAt time.Time + // ExpiresAt is when an open session becomes a candidate for abort. + ExpiresAt time.Time +} + +// Input is the data needed to open an export session. +type Input struct { + // ID identifies the session. + ID string + // Tenant is the tenant DID (did:plc) the exported bucket belongs to. + Tenant did.DID + // Bucket is the DID of the bucket (space) being exported. + Bucket did.DID + // CustomerKey is the customer's X25519 receiving key, as a multikey string. + CustomerKey string + // ExpiresAt is when the session becomes a candidate for abort if still open. + ExpiresAt time.Time +} + +// Store persists export sessions. +type Store interface { + // Add opens a session in [Opened]. It returns [store.ErrRecordExists] if a + // session with the same ID exists, and [store.ErrInvalidArgument] for an + // empty ID or customer key. + Add(ctx context.Context, input Input) error + // Get returns the session with the given ID. It returns + // [store.ErrRecordNotFound] if there is none. + Get(ctx context.Context, id string) (Record, error) + // Transition moves a session from one state to the next iff it is currently + // in from. It returns [store.ErrInvalidArgument] if from → to is not an edge + // of the state machine, or if to is [Pinned] (use Pin), and + // [store.ErrRecordNotFound] if no session with the ID is in from. + Transition(ctx context.Context, id string, from, to State) error + // Pin moves a session from [PoPVerified] to [Pinned], recording the pinned + // bucket root and the time. It returns [store.ErrRecordNotFound] if no + // session with the ID is in [PoPVerified]. + Pin(ctx context.Context, id string, root cid.Cid) error + // AppendAudit appends one JSON value to the session's audit log. It returns + // [store.ErrInvalidArgument] if entry is not JSON and + // [store.ErrRecordNotFound] if there is no session with the ID. + AppendAudit(ctx context.Context, id string, entry []byte) error + // HasOpen reports whether the tenant has any open session. + HasOpen(ctx context.Context, tenant did.DID) (bool, error) + // HasOpenForBucket reports whether the bucket has any open session. + HasOpenForBucket(ctx context.Context, bucket did.DID) (bool, error) + // DeleteByTenant removes every session of a tenant. It is used by tenant + // deletion, which refuses while a session is open, and is idempotent. + DeleteByTenant(ctx context.Context, tenant did.DID) error +} diff --git a/pkg/store/exportsession/exportsession_test.go b/pkg/store/exportsession/exportsession_test.go new file mode 100644 index 00000000..ab170f47 --- /dev/null +++ b/pkg/store/exportsession/exportsession_test.go @@ -0,0 +1,304 @@ +package exportsession_test + +import ( + "encoding/json" + "fmt" + "runtime" + "testing" + "time" + + htestutil "github.com/fil-forge/hilt/internal/testutil" + "github.com/fil-forge/hilt/pkg/store" + "github.com/fil-forge/hilt/pkg/store/exportsession" + exportsessionmemory "github.com/fil-forge/hilt/pkg/store/exportsession/memory" + exportsessionpostgres "github.com/fil-forge/hilt/pkg/store/exportsession/postgres" + providerpostgres "github.com/fil-forge/hilt/pkg/store/provider/postgres" + "github.com/fil-forge/hilt/pkg/store/tenant" + tenantpostgres "github.com/fil-forge/hilt/pkg/store/tenant/postgres" + "github.com/fil-forge/libforge/testutil" + "github.com/fil-forge/ucantone/did" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" +) + +type StoreKind string + +const ( + Memory StoreKind = "memory" + Postgres StoreKind = "postgres" +) + +var storeKinds = []StoreKind{Memory, Postgres} + +// seedFunc ensures the parent tenant (and its provider) exist so the +// export_session.tenant_id foreign key is satisfied. It is a no-op for the +// memory store, which does not enforce referential integrity. +type seedFunc func(t *testing.T, tenantID did.DID) + +func makeStore(t *testing.T, k StoreKind) (exportsession.Store, seedFunc) { + switch k { + case Memory: + return exportsessionmemory.New(), func(*testing.T, did.DID) {} + case Postgres: + pool := createPostgresPool(t) + providers := providerpostgres.New(pool) + tenants := tenantpostgres.New(pool) + seed := func(t *testing.T, tenantID did.DID) { + providerID := testutil.RandomDID(t) + require.NoError(t, providers.Add(t.Context(), providerID, tenantID.String(), nil)) + require.NoError(t, tenants.Add(t.Context(), tenantID, "ext-"+tenantID.String(), providerID, tenant.Active)) + } + return exportsessionpostgres.New(pool), seed + } + panic("unknown store kind") +} + +func createPostgresPool(t *testing.T) *pgxpool.Pool { + if htestutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !htestutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !htestutil.IsDockerAvailable(t) { + t.SkipNow() + } + return htestutil.CreatePostgres(t) +} + +// customerKey is a placeholder X25519 multikey; the store only requires a +// non-empty string. +const customerKey = "z6LSbysY2xFMRpGMhb7tFTLMpeuPRaqaWM1yECx2AtzE3KCc" + +// openSession seeds a tenant and opens a session for a fresh bucket. +func openSession(t *testing.T, s exportsession.Store, seed seedFunc) exportsession.Input { + t.Helper() + input := exportsession.Input{ + ID: fmt.Sprintf("session-%s", testutil.RandomCID(t)), + Tenant: testutil.RandomDID(t), + Bucket: testutil.RandomDID(t), + CustomerKey: customerKey, + ExpiresAt: time.Now().Add(time.Hour).Truncate(time.Microsecond), + } + seed(t, input.Tenant) + require.NoError(t, s.Add(t.Context(), input)) + return input +} + +// path is the forward walk through the state machine. +var path = []exportsession.State{ + exportsession.Opened, exportsession.PoPVerified, exportsession.Pinned, exportsession.Acked, + exportsession.ManifestBuilt, exportsession.CarReady, exportsession.Validated, exportsession.Released, +} + +// advance walks a session forward from its current state, from, to target. +func advance(t *testing.T, s exportsession.Store, id string, from, target exportsession.State) { + t.Helper() + started := false + for i := 1; i < len(path) && path[i-1] != target; i++ { + if path[i-1] == from { + started = true + } + if !started { + continue + } + if path[i] == exportsession.Pinned { + require.NoError(t, s.Pin(t.Context(), id, testutil.RandomCID(t))) + continue + } + require.NoError(t, s.Transition(t.Context(), id, path[i-1], path[i])) + } +} + +func TestValidTransition(t *testing.T) { + forward := map[exportsession.State]exportsession.State{ + exportsession.Opened: exportsession.PoPVerified, + exportsession.PoPVerified: exportsession.Pinned, + exportsession.Pinned: exportsession.Acked, + exportsession.Acked: exportsession.ManifestBuilt, + exportsession.ManifestBuilt: exportsession.CarReady, + exportsession.CarReady: exportsession.Validated, + exportsession.Validated: exportsession.Released, + } + all := []exportsession.State{ + exportsession.Opened, exportsession.PoPVerified, exportsession.Pinned, exportsession.Acked, + exportsession.ManifestBuilt, exportsession.CarReady, exportsession.Validated, + exportsession.Released, exportsession.Aborted, "", "bogus", + } + for _, from := range all { + _, open := forward[from] + require.Equal(t, open, from.Open(), "%q.Open()", from) + for _, to := range all { + want := open && (to == exportsession.Aborted || forward[from] == to) + require.Equal(t, want, exportsession.ValidTransition(from, to), "ValidTransition(%q, %q)", from, to) + } + } +} + +func TestExportSessionStore(t *testing.T) { + for _, k := range storeKinds { + t.Run(string(k), func(t *testing.T) { + s, seed := makeStore(t, k) + + t.Run("opens a session in opened", func(t *testing.T) { + input := openSession(t, s, seed) + + rec, err := s.Get(t.Context(), input.ID) + require.NoError(t, err) + require.Equal(t, input.ID, rec.ID) + require.Equal(t, input.Tenant, rec.Tenant) + require.Equal(t, input.Bucket, rec.Bucket) + require.Equal(t, customerKey, rec.CustomerKey) + require.Equal(t, exportsession.Opened, rec.State) + require.False(t, rec.Root.Defined()) + require.True(t, rec.PinnedAt.IsZero()) + require.JSONEq(t, `[]`, string(rec.Audit)) + require.False(t, rec.CreatedAt.IsZero()) + require.True(t, input.ExpiresAt.Equal(rec.ExpiresAt)) + }) + + t.Run("rejects a duplicate id", func(t *testing.T) { + input := openSession(t, s, seed) + require.ErrorIs(t, s.Add(t.Context(), input), store.ErrRecordExists) + }) + + t.Run("rejects an empty id or customer key", func(t *testing.T) { + tenantID := testutil.RandomDID(t) + seed(t, tenantID) + noKey := exportsession.Input{ID: "no-key", Tenant: tenantID, Bucket: testutil.RandomDID(t), ExpiresAt: time.Now()} + require.ErrorIs(t, s.Add(t.Context(), noKey), store.ErrInvalidArgument) + noID := exportsession.Input{Tenant: tenantID, Bucket: testutil.RandomDID(t), CustomerKey: customerKey, ExpiresAt: time.Now()} + require.ErrorIs(t, s.Add(t.Context(), noID), store.ErrInvalidArgument) + }) + + t.Run("returns ErrRecordNotFound for an unknown id", func(t *testing.T) { + _, err := s.Get(t.Context(), "missing") + require.ErrorIs(t, err, store.ErrRecordNotFound) + }) + + t.Run("walks forward to released and records the pin", func(t *testing.T) { + input := openSession(t, s, seed) + require.NoError(t, s.Transition(t.Context(), input.ID, exportsession.Opened, exportsession.PoPVerified)) + + root := testutil.RandomCID(t) + require.NoError(t, s.Pin(t.Context(), input.ID, root)) + rec, err := s.Get(t.Context(), input.ID) + require.NoError(t, err) + require.Equal(t, exportsession.Pinned, rec.State) + require.Equal(t, root, rec.Root) + require.False(t, rec.PinnedAt.IsZero()) + + advance(t, s, input.ID, exportsession.Pinned, exportsession.Released) + rec, err = s.Get(t.Context(), input.ID) + require.NoError(t, err) + require.Equal(t, exportsession.Released, rec.State) + require.Equal(t, root, rec.Root, "the pinned root outlives the session") + }) + + t.Run("transition is a compare-and-set", func(t *testing.T) { + input := openSession(t, s, seed) + require.NoError(t, s.Transition(t.Context(), input.ID, exportsession.Opened, exportsession.PoPVerified)) + require.ErrorIs(t, s.Transition(t.Context(), input.ID, exportsession.Opened, exportsession.PoPVerified), store.ErrRecordNotFound) + require.ErrorIs(t, s.Transition(t.Context(), "missing", exportsession.Opened, exportsession.PoPVerified), store.ErrRecordNotFound) + }) + + t.Run("rejects transitions outside the machine", func(t *testing.T) { + input := openSession(t, s, seed) + for _, tc := range []struct{ from, to exportsession.State }{ + {exportsession.Opened, exportsession.Acked}, // skips a step + {exportsession.PoPVerified, exportsession.Pinned}, // pinning goes through Pin + {exportsession.Released, exportsession.Aborted}, // from a terminal state + {"bogus", exportsession.Aborted}, // from an unknown state + {exportsession.Opened, exportsession.State("bogus")}, // to an unknown state + } { + require.ErrorIs(t, s.Transition(t.Context(), input.ID, tc.from, tc.to), store.ErrInvalidArgument, "%s -> %s", tc.from, tc.to) + } + rec, err := s.Get(t.Context(), input.ID) + require.NoError(t, err) + require.Equal(t, exportsession.Opened, rec.State) + }) + + t.Run("pins only from pop_verified", func(t *testing.T) { + input := openSession(t, s, seed) + require.ErrorIs(t, s.Pin(t.Context(), input.ID, testutil.RandomCID(t)), store.ErrRecordNotFound) + require.NoError(t, s.Transition(t.Context(), input.ID, exportsession.Opened, exportsession.PoPVerified)) + require.NoError(t, s.Pin(t.Context(), input.ID, testutil.RandomCID(t))) + require.ErrorIs(t, s.Pin(t.Context(), input.ID, testutil.RandomCID(t)), store.ErrRecordNotFound) + }) + + t.Run("aborts from every open state", func(t *testing.T) { + for _, state := range []exportsession.State{ + exportsession.Opened, exportsession.PoPVerified, exportsession.Pinned, exportsession.Acked, + exportsession.ManifestBuilt, exportsession.CarReady, exportsession.Validated, + } { + input := openSession(t, s, seed) + advance(t, s, input.ID, exportsession.Opened, state) + require.NoError(t, s.Transition(t.Context(), input.ID, state, exportsession.Aborted), "abort from %s", state) + open, err := s.HasOpen(t.Context(), input.Tenant) + require.NoError(t, err) + require.False(t, open, "tenant still open after abort from %s", state) + } + }) + + t.Run("appends to the audit log", func(t *testing.T) { + input := openSession(t, s, seed) + require.NoError(t, s.AppendAudit(t.Context(), input.ID, []byte(`{"event":"pop","ok":true}`))) + require.NoError(t, s.AppendAudit(t.Context(), input.ID, []byte(`"ack"`))) + require.ErrorIs(t, s.AppendAudit(t.Context(), input.ID, []byte(`not json`)), store.ErrInvalidArgument) + require.ErrorIs(t, s.AppendAudit(t.Context(), "missing", []byte(`{}`)), store.ErrRecordNotFound) + + rec, err := s.Get(t.Context(), input.ID) + require.NoError(t, err) + var audit []any + require.NoError(t, json.Unmarshal(rec.Audit, &audit)) + require.Equal(t, []any{map[string]any{"event": "pop", "ok": true}, "ack"}, audit) + }) + + t.Run("reports open sessions by tenant and bucket", func(t *testing.T) { + input := openSession(t, s, seed) + for _, check := range []func() (bool, error){ + func() (bool, error) { return s.HasOpen(t.Context(), input.Tenant) }, + func() (bool, error) { return s.HasOpenForBucket(t.Context(), input.Bucket) }, + } { + open, err := check() + require.NoError(t, err) + require.True(t, open) + } + + other, err := s.HasOpen(t.Context(), testutil.RandomDID(t)) + require.NoError(t, err) + require.False(t, other) + other, err = s.HasOpenForBucket(t.Context(), testutil.RandomDID(t)) + require.NoError(t, err) + require.False(t, other) + + advance(t, s, input.ID, exportsession.Opened, exportsession.Released) + open, err := s.HasOpen(t.Context(), input.Tenant) + require.NoError(t, err) + require.False(t, open) + open, err = s.HasOpenForBucket(t.Context(), input.Bucket) + require.NoError(t, err) + require.False(t, open) + }) + + t.Run("deletes every session of a tenant", func(t *testing.T) { + input := openSession(t, s, seed) + advance(t, s, input.ID, exportsession.Opened, exportsession.Released) + second := exportsession.Input{ + ID: input.ID + "-2", Tenant: input.Tenant, Bucket: testutil.RandomDID(t), + CustomerKey: customerKey, ExpiresAt: time.Now().Add(time.Hour), + } + require.NoError(t, s.Add(t.Context(), second)) + bystander := openSession(t, s, seed) + + require.NoError(t, s.DeleteByTenant(t.Context(), input.Tenant)) + for _, id := range []string{input.ID, second.ID} { + _, err := s.Get(t.Context(), id) + require.ErrorIs(t, err, store.ErrRecordNotFound) + } + _, err := s.Get(t.Context(), bystander.ID) + require.NoError(t, err) + require.NoError(t, s.DeleteByTenant(t.Context(), input.Tenant), "idempotent") + }) + }) + } +} diff --git a/pkg/store/exportsession/memory/store.go b/pkg/store/exportsession/memory/store.go new file mode 100644 index 00000000..a1de431b --- /dev/null +++ b/pkg/store/exportsession/memory/store.go @@ -0,0 +1,178 @@ +// Package memory provides an in-memory implementation of exportsession.Store. +package memory + +import ( + "bytes" + "context" + "encoding/json" + "sync" + "time" + + "github.com/fil-forge/hilt/pkg/store" + "github.com/fil-forge/hilt/pkg/store/exportsession" + "github.com/fil-forge/ucantone/did" + "github.com/ipfs/go-cid" +) + +// session keeps the audit log decoded so AppendAudit is a slice append. +type session struct { + rec exportsession.Record + audit []json.RawMessage +} + +type Store struct { + mutex sync.RWMutex + sessions map[string]session +} + +var _ exportsession.Store = (*Store)(nil) + +func New() *Store { + return &Store{sessions: map[string]session{}} +} + +func (s *Store) Add(ctx context.Context, input exportsession.Input) error { + // Mirrors the table's NOT NULL / CHECK constraints. + if input.ID == "" || input.CustomerKey == "" { + return store.ErrInvalidArgument + } + + s.mutex.Lock() + defer s.mutex.Unlock() + + if _, ok := s.sessions[input.ID]; ok { + return store.ErrRecordExists + } + now := time.Now().UTC() + s.sessions[input.ID] = session{rec: exportsession.Record{ + ID: input.ID, + Tenant: input.Tenant, + Bucket: input.Bucket, + CustomerKey: input.CustomerKey, + State: exportsession.Opened, + CreatedAt: now, + UpdatedAt: now, + ExpiresAt: input.ExpiresAt.UTC(), + }} + return nil +} + +func (s *Store) Get(ctx context.Context, id string) (exportsession.Record, error) { + s.mutex.RLock() + defer s.mutex.RUnlock() + + sess, ok := s.sessions[id] + if !ok { + return exportsession.Record{}, store.ErrRecordNotFound + } + return sess.record(), nil +} + +func (s *Store) Transition(ctx context.Context, id string, from, to exportsession.State) error { + if !exportsession.ValidTransition(from, to) || to == exportsession.Pinned { + return store.ErrInvalidArgument + } + + s.mutex.Lock() + defer s.mutex.Unlock() + + sess, ok := s.sessions[id] + if !ok || sess.rec.State != from { + return store.ErrRecordNotFound + } + sess.rec.State = to + sess.rec.UpdatedAt = time.Now().UTC() + s.sessions[id] = sess + return nil +} + +func (s *Store) Pin(ctx context.Context, id string, root cid.Cid) error { + if !root.Defined() { + return store.ErrInvalidArgument + } + + s.mutex.Lock() + defer s.mutex.Unlock() + + sess, ok := s.sessions[id] + if !ok || sess.rec.State != exportsession.PoPVerified { + return store.ErrRecordNotFound + } + now := time.Now().UTC() + sess.rec.State = exportsession.Pinned + sess.rec.Root = root + sess.rec.PinnedAt = now + sess.rec.UpdatedAt = now + s.sessions[id] = sess + return nil +} + +func (s *Store) AppendAudit(ctx context.Context, id string, entry []byte) error { + // Postgres rejects a non-JSON entry at the jsonb cast. + if !json.Valid(entry) { + return store.ErrInvalidArgument + } + + s.mutex.Lock() + defer s.mutex.Unlock() + + sess, ok := s.sessions[id] + if !ok { + return store.ErrRecordNotFound + } + sess.audit = append(sess.audit, json.RawMessage(bytes.Clone(entry))) + sess.rec.UpdatedAt = time.Now().UTC() + s.sessions[id] = sess + return nil +} + +func (s *Store) HasOpen(ctx context.Context, tenant did.DID) (bool, error) { + s.mutex.RLock() + defer s.mutex.RUnlock() + + for _, sess := range s.sessions { + if sess.rec.Tenant == tenant && sess.rec.State.Open() { + return true, nil + } + } + return false, nil +} + +func (s *Store) HasOpenForBucket(ctx context.Context, bucket did.DID) (bool, error) { + s.mutex.RLock() + defer s.mutex.RUnlock() + + for _, sess := range s.sessions { + if sess.rec.Bucket == bucket && sess.rec.State.Open() { + return true, nil + } + } + return false, nil +} + +func (s *Store) DeleteByTenant(ctx context.Context, tenant did.DID) error { + s.mutex.Lock() + defer s.mutex.Unlock() + + for id, sess := range s.sessions { + if sess.rec.Tenant == tenant { + delete(s.sessions, id) + } + } + return nil +} + +// record returns a copy of the session with the audit log encoded. +func (sess session) record() exportsession.Record { + rec := sess.rec + audit := sess.audit + if audit == nil { + audit = []json.RawMessage{} + } + b, err := json.Marshal(audit) + if err != nil { + panic(err) // every entry was json.Valid when appended + } + rec.Audit = b + return rec +} diff --git a/pkg/store/exportsession/postgres/store.go b/pkg/store/exportsession/postgres/store.go new file mode 100644 index 00000000..abcf8cf5 --- /dev/null +++ b/pkg/store/exportsession/postgres/store.go @@ -0,0 +1,194 @@ +// Package postgres provides a PostgreSQL-backed implementation of +// exportsession.Store. +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/fil-forge/hilt/pkg/store" + "github.com/fil-forge/hilt/pkg/store/exportsession" + "github.com/fil-forge/ucantone/did" + "github.com/ipfs/go-cid" + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +// openStates is the non-terminal predicate; it must match the partial indexes +// in the export_session migration. +const openStates = `state NOT IN ('released', 'aborted')` + +const columns = `id, tenant_id, bucket_id, customer_key, state, root_cid, pinned_at, audit, created_at, updated_at, expires_at` + +type Store struct { + pool *pgxpool.Pool +} + +var _ exportsession.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Add(ctx context.Context, input exportsession.Input) error { + if input.ID == "" || input.CustomerKey == "" { + return store.ErrInvalidArgument + } + _, err := s.pool.Exec(ctx, ` + INSERT INTO export_session (id, tenant_id, bucket_id, customer_key, state, expires_at) + VALUES ($1, $2, $3, $4, $5, $6) + `, input.ID, input.Tenant.String(), input.Bucket.String(), input.CustomerKey, exportsession.Opened, input.ExpiresAt) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == pgerrcode.UniqueViolation { + return store.ErrRecordExists + } + return fmt.Errorf("adding export session: %w", err) + } + return nil +} + +func (s *Store) Get(ctx context.Context, id string) (exportsession.Record, error) { + row := s.pool.QueryRow(ctx, `SELECT `+columns+` FROM export_session WHERE id = $1`, id) + rec, err := scanRecord(row) + if errors.Is(err, pgx.ErrNoRows) { + return exportsession.Record{}, store.ErrRecordNotFound + } + if err != nil { + return exportsession.Record{}, fmt.Errorf("getting export session: %w", err) + } + return rec, nil +} + +func (s *Store) Transition(ctx context.Context, id string, from, to exportsession.State) error { + if !exportsession.ValidTransition(from, to) || to == exportsession.Pinned { + return store.ErrInvalidArgument + } + tag, err := s.pool.Exec(ctx, ` + UPDATE export_session + SET state = $3, updated_at = NOW() + WHERE id = $1 AND state = $2 + `, id, from, to) + if err != nil { + return fmt.Errorf("transitioning export session: %w", err) + } + if tag.RowsAffected() == 0 { + return store.ErrRecordNotFound + } + return nil +} + +func (s *Store) Pin(ctx context.Context, id string, root cid.Cid) error { + if !root.Defined() { + return store.ErrInvalidArgument + } + tag, err := s.pool.Exec(ctx, ` + UPDATE export_session + SET state = $2, root_cid = $3, pinned_at = NOW(), updated_at = NOW() + WHERE id = $1 AND state = $4 + `, id, exportsession.Pinned, root.String(), exportsession.PoPVerified) + if err != nil { + return fmt.Errorf("pinning export session: %w", err) + } + if tag.RowsAffected() == 0 { + return store.ErrRecordNotFound + } + return nil +} + +func (s *Store) AppendAudit(ctx context.Context, id string, entry []byte) error { + if !json.Valid(entry) { + return store.ErrInvalidArgument + } + tag, err := s.pool.Exec(ctx, ` + UPDATE export_session + SET audit = audit || jsonb_build_array($2::jsonb), updated_at = NOW() + WHERE id = $1 + `, id, string(entry)) + if err != nil { + return fmt.Errorf("appending export session audit: %w", err) + } + if tag.RowsAffected() == 0 { + return store.ErrRecordNotFound + } + return nil +} + +func (s *Store) HasOpen(ctx context.Context, tenant did.DID) (bool, error) { + var open bool + err := s.pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM export_session WHERE tenant_id = $1 AND `+openStates+`)`, + tenant.String()).Scan(&open) + if err != nil { + return false, fmt.Errorf("checking open export sessions: %w", err) + } + return open, nil +} + +func (s *Store) HasOpenForBucket(ctx context.Context, bucket did.DID) (bool, error) { + var open bool + err := s.pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM export_session WHERE bucket_id = $1 AND `+openStates+`)`, + bucket.String()).Scan(&open) + if err != nil { + return false, fmt.Errorf("checking open export sessions for bucket: %w", err) + } + return open, nil +} + +func (s *Store) DeleteByTenant(ctx context.Context, tenant did.DID) error { + _, err := s.pool.Exec(ctx, `DELETE FROM export_session WHERE tenant_id = $1`, tenant.String()) + if err != nil { + return fmt.Errorf("deleting export sessions: %w", err) + } + return nil +} + +func scanRecord(row pgx.Row) (exportsession.Record, error) { + var ( + id, tenantStr, bucketStr string + customerKey, state string + rootStr *string + pinnedAt *time.Time + audit []byte + createdAt, updatedAt time.Time + expiresAt time.Time + ) + if err := row.Scan(&id, &tenantStr, &bucketStr, &customerKey, &state, &rootStr, &pinnedAt, + &audit, &createdAt, &updatedAt, &expiresAt); err != nil { + return exportsession.Record{}, err + } + tenant, err := did.Parse(tenantStr) + if err != nil { + return exportsession.Record{}, fmt.Errorf("parsing tenant DID: %w", err) + } + bucket, err := did.Parse(bucketStr) + if err != nil { + return exportsession.Record{}, fmt.Errorf("parsing bucket DID: %w", err) + } + rec := exportsession.Record{ + ID: id, + Tenant: tenant, + Bucket: bucket, + CustomerKey: customerKey, + State: exportsession.State(state), + Audit: audit, + CreatedAt: createdAt, + UpdatedAt: updatedAt, + ExpiresAt: expiresAt, + } + if rootStr != nil { + if rec.Root, err = cid.Parse(*rootStr); err != nil { + return exportsession.Record{}, fmt.Errorf("parsing root CID: %w", err) + } + } + if pinnedAt != nil { + rec.PinnedAt = *pinnedAt + } + return rec, nil +} From 5c17c5aef411b0af472bbc459847af9b7991ccf1 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Thu, 17 Sep 2026 12:59:26 +0200 Subject: [PATCH 2/2] feat: refuse tenant and bucket deletion during an export An open export session still needs the tenant's keys and the bucket's space, and deletion destroys both. DELETE /tenants/{id} now answers 409 while the tenant has an open session, before anything is torn down, and deletes the tenant's closed sessions in its cascade. /s3/bucket/delete fails with ExportInProgress while the bucket has an open session: a session opens here before the gateway pins anything, so the gateway's own pin cannot be relied on to refuse the delete. --- pkg/api/service/tenant/errors.go | 4 +++ pkg/api/service/tenant/service.go | 27 +++++++++++++++++--- pkg/api/service/tenant/service_test.go | 31 ++++++++++++++++++++--- pkg/api/tenants.go | 4 +-- pkg/api/tenants_test.go | 27 +++++++++++++++++--- pkg/rpc/failure.go | 1 + pkg/rpc/failure_test.go | 8 ++++++ pkg/rpc/rpc_test.go | 3 ++- pkg/rpc/service/bucket/errors.go | 5 ++++ pkg/rpc/service/bucket/policy_test.go | 3 ++- pkg/rpc/service/bucket/service.go | 30 ++++++++++++++++------ pkg/rpc/service/bucket/service_test.go | 35 ++++++++++++++++++++++---- 12 files changed, 150 insertions(+), 28 deletions(-) diff --git a/pkg/api/service/tenant/errors.go b/pkg/api/service/tenant/errors.go index 2884a4fe..28117222 100644 --- a/pkg/api/service/tenant/errors.go +++ b/pkg/api/service/tenant/errors.go @@ -14,6 +14,7 @@ const ( DIDRegistrationErrorName = "DIDRegistration" UploadRegistrationErrorName = "UploadRegistration" DIDDeactivationErrorName = "DIDDeactivation" + ExportInProgressErrorName = "ExportInProgress" ) // Known errors returned by the tenant [Service]. Handlers map these to HTTP @@ -41,4 +42,7 @@ var ( ErrUploadRegistration = errors.New(UploadRegistrationErrorName, "failed to register tenant with upload service") // ErrDIDDeactivation is returned when deactivating the tenant's did:plc fails. ErrDIDDeactivation = errors.New(DIDDeactivationErrorName, "failed to deactivate tenant DID") + // ErrExportInProgress is returned when deleting a tenant that has an open + // export session. + ErrExportInProgress = errors.New(ExportInProgressErrorName, "tenant has an export in progress") ) diff --git a/pkg/api/service/tenant/service.go b/pkg/api/service/tenant/service.go index 9854a56f..faf580da 100644 --- a/pkg/api/service/tenant/service.go +++ b/pkg/api/service/tenant/service.go @@ -18,6 +18,7 @@ import ( "github.com/fil-forge/hilt/pkg/store/bucket" bucketpolicystore "github.com/fil-forge/hilt/pkg/store/bucketpolicy" "github.com/fil-forge/hilt/pkg/store/delegation" + "github.com/fil-forge/hilt/pkg/store/exportsession" principalstore "github.com/fil-forge/hilt/pkg/store/principal" "github.com/fil-forge/hilt/pkg/store/provider" tenantstore "github.com/fil-forge/hilt/pkg/store/tenant" @@ -48,6 +49,7 @@ type Service struct { delegations delegation.Store secrets vault.Vault wrapKeys wrapkeystore.Store + exports exportsession.Store plcClient *plc.DirectoryClient upload *upload.Client revocations grant.RevocationPublisher @@ -65,6 +67,7 @@ func New( delegations delegation.Store, secrets vault.Vault, wrapKeys wrapkeystore.Store, + exports exportsession.Store, plcClient *plc.DirectoryClient, upload *upload.Client, revocations grant.RevocationPublisher, @@ -81,6 +84,7 @@ func New( secrets: secrets, plcClient: plcClient, wrapKeys: wrapKeys, + exports: exports, upload: upload, revocations: revocations, } @@ -275,9 +279,10 @@ func (s *Service) SetStatus(ctx context.Context, externalID, status string) erro return nil } -// Delete permanently deletes a tenant (which must be disabled), cascading to its -// buckets and their policies, access keys, principals, and delegations, and -// deactivating its did:plc. It is idempotent: a missing tenant is a no-op. +// Delete permanently deletes a tenant (which must be disabled and have no open +// export session), cascading to its buckets and their policies, access keys, +// principals, delegations, and export sessions, and deactivating its did:plc. +// It is idempotent: a missing tenant is a no-op. // // Out of scope: deprovisioning the tenant's spaces from the Forge upload service // (Sprue), for which there is no facility per the RFC. @@ -294,6 +299,16 @@ func (s *Service) Delete(ctx context.Context, externalID string) error { return ErrTenantNotDisabled } + // An open export still needs the tenant's keys and buckets, and deletion + // destroys both, so it waits for the export to be released or aborted. + exporting, err := s.exports.HasOpen(ctx, rec.ID) + if err != nil { + return fmt.Errorf("checking export sessions: %w", err) + } + if exporting { + return ErrExportInProgress + } + signingVaultKey := vault.TenantKeyPath(rec.ID) keys, err := s.accessKeys.ListByTenant(ctx, rec.ID) @@ -399,6 +414,12 @@ func (s *Service) Delete(ctx context.Context, externalID string) error { } } + // Cascade: export sessions, all closed by now. They reference the tenant row + // (FK RESTRICT), so they go before it. + if err := s.exports.DeleteByTenant(ctx, rec.ID); err != nil { + return fmt.Errorf("deleting export sessions: %w", err) + } + // Delegations addressed to the tenant (the bucket -> tenant grants). if err := s.delegations.DeleteByAudience(ctx, rec.ID); err != nil { return fmt.Errorf("deleting tenant delegations: %w", err) diff --git a/pkg/api/service/tenant/service_test.go b/pkg/api/service/tenant/service_test.go index 18401e38..9ccc203d 100644 --- a/pkg/api/service/tenant/service_test.go +++ b/pkg/api/service/tenant/service_test.go @@ -23,6 +23,8 @@ import ( bucketpolicystore "github.com/fil-forge/hilt/pkg/store/bucketpolicy" bucketpolicymemory "github.com/fil-forge/hilt/pkg/store/bucketpolicy/memory" delegationmemory "github.com/fil-forge/hilt/pkg/store/delegation/memory" + "github.com/fil-forge/hilt/pkg/store/exportsession" + exportsessionmemory "github.com/fil-forge/hilt/pkg/store/exportsession/memory" principalmemory "github.com/fil-forge/hilt/pkg/store/principal/memory" providermemory "github.com/fil-forge/hilt/pkg/store/provider/memory" "github.com/fil-forge/hilt/pkg/store/tenant" @@ -95,7 +97,7 @@ func provisionSetup(t *testing.T, plcStatus int) provisionEnv { require.NoError(t, err) svc := tenantsvc.New(zap.NewNop(), tenantmemory.New(), providers, bucketmemory.New(), - accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeysmemory.New(), plcClient, upload, &testutil.FakeSwarf{}) + accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeysmemory.New(), exportsessionmemory.New(), plcClient, upload, &testutil.FakeSwarf{}) return provisionEnv{svc: svc, providers: providers, sprueFailed: sprueFailed} } @@ -156,7 +158,7 @@ func TestProvision(t *testing.T) { // clients — enough for Get and SetStatus, which never touch them. func simpleService(tenants tenant.Store) *tenantsvc.Service { return tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), bucketmemory.New(), - accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeysmemory.New(), nil, nil, &testutil.FakeSwarf{}) + accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeysmemory.New(), exportsessionmemory.New(), nil, nil, &testutil.FakeSwarf{}) } func TestGetAndSetStatus(t *testing.T) { @@ -232,6 +234,7 @@ type deleteEnv struct { buckets *bucketmemory.Store policies *bucketpolicymemory.Store principals *principalmemory.Store + exports *exportsessionmemory.Store swarf *testutil.FakeSwarf // publisher is swarf as the service sees it, recording each publish's deadline. publisher *deadlinePublisher @@ -309,10 +312,11 @@ func deleteSetup(t *testing.T, status tenant.Status) deleteEnv { swarf := &testutil.FakeSwarf{} publisher := &deadlinePublisher{FakeSwarf: swarf} + exports := exportsessionmemory.New() svc := tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), buckets, - accessKeys, principals, policies, delegations, secrets, wrapkeysmemory.New(), plcClient, nil, publisher) + accessKeys, principals, policies, delegations, secrets, wrapkeysmemory.New(), exports, plcClient, nil, publisher) return deleteEnv{svc: svc, tenants: tenants, buckets: buckets, policies: policies, - principals: principals, swarf: swarf, publisher: publisher, bucketID: bucketID, tenantID: tenantID, grants: grants, directory: directory, + principals: principals, exports: exports, swarf: swarf, publisher: publisher, bucketID: bucketID, tenantID: tenantID, grants: grants, directory: directory, tombstone: tombstoneJSON.Bytes()} } @@ -401,6 +405,25 @@ func TestDelete(t *testing.T) { require.ErrorIs(t, env.svc.Delete(ctx, "tenant-1"), tenantsvc.ErrTenantNotDisabled) }) + t.Run("refuses while an export is open, then deletes once it closes", func(t *testing.T) { + env := deleteSetup(t, tenant.Disabled) + require.NoError(t, env.exports.Add(ctx, exportsession.Input{ + ID: "session-1", Tenant: env.tenantID, Bucket: env.bucketID, + CustomerKey: "z6LSbysY2xFMRpGMhb7tFTLMpeuPRaqaWM1yECx2AtzE3KCc", ExpiresAt: time.Now().Add(time.Hour), + })) + + require.ErrorIs(t, env.svc.Delete(ctx, "tenant-1"), tenantsvc.ErrExportInProgress) + _, err := env.tenants.GetByExternalID(ctx, "tenant-1") + require.NoError(t, err) + require.Equal(t, 0, env.directory.deactivations, "nothing is torn down while the export is open") + require.Empty(t, env.swarf.Revoked(), "no delegation is revoked while the export is open") + + require.NoError(t, env.exports.Transition(ctx, "session-1", exportsession.Opened, exportsession.Aborted)) + require.NoError(t, env.svc.Delete(ctx, "tenant-1")) + _, err = env.exports.Get(ctx, "session-1") + require.ErrorIs(t, err, store.ErrRecordNotFound, "closed sessions are deleted with the tenant") + }) + t.Run("maps a directory failure to ErrDIDDeactivation", func(t *testing.T) { env := deleteSetup(t, tenant.Disabled) env.directory.logLastStatus = http.StatusInternalServerError diff --git a/pkg/api/tenants.go b/pkg/api/tenants.go index 712b006e..cffdcd7d 100644 --- a/pkg/api/tenants.go +++ b/pkg/api/tenants.go @@ -22,7 +22,7 @@ func tenantHTTPError(log *zap.Logger, err error) error { return httpError(http.StatusBadRequest, err) case errors.Is(err, tenantsvc.ErrInvalidStatus): return httpError(http.StatusUnprocessableEntity, err) - case errors.Is(err, tenantsvc.ErrTenantNotDisabled): + case errors.Is(err, tenantsvc.ErrTenantNotDisabled), errors.Is(err, tenantsvc.ErrExportInProgress): return httpError(http.StatusConflict, err) case errors.Is(err, tenantsvc.ErrDIDRegistration), errors.Is(err, tenantsvc.ErrUploadRegistration), @@ -93,7 +93,7 @@ func NewUpdateTenantStatusHandler(logger *zap.Logger, tenants *tenantsvc.Service } // NewDeleteTenantHandler handles DELETE /tenants/{tenantId} — permanently delete a -// tenant (must be disabled first). Idempotent. +// tenant (must be disabled first, with no export in progress). Idempotent. func NewDeleteTenantHandler(logger *zap.Logger, tenants *tenantsvc.Service) Route { log := logger.With(zap.String("handler", "DeleteTenant")) return NewRoute(http.MethodDelete, "/tenants/:tenantId", func(c echo.Context) error { diff --git a/pkg/api/tenants_test.go b/pkg/api/tenants_test.go index 4282904c..b25c30e4 100644 --- a/pkg/api/tenants_test.go +++ b/pkg/api/tenants_test.go @@ -11,6 +11,7 @@ import ( "net/url" "sync" "testing" + "time" "github.com/fil-forge/hilt/internal/testutil" "github.com/fil-forge/hilt/pkg/api" @@ -22,6 +23,8 @@ import ( bucketmemory "github.com/fil-forge/hilt/pkg/store/bucket/memory" bucketpolicymemory "github.com/fil-forge/hilt/pkg/store/bucketpolicy/memory" delegationmemory "github.com/fil-forge/hilt/pkg/store/delegation/memory" + "github.com/fil-forge/hilt/pkg/store/exportsession" + exportsessionmemory "github.com/fil-forge/hilt/pkg/store/exportsession/memory" principalmemory "github.com/fil-forge/hilt/pkg/store/principal/memory" "github.com/fil-forge/hilt/pkg/store/provider" providermemory "github.com/fil-forge/hilt/pkg/store/provider/memory" @@ -230,7 +233,7 @@ func setupProvision(t *testing.T, cfg *setupConfig) (*echo.Echo, *provisionDeps) upload.WithHTTPClient(&http.Client{Transport: srv})) require.NoError(t, err) - svc := tenantsvc.New(zap.NewNop(), deps.tenants, deps.providers, bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), deps.secrets, deps.wrapKeys, plcClient, upload, &testutil.FakeSwarf{}) + svc := tenantsvc.New(zap.NewNop(), deps.tenants, deps.providers, bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), deps.secrets, deps.wrapKeys, exportsessionmemory.New(), plcClient, upload, &testutil.FakeSwarf{}) route := api.NewProvisionTenantHandler(zap.NewNop(), svc) e := echo.New() e.Add(route.Method, route.Path, route.Handler) @@ -513,7 +516,7 @@ func TestGetTenantHandler(t *testing.T) { ctx := t.Context() tenants := tenantmemory.New() require.NoError(t, tenants.Add(ctx, testutil.RandomDID(t), "tenant-1", testutil.RandomDID(t), tenant.Active)) - svc := tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeymemory.New(), nil, nil, &testutil.FakeSwarf{}) + svc := tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeymemory.New(), exportsessionmemory.New(), nil, nil, &testutil.FakeSwarf{}) e := serve(api.NewGetTenantHandler(zap.NewNop(), svc)) t.Run("found", func(t *testing.T) { @@ -541,7 +544,7 @@ func TestUpdateTenantStatusHandler(t *testing.T) { tenants := tenantmemory.New() id := testutil.RandomDID(t) require.NoError(t, tenants.Add(ctx, id, "tenant-1", testutil.RandomDID(t), tenant.Active)) - svc := tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeymemory.New(), nil, nil, &testutil.FakeSwarf{}) + svc := tenantsvc.New(zap.NewNop(), tenants, providermemory.New(), bucketmemory.New(), accesskeymemory.New(), principalmemory.New(), bucketpolicymemory.New(), delegationmemory.New(), vaultmemory.New(), wrapkeymemory.New(), exportsessionmemory.New(), nil, nil, &testutil.FakeSwarf{}) e := serve(api.NewUpdateTenantStatusHandler(zap.NewNop(), svc)) statusBody := func(s api.TenantStatus) []byte { @@ -613,6 +616,7 @@ type deleteDeps struct { principals *principalmemory.Store delegations *delegationmemory.Store wrapKeys *wrapkeymemory.Store + exports *exportsessionmemory.Store secrets vault.Vault directory *plcDirectory signer secp256k1.Signer @@ -668,6 +672,7 @@ func setupDelete(t *testing.T, status tenant.Status) (*echo.Echo, *deleteDeps) { principals: principalmemory.New(), delegations: delegationmemory.New(), wrapKeys: wrapkeymemory.New(), + exports: exportsessionmemory.New(), secrets: vaultmemory.New(), directory: directory, signer: signer, @@ -689,7 +694,7 @@ func setupDelete(t *testing.T, status tenant.Status) (*echo.Echo, *deleteDeps) { VaultKey: wrapkeystore.VaultKey(tenantID, 1), })) - svc := tenantsvc.New(zap.NewNop(), deps.tenants, providermemory.New(), deps.buckets, deps.accessKeys, deps.principals, bucketpolicymemory.New(), deps.delegations, deps.secrets, deps.wrapKeys, plcClient, nil, &testutil.FakeSwarf{}) + svc := tenantsvc.New(zap.NewNop(), deps.tenants, providermemory.New(), deps.buckets, deps.accessKeys, deps.principals, bucketpolicymemory.New(), deps.delegations, deps.secrets, deps.wrapKeys, deps.exports, plcClient, nil, &testutil.FakeSwarf{}) route := api.NewDeleteTenantHandler(zap.NewNop(), svc) return serve(route), deps } @@ -765,6 +770,20 @@ func TestDeleteTenantHandler(t *testing.T) { require.Equal(t, 1, deps.directory.deactivations) }) + t.Run("409 while an export is in progress", func(t *testing.T) { + e, deps := setupDelete(t, tenant.Disabled) + require.NoError(t, deps.exports.Add(ctx, exportsession.Input{ + ID: "session-1", Tenant: deps.tenantID, Bucket: testutil.RandomDID(t), + CustomerKey: "z6LSbysY2xFMRpGMhb7tFTLMpeuPRaqaWM1yECx2AtzE3KCc", ExpiresAt: time.Now().Add(time.Hour), + })) + + rec := doRequest(t, e, http.MethodDelete, "/tenants/tenant-1", nil) + require.Equal(t, http.StatusConflict, rec.Code) + _, err := deps.tenants.GetByExternalID(ctx, "tenant-1") + require.NoError(t, err) + require.Equal(t, 0, deps.directory.deactivations) + }) + t.Run("already-deactivated DID still cleans up locally", func(t *testing.T) { e, deps := setupDelete(t, tenant.Disabled) deps.serveTombstone(t) // directory reports the DID as already tombstoned diff --git a/pkg/rpc/failure.go b/pkg/rpc/failure.go index 5e6e6d53..a3268bb6 100644 --- a/pkg/rpc/failure.go +++ b/pkg/rpc/failure.go @@ -73,6 +73,7 @@ func bucketFailure(res failer, err error) error { errors.Is(err, bucketsvc.ErrUnknownBucket), errors.Is(err, bucketsvc.ErrUnknownAccessKey), errors.Is(err, bucketsvc.ErrInvalidArgument), + errors.Is(err, bucketsvc.ErrExportInProgress), // A PutBucketPolicy body that does not decode or may not be stored, // under InvalidBucketPolicy. errors.Is(err, s3bkt.ErrInvalidPolicy), diff --git a/pkg/rpc/failure_test.go b/pkg/rpc/failure_test.go index 523fa814..b7f60470 100644 --- a/pkg/rpc/failure_test.go +++ b/pkg/rpc/failure_test.go @@ -53,6 +53,14 @@ func TestBucketFailure(t *testing.T) { requireName(t, f.got, bucketsvc.BucketAlreadyOwnedErrorName) }) + t.Run("wrapped export-in-progress sentinel is set as failure with its name", func(t *testing.T) { + f := &recordingFailer{} + err := fmt.Errorf("%w: %q", bucketsvc.ErrExportInProgress, "foo") + require.NoError(t, bucketFailure(f, err)) + require.True(t, f.called) + requireName(t, f.got, bucketsvc.ExportInProgressErrorName) + }) + t.Run("a rejected policy document is set as failure under the policy name", func(t *testing.T) { f := &recordingFailer{} err := fmt.Errorf("decoding policy: unknown field %q: %w", "Version", s3bkt.ErrInvalidPolicy) diff --git a/pkg/rpc/rpc_test.go b/pkg/rpc/rpc_test.go index a6f2b310..c5347154 100644 --- a/pkg/rpc/rpc_test.go +++ b/pkg/rpc/rpc_test.go @@ -13,6 +13,7 @@ import ( bucketmemory "github.com/fil-forge/hilt/pkg/store/bucket/memory" bucketpolicymemory "github.com/fil-forge/hilt/pkg/store/bucketpolicy/memory" delegationmemory "github.com/fil-forge/hilt/pkg/store/delegation/memory" + exportsessionmemory "github.com/fil-forge/hilt/pkg/store/exportsession/memory" principalmemory "github.com/fil-forge/hilt/pkg/store/principal/memory" providermemory "github.com/fil-forge/hilt/pkg/store/provider/memory" tenantmemory "github.com/fil-forge/hilt/pkg/store/tenant/memory" @@ -35,7 +36,7 @@ func newRoutes(t *testing.T, id identity.Identity) []server.Route { require.NoError(t, err) revocations, err := swarfclient.New(testutil.RandomDID(t), url.URL{Scheme: "http", Host: "swarf.test"}) require.NoError(t, err) - buckets := bucketsvc.New(zap.NewNop(), az, bucketmemory.New(), delegationmemory.New(), accesskeymemory.New(), tenantmemory.New(), bucketpolicymemory.New(), up, revocations, nil) + buckets := bucketsvc.New(zap.NewNop(), az, bucketmemory.New(), delegationmemory.New(), accesskeymemory.New(), tenantmemory.New(), bucketpolicymemory.New(), exportsessionmemory.New(), up, revocations, nil) return []server.Route{ rpc.NewAuthorizeRequestHandler(zap.NewNop(), az), diff --git a/pkg/rpc/service/bucket/errors.go b/pkg/rpc/service/bucket/errors.go index 5d7aaa93..fa99c8b8 100644 --- a/pkg/rpc/service/bucket/errors.go +++ b/pkg/rpc/service/bucket/errors.go @@ -13,6 +13,7 @@ const ( UnknownBucketErrorName = "UnknownBucket" UnknownAccessKeyErrorName = "UnknownAccessKey" InvalidArgumentErrorName = "InvalidArgument" + ExportInProgressErrorName = "ExportInProgress" ) // Known errors returned by the bucket [Service]. Handlers pass these to @@ -41,4 +42,8 @@ var ( // of range (e.g. ListBuckets max-buckets). The name matches the canonical S3 // error code. ErrInvalidArgument = errors.New(InvalidArgumentErrorName, "invalid request parameter") + // ErrExportInProgress is returned when deleting a bucket that has an open + // export session. Ingot maps it to S3 OperationAborted, as it does for its + // own pin. + ErrExportInProgress = errors.New(ExportInProgressErrorName, "bucket has an export in progress") ) diff --git a/pkg/rpc/service/bucket/policy_test.go b/pkg/rpc/service/bucket/policy_test.go index a8948629..15c7c63f 100644 --- a/pkg/rpc/service/bucket/policy_test.go +++ b/pkg/rpc/service/bucket/policy_test.go @@ -17,6 +17,7 @@ import ( bucketmemory "github.com/fil-forge/hilt/pkg/store/bucket/memory" bucketpolicymemory "github.com/fil-forge/hilt/pkg/store/bucketpolicy/memory" delegationmemory "github.com/fil-forge/hilt/pkg/store/delegation/memory" + exportsessionmemory "github.com/fil-forge/hilt/pkg/store/exportsession/memory" principalmemory "github.com/fil-forge/hilt/pkg/store/principal/memory" providermemory "github.com/fil-forge/hilt/pkg/store/provider/memory" "github.com/fil-forge/hilt/pkg/store/tenant" @@ -77,7 +78,7 @@ func TestPolicy(t *testing.T) { swarf := &htestutil.FakeSwarf{} grants := grant.NewRotator(zap.NewNop(), delegations, accessKeys, secrets, swarf) policyWrites := bucketpolicysvc.New(zap.NewNop(), tenants, buckets, principals, policies, grants) - return fixture{bucketsvc.New(zap.NewNop(), az, buckets, delegations, accessKeys, tenants, policies, &fakeSprue{}, swarf, policyWrites), policies, bucketID, bucketGone} + return fixture{bucketsvc.New(zap.NewNop(), az, buckets, delegations, accessKeys, tenants, policies, exportsessionmemory.New(), &fakeSprue{}, swarf, policyWrites), policies, bucketID, bucketGone} } type reqOpts struct { diff --git a/pkg/rpc/service/bucket/service.go b/pkg/rpc/service/bucket/service.go index 19e855e7..2ecdcdf0 100644 --- a/pkg/rpc/service/bucket/service.go +++ b/pkg/rpc/service/bucket/service.go @@ -26,6 +26,7 @@ import ( bucketstore "github.com/fil-forge/hilt/pkg/store/bucket" bucketpolicystore "github.com/fil-forge/hilt/pkg/store/bucketpolicy" delegationstore "github.com/fil-forge/hilt/pkg/store/delegation" + "github.com/fil-forge/hilt/pkg/store/exportsession" tenantstore "github.com/fil-forge/hilt/pkg/store/tenant" s3 "github.com/fil-forge/libforge/commands/s3" s3bkt "github.com/fil-forge/libforge/commands/s3/bucket" @@ -62,6 +63,7 @@ type Service struct { accessKeys accesskey.Store tenants tenantstore.Store policies bucketpolicystore.Store + exports exportsession.Store uploads UploadClient revocations grant.RevocationPublisher // policyWrites stores a policy as the management API stores a policy PUT. @@ -77,6 +79,7 @@ func New( accessKeys accesskey.Store, tenants tenantstore.Store, policies bucketpolicystore.Store, + exports exportsession.Store, uploads UploadClient, revocations grant.RevocationPublisher, policyWrites *bucketpolicysvc.Service, @@ -89,6 +92,7 @@ func New( accessKeys: accessKeys, tenants: tenants, policies: policies, + exports: exports, uploads: uploads, revocations: revocations, policyWrites: policyWrites, @@ -269,14 +273,14 @@ func (s *Service) Create(ctx context.Context, issuer did.DID, args *s3bkt.Create } // Delete authenticates the request, checks the s3:DeleteBucket permission, -// resolves the bucket, verifies its space is empty via Sprue (acting as the -// tenant), publishes revocations for the delegations over the bucket, then -// deletes those delegations and the bucket record, which takes the policy with -// it. The policy goes without a publication: the gateway refuses a bucket it no longer -// knows, so nothing cached for it can be used. Revocations are published -// first so that a revocation service failure leaves the bucket intact and the -// call cleanly retryable — otherwise the delegations would live on with nothing -// for a verifier to check. +// resolves the bucket, refuses while an export of it is open, verifies its space +// is empty via Sprue (acting as the tenant), publishes revocations for the +// delegations over the bucket, then deletes those delegations and the bucket +// record, which takes the policy with it. The policy goes without a publication: +// the gateway refuses a bucket it no longer knows, so nothing cached for it can +// be used. Revocations are published first so that a revocation service failure +// leaves the bucket intact and the call cleanly retryable — otherwise the +// delegations would live on with nothing for a verifier to check. func (s *Service) Delete(ctx context.Context, issuer did.DID, args *s3bkt.DeleteArguments) (*s3bkt.DeleteOK, error) { authz, err := s.authorizer.Authorize(ctx, issuer, args.Request) if err != nil { @@ -287,6 +291,16 @@ func (s *Service) Delete(ctx context.Context, issuer did.DID, args *s3bkt.Delete return nil, fmt.Errorf("%w: %s", ErrOperationMismatch, authz.Operation) } + // An export session opens here before the gateway pins anything, so the + // gateway's own pin cannot be relied on to refuse this delete. + exporting, err := s.exports.HasOpenForBucket(ctx, authz.Bucket.ID) + if err != nil { + return nil, fmt.Errorf("checking export sessions: %w", err) + } + if exporting { + return nil, fmt.Errorf("%w: %q", ErrExportInProgress, authz.BucketName) + } + // Verify the bucket is empty, listing its blobs via Sprue as the tenant (the // bucket→tenant root delegation authorizes the /blob/list invocation). account, err := s.authorizer.TenantIssuer(ctx, authz.Tenant.ID) diff --git a/pkg/rpc/service/bucket/service_test.go b/pkg/rpc/service/bucket/service_test.go index 8e7b917a..9710bc1b 100644 --- a/pkg/rpc/service/bucket/service_test.go +++ b/pkg/rpc/service/bucket/service_test.go @@ -27,6 +27,8 @@ import ( delegationstore "github.com/fil-forge/hilt/pkg/store/delegation" delegationmemory "github.com/fil-forge/hilt/pkg/store/delegation/memory" delegationpostgres "github.com/fil-forge/hilt/pkg/store/delegation/postgres" + "github.com/fil-forge/hilt/pkg/store/exportsession" + exportsessionmemory "github.com/fil-forge/hilt/pkg/store/exportsession/memory" principalmemory "github.com/fil-forge/hilt/pkg/store/principal/memory" principalpostgres "github.com/fil-forge/hilt/pkg/store/principal/postgres" providermemory "github.com/fil-forge/hilt/pkg/store/provider/memory" @@ -163,7 +165,7 @@ func TestCreate(t *testing.T) { require.NoError(t, err) require.NoError(t, delegations.PutBatch(ctx, []ucan.Delegation{powerline})) az := auth.NewAuthorizer(zap.NewNop(), accessKeys, tenants, providers, buckets, principals, policies, secrets) - return bucketsvc.New(zap.NewNop(), az, buckets, delegations, accessKeys, tenants, policies, sprue, &htestutil.FakeSwarf{}, nil), buckets + return bucketsvc.New(zap.NewNop(), az, buckets, delegations, accessKeys, tenants, policies, exportsessionmemory.New(), sprue, &htestutil.FakeSwarf{}, nil), buckets } args := func() *s3bkt.CreateArguments { @@ -351,6 +353,7 @@ type deleteDeps struct { buckets *bucketmemory.Store delegations *delegationmemory.Store policies bucketpolicystore.Store + exports *exportsessionmemory.Store swarf *htestutil.FakeSwarf tenantID did.DID bucketID did.DID @@ -399,11 +402,13 @@ func TestDelete(t *testing.T) { require.NoError(t, delegations.PutBatch(ctx, []ucan.Delegation{root, grant})) az := auth.NewAuthorizer(zap.NewNop(), accessKeys, tenants, providers, buckets, principals, policies, secrets) swarf := &htestutil.FakeSwarf{} + exports := exportsessionmemory.New() return deleteDeps{ - svc: bucketsvc.New(zap.NewNop(), az, buckets, delegations, accessKeys, tenants, policies, sprue, swarf, nil), + svc: bucketsvc.New(zap.NewNop(), az, buckets, delegations, accessKeys, tenants, policies, exports, sprue, swarf, nil), buckets: buckets, delegations: delegations, policies: policies, + exports: exports, tenantID: tenantID, swarf: swarf, bucketID: bucketID, @@ -548,6 +553,26 @@ func TestDelete(t *testing.T) { require.Empty(t, d.swarf.Revocations(), "nothing is revoked when the delete is refused") }) + t.Run("refuses while an export is open and keeps the bucket", func(t *testing.T) { + sprue := &fakeSprue{empty: true} + d := setup(t, allPerms, false, sprue) + require.NoError(t, d.exports.Add(ctx, exportsession.Input{ + ID: "session-1", Tenant: d.tenantID, Bucket: d.bucketID, + CustomerKey: "z6LSbysY2xFMRpGMhb7tFTLMpeuPRaqaWM1yECx2AtzE3KCc", ExpiresAt: time.Now().Add(time.Hour), + })) + + _, err := d.svc.Delete(ctx, providerID, del(bucketName)) + require.ErrorIs(t, err, bucketsvc.ErrExportInProgress) + require.False(t, sprue.emptyCalled, "refused before touching the space") + require.Empty(t, d.swarf.Revocations()) + _, err = d.buckets.GetByName(ctx, bucketName) + require.NoError(t, err) + + require.NoError(t, d.exports.Transition(ctx, "session-1", exportsession.Opened, exportsession.Aborted)) + _, err = d.svc.Delete(ctx, providerID, del(bucketName)) + require.NoError(t, err) + }) + t.Run("propagates a SpaceEmpty error", func(t *testing.T) { d := setup(t, allPerms, false, &fakeSprue{emptyErr: errors.New("sprue unavailable")}) _, err := d.svc.Delete(ctx, providerID, del(bucketName)) @@ -573,7 +598,7 @@ func TestList(t *testing.T) { require.NoError(t, tenants.Add(ctx, tenantID, "tenant-1", providerID, tenant.Active)) seedKey(t, accessKeys, secrets, delegations, principals, signer, tenantID, perms, principalBound) az := auth.NewAuthorizer(zap.NewNop(), accessKeys, tenants, providers, buckets, principals, policies, secrets) - return bucketsvc.New(zap.NewNop(), az, buckets, delegations, accessKeys, tenants, policies, &fakeSprue{}, &htestutil.FakeSwarf{}, nil), buckets, tenantID + return bucketsvc.New(zap.NewNop(), az, buckets, delegations, accessKeys, tenants, policies, exportsessionmemory.New(), &fakeSprue{}, &htestutil.FakeSwarf{}, nil), buckets, tenantID } // listArgs presigns a ListBuckets request; extra ListBuckets query params @@ -722,7 +747,7 @@ func TestInfo(t *testing.T) { for _, w := range wrap { reads = w(reads) } - return bucketsvc.New(zap.NewNop(), az, buckets, reads, accessKeys, tenants, policies, &fakeSprue{}, &htestutil.FakeSwarf{}, nil), policies, root, accessKeys + return bucketsvc.New(zap.NewNop(), az, buckets, reads, accessKeys, tenants, policies, exportsessionmemory.New(), &fakeSprue{}, &htestutil.FakeSwarf{}, nil), policies, root, accessKeys } // grantPolicy stores a policy allowing "user-1" the given actions on the bucket. @@ -1045,7 +1070,7 @@ func infoDuringPolicyWrite(t *testing.T, straddle func(s delegationstore.Store, }) az := auth.NewAuthorizer(zap.NewNop(), accessKeys, tenants, providermemory.New(), buckets, principals, policies, vaultmemory.New()) - svc := bucketsvc.New(zap.NewNop(), az, buckets, reads, accessKeys, tenants, policies, &fakeSprue{}, &htestutil.FakeSwarf{}, nil) + svc := bucketsvc.New(zap.NewNop(), az, buckets, reads, accessKeys, tenants, policies, exportsessionmemory.New(), &fakeSprue{}, &htestutil.FakeSwarf{}, nil) type result struct { ok *s3bkt.InfoOK