diff --git a/.changeset/native-conditional-operations.md b/.changeset/native-conditional-operations.md new file mode 100644 index 00000000..7c5dde28 --- /dev/null +++ b/.changeset/native-conditional-operations.md @@ -0,0 +1,9 @@ +--- +"files-sdk": minor +--- + +Add provider-native conditional create, replace, exact-read, delete, and copy operations to the existing plugin, hook, retry, prefix, read-only, and receipt pipeline. AWS S3 implements the initial atomic primitives; unsupported adapters, filesystem, custom S3 endpoints, R2, bulk, and multipart paths fail closed. + +The S3 adapter exposes the primitives only for canonical AWS — no `endpoint` and no `AWS_ENDPOINT_URL_S3` / `AWS_ENDPOINT_URL` redirect — with a new `conditional` option to override that in either direction, and verifies per request that the resolved hostname is AWS (so a shared-config `endpoint_url` fails closed too) and that the installed `@aws-sdk/client-s3` serialized every predicate header (conditional copy needs 3.919.0+; the `@aws-sdk/client-s3` peer range moves from `^3.700.0` to `^3.1079.0`). The CLI gains `--if-match` / `--if-none-match` / `--dest-if-match` on `upload`, `download`, `delete`, and `copy`, and the MCP tools accept a matching `condition` input. `cache()` now invalidates after a failed write too; `softDelete()` forwards a conditional delete of an already-trashed key; `dedup()` rejects every conditional mode. + +A conditional mutation that commits before an awaited plugin rejects the call now surfaces as `FilesError.applied === true` (with `appliedEtag` for uploads) — on the rejected error, in `onError` / `onAction`, and in the `audit()` record — so callers can reconcile instead of retrying a predicate that can only conflict. A plugin that re-invokes `next()` after the native call failed gets that first failure as `cause`. `rejectConditional(op, plugin, reason)` is exported as the one veto shape for plugins with out-of-band side effects. Ordinary operations type `mode` as `undefined` (there never was an `"overwrite"` value to branch on). diff --git a/apps/web/docs/(concepts)/capabilities.mdx b/apps/web/docs/(concepts)/capabilities.mdx index ed2fed32..026b9e43 100644 --- a/apps/web/docs/(concepts)/capabilities.mdx +++ b/apps/web/docs/(concepts)/capabilities.mdx @@ -1,6 +1,6 @@ --- title: Capabilities -description: Query what an adapter can do up front with files.capabilities - branch on range reads, signed URLs, and server-side copy instead of catching a throw. +description: Query what an adapter can do up front with files.capabilities - including range reads, signed URLs, server-side copy, and native conditional operations. --- The unified surface is the common subset every adapter implements, but adapters differ at the edges: some honor byte-range reads, some can mint a signed URL, some copy server-side. The wrapper already gates on these per-adapter — pass a `range` to an adapter that has no range primitive and it throws _before_ any provider call. `files.capabilities` turns that implicit knowledge into a queryable surface, so you can branch up front instead of discovering a limit by catching an error. @@ -32,6 +32,19 @@ interface AdapterCapabilities { multipart: boolean; serverSideCopy: boolean; signedUrl: { supported: boolean; maxExpiresIn?: number }; + conditional: { + create: boolean; + replace: boolean; + exactRead: boolean; + delete: boolean; + copy: { + sourceEtag: boolean; + destinationCreate: boolean; + destinationReplace: boolean; + atomicSourceDestination: boolean; + }; + multipart: { create: false; replace: false }; + }; } ``` @@ -45,8 +58,15 @@ interface AdapterCapabilities { | `multipart` | the adapter exposes a resumable / multipart upload primitive | `upload({ control })` | | `serverSideCopy` | `copy()` runs server-side, with no body re-transfer through your process | `copy` | | `signedUrl` | `url()` can mint a signed or tokenized URL — see below | `url` | +| `conditional` | the requested create, replace, exact-read, delete, or copy predicate has a provider-native primitive | `upload`, `download`, `delete`, `copy` | -Every field mirrors an operation the unified API actually has. There are deliberately no flags for `raw`-only territory (object versioning, checksums, conditional writes, POST policies) — advertising a flag for a non-operation would turn the matrix into a back-door spec, where a wrong flag is worse than no flag. +Every field mirrors an operation the unified API actually has. There are deliberately no flags for `raw`-only territory such as object versioning, checksums, or POST policies — advertising a flag for a non-operation would turn the matrix into a back-door spec, where a wrong flag is worse than no flag. + +### `conditional` + +Conditional capabilities are granular because providers often support only part of the surface. `create`, `replace`, `exactRead`, and `delete` map to their native single-object primitives. Conditional copy separately declares source ETag checks, destination create/replace checks, and whether the selected source plus destination predicates are evaluated atomically in one request. Invocation rechecks the declaration and fails before provider I/O when any required part is absent. These flags describe native adapter support; a composed plugin can still veto a conditional mode when its own policy cannot preserve that atomic boundary. + +There is no probe-then-mutate fallback. See [Conditional operations](/docs/conditional-operations) for the API, ETag rules, provider matrix, and retry/observer semantics. ### `signedUrl` @@ -60,6 +80,6 @@ signedUrl: { supported: boolean; maxExpiresIn?: number } ## How it's derived -`capabilities` is computed live from the adapter on every read, so a plugin that swaps behavior is always reflected. The first six fields read the exact per-adapter flags and optional methods the wrapper already gates on (`supportsRange`, `reportsUploadProgress`, `supportsDelimiter`, `supportsMetadata`, `supportsCacheControl`, and the presence of `resumableUpload`), so they can never drift from runtime behavior. `serverSideCopy` and `signedUrl` are declared by each adapter and default to the conservative value (`false`) when an adapter declares nothing — a caller that doesn't advertise reads as "no", never a wrong "yes". +`capabilities` is computed live from the adapter on every read, so a plugin that swaps behavior is always reflected. The first six fields read the exact per-adapter flags and optional methods the wrapper already gates on (`supportsRange`, `reportsUploadProgress`, `supportsDelimiter`, `supportsMetadata`, `supportsCacheControl`, and the presence of `resumableUpload`), so they can never drift from runtime behavior. Conditional support is derived from the adapter's optional native operation group and its explicit copy guarantees. `serverSideCopy`, `signedUrl`, and every absent conditional primitive default to the conservative value (`false`) — a caller that doesn't advertise reads as "no", never a wrong "yes". If you're writing a custom adapter, set `supportsServerSideCopy` and `signedUrl` alongside the existing `supports*` flags to make your adapter introspectable; both are optional and advisory (they don't gate any operation). diff --git a/apps/web/docs/(concepts)/conditional-operations.mdx b/apps/web/docs/(concepts)/conditional-operations.mdx new file mode 100644 index 00000000..c0be5248 --- /dev/null +++ b/apps/web/docs/(concepts)/conditional-operations.mdx @@ -0,0 +1,109 @@ +--- +title: Conditional operations +description: Use provider-native create, replace, exact-read, delete, and copy predicates without bypassing plugins, hooks, receipts, or client policy. +--- + +Conditional operations apply an ETag predicate in the provider's mutation or read request. They are the safe building blocks for create-only writes and compare-and-set updates: the SDK never implements them as `head()` / `exists()` followed by an unconditional operation. + +```ts lineNumbers +// Create only when the key is absent. +const created = await files.upload("reports/q3.json", body, { + condition: { type: "create" }, +}); + +// Replace only the generation returned by the create. +const replaced = await files.upload("reports/q3.json", nextBody, { + condition: { type: "replace", etag: created.etag }, +}); + +// Read or delete exactly that replacement. +const exact = await files.download("reports/q3.json", { + condition: { etag: replaced.etag }, +}); +await files.delete("reports/q3.json", { + condition: { etag: exact.etag! }, +}); +``` + +A failed predicate rejects with [`FilesError`](/docs/api/errors) rather than weakening the request. An adapter without the requested native primitive also rejects before provider I/O. + +## Conditional copy + +A conditional copy checks the source generation and the destination predicate in one native provider request. Both predicates are required: the source must still have its supplied ETag, and the destination must either be absent or match its own supplied ETag. + +```ts lineNumbers +// Publish the staged generation, but only if nothing is published yet. +const staging = await files.head("staging/report.json"); +await files.copy("staging/report.json", "published/report.json", { + condition: { + source: { etag: staging.etag! }, + destination: { type: "create" }, + }, +}); + +// Later: replace exactly the published generation with exactly the staged one. +const nextStaging = await files.head("staging/report.json"); +const published = await files.head("published/report.json"); +await files.copy("staging/report.json", "published/report.json", { + condition: { + source: { etag: nextStaging.etag! }, + destination: { type: "replace", etag: published.etag! }, + }, +}); +``` + +There is no download-and-upload fallback for conditional copy. Check all of `sourceEtag`, the requested destination mode, and `atomicSourceDestination` under [`files.capabilities.conditional.copy`](/docs/capabilities) before planning one. + +## ETag form + +Pass the canonical bare strong ETag exposed by a conditional-capable adapter: for example `a1b2c3`, not `"a1b2c3"`. AWS S3 normalizes the values returned by `upload()`, `download()`, `head()`, and `list()` into this form. Empty values, weak validators (`W/…`), wildcards, quoted values, comma-separated lists, control characters, and excessively long values reject before provider I/O. Conditional uploads return a non-optional `etag` for the new generation; a provider commit whose response omits or corrupts that ETag rejects as an ambiguous outcome. + +## Capabilities and current support + +Each primitive is declared separately under `files.capabilities.conditional`. The declaration is conservative, and invocation checks it again. + +These flags describe the adapter's native primitives. They are necessary, not sufficient, after composition: a plugin may still veto a mode when its own side effects cannot preserve the same atomic boundary. + +```ts lineNumbers +const c = files.capabilities.conditional; + +if (c.create && c.replace && c.exactRead && c.delete) { + // This adapter can support a native single-key CAS workflow. +} + +if ( + c.copy.sourceEtag && + c.copy.destinationCreate && + c.copy.atomicSourceDestination +) { + // Safe to perform a source-exact, destination-create copy. +} +``` + +The initial implementation supports canonical AWS S3 buckets through the AWS SDK adapter. A custom S3 `endpoint`, an `AWS_ENDPOINT_URL_S3` / `AWS_ENDPOINT_URL` redirect in the environment, or a shared-config `endpoint_url` does not inherit the claim, because S3-compatible services differ in which conditional headers they honor; the [`s3()` adapter's `conditional` option](/docs/adapters/s3) overrides that detection in either direction. The adapter also checks, per request, that the resolved hostname is AWS and that every predicate it set was serialized by the installed `@aws-sdk/client-s3`, and rejects the call otherwise. Cloudflare R2 is not supported. + +The local filesystem adapter also reports every conditional capability as `false`. Its body and metadata sidecar are separate files, and a process-local lock plus a read-before-rename would not be compare-and-set against writers in another process. It fails closed until the storage layout can provide one native atomic commit boundary. + +Conditional calls are single-key only. Bulk array calls, multipart uploads (`multipart: false`, the explicit opt-out, is fine), resumable `UploadControl`, signed uploads, and `move()` do not accept a condition. Use conditional copy followed by a separately conditioned delete only when your application can tolerate the two distinct commits; the SDK does not present that sequence as an atomic move. + +## Plugins use the same operation families + +Conditional calls traverse the existing ordered plugin onion under the same `upload`, `download`, `delete`, and `copy` kinds. Their `mode` distinguishes `create`, `replace`, `exact`, `match`, and `conditional` variants. That means existing per-verb encryption, compression, validation, metadata, audit, tracing, and usage handlers see the call instead of silently passing a new verb they do not recognize. + +For a conditional root call, the SDK freezes the operation family, mode, and ETag predicates across every `next()` boundary. A plugin may transform the upload body, metadata, or returned value, or veto by throwing before `next()`. It cannot drop or change a predicate, reroute to another verb, call the native operation twice, or synthesize success without a native call. A rejected `next()` never reaches the provider, so a plugin that catches one and then calls `next(op)` with the original predicate receives the committed result as a success. Bundled plugins whose semantics require multiple backends or extra mutations reject the incompatible conditional modes before any of their provider I/O. + +The built-in [`encryption()`](/docs/plugins/encryption) plugin is compatible with conditional create, replace, and exact read: conditional uploads store its transformed bytes, and exact reads traverse its inverse transform. As with every use of that plugin, its documented key and plaintext-compatibility policy still applies. + +Other bundled plugins keep only the modes whose side effects preserve one native operation. Compression, content-type inference, and validation use their existing same-verb transforms. Exact reads bypass the ordinary cache, and the cache invalidates a key after any write settles — including a failed conditional one, since a `Conflict` is proof the cached ETag is stale. Soft delete rejects a conditional delete outside the trash prefix (a delete of an already-trashed key is a real delete and keeps its predicate); versioning rejects conditional writes and copies; deduplication rejects every conditional mode, because a pointer's ETag never reflects its content; and failover and tiering reject every conditional mode. Those vetoes happen before the plugin performs provider I/O. + +## CLI and MCP + +The [CLI](/docs/cli/commands#conditional-predicates) exposes the same predicates as flags — `--if-none-match` and `--if-match ` on `upload`, `--if-match ` on `download` and `delete`, and `--if-match ` plus `--if-none-match` / `--dest-if-match ` on `copy` — and the [MCP server](/docs/cli/mcp) accepts a `condition` input on `upload`, `download`, `delete`, and `copy` in the SDK's shapes. Both are single-key only and fail closed on adapters without native support, exactly like the SDK. + +## Hooks, receipts, and terminal outcomes + +`onAction` and `onError` keep their once-per-call semantics and `onRetry` its once-per-scheduled-retry cadence; all three include a redacted `condition` label, and no ETag predicate is copied into hook metadata (a successful conditional upload or exact read still exposes the committed ETag through `event.result`, as does an enabled receipt). A successful conditional mutation receives the ordinary upload, delete, or copy receipt with the same condition label. Reads and failed calls have no receipt. + +Hooks and receipt delivery are fire-and-forget: a hook that throws cannot change the result after the provider commits. An awaited plugin is different. If it calls `next()`, observes the committed result, and then throws, the public call rejects, `onError` and an error `onAction` fire, and no success receipt is emitted. The provider mutation cannot be rolled back, so that outcome is applied-but-unacknowledged: the rejected [`FilesError`](/docs/api/errors) carries `applied: true` (and `appliedEtag` for uploads), the same flag reaches `onError` / `onAction` and the [`audit()`](/docs/plugins/audit) record, and the right recovery is an exact read rather than a retry of the same predicate — which can now only conflict. + +Retries have the same ambiguity at the network boundary. A response can be lost after a provider commit; a retry of the same predicate can then fail because the first request already changed the object. Conditional retries never turn into an unconditional request, but callers that receive an error must still reconcile before assuming nothing changed. diff --git a/apps/web/docs/(concepts)/receipts.mdx b/apps/web/docs/(concepts)/receipts.mdx index accbc5ee..b9b67418 100644 --- a/apps/web/docs/(concepts)/receipts.mdx +++ b/apps/web/docs/(concepts)/receipts.mdx @@ -34,7 +34,7 @@ Receipts ride on the existing [`onAction`](/docs/api/onaction) hook as an additi - the call is a mutating verb (`upload`, `delete`, `copy`, `move`), and - the call **succeeded**. -Reads, `signedUploadUrl`, failures, bulk array calls (which aggregate many objects into one event), and every instance with receipts off leave `event.receipt` unset - so an existing `onAction` consumer that never opted in sees the exact payload it always has. +Reads, `signedUploadUrl`, failures, bulk array calls (which aggregate many objects into one event), and every instance with receipts off leave `event.receipt` unset - so an existing `onAction` consumer that never opted in sees the exact payload it always has. A successful [conditional mutation](/docs/conditional-operations) uses the ordinary `upload`, `delete`, or `copy` receipt and adds its redacted `condition` label; predicate ETags are never copied into the receipt. Every field except `sha256` is **derived** from the work the SDK already does for the hook - the timing, the adapter name, the caller-facing key, and `bytes` / `etag` read straight off the [`UploadResult`](/docs/api/upload). Turning receipts on with `receipts: true` therefore adds no per-call cost. @@ -60,6 +60,8 @@ const files = new Files({ With `receipts: true` (or `{ sha256: false }`), the body is never read and no hash is taken. +Receipt delivery remains observational. If `onAction` throws, the committed operation still succeeds. If an awaited plugin throws after the native operation has committed, the public call rejects and no success receipt is emitted; reconcile that applied-but-unacknowledged outcome with an exact read. + ### Plugins that transform the body The fingerprint is taken **before** any plugin runs. If you compose the instance with a body-transforming plugin - [`encryption`](/docs/plugins/encryption) writes ciphertext, [`compression`](/docs/plugins/compression) writes compressed bytes - the bytes on disk differ from `sha256`. That's deliberate: it's the hash of the content you handed in, and it matches what a [`download`](/docs/api/download) gives back, since reads reverse the same transforms. So it's the value a round-trip check can verify - and the only stable one, since `encryption` uses a fresh key per object and would otherwise hash differently on every upload of identical content. diff --git a/apps/web/docs/adapters/(system-adapters)/fs.mdx b/apps/web/docs/adapters/(system-adapters)/fs.mdx index e80ac57c..0c9cc903 100644 --- a/apps/web/docs/adapters/(system-adapters)/fs.mdx +++ b/apps/web/docs/adapters/(system-adapters)/fs.mdx @@ -44,6 +44,8 @@ const files = new Files({ Body at `` `${root}/${key}` ``; sidecar at `` `${root}/${key}.meta.json` ``. Sidecars survive `cp -r` / `git mv` / partial-tree deletion. `list()` hides them. ETag is a SHA-1-derived stable hash computed at upload time. Files written into `root` by hand without a sidecar are still readable - `contentType` falls back to `application/octet-stream` and `etag` is absent. +The separate body and sidecar layout cannot provide a cross-process atomic ETag compare-and-set boundary. The adapter therefore advertises no [conditional operations](/docs/conditional-operations) and rejects them before filesystem I/O; it does not emulate CAS with a read followed by a rename. + ## Compatibility | Method | Status | Notes | @@ -58,3 +60,4 @@ Body at `` `${root}/${key}` ``; sidecar at `` `${root}/${key}.meta.json` ``. Sid | `copy` | ✅ | | | `url` | ⚠️ | Returns a `file://` URL by default - fine for CLIs and tests, not browsers. With `urlBaseUrl` set, returns `/` so a dev server (Next.js `/public` mount, `serve-static`, etc.) can deliver the body. `responseContentDisposition` throws because neither `file://` nor static-server URLs have a signature mechanism in which to bind the override. | | `signedUploadUrl` | ❌ | Throws - the fs adapter has no built-in upload server, signer, or verifier, so it cannot bind expiry, content type, or size limits into an upload capability. Upload through `files.upload()` or an application route that enforces those controls server-side. | +| conditional operations | ❌ | Body and metadata are separate files, so there is no honest cross-process native CAS primitive. | diff --git a/apps/web/docs/adapters/(vendor-adapters)/r2.mdx b/apps/web/docs/adapters/(vendor-adapters)/r2.mdx index d9a579ca..20eaa71d 100644 --- a/apps/web/docs/adapters/(vendor-adapters)/r2.mdx +++ b/apps/web/docs/adapters/(vendor-adapters)/r2.mdx @@ -117,6 +117,10 @@ await files.signedUploadUrl("avatars/abc.png", { To cap upload sizes on R2, enforce the limit at your application gateway before issuing the URL. +## Conditional operations + +R2 does not advertise the SDK's provider-native [conditional operation](/docs/conditional-operations) contract in this release. That includes the AWS-SDK HTTP path: configuring an S3-compatible endpoint does not inherit AWS S3's capability claims. Conditional calls fail before R2 I/O. + ## Compatibility ### HTTP mode diff --git a/apps/web/docs/adapters/(vendor-adapters)/s3.mdx b/apps/web/docs/adapters/(vendor-adapters)/s3.mdx index 0488e8aa..7186762b 100644 --- a/apps/web/docs/adapters/(vendor-adapters)/s3.mdx +++ b/apps/web/docs/adapters/(vendor-adapters)/s3.mdx @@ -29,6 +29,12 @@ const files = new Files({ }); ``` +Canonical AWS S3 buckets support provider-native [conditional create, replace, exact read, delete, and copy](/docs/conditional-operations). Conditional copy sends its source ETag and destination create/replace predicate in one `CopyObject` request. + +The primitives are exposed only when the client will talk to canonical AWS: no `endpoint` option and no `AWS_ENDPOINT_URL_S3` / `AWS_ENDPOINT_URL` redirect in the environment (the AWS SDK honors those on its own). A shared-config `endpoint_url` — profile-level or under a `services` section — is invisible at construction, so it is caught at request time instead: a conditional request whose resolved hostname is not `amazonaws.com` fails closed before it is sent. S3-compatible services differ in conditional-header support, so in every one of those cases the adapter refuses rather than risk an unconditional overwrite. AWS-hosted endpoints (VPC, FIPS, dual-stack, GovCloud) resolve under `amazonaws.com` and need no override. Pass `conditional: true` to opt an S3-compatible endpoint you have verified back in (it skips both checks), or `conditional: false` to disable the primitives on a canonical bucket. + +Conditional copy needs `@aws-sdk/client-s3` 3.919.0 or newer, where `CopyObject` gained `IfMatch` / `IfNoneMatch` (the peer range now starts at 3.1079.0). Because an optional peer range is advisory, the adapter also verifies at request time that every predicate it set was actually serialized as a header and rejects the call if the installed client dropped one. Conditional uploads take a buffered body (`Blob`, `Uint8Array`, `ArrayBuffer`, or string) — a stream is rejected before I/O. + ## Options , + plugin: string, + reason: string +): never; +``` + +Veto a [conditional operation](/docs/conditional-operations) from inside `wrap`, before any provider I/O, by throwing a permanent `Provider` `FilesError` worded `: conditional is unsupported because `. Use it for the modes your plugin cannot make atomic because of a side effect of its own — a snapshot, a mirror write, a pointer rewrite — that would sit outside the native compare-and-set. Every bundled plugin that vetoes uses this shape. + +```ts lineNumbers +import { isConditionalOperation, rejectConditional } from "files-sdk"; + +const mirror: FilesPlugin = { + name: "mirror", + wrap: (op, next) => { + if (isConditionalOperation(op)) { + rejectConditional( + op, + "mirror", + "the mirror write cannot share one native compare-and-set" + ); + } + return next(op); + }, +}; +``` + ## createFiles ```ts diff --git a/apps/web/docs/plugins/audit.mdx b/apps/web/docs/plugins/audit.mdx index a086a1d6..d0f30d35 100644 --- a/apps/web/docs/plugins/audit.mdx +++ b/apps/web/docs/plugins/audit.mdx @@ -101,6 +101,7 @@ Placed **last** (innermost) it instead records the physical operations the pipel - **One record per logical operation.** Plugins run [outside retries](/docs/retries), so a call that retries three times is still **one** record - its `durationMs` spans the retries. - **Bulk fans out to one record per item.** `upload([...])` / `delete([...])` record each key individually, flagged `bulk: true`, with per-item success/error - exactly the granularity an audit log wants. +- **Conditional calls stay under their verb.** Native create/replace/exact-read/delete/copy records keep `action: "upload" | "download" | "delete" | "copy"` and add a redacted `condition`; predicate ETags are never recorded. - **Body-transparent, works on any adapter.** It never buffers, transforms, or reads the body (`size` comes from the upload result's declared metadata, not the bytes), so streaming, range downloads, `url()`, and `signedUploadUrl()` all keep working. It writes no object metadata and has no native dependencies. - **Not a security boundary.** It records operations made **through the instance**; a direct presigned `PUT` to the bucket bypasses it. Pair it with [`signedUrlPolicy()`](/docs/plugins/signed-url-policy) to keep the URLs you mint tight. - **`wrap`-only.** It adds no methods, so plain `new Files({ plugins })` works - though `createFiles` is fine too and keeps you consistent with the [extend](/docs/plugins/api#createfiles)-based plugins. diff --git a/apps/web/docs/plugins/encryption.mdx b/apps/web/docs/plugins/encryption.mdx index 1032cfa3..c2a96d0c 100644 --- a/apps/web/docs/plugins/encryption.mdx +++ b/apps/web/docs/plugins/encryption.mdx @@ -23,6 +23,8 @@ await files.upload("secret.txt", "hello"); // stored encrypted await (await files.download("secret.txt")).text(); // "hello" ``` +The same transforms cover native [conditional operations](/docs/conditional-operations): create and replace store ciphertext, and an exact read decrypts only after the provider accepts its ETag. Conditional range reads are still refused for the same AES-GCM reason as ordinary ranges. + ## How it works The plugin uses **envelope encryption**, the same pattern cloud KMS services use: diff --git a/apps/web/docs/plugins/index.mdx b/apps/web/docs/plugins/index.mdx index 5d5516fc..cc95e24f 100644 --- a/apps/web/docs/plugins/index.mdx +++ b/apps/web/docs/plugins/index.mdx @@ -85,6 +85,7 @@ Plugins run **inside** the [`onAction`](/docs/api/onaction) / [`onError`](/docs/ - A `wrap` runs **once per logical operation**, not once per retry attempt. Encryption seals the body once; a retry resends the bytes the plugin already produced. - Plugins see **caller-facing keys** - never the internal [prefixed](/docs/prefixes) path. A key-rewriting plugin rewrites before prefixing. - The hooks still fire around the whole thing, so `onAction` reports the final, plugin-produced result. +- [Conditional operations](/docs/conditional-operations) use these same verb families. Their mode and ETag predicates are immutable across `next()`, while body, metadata, and results remain transformable. ### Bulk operations too @@ -167,3 +168,5 @@ The built-in [`versioning()`](/docs/plugins/versioning) plugin is a real example - **Buffering transforms break streaming.** Encrypt / compress / scan need the whole body in memory, which is incompatible with unknown-length streams and [resumable uploads](/docs/resumable) (which re-read the original body). Gate those plugins the way the core already gates streams. - **Metadata-stashing needs adapter support.** A plugin that round-trips state through `options.metadata` (an encryption IV, say) only works on adapters that [support metadata](/docs/api/upload) - the same gate a direct `metadata` upload hits. - **`wrap` runs outside the timeout.** Per-attempt [timeouts](/docs/timeouts) bound the adapter call, not a slow plugin. The caller's `options` (including `signal`) ride on the operation, so a plugin can opt into cancellation itself. +- **Conditional roots must reach one native call.** A plugin may veto before `next()`, but it cannot downgrade or change the predicate, reroute the operation, invoke the provider twice, or synthesize success. If an awaited observer throws after `next()` committed, the caller receives an applied-but-unacknowledged error (`error.applied === true`); use a fire-and-forget hook when observation must not affect success. +- **Veto the conditional modes you cannot make atomic.** The engine catches everything that crosses `next()`, but not a side effect your plugin performs on its own — a snapshot copy, a mirror write, a pointer rewrite — which a compare-and-set cannot cover. Call [`rejectConditional(op, "my-plugin", reason)`](/docs/plugins/api#rejectconditional) for those modes before any I/O, as every bundled plugin does. Plugins that only transform the body, metadata, or result need no veto. diff --git a/apps/web/docs/plugins/tracing.mdx b/apps/web/docs/plugins/tracing.mdx index 1eb8bb03..4b650bdb 100644 --- a/apps/web/docs/plugins/tracing.mdx +++ b/apps/web/docs/plugins/tracing.mdx @@ -80,5 +80,6 @@ Placed **last** (innermost) it instead times only the provider call, with the pl - **The default tracer is a no-op until you register an SDK.** With no OpenTelemetry SDK set up, `trace.getTracer()` returns a no-op tracer, so the plugin is a cheap pass-through until you wire up an exporter. - **One span per logical operation.** Plugins run [outside retries](/docs/retries), so a call that retries three times is still **one** span, not three — the span covers the whole logical call. +- **Conditional calls keep the same span names.** They add a low-cardinality `files.condition` attribute (`create`, `replace`, `exact-read`, `match-delete`, or `conditional-copy`) and never attach predicate ETags. - **Body-transparent, works on any adapter.** It never buffers, transforms, or reads the body (`files.size` comes from declared metadata, not bytes), so streaming, range downloads, `url()`, and `signedUploadUrl()` all keep working. - **`wrap`-only.** It adds no methods, so plain `new Files({ plugins })` works — though `createFiles` is fine too and keeps you consistent with the [extend](/docs/plugins/api#createfiles)-based plugins. diff --git a/apps/web/lib/demo-files.ts b/apps/web/lib/demo-files.ts index 4769090d..e5269e15 100644 --- a/apps/web/lib/demo-files.ts +++ b/apps/web/lib/demo-files.ts @@ -15,6 +15,19 @@ const log = (op: string, ...args: unknown[]): void => { const CAPABILITIES: AdapterCapabilities = { cacheControl: true, + conditional: { + copy: { + atomicSourceDestination: false, + destinationCreate: false, + destinationReplace: false, + sourceEtag: false, + }, + create: false, + delete: false, + exactRead: false, + multipart: { create: false, replace: false }, + replace: false, + }, delimiter: true, metadata: true, multipart: true, diff --git a/bun.lock b/bun.lock index 635a9d9b..30f886c1 100644 --- a/bun.lock +++ b/bun.lock @@ -131,7 +131,7 @@ }, "peerDependencies": { "@anthropic-ai/claude-agent-sdk": "^0.3.0", - "@aws-sdk/client-s3": "^3.700.0", + "@aws-sdk/client-s3": "^3.1079.0", "@aws-sdk/lib-storage": "^3.700.0", "@aws-sdk/s3-presigned-post": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", diff --git a/packages/files-sdk/package.json b/packages/files-sdk/package.json index 51f1b758..581c73f5 100644 --- a/packages/files-sdk/package.json +++ b/packages/files-sdk/package.json @@ -443,7 +443,7 @@ }, "peerDependencies": { "@anthropic-ai/claude-agent-sdk": "^0.3.0", - "@aws-sdk/client-s3": "^3.700.0", + "@aws-sdk/client-s3": "^3.1079.0", "@aws-sdk/lib-storage": "^3.700.0", "@aws-sdk/s3-presigned-post": "^3.700.0", "@aws-sdk/s3-request-presigner": "^3.700.0", diff --git a/packages/files-sdk/src/audit/index.ts b/packages/files-sdk/src/audit/index.ts index 4798a545..8134fecb 100644 --- a/packages/files-sdk/src/audit/index.ts +++ b/packages/files-sdk/src/audit/index.ts @@ -1,4 +1,7 @@ +import { isConditionalOperation } from "../index.js"; import type { + ConditionalActionType, + ConditionalFilesOperation, FilesActionType, FilesOperation, FilesPlugin, @@ -8,6 +11,21 @@ import type { import { FilesError } from "../internal/errors.js"; import type { FilesErrorCode } from "../internal/errors.js"; +const conditionalAction = ( + op: ConditionalFilesOperation +): ConditionalActionType => { + if (op.kind === "upload") { + return op.mode; + } + if (op.kind === "download") { + return "exact-read"; + } + if (op.kind === "delete") { + return "match-delete"; + } + return "conditional-copy"; +}; + /** * The mutating verbs, audited by default — the same set the SDK treats as * writes. `signedUploadUrl` is here because minting an upload capability is a @@ -45,6 +63,8 @@ const ALL_ACTIONS: readonly FilesActionType[] = [ export interface AuditRecord { /** The verb that ran (mirrors {@link FilesActionType}). */ action: FilesActionType; + /** Native conditional primitive, without its ETag predicate. */ + condition?: ConditionalActionType; /** Caller-facing key — present for every verb except `copy` / `move` / `list`. */ key?: string; /** `copy` / `move` source. */ @@ -70,8 +90,12 @@ export interface AuditRecord { size?: number; /** Set when this record is one item of a bulk (`[...]`) call. */ bulk?: true; - /** Failure detail, on `status: "error"`. */ - error?: { code: FilesErrorCode; message: string }; + /** + * Failure detail, on `status: "error"`. `applied: true` marks a conditional + * mutation that committed at the provider before an awaited plugin rejected + * the call — the object changed despite the error status. + */ + error?: { code: FilesErrorCode; message: string; applied?: true }; } export interface AuditOptions { @@ -126,9 +150,13 @@ const auditedKinds = (events: AuditOptions["events"]): Set => { /** Normalize whatever was thrown to a stable `{ code, message }`. */ const errorInfo = ( failure: unknown -): { code: FilesErrorCode; message: string } => { +): { code: FilesErrorCode; message: string; applied?: true } => { const error = FilesError.wrap(failure); - return { code: error.code, message: error.message }; + return { + code: error.code, + message: error.message, + ...(error.applied && { applied: true }), + }; }; interface RecordContext { @@ -154,12 +182,18 @@ const buildRecord = (op: FilesOperation, ctx: RecordContext): AuditRecord => { status === "success" && op.kind === "upload" && typeof (result as UploadResult).size === "number"; + const condition: ConditionalActionType | undefined = isConditionalOperation( + op + ) + ? conditionalAction(op) + : undefined; return { action: op.kind, at, durationMs, status, ...locus, + ...(condition !== undefined && { condition }), ...(actor !== undefined && { actor }), ...("bulk" in op && op.bulk ? { bulk: true } : {}), ...(hasSize && { size: (result as UploadResult).size }), diff --git a/packages/files-sdk/src/cache/index.ts b/packages/files-sdk/src/cache/index.ts index 212b32f1..17858cce 100644 --- a/packages/files-sdk/src/cache/index.ts +++ b/packages/files-sdk/src/cache/index.ts @@ -1,4 +1,4 @@ -import { createStoredFile } from "../index.js"; +import { createStoredFile, isConditionalOperation } from "../index.js"; import type { Files, FilesOperation, @@ -8,6 +8,7 @@ import type { StoredFileMeta, } from "../index.js"; import { DEFAULT_URL_EXPIRES_IN } from "../internal/core.js"; +import { FilesError } from "../internal/errors.js"; /** The read verbs {@link cache} can serve from its store. */ export type CacheableOperation = "head" | "url" | "download"; @@ -386,6 +387,41 @@ export const cache = (options: CacheOptions = {}): FilesPlugin => { return createStoredFile(meta, { data: bytes, kind: "buffer" }); }; + /** + * Run a mutation (or exact read) and, when it fails in a way that proves the + * cached record stale, drop the affected keys before rethrowing. Only a + * conditional operation or a `Conflict` qualifies: a plain failed write is + * no evidence either way, and invalidating on every failure would charge a + * remote-store round-trip to each NotFound in a bulk delete. The store's own + * failure on this path is swallowed — the caller must see the original + * error (a `Conflict` is what drives a head() → retry loop), never a store + * hiccup dressed up as the outcome of the mutation. + */ + const invalidateOnStaleFailure = async ( + op: FilesOperation, + keys: readonly string[], + run: () => Promise + ): Promise => { + try { + return await run(); + } catch (error) { + const conflict = error instanceof FilesError && error.code === "Conflict"; + if (conflict || isConditionalOperation(op)) { + await Promise.all( + keys.map(async (key) => { + try { + await store.delete(key); + } catch { + // The original error is the outcome; a store hiccup here must + // not replace it. + } + }) + ); + } + throw error; + } + }; + const wrap = (async ( op: FilesOperation, next: PluginNext @@ -398,24 +434,42 @@ export const cache = (options: CacheOptions = {}): FilesPlugin => { return enabled.has("url") ? cachedUrl(op, next) : next(op); } case "download": { + // An exact read is a provider precondition, not a cache validator. A + // cached body may predate the supplied ETag (or have no trustworthy + // relationship to it), so always drive the native conditional read + // and never populate the ordinary download cache from its result. A + // rejected predicate does prove the cached record stale, though, so + // it invalidates like a failed conditional write would. + if (isConditionalOperation(op)) { + return invalidateOnStaleFailure(op, [op.key], () => next(op)); + } return enabled.has("download") ? cachedDownload(op, next) : next(op); } // Writes always invalidate, regardless of which reads are cached — drop - // the affected key(s) only after the mutation actually lands. + // the affected key(s) after the mutation lands. A failed write leaves + // the record alone unless the failure itself proved it stale (see + // `invalidateOnStaleFailure`). case "upload": case "delete": { - const result = await next(op); + const result = await invalidateOnStaleFailure(op, [op.key], () => + next(op) + ); await store.delete(op.key); return result; } case "copy": { - const result = await next(op); + // A conditional copy can fail on its *source* predicate too, so a + // stale cached source ETag would replay the same conflict; drop both + // ends on a stale failure. The success path touches only `op.to`. + const keys = isConditionalOperation(op) ? [op.from, op.to] : [op.to]; + const result = await invalidateOnStaleFailure(op, keys, () => next(op)); await store.delete(op.to); return result; } case "move": { - // oxlint-disable-next-line react-doctor/async-parallel -- next() must settle before invalidating; the two deletes are ordered after the write, not independent. - const result = await next(op); + const keys = [op.from, op.to]; + const result = await invalidateOnStaleFailure(op, keys, () => next(op)); + // oxlint-disable-next-line react-doctor/async-parallel -- ordered after the write settles, not independent work. await store.delete(op.from); await store.delete(op.to); return result; diff --git a/packages/files-sdk/src/cli/commands.ts b/packages/files-sdk/src/cli/commands.ts index 7263674c..a782f9f1 100644 --- a/packages/files-sdk/src/cli/commands.ts +++ b/packages/files-sdk/src/cli/commands.ts @@ -1,7 +1,9 @@ import type { BulkOptions, + CopyCondition, MultipartOptions, SearchMatch, + UploadCondition, UploadManyItem, } from "../index.js"; import { sync, transfer } from "../index.js"; @@ -93,12 +95,82 @@ export interface UploadCmdOpts extends CommonRunOpts { multipartConcurrency?: number; concurrency?: number; stopOnError?: boolean; + ifMatch?: string; + ifNoneMatch?: boolean; } +/** + * `--if-none-match` (create only) / `--if-match ` (replace exactly that + * generation) as the SDK's upload `condition`. Mutually exclusive. + */ +const buildUploadCondition = (opts: { + ifMatch?: string; + ifNoneMatch?: boolean; +}): UploadCondition | undefined => { + if (opts.ifNoneMatch && opts.ifMatch !== undefined) { + throw new FilesError( + "Provider", + "--if-match and --if-none-match are mutually exclusive" + ); + } + if (opts.ifNoneMatch) { + return { type: "create" }; + } + return opts.ifMatch === undefined + ? undefined + : { etag: opts.ifMatch, type: "replace" }; +}; + +/** + * `--if-match ` (source) plus exactly one of `--if-none-match` (create + * the destination) / `--dest-if-match ` (replace that destination + * generation) as the SDK's copy `condition`. A conditional copy needs both + * halves, so a partial set of flags is an error rather than a weaker request. + */ +const buildCopyCondition = (opts: { + destIfMatch?: string; + ifMatch?: string; + ifNoneMatch?: boolean; +}): CopyCondition | undefined => { + const hasSource = opts.ifMatch !== undefined; + const hasCreate = opts.ifNoneMatch === true; + const hasReplace = opts.destIfMatch !== undefined; + if (!(hasSource || hasCreate || hasReplace)) { + return; + } + if (hasCreate && hasReplace) { + throw new FilesError( + "Provider", + "--if-none-match and --dest-if-match are mutually exclusive" + ); + } + if (!(hasSource && (hasCreate || hasReplace))) { + throw new FilesError( + "Provider", + "a conditional copy needs --if-match and either --if-none-match or --dest-if-match " + ); + } + return { + destination: hasCreate + ? { type: "create" } + : { etag: opts.destIfMatch as string, type: "replace" }, + source: { etag: opts.ifMatch as string }, + }; +}; + +const SINGLE_KEY_CONDITION = (flag: string, verb: string): FilesError => + new FilesError( + "Provider", + `${flag} applies to a single key; conditional ${verb} has no bulk form` + ); + const runUploadDir = async ( opts: UploadCmdOpts, multipart: boolean | MultipartOptions | undefined ): Promise => { + if (opts.ifMatch !== undefined || opts.ifNoneMatch) { + throw SINGLE_KEY_CONDITION("--if-match / --if-none-match", "upload"); + } if (opts.dryRun) { // Local-only preview — don't walk the tree, matching the single-upload // dry-run which doesn't stat its --file either. @@ -136,6 +208,7 @@ const runUploadDir = async ( export const runUpload = async (opts: UploadCmdOpts): Promise => { const multipart = buildMultipart(opts); + const condition = buildUploadCondition(opts); // --dir uploads a whole local tree via the SDK's bulk array form. It's // mutually exclusive with the single-object inputs (a key, --file, --stdin). @@ -162,6 +235,7 @@ export const runUpload = async (opts: UploadCmdOpts): Promise => { "upload", { cacheControl: opts.cacheControl, + condition, contentType: opts.contentType, key, metadata: parseKeyValuePairs(opts.metadata), @@ -172,12 +246,28 @@ export const runUpload = async (opts: UploadCmdOpts): Promise => { ); } const { files } = await loadFiles(opts.global); - const { body } = await readBody({ file: opts.file, stdin: opts.stdin }); + const { body: streamed } = await readBody({ + file: opts.file, + stdin: opts.stdin, + }); + // A conditional PutObject needs its Content-Length up front and has no + // multipart fallback, so the adapters reject stream bodies; the CLI always + // reads a stream, so buffer it when a predicate is set. Unconditional + // uploads keep streaming. + const body = + condition === undefined + ? streamed + : new Uint8Array( + await new Response( + streamed as ReadableStream + ).arrayBuffer() + ); const result = await files.upload(key, body, { cacheControl: opts.cacheControl, contentType: opts.contentType, metadata: parseKeyValuePairs(opts.metadata), ...(multipart !== undefined && { multipart }), + ...(condition !== undefined && { condition }), }); emit(result, opts); }; @@ -190,6 +280,7 @@ export interface DownloadCmdOpts extends CommonRunOpts { range?: string; concurrency?: number; stopOnError?: boolean; + ifMatch?: string; } const runDownloadMany = async ( @@ -244,9 +335,14 @@ const runDownloadMany = async ( export const runDownload = async (opts: DownloadCmdOpts): Promise => { const range = parseRange(opts.range); + const condition = + opts.ifMatch === undefined ? undefined : { etag: opts.ifMatch }; // Many keys (or an explicit --out-dir) take the bulk path: each body is // written under the directory at the path its key implies. const many = opts.keys.length > 1 || opts.outDir !== undefined; + if (many && condition !== undefined) { + throw SINGLE_KEY_CONDITION("--if-match", "download"); + } if (opts.dryRun) { if (many) { @@ -258,7 +354,12 @@ export const runDownload = async (opts: DownloadCmdOpts): Promise => { } return dryRun( "download", - { dest: opts.stdout ? "" : opts.out, key: opts.keys[0], range }, + { + condition, + dest: opts.stdout ? "" : opts.out, + key: opts.keys[0], + range, + }, opts ); } @@ -269,7 +370,11 @@ export const runDownload = async (opts: DownloadCmdOpts): Promise => { const key = opts.keys[0] as string; const { files } = await loadFiles(opts.global); - const file = await files.download(key, { as: "stream", range }); + const file = await files.download(key, { + as: "stream", + range, + ...(condition !== undefined && { condition }), + }); await writeBody(file, { out: opts.out, stdout: opts.stdout }); if (!opts.stdout) { // body went to a file; emit status to stdout in the user's chosen format @@ -373,18 +478,24 @@ export interface DeleteCmdOpts extends CommonRunOpts { keys: string[]; concurrency?: number; stopOnError?: boolean; + ifMatch?: string; } export const runDelete = async (opts: DeleteCmdOpts): Promise => { + const condition = + opts.ifMatch === undefined ? undefined : { etag: opts.ifMatch }; + if (condition !== undefined && opts.keys.length !== 1) { + throw SINGLE_KEY_CONDITION("--if-match", "delete"); + } if (opts.dryRun) { - return dryRun("delete", { keys: opts.keys }, opts); + return dryRun("delete", { condition, keys: opts.keys }, opts); } const { files } = await loadFiles(opts.global); // One key keeps the original throw-on-failure contract and output shape. if (opts.keys.length === 1) { const key = opts.keys[0] as string; - await files.delete(key); + await files.delete(key, condition ? { condition } : undefined); emit({ deleted: true, key }, opts); return; } @@ -405,14 +516,18 @@ export const runDelete = async (opts: DeleteCmdOpts): Promise => { export interface CopyCmdOpts extends CommonRunOpts { from: string; to: string; + ifMatch?: string; + ifNoneMatch?: boolean; + destIfMatch?: string; } export const runCopy = async (opts: CopyCmdOpts): Promise => { + const condition = buildCopyCondition(opts); if (opts.dryRun) { - return dryRun("copy", { from: opts.from, to: opts.to }, opts); + return dryRun("copy", { condition, from: opts.from, to: opts.to }, opts); } const { files } = await loadFiles(opts.global); - await files.copy(opts.from, opts.to); + await files.copy(opts.from, opts.to, condition ? { condition } : undefined); emit({ copied: true, from: opts.from, to: opts.to }, opts); }; diff --git a/packages/files-sdk/src/cli/mcp.ts b/packages/files-sdk/src/cli/mcp.ts index 1561841a..3e8375f0 100644 --- a/packages/files-sdk/src/cli/mcp.ts +++ b/packages/files-sdk/src/cli/mcp.ts @@ -88,6 +88,41 @@ const encodeUploadBody = (text?: string, base64?: string): Uint8Array => { throw new FilesError("Provider", "expected either `text` or `base64` body"); }; +// Conditional predicates, mirroring the SDK's `condition` option shapes. The +// `capabilities` tool advertises `conditional.*`, so every tool that can +// carry a predicate must accept one — a zod object strips unknown keys, and a +// silently-dropped predicate would turn a compare-and-set into an overwrite. +const etagArg = z + .string() + .describe("Canonical bare strong ETag (no quotes, no W/ prefix)"); +const uploadConditionArg = z + .union([ + z.object({ type: z.literal("create") }), + z.object({ etag: etagArg, type: z.literal("replace") }), + ]) + .optional() + .describe( + "Create only when the key is absent ({ type: 'create' }) or replace only the generation with this ETag ({ type: 'replace', etag }). Fails closed on adapters without native support; see `capabilities.conditional`." + ); +const etagConditionArg = z + .object({ etag: etagArg }) + .optional() + .describe( + "Only proceed when the object still has this ETag. Fails closed on adapters without native support; see `capabilities.conditional`." + ); +const copyConditionArg = z + .object({ + destination: z.union([ + z.object({ type: z.literal("create") }), + z.object({ etag: etagArg, type: z.literal("replace") }), + ]), + source: z.object({ etag: etagArg }), + }) + .optional() + .describe( + "Copy only when the source still has `source.etag` and the destination is absent ({ type: 'create' }) or has `destination.etag` ({ type: 'replace' }). Both predicates are required; see `capabilities.conditional.copy`." + ); + export interface McpServerOpts { allowWrites?: boolean; destination?: GlobalCliOptions; @@ -161,6 +196,7 @@ export const buildMcpServer = async ( .optional() .describe("Base64-encoded body (mutually exclusive with text)"), cacheControl: z.string().optional(), + condition: uploadConditionArg, contentType: z.string().optional(), key: z.string().describe("Object key (path) within the bucket/store"), metadata: z @@ -194,6 +230,7 @@ export const buildMcpServer = async ( cacheControl, metadata, multipart, + condition, }) => { try { if (text !== undefined && base64 !== undefined) { @@ -208,6 +245,7 @@ export const buildMcpServer = async ( contentType, metadata, ...(multipart !== undefined && { multipart }), + ...(condition !== undefined && { condition }), }); return ok(result); } catch (error) { @@ -223,6 +261,7 @@ export const buildMcpServer = async ( description: "Download bytes for the given key. Returns metadata + base64 body so binary roundtrips safely through MCP. Bodies larger than `maxBytes` (default 10 MiB) are refused — use the CLI for larger files.", inputSchema: { + condition: etagConditionArg, key: z.string(), maxBytes: z .number() @@ -245,12 +284,15 @@ export const buildMcpServer = async ( }, title: "Download a file", }, - async ({ key, maxBytes, range }) => { + async ({ key, maxBytes, range, condition }) => { try { const cap = resolveMcpDownloadCap(maxBytes); const meta = await files.head(key); assertMcpDownloadFitsCap(key, mcpDownloadSize(meta.size, range), cap); - const file = await files.download(key, range ? { range } : undefined); + const file = await files.download(key, { + ...(range && { range }), + ...(condition && { condition }), + }); const buf = Buffer.from(await file.arrayBuffer()); assertMcpDownloadFitsCap(key, buf.byteLength, cap); return ok({ @@ -327,22 +369,31 @@ export const buildMcpServer = async ( "delete", { description: - "Permanently delete the object at `key`. Pass an array of keys to delete many in one call — that form returns a structured `{ deleted, errors? }` result instead of throwing on partial failure.", + "Permanently delete the object at `key`. Pass an array of keys to delete many in one call — that form returns a structured `{ deleted, errors? }` result instead of throwing on partial failure. A `condition` (single key only) deletes only the generation with that ETag.", inputSchema: { concurrency: concurrencyArg, + condition: etagConditionArg, key: z.union([z.string(), z.array(z.string())]), stopOnError: stopOnErrorArg, }, title: "Delete one or many keys", }, - async ({ key, concurrency, stopOnError }) => { + async ({ key, concurrency, stopOnError, condition }) => { try { if (Array.isArray(key)) { + if (condition !== undefined) { + throw new FilesError( + "Provider", + "`condition` applies to a single key — bulk delete does not support conditional predicates", + undefined, + { permanent: true } + ); + } return ok( await files.delete(key, bulkOpts(concurrency, stopOnError)) ); } - await files.delete(key); + await files.delete(key, condition ? { condition } : undefined); return ok({ deleted: true, key }); } catch (error) { return errorPayload(error); @@ -353,13 +404,18 @@ export const buildMcpServer = async ( server.registerTool( "copy", { - description: "Copy `from` to `to` within the same store.", - inputSchema: { from: z.string(), to: z.string() }, + description: + "Copy `from` to `to` within the same store. A `condition` makes it a native compare-and-set on both the source ETag and the destination state.", + inputSchema: { + condition: copyConditionArg, + from: z.string(), + to: z.string(), + }, title: "Server-side copy", }, - async ({ from, to }) => { + async ({ from, to, condition }) => { try { - await files.copy(from, to); + await files.copy(from, to, condition ? { condition } : undefined); return ok({ copied: true, from, to }); } catch (error) { return errorPayload(error); @@ -390,7 +446,7 @@ export const buildMcpServer = async ( "capabilities", { description: - "Report what the configured adapter can do — range reads, native upload progress, list delimiters, user metadata, cache-control, multipart/resumable uploads, server-side copy, and signed URLs (`supported` plus any `maxExpiresIn` cap). Pure introspection; makes no provider call. Branch on this instead of catching an unsupported-operation error.", + "Report what the configured adapter can do — range reads, native upload progress, list delimiters, user metadata, cache-control, multipart/resumable uploads, server-side copy, signed URLs (`supported` plus any `maxExpiresIn` cap), and native conditional predicates (`conditional.create` / `replace` / `exactRead` / `delete` / `copy.*`, honored by the `condition` input on upload, download, delete, and copy). Pure introspection; makes no provider call. Branch on this instead of catching an unsupported-operation error.", inputSchema: {}, title: "Adapter capabilities", }, diff --git a/packages/files-sdk/src/cli/program.ts b/packages/files-sdk/src/cli/program.ts index 61cc2b88..3ff0ce07 100644 --- a/packages/files-sdk/src/cli/program.ts +++ b/packages/files-sdk/src/cli/program.ts @@ -38,6 +38,7 @@ const VERSION = pkg.version; // option definitions consistent (and satisfy no-duplicate-string). const CONCURRENCY_FLAG = "--concurrency "; const STOP_ON_ERROR_FLAG = "--stop-on-error"; +const IF_MATCH_FLAG = "--if-match "; const STOP_FIRST_FAILURE_MANY_DESC = "stop at the first failure (many keys)"; const PREFIX_FLAG = "--prefix "; const LIMIT_FLAG = "--limit "; @@ -379,6 +380,18 @@ export const buildProgram = ( ) .option(CONCURRENCY_FLAG, "parallel uploads for --dir", intArg) .option(STOP_ON_ERROR_FLAG, "stop at the first failure (--dir)") + .addOption( + new Option( + "--if-none-match", + "create only: fail if the key already exists (single key)" + ).conflicts(["ifMatch", "dir"]) + ) + .addOption( + new Option( + IF_MATCH_FLAG, + "replace only the generation with this ETag (single key)" + ).conflicts(["ifNoneMatch", "dir"]) + ) .action( wrap(runUpload as (opts: never) => Promise, (args, common) => { const [key, opts] = args as [ @@ -392,6 +405,8 @@ export const buildProgram = ( contentType: opts.contentType as string | undefined, dir: opts.dir as string | undefined, file: opts.file as string | undefined, + ifMatch: opts.ifMatch as string | undefined, + ifNoneMatch: opts.ifNoneMatch as boolean | undefined, key, metadata: opts.metadata as readonly string[] | undefined, multipart: opts.multipart as boolean | undefined, @@ -432,12 +447,17 @@ export const buildProgram = ( ) .option(CONCURRENCY_FLAG, "parallel downloads for many keys", intArg) .option(STOP_ON_ERROR_FLAG, STOP_FIRST_FAILURE_MANY_DESC) + .option( + IF_MATCH_FLAG, + "read only the generation with this ETag (single key)" + ) .action( wrap(runDownload as (opts: never) => Promise, (args, common) => { const [keys, opts] = args as [string[], Record]; return { ...common, concurrency: opts.concurrency as number | undefined, + ifMatch: opts.ifMatch as string | undefined, keys, out: opts.out as string | undefined, outDir: opts.outDir as string | undefined, @@ -473,15 +493,53 @@ export const buildProgram = ( ) .option(CONCURRENCY_FLAG, "parallel deletes for many keys", intArg) .option(STOP_ON_ERROR_FLAG, STOP_FIRST_FAILURE_MANY_DESC) - .action(wrap(runDelete as (opts: never) => Promise, bulkBuilder)); + .option( + IF_MATCH_FLAG, + "delete only the generation with this ETag (single key)" + ) + .action( + wrap(runDelete as (opts: never) => Promise, (args, common) => { + const [, opts] = args as [string[], Record]; + return { + ...bulkBuilder(args, common), + ifMatch: opts.ifMatch as string | undefined, + } as CommonRunOpts; + }) + ); program .command("copy ") - .description("server-side copy from one key to another") + .description( + "server-side copy from one key to another (conditional with --if-match plus --if-none-match or --dest-if-match)" + ) + .option(IF_MATCH_FLAG, "copy only while the source has this ETag") + .addOption( + new Option( + "--if-none-match", + "create the destination: fail if it already exists" + ).conflicts(["destIfMatch"]) + ) + .addOption( + new Option( + "--dest-if-match ", + "replace only the destination generation with this ETag" + ).conflicts(["ifNoneMatch"]) + ) .action( wrap(runCopy as (opts: never) => Promise, (args, common) => { - const [from, to] = args as [string, string]; - return { ...common, from, to } as CommonRunOpts; + const [from, to, opts] = args as [ + string, + string, + Record, + ]; + return { + ...common, + destIfMatch: opts.destIfMatch as string | undefined, + from, + ifMatch: opts.ifMatch as string | undefined, + ifNoneMatch: opts.ifNoneMatch as boolean | undefined, + to, + } as CommonRunOpts; }) ); diff --git a/packages/files-sdk/src/dedup/index.ts b/packages/files-sdk/src/dedup/index.ts index 9adf3f8b..82e751c8 100644 --- a/packages/files-sdk/src/dedup/index.ts +++ b/packages/files-sdk/src/dedup/index.ts @@ -1,3 +1,4 @@ +import { isConditionalOperation, rejectConditional } from "../index.js"; import type { FilesOperation, FilesPlugin, @@ -304,6 +305,18 @@ export const dedup = (options: DedupOptions = {}): FilesPlugin => { op: FilesOperation, next: PluginNext ): Promise => { + // Every conditional mode is vetoed, not just the ones that touch the + // blob. A pointer's body is always empty, so its ETag is the same for + // every key and never changes when the pointer is rewritten to a new + // blob — a compare-and-set delete or copy against it would succeed even + // though the key's content had moved on, which is worse than failing. + if (isConditionalOperation(op)) { + rejectConditional( + op, + "dedup", + "a pointer's ETag never reflects its content, so no native compare-and-set can guard it" + ); + } // Direct traffic to the blob store bypasses the plugin: blobs are stored // and read verbatim, never treated as pointers or re-de-duplicated. if ("key" in op && isStoreKey(op.key)) { diff --git a/packages/files-sdk/src/failover/index.ts b/packages/files-sdk/src/failover/index.ts index cb9bedd3..a7f51b5d 100644 --- a/packages/files-sdk/src/failover/index.ts +++ b/packages/files-sdk/src/failover/index.ts @@ -1,4 +1,4 @@ -import { Files } from "../index.js"; +import { Files, isConditionalOperation, rejectConditional } from "../index.js"; import type { Adapter, Body, @@ -338,10 +338,16 @@ export const failover = (options: FailoverOptions): FilesPlugin => { } }; - const wrap = ((op: FilesOperation, next: PluginNext): Promise => - dispatch(op, [runnerViaNext(next), ...secondaryRunners])) as NonNullable< - FilesPlugin["wrap"] - >; + const wrap = ((op: FilesOperation, next: PluginNext): Promise => { + if (isConditionalOperation(op)) { + rejectConditional( + op, + "failover", + "retrying against another backend cannot preserve one native compare-and-set" + ); + } + return dispatch(op, [runnerViaNext(next), ...secondaryRunners]); + }) as NonNullable; return { name: "failover", wrap }; }; diff --git a/packages/files-sdk/src/index.ts b/packages/files-sdk/src/index.ts index eb16d612..712d4ea0 100644 --- a/packages/files-sdk/src/index.ts +++ b/packages/files-sdk/src/index.ts @@ -2,6 +2,7 @@ import { byteLengthOf, countingStream, deleteManyWithFallback, + isMultipartRequested, mapMany, } from "./internal/core.js"; import { FilesError } from "./internal/errors.js"; @@ -13,7 +14,11 @@ import { resolveReceiptsConfig, sha256Hex, } from "./internal/receipts.js"; -import type { Receipt, ReceiptsConfig } from "./internal/receipts.js"; +import type { + ConditionalActionType, + Receipt, + ReceiptsConfig, +} from "./internal/receipts.js"; import { runResumableUpload } from "./internal/resumable.js"; import type { ResumableDriver, @@ -31,6 +36,7 @@ import { } from "./internal/retry.js"; import { isSafeSearchRegex } from "./internal/search-regex.js"; +export { rejectConditional } from "./internal/conditional.js"; export { FilesError, type FilesErrorCode } from "./internal/errors.js"; export { UploadControl } from "./internal/resumable.js"; export type { @@ -42,7 +48,11 @@ export type { ResumableUploadSession, UploadControlStatus, } from "./internal/resumable.js"; -export type { Receipt, ReceiptOp } from "./internal/receipts.js"; +export type { + ConditionalActionType, + Receipt, + ReceiptOp, +} from "./internal/receipts.js"; export type { BodySource, StoredFileMeta } from "./internal/stored-file.js"; export { createStoredFile } from "./internal/stored-file.js"; export { @@ -111,6 +121,11 @@ export interface OperationOptions { retries?: RetryOptions; } +/** A native compare-and-set predicate for a single upload. */ +export type UploadCondition = + | { type: "create" } + | { type: "replace"; etag: string }; + /** * A single upload-progress report. Passed to {@link UploadOptions.onProgress} * (and {@link UploadManyOptions.onProgress}, which also carries the item `key`). @@ -147,6 +162,13 @@ export interface MultipartOptions { } export interface UploadOptions extends OperationOptions { + /** + * Apply the upload atomically only when the destination is absent + * (`create`) or still has the supplied strong ETag (`replace`). Conditional + * uploads require a native adapter primitive and are unavailable for bulk, + * multipart, and resumable uploads. + */ + condition?: UploadCondition; /** * MIME type stored alongside the object and returned to readers in the * `Content-Type` response header. Inferred from `File` / `Blob` `type` @@ -233,6 +255,9 @@ export interface UploadResult { lastModified?: number; } +/** A conditional upload always returns the new strong ETag. */ +export type ConditionalUploadResult = UploadResult & { etag: string }; + export interface StoredFile { name: string; size: number; @@ -269,6 +294,8 @@ export interface ByteRange { } export interface DownloadOptions extends OperationOptions { + /** Read only the exact object identified by this strong ETag. */ + condition?: { etag: string }; as?: "blob" | "stream"; /** * Download only a contiguous slice of the object instead of the whole thing @@ -293,6 +320,36 @@ export interface DownloadOptions extends OperationOptions { range?: ByteRange; } +/** Options passed to an adapter's native conditional upload primitive. */ +export type AdapterUploadOptions = Omit< + UploadOptions, + "condition" | "control" | "multipart" +>; + +/** Options passed to an adapter's native exact-read primitive. */ +export type AdapterDownloadOptions = Omit; + +/** Options for a single delete, including an optional native ETag predicate. */ +export interface DeleteOptions extends OperationOptions { + condition?: { etag: string }; +} + +/** The source and destination predicates for a native conditional copy. */ +export interface CopyCondition { + source: { etag: string }; + destination: UploadCondition; +} + +/** Options for a copy, including optional atomic source/destination checks. */ +export interface CopyOptions extends OperationOptions { + condition?: CopyCondition; +} + +/** The public conditional-upload overload, which excludes unsupported modes. */ +export type ConditionalUploadOptions = AdapterUploadOptions & { + condition: UploadCondition; +}; + export interface ListOptions extends OperationOptions { /** * Filter results to keys that start with this string. Omit to list @@ -634,6 +691,66 @@ export interface SignedUrlCapability { maxExpiresIn?: number; } +/** A provider-native conditional copy and the destination predicates it honors. */ +export interface AdapterConditionalCopy { + /** The implementation always checks the supplied source ETag. */ + readonly sourceEtag: true; + /** Source and destination predicates settle atomically in one provider call. */ + readonly atomicSourceDestination: true; + /** Whether `destination: { type: "create" }` is supported atomically. */ + readonly destinationCreate: boolean; + /** Whether `destination: { type: "replace" }` is supported atomically. */ + readonly destinationReplace: boolean; + run: ( + from: string, + to: string, + condition: CopyCondition, + opts?: OperationOptions + ) => Promise; +} + +/** Optional native conditional primitives supplied by an adapter. */ +export interface AdapterConditionalOperations { + create?: ( + key: string, + body: Body, + opts?: AdapterUploadOptions + ) => Promise; + replace?: ( + key: string, + body: Body, + etag: string, + opts?: AdapterUploadOptions + ) => Promise; + exactRead?: ( + key: string, + etag: string, + opts?: AdapterDownloadOptions + ) => Promise; + delete?: ( + key: string, + etag: string, + opts?: OperationOptions + ) => Promise; + copy?: AdapterConditionalCopy; +} + +/** Queryable native conditional-operation support. */ +export interface ConditionalAdapterCapabilities { + create: boolean; + replace: boolean; + exactRead: boolean; + delete: boolean; + copy: { + sourceEtag: boolean; + atomicSourceDestination: boolean; + destinationCreate: boolean; + destinationReplace: boolean; + }; + /** Conditional multipart is intentionally outside the first API slice. */ + multipart: { create: false; replace: false }; +} + /** * A queryable snapshot of what the underlying adapter can do, exposed via * {@link Files.capabilities}. Lets callers, AI tool wrappers, and validators @@ -665,6 +782,8 @@ export interface AdapterCapabilities { serverSideCopy: boolean; /** How `url()` produces a download URL. From {@link Adapter.signedUrl}. */ signedUrl: SignedUrlCapability; + /** Native compare-and-set primitives. Unsupported operations fail closed. */ + conditional: ConditionalAdapterCapabilities; } export interface Adapter { @@ -731,17 +850,22 @@ export interface Adapter { * Advisory only; it does not gate `url()`. */ readonly signedUrl?: SignedUrlCapability; + /** + * Native conditional primitives. Omitting an operation means it cannot be + * implemented atomically and the public call fails before provider I/O. + */ + readonly conditional?: AdapterConditionalOperations; upload: ( key: string, body: Body, - opts?: UploadOptions + opts?: Omit ) => Promise; /** * Download an object's body and metadata. When {@link DownloadOptions.range} * is set, adapters that advertise {@link Adapter.supportsRange} must return * only the requested bytes, with `size` set to the range length. */ - download: (key: string, opts?: DownloadOptions) => Promise; + download: (key: string, opts?: AdapterDownloadOptions) => Promise; /** * Fetch metadata only — does not transfer the body. * @@ -852,6 +976,8 @@ type WriteActionType = Extract< */ export interface FilesActionEvent { type: FilesActionType; + /** Conditional primitive used, without exposing its ETag predicate. */ + condition?: ConditionalActionType; /** Caller-facing key, for single-key operations. */ key?: string; /** Caller-facing keys, for the array form. */ @@ -885,6 +1011,7 @@ export interface FilesActionEvent { */ export interface FilesErrorEvent { type: FilesActionType; + condition?: ConditionalActionType; key?: string; keys?: string[]; from?: string; @@ -901,6 +1028,7 @@ export interface FilesErrorEvent { */ export interface FilesRetryEvent { type: FilesActionType; + condition?: ConditionalActionType; key?: string; from?: string; to?: string; @@ -970,15 +1098,21 @@ export interface FilesOptions extends OperationOptions { export interface FileHandle { readonly key: string; - upload: (body: Body, opts?: UploadOptions) => Promise; + upload: { + ( + body: Body, + opts: ConditionalUploadOptions + ): Promise; + (body: Body, opts?: UploadOptions): Promise; + }; download: (opts?: DownloadOptions) => Promise; head: (opts?: OperationOptions) => Promise; exists: (opts?: OperationOptions) => Promise; - delete: (opts?: OperationOptions) => Promise; + delete: (opts?: DeleteOptions) => Promise; url: (opts?: UrlOptions) => Promise; signedUploadUrl: (opts: SignUploadOptions) => Promise; - copyTo: (destinationKey: string, opts?: OperationOptions) => Promise; - copyFrom: (sourceKey: string, opts?: OperationOptions) => Promise; + copyTo: (destinationKey: string, opts?: CopyOptions) => Promise; + copyFrom: (sourceKey: string, opts?: CopyOptions) => Promise; /** Move this key to `destinationKey`. See {@link Files.move}. */ moveTo: (destinationKey: string, opts?: OperationOptions) => Promise; /** Move `sourceKey` onto this key. See {@link Files.move}. */ @@ -996,24 +1130,93 @@ export interface FileHandle { * single call from one element of a batch. `copy`, `move`, `list`, `url`, and * `signedUploadUrl` have no array form and are always single. */ +export type ConditionalFilesOperation = + | { + kind: "upload"; + mode: "create"; + key: string; + body: Body; + options?: AdapterUploadOptions; + } + | { + kind: "upload"; + mode: "replace"; + etag: string; + key: string; + body: Body; + options?: AdapterUploadOptions; + } + | { + kind: "download"; + mode: "exact"; + etag: string; + key: string; + options?: AdapterDownloadOptions; + } + | { + kind: "delete"; + mode: "match"; + etag: string; + key: string; + options?: OperationOptions; + } + | { + kind: "copy"; + mode: "conditional"; + from: string; + to: string; + source: { etag: string }; + destination: UploadCondition; + options?: OperationOptions; + }; + export type FilesOperation = + | ConditionalFilesOperation | { kind: "upload"; + mode?: undefined; key: string; body: Body; - options?: UploadOptions; + options?: Omit; + bulk?: true; + } + | { + kind: "download"; + mode?: undefined; + key: string; + options?: AdapterDownloadOptions; bulk?: true; } - | { kind: "download"; key: string; options?: DownloadOptions; bulk?: true } | { kind: "head"; key: string; options?: OperationOptions; bulk?: true } | { kind: "exists"; key: string; options?: OperationOptions; bulk?: true } - | { kind: "delete"; key: string; options?: OperationOptions; bulk?: true } - | { kind: "copy"; from: string; to: string; options?: OperationOptions } + | { + kind: "delete"; + mode?: undefined; + key: string; + options?: OperationOptions; + bulk?: true; + } + | { + kind: "copy"; + mode?: undefined; + from: string; + to: string; + options?: OperationOptions; + } | { kind: "move"; from: string; to: string; options?: OperationOptions } | { kind: "list"; options?: ListOptions } | { kind: "url"; key: string; options?: UrlOptions } | { kind: "signedUploadUrl"; key: string; options?: SignUploadOptions }; +/** Whether an operation carries a native conditional predicate. */ +export const isConditionalOperation = ( + op: FilesOperation +): op is ConditionalFilesOperation => + (op.kind === "upload" && (op.mode === "create" || op.mode === "replace")) || + (op.kind === "download" && op.mode === "exact") || + (op.kind === "delete" && op.mode === "match") || + (op.kind === "copy" && op.mode === "conditional"); + /** * The value a given {@link FilesOperation} resolves to — the result map that * keeps a plugin's `wrap` / `next` fully typed per verb. Mirrors the return @@ -1023,7 +1226,9 @@ export type FilesOperation = export type OperationResult = O extends { kind: "upload"; } - ? UploadResult + ? O extends { mode: "create" | "replace" } + ? ConditionalUploadResult + : UploadResult : O extends { kind: "download" | "head" } ? StoredFile : O extends { kind: "exists" } @@ -1138,6 +1343,267 @@ const assertValidKey = (key: string, label = "key"): void => { } }; +const assertCanonicalStrongEtag: ( + etag: unknown, + label?: string +) => asserts etag is string = (etag, label = "etag") => { + if (typeof etag !== "string") { + throw new FilesError( + "Provider", + `${label} must be a canonical bare strong ETag`, + undefined, + { permanent: true } + ); + } + const invalidLength = etag.length === 0 || etag.length > 1024; + const invalidForm = etag === "*" || etag.startsWith("W/"); + let invalidCharacter = false; + if (!invalidLength) { + for (const character of etag) { + const code = character.codePointAt(0) ?? 0; + if (code < 0x21 || code > 0x7e || code === 0x22 || code === 0x2c) { + invalidCharacter = true; + break; + } + } + } + if (!(invalidLength || invalidForm || invalidCharacter)) { + return; + } + throw new FilesError( + "Provider", + `${label} must be a canonical bare strong ETag`, + undefined, + { permanent: true } + ); +}; + +const assertConditionalUploadResult: ( + result: unknown +) => asserts result is ConditionalUploadResult = (result) => { + if (result === null || typeof result !== "object" || !("etag" in result)) { + throw new FilesError( + "Provider", + "a conditional upload must return its new canonical strong ETag", + undefined, + { permanent: true } + ); + } + try { + assertCanonicalStrongEtag(result.etag); + } catch (error) { + throw new FilesError( + "Provider", + "a conditional upload must return its new canonical strong ETag", + error, + { permanent: true } + ); + } +}; + +const toAdapterUploadOptions = ( + opts: UploadOptions | undefined +): AdapterUploadOptions | undefined => { + if (!opts) { + return; + } + // oxlint-disable-next-line sonarjs/no-unused-vars -- destructure-omit keeps public-only conditional controls out of the adapter contract + const { + condition: _condition, + control: _control, + multipart: _multipart, + ...adapterOptions + } = opts; + return adapterOptions; +}; + +const toAdapterDownloadOptions = ( + opts: DownloadOptions | undefined +): AdapterDownloadOptions | undefined => { + if (!opts) { + return; + } + // oxlint-disable-next-line sonarjs/no-unused-vars -- condition is a first-class operation field, never an adapter option + const { condition: _condition, ...adapterOptions } = opts; + return adapterOptions; +}; + +// Strip only the predicate: `condition` is a first-class operation field, but +// every other option keeps flowing to plugins (`op.options`) and adapters +// exactly as it did before conditional operations existed, so a custom +// adapter or third-party plugin that reads its own extra field still sees it. +const toOperationOptions = ( + opts: (OperationOptions & { condition?: unknown }) | undefined +): OperationOptions | undefined => { + if (!opts) { + return; + } + // oxlint-disable-next-line sonarjs/no-unused-vars -- destructure-omit keeps the predicate out of the options bag + const { condition: _condition, ...operationOptions } = opts; + return operationOptions; +}; + +const invalidCondition = (operation: string): never => { + throw new FilesError( + "Provider", + `${operation} condition is malformed`, + undefined, + { permanent: true } + ); +}; + +const isConditionRecord = ( + value: unknown +): value is Record => + value !== null && typeof value === "object"; + +const snapshotUploadCondition = ( + condition: unknown +): UploadCondition | undefined => { + if (condition === undefined) { + return; + } + if (!isConditionRecord(condition)) { + return invalidCondition("upload"); + } + if (condition.type === "create") { + return { type: "create" }; + } + if (condition.type !== "replace") { + return invalidCondition("upload"); + } + assertCanonicalStrongEtag(condition.etag); + return { etag: condition.etag, type: "replace" }; +}; + +const snapshotEtagCondition = ( + condition: unknown, + operation: "delete" | "download" +): { etag: string } | undefined => { + if (condition === undefined) { + return; + } + if (!isConditionRecord(condition)) { + return invalidCondition(operation); + } + assertCanonicalStrongEtag(condition.etag); + return { etag: condition.etag }; +}; + +const snapshotCopyCondition = ( + condition: unknown +): CopyCondition | undefined => { + if (condition === undefined) { + return; + } + if (!isConditionRecord(condition)) { + return invalidCondition("copy"); + } + const { destination, source } = condition; + if (!(isConditionRecord(destination) && isConditionRecord(source))) { + return invalidCondition("copy"); + } + assertCanonicalStrongEtag(source.etag, "source etag"); + if (destination.type === "create") { + return { + destination: { type: "create" }, + source: { etag: source.etag }, + }; + } + if (destination.type !== "replace") { + return invalidCondition("copy"); + } + assertCanonicalStrongEtag(destination.etag, "destination etag"); + return { + destination: { etag: destination.etag, type: "replace" }, + source: { etag: source.etag }, + }; +}; + +const assertNoBulkCondition = ( + operation: string, + values: readonly unknown[] +): void => { + // `condition: undefined` is how a spread of shared options spells "absent" + // — treat it the same way the single-key snapshots do, not as a predicate. + const hasCondition = values.some( + (value) => + value !== null && + typeof value === "object" && + (value as { condition?: unknown }).condition !== undefined + ); + if (!hasCondition) { + return; + } + throw new FilesError( + "Provider", + `${operation} does not support conditional predicates`, + undefined, + { permanent: true } + ); +}; + +const assertConditionalUploadOptions = ( + opts: UploadOptions | undefined +): void => { + // `multipart: false` is the documented opt-out, not a request — only an + // actual multipart ask (or a resumable control) is incompatible. + if (opts?.control === undefined && !isMultipartRequested(opts?.multipart)) { + return; + } + throw new FilesError( + "Provider", + "conditional uploads do not support multipart or resumable controls", + undefined, + { permanent: true } + ); +}; + +const unreachableOperation = (_op: never): never => { + throw new FilesError("Provider", "unsupported Files operation", undefined, { + permanent: true, + }); +}; + +/** + * The identity of a conditional operation's predicate — family, mode, and + * every ETag — as one string, so a candidate a plugin hands to `next()` can + * be compared to the root in a single equality check. Plain serialization: + * the root's ETags were validated when they were snapshotted, and any change + * a plugin makes (including to an invalid value) shows up as a mismatch. + */ +const conditionalOperationFingerprint = ( + op: ConditionalFilesOperation +): string => { + switch (op.kind) { + case "upload": { + return op.mode === "replace" + ? JSON.stringify(["upload", "replace", op.etag]) + : JSON.stringify(["upload", "create"]); + } + case "download": { + return JSON.stringify(["download", "exact", op.etag]); + } + case "delete": { + return JSON.stringify(["delete", "match", op.etag]); + } + case "copy": { + return op.destination.type === "replace" + ? JSON.stringify([ + "copy", + "conditional", + op.source.etag, + "replace", + op.destination.etag, + ]) + : JSON.stringify(["copy", "conditional", op.source.etag, "create"]); + } + default: { + return unreachableOperation(op); + } + } +}; + // Compile a search pattern into a key predicate once, up front, so the per-key // cost during the walk is a single test/compare and an invalid regex throws // before any provider call. @@ -1227,6 +1693,7 @@ const normalizePrefix = (prefix: string | undefined): string => { */ interface ActionContext { type: FilesActionType; + condition?: ConditionalActionType; key?: string; keys?: string[]; from?: string; @@ -1238,6 +1705,21 @@ interface ActionContext { * error is swallowed, and the return value is ignored — hooks are * fire-and-forget, like {@link UploadOptions.onProgress}. */ +const isPromiseLike = (value: unknown): value is PromiseLike => + value !== null && + (typeof value === "object" || typeof value === "function") && + typeof (value as { then?: unknown }).then === "function"; + +const consumeHookOutcome = async ( + outcome: PromiseLike +): Promise => { + try { + await outcome; + } catch { + // Observability must not break the operation. + } +}; + const emitHook = ( hook: ((event: E) => void) | undefined, event: E @@ -1246,12 +1728,32 @@ const emitHook = ( return; } try { - hook(event); + const outcome = hook(event) as unknown; + if (isPromiseLike(outcome)) { + // Hooks are deliberately not awaited, but an async observer still needs + // a rejection handler so it cannot escape after the provider settles. + void consumeHookOutcome(outcome); + } } catch { // Observability must not break the operation. } }; +const bindFileUpload = ( + files: Files, + key: string +): FileHandle["upload"] => { + function upload( + body: Body, + opts: ConditionalUploadOptions + ): Promise; + function upload(body: Body, opts?: UploadOptions): Promise; + function upload(body: Body, opts?: UploadOptions): Promise { + return files.upload(key, body, opts); + } + return upload; +}; + export class Files { readonly #adapter: A; readonly #defaults: OperationOptions; @@ -1353,16 +1855,198 @@ export class Files { op: O, base: InternalNext ): Promise> { - if (this.#wraps.length === 0) { - return base(op) as Promise>; + if (!isConditionalOperation(op)) { + if (this.#wraps.length === 0) { + return base(op) as Promise>; + } + // The one guard an ordinary root needs sits innermost: whatever a plugin + // hands to `next()` ends up here before it can reach the provider, so a + // predicate introduced anywhere in the onion is rejected exactly once + // without a per-hop check on the hottest path (every bulk item). + const rejectIntroducedConditional: InternalNext = (nextOp) => { + if (isConditionalOperation(nextOp)) { + throw new FilesError( + "Provider", + "a plugin cannot introduce a conditional predicate into an ordinary operation", + undefined, + { permanent: true } + ); + } + return base(nextOp); + }; + // Fold from the innermost wrap outward so `plugins[0]` ends up outermost. + let chain: InternalNext = rejectIntroducedConditional; + for (const wrap of this.#wraps.toReversed()) { + const next = chain; + chain = (nextOp) => wrap(nextOp, next); + } + return chain(op) as Promise>; } - // Fold from the innermost wrap outward so `plugins[0]` ends up outermost. - let chain: InternalNext = base; + + return this.#dispatchConditional(op, base) as Promise>; + } + + /** + * Conditional operations are allowed through the same onion as ordinary + * calls, but their native predicate is a non-bypassable capability boundary. + * Every `next()` must retain it exactly, and a successful chain must settle + * exactly one invocation of the real operation. + */ + async #dispatchConditional( + op: ConditionalFilesOperation, + base: InternalNext + ): Promise { + const fingerprint = conditionalOperationFingerprint(op); + // Memoized by operation identity: a plugin that forwards the same object + // costs one string comparison, one that spreads a copy costs one + // serialization — either way, one check per `next()`. + const fingerprints = new WeakMap([[op, fingerprint]]); + const settlement = { + state: "idle" as "idle" | "pending" | "success" | "error", + }; + let violation: FilesError | undefined; + let baseFailure: unknown; + let postNextFailure: unknown; + let hasPostNextFailure = false; + let nativeUploadEtag: string | undefined; + + const rejectViolation = (message: string, cause?: unknown): never => { + const error = new FilesError("Provider", message, cause, { + permanent: true, + }); + violation ??= error; + throw error; + }; + const assertSamePredicate = (candidate: FilesOperation): void => { + if (!isConditionalOperation(candidate)) { + return rejectViolation( + "a plugin cannot remove or change a conditional operation predicate" + ); + } + let candidateFingerprint = fingerprints.get(candidate); + if (candidateFingerprint === undefined) { + try { + candidateFingerprint = conditionalOperationFingerprint(candidate); + } catch (error) { + return rejectViolation( + "a plugin produced an invalid conditional operation predicate", + error + ); + } + fingerprints.set(candidate, candidateFingerprint); + } + if (candidateFingerprint !== fingerprint) { + rejectViolation( + "a plugin cannot remove or change a conditional operation predicate" + ); + } + }; + + const guardedBase: InternalNext = async (nextOp) => { + if (settlement.state !== "idle") { + // The once-only rule is documented, but when the first call failed + // that failure is the interesting part — carry it as the cause so a + // retrying plugin's caller can still see why the provider said no. + rejectViolation( + settlement.state === "error" + ? "a plugin cannot invoke a conditional provider operation more than once; the first invocation failed (see cause)" + : "a plugin cannot invoke a conditional provider operation more than once", + settlement.state === "error" ? baseFailure : undefined + ); + } + settlement.state = "pending"; + try { + const result = await base(nextOp); + if (op.kind === "upload") { + assertConditionalUploadResult(result); + nativeUploadEtag = result.etag; + } + settlement.state = "success"; + return result; + } catch (error) { + settlement.state = "error"; + baseFailure = error; + throw error; + } + }; + + // Every `next()` a plugin calls passes through exactly one predicate + // check, on the candidate it hands over; the root itself was validated + // when it was snapshotted, so the outermost call needs none. + let chain: InternalNext = guardedBase; for (const wrap of this.#wraps.toReversed()) { const next = chain; - chain = (nextOp) => wrap(nextOp, next); + // eslint-disable-next-line no-loop-func -- each iteration captures its own block-scoped wrap/next pair + chain = async (nextOp) => { + try { + return await wrap(nextOp, (candidate) => { + assertSamePredicate(candidate); + return next(candidate); + }); + } catch (error) { + // An outer plugin may swallow an inner plugin's post-commit throw + // and hand back something else; remember the first such failure so + // the caller learns about the applied-but-unacknowledged outcome + // rather than a synthesized success. + if (settlement.state === "success" && !hasPostNextFailure) { + postNextFailure = error; + hasPostNextFailure = true; + } + throw error; + } + }; + } + + // Once the native call has committed, any rejection that follows — an + // awaited plugin throwing after `next()`, or a post-commit check failing + // below — describes an applied-but-unacknowledged mutation. Mark it so + // hooks, audit, and callers can tell it apart from a veto or a provider + // failure, and know to reconcile rather than retry the same predicate. + const applied = (error: unknown): FilesError => + FilesError.applied(error, nativeUploadEtag); + + let result: unknown; + try { + result = await chain(op); + } catch (error) { + throw settlement.state === "success" ? applied(error) : error; + } + if (settlement.state !== "success") { + // A latched violation is only decisive when nothing committed. Every + // violation is rejected *before* the guarded base runs, so a plugin + // that caught one and then called `next(op)` correctly has committed + // exactly one native request with the caller's predicate — reporting + // that as a failure would hide an applied write behind a misleading + // message. When no native call landed at all, the first violation is + // the most useful error to surface. + if (violation !== undefined) { + throw FilesError.wrap(violation); + } + rejectViolation( + "a plugin cannot synthesize a successful conditional operation" + ); + } + if (hasPostNextFailure) { + throw applied(postNextFailure); + } + if (op.kind === "upload") { + try { + assertConditionalUploadResult(result); + } catch (error) { + throw applied(error); + } + if (result.etag !== nativeUploadEtag) { + throw applied( + new FilesError( + "Provider", + "a plugin cannot replace the ETag returned by a conditional upload", + undefined, + { permanent: true } + ) + ); + } } - return chain(op) as Promise>; + return result; } /** @@ -1376,20 +2060,46 @@ export class Files { * plugin re-routes to, so a cross-kind sub-op behaves the same inside a bulk * call as in a single one. */ + // eslint-disable-next-line complexity -- exhaustive provider dispatch keeps every primitive in one auditable boundary #perform(op: FilesOperation): Promise { switch (op.kind) { case "upload": { - return this.#runUpload(op.key, op.body, op.options, { - key: op.key, - type: "upload", - }); + return this.#runUpload( + op.key, + op.body, + op.options, + { + ...(op.mode === "create" && { condition: "create" as const }), + ...(op.mode === "replace" && { condition: "replace" as const }), + key: op.key, + type: "upload", + }, + op.mode === "create" || op.mode === "replace" ? op : undefined + ); } case "download": { - const ctx: ActionContext = { key: op.key, type: "download" }; + const ctx: ActionContext = { + ...(op.mode === "exact" && { condition: "exact-read" }), + key: op.key, + type: "download", + }; const path = this.#path(op.key); if (op.options?.range) { this.#assertRangeSupported(op.options.range); } + if (op.mode === "exact") { + const exactRead = this.#adapter.conditional?.exactRead; + if (!exactRead) { + return this.#unsupportedConditional("exact reads"); + } + return this.#run( + op.options, + async (attemptOpts) => + this.#storedFile(await exactRead(path, op.etag, attemptOpts)), + true, + ctx + ); + } return this.#run( op.options, async (attemptOpts) => @@ -1420,8 +2130,24 @@ export class Files { ); } case "delete": { - const ctx: ActionContext = { key: op.key, type: "delete" }; + const ctx: ActionContext = { + ...(op.mode === "match" && { condition: "match-delete" }), + key: op.key, + type: "delete", + }; const path = this.#path(op.key); + if (op.mode === "match") { + const conditionalDelete = this.#adapter.conditional?.delete; + if (!conditionalDelete) { + return this.#unsupportedConditional("conditional deletes"); + } + return this.#run( + op.options, + (attemptOpts) => conditionalDelete(path, op.etag, attemptOpts), + true, + ctx + ); + } return this.#run( op.options, (attemptOpts) => this.#adapter.delete(path, attemptOpts), @@ -1430,9 +2156,43 @@ export class Files { ); } case "copy": { - const ctx: ActionContext = { from: op.from, to: op.to, type: "copy" }; + const ctx: ActionContext = { + ...(op.mode === "conditional" && { + condition: "conditional-copy" as const, + }), + from: op.from, + to: op.to, + type: "copy", + }; const fromPath = this.#path(op.from, "copy source"); const toPath = this.#path(op.to, "copy destination"); + if (op.mode === "conditional") { + const conditionalCopy = this.#adapter.conditional?.copy; + const supportsDestination = + op.destination.type === "create" + ? conditionalCopy?.destinationCreate === true + : conditionalCopy?.destinationReplace === true; + if ( + typeof conditionalCopy?.run !== "function" || + conditionalCopy.sourceEtag !== true || + conditionalCopy.atomicSourceDestination !== true || + !supportsDestination + ) { + return this.#unsupportedConditional("conditional copies"); + } + return this.#run( + op.options, + (attemptOpts) => + conditionalCopy.run( + fromPath, + toPath, + { destination: op.destination, source: op.source }, + attemptOpts + ), + true, + ctx + ); + } return this.#run( op.options, (attemptOpts) => this.#adapter.copy(fromPath, toPath, attemptOpts), @@ -1464,7 +2224,7 @@ export class Files { ctx ); } - default: { + case "signedUploadUrl": { const ctx: ActionContext = { key: op.key, type: "signedUploadUrl" }; const path = this.#path(op.key); return this.#run( @@ -1478,6 +2238,9 @@ export class Files { ctx ); } + default: { + return unreachableOperation(op); + } } } @@ -1609,6 +2372,7 @@ export class Files { op, provider: this.#adapter.name, ts, + ...(ctx.condition !== undefined && { condition: ctx.condition }), ...(typeof upload?.size === "number" && { bytes: upload.size }), ...(typeof upload?.etag === "string" && { etag: upload.etag }), ...(sha256 !== undefined && { sha256 }), @@ -1650,8 +2414,29 @@ export class Files { */ get capabilities(): AdapterCapabilities { const a = this.#adapter; + const conditionalCopy = a.conditional?.copy; + const nativeConditionalCopy = + typeof conditionalCopy?.run === "function" && + conditionalCopy.sourceEtag === true && + conditionalCopy.atomicSourceDestination === true; return { cacheControl: a.supportsCacheControl === true, + conditional: { + copy: { + atomicSourceDestination: nativeConditionalCopy, + destinationCreate: + nativeConditionalCopy && conditionalCopy.destinationCreate === true, + destinationReplace: + nativeConditionalCopy && + conditionalCopy.destinationReplace === true, + sourceEtag: nativeConditionalCopy, + }, + create: typeof a.conditional?.create === "function", + delete: typeof a.conditional?.delete === "function", + exactRead: typeof a.conditional?.exactRead === "function", + multipart: { create: false, replace: false }, + replace: typeof a.conditional?.replace === "function", + }, delimiter: a.supportsDelimiter === true, metadata: a.supportsMetadata === true, multipart: typeof a.resumableUpload === "function", @@ -1691,7 +2476,7 @@ export class Files { moveFrom: (sourceKey, opts) => this.move(sourceKey, key, opts), moveTo: (destinationKey, opts) => this.move(key, destinationKey, opts), signedUploadUrl: (opts) => this.signedUploadUrl(key, opts), - upload: (body, opts) => this.upload(key, body, opts), + upload: bindFileUpload(this, key), url: (opts) => this.url(key, opts), }; } @@ -1748,6 +2533,11 @@ export class Files { * Both forms honor the client's `prefix`; the array form reports the keys * the caller passed, not the internal prefixed paths. */ + upload( + key: string, + body: Body, + opts: ConditionalUploadOptions + ): Promise; upload(key: string, body: Body, opts?: UploadOptions): Promise; upload( items: UploadManyItem[], @@ -1763,7 +2553,10 @@ export class Files { const bulkOpts = bodyOrOpts as UploadManyOptions | undefined; return this.#writeAction( { keys: items.map((item) => item.key), type: "upload" }, - () => this.#uploadMany(items, bulkOpts) + () => { + assertNoBulkCondition("bulk upload", [bulkOpts, ...items]); + return this.#uploadMany(items, bulkOpts); + } ); } const body = bodyOrOpts as Body; @@ -1776,11 +2569,40 @@ export class Files { // consumer — or one that fails — never reads or hashes the body. return this.#writeAction( ctx, - () => - this.#dispatch( - { body, key: keyOrItems, kind: "upload", options: opts }, - (op) => this.#perform(op) - ), + () => { + const condition = snapshotUploadCondition(opts?.condition); + if (!condition) { + return this.#dispatch( + { body, key: keyOrItems, kind: "upload", options: opts }, + (op) => this.#perform(op) + ); + } + ctx.condition = condition.type; + assertConditionalUploadOptions(opts); + const options = toAdapterUploadOptions(opts); + return condition.type === "create" + ? this.#dispatch( + { + body, + key: keyOrItems, + kind: "upload", + mode: "create", + options, + }, + (op) => this.#perform(op) + ) + : this.#dispatch( + { + body, + etag: condition.etag, + key: keyOrItems, + kind: "upload", + mode: "replace", + options, + }, + (op) => this.#perform(op) + ); + }, () => this.#uploadSha256(body) ); } @@ -1798,23 +2620,77 @@ export class Files { key: string, body: Body, opts?: UploadOptions, - ctx?: ActionContext + ctx?: ActionContext, + conditional?: Extract ): Promise { this.#assertUploadOptionsSupported(opts); const path = this.#path(key); + if ( + conditional && + (opts?.control !== undefined || isMultipartRequested(opts?.multipart)) + ) { + throw new FilesError( + "Provider", + "conditional uploads do not support multipart or resumable controls", + undefined, + { permanent: true } + ); + } + const nativeConditionalUpload = (() => { + if (!conditional) { + return; + } + if (conditional.mode === "create") { + return this.#adapter.conditional?.create; + } + const replace = this.#adapter.conditional?.replace; + if (!replace) { + return; + } + return ( + uploadPath: string, + uploadBody: Body, + uploadOptions?: AdapterUploadOptions + ): Promise => + replace(uploadPath, uploadBody, conditional.etag, uploadOptions); + })(); + if (conditional && !nativeConditionalUpload) { + return this.#unsupportedConditional( + conditional.mode === "create" + ? "conditional creates" + : "conditional replaces" + ); + } if (opts?.control) { return this.#runResumable(path, body, opts, opts.control); } const isStream = body instanceof ReadableStream; const onProgress = opts?.onProgress; + const upload = async ( + uploadBody: Body, + attemptOpts: UploadOptions | undefined + ): Promise => { + if (!nativeConditionalUpload) { + return this.#uploadResult( + await this.#adapter.upload(path, uploadBody, attemptOpts) + ); + } + const result = this.#uploadResult( + await nativeConditionalUpload( + path, + uploadBody, + toAdapterUploadOptions(attemptOpts) + ) + ); + assertConditionalUploadResult(result); + return result; + }; + if (!onProgress || this.#adapter.reportsUploadProgress) { return this.#run( opts, - async (attemptOpts) => - this.#uploadResult( - await this.#adapter.upload(path, body, attemptOpts) - ), + (attemptOpts) => upload(body, attemptOpts), !isStream, ctx ); @@ -1838,10 +2714,7 @@ export class Files { ); return this.#run( rest, - async (attemptOpts) => - this.#uploadResult( - await this.#adapter.upload(path, tracked, attemptOpts) - ), + (attemptOpts) => upload(tracked, attemptOpts), false, ctx ); @@ -1858,9 +2731,7 @@ export class Files { return this.#run( rest, async (attemptOpts) => { - const result = this.#uploadResult( - await this.#adapter.upload(path, body, attemptOpts) - ); + const result = await upload(body, attemptOpts); const done = total ?? result.size; emitHook(onProgress, { loaded: done, total: done }); return result; @@ -1986,21 +2857,44 @@ export class Files { ): Promise { if (Array.isArray(keyOrKeys)) { const keys = keyOrKeys; - return this.#action({ keys, type: "download" }, () => - this.#downloadMany(keys, opts as DownloadManyOptions | undefined) - ); + return this.#action({ keys, type: "download" }, () => { + assertNoBulkCondition("bulk download", [opts]); + return this.#downloadMany( + keys, + opts as DownloadManyOptions | undefined + ); + }); } - const ctx: ActionContext = { key: keyOrKeys, type: "download" }; - return this.#action(ctx, () => - this.#dispatch( - { - key: keyOrKeys, - kind: "download", - options: opts as DownloadOptions | undefined, - }, + const downloadOptions = opts as DownloadOptions | undefined; + const ctx: ActionContext = { + key: keyOrKeys, + type: "download", + }; + return this.#action(ctx, () => { + const condition = snapshotEtagCondition( + downloadOptions?.condition, + "download" + ); + if (condition) { + ctx.condition = "exact-read"; + } + return this.#dispatch( + condition + ? { + etag: condition.etag, + key: keyOrKeys, + kind: "download", + mode: "exact", + options: toAdapterDownloadOptions(downloadOptions), + } + : { + key: keyOrKeys, + kind: "download", + options: toAdapterDownloadOptions(downloadOptions), + }, (op) => this.#perform(op) - ) - ); + ); + }); } /** @@ -2267,29 +3161,49 @@ export class Files { * Both forms honor the client's `prefix`; the array form reports the keys * the caller passed, not the internal prefixed paths. */ - delete(key: string, opts?: OperationOptions): Promise; + delete(key: string, opts?: DeleteOptions): Promise; delete(keys: string[], opts?: DeleteManyOptions): Promise; delete( key: string | string[], - opts?: OperationOptions | DeleteManyOptions + opts?: DeleteOptions | DeleteManyOptions ): Promise { if (Array.isArray(key)) { const keys = key; - return this.#writeAction({ keys, type: "delete" }, () => - this.#deleteMany(keys, opts as DeleteManyOptions | undefined) - ); + return this.#writeAction({ keys, type: "delete" }, () => { + assertNoBulkCondition("bulk delete", [opts]); + return this.#deleteMany(keys, opts as DeleteManyOptions | undefined); + }); } - const ctx: ActionContext & { type: "delete" } = { key, type: "delete" }; - return this.#writeAction(ctx, () => - this.#dispatch( - { - key, - kind: "delete", - options: opts as OperationOptions | undefined, - }, + const deleteOptions = opts as DeleteOptions | undefined; + const ctx: ActionContext & { type: "delete" } = { + key, + type: "delete", + }; + return this.#writeAction(ctx, () => { + const condition = snapshotEtagCondition( + deleteOptions?.condition, + "delete" + ); + if (condition) { + ctx.condition = "match-delete"; + } + return this.#dispatch( + condition + ? { + etag: condition.etag, + key, + kind: "delete", + mode: "match", + options: toOperationOptions(deleteOptions), + } + : { + key, + kind: "delete", + options: toOperationOptions(deleteOptions), + }, (op) => this.#perform(op) - ) - ); + ); + }); } async #deleteMany( @@ -2395,17 +3309,37 @@ export class Files { }; } - copy(from: string, to: string, opts?: OperationOptions): Promise { + copy(from: string, to: string, opts?: CopyOptions): Promise { const ctx: ActionContext & { type: "copy" } = { from, to, type: "copy", }; - return this.#writeAction(ctx, () => - this.#dispatch({ from, kind: "copy", options: opts, to }, (op) => - this.#perform(op) - ) - ); + return this.#writeAction(ctx, () => { + const condition = snapshotCopyCondition(opts?.condition); + if (condition) { + ctx.condition = "conditional-copy"; + } + return this.#dispatch( + condition + ? { + destination: condition.destination, + from, + kind: "copy", + mode: "conditional", + options: toOperationOptions(opts), + source: condition.source, + to, + } + : { + from, + kind: "copy", + options: toOperationOptions(opts), + to, + }, + (op) => this.#perform(op) + ); + }); } /** @@ -2605,6 +3539,7 @@ export class Files { ); } + // eslint-disable-next-line complexity -- retry, timeout, abort, and hook settlement deliberately share one attempt loop async #run( opts: O | undefined, fn: (opts: O | undefined) => Promise, @@ -2643,6 +3578,7 @@ export class Files { if (ctx && this.#hooks?.onRetry) { emitHook(this.#hooks.onRetry, { attempt: attempt + 1, + ...(ctx.condition !== undefined && { condition: ctx.condition }), delayMs, error: wrapped, from: ctx.from, @@ -2685,6 +3621,15 @@ export class Files { ); } + #unsupportedConditional(operation: string): never { + throw new FilesError( + "Provider", + `${this.#adapter.name}: ${operation} are not supported by this adapter`, + undefined, + { permanent: true } + ); + } + #assertDelimiterSupported(opts?: ListOptions): void { if (opts?.delimiter === undefined) { return; diff --git a/packages/files-sdk/src/internal/conditional.ts b/packages/files-sdk/src/internal/conditional.ts new file mode 100644 index 00000000..c5649b48 --- /dev/null +++ b/packages/files-sdk/src/internal/conditional.ts @@ -0,0 +1,31 @@ +import type { ConditionalFilesOperation } from "../index.js"; +import { FilesError } from "./errors.js"; + +/** + * Veto a conditional operation from inside a plugin's `wrap`, before any + * provider I/O. + * + * The dispatcher already fails closed on everything that crosses `next()` — + * a dropped or changed predicate, a rerouted verb, a second native call, a + * synthesized result. What it cannot see is a side effect a plugin performs + * on its own (a snapshot copy, a write to another backend, a pointer rewrite) + * that would be left uncoupled from the native compare-and-set. A plugin with + * that kind of out-of-band mutation must veto the modes it cannot make atomic; + * this is the one shape every bundled plugin uses for it, so the resulting + * `FilesError` is uniform: `Provider`-coded, `permanent`, and worded as + * `: conditional is unsupported because `. + * + * @throws {FilesError} always + */ +export const rejectConditional = ( + op: Pick, + plugin: string, + reason: string +): never => { + throw new FilesError( + "Provider", + `${plugin}: conditional ${op.kind} is unsupported because ${reason}`, + undefined, + { permanent: true } + ); +}; diff --git a/packages/files-sdk/src/internal/errors.ts b/packages/files-sdk/src/internal/errors.ts index 1f3538e1..2af320a4 100644 --- a/packages/files-sdk/src/internal/errors.ts +++ b/packages/files-sdk/src/internal/errors.ts @@ -26,6 +26,18 @@ export class FilesError extends Error { * failure out of that. */ readonly permanent: boolean; + /** + * `true` when a conditional mutation **did commit** at the provider before + * this error was raised — an awaited plugin threw after `next()` returned, + * or the committed result failed a post-commit check. The object changed + * (for an upload, to the generation in {@link appliedEtag}) even though the + * call rejected, so treat it as applied-but-unacknowledged: reconcile with + * an exact read rather than re-issuing the same predicate, which can only + * conflict now. Never set on a pre-commit veto or a provider failure. + */ + readonly applied: boolean; + /** The committed generation's ETag when {@link applied} is set on an upload. */ + readonly appliedEtag?: string; /** * The original provider error, preserved for debugging. * @@ -41,7 +53,13 @@ export class FilesError extends Error { code: FilesErrorCode, message: string, cause?: unknown, - opts?: { aborted?: boolean; timedOut?: boolean; permanent?: boolean } + opts?: { + aborted?: boolean; + timedOut?: boolean; + permanent?: boolean; + applied?: boolean; + appliedEtag?: string; + } ) { super(message); this.name = "FilesError"; @@ -49,9 +67,30 @@ export class FilesError extends Error { this.aborted = opts?.aborted === true; this.timedOut = opts?.timedOut === true; this.permanent = opts?.permanent === true; + this.applied = opts?.applied === true; + if (opts?.appliedEtag !== undefined) { + this.appliedEtag = opts.appliedEtag; + } this.cause = cause; } + /** + * Re-raise `err` as the outcome of a conditional mutation that already + * committed: same code, message, and flags, with {@link applied} set (and + * {@link appliedEtag} for uploads). The original error is kept as `cause` + * so nothing about the failure is lost. + */ + static applied(err: unknown, appliedEtag?: string): FilesError { + const wrapped = FilesError.wrap(err); + return new FilesError(wrapped.code, wrapped.message, err, { + aborted: wrapped.aborted, + applied: true, + ...(appliedEtag !== undefined && { appliedEtag }), + permanent: wrapped.permanent, + timedOut: wrapped.timedOut, + }); + } + static wrap( err: unknown, fallbackCode: FilesErrorCode = "Provider" diff --git a/packages/files-sdk/src/internal/receipts.ts b/packages/files-sdk/src/internal/receipts.ts index a985e180..300ad0a4 100644 --- a/packages/files-sdk/src/internal/receipts.ts +++ b/packages/files-sdk/src/internal/receipts.ts @@ -9,6 +9,14 @@ import type { Body } from "../index.js"; */ export type ReceiptOp = "upload" | "delete" | "copy" | "move"; +/** A redacted summary of the conditional primitive used by an action. */ +export type ConditionalActionType = + | "create" + | "replace" + | "exact-read" + | "match-delete" + | "conditional-copy"; + /** * The full receipt configuration, after normalizing the * `receipts?: boolean | { sha256?: boolean }` constructor option. `enabled` @@ -63,6 +71,8 @@ export const resolveReceiptsConfig = ( export interface Receipt { /** The mutating verb that produced this receipt. */ op: ReceiptOp; + /** The native conditional primitive, when this was a conditional mutation. */ + condition?: ConditionalActionType; /** The storage provider, from the adapter's `name` (e.g. `"s3"`, `"r2"`). */ provider: string; /** @@ -88,6 +98,7 @@ export interface Receipt { /** The action-derived inputs a {@link Receipt} is assembled from. */ export interface ReceiptInput { op: ReceiptOp; + condition?: ConditionalActionType; provider: string; key: string; bytes?: number; @@ -109,6 +120,7 @@ export const buildReceipt = (input: ReceiptInput): Receipt => ({ op: input.op, provider: input.provider, ts: input.ts, + ...(input.condition !== undefined && { condition: input.condition }), ...(input.bytes !== undefined && { bytes: input.bytes }), ...(input.etag !== undefined && { etag: input.etag }), ...(input.sha256 !== undefined && { sha256: input.sha256 }), diff --git a/packages/files-sdk/src/internal/resumable.ts b/packages/files-sdk/src/internal/resumable.ts index 4a0495f4..c78a693e 100644 --- a/packages/files-sdk/src/internal/resumable.ts +++ b/packages/files-sdk/src/internal/resumable.ts @@ -419,7 +419,12 @@ const resolveConcurrency = ( return concurrency && concurrency > 0 ? concurrency : DEFAULT_CONCURRENCY; }; -const reportProgress = ( +/** + * Deliver an upload progress event to an optional listener. Fire-and-forget: + * a throwing reporter can never fail (or, after a commit, retry) the upload + * it observes. Shared by the resumable driver and the S3 conditional path. + */ +export const reportProgress = ( onProgress: ((progress: UploadProgress) => void) | undefined, progress: UploadProgress ): void => { diff --git a/packages/files-sdk/src/s3/core.ts b/packages/files-sdk/src/s3/core.ts index b44b9801..a0343dda 100644 --- a/packages/files-sdk/src/s3/core.ts +++ b/packages/files-sdk/src/s3/core.ts @@ -5,9 +5,15 @@ import type * as RequestPresigner from "@aws-sdk/s3-request-presigner"; import type { Adapter, + AdapterDownloadOptions, + AdapterUploadOptions, + Body, + ConditionalUploadResult, + CopyCondition, DeleteManyOptions, DeleteManyResult, MultipartOptions, + OperationOptions, PartMeta, PartsResumableDriver, ResumableDriverOptions, @@ -15,6 +21,7 @@ import type { SignedUpload, StoredFile, UploadProgress, + UploadOptions, UploadResult, } from "../index.js"; import { @@ -31,6 +38,7 @@ import { readEnv } from "../internal/env.js"; import { FilesError } from "../internal/errors.js"; import type { ProviderFilesErrorCode } from "../internal/errors.js"; import { inferTypeFromName } from "../internal/mime.js"; +import { reportProgress } from "../internal/resumable.js"; import { createStoredFile } from "../internal/stored-file.js"; /** @@ -83,6 +91,28 @@ export interface S3AdapterOptions { * S3-compatible services and by LocalStack. */ forcePathStyle?: boolean; + /** + * Whether to expose the native conditional primitives (`If-Match` / + * `If-None-Match` create, replace, exact read, delete, and copy). + * + * Defaults to `true` only when the client will talk to canonical AWS S3: + * no `endpoint` here and no `AWS_ENDPOINT_URL_S3` / `AWS_ENDPOINT_URL` + * redirect in the environment. S3-compatible services differ in which + * conditional headers they honor, so the adapter fails closed for them + * rather than risk an unconditional overwrite. + * + * A shared-config `endpoint_url` (profile- or service-level) is invisible + * at construction, so it is caught at request time instead: a conditional + * request whose resolved hostname is not `amazonaws.com` fails closed + * before it is sent. AWS-hosted endpoints (VPC, FIPS, dual-stack, GovCloud) + * all resolve under that suffix and need no override. + * + * Set `true` to opt an S3-compatible endpoint that you have verified + * honors `If-Match` / `If-None-Match` in — this skips both the constructor + * check and the request-time hostname check — or `false` to disable the + * primitives on a canonical bucket. + */ + conditional?: boolean; /** * Static credentials. Skip to use the AWS credential chain (env vars, * IAM role, shared profile, EC2/ECS/EKS instance metadata). @@ -130,6 +160,172 @@ const stripEtag = (etag: string | undefined): string | undefined => { return etag.replaceAll(/^"+|"+$/gu, ""); }; +// Conditional requests accept one strong entity tag, not an HTTP list or +// wildcard. Keep the adapter boundary strict even when callers invoke the +// optional primitive directly instead of going through `Files` validation. +const CANONICAL_ETAG = /^(?!\*$)(?!W\/)[\u0021\u0023-\u002B\u002D-\u007E]+$/u; +const MAX_ETAG_LENGTH = 1024; + +const assertCanonicalEtag = (etag: string): string => { + if (etag.length > MAX_ETAG_LENGTH || !CANONICAL_ETAG.test(etag)) { + throw new FilesError( + "Provider", + "s3 adapter: conditional ETags must be canonical bare strong values", + undefined, + { permanent: true } + ); + } + return etag; +}; + +const quoteCanonicalEtag = (etag: string): string => + `"${assertCanonicalEtag(etag)}"`; + +function normalizeConditionalResponseEtag( + etag: string | undefined, + operation: string, + required: true +): string; +function normalizeConditionalResponseEtag( + etag: string | undefined, + operation: string, + required: false +): string | undefined; +function normalizeConditionalResponseEtag( + etag: string | undefined, + operation: string, + required: boolean +): string | undefined { + if (etag === undefined) { + if (required) { + throw new FilesError( + "Provider", + `S3 returned no ETag after ${operation}; the object may already have been committed`, + undefined, + { permanent: true } + ); + } + return; + } + const bare = + etag.length >= 2 && etag.startsWith('"') && etag.endsWith('"') + ? etag.slice(1, -1) + : etag; + try { + return assertCanonicalEtag(bare); + } catch (error) { + throw new FilesError( + "Provider", + `S3 returned an invalid ETag after ${operation}`, + error, + { permanent: true } + ); + } +} + +const abortOptions = (signal: AbortSignal | undefined) => + signal ? { abortSignal: signal } : undefined; + +// Each conditional input field and the wire header it must serialize to. A +// client whose model predates the field leaves the header out, so the guard +// compares the built request against the input rather than trusting the +// peer range. +const CONDITIONAL_HEADERS: readonly (readonly [string, string])[] = [ + ["IfMatch", "if-match"], + ["IfNoneMatch", "if-none-match"], + ["CopySourceIfMatch", "x-amz-copy-source-if-match"], +]; + +// Canonical AWS S3 hostnames: the only backends known to honor every +// conditional header. VPC / FIPS / dual-stack / GovCloud endpoints all live +// under these suffixes; S3-compatible services never do. +const AWS_HOST_SUFFIXES = ["amazonaws.com", "amazonaws.com.cn"]; +const isAwsHost = (hostname: string): boolean => { + const host = hostname.toLowerCase(); + return AWS_HOST_SUFFIXES.some( + (suffix) => host === suffix || host.endsWith(`.${suffix}`) + ); +}; + +/** + * Build-step middleware that keeps a conditional request from ever going out + * without its predicate on the wire, and — unless the caller opted in with + * `conditional: true` — from going anywhere other than AWS. It runs after + * serialization and endpoint resolution, so `args.request` is exactly what + * would be sent: the resolved hostname covers an `endpoint` option, an + * `AWS_ENDPOINT_URL*` variable, and a shared-config `endpoint_url` alike, + * none of which the synchronous constructor can see. Generic over the + * handler shapes so it slots into `middlewareStack.add` without pulling + * `@smithy/types` in as a dependency. + */ +const conditionalRequestGuard = + (allowAnyHost: boolean) => + ( + next: (args: Args) => Promise + ) => + (args: Args): Promise => { + const input = args.input as Record; + const expected = CONDITIONAL_HEADERS.filter( + ([field]) => input[field] !== undefined + ); + // The common case: no predicate on this command, nothing to scan. + if (expected.length === 0) { + return next(args); + } + const request = args.request as { + headers?: Record; + hostname?: string; + }; + if (!(allowAnyHost || isAwsHost(request.hostname ?? ""))) { + throw new FilesError( + "Provider", + `s3 adapter: conditional requests are only sent to AWS S3, but this client resolves to ${request.hostname ?? "an unknown host"}; pass \`conditional: true\` to opt an S3-compatible endpoint in`, + undefined, + { permanent: true } + ); + } + const sent = new Set( + Object.keys(request.headers ?? {}).map((name) => name.toLowerCase()) + ); + for (const [, header] of expected) { + if (!sent.has(header)) { + throw new FilesError( + "Provider", + `s3 adapter: the installed @aws-sdk/client-s3 did not serialize ${header}; conditional requests need 3.919.0 or newer`, + undefined, + { permanent: true } + ); + } + } + return next(args); + }; + +const assertConditionalUploadOptions = ( + options: AdapterUploadOptions | undefined +): void => { + // AdapterUploadOptions excludes both fields statically; retain a runtime + // fail-closed guard for direct JavaScript/structural calls. + const untrusted = options as + | (AdapterUploadOptions & Pick) + | undefined; + if (isMultipartRequested(untrusted?.multipart)) { + throw new FilesError( + "Provider", + "s3 adapter: conditional multipart uploads are not supported", + undefined, + { permanent: true } + ); + } + if (untrusted?.control !== undefined) { + throw new FilesError( + "Provider", + "s3 adapter: resumable upload control is not supported for conditional uploads", + undefined, + { permanent: true } + ); + } +}; + // `@aws-sdk/lib-storage` is an optional peer dependency, pulled in only when an // upload needs the multipart/progress path. Loaded lazily (the return type is // inferred from the dynamic import) so it isn't required by callers who only do @@ -409,6 +605,9 @@ const S3_NOT_FOUND_CODES: ReadonlySet = new Set([ ]); const S3_UNAUTH_CODES: ReadonlySet = new Set(["AccessDenied"]); const S3_CONFLICT_CODES: ReadonlySet = new Set(["PreconditionFailed"]); +const S3_RETRYABLE_CONDITIONAL_CONFLICT_CODES: ReadonlySet = new Set([ + "ConditionalRequestConflict", +]); // `DeleteObjects` rejects requests with more than 1000 keys, so the bulk path // has to chunk longer key lists into separate requests. const S3_DELETE_BATCH_LIMIT = 1000; @@ -431,8 +630,8 @@ const extractS3Error = ( }; }; -const buildMapS3Error = (providerLabel = "S3 error") => - makeErrorMapper({ +const buildMapS3Error = (providerLabel = "S3 error") => { + const mapDefault = makeErrorMapper({ codes: { conflict: S3_CONFLICT_CODES, notFound: S3_NOT_FOUND_CODES, @@ -441,6 +640,27 @@ const buildMapS3Error = (providerLabel = "S3 error") => extract: extractS3Error, providerLabel, }); + return (err: unknown): FilesError => { + if (err instanceof FilesError) { + return err; + } + const extracted = extractS3Error(err); + // Unlike PreconditionFailed (412), AWS documents this 409 as a transient + // race that clients should retry. Keep it Provider-coded so Files' retry + // policy can safely reissue the same native conditional request. + if ( + extracted.code && + S3_RETRYABLE_CONDITIONAL_CONFLICT_CODES.has(extracted.code) + ) { + return new FilesError( + "Provider", + extracted.message ?? providerLabel, + err + ); + } + return mapDefault(err); + }; +}; const _defaultMapS3Error = buildMapS3Error(); @@ -515,6 +735,20 @@ export const createS3Adapter = ( }; const client = new S3Client(config); + // A no-op unless the command carries a conditional input. Two gaps it + // closes at the last moment before the wire: the SDK-version gap (the peer + // floor is advisory, and a `@aws-sdk/client-s3` predating a conditional + // input field — CopyObject `IfMatch` / `IfNoneMatch` arrived in 3.919.0 — + // accepts the field and silently omits the header), and the endpoint gap + // (a shared-config `endpoint_url` redirects the client to a service whose + // conditional support is unknown, and only the resolved request shows it). + client.middlewareStack.add( + conditionalRequestGuard(opts.conditional === true), + { + name: "filesSdkConditionalRequestGuard", + step: "build", + } + ); const { bucket } = opts; const { publicBaseUrl } = opts; const defaultUrlExpiresIn = @@ -540,38 +774,270 @@ export const createS3Adapter = ( { expiresIn } ); - return { - bucket, - async copy(from, to, operationOpts) { - try { + // One request builder per verb, shared by the ordinary method and its + // conditional twin so a predicate is the *only* thing that differs on the + // wire — a header added, a metadata encoding fixed, a Content-Length + // fallback changed on one path reaches the other automatically. + const putParams = ( + key: string, + normalized: Awaited>, + options: AdapterUploadOptions | undefined + ) => ({ + Body: normalized.data, + Bucket: bucket, + ContentType: normalized.contentType, + Key: key, + ...(options?.cacheControl && { CacheControl: options.cacheControl }), + ...(options?.metadata && { Metadata: options.metadata }), + ...(normalized.contentLength !== undefined && { + ContentLength: normalized.contentLength, + }), + }); + + const getObject = ( + key: string, + downloadOpts: AdapterDownloadOptions | undefined, + predicate?: { IfMatch: string } + ) => + client.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + ...predicate, + // S3 replies 206 with ContentLength set to the slice length and a + // ranged body, so the size/byte handling below needs no special + // casing — the range just rides along on the GET. + ...(downloadOpts?.range && { + Range: httpRangeHeader(downloadOpts.range), + }), + }), + abortOptions(downloadOpts?.signal) + ); + + // Turn a GetObject response into a StoredFile: a lazy stream that trusts + // S3's ContentLength (falling back to 0 only if the header is missing, + // which is rare in practice), or a buffer whose size is the real byte + // length so what we surface always matches what the caller can read. + const toStoredFile = async ( + result: ClientS3.GetObjectCommandOutput, + meta: { etag: string | undefined; key: string }, + downloadOpts: AdapterDownloadOptions | undefined + ): Promise => { + const baseMeta = { + ...meta, + lastModified: result.LastModified?.getTime(), + metadata: result.Metadata, + type: result.ContentType ?? DEFAULT_CONTENT_TYPE, + }; + if (downloadOpts?.as === "stream") { + const stream = result.Body?.transformToWebStream(); + return createStoredFile( + { ...baseMeta, size: Number(result.ContentLength ?? 0) }, + { factory: () => stream ?? emptyStream(), kind: "stream" } + ); + } + const bytes = + (await result.Body?.transformToByteArray()) ?? new Uint8Array(); + return createStoredFile( + { ...baseMeta, size: bytes.byteLength }, + { data: bytes, kind: "buffer" } + ); + }; + + const copyObject = ( + from: string, + to: string, + operationOpts: OperationOptions | undefined, + predicate?: Pick< + ClientS3.CopyObjectCommandInput, + "CopySourceIfMatch" | "IfMatch" | "IfNoneMatch" + > + ) => + client.send( + new CopyObjectCommand({ + Bucket: bucket, // CopySource must be URL-encoded per // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html. // S3 bucket naming rules don't require encoding in practice, but we // encode both halves defensively in case a custom endpoint (e.g. // MinIO) accepts looser names. `Key:` is passed unencoded — the SDK // signs and serializes it as part of the request, not as a URL value. - await client.send( - new CopyObjectCommand({ - Bucket: bucket, - CopySource: `${encodeURIComponent(bucket)}/${encodeURIComponent(from)}`, - Key: to, - }), - operationOpts?.signal - ? { abortSignal: operationOpts.signal } - : undefined + CopySource: `${encodeURIComponent(bucket)}/${encodeURIComponent(from)}`, + Key: to, + ...predicate, + }), + abortOptions(operationOpts?.signal) + ); + + const deleteObject = ( + key: string, + operationOpts: OperationOptions | undefined, + predicate?: { IfMatch: string } + ) => + client.send( + new DeleteObjectCommand({ Bucket: bucket, Key: key, ...predicate }), + abortOptions(operationOpts?.signal) + ); + + const conditionalUpload = async ( + key: string, + body: Body, + condition: { type: "create" } | { type: "replace"; etag: string }, + options?: AdapterUploadOptions + ): Promise => { + assertConditionalUploadOptions(options); + const predicate = + condition.type === "create" + ? { IfNoneMatch: "*" } + : { IfMatch: quoteCanonicalEtag(condition.etag) }; + const normalized = await normalizeBody(body, options?.contentType); + // `normalizeBody` never sizes a stream, so this rejects every stream + // body: PutObject with a predicate needs Content-Length up front, and + // there is no multipart fallback for conditional writes. + if ( + normalized.data instanceof ReadableStream && + normalized.contentLength === undefined + ) { + throw new FilesError( + "Provider", + "s3 adapter: conditional uploads do not accept stream bodies; buffer to a Blob or Uint8Array first", + undefined, + { permanent: true } + ); + } + const total = normalized.contentLength ?? 0; + reportProgress(options?.onProgress, { loaded: 0, total }); + try { + const result = await client.send( + new PutObjectCommand({ + ...putParams(key, { ...normalized, contentLength: total }, options), + ...predicate, + }), + abortOptions(options?.signal) + ); + const etag = normalizeConditionalResponseEtag( + result.ETag, + "a conditional upload", + true + ); + reportProgress(options?.onProgress, { loaded: total, total }); + return { contentType: normalized.contentType, etag, key, size: total }; + } catch (error) { + throw wrapErr(error); + } + }; + + const conditionalDownload = async ( + key: string, + etag: string, + downloadOpts?: AdapterDownloadOptions + ): Promise => { + const canonicalEtag = assertCanonicalEtag(etag); + try { + const result = await getObject(key, downloadOpts, { + IfMatch: quoteCanonicalEtag(canonicalEtag), + }); + const responseEtag = normalizeConditionalResponseEtag( + result.ETag, + "an exact read", + false + ); + if (responseEtag !== undefined && responseEtag !== canonicalEtag) { + throw new FilesError( + "Provider", + "S3 returned an ETag that did not match the exact-read predicate", + undefined, + { permanent: true } ); + } + // S3 validated If-Match even when a test double/proxy omits ETag from + // the response, so retain the exact canonical predicate as provenance. + return await toStoredFile( + result, + { etag: responseEtag ?? canonicalEtag, key }, + downloadOpts + ); + } catch (error) { + throw wrapErr(error); + } + }; + + // Canonical AWS only, unless the caller says otherwise: an explicit + // `endpoint` or an `AWS_ENDPOINT_URL*` redirect (which `S3Client` honors + // on its own) both point at a service whose conditional-header support is + // unknown, so the primitives stay off and every conditional call fails + // closed before provider I/O. + const nativeConditional = + opts.conditional ?? + (opts.endpoint === undefined && + readEnv("AWS_ENDPOINT_URL_S3") === undefined && + readEnv("AWS_ENDPOINT_URL") === undefined); + const conditional: S3Adapter["conditional"] = nativeConditional + ? { + copy: { + atomicSourceDestination: true, + destinationCreate: true, + destinationReplace: true, + async run( + from: string, + to: string, + condition: CopyCondition, + operationOpts + ): Promise { + const destinationPredicate = + condition.destination.type === "create" + ? { IfNoneMatch: "*" } + : { + IfMatch: quoteCanonicalEtag(condition.destination.etag), + }; + try { + await copyObject(from, to, operationOpts, { + CopySourceIfMatch: quoteCanonicalEtag(condition.source.etag), + ...destinationPredicate, + }); + } catch (error) { + throw wrapErr(error); + } + }, + sourceEtag: true, + }, + create(key, body, options) { + return conditionalUpload(key, body, { type: "create" }, options); + }, + async delete(key, etag, operationOpts): Promise { + try { + await deleteObject(key, operationOpts, { + IfMatch: quoteCanonicalEtag(etag), + }); + } catch (error) { + throw wrapErr(error); + } + }, + exactRead: conditionalDownload, + replace(key, body, etag, options) { + return conditionalUpload( + key, + body, + { etag, type: "replace" }, + options + ); + }, + } + : undefined; + + return { + bucket, + ...(conditional && { conditional }), + async copy(from, to, operationOpts) { + try { + await copyObject(from, to, operationOpts); } catch (error) { throw wrapErr(error); } }, async delete(key, operationOpts) { try { - await client.send( - new DeleteObjectCommand({ Bucket: bucket, Key: key }), - operationOpts?.signal - ? { abortSignal: operationOpts.signal } - : undefined - ); + await deleteObject(key, operationOpts); } catch (error) { throw wrapErr(error); } @@ -647,47 +1113,11 @@ export const createS3Adapter = ( }, async download(key, downloadOpts) { try { - const result = await client.send( - new GetObjectCommand({ - Bucket: bucket, - Key: key, - // S3 replies 206 with ContentLength set to the slice length and a - // ranged body, so the size/byte handling below needs no special - // casing — the range just rides along on the GET. - ...(downloadOpts?.range && { - Range: httpRangeHeader(downloadOpts.range), - }), - }), - downloadOpts?.signal - ? { abortSignal: downloadOpts.signal } - : undefined - ); - const baseMeta = { - etag: stripEtag(result.ETag), - key, - lastModified: result.LastModified?.getTime(), - metadata: result.Metadata, - type: result.ContentType ?? DEFAULT_CONTENT_TYPE, - }; - if (downloadOpts?.as === "stream") { - const stream = result.Body?.transformToWebStream(); - // Stream path: we trust S3's ContentLength header. Falls back to 0 - // only if the header is missing, which is rare in practice. - return createStoredFile( - { ...baseMeta, size: Number(result.ContentLength ?? 0) }, - { - factory: () => stream ?? emptyStream(), - kind: "stream", - } - ); - } - const bytes = - (await result.Body?.transformToByteArray()) ?? new Uint8Array(); - // Buffer path: prefer the real byte length over ContentLength so the - // size we surface always matches the bytes the caller can actually read. - return createStoredFile( - { ...baseMeta, size: bytes.byteLength }, - { data: bytes, kind: "buffer" } + const result = await getObject(key, downloadOpts); + return await toStoredFile( + result, + { etag: stripEtag(result.ETag), key }, + downloadOpts ); } catch (error) { throw wrapErr(error); @@ -858,21 +1288,10 @@ export const createS3Adapter = ( // `copy()` issues a CopyObject — server-side, no body round-trip. supportsServerSideCopy: true, async upload(key, body, options) { - const { cacheControl, metadata, multipart, onProgress, signal } = - options ?? {}; - const { data, contentType, contentLength } = await normalizeBody( - body, - options?.contentType - ); - const params = { - Body: data, - Bucket: bucket, - ContentType: contentType, - Key: key, - ...(cacheControl && { CacheControl: cacheControl }), - ...(metadata && { Metadata: metadata }), - ...(contentLength !== undefined && { ContentLength: contentLength }), - }; + const { multipart, onProgress, signal } = options ?? {}; + const normalized = await normalizeBody(body, options?.contentType); + const { data, contentType, contentLength } = normalized; + const params = putParams(key, normalized, options); // lib-storage's Upload is the path for explicit multipart, for progress // reporting, and for unknown-length streams — a single PutObject can't // reliably send a stream without a Content-Length, so auto-engage there. diff --git a/packages/files-sdk/src/soft-delete/index.ts b/packages/files-sdk/src/soft-delete/index.ts index 245b0614..c908ce0a 100644 --- a/packages/files-sdk/src/soft-delete/index.ts +++ b/packages/files-sdk/src/soft-delete/index.ts @@ -1,3 +1,4 @@ +import { isConditionalOperation, rejectConditional } from "../index.js"; import type { Files, FilesOperation, @@ -235,10 +236,22 @@ export const softDelete = ( switch (op.kind) { case "delete": { // A delete inside the trash is a real delete — this is how `purge()` - // and any manual trash cleanup actually remove bytes. + // and any manual trash cleanup actually remove bytes. It is forwarded + // unchanged, so a conditional one keeps its native compare-and-set + // (useful for purging one trashed generation atomically against a + // concurrent restore). if (isTrashKey(op.key)) { return next(op); } + // Outside the trash a delete becomes a move, and no single native + // predicate spans that copy + delete, so the mode is vetoed. + if (isConditionalOperation(op)) { + rejectConditional( + op, + "soft-delete", + "trash routing cannot preserve the native compare-and-set" + ); + } try { // Thread the caller's options through — the re-routed move IS the // user's delete, so its `signal`/`timeout`/`retries` must apply. diff --git a/packages/files-sdk/src/tiering/index.ts b/packages/files-sdk/src/tiering/index.ts index 6ae7b6ea..d9868437 100644 --- a/packages/files-sdk/src/tiering/index.ts +++ b/packages/files-sdk/src/tiering/index.ts @@ -1,4 +1,4 @@ -import { Files } from "../index.js"; +import { Files, isConditionalOperation, rejectConditional } from "../index.js"; import type { Adapter, Body, @@ -677,8 +677,16 @@ export const tiering = (options: TieringOptions): FilesPlugin => { } }; - const wrap = ((op: FilesOperation, next: PluginNext): Promise => - dispatch(runnerViaNext(next), op)) as NonNullable; + const wrap = ((op: FilesOperation, next: PluginNext): Promise => { + if (isConditionalOperation(op)) { + rejectConditional( + op, + "tiering", + "cross-tier routing cannot preserve one native compare-and-set" + ); + } + return dispatch(runnerViaNext(next), op); + }) as NonNullable; return { extend: (files) => { diff --git a/packages/files-sdk/src/tracing/index.ts b/packages/files-sdk/src/tracing/index.ts index 1ba2e18f..1bf23c1a 100644 --- a/packages/files-sdk/src/tracing/index.ts +++ b/packages/files-sdk/src/tracing/index.ts @@ -1,7 +1,10 @@ import { SpanStatusCode, trace } from "@opentelemetry/api"; import type { Attributes, Span, Tracer } from "@opentelemetry/api"; +import { isConditionalOperation } from "../index.js"; import type { + ConditionalActionType, + ConditionalFilesOperation, FilesOperation, FilesPlugin, ListResult, @@ -16,6 +19,21 @@ const DEFAULT_SPAN_PREFIX = "files."; /** Instrumentation name used for the default tracer. */ const INSTRUMENTATION_NAME = "files-sdk"; +const conditionalAction = ( + op: ConditionalFilesOperation +): ConditionalActionType => { + if (op.kind === "upload") { + return op.mode; + } + if (op.kind === "download") { + return "exact-read"; + } + if (op.kind === "delete") { + return "match-delete"; + } + return "conditional-copy"; +}; + export interface TracingOptions { /** * The tracer spans are created on. Defaults to @@ -44,6 +62,9 @@ export interface TracingOptions { /** Caller-facing attributes known before the operation runs. */ const baseAttributes = (op: FilesOperation): Attributes => { const attributes: Attributes = { "files.operation": op.kind }; + if (isConditionalOperation(op)) { + attributes["files.condition"] = conditionalAction(op); + } if (op.kind === "copy" || op.kind === "move") { attributes["files.from"] = op.from; attributes["files.to"] = op.to; diff --git a/packages/files-sdk/src/versioning/index.ts b/packages/files-sdk/src/versioning/index.ts index 4d2f1b6d..cd733843 100644 --- a/packages/files-sdk/src/versioning/index.ts +++ b/packages/files-sdk/src/versioning/index.ts @@ -1,3 +1,4 @@ +import { isConditionalOperation, rejectConditional } from "../index.js"; import type { Files, FilesOperation, @@ -368,6 +369,13 @@ export const versioning = ( op: FilesOperation, next: PluginNext ): Promise => { + if (isConditionalOperation(op) && op.kind !== "download") { + rejectConditional( + op, + "versioning", + "snapshot side effects cannot be coupled to the native compare-and-set" + ); + } switch (op.kind) { case "upload": case "delete": { diff --git a/packages/files-sdk/test/audit.test.ts b/packages/files-sdk/test/audit.test.ts index 6f06edcf..3bc52820 100644 --- a/packages/files-sdk/test/audit.test.ts +++ b/packages/files-sdk/test/audit.test.ts @@ -2,8 +2,12 @@ import { describe, expect, test } from "bun:test"; import { audit } from "../src/audit/index.js"; import type { AuditOptions, AuditRecord } from "../src/audit/index.js"; -import { createFiles, Files } from "../src/index.js"; -import type { FilesPlugin } from "../src/index.js"; +import { createFiles, Files, FilesError } from "../src/index.js"; +import type { + ConditionalFilesOperation, + FilesPlugin, + PluginNext, +} from "../src/index.js"; import { memory } from "../src/memory/index.js"; const bytes = (data: string): Uint8Array => new TextEncoder().encode(data); @@ -266,4 +270,95 @@ describe("audit", () => { await files.upload("a.txt", bytes("hi")); expect(records).toHaveLength(1); }); + + test("records terminal outcomes for conditional operations", async () => { + const records: AuditRecord[] = []; + const plugin = audit({ + events: "all", + sink: (record) => { + records.push(record); + }, + }); + const { wrap } = plugin; + if (!wrap) { + throw new Error("audit wrap missing"); + } + const create: Extract< + ConditionalFilesOperation, + { kind: "upload"; mode: "create" } + > = { + body: bytes("hello"), + key: "a.txt", + kind: "upload", + mode: "create", + }; + await wrap(create, (() => + Promise.resolve({ + contentType: "text/plain", + etag: "etag-1", + key: "a.txt", + size: 5, + })) as PluginNext); + const exact: Extract = { + etag: "stale-etag", + key: "a.txt", + kind: "download", + mode: "exact", + }; + await wrap(exact, (() => + Promise.reject( + new FilesError("Conflict", "stale ETag") + )) as PluginNext).catch(() => {}); + const matchedDelete: Extract< + ConditionalFilesOperation, + { kind: "delete" } + > = { + etag: "etag-1", + key: "delete.txt", + kind: "delete", + mode: "match", + }; + await wrap(matchedDelete, (() => Promise.resolve()) as PluginNext); + const conditionalCopy: Extract< + ConditionalFilesOperation, + { kind: "copy" } + > = { + destination: { type: "create" }, + from: "a.txt", + kind: "copy", + mode: "conditional", + source: { etag: "etag-1" }, + to: "copy.txt", + }; + await wrap(conditionalCopy, (() => Promise.resolve()) as PluginNext); + + expect(records).toHaveLength(4); + expect(records[0]).toMatchObject({ + action: "upload", + condition: "create", + key: "a.txt", + size: 5, + status: "success", + }); + expect(records[1]).toMatchObject({ + action: "download", + condition: "exact-read", + error: { code: "Conflict", message: "stale ETag" }, + key: "a.txt", + status: "error", + }); + expect(records[2]).toMatchObject({ + action: "delete", + condition: "match-delete", + key: "delete.txt", + status: "success", + }); + expect(records[3]).toMatchObject({ + action: "copy", + condition: "conditional-copy", + from: "a.txt", + status: "success", + to: "copy.txt", + }); + }); }); diff --git a/packages/files-sdk/test/cache.test.ts b/packages/files-sdk/test/cache.test.ts index bb0a000f..9c1a352a 100644 --- a/packages/files-sdk/test/cache.test.ts +++ b/packages/files-sdk/test/cache.test.ts @@ -6,12 +6,14 @@ import type { CacheRecord, CacheStore, } from "../src/cache/index.js"; -import { createFiles } from "../src/index.js"; +import { createFiles, createStoredFile, FilesError } from "../src/index.js"; import type { Adapter, + ConditionalFilesOperation, DownloadOptions, Files, OperationOptions, + PluginNext, UploadOptions, UrlOptions, } from "../src/index.js"; @@ -238,9 +240,233 @@ describe("cache plugin — download", () => { // The ranged read and the full read each hit the provider once. expect(calls.download).toEqual(["a.txt", "a.txt"]); }); + + test("never serves or populates an exact read from the ordinary cache", async () => { + const plugin = cache({ operations: ["download"] }); + const { wrap } = plugin; + if (!wrap) { + throw new Error("cache wrap missing"); + } + let nextCalls = 0; + const next = (() => { + nextCalls += 1; + const body = `version-${nextCalls}`; + return Promise.resolve( + createStoredFile( + { + etag: "etag-1", + key: "a.txt", + size: body.length, + type: "text/plain", + }, + { data: new TextEncoder().encode(body), kind: "buffer" } + ) + ); + }) as PluginNext; + const operation: Extract = + { + etag: "etag-1", + key: "a.txt", + kind: "download", + mode: "exact", + }; + + const first = await wrap(operation, next); + const second = await wrap(operation, next); + + expect(await first.text()).toBe("version-1"); + expect(await second.text()).toBe("version-2"); + expect(nextCalls).toBe(2); + }); }); describe("cache plugin — invalidation", () => { + test("a failed write still invalidates, so a stale ETag cannot poison a CAS retry loop", async () => { + const inner = fakeAdapter(); + let generation = 0; + const conditional: Adapter = { + ...inner, + conditional: { + // Always reject: the caller's ETag is stale as far as the provider is + // concerned, which is exactly the case where a cached head() must not + // keep serving that ETag. + replace: () => + Promise.reject(new FilesError("Conflict", "precondition failed")), + }, + head: async (key, opts) => { + generation += 1; + const meta = await inner.head(key, opts); + return { ...meta, etag: `gen-${generation}` }; + }, + }; + const files = createFiles({ + adapter: conditional, + plugins: [cache({ ttl: 60_000 })], + }); + await files.upload("doc.txt", "v1"); + const first = await files.head("doc.txt"); + // Served from the cache. + const again = await files.head("doc.txt"); + expect(again.etag).toBe(first.etag); + + await expect( + files.upload("doc.txt", "v2", { + condition: { etag: first.etag as string, type: "replace" }, + }) + ).rejects.toMatchObject({ code: "Conflict" }); + + // The conflict proved the cached record stale; the next head() must go to + // the provider rather than replaying the ETag that just conflicted. + const refreshed = await files.head("doc.txt"); + expect(refreshed.etag).not.toBe(first.etag); + }); + + test("a rejected exact read invalidates the cached record too", async () => { + const inner = fakeAdapter(); + let generation = 0; + const conditional: Adapter = { + ...inner, + conditional: { + exactRead: () => + Promise.reject(new FilesError("Conflict", "precondition failed")), + }, + head: async (key, opts) => { + generation += 1; + const meta = await inner.head(key, opts); + return { ...meta, etag: `gen-${generation}` }; + }, + }; + const files = createFiles({ + adapter: conditional, + plugins: [cache({ ttl: 60_000 })], + }); + await files.upload("doc.txt", "v1"); + const first = await files.head("doc.txt"); + await expect( + files.download("doc.txt", { condition: { etag: first.etag as string } }) + ).rejects.toMatchObject({ code: "Conflict" }); + const refreshed = await files.head("doc.txt"); + expect(refreshed.etag).not.toBe(first.etag); + }); + + test("a conditional copy that conflicts on its source invalidates the cached source", async () => { + const inner = fakeAdapter(); + let generation = 0; + const conditional: Adapter = { + ...inner, + conditional: { + copy: { + atomicSourceDestination: true, + destinationCreate: true, + destinationReplace: true, + run: () => + Promise.reject(new FilesError("Conflict", "source ETag mismatch")), + sourceEtag: true, + }, + }, + head: async (key, opts) => { + generation += 1; + const meta = await inner.head(key, opts); + return { ...meta, etag: `gen-${generation}` }; + }, + }; + const files = createFiles({ + adapter: conditional, + plugins: [cache({ ttl: 60_000 })], + }); + await files.upload("src.txt", "v1"); + const source = await files.head("src.txt"); + await expect( + files.copy("src.txt", "dst.txt", { + condition: { + destination: { type: "create" }, + source: { etag: source.etag as string }, + }, + }) + ).rejects.toMatchObject({ code: "Conflict" }); + // The source's cached ETag is what just conflicted; it must not replay. + const refreshed = await files.head("src.txt"); + expect(refreshed.etag).not.toBe(source.etag); + }); + + test("a store failure while invalidating after a conflict never masks the conflict", async () => { + const inner = fakeAdapter(); + const conditional: Adapter = { + ...inner, + conditional: { + replace: () => + Promise.reject(new FilesError("Conflict", "precondition failed")), + }, + }; + const deletes: string[] = []; + const memory = new Map(); + const store: CacheStore = { + clear: () => memory.clear(), + delete: (key) => { + deletes.push(key); + return Promise.reject(new Error("redis: connection reset")); + }, + get: (key) => Promise.resolve(memory.get(key)), + set: (key, entry) => { + memory.set(key, entry); + return Promise.resolve(); + }, + }; + const files = createFiles({ + adapter: conditional, + plugins: [cache({ store })], + }); + await expect(files.upload("doc.txt", "v1")).rejects.toThrow( + /connection reset/u + ); + const { etag } = await files.head("doc.txt"); + await expect( + files.upload("doc.txt", "v2", { + condition: { + etag: (etag as string).replaceAll('"', ""), + type: "replace", + }, + }) + ).rejects.toMatchObject({ code: "Conflict" }); + // The invalidation was attempted, but its failure stayed out of the way. + expect(deletes.filter((key) => key === "doc.txt").length).toBeGreaterThan( + 1 + ); + }); + + test("a plain failed write leaves the cache alone and costs no store round-trip", async () => { + const inner = fakeAdapter(); + const flaky: Adapter = { + ...inner, + delete: () => Promise.reject(new Error("delete boom")), + }; + const deletes: string[] = []; + const memory = new Map(); + const store: CacheStore = { + clear: () => memory.clear(), + delete: (key) => { + deletes.push(key); + memory.delete(key); + return Promise.resolve(); + }, + get: (key) => Promise.resolve(memory.get(key)), + set: (key, entry) => { + memory.set(key, entry); + return Promise.resolve(); + }, + }; + const { adapter, calls } = counting(flaky); + const files = createFiles({ adapter, plugins: [cache({ store })] }); + await files.upload("a.txt", "hello"); + await files.head("a.txt"); + deletes.length = 0; + + await expect(files.delete("a.txt")).rejects.toThrow(/delete boom/u); + expect(deletes).toEqual([]); + await files.head("a.txt"); + expect(calls.head).toEqual(["a.txt"]); + }); + test("upload invalidates the cached read", async () => { const { adapter, calls } = counting(); const files = withCache({}, adapter); diff --git a/packages/files-sdk/test/cli-commands.test.ts b/packages/files-sdk/test/cli-commands.test.ts index 19aaa758..31774bcd 100644 --- a/packages/files-sdk/test/cli-commands.test.ts +++ b/packages/files-sdk/test/cli-commands.test.ts @@ -601,6 +601,163 @@ describe("cli/commands real (fs adapter)", () => { }); }); +describe("cli/commands conditional flags", () => { + test("upload / download / delete / copy dry-runs echo the resolved condition", async () => { + await runUpload({ + ...baseOpts({ dryRun: true }), + file: "./local.txt", + ifNoneMatch: true, + key: "k", + }); + expect(lastJson(cap.stdout).condition).toEqual({ type: "create" }); + cap.stdout.length = 0; + + await runUpload({ + ...baseOpts({ dryRun: true }), + file: "./local.txt", + ifMatch: "abc", + key: "k", + }); + expect(lastJson(cap.stdout).condition).toEqual({ + etag: "abc", + type: "replace", + }); + cap.stdout.length = 0; + + await runDownload({ + ...baseOpts({ dryRun: true }), + ifMatch: "abc", + keys: ["k"], + out: "./k", + }); + expect(lastJson(cap.stdout).condition).toEqual({ etag: "abc" }); + cap.stdout.length = 0; + + await runDelete({ + ...baseOpts({ dryRun: true }), + ifMatch: "abc", + keys: ["k"], + }); + expect(lastJson(cap.stdout).condition).toEqual({ etag: "abc" }); + cap.stdout.length = 0; + + await runCopy({ + ...baseOpts({ dryRun: true }), + from: "a", + ifMatch: "src", + ifNoneMatch: true, + to: "b", + }); + expect(lastJson(cap.stdout).condition).toEqual({ + destination: { type: "create" }, + source: { etag: "src" }, + }); + cap.stdout.length = 0; + + await runCopy({ + ...baseOpts({ dryRun: true }), + destIfMatch: "dst", + from: "a", + ifMatch: "src", + to: "b", + }); + expect(lastJson(cap.stdout).condition).toEqual({ + destination: { etag: "dst", type: "replace" }, + source: { etag: "src" }, + }); + }); + + test("contradictory, partial, and bulk conditional flags are rejected before any I/O", async () => { + await expect( + runUpload({ + ...baseOpts({ dryRun: true }), + file: "./x", + ifMatch: "abc", + ifNoneMatch: true, + key: "k", + }) + ).rejects.toThrow(/mutually exclusive/u); + await expect( + runUpload({ ...baseOpts({ dryRun: true }), dir: root, ifNoneMatch: true }) + ).rejects.toThrow(/single key/u); + await expect( + runDownload({ + ...baseOpts({ dryRun: true }), + ifMatch: "abc", + keys: ["a", "b"], + outDir: root, + }) + ).rejects.toThrow(/single key/u); + await expect( + runDelete({ + ...baseOpts({ dryRun: true }), + ifMatch: "abc", + keys: ["a", "b"], + }) + ).rejects.toThrow(/single key/u); + // A conditional copy needs both halves. + await expect( + runCopy({ + ...baseOpts({ dryRun: true }), + from: "a", + ifMatch: "src", + to: "b", + }) + ).rejects.toThrow(/--if-none-match or --dest-if-match/u); + await expect( + runCopy({ + ...baseOpts({ dryRun: true }), + from: "a", + ifNoneMatch: true, + to: "b", + }) + ).rejects.toThrow(/--if-match/u); + await expect( + runCopy({ + ...baseOpts({ dryRun: true }), + destIfMatch: "dst", + from: "a", + ifMatch: "src", + ifNoneMatch: true, + to: "b", + }) + ).rejects.toThrow(/mutually exclusive/u); + expect(cap.stdout).toHaveLength(0); + }); + + test("a condition reaches the SDK and fails closed on an adapter without native support", async () => { + // The fs adapter has no conditional primitives, so every predicate must + // surface as the SDK's fail-closed error — proof the flag was forwarded + // rather than dropped into an unconditional call. + const local = path.join(root, "in.txt"); + await fsp.writeFile(local, "payload"); + await expect( + runUpload({ ...baseOpts(), file: local, ifNoneMatch: true, key: "k" }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect(fsp.stat(path.join(root, "k"))).rejects.toThrow(); + + await runUpload({ ...baseOpts(), file: local, key: "k" }); + await expect( + runDownload({ ...baseOpts(), ifMatch: "abc", keys: ["k"], stdout: true }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + runDelete({ ...baseOpts(), ifMatch: "abc", keys: ["k"] }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + runCopy({ + ...baseOpts(), + from: "k", + ifMatch: "abc", + ifNoneMatch: true, + to: "k2", + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + // The unconditional object is untouched and no copy was made. + expect(await fsp.readFile(path.join(root, "k"), "utf-8")).toBe("payload"); + await expect(fsp.stat(path.join(root, "k2"))).rejects.toThrow(); + }); +}); + describe("cli/commands new surface", () => { // Write the upload source OUTSIDE the fs root — a file under the root would // itself show up as a stray object key in list/transfer results. diff --git a/packages/files-sdk/test/cli-conditional-s3.test.ts b/packages/files-sdk/test/cli-conditional-s3.test.ts new file mode 100644 index 00000000..9435aa84 --- /dev/null +++ b/packages/files-sdk/test/cli-conditional-s3.test.ts @@ -0,0 +1,108 @@ +import { + afterAll, + afterEach, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import * as fsp from "node:fs/promises"; +import * as os from "node:os"; +import path from "node:path"; + +import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { mockClient } from "aws-sdk-client-mock"; + +import { runUpload } from "../src/cli/commands.js"; +import type { CommonRunOpts } from "../src/cli/commands.js"; + +// The CLI always reads its body as a stream, and the S3 adapter — the only +// one with native conditional primitives — rejects stream bodies for a +// conditional PutObject. This drives `upload --if-none-match` end to end +// through a mocked S3Client to prove the CLI buffers the body first. + +const s3Mock = mockClient(S3Client); + +type WriteFn = typeof process.stdout.write; +const origOut = process.stdout.write.bind(process.stdout) as WriteFn; +const stdout: string[] = []; + +let root: string; + +beforeEach(async () => { + root = await fsp.mkdtemp(path.join(os.tmpdir(), "files-sdk-cli-cas-")); + stdout.length = 0; + s3Mock.reset(); + (process.stdout as { write: WriteFn }).write = ((chunk: unknown) => { + stdout.push( + typeof chunk === "string" + ? chunk + : Buffer.from(chunk as Uint8Array).toString("utf-8") + ); + return true; + }) as WriteFn; +}); + +afterEach(async () => { + (process.stdout as { write: WriteFn }).write = origOut; + await fsp.rm(root, { force: true, recursive: true }); +}); + +afterAll(() => { + s3Mock.restore(); +}); + +const baseOpts = (): CommonRunOpts => ({ + dryRun: false, + global: { + accessKeyId: "AKIA", + bucket: "b", + provider: "s3", + region: "us-east-1", + secretAccessKey: "secret", + }, + json: true, + pretty: false, + verbose: false, +}); + +describe("cli upload with a condition against s3", () => { + test("--if-none-match buffers the file body so the conditional PutObject has a length", async () => { + s3Mock.on(PutObjectCommand).resolves({ ETag: '"created"' }); + const local = path.join(root, "config.json"); + await fsp.writeFile(local, '{"ok":true}'); + + await runUpload({ + ...baseOpts(), + file: local, + ifNoneMatch: true, + key: "k", + }); + + const calls = s3Mock.commandCalls(PutObjectCommand); + expect(calls).toHaveLength(1); + const [{ input }] = (calls[0] as NonNullable<(typeof calls)[0]>).args; + expect(input.IfNoneMatch).toBe("*"); + expect(input.ContentLength).toBe('{"ok":true}'.length); + expect(input.Body).toBeInstanceOf(Uint8Array); + expect(Buffer.from(input.Body as Uint8Array).toString()).toBe( + '{"ok":true}' + ); + const emitted = JSON.parse(stdout.join("").trim().split("\n").at(-1) ?? ""); + expect(emitted).toMatchObject({ etag: "created", key: "k" }); + }); + + test("--if-match sends If-Match with the quoted ETag", async () => { + s3Mock.on(PutObjectCommand).resolves({ ETag: '"next"' }); + const local = path.join(root, "config.json"); + await fsp.writeFile(local, "v2"); + + await runUpload({ ...baseOpts(), file: local, ifMatch: "prev", key: "k" }); + + const calls = s3Mock.commandCalls(PutObjectCommand); + expect(calls).toHaveLength(1); + const [{ input }] = (calls[0] as NonNullable<(typeof calls)[0]>).args; + expect(input.IfMatch).toBe('"prev"'); + expect(input.ContentLength).toBe(2); + }); +}); diff --git a/packages/files-sdk/test/cli-mcp.test.ts b/packages/files-sdk/test/cli-mcp.test.ts index a39823f7..4eb40a40 100644 --- a/packages/files-sdk/test/cli-mcp.test.ts +++ b/packages/files-sdk/test/cli-mcp.test.ts @@ -288,6 +288,70 @@ describe("cli/mcp tools (write-enabled)", () => { expect(after.data.exists).toBe(false); }); + test("condition inputs are forwarded, not stripped, and fail closed on fs", async () => { + await call(h.client, "upload", { key: "cas.txt", text: "v1" }); + + const create = await call(h.client, "upload", { + condition: { type: "create" }, + key: "cas-new.txt", + text: "x", + }); + expect(create.isError).toBe(true); + expect((create.data.error as { message: string }).message).toMatch( + /conditional/iu + ); + const missing = await call(h.client, "exists", { key: "cas-new.txt" }); + expect(missing.data.exists).toBe(false); + + const replace = await call(h.client, "upload", { + condition: { etag: "abc", type: "replace" }, + key: "cas.txt", + text: "v2", + }); + expect(replace.isError).toBe(true); + + const read = await call(h.client, "download", { + condition: { etag: "abc" }, + key: "cas.txt", + }); + expect(read.isError).toBe(true); + + const remove = await call(h.client, "delete", { + condition: { etag: "abc" }, + key: "cas.txt", + }); + expect(remove.isError).toBe(true); + + const copy = await call(h.client, "copy", { + condition: { destination: { type: "create" }, source: { etag: "abc" } }, + from: "cas.txt", + to: "cas-copy.txt", + }); + expect(copy.isError).toBe(true); + + // Nothing above degraded into an unconditional mutation. + const body = await call(h.client, "download", { key: "cas.txt" }); + expect(Buffer.from(body.data.base64 as string, "base64").toString()).toBe( + "v1" + ); + const copied = await call(h.client, "exists", { key: "cas-copy.txt" }); + expect(copied.data.exists).toBe(false); + }); + + test("bulk delete rejects a condition instead of ignoring it", async () => { + await call(h.client, "upload", { key: "bulk-a.txt", text: "x" }); + const result = await call(h.client, "delete", { + condition: { etag: "abc" }, + key: ["bulk-a.txt"], + }); + expect(result.isError).toBe(true); + expect((result.data.error as { message: string }).message).toMatch( + /single key/u + ); + const still = await call(h.client, "exists", { key: "bulk-a.txt" }); + expect(still.data.exists).toBe(true); + }); + test("copy and move", async () => { await call(h.client, "upload", { key: "src.txt", text: "payload" }); diff --git a/packages/files-sdk/test/cli-program.test.ts b/packages/files-sdk/test/cli-program.test.ts index 7ac1f746..fbcb757f 100644 --- a/packages/files-sdk/test/cli-program.test.ts +++ b/packages/files-sdk/test/cli-program.test.ts @@ -581,6 +581,87 @@ describe("cli/program parseAsync (fs end-to-end)", () => { ); }); + test("conditional flags route through the upload/download/delete/copy builders", async () => { + const local = path.join(root, "in.txt"); + await fsp.writeFile(local, "payload"); + await run( + "--provider", + "fs", + "--root", + root, + "--dry-run", + "upload", + "k", + "--file", + local, + "--if-match", + "abc" + ); + expect(lastJson(cap.stdout)).toMatchObject({ + action: "upload", + condition: { etag: "abc", type: "replace" }, + }); + cap.stdout.length = 0; + + await run( + "--provider", + "fs", + "--root", + root, + "--dry-run", + "download", + "k", + "--out", + local, + "--if-match", + "abc" + ); + expect(lastJson(cap.stdout)).toMatchObject({ + action: "download", + condition: { etag: "abc" }, + }); + cap.stdout.length = 0; + + await run( + "--provider", + "fs", + "--root", + root, + "--dry-run", + "delete", + "k", + "--if-match", + "abc" + ); + expect(lastJson(cap.stdout)).toMatchObject({ + action: "delete", + condition: { etag: "abc" }, + }); + cap.stdout.length = 0; + + await run( + "--provider", + "fs", + "--root", + root, + "--dry-run", + "copy", + "a", + "b", + "--if-match", + "src", + "--dest-if-match", + "dst" + ); + expect(lastJson(cap.stdout)).toMatchObject({ + action: "copy", + condition: { + destination: { etag: "dst", type: "replace" }, + source: { etag: "src" }, + }, + }); + }); + test("upload/download new flags route through their builders", async () => { const local = path.join(root, "in.txt"); await fsp.writeFile(local, "payload"); diff --git a/packages/files-sdk/test/conditional.test.ts b/packages/files-sdk/test/conditional.test.ts new file mode 100644 index 00000000..b6c5e000 --- /dev/null +++ b/packages/files-sdk/test/conditional.test.ts @@ -0,0 +1,1131 @@ +import { describe, expect, mock, test } from "bun:test"; + +import { + Files, + FilesError, + handlers, + isConditionalOperation, + rejectConditional, +} from "../src/index.js"; +import type { + Adapter, + AdapterDownloadOptions, + AdapterUploadOptions, + Body, + ConditionalUploadResult, + CopyCondition, + FilesActionEvent, + FilesErrorEvent, + FilesPlugin, + OperationOptions, + Receipt, + StoredFile, +} from "../src/index.js"; +import { fakeAdapter } from "./fake-adapter.js"; + +const bareEtag = (etag: string | undefined): string => { + if (!etag) { + throw new Error("test adapter did not produce an ETag"); + } + return etag.startsWith('"') && etag.endsWith('"') ? etag.slice(1, -1) : etag; +}; + +const bodyText = async (body: Body): Promise => { + if (typeof body === "string") { + return body; + } + if (body instanceof Blob) { + return body.text(); + } + if (body instanceof ArrayBuffer) { + return new TextDecoder().decode(body); + } + if (ArrayBuffer.isView(body)) { + return new TextDecoder().decode( + new Uint8Array(body.buffer, body.byteOffset, body.byteLength) + ); + } + const reader = body.getReader(); + const chunks: Uint8Array[] = []; + while (true) { + // eslint-disable-next-line no-await-in-loop -- a test helper drains the stream in order + const { done, value } = await reader.read(); + if (done) { + break; + } + if (value) { + chunks.push(value); + } + } + const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(bytes); +}; + +interface ConditionalHarness { + adapter: Adapter; + base: ReturnType; + calls: { + copy: CopyCondition[]; + create: string[]; + delete: string[]; + exactRead: string[]; + replace: string[]; + uploadBodies: string[]; + uploadKeys: string[]; + uploadMetadata: (Record | undefined)[]; + }; +} + +const conditionalHarness = (opts?: { + createFailures?: number; +}): ConditionalHarness => { + const base = fakeAdapter({ supportsRange: true }); + let createFailures = opts?.createFailures ?? 0; + const calls: ConditionalHarness["calls"] = { + copy: [], + create: [], + delete: [], + exactRead: [], + replace: [], + uploadBodies: [], + uploadKeys: [], + uploadMetadata: [], + }; + + const currentEtag = async (key: string): Promise => { + const file = await base.head(key); + return bareEtag(file.etag); + }; + const upload = async ( + mode: "create" | "replace", + key: string, + body: Body, + uploadOptions?: AdapterUploadOptions + ): Promise => { + calls[mode].push(key); + calls.uploadKeys.push(key); + calls.uploadBodies.push(await bodyText(body)); + calls.uploadMetadata.push(uploadOptions?.metadata); + if (mode === "create" && createFailures > 0) { + createFailures -= 1; + throw new FilesError("Provider", "temporary conditional failure"); + } + const result = await base.upload(key, body, uploadOptions); + return { ...result, etag: bareEtag(result.etag) }; + }; + + const adapter: Adapter = { + ...base, + conditional: { + copy: { + atomicSourceDestination: true, + destinationCreate: true, + destinationReplace: true, + async run( + from: string, + to: string, + condition: CopyCondition + ): Promise { + calls.copy.push(condition); + if ((await currentEtag(from)) !== condition.source.etag) { + throw new FilesError("Conflict", "source ETag mismatch"); + } + if (condition.destination.type === "create") { + if (base.has(to)) { + throw new FilesError("Conflict", "destination exists"); + } + } else if ( + !base.has(to) || + (await currentEtag(to)) !== condition.destination.etag + ) { + throw new FilesError("Conflict", "destination ETag mismatch"); + } + await base.copy(from, to); + }, + sourceEtag: true, + }, + create: (key, body, uploadOptions) => + upload("create", key, body, uploadOptions), + async delete( + key: string, + etag: string, + deleteOptions?: OperationOptions + ): Promise { + calls.delete.push(key); + if ((await currentEtag(key)) !== etag) { + throw new FilesError("Conflict", "delete ETag mismatch"); + } + await base.delete(key, deleteOptions); + }, + async exactRead( + key: string, + etag: string, + downloadOptions?: AdapterDownloadOptions + ): Promise { + calls.exactRead.push(key); + if ((await currentEtag(key)) !== etag) { + throw new FilesError("Conflict", "read ETag mismatch"); + } + const file = await base.download(key, downloadOptions); + return { ...file, etag: bareEtag(file.etag) }; + }, + async replace(key, body, etag, uploadOptions) { + if (!base.has(key) || (await currentEtag(key)) !== etag) { + throw new FilesError("Conflict", "replace ETag mismatch"); + } + return upload("replace", key, body, uploadOptions); + }, + }, + }; + return { adapter, base, calls }; +}; + +describe("native conditional operations", () => { + test("all primitives use native adapter operations, prefixing, and plugin transforms", async () => { + const harness = conditionalHarness(); + const seen: string[] = []; + const protection: FilesPlugin = { + name: "protection", + wrap: handlers({ + upload: (op, next) => { + if (isConditionalOperation(op)) { + seen.push(`${op.kind}:${op.mode}:${op.key}`); + } + return next({ + ...op, + body: "ciphertext", + options: { + ...op.options, + metadata: { protected: "true" }, + }, + }); + }, + }), + }; + const files = new Files({ + adapter: harness.adapter, + plugins: [protection], + prefix: "tenant", + }); + + const created = await files.upload("record", "plaintext", { + condition: { type: "create" }, + }); + expect(created.etag).toBeString(); + expect(harness.calls.uploadKeys).toEqual(["tenant/record"]); + expect(harness.calls.uploadBodies).toEqual(["ciphertext"]); + expect(harness.calls.uploadMetadata).toEqual([{ protected: "true" }]); + expect(seen).toEqual(["upload:create:record"]); + + const exact = await files.download("record", { + condition: { etag: created.etag }, + range: { end: 5, start: 0 }, + }); + expect(await exact.text()).toBe("cipher"); + expect(exact.key).toBe("record"); + + const replaced = await files.upload("record", "new plaintext", { + condition: { etag: created.etag, type: "replace" }, + }); + expect(replaced.etag).not.toBe(created.etag); + await files.delete("record", { condition: { etag: replaced.etag } }); + expect(harness.base.has("tenant/record")).toBe(false); + }); + + test("conditional copy requires and forwards both predicates atomically", async () => { + const harness = conditionalHarness(); + const files = new Files({ adapter: harness.adapter }); + const source = await files.upload("source", "value", { + condition: { type: "create" }, + }); + + await files.copy("source", "destination", { + condition: { + destination: { type: "create" }, + source: { etag: source.etag }, + }, + }); + const destination = await files.download("destination"); + expect(await destination.text()).toBe("value"); + expect(harness.calls.copy).toEqual([ + { + destination: { type: "create" }, + source: { etag: source.etag }, + }, + ]); + }); + + test("copy snapshots caller predicate objects before plugins can mutate them", async () => { + const harness = conditionalHarness(); + const seed = new Files({ adapter: harness.adapter }); + const source = await seed.upload("source", "value", { + condition: { type: "create" }, + }); + const sourcePredicate = { etag: source.etag }; + const destinationPredicate = { type: "create" as const }; + const mutateCallerReferences: FilesPlugin = { + name: "mutate-caller-references", + wrap: handlers({ + copy: (op, next) => { + sourcePredicate.etag = "attacker-source"; + Reflect.set(destinationPredicate, "type", "replace"); + Reflect.set(destinationPredicate, "etag", "attacker-destination"); + return next(op); + }, + }), + }; + const files = new Files({ + adapter: harness.adapter, + plugins: [mutateCallerReferences], + }); + + await files.copy("source", "destination", { + condition: { + destination: destinationPredicate, + source: sourcePredicate, + }, + }); + expect(harness.calls.copy.at(-1)).toEqual({ + destination: { type: "create" }, + source: { etag: source.etag }, + }); + }); + + test("capabilities are conservative and unsupported operations perform no provider I/O", async () => { + const base = fakeAdapter(); + const upload = mock(base.upload); + const download = mock(base.download); + const deleteOne = mock(base.delete); + const copy = mock(base.copy); + const files = new Files({ + adapter: { + ...base, + copy, + delete: deleteOne, + download, + upload, + }, + }); + + expect(files.capabilities.conditional).toEqual({ + copy: { + atomicSourceDestination: false, + destinationCreate: false, + destinationReplace: false, + sourceEtag: false, + }, + create: false, + delete: false, + exactRead: false, + multipart: { create: false, replace: false }, + replace: false, + }); + await expect( + files.upload("a", "v", { condition: { type: "create" } }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.upload("a", "v", { + condition: { etag: "etag", type: "replace" }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.download("a", { condition: { etag: "etag" } }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.delete("a", { condition: { etag: "etag" } }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.copy("a", "b", { + condition: { + destination: { type: "create" }, + source: { etag: "etag" }, + }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + expect(upload).not.toHaveBeenCalled(); + expect(download).not.toHaveBeenCalled(); + expect(deleteOne).not.toHaveBeenCalled(); + expect(copy).not.toHaveBeenCalled(); + + const partial = conditionalHarness(); + const partialCopy = partial.adapter.conditional?.copy; + if (!partialCopy) { + throw new Error("test adapter lacks copy"); + } + const partialRun = mock(partialCopy.run); + partialCopy.run = partialRun; + Reflect.set(partialCopy, "destinationCreate", false); + const partialFiles = new Files({ adapter: partial.adapter }); + expect(partialFiles.capabilities.conditional.copy).toEqual({ + atomicSourceDestination: true, + destinationCreate: false, + destinationReplace: true, + sourceEtag: true, + }); + await expect( + partialFiles.copy("source", "destination", { + condition: { + destination: { type: "create" }, + source: { etag: "etag" }, + }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + expect(partialRun).not.toHaveBeenCalled(); + }); + + test("malformed, weak, wildcard, quoted, and list ETags fail before I/O", async () => { + const harness = conditionalHarness(); + const files = new Files({ adapter: harness.adapter }); + await Promise.all( + ["", '"quoted"', "W/weak", "*", "one,two", "has space"].map((etag) => + expect( + files.upload("a", "v", { + condition: { etag, type: "replace" }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }) + ) + ); + expect(harness.calls.replace).toHaveLength(0); + }); + + test("malformed runtime conditions never downgrade to ordinary operations", async () => { + const harness = conditionalHarness(); + const ordinaryUpload = mock(harness.adapter.upload); + const ordinaryDownload = mock(harness.adapter.download); + const ordinaryDelete = mock(harness.adapter.delete); + const ordinaryCopy = mock(harness.adapter.copy); + harness.adapter.upload = ordinaryUpload; + harness.adapter.download = ordinaryDownload; + harness.adapter.delete = ordinaryDelete; + harness.adapter.copy = ordinaryCopy; + const files = new Files({ adapter: harness.adapter }); + + await Promise.all( + [null, false, 0, ""].map((invalid, index) => { + const options = {}; + Reflect.set(options, "condition", invalid); + return expect( + files.upload(`upload-${index}`, "plaintext", options) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + }) + ); + + const downloadOptions = {}; + Reflect.set(downloadOptions, "condition", false); + await expect( + files.download("download", downloadOptions) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + + const deleteOptions = {}; + Reflect.set(deleteOptions, "condition", 0); + await expect(files.delete("delete", deleteOptions)).rejects.toMatchObject({ + code: "Provider", + permanent: true, + }); + + const copyOptions = {}; + Reflect.set(copyOptions, "condition", { + destination: { type: "create" }, + source: null, + }); + await expect( + files.copy("source", "destination", copyOptions) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + + expect(ordinaryUpload).not.toHaveBeenCalled(); + expect(ordinaryDownload).not.toHaveBeenCalled(); + expect(ordinaryDelete).not.toHaveBeenCalled(); + expect(ordinaryCopy).not.toHaveBeenCalled(); + expect(harness.calls.create).toHaveLength(0); + expect(harness.calls.replace).toHaveLength(0); + expect(harness.calls.exactRead).toHaveLength(0); + expect(harness.calls.delete).toHaveLength(0); + expect(harness.calls.copy).toHaveLength(0); + }); + + test("conditional multipart, resumable, and casted bulk predicates fail closed", async () => { + const base = fakeAdapter(); + const upload = mock(base.upload); + const download = mock(base.download); + const deleteOne = mock(base.delete); + const harness = conditionalHarness(); + const files = new Files({ + adapter: { + ...harness.adapter, + delete: deleteOne, + download, + upload, + }, + }); + + await expect( + files.upload("a", "v", { + condition: { type: "create" }, + multipart: true, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + + const uploadItem = { body: "v", key: "bulk-upload" }; + Reflect.set(uploadItem, "condition", { type: "create" }); + await expect(files.upload([uploadItem])).rejects.toMatchObject({ + code: "Provider", + permanent: true, + }); + const uploadOptions = {}; + Reflect.set(uploadOptions, "condition", { type: "create" }); + await expect( + files.upload([{ body: "v", key: "bulk-options" }], uploadOptions) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + + const downloadOptions = {}; + Reflect.set(downloadOptions, "condition", { etag: "etag" }); + await expect( + files.download(["bulk-download"], downloadOptions) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + const deleteOptions = {}; + Reflect.set(deleteOptions, "condition", { etag: "etag" }); + await expect( + files.delete(["bulk-delete"], deleteOptions) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + + expect(harness.calls.create).toHaveLength(0); + expect(upload).not.toHaveBeenCalled(); + expect(download).not.toHaveBeenCalled(); + expect(deleteOne).not.toHaveBeenCalled(); + }); +}); + +describe("conditional option handling", () => { + test("multipart: false is the documented opt-out, not a multipart request", async () => { + const harness = conditionalHarness(); + const files = new Files({ adapter: harness.adapter }); + const result = await files.upload("a", "v", { + condition: { type: "create" }, + multipart: false, + }); + expect(result.etag).toBeString(); + expect(harness.calls.create).toEqual(["a"]); + // A real multipart ask is still incompatible. + await expect( + files.upload("b", "v", { + condition: { type: "create" }, + multipart: { partSize: 5 * 1024 * 1024 }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + expect(harness.calls.create).toEqual(["a"]); + }); + + test("bulk calls treat condition: undefined as absent, like a spread of shared options", async () => { + const harness = conditionalHarness(); + const files = new Files({ adapter: harness.adapter }); + await files.upload("a", "v"); + await files.upload("b", "v"); + + const uploaded = await files.upload([{ body: "x", key: "c" }], { + condition: undefined, + } as never); + expect(uploaded.uploaded.map((item) => item.key)).toEqual(["c"]); + + const downloaded = await files.download(["a", "b"], { + as: "buffer", + condition: undefined, + } as never); + expect(downloaded.downloaded).toHaveLength(2); + + const deleted = await files.delete(["a", "b"], { + condition: undefined, + } as never); + expect(deleted.deleted).toEqual(["a", "b"]); + }); + + test("ordinary delete and copy still pass extra options through to plugins and the adapter", async () => { + const harness = conditionalHarness(); + const seenByPlugin: unknown[] = []; + const seenByAdapter: unknown[] = []; + const adapter: Adapter = { + ...harness.adapter, + copy(from, to, opts) { + seenByAdapter.push(opts); + return harness.adapter.copy(from, to, opts); + }, + delete(key, opts) { + seenByAdapter.push(opts); + return harness.adapter.delete(key, opts); + }, + }; + const observe: FilesPlugin = { + name: "observe", + wrap: (op, next) => { + if (op.kind === "delete" || op.kind === "copy") { + seenByPlugin.push(op.options); + } + return next(op); + }, + }; + const files = new Files({ adapter, plugins: [observe] }); + await files.upload("a", "v"); + await files.copy("a", "b", { reason: "audit" } as never); + await files.delete("a", { reason: "gdpr" } as never); + expect(seenByPlugin).toEqual([{ reason: "audit" }, { reason: "gdpr" }]); + expect(seenByAdapter).toEqual([{ reason: "audit" }, { reason: "gdpr" }]); + + // The conditional forms strip only the predicate itself. + const { etag } = await files.head("b"); + await files.delete("b", { + condition: { etag: bareEtag(etag) }, + timeout: 5000, + }); + expect(seenByPlugin.at(-1)).toEqual({ timeout: 5000 }); + }); +}); + +describe("conditional plugin boundary", () => { + test("a veto or a provider failure is never reported as applied", async () => { + const vetoHarness = conditionalHarness(); + const veto: FilesPlugin = { + name: "veto", + wrap: handlers({ + upload: () => Promise.reject(new FilesError("Unauthorized", "blocked")), + }), + }; + await expect( + new Files({ adapter: vetoHarness.adapter, plugins: [veto] }).upload( + "a", + "v", + { condition: { type: "create" } } + ) + ).rejects.toMatchObject({ applied: false, code: "Unauthorized" }); + + const failingHarness = conditionalHarness({ createFailures: 1 }); + await expect( + new Files({ adapter: failingHarness.adapter }).upload("a", "v", { + condition: { type: "create" }, + retries: { max: 0 }, + }) + ).rejects.toMatchObject({ applied: false, code: "Provider" }); + expect(failingHarness.base.has("a")).toBe(false); + }); + + test("retrying next() after the native call failed carries the first failure as cause", async () => { + const harness = conditionalHarness({ createFailures: 1 }); + let seen: unknown; + const retry: FilesPlugin = { + name: "retry-after-failure", + wrap: handlers({ + upload: async (op, next) => { + try { + return await next(op); + } catch (error) { + seen = error; + return await next(op); + } + }, + }), + }; + const failure = await new Files({ + adapter: harness.adapter, + plugins: [retry], + }) + .upload("a", "v", { condition: { type: "create" }, retries: { max: 0 } }) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(FilesError); + expect((failure as FilesError).permanent).toBe(true); + expect((failure as Error).message).toMatch( + /more than once.*first invocation failed/u + ); + expect((failure as FilesError).cause).toBe(seen); + expect((failure as FilesError).applied).toBe(false); + expect(harness.calls.create).toEqual(["a"]); + }); + + test("rejectConditional is the uniform plugin veto", async () => { + const harness = conditionalHarness(); + const mirror: FilesPlugin = { + name: "mirror", + wrap: (op, next) => { + if (isConditionalOperation(op)) { + rejectConditional( + op, + "mirror", + "the mirror write cannot share one native compare-and-set" + ); + } + return next(op); + }, + }; + const files = new Files({ adapter: harness.adapter, plugins: [mirror] }); + const failure = await files + .delete("a", { condition: { etag: "abc" } }) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(FilesError); + expect(failure).toMatchObject({ + applied: false, + code: "Provider", + message: + "mirror: conditional delete is unsupported because the mirror write cannot share one native compare-and-set", + permanent: true, + }); + expect(harness.calls.delete).toHaveLength(0); + // Ordinary operations are untouched. + await files.upload("b", "v"); + expect(harness.base.has("b")).toBe(true); + }); + + test("a plugin that catches a rejected predicate change and retries correctly gets the committed result", async () => { + const harness = conditionalHarness(); + let rejected: unknown; + const retry: FilesPlugin = { + name: "catch-then-retry", + wrap: handlers({ + upload: async (op, next) => { + try { + // Dropping the predicate is a violation, rejected before the + // provider is reached. + const downgraded = { ...op }; + Reflect.deleteProperty(downgraded, "mode"); + return await next(downgraded as typeof op); + } catch (error) { + rejected = error; + // The bad call never reached the provider; this one is exactly + // the caller's predicate and must be reported as what it is: a + // committed conditional write. + return await next(op); + } + }, + }), + }; + const files = new Files({ adapter: harness.adapter, plugins: [retry] }); + const result = await files.upload("a", "v", { + condition: { type: "create" }, + }); + expect(rejected).toBeInstanceOf(FilesError); + expect(harness.calls.create).toEqual(["a"]); + expect(harness.calls.uploadBodies).toEqual(["v"]); + const committed = await files.head("a"); + expect(result.etag).toBe(bareEtag(committed.etag)); + }); + + test("a plugin cannot introduce a condition into an ordinary operation", async () => { + const harness = conditionalHarness(); + const ordinaryUpload = mock(harness.adapter.upload); + harness.adapter.upload = ordinaryUpload; + const introduce: FilesPlugin = { + name: "introduce-condition", + wrap: handlers({ + upload: (op, next) => { + Reflect.set(op, "mode", "create"); + return next(op); + }, + }), + }; + const files = new Files({ adapter: harness.adapter, plugins: [introduce] }); + await expect(files.upload("a", "v")).rejects.toThrow( + "cannot introduce a conditional predicate" + ); + expect(ordinaryUpload).not.toHaveBeenCalled(); + expect(harness.calls.create).toHaveLength(0); + }); + + test("a veto runs before provider I/O", async () => { + const harness = conditionalHarness(); + const veto: FilesPlugin = { + name: "veto", + wrap: handlers({ + upload: () => Promise.reject(new FilesError("Unauthorized", "blocked")), + }), + }; + const files = new Files({ adapter: harness.adapter, plugins: [veto] }); + await expect( + files.upload("a", "v", { condition: { type: "create" } }) + ).rejects.toThrow("blocked"); + expect(harness.calls.create).toHaveLength(0); + }); + + test("predicate downgrade, mutation, and delimiter-collision attacks fail closed", async () => { + const downgradeHarness = conditionalHarness(); + const downgrade: FilesPlugin = { + name: "downgrade", + wrap: handlers({ + upload: (op, next) => { + Reflect.set(op, "mode", "overwrite"); + return next(op); + }, + }), + }; + await expect( + new Files({ + adapter: downgradeHarness.adapter, + plugins: [downgrade], + }).upload("a", "v", { condition: { type: "create" } }) + ).rejects.toThrow("cannot remove or change"); + expect(downgradeHarness.calls.create).toHaveLength(0); + + const collisionHarness = conditionalHarness(); + const collision: FilesPlugin = { + name: "collision", + wrap: handlers({ + copy: (op, next) => { + if (op.mode !== "conditional") { + return next(op); + } + const candidate = { + ...op, + destination: { etag: "b:replace:c", type: "replace" as const }, + source: { etag: "a" }, + }; + return next(candidate); + }, + }), + }; + await expect( + new Files({ + adapter: collisionHarness.adapter, + plugins: [collision], + }).copy("a", "b", { + condition: { + destination: { etag: "c", type: "replace" }, + source: { etag: "a:replace:b" }, + }, + }) + ).rejects.toThrow("cannot remove or change"); + expect(collisionHarness.calls.copy).toHaveLength(0); + }); + + test("synthetic success and a second next call are rejected", async () => { + const syntheticHarness = conditionalHarness(); + const synthetic: FilesPlugin = { + name: "synthetic", + wrap: handlers({ + upload: (op, next) => + isConditionalOperation(op) + ? Promise.resolve({ + contentType: "application/octet-stream", + etag: "invented", + key: op.key, + size: 0, + }) + : next(op), + }), + }; + await expect( + new Files({ + adapter: syntheticHarness.adapter, + plugins: [synthetic], + }).upload("a", "v", { condition: { type: "create" } }) + ).rejects.toThrow("cannot synthesize"); + expect(syntheticHarness.calls.create).toHaveLength(0); + + const twiceHarness = conditionalHarness(); + const twice: FilesPlugin = { + name: "twice", + wrap: handlers({ + upload: async (op, next) => { + const result = await next(op); + await next(op); + return result; + }, + }), + }; + await expect( + new Files({ adapter: twiceHarness.adapter, plugins: [twice] }).upload( + "a", + "v", + { condition: { type: "create" } } + ) + ).rejects.toThrow("more than once"); + expect(twiceHarness.calls.create).toHaveLength(1); + }); + + test("dropping or replacing the result ETag rejects after the mutation", async () => { + const harness = conditionalHarness(); + const dropEtag: FilesPlugin = { + name: "drop-etag", + wrap: handlers({ + upload: async (op, next) => { + const result = await next(op); + if (isConditionalOperation(op)) { + Reflect.deleteProperty(result, "etag"); + } + return result; + }, + }), + }; + const files = new Files({ adapter: harness.adapter, plugins: [dropEtag] }); + await expect( + files.upload("a", "v", { condition: { type: "create" } }) + ).rejects.toMatchObject({ + applied: true, + message: expect.stringContaining( + "must return its new canonical strong ETag" + ), + }); + expect(harness.base.has("a")).toBe(true); + expect(harness.calls.create).toHaveLength(1); + + const changedHarness = conditionalHarness(); + const changeEtag: FilesPlugin = { + name: "change-etag", + wrap: handlers({ + upload: async (op, next) => { + const result = await next(op); + if (isConditionalOperation(op)) { + Reflect.set(result, "etag", "invented"); + } + return result; + }, + }), + }; + const changed = new Files({ + adapter: changedHarness.adapter, + plugins: [changeEtag], + }); + await expect( + changed.upload("a", "v", { condition: { type: "create" } }) + ).rejects.toMatchObject({ + applied: true, + message: expect.stringContaining("cannot replace the ETag"), + }); + expect(changedHarness.base.has("a")).toBe(true); + expect(changedHarness.calls.create).toHaveLength(1); + }); + + test("observer failure after mutation rejects, is not retried, and emits no receipt", async () => { + const harness = conditionalHarness(); + const actions: FilesActionEvent[] = []; + const errors: FilesErrorEvent[] = []; + const observer: FilesPlugin = { + name: "observer", + wrap: handlers({ + upload: async (op, next) => { + await next(op); + throw new Error("observer failed"); + }, + }), + }; + const swallowingOuter: FilesPlugin = { + name: "swallowing-outer", + wrap: handlers({ + upload: async (op, next) => { + try { + return await next(op); + } catch { + return { + contentType: "application/octet-stream", + etag: "invented", + key: op.key, + size: 0, + }; + } + }, + }), + }; + const files = new Files({ + adapter: harness.adapter, + hooks: { + onAction: (event) => actions.push(event), + onError: (event) => errors.push(event), + }, + plugins: [swallowingOuter, observer], + receipts: true, + }); + + const failure = await files + .upload("a", "v", { + condition: { type: "create" }, + retries: 3, + }) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(FilesError); + expect((failure as Error).message).toBe("observer failed"); + expect((failure as FilesError).applied).toBe(true); + expect((failure as FilesError).appliedEtag).toBeString(); + expect(harness.base.has("a")).toBe(true); + expect(harness.calls.create).toHaveLength(1); + expect(errors).toHaveLength(1); + expect(errors[0]?.condition).toBe("create"); + // Hooks see the same applied-but-unacknowledged signal as the caller. + expect(errors[0]?.error.applied).toBe(true); + expect(errors[0]?.error.appliedEtag).toBeString(); + expect(actions).toHaveLength(1); + expect(actions[0]).toMatchObject({ + condition: "create", + status: "error", + type: "upload", + }); + expect(actions[0]?.receipt).toBeUndefined(); + }); +}); + +describe("conditional hooks, retries, receipts, and policy", () => { + test("async hook rejections remain fire-and-forget terminal observations", async () => { + const harness = conditionalHarness(); + let actionCalls = 0; + let errorCalls = 0; + const vetoBlocked: FilesPlugin = { + name: "veto-blocked", + wrap: handlers({ + upload: (op, next) => { + if (op.key === "blocked") { + throw new FilesError("Unauthorized", "blocked by policy"); + } + return next(op); + }, + }), + }; + const files = new Files({ + adapter: harness.adapter, + hooks: { + async onAction() { + actionCalls += 1; + await Promise.resolve(); + throw new Error("async action observer failed"); + }, + async onError() { + errorCalls += 1; + await Promise.resolve(); + throw new Error("async error observer failed"); + }, + }, + plugins: [vetoBlocked], + receipts: true, + }); + + await expect( + files.upload("created", "value", { condition: { type: "create" } }) + ).resolves.toMatchObject({ etag: expect.any(String) }); + await expect( + files.upload("blocked", "value", { condition: { type: "create" } }) + ).rejects.toMatchObject({ code: "Unauthorized" }); + await Promise.resolve(); + + expect(actionCalls).toBe(2); + expect(errorCalls).toBe(1); + expect(harness.calls.create).toEqual(["created"]); + }); + + test("a retry preserves the predicate and reports its redacted condition", async () => { + const harness = conditionalHarness({ createFailures: 1 }); + const retries: FilesActionEvent[] = []; + const retryConditions: string[] = []; + const files = new Files({ + adapter: harness.adapter, + hooks: { + onAction: (event) => retries.push(event), + onRetry: (event) => { + if (event.condition) { + retryConditions.push(event.condition); + } + }, + }, + }); + const result = await files.upload("a", "v", { + condition: { type: "create" }, + retries: { backoff: () => 0, max: 1 }, + }); + expect(result.etag).toBeString(); + expect(harness.calls.create).toEqual(["a", "a"]); + expect(retryConditions).toEqual(["create"]); + expect(retries[0]).toMatchObject({ + condition: "create", + status: "success", + }); + }); + + test("success receipts keep base verbs plus conditional summaries", async () => { + const harness = conditionalHarness(); + const receipts: Receipt[] = []; + const files = new Files({ + adapter: harness.adapter, + hooks: { + onAction(event) { + if (event.receipt) { + receipts.push(event.receipt); + } + }, + }, + receipts: true, + }); + + const created = await files.upload("source", "one", { + condition: { type: "create" }, + }); + const replaced = await files.upload("source", "two", { + condition: { etag: created.etag, type: "replace" }, + }); + await files.download("source", { + condition: { etag: replaced.etag }, + }); + await files.copy("source", "copy", { + condition: { + destination: { type: "create" }, + source: { etag: replaced.etag }, + }, + }); + await files.delete("source", { condition: { etag: replaced.etag } }); + + expect(receipts.map(({ condition, op }) => [op, condition])).toEqual([ + ["upload", "create"], + ["upload", "replace"], + ["copy", "conditional-copy"], + ["delete", "match-delete"], + ]); + }); + + test("readonly blocks conditional writes but permits exact reads", async () => { + const harness = conditionalHarness(); + const writable = new Files({ adapter: harness.adapter }); + const created = await writable.upload("a", "value", { + condition: { type: "create" }, + }); + const readonly = writable.readonly(); + const exact = await readonly.download("a", { + condition: { etag: created.etag }, + }); + expect(await exact.text()).toBe("value"); + await expect( + readonly.upload("b", "value", { condition: { type: "create" } }) + ).rejects.toMatchObject({ code: "ReadOnly" }); + expect(harness.calls.create).toEqual(["a"]); + }); + + test("timeout aborts the native attempt without retrying", async () => { + const harness = conditionalHarness(); + let attempts = 0; + let sawSignal = false; + const { conditional } = harness.adapter; + if (!conditional?.create) { + throw new Error("test adapter lacks create"); + } + conditional.create = ( + _key, + _body, + uploadOptions + ): Promise => { + attempts += 1; + sawSignal = uploadOptions?.signal !== undefined; + const pending = Promise.withResolvers(); + uploadOptions?.signal?.addEventListener( + "abort", + () => pending.reject(uploadOptions.signal?.reason), + { once: true } + ); + return pending.promise; + }; + const files = new Files({ adapter: harness.adapter }); + await expect( + files.upload("a", "v", { + condition: { type: "create" }, + retries: 2, + timeout: 5, + }) + ).rejects.toMatchObject({ aborted: true, timedOut: true }); + expect(attempts).toBe(1); + expect(sawSignal).toBe(true); + }); +}); diff --git a/packages/files-sdk/test/dedup.test.ts b/packages/files-sdk/test/dedup.test.ts index d7812cc9..14d8a0fe 100644 --- a/packages/files-sdk/test/dedup.test.ts +++ b/packages/files-sdk/test/dedup.test.ts @@ -2,8 +2,13 @@ import { describe, expect, test } from "bun:test"; import { dedup } from "../src/dedup/index.js"; import type { DedupOptions } from "../src/dedup/index.js"; -import { createFiles } from "../src/index.js"; -import type { Adapter, Files } from "../src/index.js"; +import { createFiles, FilesError } from "../src/index.js"; +import type { + Adapter, + ConditionalFilesOperation, + Files, + PluginNext, +} from "../src/index.js"; import { fakeAdapter } from "./fake-adapter.js"; import type { FakeAdapter } from "./fake-adapter.js"; @@ -316,3 +321,64 @@ describe("dedup plugin — options", () => { expect(() => dedup({ prefix: "///" })).toThrow(/must not be empty/u); }); }); + +describe("dedup plugin — conditional policy", () => { + test("rejects every conditional mode before pointer or blob I/O", async () => { + const plugin = dedup(); + const { wrap } = plugin; + if (!wrap) { + throw new Error("dedup wrap missing"); + } + let nextCalls = 0; + const next = (() => { + nextCalls += 1; + return Promise.resolve(); + }) as PluginNext; + // Uploads and exact reads span pointer + blob; delete and copy touch only + // the pointer, but a pointer's ETag is the hash of an always-empty body — + // identical for every key and unchanged when the pointer is rewritten to + // a new blob — so a compare-and-set against it could never fail and would + // silently drop or duplicate content that had moved on. + const operations: ConditionalFilesOperation[] = [ + { body: "content", key: "a.txt", kind: "upload", mode: "create" }, + { + etag: "etag-1", + key: "a.txt", + kind: "download", + mode: "exact", + }, + { etag: "etag-1", key: "a.txt", kind: "delete", mode: "match" }, + { + destination: { type: "create" }, + from: "a.txt", + kind: "copy", + mode: "conditional", + source: { etag: "etag-1" }, + to: "b.txt", + }, + ]; + + for (const operation of operations) { + // eslint-disable-next-line no-await-in-loop -- each rejected operation must be inspected independently + const failure = await wrap(operation, next).catch( + (error: unknown) => error + ); + expect(failure).toBeInstanceOf(FilesError); + expect((failure as FilesError).permanent).toBe(true); + expect((failure as Error).message).toMatch(/native compare-and-set/u); + } + expect(nextCalls).toBe(0); + }); + + test("a CAS delete through the Files instance is refused and the content survives", async () => { + const files = createFiles({ adapter: fakeAdapter(), plugins: [dedup()] }); + await files.upload("k.txt", "first"); + const { etag } = await files.head("k.txt"); + await files.upload("k.txt", "second"); + await expect( + files.delete("k.txt", { condition: { etag: etag as string } }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + const survived = await files.download("k.txt"); + expect(await survived.text()).toBe("second"); + }); +}); diff --git a/packages/files-sdk/test/encryption.test.ts b/packages/files-sdk/test/encryption.test.ts index d6da5d6d..0842f757 100644 --- a/packages/files-sdk/test/encryption.test.ts +++ b/packages/files-sdk/test/encryption.test.ts @@ -1,8 +1,13 @@ import { describe, expect, test } from "bun:test"; import { encryption, generateEncryptionKey } from "../src/encryption/index.js"; -import { Files } from "../src/index.js"; -import type { Adapter } from "../src/index.js"; +import { createStoredFile, Files } from "../src/index.js"; +import type { + Adapter, + ConditionalFilesOperation, + FilesOperation, + PluginNext, +} from "../src/index.js"; import { fakeAdapter } from "./fake-adapter.js"; const encrypted = async (adapter = fakeAdapter()): Promise => @@ -72,6 +77,130 @@ describe("encryption plugin — round-trips", () => { }); }); +describe("encryption plugin — conditional pipeline", () => { + test("encrypts create and replace, then decrypts an exact read", async () => { + const plugin = encryption(await generateEncryptionKey()); + const { wrap } = plugin; + if (!wrap) { + throw new Error("encryption wrap missing"); + } + const stored: { + body: Uint8Array; + metadata: Record; + type: string; + }[] = []; + const uploadNext = ((candidate: FilesOperation) => { + if (candidate.kind !== "upload") { + throw new Error("expected upload"); + } + if (!(candidate.body instanceof Uint8Array)) { + throw new Error("expected buffered ciphertext"); + } + const metadata = candidate.options?.metadata; + if (!metadata) { + throw new Error("expected encryption metadata"); + } + stored.push({ + body: candidate.body, + metadata, + type: candidate.options?.contentType ?? "application/octet-stream", + }); + return Promise.resolve({ + contentType: + candidate.options?.contentType ?? "application/octet-stream", + etag: `etag-${stored.length}`, + key: candidate.key, + size: candidate.body.byteLength, + }); + }) as PluginNext; + const createOperation: Extract< + ConditionalFilesOperation, + { kind: "upload"; mode: "create" } + > = { + body: "created secret", + key: "secret.txt", + kind: "upload", + mode: "create", + }; + const replaceOperation: Extract< + ConditionalFilesOperation, + { kind: "upload"; mode: "replace" } + > = { + body: "replaced secret", + etag: "etag-1", + key: "secret.txt", + kind: "upload", + mode: "replace", + }; + + await wrap(createOperation, uploadNext); + const replaced = await wrap(replaceOperation, uploadNext); + + expect(stored).toHaveLength(2); + for (const [index, record] of stored.entries()) { + const plaintext = index === 0 ? "created secret" : "replaced secret"; + expect(new TextDecoder().decode(record.body)).not.toBe(plaintext); + expect(record.metadata.fsenc_scheme).toBe("aes-gcm/envelope/v1"); + expect(record.metadata.fsenc_iv).toBeDefined(); + expect(record.metadata.fsenc_dek).toBeDefined(); + } + expect(replaced.size).toBe("replaced secret".length); + + const [, latest] = stored; + if (!latest) { + throw new Error("expected replaced ciphertext"); + } + const raw = createStoredFile( + { + etag: "etag-2", + key: "secret.txt", + metadata: latest.metadata, + size: latest.body.byteLength, + type: latest.type, + }, + { data: latest.body, kind: "buffer" } + ); + const exactOperation: Extract< + ConditionalFilesOperation, + { kind: "download" } + > = { + etag: "etag-2", + key: "secret.txt", + kind: "download", + mode: "exact", + }; + const exact = await wrap(exactOperation, (() => + Promise.resolve(raw)) as PluginNext); + + expect(await exact.text()).toBe("replaced secret"); + expect(exact.metadata).toBeUndefined(); + }); + + test("vetoes a ranged exact read before the provider pipeline", async () => { + const plugin = encryption(await generateEncryptionKey()); + const { wrap } = plugin; + if (!wrap) { + throw new Error("encryption wrap missing"); + } + let nextCalls = 0; + const next = (() => { + nextCalls += 1; + return Promise.resolve(); + }) as PluginNext; + const operation: Extract = + { + etag: "etag-1", + key: "secret.txt", + kind: "download", + mode: "exact", + options: { range: { end: 3, start: 0 } }, + }; + + await expect(wrap(operation, next)).rejects.toThrow(/range downloads/u); + expect(nextCalls).toBe(0); + }); +}); + describe("encryption plugin — metadata", () => { test("preserves user metadata and strips internal fields on read", async () => { const files = await encrypted(); diff --git a/packages/files-sdk/test/failover.test.ts b/packages/files-sdk/test/failover.test.ts index 81c0208c..35539e1c 100644 --- a/packages/files-sdk/test/failover.test.ts +++ b/packages/files-sdk/test/failover.test.ts @@ -3,7 +3,11 @@ import { describe, expect, test } from "bun:test"; import { failover } from "../src/failover/index.js"; import type { FailoverEvent, FailoverOptions } from "../src/failover/index.js"; import { Files } from "../src/index.js"; -import type { Adapter } from "../src/index.js"; +import type { + Adapter, + ConditionalFilesOperation, + PluginNext, +} from "../src/index.js"; import { FilesError } from "../src/internal/errors.js"; import { fakeAdapter } from "./fake-adapter.js"; import type { FakeAdapter } from "./fake-adapter.js"; @@ -345,3 +349,54 @@ describe("failover — a single secondary passed directly", () => { expect(await files.download("a.txt").then((f) => f.text())).toBe("solo"); }); }); + +describe("failover — conditional policy", () => { + test("rejects every conditional primitive before touching any backend", async () => { + const plugin = failover({ secondaries: fakeAdapter() }); + const { wrap } = plugin; + if (!wrap) { + throw new Error("failover wrap missing"); + } + let nextCalls = 0; + const next = (() => { + nextCalls += 1; + return Promise.resolve(); + }) as PluginNext; + const operations: ConditionalFilesOperation[] = [ + { body: "x", key: "a.txt", kind: "upload", mode: "create" }, + { + body: "x", + etag: "etag-1", + key: "a.txt", + kind: "upload", + mode: "replace", + }, + { + etag: "etag-1", + key: "a.txt", + kind: "download", + mode: "exact", + }, + { etag: "etag-1", key: "a.txt", kind: "delete", mode: "match" }, + { + destination: { type: "create" }, + from: "a.txt", + kind: "copy", + mode: "conditional", + source: { etag: "etag-1" }, + to: "b.txt", + }, + ]; + + for (const operation of operations) { + // eslint-disable-next-line no-await-in-loop -- each rejected operation must be inspected independently + const failure = await Promise.resolve() + .then(() => wrap(operation, next)) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(FilesError); + expect((failure as FilesError).permanent).toBe(true); + expect((failure as Error).message).toMatch(/native compare-and-set/u); + } + expect(nextCalls).toBe(0); + }); +}); diff --git a/packages/files-sdk/test/files.test.ts b/packages/files-sdk/test/files.test.ts index cd3f0b05..478b4b7e 100644 --- a/packages/files-sdk/test/files.test.ts +++ b/packages/files-sdk/test/files.test.ts @@ -1377,6 +1377,19 @@ describe("Files class", () => { const files = new Files({ adapter: fakeAdapter() }); expect(files.capabilities).toEqual({ cacheControl: true, + conditional: { + copy: { + atomicSourceDestination: false, + destinationCreate: false, + destinationReplace: false, + sourceEtag: false, + }, + create: false, + delete: false, + exactRead: false, + multipart: { create: false, replace: false }, + replace: false, + }, delimiter: false, metadata: true, multipart: false, @@ -1409,6 +1422,19 @@ describe("Files class", () => { const files = new Files({ adapter }); expect(files.capabilities).toEqual({ cacheControl: true, + conditional: { + copy: { + atomicSourceDestination: false, + destinationCreate: false, + destinationReplace: false, + sourceEtag: false, + }, + create: false, + delete: false, + exactRead: false, + multipart: { create: false, replace: false }, + replace: false, + }, delimiter: false, metadata: true, multipart: true, diff --git a/packages/files-sdk/test/fs.test.ts b/packages/files-sdk/test/fs.test.ts index a2e7b422..86e203b6 100644 --- a/packages/files-sdk/test/fs.test.ts +++ b/packages/files-sdk/test/fs.test.ts @@ -73,6 +73,61 @@ describe("fs adapter", () => { const adapter = fsAdapter({ root: "./.tmp-relative-root" }); expect(path.isAbsolute(adapter.root)).toBe(true); }); + + test("fails every conditional primitive before filesystem adapter I/O", async () => { + const root = await makeRoot(); + const adapter = fsAdapter({ root }); + const upload = spyOn(adapter, "upload"); + const download = spyOn(adapter, "download"); + const remove = spyOn(adapter, "delete"); + const copy = spyOn(adapter, "copy"); + const files = new Files({ adapter }); + + expect(files.capabilities.conditional).toEqual({ + copy: { + atomicSourceDestination: false, + destinationCreate: false, + destinationReplace: false, + sourceEtag: false, + }, + create: false, + delete: false, + exactRead: false, + multipart: { create: false, replace: false }, + replace: false, + }); + + await expect( + files.upload("created.txt", "body", { + condition: { type: "create" }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.upload("replaced.txt", "body", { + condition: { etag: "old-etag", type: "replace" }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.download("record.txt", { condition: { etag: "read-etag" } }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.delete("record.txt", { condition: { etag: "delete-etag" } }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.copy("from.txt", "to.txt", { + condition: { + destination: { type: "create" }, + source: { etag: "source-etag" }, + }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + + expect(upload).toHaveBeenCalledTimes(0); + expect(download).toHaveBeenCalledTimes(0); + expect(remove).toHaveBeenCalledTimes(0); + expect(copy).toHaveBeenCalledTimes(0); + expect(await fsp.readdir(root)).toEqual([]); + }); }); describe("upload + download", () => { diff --git a/packages/files-sdk/test/r2.test.ts b/packages/files-sdk/test/r2.test.ts index 5bcd1d05..5ca071d1 100644 --- a/packages/files-sdk/test/r2.test.ts +++ b/packages/files-sdk/test/r2.test.ts @@ -35,6 +35,24 @@ const streamBody = (text: string) => sdkStreamMixin(Readable.from(Buffer.from(text))); describe("r2 adapter — HTTP path", () => { + test("does not advertise native conditional operations", () => { + const files = new Files({ adapter: makeAdapter() }); + + expect(files.capabilities.conditional).toEqual({ + copy: { + atomicSourceDestination: false, + destinationCreate: false, + destinationReplace: false, + sourceEtag: false, + }, + create: false, + delete: false, + exactRead: false, + multipart: { create: false, replace: false }, + replace: false, + }); + }); + test("uses S3-compatible endpoint with auto region and path-style", async () => { const adapter = r2({ accessKeyId: "AKID", diff --git a/packages/files-sdk/test/s3-conditional-guard.test.ts b/packages/files-sdk/test/s3-conditional-guard.test.ts new file mode 100644 index 00000000..4e34f53e --- /dev/null +++ b/packages/files-sdk/test/s3-conditional-guard.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, test } from "bun:test"; +import { Readable } from "node:stream"; + +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CopyObjectCommand, + CreateMultipartUploadCommand, + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectCommand, + HeadObjectCommand, + ListObjectsV2Command, + ListPartsCommand, + PutObjectCommand, + S3Client, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import type { S3ClientConfig } from "@aws-sdk/client-s3"; +import { createPresignedPost } from "@aws-sdk/s3-presigned-post"; +import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; + +import { FilesError } from "../src/index.js"; +import { createS3Adapter } from "../src/s3/core.js"; +import type { S3Adapter, S3Sdk } from "../src/s3/core.js"; + +// This file deliberately does NOT use aws-sdk-client-mock: that stubs +// `send`, so the middleware stack — where the header guard lives — never +// runs. Instead the client is real and its request handler is a fake that +// records what would have gone on the wire and answers with a canned +// response, so the serializer, our build-step guard, and signing all run. + +interface SentRequest { + headers: Record; + method: string; + path: string; +} + +const xml = (body: string) => Readable.from([Buffer.from(body)]); + +const fakeHandler = (sent: SentRequest[]) => ({ + handle: (request: SentRequest) => { + sent.push(request); + return Promise.resolve({ + response: { + body: xml( + request.method === "PUT" && !request.headers["x-amz-copy-source"] + ? "" + : '"copied"' + ), + headers: { etag: '"committed"' }, + statusCode: 200, + }, + }); + }, +}); + +/** + * Build the adapter over a real client. `strip` simulates an older + * `@aws-sdk/client-s3` whose CopyObject model predates `IfMatch` / + * `IfNoneMatch`: the input field is accepted but never serialized. + */ +const adapterOver = ( + sent: SentRequest[], + strip?: string, + simulate?: { endpoint?: string; conditional?: boolean } +): S3Adapter => { + class TestClient extends S3Client { + // aws-sdk-client-mock (used by s3.test.ts) stubs `S3Client.prototype.send` + // for the whole process and never restores it, so reach past that own + // property to the inherited, real `Client#send` — this file exists to run + // the middleware stack, which the stub short-circuits. + override send = ( + Object.getPrototypeOf(S3Client.prototype) as { send: S3Client["send"] } + ).send; + + constructor(config: S3ClientConfig) { + super({ + ...config, + credentials: { accessKeyId: "AKIA", secretAccessKey: "secret" }, + // Stands in for a shared-config `endpoint_url`: the adapter never + // saw an `endpoint` option, but the client resolves elsewhere. + ...(simulate?.endpoint && { endpoint: simulate.endpoint }), + requestHandler: fakeHandler(sent), + }); + if (strip) { + this.middlewareStack.add( + (next) => (args) => { + const request = args.request as { headers: Record }; + Reflect.deleteProperty(request.headers, strip); + return next(args); + }, + { name: "simulateOldSdk", priority: "high", step: "build" } + ); + } + } + } + const sdk: S3Sdk = { + clientS3: { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CopyObjectCommand, + CreateMultipartUploadCommand, + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectCommand, + HeadObjectCommand, + ListObjectsV2Command, + ListPartsCommand, + PutObjectCommand, + S3Client: TestClient, + UploadPartCommand, + }, + presignedPost: { createPresignedPost }, + requestPresigner: { getSignedUrl }, + }; + return createS3Adapter(sdk, { + bucket: "b", + region: "us-east-1", + ...(simulate?.conditional !== undefined && { + conditional: simulate.conditional, + }), + }); +}; + +const native = ( + adapter: S3Adapter +): Required> => { + const { + copy, + create, + delete: remove, + exactRead, + replace, + } = adapter.conditional ?? {}; + if (!(copy && create && remove && exactRead && replace)) { + throw new Error("expected native conditional primitives"); + } + return { copy, create, delete: remove, exactRead, replace }; +}; + +describe("s3 adapter — conditional header guard", () => { + test("a current SDK serializes every predicate and the request goes out", async () => { + const sent: SentRequest[] = []; + const conditional = native(adapterOver(sent)); + + await conditional.copy.run("from.txt", "to.txt", { + destination: { type: "create" }, + source: { etag: "src" }, + }); + await conditional.create("new.txt", "body"); + await conditional.delete("old.txt", "gone"); + + expect(sent).toHaveLength(3); + const [copy, put, del] = sent as [SentRequest, SentRequest, SentRequest]; + expect(copy.headers["x-amz-copy-source-if-match"]).toBe('"src"'); + expect(copy.headers["if-none-match"]).toBe("*"); + expect(put.headers["if-none-match"]).toBe("*"); + expect(del.headers["if-match"]).toBe('"gone"'); + }); + + test("an SDK that drops a conditional copy predicate fails closed before the request is sent", async () => { + const sent: SentRequest[] = []; + const conditional = native(adapterOver(sent, "if-none-match")); + + const failure = await conditional.copy + .run("from.txt", "to.txt", { + destination: { type: "create" }, + source: { etag: "src" }, + }) + .catch((error: unknown) => error); + + expect(failure).toBeInstanceOf(FilesError); + expect((failure as FilesError).code).toBe("Provider"); + expect((failure as FilesError).permanent).toBe(true); + expect((failure as Error).message).toMatch( + /did not serialize if-none-match/u + ); + // Nothing reached the wire — no unconditional overwrite happened. + expect(sent).toHaveLength(0); + }); + + test("the guard covers the source predicate and the single-object primitives too", async () => { + const copySent: SentRequest[] = []; + await expect( + native(adapterOver(copySent, "x-amz-copy-source-if-match")).copy.run( + "from.txt", + "to.txt", + { + destination: { etag: "dst", type: "replace" }, + source: { etag: "src" }, + } + ) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + expect(copySent).toHaveLength(0); + + const putSent: SentRequest[] = []; + await expect( + native(adapterOver(putSent, "if-match")).replace("k", "body", "old") + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + expect(putSent).toHaveLength(0); + }); + + test("ordinary requests are untouched by the guard", async () => { + const sent: SentRequest[] = []; + const adapter = adapterOver(sent, "if-match"); + await adapter.copy("from.txt", "to.txt"); + expect(sent).toHaveLength(1); + expect(sent[0]?.headers["if-match"]).toBeUndefined(); + }); + + test("a shared-config endpoint_url that resolves off AWS fails closed at request time", async () => { + const sent: SentRequest[] = []; + // The constructor gate still exposes the primitives (no `endpoint`, no + // env redirect), so this is exactly the gap a profile `endpoint_url` + // opens — closed by the resolved hostname on the built request. + const adapter = adapterOver(sent, undefined, { + endpoint: "http://localhost:9000", + }); + const conditional = native(adapter); + const failure = await conditional + .create("k", "body") + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(FilesError); + expect((failure as FilesError).permanent).toBe(true); + expect((failure as Error).message).toMatch( + /only sent to AWS S3.*localhost.*conditional: true/u + ); + expect(sent).toHaveLength(0); + // Ordinary traffic to that endpoint is unaffected. + await adapter.copy("from.txt", "to.txt"); + expect(sent).toHaveLength(1); + }); + + test("conditional: true opts a verified S3-compatible endpoint in", async () => { + const sent: SentRequest[] = []; + const conditional = native( + adapterOver(sent, undefined, { + conditional: true, + endpoint: "http://localhost:9000", + }) + ); + await conditional.create("k", "body"); + expect(sent).toHaveLength(1); + expect(sent[0]?.headers["if-none-match"]).toBe("*"); + }); + + test("AWS-hosted endpoints — VPC, FIPS, dual-stack, GovCloud — pass the hostname check", async () => { + const hosts = [ + "https://bucket.vpce-0a1b2c.s3.us-east-1.vpce.amazonaws.com", + "https://s3-fips.us-gov-west-1.amazonaws.com", + "https://s3.dualstack.us-east-1.amazonaws.com", + "https://s3.cn-north-1.amazonaws.com.cn", + ]; + for (const endpoint of hosts) { + const sent: SentRequest[] = []; + // eslint-disable-next-line no-await-in-loop -- each host is an independent case + await native(adapterOver(sent, undefined, { endpoint })).delete( + "k", + "etag" + ); + expect(sent).toHaveLength(1); + } + }); +}); diff --git a/packages/files-sdk/test/s3.test.ts b/packages/files-sdk/test/s3.test.ts index 5fced2d6..8cc5b049 100644 --- a/packages/files-sdk/test/s3.test.ts +++ b/packages/files-sdk/test/s3.test.ts @@ -20,8 +20,12 @@ import { sdkStreamMixin } from "@smithy/util-stream"; import { mockClient } from "aws-sdk-client-mock"; import { Files, FilesError, UploadControl } from "../src/index.js"; -import type { ResumableUploadSession } from "../src/index.js"; +import type { + AdapterUploadOptions, + ResumableUploadSession, +} from "../src/index.js"; import { mapS3Error, s3 } from "../src/s3/index.js"; +import type { S3Adapter } from "../src/s3/index.js"; const s3Mock = mockClient(S3Client); @@ -94,6 +98,29 @@ const firstCall = (calls: T[]): T => { return first; }; +const requireNativeConditional = ( + adapter: S3Adapter +): Required> => { + const { conditional } = adapter; + const { + copy, + create, + delete: remove, + exactRead, + replace, + } = conditional ?? {}; + if (!copy || !create || !remove || !exactRead || !replace) { + throw new Error("expected the native AWS S3 conditional primitives"); + } + return { + copy, + create, + delete: remove, + exactRead, + replace, + }; +}; + describe("s3 adapter", () => { test("upload sends PutObjectCommand with bucket/key/contentType/metadata", async () => { s3Mock.on(PutObjectCommand).resolves({ ETag: '"abc"' }); @@ -384,6 +411,514 @@ describe("s3 adapter", () => { expect(input.CopySource).toBe("test-bucket/foo%20bar.txt"); }); + test("native conditional primitives are exposed only for AWS S3 endpoints", () => { + const native = s3({ bucket: "b", region: "us-east-1" }); + const conditional = requireNativeConditional(native); + expect(conditional.copy).toMatchObject({ + atomicSourceDestination: true, + destinationCreate: true, + destinationReplace: true, + sourceEtag: true, + }); + + const compatible = s3({ + bucket: "b", + endpoint: "https://storage.example.test", + region: "us-east-1", + }); + expect(compatible.conditional).toBeUndefined(); + expect(s3Mock.calls()).toHaveLength(0); + }); + + test("an AWS_ENDPOINT_URL* redirect hides the primitives; `conditional` overrides either way", () => { + const saved = { + base: process.env.AWS_ENDPOINT_URL, + s3: process.env.AWS_ENDPOINT_URL_S3, + }; + delete process.env.AWS_ENDPOINT_URL; + delete process.env.AWS_ENDPOINT_URL_S3; + try { + // S3Client honors these on its own, so an env-redirected client is an + // S3-compatible service as far as conditional-header support goes. + process.env.AWS_ENDPOINT_URL_S3 = "http://localhost:9000"; + expect( + s3({ bucket: "b", region: "us-east-1" }).conditional + ).toBeUndefined(); + delete process.env.AWS_ENDPOINT_URL_S3; + process.env.AWS_ENDPOINT_URL = "http://localhost:4566"; + expect( + s3({ bucket: "b", region: "us-east-1" }).conditional + ).toBeUndefined(); + // An explicit opt-in wins over both the env redirect and a custom + // endpoint (VPC / FIPS / GovCloud hostnames are still AWS). + expect( + s3({ bucket: "b", conditional: true, region: "us-east-1" }).conditional + ).toBeDefined(); + delete process.env.AWS_ENDPOINT_URL; + expect( + s3({ + bucket: "b", + conditional: true, + endpoint: "https://bucket.vpce-abc.s3.us-east-1.vpce.amazonaws.com", + region: "us-east-1", + }).conditional + ).toBeDefined(); + // And an explicit opt-out disables them on a canonical bucket. + const optedOut = new Files({ + adapter: s3({ bucket: "b", conditional: false, region: "us-east-1" }), + }); + expect(optedOut.capabilities.conditional.create).toBe(false); + } finally { + if (saved.base === undefined) { + delete process.env.AWS_ENDPOINT_URL; + } else { + process.env.AWS_ENDPOINT_URL = saved.base; + } + if (saved.s3 === undefined) { + delete process.env.AWS_ENDPOINT_URL_S3; + } else { + process.env.AWS_ENDPOINT_URL_S3 = saved.s3; + } + } + expect(s3Mock.calls()).toHaveLength(0); + }); + + test("custom endpoints fail every Files conditional operation before provider I/O", async () => { + const files = new Files({ + adapter: s3({ + bucket: "b", + endpoint: "https://storage.example.test", + region: "us-east-1", + }), + }); + + expect(files.capabilities.conditional).toEqual({ + copy: { + atomicSourceDestination: false, + destinationCreate: false, + destinationReplace: false, + sourceEtag: false, + }, + create: false, + delete: false, + exactRead: false, + multipart: { create: false, replace: false }, + replace: false, + }); + await expect( + files.upload("created.txt", "body", { + condition: { type: "create" }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.upload("replaced.txt", "body", { + condition: { etag: "old-etag", type: "replace" }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.download("record.txt", { condition: { etag: "exact-etag" } }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.delete("record.txt", { condition: { etag: "delete-etag" } }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + files.copy("from.txt", "to.txt", { + condition: { + destination: { type: "create" }, + source: { etag: "source-etag" }, + }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + expect(s3Mock.calls()).toHaveLength(0); + }); + + test("Files preserves prefixes and retries AWS conditional request conflicts natively", async () => { + s3Mock + .on(PutObjectCommand) + .rejectsOnce( + Object.assign(new Error("retry the request"), { + $metadata: { httpStatusCode: 409 }, + name: "ConditionalRequestConflict", + }) + ) + .resolvesOnce({ ETag: '"created-etag"' }); + const files = new Files({ + adapter: s3({ bucket: "b", region: "us-east-1" }), + prefix: "tenant/workspace", + }); + + expect(files.capabilities.conditional).toMatchObject({ + copy: { + atomicSourceDestination: true, + destinationCreate: true, + destinationReplace: true, + sourceEtag: true, + }, + create: true, + delete: true, + exactRead: true, + replace: true, + }); + const result = await files.upload("record.txt", "body", { + condition: { type: "create" }, + retries: { backoff: () => 0, max: 1 }, + }); + + expect(result.etag).toBe("created-etag"); + const calls = s3Mock.commandCalls(PutObjectCommand); + expect(calls).toHaveLength(2); + for (const call of calls) { + const [{ input }] = call.args; + expect(input.Key).toBe("tenant/workspace/record.txt"); + expect(input.IfNoneMatch).toBe("*"); + } + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + }); + + test("conditional create uses one PutObject with If-None-Match and preserves upload options", async () => { + s3Mock.on(PutObjectCommand).resolves({ ETag: '"created-etag"' }); + const conditional = requireNativeConditional( + s3({ bucket: "test-bucket", region: "us-east-1" }) + ); + const { signal } = new AbortController(); + const progress: { loaded: number; total?: number }[] = []; + + const result = await conditional.create("created.txt", "hello", { + cacheControl: "private, max-age=60", + contentType: "text/plain", + metadata: { owner: "alice" }, + onProgress: (event) => progress.push(event), + signal, + }); + + expect(result).toEqual({ + contentType: "text/plain", + etag: "created-etag", + key: "created.txt", + size: 5, + }); + expect(progress).toEqual([ + { loaded: 0, total: 5 }, + { loaded: 5, total: 5 }, + ]); + const calls = s3Mock.commandCalls(PutObjectCommand); + expect(calls).toHaveLength(1); + const [{ input }, sendOptions] = firstCall(calls).args as [ + PutObjectCommand, + { abortSignal?: AbortSignal }?, + ]; + expect(input).toMatchObject({ + Bucket: "test-bucket", + CacheControl: "private, max-age=60", + ContentLength: 5, + ContentType: "text/plain", + IfNoneMatch: "*", + Key: "created.txt", + Metadata: { owner: "alice" }, + }); + expect(input.IfMatch).toBeUndefined(); + expect(sendOptions).toEqual({ abortSignal: signal }); + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + }); + + test("conditional replace quotes one canonical ETag and returns the new bare ETag", async () => { + s3Mock.on(PutObjectCommand).resolves({ ETag: '"new-etag"' }); + const conditional = requireNativeConditional( + s3({ bucket: "b", region: "us-east-1" }) + ); + + const result = await conditional.replace( + "record.bin", + new Uint8Array([1, 2, 3]), + "old\\etag" + ); + + expect(result.etag).toBe("new-etag"); + const calls = s3Mock.commandCalls(PutObjectCommand); + expect(calls).toHaveLength(1); + const [{ input }] = firstCall(calls).args; + expect(input.IfMatch).toBe('"old\\etag"'); + expect(input.IfNoneMatch).toBeUndefined(); + expect(input.ContentLength).toBe(3); + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + }); + + test("exact read combines If-Match, range, signal, and normalized response metadata", async () => { + s3Mock.on(GetObjectCommand).resolves({ + Body: streamBody("234") as unknown as undefined, + ContentLength: 3, + ContentType: "text/plain", + ETag: '"exact-etag"', + Metadata: { protected: "true" }, + }); + const conditional = requireNativeConditional( + s3({ bucket: "b", region: "us-east-1" }) + ); + const { signal } = new AbortController(); + + const got = await conditional.exactRead("record.txt", "exact-etag", { + range: { end: 4, start: 2 }, + signal, + }); + + expect(await got.text()).toBe("234"); + expect(got.etag).toBe("exact-etag"); + expect(got.metadata).toEqual({ protected: "true" }); + const calls = s3Mock.commandCalls(GetObjectCommand); + expect(calls).toHaveLength(1); + const [{ input }, sendOptions] = firstCall(calls).args as [ + GetObjectCommand, + { abortSignal?: AbortSignal }?, + ]; + expect(input).toMatchObject({ + Bucket: "b", + IfMatch: '"exact-etag"', + Key: "record.txt", + Range: "bytes=2-4", + }); + expect(sendOptions).toEqual({ abortSignal: signal }); + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + }); + + test("exact read surfaces its validated predicate when S3 omits the response ETag", async () => { + s3Mock.on(GetObjectCommand).resolves({ + Body: streamBody("body") as unknown as undefined, + ContentLength: 4, + }); + const conditional = requireNativeConditional( + s3({ bucket: "b", region: "us-east-1" }) + ); + + const got = await conditional.exactRead("record.txt", "known-etag"); + + expect(got.etag).toBe("known-etag"); + }); + + test("exact read preserves stream mode when S3 returns an empty body", async () => { + s3Mock.on(GetObjectCommand).resolves({ + ContentLength: 0, + ETag: '"exact-etag"', + }); + const conditional = requireNativeConditional( + s3({ bucket: "b", region: "us-east-1" }) + ); + + const got = await conditional.exactRead("record.txt", "exact-etag", { + as: "stream", + }); + + expect(await got.text()).toBe(""); + expect(got.size).toBe(0); + expect(got.etag).toBe("exact-etag"); + }); + + test("exact read fails closed when the response ETag disagrees with its predicate", async () => { + s3Mock.on(GetObjectCommand).resolves({ + Body: streamBody("body") as unknown as undefined, + ContentLength: 4, + ETag: '"different-etag"', + }); + const conditional = requireNativeConditional( + s3({ bucket: "b", region: "us-east-1" }) + ); + + await expect( + conditional.exactRead("record.txt", "expected-etag") + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(1); + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + }); + + test("conditional delete uses one DeleteObject with If-Match and signal", async () => { + s3Mock.on(DeleteObjectCommand).resolves({}); + const conditional = requireNativeConditional( + s3({ bucket: "b", region: "us-east-1" }) + ); + const { signal } = new AbortController(); + + await conditional.delete("record.txt", "delete-etag", { signal }); + + const calls = s3Mock.commandCalls(DeleteObjectCommand); + expect(calls).toHaveLength(1); + const [{ input }, sendOptions] = firstCall(calls).args as [ + DeleteObjectCommand, + { abortSignal?: AbortSignal }?, + ]; + expect(input).toMatchObject({ + Bucket: "b", + IfMatch: '"delete-etag"', + Key: "record.txt", + }); + expect(sendOptions).toEqual({ abortSignal: signal }); + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + }); + + test("conditional copy enforces source and destination predicates in the same command", async () => { + s3Mock.on(CopyObjectCommand).resolves({}); + const conditional = requireNativeConditional( + s3({ bucket: "test-bucket", region: "us-east-1" }) + ); + const { signal } = new AbortController(); + + await conditional.copy.run( + "source folder/a.txt", + "created.txt", + { + destination: { type: "create" }, + source: { etag: "source-etag" }, + }, + { signal } + ); + await conditional.copy.run("source.txt", "replaced.txt", { + destination: { etag: "destination-etag", type: "replace" }, + source: { etag: "source-etag-2" }, + }); + + const calls = s3Mock.commandCalls(CopyObjectCommand); + expect(calls).toHaveLength(2); + const [createCommand, createSendOptions] = firstCall(calls).args as [ + CopyObjectCommand, + { abortSignal?: AbortSignal }?, + ]; + const { input: createInput } = createCommand; + expect(createInput).toMatchObject({ + Bucket: "test-bucket", + CopySource: "test-bucket/source%20folder%2Fa.txt", + CopySourceIfMatch: '"source-etag"', + IfNoneMatch: "*", + Key: "created.txt", + }); + expect(createInput?.IfMatch).toBeUndefined(); + expect(createSendOptions).toEqual({ abortSignal: signal }); + + const [replaceCommand] = firstCall(calls.slice(1)).args as [ + CopyObjectCommand, + { abortSignal?: AbortSignal }?, + ]; + const { input: replaceInput } = replaceCommand; + expect(replaceInput).toMatchObject({ + CopySourceIfMatch: '"source-etag-2"', + IfMatch: '"destination-etag"', + Key: "replaced.txt", + }); + expect(replaceInput?.IfNoneMatch).toBeUndefined(); + expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(0); + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + test("conditional copy maps a provider failure without issuing fallback I/O", async () => { + s3Mock + .on(CopyObjectCommand) + .rejects(Object.assign(new Error("denied"), { name: "AccessDenied" })); + const conditional = requireNativeConditional( + s3({ bucket: "test-bucket", region: "us-east-1" }) + ); + + await expect( + conditional.copy.run("source.txt", "destination.txt", { + destination: { type: "create" }, + source: { etag: "source-etag" }, + }) + ).rejects.toMatchObject({ code: "Unauthorized" }); + + expect(s3Mock.commandCalls(CopyObjectCommand)).toHaveLength(1); + expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(0); + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + test("conditional uploads reject multipart, resumable control, and unsized streams before provider I/O", async () => { + const conditional = requireNativeConditional( + s3({ bucket: "b", region: "us-east-1" }) + ); + const multipart = { + multipart: true, + } as unknown as AdapterUploadOptions; + const controlled = { + control: new UploadControl(), + } as unknown as AdapterUploadOptions; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1])); + controller.close(); + }, + }); + + await expect( + conditional.create("multipart.bin", "body", multipart) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + conditional.create("controlled.bin", "body", controlled) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + conditional.create("stream.bin", stream) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + expect(s3Mock.calls()).toHaveLength(0); + expect(FakeUpload.instances).toBe(0); + }); + + test("conditional operations reject non-canonical ETags before provider I/O", async () => { + const conditional = requireNativeConditional( + s3({ bucket: "b", region: "us-east-1" }) + ); + + await expect( + conditional.replace("record.txt", "body", '"already-quoted"') + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + conditional.exactRead("record.txt", "W/weak-etag") + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + conditional.delete("record.txt", "first,second") + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + conditional.copy.run("from", "to", { + destination: { type: "create" }, + source: { etag: "*" }, + }) + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + expect(s3Mock.calls()).toHaveLength(0); + }); + + test("conditional create and replace fail permanently when S3 omits the committed ETag", async () => { + s3Mock.on(PutObjectCommand).resolves({}); + const conditional = requireNativeConditional( + s3({ bucket: "b", region: "us-east-1" }) + ); + + await expect( + conditional.create("created.txt", "body") + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + conditional.replace("replaced.txt", "body", "old-etag") + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(2); + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + }); + + test("conditional output ETags are validated and progress observers cannot retry a committed write", async () => { + s3Mock + .on(PutObjectCommand) + .resolvesOnce({ ETag: '"bad,etag"' }) + .resolvesOnce({ ETag: '"good-etag"' }); + const conditional = requireNativeConditional( + s3({ bucket: "b", region: "us-east-1" }) + ); + + await expect( + conditional.create("invalid.txt", "body") + ).rejects.toMatchObject({ code: "Provider", permanent: true }); + await expect( + conditional.create("observed.txt", "body", { + onProgress() { + throw new Error("observer failed"); + }, + }) + ).resolves.toMatchObject({ etag: "good-etag" }); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(2); + }); + test("operation signals are forwarded to the AWS client", async () => { const { signal } = new AbortController(); s3Mock.on(HeadObjectCommand).resolves({}); @@ -836,6 +1371,19 @@ describe("s3 adapter", () => { } }); + test("ConditionalRequestConflict maps to a retryable Provider error", () => { + const providerError = Object.assign(new Error("retry the request"), { + $metadata: { httpStatusCode: 409 }, + name: "ConditionalRequestConflict", + }); + + const error = mapS3Error(providerError); + + expect(error.code).toBe("Provider"); + expect(error.permanent).toBe(false); + expect(error.cause).toBe(providerError); + }); + test("upload error is mapped to Provider for unknown S3 errors", async () => { s3Mock.on(PutObjectCommand).rejects( Object.assign(new Error("server error"), { diff --git a/packages/files-sdk/test/soft-delete.test.ts b/packages/files-sdk/test/soft-delete.test.ts index d42346c2..839c667f 100644 --- a/packages/files-sdk/test/soft-delete.test.ts +++ b/packages/files-sdk/test/soft-delete.test.ts @@ -1,7 +1,15 @@ import { describe, expect, test } from "bun:test"; -import { createFiles } from "../src/index.js"; -import type { Adapter, Files, ListOptions, ListResult } from "../src/index.js"; +import { createFiles, FilesError } from "../src/index.js"; +import type { + FilesOperation, + Adapter, + ConditionalFilesOperation, + Files, + ListOptions, + ListResult, + PluginNext, +} from "../src/index.js"; import { softDelete } from "../src/soft-delete/index.js"; import type { SoftDeleteOptions } from "../src/soft-delete/index.js"; import { fakeAdapter } from "./fake-adapter.js"; @@ -304,3 +312,93 @@ describe("soft-delete plugin — error propagation", () => { await expect(files.delete("a.txt")).rejects.toThrow(/boom/u); }); }); + +describe("soft-delete plugin — conditional policy", () => { + test("a conditional delete of a key already in the trash is forwarded unchanged", async () => { + // Trash-key deletes are real deletes (how purge works), forwarded to + // `next` as-is — so the native compare-and-set is preserved and a caller + // can purge one trashed generation atomically against a concurrent + // restore. Only the non-trash path (delete → move) is vetoed. + const plugin = softDelete(); + const { wrap } = plugin; + if (!wrap) { + throw new Error("soft-delete wrap missing"); + } + const forwarded: FilesOperation[] = []; + const next = ((op: FilesOperation) => { + forwarded.push(op); + return Promise.resolve(); + }) as PluginNext; + const trashed: ConditionalFilesOperation = { + etag: "etag-1", + key: ".trash/notes.txt", + kind: "delete", + mode: "match", + }; + await wrap(trashed, next); + expect(forwarded).toEqual([trashed]); + + const live: ConditionalFilesOperation = { + etag: "etag-1", + key: "notes.txt", + kind: "delete", + mode: "match", + }; + await expect(wrap(live, next)).rejects.toMatchObject({ + code: "Provider", + permanent: true, + }); + expect(forwarded).toHaveLength(1); + }); + + test("rejects conditional delete before invoking the provider pipeline", async () => { + const plugin = softDelete(); + const { wrap } = plugin; + if (!wrap) { + throw new Error("soft-delete wrap missing"); + } + let nextCalls = 0; + const next = (() => { + nextCalls += 1; + return Promise.resolve(); + }) as PluginNext; + const operation: ConditionalFilesOperation = { + etag: "etag-1", + key: "notes.txt", + kind: "delete", + mode: "match", + }; + + const failure = await wrap(operation, next).catch( + (error: unknown) => error + ); + + expect(failure).toBeInstanceOf(FilesError); + expect((failure as FilesError).permanent).toBe(true); + expect((failure as Error).message).toMatch(/native compare-and-set/u); + expect(nextCalls).toBe(0); + + const compatible: ConditionalFilesOperation[] = [ + { body: "new", key: "new.txt", kind: "upload", mode: "create" }, + { + etag: "etag-1", + key: "notes.txt", + kind: "download", + mode: "exact", + }, + { + destination: { type: "create" }, + from: "notes.txt", + kind: "copy", + mode: "conditional", + source: { etag: "etag-1" }, + to: "copy.txt", + }, + ]; + for (const candidate of compatible) { + // eslint-disable-next-line no-await-in-loop -- each compatible conditional family must pass through once + await wrap(candidate, next); + } + expect(nextCalls).toBe(3); + }); +}); diff --git a/packages/files-sdk/test/tiering.test.ts b/packages/files-sdk/test/tiering.test.ts index 54624d0b..b445c60b 100644 --- a/packages/files-sdk/test/tiering.test.ts +++ b/packages/files-sdk/test/tiering.test.ts @@ -1,7 +1,13 @@ import { describe, expect, test } from "bun:test"; -import { createFiles } from "../src/index.js"; -import type { Adapter, ListOptions, ListResult } from "../src/index.js"; +import { createFiles, FilesError } from "../src/index.js"; +import type { + Adapter, + ConditionalFilesOperation, + ListOptions, + ListResult, + PluginNext, +} from "../src/index.js"; import { tiering } from "../src/tiering/index.js"; import type { TieringOptions, TierRouter } from "../src/tiering/index.js"; import { fakeAdapter } from "./fake-adapter.js"; @@ -543,3 +549,54 @@ describe("tiering — copy with a missing source under fallback", () => { await expect(files.copy("ghost", "dest")).rejects.toThrow(/not found/u); }); }); + +describe("tiering — conditional policy", () => { + test("rejects every conditional primitive before touching either tier", async () => { + const plugin = tiering({ cold: fakeAdapter(), route: prefixRoute }); + const { wrap } = plugin; + if (!wrap) { + throw new Error("tiering wrap missing"); + } + let nextCalls = 0; + const next = (() => { + nextCalls += 1; + return Promise.resolve(); + }) as PluginNext; + const operations: ConditionalFilesOperation[] = [ + { body: "x", key: "a.txt", kind: "upload", mode: "create" }, + { + body: "x", + etag: "etag-1", + key: "a.txt", + kind: "upload", + mode: "replace", + }, + { + etag: "etag-1", + key: "a.txt", + kind: "download", + mode: "exact", + }, + { etag: "etag-1", key: "a.txt", kind: "delete", mode: "match" }, + { + destination: { type: "create" }, + from: "a.txt", + kind: "copy", + mode: "conditional", + source: { etag: "etag-1" }, + to: "b.txt", + }, + ]; + + for (const operation of operations) { + // eslint-disable-next-line no-await-in-loop -- each rejected operation must be inspected independently + const failure = await Promise.resolve() + .then(() => wrap(operation, next)) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(FilesError); + expect((failure as FilesError).permanent).toBe(true); + expect((failure as Error).message).toMatch(/native compare-and-set/u); + } + expect(nextCalls).toBe(0); + }); +}); diff --git a/packages/files-sdk/test/tracing.test.ts b/packages/files-sdk/test/tracing.test.ts index 49d4c01c..f56fc4c8 100644 --- a/packages/files-sdk/test/tracing.test.ts +++ b/packages/files-sdk/test/tracing.test.ts @@ -16,8 +16,13 @@ import { } from "@opentelemetry/sdk-trace-base"; import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; -import { createFiles, Files } from "../src/index.js"; -import type { FilesPlugin, OperationOptions } from "../src/index.js"; +import { createFiles, Files, FilesError } from "../src/index.js"; +import type { + ConditionalFilesOperation, + FilesPlugin, + OperationOptions, + PluginNext, +} from "../src/index.js"; import { memory } from "../src/memory/index.js"; import { tracing } from "../src/tracing/index.js"; @@ -217,6 +222,92 @@ describe("tracing", () => { await files.upload("a.txt", bytes("hi")); expect(named("files.upload")).toHaveLength(1); }); + + test("records redacted conditional success and error outcomes", async () => { + const plugin = tracing({ tracer }); + const { wrap } = plugin; + if (!wrap) { + throw new Error("tracing wrap missing"); + } + const create: Extract< + ConditionalFilesOperation, + { kind: "upload"; mode: "create" } + > = { + body: bytes("hello"), + key: "a.txt", + kind: "upload", + mode: "create", + }; + await wrap(create, (() => + Promise.resolve({ + contentType: "text/plain", + etag: "secret-etag", + key: "a.txt", + size: 5, + })) as PluginNext); + const exact: Extract = { + etag: "secret-etag", + key: "a.txt", + kind: "download", + mode: "exact", + }; + await wrap(exact, (() => + Promise.reject( + new FilesError("Conflict", "stale ETag") + )) as PluginNext).catch(() => {}); + const matchedDelete: Extract< + ConditionalFilesOperation, + { kind: "delete" } + > = { + etag: "secret-etag", + key: "delete.txt", + kind: "delete", + mode: "match", + }; + await wrap(matchedDelete, (() => Promise.resolve()) as PluginNext); + const conditionalCopy: Extract< + ConditionalFilesOperation, + { kind: "copy" } + > = { + destination: { type: "create" }, + from: "a.txt", + kind: "copy", + mode: "conditional", + source: { etag: "secret-etag" }, + to: "copy.txt", + }; + await wrap(conditionalCopy, (() => Promise.resolve()) as PluginNext); + + const [upload] = named("files.upload"); + const [download] = named("files.download"); + const [remove] = named("files.delete"); + const [copy] = named("files.copy"); + expect(upload?.attributes).toMatchObject({ + "files.condition": "create", + "files.operation": "upload", + "files.size": 5, + }); + expect(upload?.attributes["files.etag"]).toBeUndefined(); + expect(download?.attributes).toMatchObject({ + "files.condition": "exact-read", + "files.operation": "download", + }); + expect(download?.attributes["files.etag"]).toBeUndefined(); + expect(download?.status.code).toBe(SpanStatusCode.ERROR); + expect(download?.events.some((event) => event.name === "exception")).toBe( + true + ); + expect(remove?.attributes).toMatchObject({ + "files.condition": "match-delete", + "files.operation": "delete", + }); + expect(copy?.attributes).toMatchObject({ + "files.condition": "conditional-copy", + "files.from": "a.txt", + "files.operation": "copy", + "files.to": "copy.txt", + }); + }); }); // Context propagation needs an async context manager registered globally. Scope diff --git a/packages/files-sdk/test/usage.test.ts b/packages/files-sdk/test/usage.test.ts index de8c150c..1f0e4af0 100644 --- a/packages/files-sdk/test/usage.test.ts +++ b/packages/files-sdk/test/usage.test.ts @@ -1,6 +1,11 @@ import { describe, expect, test } from "bun:test"; -import { createFiles } from "../src/index.js"; +import { createFiles, createStoredFile, FilesError } from "../src/index.js"; +import type { + ConditionalFilesOperation, + Files, + PluginNext, +} from "../src/index.js"; import { memory } from "../src/memory/index.js"; import { usage } from "../src/usage/index.js"; @@ -211,4 +216,62 @@ describe("usage", () => { expect(files.usage().operations).toBe(1); expect(files.usage().operationsByKind.upload).toBe(1); }); + + test("counts successful conditional calls and excludes terminal errors", async () => { + const plugin = usage(); + const { wrap } = plugin; + const { extend } = plugin; + if (!(wrap && extend)) { + throw new Error("usage plugin surface missing"); + } + const stats = extend({} as Files); + const create: Extract< + ConditionalFilesOperation, + { kind: "upload"; mode: "create" } + > = { + body: bytes("hello"), + key: "a.txt", + kind: "upload", + mode: "create", + }; + await wrap(create, (() => + Promise.resolve({ + contentType: "text/plain", + etag: "etag-1", + key: "a.txt", + size: 5, + })) as PluginNext); + const exact: Extract = { + etag: "etag-1", + key: "a.txt", + kind: "download", + mode: "exact", + }; + const downloaded = await wrap(exact, (() => + Promise.resolve( + createStoredFile( + { etag: "etag-1", key: "a.txt", size: 5, type: "text/plain" }, + { data: bytes("hello"), kind: "buffer" } + ) + )) as PluginNext); + expect(await downloaded.text()).toBe("hello"); + + const deletion: Extract = { + etag: "stale-etag", + key: "a.txt", + kind: "delete", + mode: "match", + }; + await wrap(deletion, (() => + Promise.reject( + new FilesError("Conflict", "stale ETag") + )) as PluginNext).catch(() => {}); + + expect(stats.usage()).toMatchObject({ + bytesDown: 5, + bytesUp: 5, + operations: 2, + operationsByKind: { delete: 0, download: 1, upload: 1 }, + }); + }); }); diff --git a/packages/files-sdk/test/versioning.test.ts b/packages/files-sdk/test/versioning.test.ts index c0366576..f9f806a7 100644 --- a/packages/files-sdk/test/versioning.test.ts +++ b/packages/files-sdk/test/versioning.test.ts @@ -1,7 +1,14 @@ import { describe, expect, test } from "bun:test"; -import { createFiles } from "../src/index.js"; -import type { Adapter, Files, ListOptions, ListResult } from "../src/index.js"; +import { createFiles, FilesError } from "../src/index.js"; +import type { + Adapter, + ConditionalFilesOperation, + Files, + ListOptions, + ListResult, + PluginNext, +} from "../src/index.js"; import { versioning } from "../src/versioning/index.js"; import type { VersioningOptions } from "../src/versioning/index.js"; import { fakeAdapter } from "./fake-adapter.js"; @@ -422,3 +429,52 @@ describe("versioning plugin — error propagation", () => { await expect(files.upload("a.txt", "x")).rejects.toThrow(/boom/u); }); }); + +describe("versioning plugin — conditional policy", () => { + test("rejects conditional mutations before snapshot or provider I/O", async () => { + const plugin = versioning(); + const { wrap } = plugin; + if (!wrap) { + throw new Error("versioning wrap missing"); + } + let nextCalls = 0; + const next = (() => { + nextCalls += 1; + return Promise.resolve(); + }) as PluginNext; + const operations: ConditionalFilesOperation[] = [ + { body: "v2", key: "notes.txt", kind: "upload", mode: "create" }, + { etag: "etag-1", key: "notes.txt", kind: "delete", mode: "match" }, + { + destination: { type: "create" }, + from: "staging.txt", + kind: "copy", + mode: "conditional", + source: { etag: "etag-1" }, + to: "notes.txt", + }, + ]; + + for (const operation of operations) { + // eslint-disable-next-line no-await-in-loop -- each rejected operation must be inspected independently + const failure = await wrap(operation, next).catch( + (error: unknown) => error + ); + expect(failure).toBeInstanceOf(FilesError); + expect((failure as FilesError).permanent).toBe(true); + expect((failure as Error).message).toMatch(/native compare-and-set/u); + } + expect(nextCalls).toBe(0); + + await wrap( + { + etag: "etag-1", + key: "notes.txt", + kind: "download", + mode: "exact", + }, + next + ); + expect(nextCalls).toBe(1); + }); +});