diff --git a/.changeset/harden-storage-boundaries.md b/.changeset/harden-storage-boundaries.md new file mode 100644 index 0000000..3fd747e --- /dev/null +++ b/.changeset/harden-storage-boundaries.md @@ -0,0 +1,8 @@ +--- +'@nestm/storage': minor +--- + +Harden storage integration boundaries with cross-copy-safe `StorageError` +detection, a package-owned S3 driver factory, conditional staged-object +promotion, and a mandatory parsed key policy plus signed-transfer limits for +the optional HTTP gateway. diff --git a/.changeset/map-foreign-files-not-found.md b/.changeset/map-foreign-files-not-found.md new file mode 100644 index 0000000..cfb7c47 --- /dev/null +++ b/.changeset/map-foreign-files-not-found.md @@ -0,0 +1,7 @@ +--- +'@nestm/storage': patch +--- + +Map structurally branded `files-sdk` errors, including errors wrapped across +duplicate package copies, so missing objects retain the `NOT_FOUND` storage +error code. diff --git a/README.md b/README.md index 49deaee..aa62bcb 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,15 @@ consumers do not download NestJS. ## Install ```sh -pnpm add @nestm/storage@alpha files-sdk@2.2.2 +pnpm add @nestm/storage@alpha +``` + +The package-owned S3 factory needs no direct `files-sdk` import. Install the +pinned engine explicitly only when the application imports another adapter +such as `files-sdk/gcs`: + +```sh +pnpm add files-sdk@2.2.2 ``` Install only the native SDKs required by the chosen provider. For example: @@ -82,14 +90,15 @@ runtime or declaration imports. Provider adapters remain available through ## Configure named stores -Create provider adapters with `files-sdk`, wrap them through the explicit -bridge, and register the resulting drivers: +Use the package-owned S3 factory when applicable. For other providers, create a +`files-sdk` adapter, wrap it through the explicit bridge, and register the +resulting driver: ```ts import { Module } from '@nestjs/common'; -import { s3 } from 'files-sdk/s3'; import { gcs } from 'files-sdk/gcs'; import { createFilesSdkDriver } from '@nestm/storage/files-sdk'; +import { createS3StorageDriver } from '@nestm/storage/files-sdk/s3'; import { StorageModule } from '@nestm/storage'; export const StorageKey = { @@ -104,11 +113,11 @@ export const StorageKey = { stores: [ { name: StorageKey.MEDIA, - driver: createFilesSdkDriver({ - adapter: s3({ + driver: createS3StorageDriver({ + adapter: { bucket: 'media', region: 'us-east-1', - }), + }, }), }, { @@ -194,11 +203,11 @@ StorageModule.forRootAsync({ name: 'media', inject: [ConfigService], useFactory: (config: ConfigService) => - createFilesSdkDriver({ - adapter: s3({ + createS3StorageDriver({ + adapter: { bucket: config.getOrThrow('MEDIA_BUCKET'), region: config.getOrThrow('AWS_REGION'), - }), + }, }), }, ], @@ -221,6 +230,7 @@ whitespace. `StorageClient` exposes: - `upload`, `downloadStream`, `head`, `exists`, `delete`, `copy`, and `move`; +- conditional staged-object `promote` when the driver advertises it; - `list`, cursor-aware `listAll`, and lazy `search`; - `signDownload` and discriminated PUT/POST `signUpload`; - `uploadMany`, `downloadMany`, `headMany`, `existsMany`, and `deleteMany`; @@ -250,6 +260,37 @@ Node `Readable` uploads are accepted and converted to Web streams without buffering. Provider capability gaps fail closed with `StorageError` rather than silently discarding a range, metadata, or cache-control request. +### Race-free staged-object promotion + +The S3 bridge advertises ETag- and version-conditional server-side copy. This +lets an application validate a staged object and copy that exact source to its +final key instead of re-reading whichever bytes occupy the staging key later: + +```ts +import { StorageError, StorageErrorCode } from '@nestm/storage'; + +const staged = await media.head(stagingKey); +if (staged.etag === undefined) { + throw new StorageError('Provider did not return a source ETag.', { + code: StorageErrorCode.NOT_SUPPORTED, + }); +} + +// Validate size, declared MIME, and magic bytes before this call. +await media.file(stagingKey).promoteTo(finalKey, { + sourceEtag: staged.etag, +}); + +// Commit ready metadata first. Promotion deliberately retains the staged +// object so a failed database commit remains recoverable. +await media.delete(stagingKey); +``` + +`sourceVersion` can select an immutable S3 version and may be combined with +`sourceEtag`. A promotion without either identity is rejected. Drivers that do +not publish `capabilities.conditionalCopy` fail with `NOT_SUPPORTED` rather +than falling back to an unsafe ordinary copy. + ### Resumable uploads ```ts @@ -301,20 +342,42 @@ The gateway lives at `@nestm/storage/gateway` and is never mounted by `StorageModule`. ```ts -import { Module } from '@nestjs/common'; +import { Injectable, Module } from '@nestjs/common'; import { StorageGatewayModule, StorageGatewayOperation, + type StorageGatewayKeyPolicy, } from '@nestm/storage/gateway'; +@Injectable() +class TenantStorageKeyPolicy implements StorageGatewayKeyPolicy { + resolve({ input, request, target }) { + const tenantId = tenantIdFromAuthenticatedRequest(request); + const root = `tenants/${base64url(tenantId)}`; + if (target === 'pattern') { + // Search is already constrained by the separately resolved prefix. + return input?.value ?? '*'; + } + return `${root}/${input?.value ?? ''}`; + } +} + +@Module({ + providers: [TenantStorageKeyPolicy], + exports: [TenantStorageKeyPolicy], +}) +class StoragePolicyModule {} + @Module({ imports: [ AppStorageModule, AuthModule, + StoragePolicyModule, StorageGatewayModule.register({ - imports: [AppStorageModule, AuthModule], + imports: [AppStorageModule, AuthModule, StoragePolicyModule], store: 'media', guards: [JwtAuthGuard], + keyPolicy: TenantStorageKeyPolicy, mode: 'hybrid', operations: [ StorageGatewayOperation.DOWNLOAD, @@ -325,6 +388,9 @@ import { StorageGatewayOperation.SIGN_UPLOAD, ], maxUploadBytes: 100 * 1024 * 1024, + maxSignedUploadBytes: 10 * 1024 * 1024, + signedUploadContentTypes: ['image/jpeg', 'image/png'], + maxSignedUrlExpiresIn: 900, maxListResults: 1000, maxSearchResults: 1000, proxyInlineContentTypes: ['image/jpeg', 'image/png'], @@ -338,12 +404,39 @@ Registration fails without at least one existing Nest guard. The only bypass is the explicit `allowUnauthenticated: true` development escape hatch. Operations are deny-by-default and must be allowlisted individually. +Registration also fails without a `keyPolicy`. Guards answer whether a request +may reach the gateway; the key policy independently resolves every parsed +`key`, `prefix`, search `pattern`, `from`, and `to` value to the exact provider +path. It runs even when a list/search prefix was omitted, so the policy can +always impose a tenant root. Key-policy providers may be request scoped. +Returned paths are parsed again and reject absolute paths, backslashes, control +characters, empty segments, and dot/parent segments. + +Existing single-tenant applications can temporarily set +`unsafeAllowUnscopedKeys: true` instead of `keyPolicy`. The name is intentional: +it preserves caller-controlled provider keys and must not be used on an exposed +or multi-tenant gateway. It cannot be combined with `keyPolicy`. + Proxy downloads default to `Content-Disposition: attachment` and always send `X-Content-Type-Options: nosniff`. Add only trusted, non-active MIME types to `proxyInlineContentTypes` when browser rendering is required. Search responses are capped by `maxSearchResults`, and list pages by `maxListResults` (both 1,000 by default). +Every signed URL is capped by `maxSignedUrlExpiresIn` (3,600 seconds by +default). Signed uploads always carry a provider-enforced maximum size, capped +by `maxSignedUploadBytes`, and require an exact lowercase MIME type from +`signedUploadContentTypes`. The default direct-upload allowlist contains only +`application/octet-stream`. Gateway callers may request only the literal +`attachment` or `inline` response disposition; arbitrary response-header text +and filenames are rejected. A driver must also advertise +`signedUploadPolicy.contentType` and `signedUploadPolicy.sizeRange`; otherwise +the gateway refuses to mint the URL. `createS3StorageDriver()` advertises both +and uses S3 POST policy conditions. Signed downloads similarly require +`signedDownloadPolicy.expiresIn`. The S3 factory advertises it only when no +permanent `publicBaseUrl` was configured, preventing a configured TTL from +silently returning a non-expiring public link. + The fixed gateway prefix is `/storage`: | Method | Path | Operation | @@ -394,9 +487,14 @@ try { } ``` +`files-sdk` `NotFound` failures retain `StorageErrorCode.NOT_FOUND`, including +when a provider adapter and this driver resolve separate copies of `files-sdk`. +`isStorageError()` likewise recognizes branded and exact legacy structural +errors produced by a duplicated `@nestm/storage` package copy. + Capability flags cover range reads, native byte-level upload progress, delimiter listing, metadata, cache control, resumable uploads, server-side -copy, and signed transfers. +copy, conditional promotion, and signed transfers. Provider-specific native clients are intentionally not exposed from the root package. diff --git a/SECURITY.md b/SECURITY.md index db9acc1..dc2ceff 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -23,3 +23,26 @@ cross-store pruning, or dependency compromise are especially useful. For a vulnerability originating in NestJS, `files-sdk`, or a provider SDK, follow that project's security policy as well. You may still report it here privately when this integration needs a mitigation. + +## Gateway security boundary + +Mounting the optional HTTP gateway requires both authentication guards and a +`StorageGatewayKeyPolicy`. The policy receives parsed paths and must derive the +provider key space from trusted request context. Do not treat a client prefix +or object key as a tenant identifier. `unsafeAllowUnscopedKeys` exists only to +migrate trusted single-tenant deployments and is unsafe on an exposed gateway. + +Signed URL expiry, signed-upload byte size, and signed-upload MIME are bounded +by module configuration and cannot be increased by a request. Only `inline` +and `attachment` content dispositions are accepted. File type still must be +verified from magic bytes after upload; a signed MIME condition proves only +what header the uploader supplied. The gateway rejects signed-upload drivers +that do not advertise provider-enforced content-type and size-range policies. +It also rejects download adapters that cannot guarantee the requested expiry; +in particular, the S3 bridge does not treat `publicBaseUrl` links as expiring. + +For staged uploads on S3, use `promote`/`promoteTo` with the ETag returned by +the validated `head` or with an immutable provider version. Ordinary `copy` +does not protect against a staging-key replay between validation and copy. +Conditional promotion keeps the staging source; delete it only after the +application's metadata transaction commits. diff --git a/package.json b/package.json index fe7e4aa..d38e47f 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,10 @@ "types": "./dist/files-sdk/index.d.ts", "import": "./dist/files-sdk/index.js" }, + "./files-sdk/s3": { + "types": "./dist/files-sdk/s3/index.d.ts", + "import": "./dist/files-sdk/s3/index.js" + }, "./gateway": { "types": "./dist/gateway/index.d.ts", "import": "./dist/gateway/index.js" @@ -89,12 +93,28 @@ "files-sdk": "2.2.2" }, "peerDependencies": { + "@aws-sdk/client-s3": "^3.700.0", + "@aws-sdk/lib-storage": "^3.700.0", + "@aws-sdk/s3-presigned-post": "^3.700.0", + "@aws-sdk/s3-request-presigner": "^3.700.0", "@nestjs/common": "^12.0.0-alpha.5", "@nestjs/core": "^12.0.0-alpha.5", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1" }, "peerDependenciesMeta": { + "@aws-sdk/client-s3": { + "optional": true + }, + "@aws-sdk/lib-storage": { + "optional": true + }, + "@aws-sdk/s3-presigned-post": { + "optional": true + }, + "@aws-sdk/s3-request-presigner": { + "optional": true + }, "@nestjs/common": { "optional": true }, @@ -109,6 +129,10 @@ } }, "devDependencies": { + "@aws-sdk/client-s3": "3.1101.0", + "@aws-sdk/lib-storage": "3.1101.0", + "@aws-sdk/s3-presigned-post": "3.1101.0", + "@aws-sdk/s3-request-presigner": "3.1101.0", "@changesets/cli": "2.31.1", "@nestjs/common": "12.0.0-alpha.5", "@nestjs/core": "12.0.0-alpha.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ebdcdf8..af10199 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,8 +13,20 @@ importers: dependencies: files-sdk: specifier: 2.2.2 - version: 2.2.2(@nestjs/common@12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0))(fastify@5.10.0)(hono@4.12.32)(supports-color@7.2.0)(zod@4.4.3) + version: 2.2.2(@aws-sdk/client-s3@3.1101.0)(@aws-sdk/lib-storage@3.1101.0(@aws-sdk/client-s3@3.1101.0))(@aws-sdk/s3-presigned-post@3.1101.0)(@aws-sdk/s3-request-presigner@3.1101.0)(@nestjs/common@12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0))(fastify@5.10.0)(hono@4.12.32)(supports-color@7.2.0)(zod@4.4.3) devDependencies: + '@aws-sdk/client-s3': + specifier: 3.1101.0 + version: 3.1101.0 + '@aws-sdk/lib-storage': + specifier: 3.1101.0 + version: 3.1101.0(@aws-sdk/client-s3@3.1101.0) + '@aws-sdk/s3-presigned-post': + specifier: 3.1101.0 + version: 3.1101.0 + '@aws-sdk/s3-request-presigner': + specifier: 3.1101.0 + version: 3.1101.0 '@changesets/cli': specifier: 2.31.1 version: 2.31.1(@types/node@26.1.2) @@ -72,6 +84,92 @@ importers: packages: + '@aws-sdk/checksums@3.1000.24': + resolution: {integrity: sha512-7TWLjypP8kk3savsDBRuhZJx7mBuFFA2136BQhwwLllsAnO4Tmq/p+SXZaNxbuulkzUFz3BZzj0bb4YzexZcNQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-s3@3.1101.0': + resolution: {integrity: sha512-16EFb1aTEBgPcfUAWAjjlB57IZCyn7B3rlfT+xqE7M6WoH8AMMU3vFZO0UOitwh/xvvzVx73YED1/n0PU4qBMw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.977.4': + resolution: {integrity: sha512-CEkcQlMOQJCvul60U7wdAOACjtdgFWDsfJI+6wUOGdhGNV2lGbuJpi/R50QLpFG3Tp+sQxa/RmzC3X7KHbhuTA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.65': + resolution: {integrity: sha512-lJT2aRw9wCV8jPHyFJjdZLD4HTydL6/22AnCSOB8e/LqOc55nEJGLHkJQeSxhn8QiqyjFwPKQFtMw0ovjRUY/g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.67': + resolution: {integrity: sha512-N7fw/15hSwI/CPxe5ohOyb7O4ge9f5me1gVIn8OIkBRB0squ8OJqQyDyH/HoL+Sb1W5xdC88jVC+bHkw73iu+Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.973.10': + resolution: {integrity: sha512-Zh9XRaPnDN9buO7GfWBubS22R6Nq5D6hbyYEMN05LiOnXugm/8WDjUx6y756bSPbdn3aJB2qG4zFW3bN82QhoQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.72': + resolution: {integrity: sha512-zZapIKwaHp7TdTf9hbH1I3CVUdEupmt7FXO/BoTQGC+4h6NkXKWpqF2p5WyfpjurDLHCpSyh+BzMlAg8arqWLA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.76': + resolution: {integrity: sha512-1yzLmRiYSgGC25v7ZZEwJn/auhHHTIHgFOmzL2f36hf1+7jSLcX+1QrAz4760WEzPiiQl8xmlpFhHfl2OoyVzA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.65': + resolution: {integrity: sha512-e5DbbNteOSalN58U83G6kFa4ECLEuGbGqNBHIXE7zYXA/m4GHblIGjFbSH7wYv6gBV8iNSDcRZBKfQZF5vF9nw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.973.9': + resolution: {integrity: sha512-0V0u4t+KBku9fbh5CPCaC5hUWwSzDafp8nCuDy817zWbp2gz80jO44rMQkiwnZ+k54B+tjAtzRy00DJRGTKGBg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.71': + resolution: {integrity: sha512-e4dwiRltGAaQ+2yxw57Hj0l/BF3BHiG14+QpYE7bGYBlpAq/fkIri2BDhjWon8c0mhhtd2txQBAkQb9BcTStFg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/lib-storage@3.1101.0': + resolution: {integrity: sha512-S2kYJzmX8a9SPgesOn8la1Ezuto8OqI4e85yNdiOJ/trLMrBxxEZoz8CtOk0uT3FjEJXVbYApzLNmVGVLvUwxA==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@aws-sdk/client-s3': ^3.1101.0 + + '@aws-sdk/middleware-sdk-s3@3.972.70': + resolution: {integrity: sha512-APdP0iODt39AkjCjzTFIoFrxDH/Cz3CpWRDKLcsJg7eOnfE1htkxL9BhDoe/xL7cXdoMwh2HBYv3DiT1uf64NQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.39': + resolution: {integrity: sha512-wU5NPnj62Sb7A8xn/Zb+xThe05P3otNtDl37iOIi5DDMeCesNeCckaG+eXWGUs12Z9R34I8CD05TaTe6SIa61g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/s3-presigned-post@3.1101.0': + resolution: {integrity: sha512-OA8D+mojjQ3SrO0cLo7PG+PMvXirosb5a4fzRx5lgNL3t60m/gRaOhrTZPOAvs+z8Doj0Xc1BOkYiA6PUA90AA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/s3-request-presigner@3.1101.0': + resolution: {integrity: sha512-2raRfiex9qYrdQ60ATeEmOVVC7Do2GEDGSTnoPbgv+xyN3oB48tZeREVw0brI7AlRRvm9fygXTHmyRR0I/LskA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.43': + resolution: {integrity: sha512-lKekx8bLBXSv4O+cslk9Zfnw2XKSkWBs3uWL5QGhH2ZAQfNS7FE0vcSSN2vD/AhxX54ZTywWxR4STThoeOXlBA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1100.0': + resolution: {integrity: sha512-THf3MkgY3fNJZ3zdgSenLqR7gSE68KccCj1RCKretlG73Ppszvues02VpCUO9NlB/tZDC483FvGCld+AiPCkvg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.974.2': + resolution: {integrity: sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.37': + resolution: {integrity: sha512-zKq4HQum8JwDyEuyfuI4bbiAcU0KxP6qy+9PR/IsR92IyE/DaBAikzAS50tjxip4bqIIANpCcG+Yyj6CVhXupg==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.3.0': + resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} + engines: {node: '>=18.0.0'} + '@babel/helper-string-parser@7.29.7': resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} engines: {node: '>=6.9.0'} @@ -557,6 +655,30 @@ packages: '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + '@smithy/core@3.31.1': + resolution: {integrity: sha512-CyogUINxvi7C7LDsh8Syo6hVJOT9ckz4rG8dRZfTJ8r91HkMY59PnNooaj7WcHyxEkxPfBAmbgztZU+xTo76lg==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.4.16': + resolution: {integrity: sha512-QfuLWAkLzptffFW980AFeHZFdqds2B64rpEd3uJ6lgs3xVn9QegGMUgUcj+4d7dRrAsya3r58ZKpku97WcFb4w==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.6.13': + resolution: {integrity: sha512-4fW86pEUOMbrD5nkbyl/tTvPHHWJFbuB2odl6ps9lWfHoXf9HWh3Q/Smh59qH1g7+c/BSZghX6bbUk4gsiMs8A==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.9.13': + resolution: {integrity: sha512-Nmd/Nl35zfYrd+a6OO2cDJb3GPh9bgTjIUhcM+JFfjpp8/osCgboDV5nCT1I01Pv6R13eSKDKLSoVa5ZB6Zsfw==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.6.12': + resolution: {integrity: sha512-I6KLtq3H0qqSuV9vLglfi8puHqzygzWHOnI4z/Rdoo+q50vvo18vBRdPAvvEtcaKROz7Zn6qnPa14kRfPH6PcQ==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.16.1': + resolution: {integrity: sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==} + engines: {node: '>=18.0.0'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -697,6 +819,9 @@ packages: aws4fetch@1.0.20: resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==} + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + better-path-resolve@1.0.0: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} @@ -705,6 +830,9 @@ packages: resolution: {integrity: sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==} engines: {node: '>=18'} + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + braces@3.0.3: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} @@ -712,6 +840,9 @@ packages: buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + buffer@5.6.0: + resolution: {integrity: sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==} + busboy@1.6.0: resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} engines: {node: '>=10.16.0'} @@ -877,6 +1008,10 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + eventsource-parser@3.1.0: resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==} engines: {node: '>=18.0.0'} @@ -1786,6 +1921,9 @@ packages: std-env@4.2.0: resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} + stream-browserify@3.0.0: + resolution: {integrity: sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==} + streamsearch@1.1.0: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} @@ -2008,6 +2146,200 @@ packages: snapshots: + '@aws-sdk/checksums@3.1000.24': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.1101.0': + dependencies: + '@aws-sdk/checksums': 3.1000.24 + '@aws-sdk/core': 3.977.4 + '@aws-sdk/credential-provider-node': 3.972.76 + '@aws-sdk/middleware-sdk-s3': 3.972.70 + '@aws-sdk/signature-v4-multi-region': 3.996.43 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/fetch-http-handler': 5.6.13 + '@smithy/node-http-handler': 4.9.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/core@3.977.4': + dependencies: + '@aws-sdk/types': 3.974.2 + '@aws-sdk/xml-builder': 3.972.37 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.31.1 + '@smithy/signature-v4': 5.6.12 + '@smithy/types': 4.16.1 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.65': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.67': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/fetch-http-handler': 5.6.13 + '@smithy/node-http-handler': 4.9.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.973.10': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/credential-provider-env': 3.972.65 + '@aws-sdk/credential-provider-http': 3.972.67 + '@aws-sdk/credential-provider-login': 3.972.72 + '@aws-sdk/credential-provider-process': 3.972.65 + '@aws-sdk/credential-provider-sso': 3.973.9 + '@aws-sdk/credential-provider-web-identity': 3.972.71 + '@aws-sdk/nested-clients': 3.997.39 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/credential-provider-imds': 4.4.16 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.72': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/nested-clients': 3.997.39 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.76': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.65 + '@aws-sdk/credential-provider-http': 3.972.67 + '@aws-sdk/credential-provider-ini': 3.973.10 + '@aws-sdk/credential-provider-process': 3.972.65 + '@aws-sdk/credential-provider-sso': 3.973.9 + '@aws-sdk/credential-provider-web-identity': 3.972.71 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/credential-provider-imds': 4.4.16 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.65': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.973.9': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/nested-clients': 3.997.39 + '@aws-sdk/token-providers': 3.1100.0 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.71': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/nested-clients': 3.997.39 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/lib-storage@3.1101.0(@aws-sdk/client-s3@3.1101.0)': + dependencies: + '@aws-sdk/client-s3': 3.1101.0 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + buffer: 5.6.0 + events: 3.3.0 + stream-browserify: 3.0.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.70': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/signature-v4-multi-region': 3.996.43 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.39': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/signature-v4-multi-region': 3.996.43 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/fetch-http-handler': 5.6.13 + '@smithy/node-http-handler': 4.9.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/s3-presigned-post@3.1101.0': + dependencies: + '@aws-sdk/client-s3': 3.1101.0 + '@aws-sdk/core': 3.977.4 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/signature-v4': 5.6.12 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/s3-request-presigner@3.1101.0': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/signature-v4-multi-region': 3.996.43 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.43': + dependencies: + '@aws-sdk/types': 3.974.2 + '@smithy/signature-v4': 5.6.12 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1100.0': + dependencies: + '@aws-sdk/core': 3.977.4 + '@aws-sdk/nested-clients': 3.997.39 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/types@3.974.2': + dependencies: + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.37': + dependencies: + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.3.0': {} + '@babel/helper-string-parser@7.29.7': {} '@babel/helper-validator-identifier@7.29.7': {} @@ -2487,6 +2819,39 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} + '@smithy/core@3.31.1': + dependencies: + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.4.16': + dependencies: + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.6.13': + dependencies: + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.9.13': + dependencies: + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@smithy/signature-v4@5.6.12': + dependencies: + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@smithy/types@4.16.1': + dependencies: + tslib: 2.8.1 + '@standard-schema/spec@1.1.0': {} '@tokenizer/inflate@0.4.1(supports-color@7.2.0)': @@ -2642,6 +3007,8 @@ snapshots: aws4fetch@1.0.20: {} + base64-js@1.5.1: {} + better-path-resolve@1.0.0: dependencies: is-windows: 1.0.2 @@ -2660,12 +3027,19 @@ snapshots: transitivePeerDependencies: - supports-color + bowser@2.14.1: {} + braces@3.0.3: dependencies: fill-range: 7.1.1 buffer-from@1.1.2: {} + buffer@5.6.0: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + busboy@1.6.0: dependencies: streamsearch: 1.1.0 @@ -2797,6 +3171,8 @@ snapshots: etag@1.8.1: {} + events@3.3.0: {} + eventsource-parser@3.1.0: optional: true @@ -2946,13 +3322,17 @@ snapshots: transitivePeerDependencies: - supports-color - files-sdk@2.2.2(@nestjs/common@12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0))(fastify@5.10.0)(hono@4.12.32)(supports-color@7.2.0)(zod@4.4.3): + files-sdk@2.2.2(@aws-sdk/client-s3@3.1101.0)(@aws-sdk/lib-storage@3.1101.0(@aws-sdk/client-s3@3.1101.0))(@aws-sdk/s3-presigned-post@3.1101.0)(@aws-sdk/s3-request-presigner@3.1101.0)(@nestjs/common@12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0))(fastify@5.10.0)(hono@4.12.32)(supports-color@7.2.0)(zod@4.4.3): dependencies: aws4fetch: 1.0.20 commander: 15.0.0 picomatch: 4.0.5 safe-regex2: 5.1.1 optionalDependencies: + '@aws-sdk/client-s3': 3.1101.0 + '@aws-sdk/lib-storage': 3.1101.0(@aws-sdk/client-s3@3.1101.0) + '@aws-sdk/s3-presigned-post': 3.1101.0 + '@aws-sdk/s3-request-presigner': 3.1101.0 '@modelcontextprotocol/sdk': 1.30.0(supports-color@7.2.0)(zod@4.4.3) '@nestjs/common': 12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0) fastify: 5.10.0 @@ -3618,6 +3998,11 @@ snapshots: std-env@4.2.0: {} + stream-browserify@3.0.0: + dependencies: + inherits: 2.0.4 + readable-stream: 3.6.2 + streamsearch@1.1.0: {} string_decoder@1.3.0: diff --git a/scripts/test-packed-core-consumer.mjs b/scripts/test-packed-core-consumer.mjs index da11cd0..9739b3f 100644 --- a/scripts/test-packed-core-consumer.mjs +++ b/scripts/test-packed-core-consumer.mjs @@ -43,6 +43,14 @@ try { private: true, type: 'module', dependencies: { + '@aws-sdk/client-s3': + rootPackage.devDependencies['@aws-sdk/client-s3'], + '@aws-sdk/lib-storage': + rootPackage.devDependencies['@aws-sdk/lib-storage'], + '@aws-sdk/s3-presigned-post': + rootPackage.devDependencies['@aws-sdk/s3-presigned-post'], + '@aws-sdk/s3-request-presigner': + rootPackage.devDependencies['@aws-sdk/s3-request-presigner'], '@nestm/storage': `file:${tarballPath}`, }, devDependencies: { @@ -113,10 +121,12 @@ import { StorageClient, StorageErrorCode, StorageUploadControl, + isStorageError, type StorageCapabilities, type StorageDriver, type StorageObjectMetadata, } from '@nestm/storage/core'; +import { createS3StorageDriver } from '@nestm/storage/files-sdk/s3'; const capabilities = { cacheControl: true, @@ -249,6 +259,38 @@ assert.equal(nestResolved, false); assert.equal(DEFAULT_BUFFER_LIMIT, 10 * 1024 * 1024); assert.equal(StorageErrorCode.NOT_FOUND, 'NOT_FOUND'); assert.equal(new StorageUploadControl().status, 'idle'); +const foreignStorageError = Object.assign(new Error('foreign'), { + [Symbol.for('@nestm/storage/StorageError')]: true, + aborted: false, + code: StorageErrorCode.NOT_FOUND, + key: 'missing.bin', + name: 'StorageError', + operation: 'head', + permanent: true, + store: 'foreign', + timedOut: false, +}); +assert.equal(isStorageError(foreignStorageError), true); + +const s3Driver = createS3StorageDriver({ + adapter: { + bucket: 'packed-test', + credentials: { accessKeyId: 'test', secretAccessKey: 'test' }, + region: 'us-east-1', + }, +}); +assert.deepEqual(s3Driver.capabilities.conditionalCopy, { + etag: true, + supported: true, + version: true, +}); +assert.deepEqual(s3Driver.capabilities.signedUploadPolicy, { + contentType: true, + sizeRange: true, +}); +assert.deepEqual(s3Driver.capabilities.signedDownloadPolicy, { + expiresIn: true, +}); const client = new StorageClient('packed', driver); const uploaded = await client.upload('hello.txt', 'hello core', { diff --git a/src/files-sdk/files-sdk.driver.spec.ts b/src/files-sdk/files-sdk.driver.spec.ts index e477466..a7c75c8 100644 --- a/src/files-sdk/files-sdk.driver.spec.ts +++ b/src/files-sdk/files-sdk.driver.spec.ts @@ -7,6 +7,43 @@ import { StorageError, StorageErrorCode } from '../storage.error.js'; import { createFilesSdkDriver } from './files-sdk.driver.js'; describe('FilesSdkStorageDriver', () => { + it('maps a not-found FilesError from another package copy', async () => { + class ForeignFilesError extends Error { + override readonly name = 'FilesError'; + readonly code = 'NotFound'; + readonly aborted = false; + readonly timedOut = false; + readonly permanent = true; + } + + const adapter = memory(); + adapter.head = async () => { + throw new ForeignFilesError('missing object'); + }; + const driver = createFilesSdkDriver({ adapter }); + + await expect(driver.head('missing.bin')).rejects.toMatchObject({ + code: StorageErrorCode.NOT_FOUND, + message: 'missing object', + permanent: true, + }); + }); + + it('does not classify an unrelated provider error from its code alone', async () => { + const adapter = memory(); + adapter.head = async () => { + throw Object.assign(new Error('provider used a coincidental code'), { + code: 'NotFound', + }); + }; + const driver = createFilesSdkDriver({ adapter }); + + await expect(driver.head('unknown.bin')).rejects.toMatchObject({ + code: StorageErrorCode.PROVIDER, + message: 'provider used a coincidental code', + }); + }); + it('preserves owned storage errors raised while consuming upload streams', async () => { const expected = new StorageError('stream limit reached', { code: StorageErrorCode.LIMIT_EXCEEDED, diff --git a/src/files-sdk/files-sdk.driver.ts b/src/files-sdk/files-sdk.driver.ts index 3efe90d..32ce57c 100644 --- a/src/files-sdk/files-sdk.driver.ts +++ b/src/files-sdk/files-sdk.driver.ts @@ -18,7 +18,11 @@ import { type UrlOptions, } from 'files-sdk'; -import { StorageError, StorageErrorCode } from '../storage.error.js'; +import { + StorageError, + StorageErrorCode, + isStorageError, +} from '../storage.error.js'; import type { StorageDriver } from '../storage.driver.js'; import type { StorageBody, @@ -28,8 +32,12 @@ import type { StorageObject, StorageObjectMetadata, StorageOperationOptions, + StorageConditionalCopyCapability, + StoragePromotionOptions, StorageRetryOptions, StorageSearchOptions, + StorageSignedUploadPolicyCapability, + StorageSignedDownloadPolicyCapability, StorageSignedDownloadOptions, StorageSignedUpload, StorageSignedUploadOptions, @@ -41,11 +49,166 @@ import { getFilesSdkUploadControl } from '../storage-upload-control.js'; export type FilesSdkDriverOptions = FilesOptions; -function mapFilesError(error: unknown): StorageError { - if (error instanceof StorageError) { +/** + * Optional adapter extension for providers that can conditionally copy an + * immutable source identity. Plain files-sdk adapters remain fully supported. + */ +export interface FilesSdkConditionalCopyAdapter { + readonly conditionalCopy: StorageConditionalCopyCapability; + promote( + sourceKey: string, + destinationKey: string, + options: StoragePromotionOptions, + ): Promise; +} + +export interface FilesSdkSignedUploadPolicyAdapter { + readonly signedUploadPolicy: StorageSignedUploadPolicyCapability; +} + +export interface FilesSdkSignedDownloadPolicyAdapter { + readonly signedDownloadPolicy: StorageSignedDownloadPolicyCapability; +} + +function conditionalCopyAdapterOf( + adapter: Adapter, +): FilesSdkConditionalCopyAdapter | undefined { + if ( + typeof adapter !== 'object' || + adapter === null || + !('conditionalCopy' in adapter) || + !('promote' in adapter) + ) { + return undefined; + } + const candidate = adapter as Adapter & + Partial; + const capability = candidate.conditionalCopy; + if ( + capability === undefined || + typeof capability.supported !== 'boolean' || + typeof capability.etag !== 'boolean' || + typeof capability.version !== 'boolean' || + typeof candidate.promote !== 'function' + ) { + return undefined; + } + return candidate as Adapter & FilesSdkConditionalCopyAdapter; +} + +function signedUploadPolicyAdapterOf( + adapter: Adapter, +): FilesSdkSignedUploadPolicyAdapter | undefined { + if ( + typeof adapter !== 'object' || + adapter === null || + !('signedUploadPolicy' in adapter) + ) { + return undefined; + } + const candidate = adapter as Adapter & + Partial; + const capability = candidate.signedUploadPolicy; + if ( + capability === undefined || + typeof capability.contentType !== 'boolean' || + typeof capability.sizeRange !== 'boolean' + ) { + return undefined; + } + return candidate as Adapter & FilesSdkSignedUploadPolicyAdapter; +} + +function signedDownloadPolicyAdapterOf( + adapter: Adapter, +): FilesSdkSignedDownloadPolicyAdapter | undefined { + if ( + typeof adapter !== 'object' || + adapter === null || + !('signedDownloadPolicy' in adapter) + ) { + return undefined; + } + const candidate = adapter as Adapter & + Partial; + const capability = candidate.signedDownloadPolicy; + if (capability === undefined || typeof capability.expiresIn !== 'boolean') { + return undefined; + } + return candidate as Adapter & FilesSdkSignedDownloadPolicyAdapter; +} + +interface FilesErrorLike { + readonly name: string; + readonly message: string; + readonly code: FilesError['code']; + readonly aborted: boolean; + readonly timedOut: boolean; + readonly permanent: boolean; + readonly cause?: unknown; +} + +function isFilesErrorCode(value: unknown): value is FilesError['code'] { + return ( + value === 'NotFound' || + value === 'Unauthorized' || + value === 'Conflict' || + value === 'ReadOnly' || + value === 'Provider' + ); +} + +function isFilesErrorLike(error: unknown): error is FilesErrorLike { + if (error instanceof FilesError) { + return true; + } + if (!(error instanceof Error)) { + return false; + } + + try { + return ( + error.name === 'FilesError' && + typeof error.message === 'string' && + 'code' in error && + isFilesErrorCode(error.code) && + 'aborted' in error && + typeof error.aborted === 'boolean' && + 'timedOut' in error && + typeof error.timedOut === 'boolean' && + 'permanent' in error && + typeof error.permanent === 'boolean' + ); + } catch { + return false; + } +} + +function unwrapFilesError(error: FilesErrorLike): FilesErrorLike { + const seen = new Set(); + let current = error; + + while ( + current.code === 'Provider' && + !current.aborted && + !current.timedOut && + !current.permanent && + isFilesErrorLike(current.cause) && + current.message === current.cause.message && + !seen.has(current.cause) + ) { + seen.add(current); + current = current.cause; + } + + return current; +} + +export function mapFilesSdkError(error: unknown): StorageError { + if (isStorageError(error)) { return error; } - if (!(error instanceof FilesError)) { + if (!isFilesErrorLike(error)) { return new StorageError( error instanceof Error ? error.message : String(error), { @@ -55,17 +218,19 @@ function mapFilesError(error: unknown): StorageError { ); } - if (error.cause instanceof StorageError) { - return error.cause; + const filesError = unwrapFilesError(error); + + if (isStorageError(filesError.cause)) { + return filesError.cause; } let code: StorageErrorCode; - if (error.timedOut) { + if (filesError.timedOut) { code = StorageErrorCode.TIMEOUT; - } else if (error.aborted) { + } else if (filesError.aborted) { code = StorageErrorCode.ABORTED; } else { - switch (error.code) { + switch (filesError.code) { case 'NotFound': code = StorageErrorCode.NOT_FOUND; break; @@ -80,7 +245,7 @@ function mapFilesError(error: unknown): StorageError { break; case 'Provider': code = /(?:not supported|does not support|unsupported)/iu.test( - error.message, + filesError.message, ) ? StorageErrorCode.NOT_SUPPORTED : StorageErrorCode.PROVIDER; @@ -88,12 +253,12 @@ function mapFilesError(error: unknown): StorageError { } } - return new StorageError(error.message, { - aborted: error.aborted, - cause: error.cause ?? error, + return new StorageError(filesError.message, { + aborted: filesError.aborted, + cause: filesError.cause ?? error, code, - permanent: error.permanent, - timedOut: error.timedOut, + permanent: filesError.permanent, + timedOut: filesError.timedOut, }); } @@ -110,7 +275,7 @@ function mapRetryOptions( backoff: ({ attempt, error }) => retries.backoff?.({ attempt, - error: mapFilesError(error), + error: mapFilesSdkError(error), }) ?? 0, }), }; @@ -182,7 +347,7 @@ function normalizeDownloadStream( try { await activeReader.cancel(reason); } catch (error) { - throw mapFilesError(error); + throw mapFilesSdkError(error); } finally { release(); } @@ -203,7 +368,7 @@ function normalizeDownloadStream( controller.enqueue(result.value); } catch (error) { release(); - controller.error(mapFilesError(error)); + controller.error(mapFilesSdkError(error)); } }, }); @@ -341,10 +506,17 @@ export class FilesSdkStorageDriver< > implements StorageDriver { readonly #files: Files; readonly #name: string; + readonly #conditionalCopy: FilesSdkConditionalCopyAdapter | undefined; + readonly #signedUploadPolicy: FilesSdkSignedUploadPolicyAdapter | undefined; + readonly #signedDownloadPolicy: + FilesSdkSignedDownloadPolicyAdapter | undefined; constructor(options: FilesSdkDriverOptions) { this.#files = new Files(options); this.#name = options.adapter.name; + this.#conditionalCopy = conditionalCopyAdapterOf(options.adapter); + this.#signedUploadPolicy = signedUploadPolicyAdapterOf(options.adapter); + this.#signedDownloadPolicy = signedDownloadPolicyAdapterOf(options.adapter); } get name(): string { @@ -360,8 +532,19 @@ export class FilesSdkStorageDriver< rangeRead: capabilities.rangeRead, resumableUpload: capabilities.multipart, serverSideCopy: capabilities.serverSideCopy, + ...(this.#conditionalCopy !== undefined && { + conditionalCopy: { ...this.#conditionalCopy.conditionalCopy }, + }), signedDownload: { ...capabilities.signedUrl }, + ...(this.#signedDownloadPolicy !== undefined && { + signedDownloadPolicy: { + ...this.#signedDownloadPolicy.signedDownloadPolicy, + }, + }), signedUpload: 'runtime' as const, + ...(this.#signedUploadPolicy !== undefined && { + signedUploadPolicy: { ...this.#signedUploadPolicy.signedUploadPolicy }, + }), nativeUploadProgress: capabilities.uploadProgress, }; } @@ -430,6 +613,30 @@ export class FilesSdkStorageDriver< ); } + promote( + sourceKey: string, + destinationKey: string, + options: StoragePromotionOptions, + ): Promise { + const adapter = this.#conditionalCopy; + if (adapter === undefined || !adapter.conditionalCopy.supported) { + return Promise.reject( + new StorageError( + `Storage adapter "${this.#name}" does not support conditional promotion.`, + { + code: StorageErrorCode.NOT_SUPPORTED, + key: sourceKey, + operation: 'promote', + permanent: true, + }, + ), + ); + } + return this.#call(() => + adapter.promote(sourceKey, destinationKey, options), + ); + } + async list(options?: StorageListOptions): Promise { return this.#call(async () => { const result = await this.#files.list(listOptions(options)); @@ -462,7 +669,7 @@ export class FilesSdkStorageDriver< yield metadataOf(file); } } catch (error) { - throw mapFilesError(error); + throw mapFilesSdkError(error); } } @@ -488,7 +695,7 @@ export class FilesSdkStorageDriver< try { return await operation(); } catch (error) { - throw mapFilesError(error); + throw mapFilesSdkError(error); } } } diff --git a/src/files-sdk/index.ts b/src/files-sdk/index.ts index cf74bb0..b75c051 100644 --- a/src/files-sdk/index.ts +++ b/src/files-sdk/index.ts @@ -1,6 +1,9 @@ export { FilesSdkStorageDriver, createFilesSdkDriver, + type FilesSdkConditionalCopyAdapter, + type FilesSdkSignedUploadPolicyAdapter, + type FilesSdkSignedDownloadPolicyAdapter, type FilesSdkDriverOptions, } from './files-sdk.driver.js'; diff --git a/src/files-sdk/s3/index.ts b/src/files-sdk/s3/index.ts new file mode 100644 index 0000000..f9b07fd --- /dev/null +++ b/src/files-sdk/s3/index.ts @@ -0,0 +1,173 @@ +import { CopyObjectCommand } from '@aws-sdk/client-s3'; +import { + mapS3Error, + s3, + type S3Adapter, + type S3AdapterOptions, +} from 'files-sdk/s3'; + +import type { StoragePromotionOptions } from '../../storage.types.js'; +import { + createFilesSdkDriver, + type FilesSdkConditionalCopyAdapter, + type FilesSdkDriverOptions, + type FilesSdkSignedUploadPolicyAdapter, + type FilesSdkSignedDownloadPolicyAdapter, + type FilesSdkStorageDriver, + mapFilesSdkError, +} from '../files-sdk.driver.js'; + +export interface S3StorageDriverOptions extends Omit< + FilesSdkDriverOptions, + 'adapter' +> { + adapter: S3AdapterOptions; +} + +type EnhancedS3Adapter = S3Adapter & + FilesSdkConditionalCopyAdapter & + FilesSdkSignedDownloadPolicyAdapter & + FilesSdkSignedUploadPolicyAdapter; + +function copySource( + bucket: string, + key: string, + version: string | undefined, +): string { + const source = `${encodeURIComponent(bucket)}/${encodeURIComponent(key)}`; + return version === undefined + ? source + : `${source}?versionId=${encodeURIComponent(version)}`; +} + +function operationSignal( + options: StoragePromotionOptions, +): AbortSignal | undefined { + const timeoutSignal = + options.timeout === undefined || options.timeout <= 0 + ? undefined + : AbortSignal.timeout(options.timeout); + if (options.signal === undefined) { + return timeoutSignal; + } + return timeoutSignal === undefined + ? options.signal + : AbortSignal.any([options.signal, timeoutSignal]); +} + +function maxRetries(options: StoragePromotionOptions): number { + const configured = + typeof options.retries === 'number' + ? options.retries + : options.retries?.max; + return Math.max(0, Math.floor(configured ?? 0)); +} + +async function waitForRetry( + milliseconds: number, + signal: AbortSignal | undefined, +): Promise { + if (signal?.aborted === true) { + throw signal.reason; + } + await new Promise((resolve, reject) => { + const cleanup = (): void => signal?.removeEventListener('abort', abort); + const timer = setTimeout( + () => { + cleanup(); + resolve(); + }, + Math.max(0, milliseconds), + ); + const abort = (): void => { + clearTimeout(timer); + cleanup(); + reject(signal?.reason); + }; + signal?.addEventListener('abort', abort, { once: true }); + }); +} + +/** + * Creates the files-sdk S3 driver from the storage package's own dependency + * context and adds an ETag/version-conditional server-side promotion. + */ +export function createS3StorageDriver( + options: S3StorageDriverOptions, +): FilesSdkStorageDriver { + const { adapter: adapterOptions, ...filesOptions } = options; + const base = s3(adapterOptions); + const adapter: EnhancedS3Adapter = Object.assign(base, { + conditionalCopy: Object.freeze({ + etag: true, + supported: true, + version: true, + }), + signedUploadPolicy: Object.freeze({ + contentType: true, + sizeRange: true, + }), + signedDownloadPolicy: Object.freeze({ + expiresIn: adapterOptions.publicBaseUrl === undefined, + }), + async promote( + sourceKey: string, + destinationKey: string, + promotion: StoragePromotionOptions, + ): Promise { + const retries = maxRetries(promotion); + for (let attempt = 0; ; attempt += 1) { + const signal = operationSignal(promotion); + try { + await base.raw.send( + new CopyObjectCommand({ + Bucket: base.bucket, + CopySource: copySource( + base.bucket, + sourceKey, + promotion.sourceVersion, + ), + ...(promotion.sourceEtag !== undefined && { + CopySourceIfMatch: promotion.sourceEtag, + }), + Key: destinationKey, + }), + signal === undefined ? undefined : { abortSignal: signal }, + ); + return; + } catch (error) { + const mapped = mapS3Error(error); + if ( + attempt >= retries || + mapped.code !== 'Provider' || + mapped.aborted || + mapped.permanent || + signal?.aborted === true + ) { + throw mapped; + } + const storageError = mapFilesSdkError(mapped); + const delay = + typeof promotion.retries === 'object' && + promotion.retries.backoff !== undefined + ? promotion.retries.backoff({ + attempt: attempt + 1, + error: storageError, + }) + : Math.min(1000, 100 * 2 ** attempt); + await waitForRetry(delay, promotion.signal); + } + } + }, + } satisfies FilesSdkConditionalCopyAdapter & + FilesSdkSignedDownloadPolicyAdapter & + FilesSdkSignedUploadPolicyAdapter); + + return createFilesSdkDriver({ + ...filesOptions, + adapter, + }); +} + +export { mapS3Error, s3 } from 'files-sdk/s3'; +export type { S3Adapter, S3AdapterOptions, S3Sdk } from 'files-sdk/s3'; diff --git a/src/files-sdk/s3/s3.driver.spec.ts b/src/files-sdk/s3/s3.driver.spec.ts new file mode 100644 index 0000000..3cde238 --- /dev/null +++ b/src/files-sdk/s3/s3.driver.spec.ts @@ -0,0 +1,100 @@ +import { CopyObjectCommand, S3Client } from '@aws-sdk/client-s3'; + +import { StorageClient } from '../../storage.client.js'; +import { StorageErrorCode } from '../../storage.error.js'; +import { createS3StorageDriver } from './index.js'; + +describe('createS3StorageDriver', () => { + it('uses the package-owned S3 adapter and exposes conditional promotion', async () => { + const send = vi + .spyOn(S3Client.prototype, 'send') + .mockResolvedValue({} as never); + const client = new StorageClient( + 'objects', + createS3StorageDriver({ + adapter: { + bucket: 'private-bucket', + credentials: { + accessKeyId: 'test', + secretAccessKey: 'test', + }, + region: 'us-east-1', + }, + }), + ); + + expect(client.capabilities.conditionalCopy).toEqual({ + etag: true, + supported: true, + version: true, + }); + expect(client.capabilities.signedUploadPolicy).toEqual({ + contentType: true, + sizeRange: true, + }); + expect(client.capabilities.signedDownloadPolicy).toEqual({ + expiresIn: true, + }); + + await client.promote('staging/a b.png', 'final/image.png', { + sourceEtag: '"etag-1"', + sourceVersion: 'version/1', + }); + + expect(send).toHaveBeenCalledTimes(1); + const command = send.mock.calls[0]?.[0]; + expect(command).toBeInstanceOf(CopyObjectCommand); + expect((command as CopyObjectCommand).input).toEqual({ + Bucket: 'private-bucket', + CopySource: 'private-bucket/staging%2Fa%20b.png?versionId=version%2F1', + CopySourceIfMatch: '"etag-1"', + Key: 'final/image.png', + }); + }); + + it('does not claim expiring downloads for a permanent public base URL', () => { + const driver = createS3StorageDriver({ + adapter: { + bucket: 'public-bucket', + credentials: { + accessKeyId: 'test', + secretAccessKey: 'test', + }, + publicBaseUrl: 'https://cdn.example.test', + region: 'us-east-1', + }, + }); + + expect(driver.capabilities.signedDownloadPolicy).toEqual({ + expiresIn: false, + }); + }); + + it('maps a failed S3 source precondition to a storage conflict', async () => { + vi.spyOn(S3Client.prototype, 'send').mockRejectedValue( + Object.assign(new Error('source changed'), { + $metadata: { httpStatusCode: 412 }, + name: 'PreconditionFailed', + }), + ); + const client = new StorageClient( + 'objects', + createS3StorageDriver({ + adapter: { + bucket: 'private-bucket', + credentials: { + accessKeyId: 'test', + secretAccessKey: 'test', + }, + region: 'us-east-1', + }, + }), + ); + + await expect( + client.promote('staging/image.png', 'final/image.png', { + sourceEtag: '"old-etag"', + }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + }); +}); diff --git a/src/gateway/index.ts b/src/gateway/index.ts index 4bfc4b5..38300b6 100644 --- a/src/gateway/index.ts +++ b/src/gateway/index.ts @@ -1,6 +1,10 @@ export { StorageGatewayModule } from './storage-gateway.module.js'; export { StorageGatewayOperation, + type ParsedStorageGatewayKey, + type StorageGatewayKeyPolicy, + type StorageGatewayKeyPolicyContext, + type StorageGatewayKeyTarget, type StorageGatewayMode, type StorageGatewayOperation as StorageGatewayOperationName, type StorageGatewayOptions, diff --git a/src/gateway/storage-gateway.controller.ts b/src/gateway/storage-gateway.controller.ts index 14ffc27..9506d24 100644 --- a/src/gateway/storage-gateway.controller.ts +++ b/src/gateway/storage-gateway.controller.ts @@ -21,7 +21,11 @@ import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import type { ReadableStream as NodeReadableStream } from 'node:stream/web'; -import { StorageError, StorageErrorCode } from '../storage.error.js'; +import { + StorageError, + StorageErrorCode, + isStorageError, +} from '../storage.error.js'; import { StorageService } from '../storage.service.js'; import type { StorageByteRange, @@ -32,10 +36,16 @@ import type { } from '../storage.types.js'; import { StorageGatewayGuard } from './storage-gateway.guard.js'; import { FASTIFY_STORAGE_UPLOAD_CONFIG } from './storage-gateway-fastify-parser.js'; -import { STORAGE_GATEWAY_OPTIONS } from './storage-gateway.tokens.js'; +import { + STORAGE_GATEWAY_KEY_POLICY, + STORAGE_GATEWAY_OPTIONS, +} from './storage-gateway.tokens.js'; import { StorageGatewayOperation, + type ParsedStorageGatewayKey, type ResolvedStorageGatewayOptions, + type StorageGatewayKeyPolicy, + type StorageGatewayKeyTarget, type StorageGatewayOperation as StorageGatewayOperationName, } from './storage-gateway.types.js'; @@ -54,6 +64,106 @@ interface RequestWrapper { headers?: Record; } +const MAX_GATEWAY_PATH_LENGTH = 1024; + +function parseGatewayPath( + value: string | undefined, + target: StorageGatewayKeyTarget, +): ParsedStorageGatewayKey | undefined { + if (value === undefined) { + if (target === 'prefix') { + return undefined; + } + throw new BadRequestException(`${target} is required.`); + } + if (value.length > MAX_GATEWAY_PATH_LENGTH) { + throw new BadRequestException( + `${target} cannot exceed ${MAX_GATEWAY_PATH_LENGTH} characters.`, + ); + } + if (value.length === 0) { + if (target === 'prefix') { + return Object.freeze({ + segments: Object.freeze([]), + trailingSlash: false, + value, + }); + } + throw new BadRequestException(`${target} must be a non-empty string.`); + } + if ( + [...value].some((character) => { + const codePoint = character.codePointAt(0); + return ( + character === '\\' || + codePoint === undefined || + codePoint <= 0x1f || + codePoint === 0x7f + ); + }) + ) { + throw new BadRequestException( + `${target} cannot contain control characters or backslashes.`, + ); + } + if (target === 'pattern') { + return Object.freeze({ + segments: Object.freeze(value.split('/')), + trailingSlash: value.endsWith('/'), + value, + }); + } + if (value.startsWith('/') || value.trim() !== value) { + throw new BadRequestException( + `${target} must be a relative object path without surrounding whitespace.`, + ); + } + const trailingSlash = value.endsWith('/'); + const segments = value.split('/'); + if (trailingSlash) { + segments.pop(); + } + if ( + segments.some( + (segment) => segment.length === 0 || segment === '.' || segment === '..', + ) + ) { + throw new BadRequestException( + `${target} cannot contain empty, dot, or parent path segments.`, + ); + } + return Object.freeze({ + segments: Object.freeze(segments), + trailingSlash, + value, + }); +} + +function exactContentType(value: string, field: string): string { + const normalized = value.toLowerCase(); + if ( + value !== normalized || + !/^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/u.test(value) + ) { + throw new BadRequestException( + `${field} must be a lowercase exact MIME type without parameters.`, + ); + } + return normalized; +} + +function safeContentDisposition(value: string | undefined): string | undefined { + if (value === undefined) { + return undefined; + } + if (value !== 'attachment' && value !== 'inline') { + throw new BadRequestException( + 'responseContentDisposition must be either "attachment" or "inline".', + ); + } + return value; +} + function recordOf(value: unknown, label: string): Record { if (typeof value !== 'object' || value === null || Array.isArray(value)) { throw new BadRequestException(`${label} must be a JSON object.`); @@ -273,7 +383,7 @@ function toHttpException(error: unknown): HttpException { if (error instanceof HttpException) { return error; } - if (error instanceof StorageError) { + if (isStorageError(error)) { return new HttpException( { error: { @@ -304,6 +414,8 @@ export class StorageGatewayController { private readonly storage: StorageService, @Inject(STORAGE_GATEWAY_OPTIONS) private readonly options: ResolvedStorageGatewayOptions, + @Inject(STORAGE_GATEWAY_KEY_POLICY) + private readonly keyPolicy: StorageGatewayKeyPolicy, ) {} @Get('object') @@ -313,23 +425,31 @@ export class StorageGatewayController { @Query('rangeStart') rangeStart: string | undefined, @Query('rangeEnd') rangeEnd: string | undefined, @Query('disposition') disposition: string | undefined, + @Req() request: unknown, @Res() response: unknown, ): Promise { await this.#run(async () => { this.#assertAllowed(StorageGatewayOperation.DOWNLOAD); - const objectKey = this.#key(key); + const responseContentDisposition = safeContentDisposition(disposition); const client = this.storage.use(this.options.store); const forceProxy = proxy === 'true' || proxy === '1'; const canSign = this.options.operations.has(StorageGatewayOperation.SIGN_DOWNLOAD) && - client.capabilities.signedDownload.supported; + client.capabilities.signedDownload.supported && + client.capabilities.signedDownloadPolicy?.expiresIn === true; if (!forceProxy && this.options.mode !== 'proxy' && canSign) { try { - const url = await client.signDownload(objectKey, { + const signedObjectKey = await this.#resolvePath( + request, + StorageGatewayOperation.SIGN_DOWNLOAD, + 'key', + key, + ); + const url = await client.signDownload(signedObjectKey, { expiresIn: this.options.defaultSignedUrlExpiresIn, - ...(disposition !== undefined && { - responseContentDisposition: disposition, + ...(responseContentDisposition !== undefined && { + responseContentDisposition, }), }); const raw = rawResponseOf(response); @@ -340,7 +460,7 @@ export class StorageGatewayController { } catch (error) { if ( this.options.mode !== 'hybrid' || - !(error instanceof StorageError) || + !isStorageError(error) || error.code !== StorageErrorCode.NOT_SUPPORTED ) { throw error; @@ -353,12 +473,18 @@ export class StorageGatewayController { 'Signed downloads are unavailable for this store.', { code: StorageErrorCode.NOT_SUPPORTED, - key: objectKey, permanent: true, }, ); } + const objectKey = await this.#resolvePath( + request, + StorageGatewayOperation.DOWNLOAD, + 'key', + key, + ); + const start = queryInteger(rangeStart, 'rangeStart'); const end = queryInteger(rangeEnd, 'rangeEnd'); if (end !== undefined && start === undefined) { @@ -435,7 +561,12 @@ export class StorageGatewayController { }, ); } - const objectKey = this.#key(key); + const objectKey = await this.#resolvePath( + request, + StorageGatewayOperation.UPLOAD, + 'key', + key, + ); const transportContentType = requestHeader(request, 'content-type') ?.split(';', 1)[0] ?.trim() @@ -467,9 +598,11 @@ export class StorageGatewayController { const body = Readable.toWeb( Readable.from(enforceUploadLimit(source, this.options.maxUploadBytes)), ) as ReadableStream; - const contentType = + const contentType = exactContentType( requestHeader(request, 'x-storage-content-type') ?? - 'application/octet-stream'; + 'application/octet-stream', + 'x-storage-content-type', + ); const result = await this.storage .use(this.options.store) .upload(objectKey, body, { contentType, multipart: true }); @@ -480,13 +613,20 @@ export class StorageGatewayController { @Head('metadata') async head( @Query('key') key: string | undefined, + @Req() request: unknown, @Res() response: unknown, ): Promise { await this.#run(async () => { this.#assertAllowed(StorageGatewayOperation.HEAD); + const objectKey = await this.#resolvePath( + request, + StorageGatewayOperation.HEAD, + 'key', + key, + ); const metadata = await this.storage .use(this.options.store) - .head(this.#key(key)); + .head(objectKey); const raw = rawResponseOf(response); raw.statusCode = HttpStatus.OK; setObjectHeaders(raw, metadata); @@ -497,10 +637,17 @@ export class StorageGatewayController { @Delete('object') async delete( @Query('key') key: string | undefined, + @Req() request: unknown, ): Promise> { return this.#run(async () => { this.#assertAllowed(StorageGatewayOperation.DELETE); - await this.storage.use(this.options.store).delete(this.#key(key)); + const objectKey = await this.#resolvePath( + request, + StorageGatewayOperation.DELETE, + 'key', + key, + ); + await this.storage.use(this.options.store).delete(objectKey); return { data: { deleted: true } }; }); } @@ -511,6 +658,7 @@ export class StorageGatewayController { @Query('cursor') cursor: string | undefined, @Query('limit') limit: string | undefined, @Query('delimiter') delimiter: string | undefined, + @Req() request: unknown, ): Promise> { return this.#run(async () => { this.#assertAllowed(StorageGatewayOperation.LIST); @@ -521,11 +669,17 @@ export class StorageGatewayController { `limit must be between 1 and ${this.options.maxListResults}.`, ); } + const resolvedPrefix = await this.#resolvePath( + request, + StorageGatewayOperation.LIST, + 'prefix', + prefix, + ); const result = await this.storage.use(this.options.store).list({ ...(cursor !== undefined && { cursor }), ...(delimiter !== undefined && { delimiter }), limit: parsedLimit, - ...(prefix !== undefined && { prefix }), + prefix: resolvedPrefix, }); return { data: result }; }); @@ -539,10 +693,22 @@ export class StorageGatewayController { @Query('maxResults') maxResults: string | undefined, @Query('match') match: string | undefined, @Query('caseInsensitive') caseInsensitive: string | undefined, + @Req() request: unknown, ): Promise> { return this.#run(async () => { this.#assertAllowed(StorageGatewayOperation.SEARCH); - const searchPattern = this.#key(pattern, 'pattern'); + const searchPattern = await this.#resolvePath( + request, + StorageGatewayOperation.SEARCH, + 'pattern', + pattern, + ); + const resolvedPrefix = await this.#resolvePath( + request, + StorageGatewayOperation.SEARCH, + 'prefix', + prefix, + ); if ( match !== undefined && !['glob', 'regex', 'substring', 'exact'].includes(match) @@ -582,7 +748,7 @@ export class StorageGatewayController { match: match as 'glob' | 'regex' | 'substring' | 'exact', }), maxResults: parsedMaxResults, - ...(prefix !== undefined && { prefix }), + prefix: resolvedPrefix, })) { objects.push(object); if (objects.length === parsedMaxResults) { @@ -596,6 +762,7 @@ export class StorageGatewayController { @Post('sign-download') async signDownload( @Body() body: unknown, + @Req() request: unknown, ): Promise> { return this.#run(async () => { this.#assertAllowed(StorageGatewayOperation.SIGN_DOWNLOAD); @@ -609,27 +776,43 @@ export class StorageGatewayController { ); } const input = recordOf(body, 'body'); - const responseContentDisposition = optionalString( - input, - 'responseContentDisposition', + const responseContentDisposition = safeContentDisposition( + optionalString(input, 'responseContentDisposition'), ); const options: StorageSignedDownloadOptions = { - expiresIn: - optionalPositiveInteger(input, 'expiresIn') ?? - this.options.defaultSignedUrlExpiresIn, + expiresIn: this.#signedExpiry(input), ...(responseContentDisposition !== undefined && { responseContentDisposition, }), }; - const url = await this.storage - .use(this.options.store) - .signDownload(requiredString(input, 'key'), options); + const objectKey = await this.#resolvePath( + request, + StorageGatewayOperation.SIGN_DOWNLOAD, + 'key', + requiredString(input, 'key'), + ); + const client = this.storage.use(this.options.store); + if (client.capabilities.signedDownloadPolicy?.expiresIn !== true) { + throw new StorageError( + 'This store cannot prove that signed-download expiry is provider-enforced.', + { + code: StorageErrorCode.NOT_SUPPORTED, + key: objectKey, + operation: 'signDownload', + permanent: true, + }, + ); + } + const url = await client.signDownload(objectKey, options); return { data: { url } }; }); } @Post('sign-upload') - async signUpload(@Body() body: unknown): Promise> { + async signUpload( + @Body() body: unknown, + @Req() request: unknown, + ): Promise> { return this.#run(async () => { this.#assertAllowed(StorageGatewayOperation.SIGN_UPLOAD); if (this.options.mode === 'proxy') { @@ -642,8 +825,32 @@ export class StorageGatewayController { ); } const input = recordOf(body, 'body'); - const contentType = optionalString(input, 'contentType'); - const maxSize = optionalPositiveInteger(input, 'maxSize'); + const rawContentType = optionalString(input, 'contentType'); + if (rawContentType === undefined) { + throw new BadRequestException( + 'contentType is required for signed uploads.', + ); + } + const contentType = exactContentType(rawContentType, 'contentType'); + if (!this.options.signedUploadContentTypes.has(contentType)) { + throw new BadRequestException( + `contentType "${contentType}" is not allowed for signed uploads.`, + ); + } + const requestedMaxSize = optionalPositiveInteger(input, 'maxSize'); + if ( + requestedMaxSize !== undefined && + requestedMaxSize > this.options.maxSignedUploadBytes + ) { + throw new StorageError( + `Signed upload exceeds the ${this.options.maxSignedUploadBytes}-byte gateway limit.`, + { + code: StorageErrorCode.LIMIT_EXCEEDED, + permanent: true, + }, + ); + } + const maxSize = requestedMaxSize ?? this.options.maxSignedUploadBytes; const minSize = optionalNonNegativeInteger(input, 'minSize'); if (maxSize !== undefined && minSize !== undefined && minSize > maxSize) { throw new BadRequestException( @@ -651,16 +858,34 @@ export class StorageGatewayController { ); } const options: StorageSignedUploadOptions = { - expiresIn: - optionalPositiveInteger(input, 'expiresIn') ?? - this.options.defaultSignedUrlExpiresIn, - ...(contentType !== undefined && { contentType }), - ...(maxSize !== undefined && { maxSize }), + contentType, + expiresIn: this.#signedExpiry(input), + maxSize, ...(minSize !== undefined && { minSize }), }; - const result = await this.storage - .use(this.options.store) - .signUpload(requiredString(input, 'key'), options); + const objectKey = await this.#resolvePath( + request, + StorageGatewayOperation.SIGN_UPLOAD, + 'key', + requiredString(input, 'key'), + ); + const client = this.storage.use(this.options.store); + const policyCapability = client.capabilities.signedUploadPolicy; + if ( + policyCapability?.contentType !== true || + policyCapability.sizeRange !== true + ) { + throw new StorageError( + 'This store cannot prove that signed-upload MIME and size constraints are provider-enforced.', + { + code: StorageErrorCode.NOT_SUPPORTED, + key: objectKey, + operation: 'signUpload', + permanent: true, + }, + ); + } + const result = await client.signUpload(objectKey, options); return { data: result }; }); } @@ -668,13 +893,26 @@ export class StorageGatewayController { @Post('copy') async copy( @Body() body: unknown, + @Req() request: unknown, ): Promise> { return this.#run(async () => { this.#assertAllowed(StorageGatewayOperation.COPY); const input = recordOf(body, 'body'); - await this.storage - .use(this.options.store) - .copy(requiredString(input, 'from'), requiredString(input, 'to')); + const [from, to] = await Promise.all([ + this.#resolvePath( + request, + StorageGatewayOperation.COPY, + 'from', + requiredString(input, 'from'), + ), + this.#resolvePath( + request, + StorageGatewayOperation.COPY, + 'to', + requiredString(input, 'to'), + ), + ]); + await this.storage.use(this.options.store).copy(from, to); return { data: { copied: true } }; }); } @@ -682,13 +920,26 @@ export class StorageGatewayController { @Post('move') async move( @Body() body: unknown, + @Req() request: unknown, ): Promise> { return this.#run(async () => { this.#assertAllowed(StorageGatewayOperation.MOVE); const input = recordOf(body, 'body'); - await this.storage - .use(this.options.store) - .move(requiredString(input, 'from'), requiredString(input, 'to')); + const [from, to] = await Promise.all([ + this.#resolvePath( + request, + StorageGatewayOperation.MOVE, + 'from', + requiredString(input, 'from'), + ), + this.#resolvePath( + request, + StorageGatewayOperation.MOVE, + 'to', + requiredString(input, 'to'), + ), + ]); + await this.storage.use(this.options.store).move(from, to); return { data: { moved: true } }; }); } @@ -707,11 +958,43 @@ export class StorageGatewayController { } } - #key(value: string | undefined, label = 'key'): string { - if (value === undefined || value.length === 0) { - throw new BadRequestException(`${label} is required.`); + async #resolvePath( + request: unknown, + operation: StorageGatewayOperationName, + target: StorageGatewayKeyTarget, + value: string | undefined, + ): Promise { + const input = parseGatewayPath(value, target); + const resolved = await this.keyPolicy.resolve({ + input, + operation, + request, + target, + }); + const parsed = parseGatewayPath(resolved, target); + if (parsed === undefined) { + throw new StorageError( + `Storage gateway key policy did not resolve ${target}.`, + { + code: StorageErrorCode.UNAUTHORIZED, + operation, + permanent: true, + }, + ); + } + return parsed.value; + } + + #signedExpiry(input: Record): number { + const expiresIn = + optionalPositiveInteger(input, 'expiresIn') ?? + this.options.defaultSignedUrlExpiresIn; + if (expiresIn > this.options.maxSignedUrlExpiresIn) { + throw new BadRequestException( + `expiresIn cannot exceed ${this.options.maxSignedUrlExpiresIn} seconds.`, + ); } - return value; + return expiresIn; } async #run(operation: () => Promise): Promise { diff --git a/src/gateway/storage-gateway.module.ts b/src/gateway/storage-gateway.module.ts index 67b5ac8..c7f5f95 100644 --- a/src/gateway/storage-gateway.module.ts +++ b/src/gateway/storage-gateway.module.ts @@ -10,16 +10,25 @@ import { StorageGatewayFastifyParser } from './storage-gateway-fastify-parser.js import { StorageGatewayGuard } from './storage-gateway.guard.js'; import { STORAGE_GATEWAY_GUARDS, + STORAGE_GATEWAY_KEY_POLICY, STORAGE_GATEWAY_OPTIONS, } from './storage-gateway.tokens.js'; import { StorageGatewayOperation, type ResolvedStorageGatewayOptions, + type StorageGatewayKeyPolicy, type StorageGatewayOptions, } from './storage-gateway.types.js'; const operations = new Set(Object.values(StorageGatewayOperation)); +function isExactContentType(value: unknown): value is string { + return ( + typeof value === 'string' && + /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/u.test(value) + ); +} + function resolveOptions( options: StorageGatewayOptions, ): ResolvedStorageGatewayOptions { @@ -43,6 +52,22 @@ function resolveOptions( 'StorageGatewayModule requires at least one Nest guard. Set allowUnauthenticated: true only for explicit development use.', ); } + if ( + options.keyPolicy === undefined && + options.unsafeAllowUnscopedKeys !== true + ) { + throw new TypeError( + 'StorageGatewayModule requires a keyPolicy. Set unsafeAllowUnscopedKeys: true only as an explicit migration escape hatch.', + ); + } + if ( + options.keyPolicy !== undefined && + options.unsafeAllowUnscopedKeys === true + ) { + throw new TypeError( + 'StorageGatewayModule cannot combine keyPolicy with unsafeAllowUnscopedKeys.', + ); + } if ( options.mode !== undefined && !['proxy', 'signed', 'hybrid'].includes(options.mode) @@ -70,6 +95,44 @@ function resolveOptions( 'defaultSignedUrlExpiresIn must be a positive safe integer.', ); } + const maxSignedUrlExpiresIn = options.maxSignedUrlExpiresIn ?? 3600; + if ( + !Number.isSafeInteger(maxSignedUrlExpiresIn) || + maxSignedUrlExpiresIn <= 0 + ) { + throw new TypeError( + 'maxSignedUrlExpiresIn must be a positive safe integer.', + ); + } + if (defaultSignedUrlExpiresIn > maxSignedUrlExpiresIn) { + throw new TypeError( + 'defaultSignedUrlExpiresIn cannot exceed maxSignedUrlExpiresIn.', + ); + } + const maxSignedUploadBytes = options.maxSignedUploadBytes ?? maxUploadBytes; + if ( + !Number.isSafeInteger(maxSignedUploadBytes) || + maxSignedUploadBytes <= 0 + ) { + throw new TypeError( + 'maxSignedUploadBytes must be a positive safe integer.', + ); + } + const signedUploadContentTypes = options.signedUploadContentTypes ?? [ + 'application/octet-stream', + ]; + if (signedUploadContentTypes.length === 0) { + throw new TypeError( + 'signedUploadContentTypes must contain at least one exact MIME type.', + ); + } + for (const contentType of signedUploadContentTypes) { + if (!isExactContentType(contentType)) { + throw new TypeError( + 'signedUploadContentTypes must contain lowercase exact MIME types without parameters.', + ); + } + } const proxyInlineContentTypes = options.proxyInlineContentTypes ?? []; for (const contentType of proxyInlineContentTypes) { if ( @@ -89,12 +152,15 @@ function resolveOptions( defaultSignedUrlExpiresIn, maxListResults, maxSearchResults, + maxSignedUploadBytes, + maxSignedUrlExpiresIn, maxUploadBytes, mode: options.mode ?? 'hybrid', operations: new Set(options.operations), proxyInlineContentTypes: new Set( proxyInlineContentTypes.map((contentType) => contentType.toLowerCase()), ), + signedUploadContentTypes: new Set(signedUploadContentTypes), ...(options.store !== undefined && { store: options.store }), }; } @@ -112,6 +178,26 @@ export class StorageGatewayModule { inject: [...guardTokens], useFactory: (...guards: CanActivate[]) => guards, }; + const keyPolicyProvider: Provider = + options.keyPolicy === undefined + ? { + provide: STORAGE_GATEWAY_KEY_POLICY, + useValue: { + resolve({ input, target }) { + if (input !== undefined) { + return input.value; + } + if (target === 'prefix') { + return ''; + } + throw new TypeError(`${target} is required.`); + }, + } satisfies StorageGatewayKeyPolicy, + } + : { + provide: STORAGE_GATEWAY_KEY_POLICY, + useExisting: options.keyPolicy, + }; return { module: StorageGatewayModule, @@ -120,6 +206,7 @@ export class StorageGatewayModule { providers: [ { provide: STORAGE_GATEWAY_OPTIONS, useValue: resolved }, guardsProvider, + keyPolicyProvider, StorageGatewayGuard, StorageGatewayFastifyParser, ], diff --git a/src/gateway/storage-gateway.tokens.ts b/src/gateway/storage-gateway.tokens.ts index 091e439..82156bf 100644 --- a/src/gateway/storage-gateway.tokens.ts +++ b/src/gateway/storage-gateway.tokens.ts @@ -5,3 +5,7 @@ export const STORAGE_GATEWAY_OPTIONS = Symbol.for( export const STORAGE_GATEWAY_GUARDS = Symbol.for( '@nestm/storage/gateway/guards', ); + +export const STORAGE_GATEWAY_KEY_POLICY = Symbol.for( + '@nestm/storage/gateway/key-policy', +); diff --git a/src/gateway/storage-gateway.types.ts b/src/gateway/storage-gateway.types.ts index 187d663..8dad76f 100644 --- a/src/gateway/storage-gateway.types.ts +++ b/src/gateway/storage-gateway.types.ts @@ -22,6 +22,32 @@ export type StorageGatewayOperation = export type StorageGatewayMode = 'proxy' | 'signed' | 'hybrid'; +export type StorageGatewayKeyTarget = + 'key' | 'prefix' | 'pattern' | 'from' | 'to'; + +/** A path parsed before it crosses the application's key policy boundary. */ +export interface ParsedStorageGatewayKey { + readonly value: string; + readonly segments: readonly string[]; + readonly trailingSlash: boolean; +} + +export interface StorageGatewayKeyPolicyContext { + readonly operation: StorageGatewayOperation; + readonly target: StorageGatewayKeyTarget; + readonly request: unknown; + /** Undefined only for an omitted list/search prefix. */ + readonly input: ParsedStorageGatewayKey | undefined; +} + +/** + * Resolves an untrusted, parsed gateway path to the exact provider key/prefix. + * Throw to deny. The returned value is parsed again before provider use. + */ +export interface StorageGatewayKeyPolicy { + resolve(context: StorageGatewayKeyPolicyContext): string | Promise; +} + export interface StorageGatewayOptions { /** * Module exporting StorageService. Required when StorageModule is not global. @@ -41,6 +67,16 @@ export interface StorageGatewayOptions { * gateway. */ allowUnauthenticated?: boolean; + /** + * Existing provider implementing the mandatory key authorization/scope + * boundary. It may be request scoped. + */ + keyPolicy?: InjectionToken; + /** + * Migration-only escape hatch preserving caller-controlled keys. It cannot + * be combined with keyPolicy and must never be used on an exposed gateway. + */ + unsafeAllowUnscopedKeys?: boolean; /** Defaults to hybrid: prefer signed downloads, proxy when unavailable. */ mode?: StorageGatewayMode; /** Proxy upload ceiling. Defaults to 100 MiB. */ @@ -59,6 +95,15 @@ export interface StorageGatewayOptions { proxyInlineContentTypes?: readonly string[]; /** Default signed-download lifetime. Defaults to 300 seconds. */ defaultSignedUrlExpiresIn?: number; + /** Hard ceiling for every signed URL. Defaults to 3,600 seconds. */ + maxSignedUrlExpiresIn?: number; + /** Hard signed-upload ceiling. Defaults to maxUploadBytes. */ + maxSignedUploadBytes?: number; + /** + * Exact MIME allowlist for direct uploads. Defaults to only + * application/octet-stream; signed-upload callers must always provide one. + */ + signedUploadContentTypes?: readonly string[]; } export interface ResolvedStorageGatewayOptions { @@ -71,4 +116,7 @@ export interface ResolvedStorageGatewayOptions { maxSearchResults: number; proxyInlineContentTypes: ReadonlySet; defaultSignedUrlExpiresIn: number; + maxSignedUrlExpiresIn: number; + maxSignedUploadBytes: number; + signedUploadContentTypes: ReadonlySet; } diff --git a/src/storage.client.spec.ts b/src/storage.client.spec.ts index bf46161..ae91d31 100644 --- a/src/storage.client.spec.ts +++ b/src/storage.client.spec.ts @@ -87,6 +87,52 @@ describe('StorageClient', () => { await expect(client.exists('archive/one.txt')).resolves.toBe(true); }); + it('promotes only through a driver-declared conditional copy capability', async () => { + const driver = createMemoryStorageDriver(); + const promote = vi.fn(async () => undefined); + Object.defineProperty(driver, 'capabilities', { + value: { + ...driver.capabilities, + conditionalCopy: { + etag: true, + supported: true, + version: false, + }, + }, + }); + driver.promote = promote; + const client = new StorageClient('media', driver); + + await client.file('staging/image.png').promoteTo('final/image.png', { + sourceEtag: '"verified-etag"', + }); + expect(promote).toHaveBeenCalledWith( + 'staging/image.png', + 'final/image.png', + { sourceEtag: '"verified-etag"' }, + ); + expect(() => + client.promote('staging/image.png', 'final/image.png', { + sourceVersion: 'v1', + }), + ).toThrow( + expect.objectContaining({ code: StorageErrorCode.NOT_SUPPORTED }), + ); + }); + + it('rejects unconditional promotion and unsupported drivers', async () => { + const client = new StorageClient('media', createMemoryStorageDriver()); + + expect(() => client.promote('staging.bin', 'final.bin', {})).toThrow( + 'requires sourceEtag', + ); + expect(() => + client.promote('staging.bin', 'final.bin', { sourceEtag: 'etag' }), + ).toThrow( + expect.objectContaining({ code: StorageErrorCode.NOT_SUPPORTED }), + ); + }); + it('classifies invalid owned options before calling the provider', async () => { const client = new StorageClient('media', createMemoryStorageDriver()); diff --git a/src/storage.client.ts b/src/storage.client.ts index 748704d..392646a 100644 --- a/src/storage.client.ts +++ b/src/storage.client.ts @@ -22,6 +22,7 @@ import type { StorageOperationContext, StorageOperationOptions, StoragePlugin, + StoragePromotionOptions, StorageSearchOptions, StorageSignedDownloadOptions, StorageSignedUpload, @@ -223,6 +224,11 @@ export interface StorageFileHandle { destinationKey: string, options?: StorageOperationOptions, ): Promise; + /** Conditionally copies this staged object and deliberately retains it. */ + promoteTo( + destinationKey: string, + options: StoragePromotionOptions, + ): Promise; } export class StorageClient { @@ -260,6 +266,8 @@ export class StorageClient { key, moveTo: (destinationKey, options) => this.move(key, destinationKey, options), + promoteTo: (destinationKey, options) => + this.promote(key, destinationKey, options), signDownload: (options) => this.signDownload(key, options), signUpload: (options) => this.signUpload(key, options), bytes: (options) => this.downloadBytes(key, options), @@ -385,6 +393,60 @@ export class StorageClient { ); } + /** + * Conditionally copies a staged object to a final key. This operation does + * not delete the source; delete it after the application commit succeeds. + */ + promote( + sourceKey: string, + destinationKey: string, + options: StoragePromotionOptions, + ): Promise { + assertKey(sourceKey, 'source key'); + assertKey(destinationKey, 'destination key'); + const sourceEtag = options.sourceEtag; + const sourceVersion = options.sourceVersion; + if (sourceEtag !== undefined && sourceEtag.length === 0) { + invalidArgument('sourceEtag must be a non-empty string.'); + } + if (sourceVersion !== undefined && sourceVersion.length === 0) { + invalidArgument('sourceVersion must be a non-empty string.'); + } + if (sourceEtag === undefined && sourceVersion === undefined) { + invalidArgument('promote requires sourceEtag, sourceVersion, or both.'); + } + + const capability = this.#driver.capabilities.conditionalCopy; + const promote = this.#driver.promote; + if ( + capability?.supported !== true || + promote === undefined || + (sourceEtag !== undefined && !capability.etag) || + (sourceVersion !== undefined && !capability.version) + ) { + throw new StorageError( + `Store "${this.name}" does not support the requested conditional promotion.`, + { + code: StorageErrorCode.NOT_SUPPORTED, + key: sourceKey, + operation: 'promote', + permanent: true, + store: this.name, + }, + ); + } + + return this.#execute( + { + from: sourceKey, + operation: 'promote', + store: this.name, + to: destinationKey, + }, + () => promote.call(this.#driver, sourceKey, destinationKey, options), + ); + } + list(options?: StorageListOptions): Promise { assertListOptions(options); return this.#execute({ operation: 'list', store: this.name }, () => diff --git a/src/storage.driver.ts b/src/storage.driver.ts index 40287a2..c596f01 100644 --- a/src/storage.driver.ts +++ b/src/storage.driver.ts @@ -7,6 +7,7 @@ import type { StorageObject, StorageObjectMetadata, StorageOperationOptions, + StoragePromotionOptions, StorageSearchOptions, StorageSignedDownloadOptions, StorageSignedUpload, @@ -44,6 +45,15 @@ export interface StorageDriver { destinationKey: string, options?: StorageOperationOptions, ): Promise; + /** + * Conditionally copies a staged object to its final key. The source remains + * in place so applications can delete it only after their metadata commit. + */ + promote?( + sourceKey: string, + destinationKey: string, + options: StoragePromotionOptions, + ): Promise; list(options?: StorageListOptions): Promise; search( pattern: string | RegExp, diff --git a/src/storage.error.spec.ts b/src/storage.error.spec.ts new file mode 100644 index 0000000..210ddca --- /dev/null +++ b/src/storage.error.spec.ts @@ -0,0 +1,75 @@ +import { + StorageError, + StorageErrorCode, + isStorageError, + normalizeStorageError, +} from './storage.error.js'; + +describe('StorageError', () => { + it('is recognized across duplicated package copies', () => { + const brand = Symbol.for('@nestm/storage/StorageError'); + class ForeignStorageError extends Error { + override readonly name = 'StorageError'; + readonly [brand] = true; + readonly code = StorageErrorCode.NOT_FOUND; + readonly store = 'foreign'; + readonly operation = 'head'; + readonly key = 'missing.bin'; + readonly aborted = false; + readonly timedOut = false; + readonly permanent = true; + } + + const error = new ForeignStorageError('missing'); + expect(isStorageError(error)).toBe(true); + expect(normalizeStorageError(error)).toBe(error); + }); + + it('recognizes the exact legacy structural shape without a brand', () => { + const error = Object.assign(new Error('legacy'), { + aborted: false, + code: StorageErrorCode.PROVIDER, + key: undefined, + name: 'StorageError', + operation: undefined, + permanent: false, + store: undefined, + timedOut: false, + }); + + expect(isStorageError(error)).toBe(true); + }); + + it('does not classify unrelated errors from a coincidental code or name', () => { + expect( + isStorageError( + Object.assign(new Error('provider'), { + code: StorageErrorCode.NOT_FOUND, + }), + ), + ).toBe(false); + expect( + isStorageError( + Object.assign(new Error('incomplete'), { + code: StorageErrorCode.NOT_FOUND, + name: 'StorageError', + }), + ), + ).toBe(false); + expect(isStorageError({ code: StorageErrorCode.NOT_FOUND })).toBe(false); + }); + + it('brands owned errors without exposing the marker during enumeration', () => { + const error = new StorageError('missing', { + code: StorageErrorCode.NOT_FOUND, + }); + + expect(isStorageError(error)).toBe(true); + expect( + Object.getOwnPropertyDescriptor( + error, + Symbol.for('@nestm/storage/StorageError'), + ), + ).toMatchObject({ enumerable: false, value: true, writable: false }); + }); +}); diff --git a/src/storage.error.ts b/src/storage.error.ts index 6d215d5..971afc9 100644 --- a/src/storage.error.ts +++ b/src/storage.error.ts @@ -14,6 +14,12 @@ export const StorageErrorCode = { export type StorageErrorCode = (typeof StorageErrorCode)[keyof typeof StorageErrorCode]; +const STORAGE_ERROR_BRAND = Symbol.for('@nestm/storage/StorageError'); + +function isStorageErrorCode(value: unknown): value is StorageErrorCode { + return Object.values(StorageErrorCode).includes(value as StorageErrorCode); +} + export interface StorageErrorOptions { code: StorageErrorCode; store?: string; @@ -26,6 +32,7 @@ export interface StorageErrorOptions { } export class StorageError extends Error { + declare readonly [STORAGE_ERROR_BRAND]: true; readonly code: StorageErrorCode; readonly store: string | undefined; readonly operation: string | undefined; @@ -37,6 +44,12 @@ export class StorageError extends Error { constructor(message: string, options: StorageErrorOptions) { super(message, { cause: options.cause }); + Object.defineProperty(this, STORAGE_ERROR_BRAND, { + configurable: false, + enumerable: false, + value: true, + writable: false, + }); this.name = 'StorageError'; this.code = options.code; this.store = options.store; @@ -50,7 +63,41 @@ export class StorageError extends Error { } export function isStorageError(error: unknown): error is StorageError { - return error instanceof StorageError; + if (error instanceof StorageError) { + return true; + } + if (!(error instanceof Error) || error.name !== 'StorageError') { + return false; + } + + try { + const candidate = error as Error & { + readonly [STORAGE_ERROR_BRAND]?: unknown; + readonly aborted?: unknown; + readonly code?: unknown; + readonly key?: unknown; + readonly operation?: unknown; + readonly permanent?: unknown; + readonly store?: unknown; + readonly timedOut?: unknown; + }; + const hasCompatibleBrand = + candidate[STORAGE_ERROR_BRAND] === true || + candidate[STORAGE_ERROR_BRAND] === undefined; + return ( + hasCompatibleBrand && + isStorageErrorCode(candidate.code) && + typeof candidate.aborted === 'boolean' && + typeof candidate.timedOut === 'boolean' && + typeof candidate.permanent === 'boolean' && + (candidate.store === undefined || typeof candidate.store === 'string') && + (candidate.operation === undefined || + typeof candidate.operation === 'string') && + (candidate.key === undefined || typeof candidate.key === 'string') + ); + } catch { + return false; + } } export function normalizeStorageError( diff --git a/src/storage.types.ts b/src/storage.types.ts index cdb465e..4055279 100644 --- a/src/storage.types.ts +++ b/src/storage.types.ts @@ -31,6 +31,17 @@ export interface StorageOperationOptions { retries?: StorageRetryOptions; } +/** + * Preconditions for promoting a staged object to its final key. At least one + * source identity must be supplied; unsupported identities fail closed. + */ +export interface StoragePromotionOptions extends StorageOperationOptions { + /** Copy only the source object whose provider ETag exactly matches. */ + sourceEtag?: string; + /** Copy this immutable provider version of the source object. */ + sourceVersion?: string; +} + export interface StorageMultipartOptions { partSize?: number; concurrency?: number; @@ -141,6 +152,24 @@ export interface StorageSignedUrlCapability { maxExpiresIn?: number; } +export interface StorageConditionalCopyCapability { + supported: boolean; + etag: boolean; + version: boolean; +} + +export interface StorageSignedUploadPolicyCapability { + /** The signed request fixes the exact declared content type. */ + contentType: boolean; + /** The signed request enforces the requested min/max byte range. */ + sizeRange: boolean; +} + +export interface StorageSignedDownloadPolicyCapability { + /** Every generated URL honors the requested expiry. */ + expiresIn: boolean; +} + export interface StorageCapabilities { rangeRead: boolean; /** True when the provider reports native byte-level upload progress. */ @@ -150,12 +179,25 @@ export interface StorageCapabilities { cacheControl: boolean; resumableUpload: boolean; serverSideCopy: boolean; + /** + * Conditional server-side copy used to promote an already verified staged + * object without a validation/copy race. Absent means unsupported for + * compatibility with drivers built against earlier package versions. + */ + conditionalCopy?: StorageConditionalCopyCapability; signedDownload: StorageSignedUrlCapability; + /** Expiry guarantees enforced by the provider adapter. */ + signedDownloadPolicy?: StorageSignedDownloadPolicyCapability; /** * Some providers decide support from credentials or requested constraints, * so direct-upload support can only be known when the call is attempted. */ signedUpload: boolean | 'runtime'; + /** + * Constraints cryptographically/provider-policy enforced by direct upload. + * Absent means callers must not assume request options are enforced. + */ + signedUploadPolicy?: StorageSignedUploadPolicyCapability; } export interface StorageBulkOptions { @@ -216,6 +258,7 @@ export type StorageOperationName = | 'delete' | 'copy' | 'move' + | 'promote' | 'list' | 'search' | 'signDownload' diff --git a/test/gateway.e2e-spec.ts b/test/gateway.e2e-spec.ts index 6630dcc..9657283 100644 --- a/test/gateway.e2e-spec.ts +++ b/test/gateway.e2e-spec.ts @@ -17,11 +17,21 @@ import request from 'supertest'; import { StorageGatewayModule, StorageGatewayOperation, + type StorageGatewayKeyPolicy, + type StorageGatewayKeyPolicyContext, + type StorageGatewayMode, + type StorageGatewayOperationName, } from '../src/gateway/index.js'; import { StorageModule } from '../src/storage.module.js'; +import type { StorageDriver } from '../src/storage.driver.js'; +import { createS3StorageDriver } from '../src/files-sdk/s3/index.js'; import { createMemoryStorageDriver } from '../src/testing/index.js'; let guardCalls = 0; +const keyPolicyCalls: Array<{ + operation: StorageGatewayOperationName; + target: StorageGatewayKeyPolicyContext['target']; +}> = []; @Injectable() class AllowGuard implements CanActivate { @@ -37,6 +47,24 @@ class AllowGuard implements CanActivate { }) class GuardModule {} +@Injectable() +class ScopedKeyPolicy implements StorageGatewayKeyPolicy { + resolve(context: StorageGatewayKeyPolicyContext): string { + keyPolicyCalls.push({ + operation: context.operation, + target: context.target, + }); + const input = context.input?.value ?? ''; + return `scoped/${input}`; + } +} + +@Module({ + providers: [ScopedKeyPolicy], + exports: [ScopedKeyPolicy], +}) +class KeyPolicyModule {} + @Controller() class UnrelatedBinaryController { @Put('unrelated-binary') @@ -50,11 +78,24 @@ type AdapterName = 'express' | 'fastify'; async function createApp( adapterName: AdapterName, maxUploadBytes = 1024, + gateway: { + defaultSignedUrlExpiresIn?: number; + driver?: StorageDriver; + maxSignedUploadBytes?: number; + maxSignedUrlExpiresIn?: number; + mode?: StorageGatewayMode; + operations?: readonly StorageGatewayOperationName[]; + signedUploadContentTypes?: readonly string[]; + } = {}, ): Promise { const storage = StorageModule.forRoot({ stores: [ { - driver: createMemoryStorageDriver(), + driver: + gateway.driver ?? + createMemoryStorageDriver({ + adapter: { initial: { 'outside/secret.txt': 'secret' } }, + }), name: 'gateway', }, ], @@ -63,11 +104,21 @@ async function createApp( controllers: [UnrelatedBinaryController], imports: [ StorageGatewayModule.register({ + ...(gateway.defaultSignedUrlExpiresIn !== undefined && { + defaultSignedUrlExpiresIn: gateway.defaultSignedUrlExpiresIn, + }), guards: [AllowGuard], - imports: [storage, GuardModule], + imports: [storage, GuardModule, KeyPolicyModule], + keyPolicy: ScopedKeyPolicy, maxUploadBytes, - mode: 'proxy', - operations: [ + ...(gateway.maxSignedUploadBytes !== undefined && { + maxSignedUploadBytes: gateway.maxSignedUploadBytes, + }), + ...(gateway.maxSignedUrlExpiresIn !== undefined && { + maxSignedUrlExpiresIn: gateway.maxSignedUrlExpiresIn, + }), + mode: gateway.mode ?? 'proxy', + operations: gateway.operations ?? [ StorageGatewayOperation.UPLOAD, StorageGatewayOperation.DOWNLOAD, StorageGatewayOperation.HEAD, @@ -77,6 +128,9 @@ async function createApp( StorageGatewayOperation.MOVE, StorageGatewayOperation.DELETE, ], + ...(gateway.signedUploadContentTypes !== undefined && { + signedUploadContentTypes: gateway.signedUploadContentTypes, + }), store: 'gateway', }), ], @@ -98,6 +152,7 @@ describe.each(['express', 'fastify'])( beforeAll(async () => { guardCalls = 0; + keyPolicyCalls.length = 0; app = await createApp(adapterName); }); @@ -124,6 +179,12 @@ describe.each(['express', 'fastify'])( expect(response.headers['content-disposition']).toBe('attachment'); expect(response.headers['x-content-type-options']).toBe('nosniff'); expect(guardCalls).toBeGreaterThanOrEqual(2); + expect( + keyPolicyCalls.some( + ({ operation, target }) => + operation === StorageGatewayOperation.UPLOAD && target === 'key', + ), + ).toBe(true); }); it('lists, searches, copies, moves, heads, and deletes allowed objects', async () => { @@ -132,6 +193,12 @@ describe.each(['express', 'fastify'])( .query({ prefix: 'folder/' }) .expect(200); expect(list.body.data.items).toHaveLength(1); + expect(list.body.data.items[0].key).toBe('scoped/folder/hello.txt'); + const tenantRoot = await request(app.getHttpServer()) + .get('/storage/list') + .expect(200); + expect(tenantRoot.body.data.items).toHaveLength(1); + expect(tenantRoot.body.data.items[0].key).not.toContain('outside/'); const search = await request(app.getHttpServer()) .get('/storage/search') @@ -156,6 +223,19 @@ describe.each(['express', 'fastify'])( .delete('/storage/object') .query({ key: 'moved.txt' }) .expect(200); + expect(keyPolicyCalls).toEqual( + expect.arrayContaining([ + { operation: StorageGatewayOperation.LIST, target: 'prefix' }, + { operation: StorageGatewayOperation.SEARCH, target: 'pattern' }, + { operation: StorageGatewayOperation.SEARCH, target: 'prefix' }, + { operation: StorageGatewayOperation.COPY, target: 'from' }, + { operation: StorageGatewayOperation.COPY, target: 'to' }, + { operation: StorageGatewayOperation.MOVE, target: 'from' }, + { operation: StorageGatewayOperation.MOVE, target: 'to' }, + { operation: StorageGatewayOperation.HEAD, target: 'key' }, + { operation: StorageGatewayOperation.DELETE, target: 'key' }, + ]), + ); }); it('returns 403 for operations outside the allowlist', async () => { @@ -175,6 +255,17 @@ describe.each(['express', 'fastify'])( .expect(400); }); + it('rejects ambiguous object paths before the key policy runs', async () => { + await request(app.getHttpServer()) + .get('/storage/object') + .query({ key: '../escape.txt' }) + .expect(400); + await request(app.getHttpServer()) + .get('/storage/list') + .query({ prefix: 'folder//nested' }) + .expect(400); + }); + it('caps materialized search results', async () => { await request(app.getHttpServer()) .get('/storage/search') @@ -211,6 +302,52 @@ describe.each(['express', 'fastify'])( }, ); +describe.each(['express', 'fastify'])( + 'StorageGatewayModule signed S3 policies (%s)', + (adapterName) => { + it('returns only provider-enforced upload and download policies', async () => { + const app = await createApp(adapterName, 1024, { + defaultSignedUrlExpiresIn: 60, + driver: createS3StorageDriver({ + adapter: { + bucket: 'private-bucket', + credentials: { + accessKeyId: 'test', + secretAccessKey: 'test', + }, + region: 'us-east-1', + }, + }), + maxSignedUploadBytes: 8, + maxSignedUrlExpiresIn: 60, + mode: 'signed', + operations: [ + StorageGatewayOperation.SIGN_DOWNLOAD, + StorageGatewayOperation.SIGN_UPLOAD, + ], + signedUploadContentTypes: ['image/png'], + }); + try { + const upload = await request(app.getHttpServer()) + .post('/storage/sign-upload') + .send({ contentType: 'image/png', key: 'image.png', maxSize: 8 }) + .expect(201); + expect(upload.body.data.method).toBe('POST'); + expect(upload.body.data.fields['Content-Type']).toBe('image/png'); + + const download = await request(app.getHttpServer()) + .post('/storage/sign-download') + .send({ expiresIn: 60, key: 'image.png' }) + .expect(201); + const url = new URL(download.body.data.url); + expect(url.searchParams.get('X-Amz-Expires')).toBe('60'); + } finally { + await app.close(); + } + }); + }, +); + describe('StorageGatewayModule security defaults', () => { it('rejects registration without a guard or explicit development override', () => { expect(() => @@ -226,10 +363,85 @@ describe('StorageGatewayModule security defaults', () => { allowUnauthenticated: true, mode: 'signned' as 'signed', operations: [StorageGatewayOperation.DOWNLOAD], + unsafeAllowUnscopedKeys: true, }), ).toThrow('Unknown storage gateway mode'); }); + it('requires a key policy or the explicitly unsafe migration escape hatch', () => { + expect(() => + StorageGatewayModule.register({ + allowUnauthenticated: true, + operations: [StorageGatewayOperation.DOWNLOAD], + }), + ).toThrow('requires a keyPolicy'); + + expect(() => + StorageGatewayModule.register({ + allowUnauthenticated: true, + keyPolicy: ScopedKeyPolicy, + operations: [StorageGatewayOperation.DOWNLOAD], + unsafeAllowUnscopedKeys: true, + }), + ).toThrow('cannot combine keyPolicy'); + }); + + it('hard-caps signed URL expiry, upload size, and upload MIME', async () => { + const app = await createApp('express', 1024, { + defaultSignedUrlExpiresIn: 60, + maxSignedUploadBytes: 8, + maxSignedUrlExpiresIn: 60, + mode: 'signed', + operations: [ + StorageGatewayOperation.SIGN_DOWNLOAD, + StorageGatewayOperation.SIGN_UPLOAD, + ], + signedUploadContentTypes: ['image/png'], + }); + try { + await request(app.getHttpServer()) + .post('/storage/sign-download') + .send({ expiresIn: 61, key: 'image.png' }) + .expect(400); + await request(app.getHttpServer()) + .post('/storage/sign-download') + .send({ + expiresIn: 60, + key: 'image.png', + responseContentDisposition: 'attachment\r\nx-injected: yes', + }) + .expect(400); + const unsupportedDownload = await request(app.getHttpServer()) + .post('/storage/sign-download') + .send({ + expiresIn: 60, + key: 'image.png', + responseContentDisposition: 'attachment', + }) + .expect(501); + expect(unsupportedDownload.body.error.code).toBe('NOT_SUPPORTED'); + await request(app.getHttpServer()) + .post('/storage/sign-upload') + .send({ contentType: 'text/html', key: 'image.png', maxSize: 8 }) + .expect(400); + await request(app.getHttpServer()) + .post('/storage/sign-upload') + .send({ contentType: 'image/png', key: 'image.png', maxSize: 9 }) + .expect(413); + await request(app.getHttpServer()) + .post('/storage/sign-upload') + .send({ key: 'image.png', maxSize: 8 }) + .expect(400); + const unsupported = await request(app.getHttpServer()) + .post('/storage/sign-upload') + .send({ contentType: 'image/png', key: 'image.png', maxSize: 8 }) + .expect(501); + expect(unsupported.body.error.code).toBe('NOT_SUPPORTED'); + } finally { + await app.close(); + } + }); + it('enforces the streaming upload limit', async () => { const app = await createApp('express', 4); try {