From 8556cc1488cabb274e873c71f8bbeeebb0d6eabb Mon Sep 17 00:00:00 2001 From: Kauan Guesser Date: Tue, 11 Aug 2026 20:24:19 -0300 Subject: [PATCH] feat: add encrypted artifact storage protocol --- .changeset/add-artifact-storage.md | 21 + README.md | 114 +++ package.json | 16 + pnpm-lock.yaml | 135 ++++ scripts/test-packed-core-consumer.mjs | 84 +- src/artifacts/artifact-storage.ts | 802 +++++++++++++++++++ src/artifacts/compatibility.spec.ts | 202 +++++ src/artifacts/crypto/codec.spec.ts | 394 ++++++++++ src/artifacts/crypto/codec.ts | 349 +++++++++ src/artifacts/crypto/context.ts | 71 ++ src/artifacts/crypto/dek-cache.spec.ts | 215 +++++ src/artifacts/crypto/dek-cache.ts | 182 +++++ src/artifacts/crypto/env-config.spec.ts | 208 +++++ src/artifacts/crypto/golden.spec.ts | 66 ++ src/artifacts/crypto/index.ts | 71 ++ src/artifacts/crypto/key-provider.ts | 86 ++ src/artifacts/crypto/kms-provider.spec.ts | 119 +++ src/artifacts/crypto/kms-provider.ts | 115 +++ src/artifacts/crypto/local-provider.spec.ts | 52 ++ src/artifacts/crypto/local-provider.ts | 75 ++ src/artifacts/env-config.ts | 124 +++ src/artifacts/index.spec.ts | 824 ++++++++++++++++++++ src/artifacts/index.ts | 15 + src/artifacts/nest/index.ts | 194 +++++ src/artifacts/nest/storage.module.spec.ts | 121 +++ src/artifacts/object-store.spec.ts | 344 ++++++++ src/artifacts/object-store.ts | 307 ++++++++ src/artifacts/storage-driver.ts | 158 ++++ test/fixtures/cae1-golden.ts | 39 + 29 files changed, 5499 insertions(+), 4 deletions(-) create mode 100644 .changeset/add-artifact-storage.md create mode 100644 src/artifacts/artifact-storage.ts create mode 100644 src/artifacts/compatibility.spec.ts create mode 100644 src/artifacts/crypto/codec.spec.ts create mode 100644 src/artifacts/crypto/codec.ts create mode 100644 src/artifacts/crypto/context.ts create mode 100644 src/artifacts/crypto/dek-cache.spec.ts create mode 100644 src/artifacts/crypto/dek-cache.ts create mode 100644 src/artifacts/crypto/env-config.spec.ts create mode 100644 src/artifacts/crypto/golden.spec.ts create mode 100644 src/artifacts/crypto/index.ts create mode 100644 src/artifacts/crypto/key-provider.ts create mode 100644 src/artifacts/crypto/kms-provider.spec.ts create mode 100644 src/artifacts/crypto/kms-provider.ts create mode 100644 src/artifacts/crypto/local-provider.spec.ts create mode 100644 src/artifacts/crypto/local-provider.ts create mode 100644 src/artifacts/env-config.ts create mode 100644 src/artifacts/index.spec.ts create mode 100644 src/artifacts/index.ts create mode 100644 src/artifacts/nest/index.ts create mode 100644 src/artifacts/nest/storage.module.spec.ts create mode 100644 src/artifacts/object-store.spec.ts create mode 100644 src/artifacts/object-store.ts create mode 100644 src/artifacts/storage-driver.ts create mode 100644 test/fixtures/cae1-golden.ts diff --git a/.changeset/add-artifact-storage.md b/.changeset/add-artifact-storage.md new file mode 100644 index 0000000..3926b9b --- /dev/null +++ b/.changeset/add-artifact-storage.md @@ -0,0 +1,21 @@ +--- +'@nestm/storage': minor +--- + +Add byte-compatible encrypted artifact storage through the +`@nestm/storage/artifacts` and `@nestm/storage/artifacts/nest` entry points. + +The framework-neutral facade stores self-contained CAE1 AES-256-GCM envelopes, +binds ciphertext to its tenant, artifact, version, and path context, supports +local and AWS KMS key providers, and preserves authenticated content types, +bounded reads, versioned bundles, legacy migration reads, and encrypted generic +objects. The Nest entry point composes artifact and object adapters over two +named storage clients and clears cached data keys on shutdown. + +The public protocol keeps the deployed filesystem and object-provider layouts, +reserves configurable ObjectStore namespaces from artifact ids, validates all +artifact/version paths, and bounds ZIP entry count and expansion before any +write. CAE1 now rejects unauthenticated empty content types and writer-side +oversized headers, KMS verifies the stored key id, DEK cache lifetimes are +capped at five minutes, filesystem `.ct` collisions are rejected, and sweepers +skip objects without trustworthy timestamps. diff --git a/README.md b/README.md index 2856f48..0ca9994 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,9 @@ Install only the native SDKs required by the chosen provider. For example: pnpm add @aws-sdk/client-s3 @aws-sdk/s3-presigned-post \ @aws-sdk/s3-request-presigner @aws-sdk/lib-storage +# Encrypted artifact storage with AWS KMS +pnpm add @aws-sdk/client-kms + # Google Cloud Storage pnpm add @google-cloud/storage google-auth-library @@ -88,6 +91,117 @@ storage errors and operation types, and `StorageUploadControl`. It has no NestJS runtime or declaration imports. Provider adapters remain available through `@nestm/storage/files-sdk`. +## Encrypted artifact storage + +The optional `@nestm/storage/artifacts` entry point adds a framework-neutral +artifact facade over the same provider drivers. Every write is a self-contained +CAE1 AES-256-GCM envelope; plaintext mode does not exist. The authenticated +context binds each object to its tenant scope, artifact id, version, path, and +content type so moving ciphertext to another address fails closed. + +Install `@aws-sdk/client-kms` in every artifact-storage consumer. It is an +optional peer so core-only applications do not download AWS KMS, while artifact +applications can choose either the local key provider or KMS at runtime. + +Set `ARTIFACT_KEY_PROVIDER=local` with a base64-encoded 32-byte `ARTIFACT_KEK` +(`ARTIFACT_KEK_NAME` is optional), or set `ARTIFACT_KEY_PROVIDER=kms` with +`ARTIFACT_KMS_KEY_ID` (`ARTIFACT_KMS_REGION` is optional). Invalid or missing +key configuration aborts startup. `ARTIFACT_ENCRYPTION_READ_LEGACY=true` is a +temporary, explicit plaintext-read migration flag and never enables plaintext +writes. + +```ts +import { + artifactScope, + artifactStorageConfigFromEnv, + createArtifactStorage, + storageCryptoFromEnv, +} from '@nestm/storage/artifacts'; + +const storage = await createArtifactStorage( + artifactStorageConfigFromEnv(process.env), + storageCryptoFromEnv(process.env), +); + +const ref = artifactScope({ + organizationId: 'org-1', + ownerUserId: 'user-1', +}); + +await storage.writeHtml('artifact-1', Buffer.from('

Encrypted

'), { + scope: ref, +}); + +const html = await storage.read('artifact-1', 'index.html', { + scope: ref, +}); +``` + +`writeBundle()` expands a zip into separately authenticated objects, skips +traversal and reserved-namespace entries, and rejects case-folded duplicate +paths, more than 1,000 entries, entries larger than 64 MiB, or more than 256 MiB +total expanded data. Callers may lower those limits per write. `read()` and +`readWithInfo()` support directory-to-`index.html` resolution, optional +plaintext-size limits, and version-bound paths. + +`ObjectStore` exposes the same encryption guarantees for non-artifact objects +such as upload staging and organization media. Keys must begin with a configured +top-level namespace; the defaults are `_staging` and `org-logos`. Those names +are reserved as artifact ids, case-insensitively, so artifact and object +operations cannot address one another even when an object-store provider uses a +shared bucket root. Override the top-level +`ArtifactStorageConfig.objectNamespaces` field when an application owns +different object families. + +Keep `objectNamespaces` identical across every reader and writer for a deployed +store; changing the set is a storage-protocol migration, not a per-process +preference. On case-insensitive filesystems, use case-stable artifact ids, +version ids, and object keys (Concepta's lowercase UUID ids satisfy this). + +The CAE1 framing and its `{ scope, artifactId, version, path }` context are a +compatibility contract. Existing envelopes remain readable. Filesystems retain +the layouts `//` and `/_objects/`; S3 and +other object-store providers retain their deployed unprefixed object keys, so a +rolling upgrade does not split old and new writers across keyspaces. Artifact +and version ids must each be one safe storage segment. Legacy plaintext reads +are available only through the explicit +`ARTIFACT_ENCRYPTION_READ_LEGACY=true` migration flag; writes are always +encrypted. + +The convenience `createArtifactStorage()` and `createObjectStore()` factories +are ideal for scripts. Long-running framework-neutral processes that need an +explicit shutdown path should create the raw clients with +`createArtifactStorageClient()` / `createObjectStorageClient()`, compose them +with the corresponding `create*WithClient()` function, call each client's +`onApplicationShutdown()`, and finally call `crypto.keyProvider.clear()`. The +Nest module owns that lifecycle automatically. + +Nest applications can register both adapters with one dynamic module: + +```ts +import { Module } from '@nestjs/common'; +import { + artifactStorageConfigFromEnv, + storageCryptoFromEnv, +} from '@nestm/storage/artifacts'; +import { ArtifactStorageModule } from '@nestm/storage/artifacts/nest'; + +@Module({ + imports: [ + ArtifactStorageModule.forRoot({ + config: artifactStorageConfigFromEnv(process.env), + crypto: storageCryptoFromEnv(process.env), + }), + ], +}) +export class AppModule {} +``` + +Inject `ArtifactStorage` with `@InjectArtifactStorage()` and `ObjectStore` with +`@InjectObjectStore()`. The module owns named raw clients (`artifacts` and +`objects`) and clears cached data keys during application shutdown. Set +`isGlobal: true` only when both adapters are intentionally application-wide. + ## Configure named stores Use the package-owned S3 factory when applicable. For other providers, create a diff --git a/package.json b/package.json index 807b513..29837da 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,14 @@ "types": "./dist/index.d.ts", "import": "./dist/index.js" }, + "./artifacts": { + "types": "./dist/artifacts/index.d.ts", + "import": "./dist/artifacts/index.js" + }, + "./artifacts/nest": { + "types": "./dist/artifacts/nest/index.d.ts", + "import": "./dist/artifacts/nest/index.js" + }, "./core": { "types": "./dist/core/index.d.ts", "import": "./dist/core/index.js" @@ -98,9 +106,11 @@ "release": "node scripts/publish.mjs" }, "dependencies": { + "adm-zip": "0.5.18", "files-sdk": "2.2.3" }, "peerDependencies": { + "@aws-sdk/client-kms": "^3.700.0", "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/lib-storage": "^3.700.0", "@aws-sdk/s3-presigned-post": "^3.700.0", @@ -111,6 +121,9 @@ "rxjs": "^7.8.1" }, "peerDependenciesMeta": { + "@aws-sdk/client-kms": { + "optional": true + }, "@aws-sdk/client-s3": { "optional": true }, @@ -137,6 +150,7 @@ } }, "devDependencies": { + "@aws-sdk/client-kms": "3.1103.0", "@aws-sdk/client-s3": "3.1103.0", "@aws-sdk/lib-storage": "3.1103.0", "@aws-sdk/s3-presigned-post": "3.1103.0", @@ -147,9 +161,11 @@ "@nestjs/platform-express": "12.0.0-alpha.5", "@nestjs/platform-fastify": "12.0.0-alpha.5", "@nestjs/testing": "12.0.0-alpha.5", + "@types/adm-zip": "0.5.8", "@types/node": "26.1.2", "@types/supertest": "7.2.1", "@vitest/coverage-v8": "4.1.10", + "aws-sdk-client-mock": "4.1.0", "fastify": "5.11.2", "oxlint": "1.77.0", "prettier": "3.9.6", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09c3514..8fe1d93 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,10 +11,16 @@ importers: .: dependencies: + adm-zip: + specifier: 0.5.18 + version: 0.5.18 files-sdk: specifier: 2.2.3 version: 2.2.3(@aws-sdk/client-s3@3.1103.0)(@aws-sdk/lib-storage@3.1103.0(@aws-sdk/client-s3@3.1103.0))(@aws-sdk/s3-presigned-post@3.1103.0)(@aws-sdk/s3-request-presigner@3.1103.0)(@nestjs/common@12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0))(fastify@5.11.2)(hono@4.13.0)(supports-color@7.2.0)(zod@4.4.3) devDependencies: + '@aws-sdk/client-kms': + specifier: 3.1103.0 + version: 3.1103.0 '@aws-sdk/client-s3': specifier: 3.1103.0 version: 3.1103.0 @@ -45,6 +51,9 @@ importers: '@nestjs/testing': specifier: 12.0.0-alpha.5 version: 12.0.0-alpha.5(@nestjs/common@12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0))(@nestjs/core@12.0.0-alpha.5)(@nestjs/platform-express@12.0.0-alpha.5) + '@types/adm-zip': + specifier: 0.5.8 + version: 0.5.8 '@types/node': specifier: 26.1.2 version: 26.1.2 @@ -54,6 +63,9 @@ importers: '@vitest/coverage-v8': specifier: 4.1.10 version: 4.1.10(vitest@4.1.10) + aws-sdk-client-mock: + specifier: 4.1.0 + version: 4.1.0 fastify: specifier: 5.11.2 version: 5.11.2 @@ -88,6 +100,10 @@ packages: resolution: {integrity: sha512-CGznePoL+1oWCSzqmkvlYMpWQEZohPR3LntjKXXfVH+oh6Kd8d+yHLkjpiAGwPIHAxFe4OAq3aKfRnv/wqWIyA==} engines: {node: '>=20.0.0'} + '@aws-sdk/client-kms@3.1103.0': + resolution: {integrity: sha512-gYdlyiASPS3Zu9a9jpJlKfEw3FGD4CB8lnuvUTpio1xmboltQjbnaTZ7JR12raiPcv51ODFfES1sUT+QXu3+Fw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-s3@3.1103.0': resolution: {integrity: sha512-FO7SB2vLhZRN3lBHVhMhRm3YKSHK8e917qjB8LMEEhU69cQ0Ahd/OMyezu+KqNANqEtyxfSnFVvYt81x6ZKGZA==} engines: {node: '>=20.0.0'} @@ -655,6 +671,18 @@ packages: '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + '@sinonjs/commons@3.0.1': + resolution: {integrity: sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==} + + '@sinonjs/fake-timers@11.2.2': + resolution: {integrity: sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==} + + '@sinonjs/fake-timers@15.4.0': + resolution: {integrity: sha512-DsG+8/LscQIQg68J6Ef3dv10u6nVyetYn923s3/sus5eaGfTo1of5WMZSLf0UJc9KDuKPilPH0UDJCjvNbDNCA==} + + '@sinonjs/samsam@8.0.3': + resolution: {integrity: sha512-hw6HbX+GyVZzmaYNh82Ecj1vdGZrqVIn/keDTg63IgAwiQPO+xCz99uG6Woqgb4tM0mUiFENKZ4cqd7IX94AXQ==} + '@smithy/core@3.31.1': resolution: {integrity: sha512-CyogUINxvi7C7LDsh8Syo6hVJOT9ckz4rG8dRZfTJ8r91HkMY59PnNooaj7WcHyxEkxPfBAmbgztZU+xTo76lg==} engines: {node: '>=18.0.0'} @@ -692,6 +720,9 @@ packages: '@tybys/wasm-util@0.10.3': resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} + '@types/adm-zip@0.5.8': + resolution: {integrity: sha512-RVVH7QvZYbN+ihqZ4kX/dMiowf6o+Jk1fNwiSdx0NahBJLU787zkULhGhJM8mf/obmLGmgdMM0bXsQTmyfbR7Q==} + '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -713,6 +744,12 @@ packages: '@types/node@26.1.2': resolution: {integrity: sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==} + '@types/sinon@17.0.4': + resolution: {integrity: sha512-RHnIrhfPO3+tJT0s7cFaXGZvsL4bbR3/k7z3P312qMS4JaS2Tk+KiwiLx1S0rQ56ERj00u1/BtdyVd0FY+Pdew==} + + '@types/sinonjs__fake-timers@15.0.1': + resolution: {integrity: sha512-Ko2tjWJq8oozHzHV+reuvS5KYIRAokHnGbDwGh/J64LntgpbuylF74ipEL24HCyRjf9FOlBiBHWBR1RlVKsI1w==} + '@types/superagent@8.1.11': resolution: {integrity: sha512-KA7srSW/HENDtOw9DOqaFLgWuMqN9WgjEw62lh9dpvRaZDkhdOkazASd7X7i2eMUYLHa1U37ZttnePsH5zTDHw==} @@ -884,6 +921,10 @@ packages: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} + adm-zip@0.5.18: + resolution: {integrity: sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==} + engines: {node: '>=12.0'} + ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} peerDependencies: @@ -936,6 +977,9 @@ packages: avvio@9.3.0: resolution: {integrity: sha512-g2tQ7LE7oOSqDfwEm3M+ZCMTJc7KiZCdJ4UwyZJb5ckTKyYu50OYmvv0mCFXPuYXoM4zkSt8zM9XQ9KCvxA74A==} + aws-sdk-client-mock@4.1.0: + resolution: {integrity: sha512-h/tOYTkXEsAcV3//6C1/7U4ifSpKyJvb6auveAepqqNJl6TdZaPFEtKjBQNf8UxQdDP850knB2i/whq4zlsxJw==} + aws4fetch@1.0.20: resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==} @@ -1075,6 +1119,10 @@ packages: dezalgo@1.0.4: resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==} + diff@5.2.2: + resolution: {integrity: sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A==} + engines: {node: '>=0.3.1'} + dir-glob@3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} @@ -1554,6 +1602,9 @@ packages: jsonfile@4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} + just-extend@6.2.0: + resolution: {integrity: sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==} + light-my-request@6.6.0: resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} @@ -1721,6 +1772,9 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} + nise@6.1.5: + resolution: {integrity: sha512-SnRDPDBjxZZoU2n0+gzzLtSvo1OZo7j6jnbXsoh3AFxEGhaFU7ZF0TmefuKERq79wxR2U+MPn7ArW+Tl+clC3A==} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -2010,6 +2064,9 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + sinon@18.0.1: + resolution: {integrity: sha512-a2N2TDY1uGviajJ6r4D1CyRAkzE9NNVlYOV1wX5xQDuAk0ONgzgRl0EjCQuRCPxOwp13ghsMwt9Gdldujs39qw==} + slash@3.0.0: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} @@ -2117,6 +2174,14 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + type-detect@4.0.8: + resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==} + engines: {node: '>=4'} + + type-detect@4.1.0: + resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==} + engines: {node: '>=4'} + type-is@1.6.18: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} @@ -2274,6 +2339,17 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/client-kms@3.1103.0': + dependencies: + '@aws-sdk/core': 3.977.6 + '@aws-sdk/credential-provider-node': 3.972.78 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.31.1 + '@smithy/fetch-http-handler': 5.6.13 + '@smithy/node-http-handler': 4.9.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/client-s3@3.1103.0': dependencies: '@aws-sdk/checksums': 3.1000.26 @@ -2939,6 +3015,23 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} + '@sinonjs/commons@3.0.1': + dependencies: + type-detect: 4.0.8 + + '@sinonjs/fake-timers@11.2.2': + dependencies: + '@sinonjs/commons': 3.0.1 + + '@sinonjs/fake-timers@15.4.0': + dependencies: + '@sinonjs/commons': 3.0.1 + + '@sinonjs/samsam@8.0.3': + dependencies: + '@sinonjs/commons': 3.0.1 + type-detect: 4.1.0 + '@smithy/core@3.31.1': dependencies: '@smithy/types': 4.16.1 @@ -2988,6 +3081,10 @@ snapshots: tslib: 2.8.1 optional: true + '@types/adm-zip@0.5.8': + dependencies: + '@types/node': 26.1.2 + '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 @@ -3007,6 +3104,12 @@ snapshots: dependencies: undici-types: 8.3.0 + '@types/sinon@17.0.4': + dependencies: + '@types/sinonjs__fake-timers': 15.0.1 + + '@types/sinonjs__fake-timers@15.0.1': {} + '@types/superagent@8.1.11': dependencies: '@types/cookiejar': 2.1.5 @@ -3141,6 +3244,8 @@ snapshots: mime-types: 3.0.2 negotiator: 1.0.0 + adm-zip@0.5.18: {} + ajv-formats@3.0.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -3185,6 +3290,12 @@ snapshots: '@fastify/error': 4.2.0 fastq: 1.20.1 + aws-sdk-client-mock@4.1.0: + dependencies: + '@types/sinon': 17.0.4 + sinon: 18.0.1 + tslib: 2.8.1 + aws4fetch@1.0.20: {} base64-js@1.5.1: {} @@ -3305,6 +3416,8 @@ snapshots: asap: 2.0.6 wrappy: 1.0.2 + diff@5.2.2: {} + dir-glob@3.0.1: dependencies: path-type: 4.0.0 @@ -3728,6 +3841,8 @@ snapshots: optionalDependencies: graceful-fs: 4.2.11 + just-extend@6.2.0: {} + light-my-request@6.6.0: dependencies: cookie: 1.1.1 @@ -3851,6 +3966,13 @@ snapshots: negotiator@1.0.0: {} + nise@6.1.5: + dependencies: + '@sinonjs/commons': 3.0.1 + '@sinonjs/fake-timers': 15.4.0 + just-extend: 6.2.0 + path-to-regexp: 8.4.2 + object-assign@4.1.1: {} object-inspect@1.13.4: {} @@ -4155,6 +4277,15 @@ snapshots: signal-exit@4.1.0: {} + sinon@18.0.1: + dependencies: + '@sinonjs/commons': 3.0.1 + '@sinonjs/fake-timers': 11.2.2 + '@sinonjs/samsam': 8.0.3 + diff: 5.2.2 + nise: 6.1.5 + supports-color: 7.2.0 + slash@3.0.0: {} sonic-boom@4.2.1: @@ -4258,6 +4389,10 @@ snapshots: tslib@2.8.1: {} + type-detect@4.0.8: {} + + type-detect@4.1.0: {} + type-is@1.6.18: dependencies: media-typer: 0.3.0 diff --git a/scripts/test-packed-core-consumer.mjs b/scripts/test-packed-core-consumer.mjs index 9739b3f..e8591e1 100644 --- a/scripts/test-packed-core-consumer.mjs +++ b/scripts/test-packed-core-consumer.mjs @@ -43,6 +43,8 @@ try { private: true, type: 'module', dependencies: { + '@aws-sdk/client-kms': + rootPackage.devDependencies['@aws-sdk/client-kms'], '@aws-sdk/client-s3': rootPackage.devDependencies['@aws-sdk/client-s3'], '@aws-sdk/lib-storage': @@ -126,6 +128,13 @@ import { type StorageDriver, type StorageObjectMetadata, } from '@nestm/storage/core'; +import { + createArtifactStorageWithClient, + createObjectStoreWithClient, + LocalKeyProvider, + open, + seal, +} from '@nestm/storage/artifacts'; import { createS3StorageDriver } from '@nestm/storage/files-sdk/s3'; const capabilities = { @@ -159,11 +168,16 @@ const driver = { capabilities, name: 'packed-memory', async upload(key, body, options) { - if (typeof body !== 'string') { - throw new TypeError('The smoke driver accepts string bodies only.'); - } + const bytes = + typeof body === 'string' + ? new TextEncoder().encode(body) + : body instanceof Uint8Array + ? body.slice() + : undefined; + if (bytes === undefined) + throw new TypeError('The smoke driver accepts string or byte bodies.'); const stored = { - body: new TextEncoder().encode(body), + body: bytes, contentType: options?.contentType ?? 'application/octet-stream', }; objects.set(key, stored); @@ -256,6 +270,10 @@ try { } assert.equal(nestResolved, false); +assert.match( + import.meta.resolve('@nestm/storage/artifacts/nest'), + new RegExp('dist/artifacts/nest/index[.]js$'), +); assert.equal(DEFAULT_BUFFER_LIMIT, 10 * 1024 * 1024); assert.equal(StorageErrorCode.NOT_FOUND, 'NOT_FOUND'); assert.equal(new StorageUploadControl().status, 'idle'); @@ -272,6 +290,24 @@ const foreignStorageError = Object.assign(new Error('foreign'), { }); assert.equal(isStorageError(foreignStorageError), true); +const envelopeContext = { + artifactId: 'packed-artifact', + path: 'index.html', + scope: 'org:packed', + version: null, +}; +const keyProvider = new LocalKeyProvider('packed', Buffer.alloc(32, 7)); +const envelope = await seal( + Buffer.from('packed artifact'), + envelopeContext, + keyProvider, + 'text/html; charset=utf-8', +); +const opened = await open(envelope, envelopeContext, keyProvider); +assert.equal(envelope.subarray(0, 4).toString(), 'CAE1'); +assert.equal(opened.plain.toString(), 'packed artifact'); +assert.equal(opened.contentType, 'text/html; charset=utf-8'); + const s3Driver = createS3StorageDriver({ adapter: { bucket: 'packed-test', @@ -299,6 +335,46 @@ const uploaded = await client.upload('hello.txt', 'hello core', { assert.equal(uploaded.key, 'hello.txt'); assert.equal(await client.downloadText('hello.txt'), 'hello core'); +const crypto = { keyProvider }; +const artifactStorage = createArtifactStorageWithClient( + { provider: 's3' }, + crypto, + client, +); +await artifactStorage.writeHtml( + 'packed-artifact', + Buffer.from('packed facade'), + { scope: 'org:packed' }, +); +assert.equal( + ( + await artifactStorage.read('packed-artifact', 'index.html', { + scope: 'org:packed', + }) + )?.toString(), + 'packed facade', +); + +const objectStore = createObjectStoreWithClient( + { provider: 's3' }, + crypto, + client, +); +await objectStore.putObject( + 'org-logos/packed', + Buffer.from('packed logo'), + 'image/png', + { scope: 'org:packed' }, +); +assert.equal( + ( + await objectStore.getObject('org-logos/packed', { + scope: 'org:packed', + }) + )?.body.toString(), + 'packed logo', +); + await client.onApplicationShutdown(); await client.onApplicationShutdown(); assert.equal(closeCalls, 1); diff --git a/src/artifacts/artifact-storage.ts b/src/artifacts/artifact-storage.ts new file mode 100644 index 0000000..1019d1d --- /dev/null +++ b/src/artifacts/artifact-storage.ts @@ -0,0 +1,802 @@ +import { + StorageErrorCode, + isStorageError, + type StorageClient, +} from '../core/index.js'; +import AdmZip from 'adm-zip'; +import { extname } from 'node:path'; + +import { + ENVELOPE_MAX_OVERHEAD_BYTES, + open, + seal, + SCOPE_FROM_HEADER, + type EnvelopeContext, + type StorageCrypto, +} from './crypto/index.js'; +import { + createArtifactStorageClient, + DEFAULT_OBJECT_NAMESPACES, + resolveObjectNamespaces, + type ArtifactStorageConfig, +} from './storage-driver.js'; + +export { + assertStorageProvider, + defaultFsRoot, + type ArtifactStorageConfig, +} from './storage-driver.js'; + +/** + * Storage for uploaded artifact files, keyed `/` (e.g. `/index.html`, + * `/assets/app.js`). The api writes; the sandbox reads. The `fs` provider keeps both pointed + * at one directory (fine for a single host / shared volume); any object-store provider — S3, GCS, + * Azure, R2, and the rest — lets the two services share storage without a shared volume, which is + * what running them on separate hosts (as on ECS) requires. + * + * Every object is stored as a CAE1 encryption envelope (ADR-0001, SEC-03): callers must say who + * owns what they write (`ScopeRef`) and what they expect to read (`ReadRef`) — the codec binds + * ciphertext to `{scope, artifactId, version, path}` so a swapped or replayed object fails closed. + */ +export interface ArtifactStorage { + /** Single self-contained HTML artifact → `/index.html`. */ + writeHtml(artifactId: string, buffer: Buffer, ref: ScopeRef): Promise; + /** + * Pre-built SPA bundle (a zip whose root contains index.html). Encrypted per extracted entry. + * With `opts.versionId` (VER-03), entries land under `v//…` and are sealed with + * `ctx.version = versionId` (the caller's `ref.version` is overridden to match — the path⇔version + * biconditional is derived, never trusted). Reserved-namespace screening applies to the zip + * ENTRY name in both modes: a bundle shipping `v/…` or `__…` files has those skipped, not nested. + */ + writeBundle( + artifactId: string, + zipBuffer: Buffer, + ref: ScopeRef, + opts?: WriteBundleOptions, + ): Promise; + /** + * Write an arbitrary file within the artifact (e.g. `__vendor/.js`, `__meta.json`). + * Rejects any `relPath` escaping the artifact directory. `contentType` rides the envelope + * header (authenticated) on both backends. + */ + writeFile( + artifactId: string, + relPath: string, + buffer: Buffer, + ref: ScopeRef, + contentType?: string, + ): Promise; + /** Server-rendered PNG preview → reserved key `/__thumb.png`. */ + writeThumbnail( + artifactId: string, + pngBuffer: Buffer, + ref: ScopeRef, + ): Promise; + /** Delete every file under the artifact (includes the thumbnail). */ + remove(artifactId: string): Promise; + /** + * Read a file within the artifact; `relPath === ""` resolves to index.html. `null` if absent. + * Throws `EnvelopeError` on any decrypt failure — a corrupt or swapped object must never be + * mistaken for a missing one (no SPA fallback over tampering). + */ + read( + artifactId: string, + relPath: string, + ref: ReadRef, + options?: ArtifactReadOptions, + ): Promise; + /** + * Like `read`, but also returns the envelope's AUTHENTICATED content type (VER-03). This is the + * read primitive for publish/CoW copy loops: a decrypt→re-encrypt copy must re-seal with the + * same `ct` it opened (the type is AAD-bound and steers serving), and `read()` discards it. + * `contentType` is undefined only for legacy plaintext pass-throughs (nothing was verified). + */ + readWithInfo( + artifactId: string, + relPath: string, + ref: ReadRef, + options?: ArtifactReadOptions, + ): Promise; + /** + * Delete specific files under the artifact (VER-03: publish prunes root paths removed by the + * newly-materialized version). Paths escaping the artifact directory and missing paths are + * ignored (idempotent — publish repair re-runs the same prune). + */ + deleteFiles(artifactId: string, relPaths: string[]): Promise; + /** + * List stored objects under the artifact (optionally under `subPrefix`, e.g. `v//`). + * Returns artifact-relative forward-slash paths + last-modified stamps, sorted by path. Reads + * no object bodies and needs no crypto — it serves the sweepers/invariants (version-prefix + * existence, orphan-prefix age, canonical bundle enumeration), never the serving plane. + */ + listFiles(artifactId: string, subPrefix?: string): Promise; +} + +/** One listed storage object (metadata only — the body stays sealed). */ +export interface StoredFileInfo { + /** Artifact-relative path, forward slashes (e.g. `index.html`, `v//index.html`). */ + path: string; + lastModified: Date; +} + +/** A decrypted object plus the authenticated content type it was sealed with (see readWithInfo). */ +export interface StoredObject { + plain: Buffer; + /** From the envelope header (`ct`, AAD-bound); undefined for legacy plaintext pass-throughs. */ + contentType?: string; +} + +/** Bounds decrypted data while also bounding the opaque envelope downloaded before decryption. */ +export interface ArtifactReadOptions { + readonly maxPlainBytes?: number; +} + +/** Permanent, address-independent read rejection safe for callers to map to a bounded 4xx. */ +export class ArtifactReadLimitError extends Error { + constructor(readonly maxPlainBytes: number) { + super(`artifact plaintext exceeds the ${maxPlainBytes} byte read limit`); + this.name = 'ArtifactReadLimitError'; + } +} + +/** Permanent rejection for a ZIP whose declared expansion exceeds the public protocol limits. */ +export class ArtifactBundleLimitError extends Error { + constructor(readonly reason: 'entries' | 'entry-bytes' | 'total-bytes') { + super(`artifact bundle exceeds the ${reason} limit`); + this.name = 'ArtifactBundleLimitError'; + } +} + +export const ARTIFACT_BUNDLE_MAX_ENTRIES = 1_000; +export const ARTIFACT_BUNDLE_MAX_ENTRY_BYTES = 64 * 1024 * 1024; +export const ARTIFACT_BUNDLE_MAX_TOTAL_BYTES = 256 * 1024 * 1024; + +/** Reserved storage path for the generated PNG preview, served by the sandbox as `/__thumb.png`. */ +export const THUMBNAIL_KEY = '__thumb.png'; + +/** Options for writeBundle (VER-03 versioned bundle writes). */ +export interface WriteBundleOptions { + /** Write entries under `v//…` with `ctx.version = versionId` instead of the root. */ + versionId?: string; + /** Optional stricter limits; values may lower, but never raise, the protocol hard limits. */ + limits?: { + maxEntries?: number; + maxEntryBytes?: number; + maxTotalBytes?: number; + }; +} + +/** Who owns an object being written; `version` stays null until VER-02 introduces versions. */ +export interface ScopeRef { + /** `org:` | `user:` | `upload:` — see artifactScope(). */ + scope: string; + version?: string | null; +} + +/** + * What the reader independently knows. The api asserts the scope it fetched from the DB; the + * sandbox — which only knows the artifactId — passes SCOPE_FROM_HEADER and relies on the wrap + * layer to enforce scope cryptographically (see crypto/context.ts). + */ +export interface ReadRef { + // `string` absorbs the literal for the checker, but the sentinel is domain vocabulary + // here — collapsing the union would erase what @from-header means to a reader. + // eslint-disable-next-line typescript/no-redundant-type-constituents + scope: string | typeof SCOPE_FROM_HEADER; + version?: string | null; +} + +export async function createArtifactStorage( + cfg: ArtifactStorageConfig, + crypto: StorageCrypto, +): Promise { + return createArtifactStorageWithClient( + cfg, + crypto, + await createArtifactStorageClient(cfg), + ); +} + +/** Framework-neutral domain adapter used by both direct consumers and the Nest composition entry. */ +export function createArtifactStorageWithClient( + cfg: ArtifactStorageConfig, + crypto: StorageCrypto, + client: StorageClient, +): ArtifactStorage { + return new ClientArtifactStorage( + crypto, + client, + resolveObjectNamespaces(cfg.objectNamespaces ?? DEFAULT_OBJECT_NAMESPACES), + cfg.provider === 'fs', + ); +} + +/** + * Storage path prefix for a version's objects (AP-002): `""` for a backfilled `canonical_v1` + * version (its bytes ARE the canonical root objects) and `v//` for a `versioned` one. + */ +export function versionPrefix( + layout: 'canonical_v1' | 'versioned', + versionId: string, +): string { + assertVersionId(versionId); + return layout === 'canonical_v1' ? '' : `v/${versionId}/`; +} + +/** + * Envelope `ctx.version` for a version's objects (AP-002): `null` for `canonical_v1` (root AAD — + * the serving plane is version-free forever) and the version UUID, byte-identical to the path + * segment, for `versioned`. + */ +export function versionAad( + layout: 'canonical_v1' | 'versioned', + versionId: string, +): string | null { + assertVersionId(versionId); + return layout === 'canonical_v1' ? null : versionId; +} + +/** Artifact ids are one storage segment; `_objects` is the filesystem object-store namespace. */ +export function assertArtifactId(artifactId: string): void { + assertStorageSegment(artifactId, 'artifact id'); + if (artifactId.toLowerCase() === '_objects') { + throw new Error('storage: artifact id "_objects" is reserved'); + } +} + +/** Version ids are one storage segment because they are interpolated under `v//`. */ +export function assertVersionId(versionId: string): void { + assertStorageSegment(versionId, 'version id'); +} + +function assertStorageSegment(value: string, label: string): void { + if ( + value === '' || + value === '.' || + value === '..' || + value.includes('/') || + value.includes('\\') || + value.includes('\0') || + Buffer.byteLength(value, 'utf8') > 255 + ) { + throw new Error(`storage: invalid ${label} ${JSON.stringify(value)}`); + } +} + +function normalizeRelativePath(path: string): string { + return path.replace(/\\/g, '/'); +} + +function isUnsafeRelativePath( + path: string, + allowEmpty = false, + allowTrailingSlash = false, +): boolean { + if (path === '') return !allowEmpty; + if (path.startsWith('/') || path.includes('\0')) { + return true; + } + const segments = path.split('/'); + if (allowTrailingSlash && segments.at(-1) === '') segments.pop(); + return segments.some( + (segment) => segment === '' || segment === '.' || segment === '..', + ); +} + +/** + * Is this artifact-relative path inside the namespaces VER-02 reserves for the platform — the + * `v/` version prefix and the `__`-internal families (meta/thumb/source/snapshot/overflow) — + * with the one serving exemption, top-level single-segment `__vendor/` pool assets? + * User-supplied bundle entries matching this are SKIPPED at write time (writeBundle), exactly + * like zip-slip entries: the sandbox deny-guard will never serve them, and letting them through + * to the seal chokepoint would turn a hostile/quirky zip into a mid-write throw. + */ +export function isReservedArtifactPath(relPath: string): boolean { + const segments = relPath.split('/'); + if (segments[0]?.toLowerCase() === 'v') return true; + for (let i = 0; i < segments.length; i++) { + const seg = segments[i]; + if (seg === undefined || !seg.startsWith('__')) continue; + if ( + i === 0 && + seg === '__vendor' && + segments.length === 2 && + segments[1] !== '' + ) + continue; + return true; + } + return false; +} + +/** + * AP-002 storage rule 2: `path.startsWith("v/" + version + "/") ⇔ version !== null`. Root objects + * never live under the reserved `v/` namespace and versioned objects never live anywhere else — + * asserted at the single ctx chokepoint (covers reads and writes, both backends), so a + * mis-addressed artifact object is impossible rather than unlikely. Throws (fail-closed): this is + * a programming error, never user input — every USER-named path (zip entries) is screened by + * `isReservedArtifactPath` before it can reach this assertion. + */ +export function assertVersionPath(path: string, version: string | null): void { + if (isUnsafeRelativePath(path)) { + throw new Error(`storage: invalid artifact path ${JSON.stringify(path)}`); + } + if (version !== null) { + assertVersionId(version); + if (!path.startsWith(`v/${version}/`)) { + throw new Error( + `storage: versioned object path "${path}" must live under "v/${version}/" (AP-002 rule 2)`, + ); + } + } else if (path.split('/', 1)[0]?.toLowerCase() === 'v') { + throw new Error( + `storage: root object path "${path}" is inside the reserved v/ namespace (AP-002 rule 2)`, + ); + } +} + +type RawRead = + | { kind: 'hit'; raw: Buffer; rel: string } + | { kind: 'miss' } + | { kind: 'error'; error: unknown }; + +interface PreparedBundleEntry { + name: string; + body: Buffer; +} + +function bundleLimit( + requested: number | undefined, + hardLimit: number, + name: string, +): number { + const value = requested ?? hardLimit; + if (!Number.isSafeInteger(value) || value < 1 || value > hardLimit) { + throw new RangeError( + `${name} must be an integer between 1 and ${hardLimit}`, + ); + } + return value; +} + +function prepareBundle( + zipBuffer: Buffer, + limits: WriteBundleOptions['limits'], +): PreparedBundleEntry[] { + const maxEntries = bundleLimit( + limits?.maxEntries, + ARTIFACT_BUNDLE_MAX_ENTRIES, + 'maxEntries', + ); + const maxEntryBytes = bundleLimit( + limits?.maxEntryBytes, + ARTIFACT_BUNDLE_MAX_ENTRY_BYTES, + 'maxEntryBytes', + ); + const maxTotalBytes = bundleLimit( + limits?.maxTotalBytes, + ARTIFACT_BUNDLE_MAX_TOTAL_BYTES, + 'maxTotalBytes', + ); + const entries = new AdmZip(zipBuffer).getEntries(); + if (entries.length > maxEntries) { + throw new ArtifactBundleLimitError('entries'); + } + + const accepted: Array<{ entry: (typeof entries)[number]; name: string }> = []; + const names = new Set(); + let declaredTotal = 0; + for (const entry of entries) { + if (entry.isDirectory) continue; + const name = normalizeRelativePath(entry.entryName).replace(/^\/+/, ''); + if (isUnsafeRelativePath(name) || isReservedArtifactPath(name)) { + continue; + } + const portableName = name.toLowerCase(); + if (names.has(portableName)) { + throw new Error(`artifact bundle contains duplicate entry "${name}"`); + } + names.add(portableName); + const declaredSize = entry.header.size; + if ( + !Number.isSafeInteger(declaredSize) || + declaredSize < 0 || + declaredSize > maxEntryBytes + ) { + throw new ArtifactBundleLimitError('entry-bytes'); + } + declaredTotal += declaredSize; + if (!Number.isSafeInteger(declaredTotal) || declaredTotal > maxTotalBytes) { + throw new ArtifactBundleLimitError('total-bytes'); + } + accepted.push({ entry, name }); + } + + // Inflate and validate every accepted entry before the first storage write. This bounds peak + // memory by maxTotalBytes and prevents a late malformed/oversized entry from leaving a partial + // bundle behind. + const prepared: PreparedBundleEntry[] = []; + let actualTotal = 0; + for (const { entry, name } of accepted) { + const body = entry.getData(); + if (body.length > maxEntryBytes) { + throw new ArtifactBundleLimitError('entry-bytes'); + } + actualTotal += body.length; + if (!Number.isSafeInteger(actualTotal) || actualTotal > maxTotalBytes) { + throw new ArtifactBundleLimitError('total-bytes'); + } + prepared.push({ name, body }); + } + return prepared; +} + +/** Envelope context for an artifact-relative path. */ +function artifactCtx( + artifactId: string, + path: string, + ref: ScopeRef | ReadRef, +): EnvelopeContext { + assertArtifactId(artifactId); + const version = ref.version ?? null; + assertVersionPath(path, version); + return { scope: ref.scope as string, artifactId, version, path }; +} + +/** Structured, greppable trace of a legacy plaintext pass-through (SEC-04 migration window). */ +export function warnLegacyRead(where: string, key: string): void { + // `console` is deliberate: this package is consumed by apps/sandbox as well as the Nest API, + // so a framework logger is not available here. It is the only portable sink. + // eslint-disable-next-line no-console + console.warn( + `[storage] legacy plaintext read (${where}): ${key} — pending SEC-04 backfill`, + ); +} + +// ── storage-client backend ─────────────────────────────────────────────────── +/** + * The one implementation, for every provider. It reaches storage only through `StorageClient`, + * so filesystem, S3, GCS, Azure, R2 and the rest differ by driver, not by code path here. The + * filesystem once had a parallel implementation that resolved real paths and probed with + * `existsSync`; it is gone, and with it the chance of the two drifting on reserved namespaces, + * version prefixes, or traversal guards. + */ +class ClientArtifactStorage implements ArtifactStorage { + constructor( + private readonly crypto: StorageCrypto, + private readonly client: StorageClient, + private readonly objectNamespaces: ReadonlySet, + private readonly filesystemLayout: boolean, + ) {} + + private assertArtifactId(artifactId: string): void { + assertArtifactId(artifactId); + if (this.objectNamespaces.has(artifactId.toLowerCase())) { + throw new Error( + `storage: artifact id ${JSON.stringify(artifactId)} is a reserved object namespace`, + ); + } + } + + async writeHtml( + artifactId: string, + buffer: Buffer, + ref: ScopeRef, + ): Promise { + await this.put( + artifactId, + 'index.html', + buffer, + ref, + 'text/html; charset=utf-8', + ); + } + + async writeBundle( + artifactId: string, + zipBuffer: Buffer, + ref: ScopeRef, + opts?: WriteBundleOptions, + ): Promise { + this.assertArtifactId(artifactId); + // Versioned mode (VER-03): mirror the fs backend — prefix + derived ctx.version, entry-name + // screening unchanged. + const versionId = opts?.versionId; + if (versionId !== undefined) assertVersionId(versionId); + const prefix = versionId === undefined ? '' : `v/${versionId}/`; + const effRef: ScopeRef = + versionId === undefined ? ref : { ...ref, version: versionId }; + const entries = prepareBundle(zipBuffer, opts?.limits); + for (const { name, body } of entries) { + // `__vendor/` stays ROOT-scoped in versioned mode — see the fs backend. + if (prefix !== '' && /^__vendor\/[^/]+$/.test(name)) { + await this.put( + artifactId, + name, + body, + { scope: ref.scope }, + contentTypeFor(name), + ); + continue; + } + await this.put( + artifactId, + prefix + name, + body, + effRef, + contentTypeFor(name), + ); + } + } + + async writeFile( + artifactId: string, + relPath: string, + buffer: Buffer, + ref: ScopeRef, + contentType?: string, + ): Promise { + const name = normalizeRelativePath(relPath); + // containment guard at the key level: reject absolute paths, empty, or any traversal segment. + if (isUnsafeRelativePath(name)) { + throw new Error(`writeFile: path escapes artifact directory: ${relPath}`); + } + await this.put( + artifactId, + name, + buffer, + ref, + contentType ?? contentTypeFor(name), + ); + } + + async writeThumbnail( + artifactId: string, + pngBuffer: Buffer, + ref: ScopeRef, + ): Promise { + await this.put(artifactId, THUMBNAIL_KEY, pngBuffer, ref, 'image/png'); + } + + async remove(artifactId: string): Promise { + this.assertArtifactId(artifactId); + for await (const object of this.client.listAll({ + prefix: `${artifactId}/`, + })) { + await this.client.delete(object.key); + } + } + + async listFiles( + artifactId: string, + subPrefix = '', + ): Promise { + this.assertArtifactId(artifactId); + const normalizedPrefix = normalizeRelativePath(subPrefix); + if (isUnsafeRelativePath(normalizedPrefix, true, true)) { + throw new Error( + `storage: invalid artifact list prefix ${JSON.stringify(subPrefix)}`, + ); + } + const base = `${artifactId}/`; + const out: StoredFileInfo[] = []; + for await (const object of this.client.listAll({ + prefix: `${base}${normalizedPrefix}`, + })) { + if (!object.key.startsWith(base)) continue; + out.push({ + path: object.key.slice(base.length), + lastModified: object.lastModified ?? new Date(0), + }); + } + return out.sort((a, b) => (a.path < b.path ? -1 : 1)); + } + + async read( + artifactId: string, + relPath: string, + ref: ReadRef, + options?: ArtifactReadOptions, + ): Promise { + const out = await this.readWithInfo(artifactId, relPath, ref, options); + return out === null ? null : out.plain; + } + + /** + * Raw bytes for one artifact-relative path. Reports a plain miss and a provider failure + * separately so the caller can retry a directory-shaped key without ever turning a genuine + * backend failure into a 404. + */ + private async fetchRaw( + artifactId: string, + rel: string, + options: ArtifactReadOptions | undefined, + preflight = true, + ): Promise { + const key = `${artifactId}/${rel}`; + if (preflight && !(await preflightRawRead(this.client, key, options))) + return { kind: 'miss' }; + try { + const bytes = await this.client.downloadBytes(key, { + maxBytes: rawReadLimit(options), + }); + return { kind: 'hit', raw: Buffer.from(bytes), rel }; + } catch (error) { + if (isStorageError(error) && error.code === StorageErrorCode.NOT_FOUND) { + return { kind: 'miss' }; + } + // A read limit is the caller's own bound, not a backend condition: it must surface as + // itself rather than as a miss the index.html rewrite would paper over. + if ( + isStorageError(error) && + error.code === StorageErrorCode.LIMIT_EXCEEDED + ) { + throw new ArtifactReadLimitError(requirePlainReadLimit(options)); + } + return { kind: 'error', error }; + } + } + + async readWithInfo( + artifactId: string, + relPath: string, + ref: ReadRef, + options?: ArtifactReadOptions, + ): Promise { + this.assertArtifactId(artifactId); + const normalized = normalizeRelativePath(relPath) + .replace(/\/{2,}/g, '/') + .replace(/^\/+|\/+$/g, ''); + const rel = normalized === '' ? 'index.html' : normalized; + if (isUnsafeRelativePath(rel)) return null; + + // A directory-shaped request ("" or "docs") serves that directory's index.html. Providers + // disagree on how a directory key fails — an object store has no such key at all, while the + // filesystem reports reading a directory as its own error — so the rewrite is driven by the + // miss, not by probing the backend. The AAD binds to the path actually read, never to the + // one requested (ADR D6.1). + // A filesystem HEAD cannot distinguish a file from a directory candidate. Skip that one + // preflight so a directory inode size cannot trigger the caller's plaintext limit before we + // try its bounded `index.html`; downloadBytes still enforces the raw-byte ceiling. + const direct = await this.fetchRaw( + artifactId, + rel, + options, + !this.filesystemLayout, + ); + if (direct.kind === 'error' && !this.filesystemLayout) { + throw direct.error; + } + let resolved = direct; + if (resolved.kind !== 'hit') { + const fallback = await this.fetchRaw( + artifactId, + `${rel}/index.html`, + options, + ); + if (fallback.kind === 'error') throw fallback.error; + if (fallback.kind === 'miss') { + if (direct.kind === 'error') throw direct.error; + return null; + } + resolved = fallback; + } + + const out = await open( + resolved.raw, + artifactCtx(artifactId, resolved.rel, ref), + this.crypto.keyProvider, + this.crypto.allowLegacyPlaintext === undefined + ? {} + : { allowLegacyPlaintext: this.crypto.allowLegacyPlaintext }, + ); + enforcePlainReadLimit(out.plain, options); + if (out.legacy) warnLegacyRead('storage', `${artifactId}/${resolved.rel}`); + return { + plain: out.plain, + ...(out.contentType === undefined + ? {} + : { contentType: out.contentType }), + }; + } + + async deleteFiles(artifactId: string, relPaths: string[]): Promise { + this.assertArtifactId(artifactId); + for (const relPath of relPaths) { + const name = normalizeRelativePath(relPath); + // containment at the key level: reject absolute/empty paths and traversal segments. + if (isUnsafeRelativePath(name)) { + continue; + } + await this.client.delete(`${artifactId}/${name}`); + } + } + + /** The single s3 write primitive; enveloped objects are opaque octet streams to the bucket. */ + private async put( + artifactId: string, + relPath: string, + body: Buffer, + ref: ScopeRef, + contentType: string, + ): Promise { + this.assertArtifactId(artifactId); + const sealed = await seal( + body, + artifactCtx(artifactId, relPath, ref), + this.crypto.keyProvider, + contentType, + ); + await this.client.upload(`${artifactId}/${relPath}`, sealed, { + // The real content type is authenticated inside CAE1 and must not leak through S3 metadata. + contentType: 'application/octet-stream', + }); + } +} + +function requirePlainReadLimit( + options: ArtifactReadOptions | undefined, +): number { + const limit = options?.maxPlainBytes; + if (limit === undefined || !Number.isSafeInteger(limit) || limit < 0) { + throw new Error('maxPlainBytes must be a non-negative safe integer'); + } + return limit; +} + +function rawReadLimit(options: ArtifactReadOptions | undefined): number { + if (options?.maxPlainBytes === undefined) return Infinity; + return requirePlainReadLimit(options) + ENVELOPE_MAX_OVERHEAD_BYTES; +} + +async function preflightRawRead( + client: StorageClient, + key: string, + options: ArtifactReadOptions | undefined, +): Promise { + if (options?.maxPlainBytes === undefined) return true; + try { + const metadata = await client.head(key); + if (metadata.size > rawReadLimit(options)) { + throw new ArtifactReadLimitError(requirePlainReadLimit(options)); + } + return true; + } catch (error) { + if (isStorageError(error) && error.code === StorageErrorCode.NOT_FOUND) + return false; + throw error; + } +} + +function enforcePlainReadLimit( + plain: Buffer, + options: ArtifactReadOptions | undefined, +): void { + if (options?.maxPlainBytes === undefined) return; + const limit = requirePlainReadLimit(options); + if (plain.length > limit) throw new ArtifactReadLimitError(limit); +} + +const MIME: Record = { + '.html': 'text/html; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', + '.mjs': 'text/javascript; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.svg': 'image/svg+xml', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.ico': 'image/x-icon', + '.woff': 'font/woff', + '.woff2': 'font/woff2', + '.map': 'application/json; charset=utf-8', + '.wasm': 'application/wasm', +}; + +/** content-type for an artifact file path, falling back to octet-stream. */ +export function contentTypeFor(path: string): string { + return MIME[extname(path).toLowerCase()] ?? 'application/octet-stream'; +} diff --git a/src/artifacts/compatibility.spec.ts b/src/artifacts/compatibility.spec.ts new file mode 100644 index 0000000..55bccfa --- /dev/null +++ b/src/artifacts/compatibility.spec.ts @@ -0,0 +1,202 @@ +import { createDecipheriv } from 'node:crypto'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + GOLDEN_CTX, + GOLDEN_ENVELOPE, + GOLDEN_OBJECT_CTX, + GOLDEN_OBJECT_ENVELOPE, + GOLDEN_OBJECT_PLAINTEXT, + GOLDEN_PLAINTEXT, +} from '../../test/fixtures/cae1-golden.js'; + +import { createArtifactStorage } from './artifact-storage.js'; +import { LocalKeyProvider, type EnvelopeContext } from './crypto/index.js'; +import { createObjectStore } from './object-store.js'; + +const KEK = Buffer.from( + '2ea36d37ad929a52dd5e0f8ea3669bc7ec7fc2f1d35ff3f9a59495ff70fc36d8', + 'hex', +); +const SCOPE = 'org:compat-org'; + +interface FrozenLegacyHeader { + v: number; + alg: string; + kid: string; + wdk: string; + iv: string; + ctx: EnvelopeContext; + ct?: string; +} + +function legacyCanonicalJson(ctx: EnvelopeContext): string { + return JSON.stringify({ + artifactId: ctx.artifactId, + path: ctx.path, + scope: ctx.scope, + version: ctx.version, + }); +} + +/** + * Frozen copy of the pre-consolidation local reader's v1 algorithm. Keep it independent from the + * production decoder: this is the new-writer → deployed-old-reader half of the compatibility + * contract, and a symmetric codec change must not make the test pass. + */ +function openWithFrozenLegacyLocalReader( + bytes: Buffer, + expectContext: EnvelopeContext, + kek: Buffer, + expectedKid: string, +): { plain: Buffer; contentType?: string } { + if (bytes.subarray(0, 4).toString() !== 'CAE1') { + throw new Error('legacy reader: bad magic'); + } + const headerLength = bytes.readUInt32BE(4); + const header = JSON.parse( + bytes.subarray(8, 8 + headerLength).toString('utf8'), + ) as FrozenLegacyHeader; + if ( + header.v !== 1 || + header.alg !== 'A256GCM' || + header.kid !== expectedKid || + legacyCanonicalJson(header.ctx) !== legacyCanonicalJson(expectContext) + ) { + throw new Error('legacy reader: incompatible header'); + } + + const wrappedDataKey = Buffer.from(header.wdk, 'base64'); + const wrapDecipher = createDecipheriv( + 'aes-256-gcm', + kek, + wrappedDataKey.subarray(0, 12), + ); + wrapDecipher.setAAD(Buffer.from(legacyCanonicalJson(header.ctx))); + wrapDecipher.setAuthTag(wrappedDataKey.subarray(12, 28)); + const dek = Buffer.concat([ + wrapDecipher.update(wrappedDataKey.subarray(28)), + wrapDecipher.final(), + ]); + try { + const body = bytes.subarray(8 + headerLength); + const decipher = createDecipheriv( + 'aes-256-gcm', + dek, + Buffer.from(header.iv, 'base64'), + ); + decipher.setAAD( + Buffer.from( + header.ct + ? `${legacyCanonicalJson(header.ctx)}\0${header.ct}` + : legacyCanonicalJson(header.ctx), + ), + ); + decipher.setAuthTag(body.subarray(body.length - 16)); + return { + plain: Buffer.concat([ + decipher.update(body.subarray(0, body.length - 16)), + decipher.final(), + ]), + ...(header.ct === undefined ? {} : { contentType: header.ct }), + }; + } finally { + dek.fill(0); + } +} + +describe('@nestm raw-driver compatibility', () => { + let root = ''; + + afterEach(() => { + if (root) rmSync(root, { recursive: true, force: true }); + }); + + it('reads artifact bytes written by the legacy raw-layout writer', async () => { + root = mkdtempSync(join(tmpdir(), 'storage-legacy-to-nestm-')); + const legacyPath = join(root, GOLDEN_CTX.artifactId, GOLDEN_CTX.path); + mkdirSync(dirname(legacyPath), { recursive: true }); + writeFileSync(legacyPath, GOLDEN_ENVELOPE); + + const storage = await createArtifactStorage( + { provider: 'fs', config: { root } }, + { keyProvider: new LocalKeyProvider('golden', Buffer.alloc(32, 42)) }, + ); + const result = await storage.readWithInfo( + GOLDEN_CTX.artifactId, + GOLDEN_CTX.path, + { scope: GOLDEN_CTX.scope }, + ); + + expect(result?.plain.toString()).toBe(GOLDEN_PLAINTEXT); + expect(result?.contentType).toBe('text/html; charset=utf-8'); + expect(readFileSync(legacyPath)).toEqual(GOLDEN_ENVELOPE); + }); + + it('writes artifact bytes the legacy codec and layout can read unchanged', async () => { + root = mkdtempSync(join(tmpdir(), 'storage-nestm-to-legacy-')); + const storage = await createArtifactStorage( + { provider: 'fs', config: { root } }, + { keyProvider: new LocalKeyProvider('compat', Buffer.from(KEK)) }, + ); + await storage.writeFile( + 'artifact-2', + 'assets/app.js', + Buffer.from("console.log('compatible')"), + { scope: SCOPE }, + ); + + const raw = readFileSync(join(root, 'artifact-2', 'assets', 'app.js')); + const legacyRead = openWithFrozenLegacyLocalReader( + raw, + { + scope: SCOPE, + artifactId: 'artifact-2', + version: null, + path: 'assets/app.js', + }, + Buffer.from(KEK), + 'local:compat', + ); + + expect(raw.subarray(0, 4).toString()).toBe('CAE1'); + expect(legacyRead.plain.toString()).toBe("console.log('compatible')"); + expect(legacyRead.contentType).toBe('text/javascript; charset=utf-8'); + }); + + it('preserves the legacy _objects layout and object-store AAD in both directions', async () => { + root = mkdtempSync(join(tmpdir(), 'object-storage-compat-')); + const key = GOLDEN_OBJECT_CTX.path; + const objectPath = join(root, '_objects', key); + mkdirSync(dirname(objectPath), { recursive: true }); + writeFileSync(objectPath, GOLDEN_OBJECT_ENVELOPE); + + const store = await createObjectStore( + { provider: 'fs', config: { root } }, + { keyProvider: new LocalKeyProvider('compat', Buffer.from(KEK)) }, + ); + expect( + (await store.getObject(key, { scope: SCOPE }))?.body.toString(), + ).toBe(GOLDEN_OBJECT_PLAINTEXT); + + await store.putObject(key, Buffer.from('new-logo'), 'image/webp', { + scope: SCOPE, + }); + const legacyRead = openWithFrozenLegacyLocalReader( + readFileSync(objectPath), + GOLDEN_OBJECT_CTX, + Buffer.from(KEK), + 'local:compat', + ); + expect(legacyRead.plain.toString()).toBe('new-logo'); + expect(legacyRead.contentType).toBe('image/webp'); + }); +}); diff --git a/src/artifacts/crypto/codec.spec.ts b/src/artifacts/crypto/codec.spec.ts new file mode 100644 index 0000000..dc2d66f --- /dev/null +++ b/src/artifacts/crypto/codec.spec.ts @@ -0,0 +1,394 @@ +import { randomBytes } from 'node:crypto'; +import { describe, expect, it } from 'vitest'; + +import { + ENVELOPE_SUPPORTED_VERSIONS, + ENVELOPE_WRITE_VERSION, + EnvelopeError, + isEnvelope, + open, + seal, +} from './codec.js'; +import { + SCOPE_FROM_HEADER, + type EnvelopeContext, + type ExpectedContext, +} from './context.js'; +import type { DataKey, KeyProvider } from './key-provider.js'; +import { LocalKeyProvider } from './local-provider.js'; + +const KEK = randomBytes(32); +const provider = new LocalKeyProvider('test', KEK); + +const CTX: EnvelopeContext = { + scope: 'org:org-1', + artifactId: 'art-1', + version: null, + path: 'index.html', +}; +const EXPECT: ExpectedContext = { ...CTX }; + +function flip(buf: Buffer, i: number, mask = 0x01): void { + buf.writeUInt8(buf.readUInt8(i) ^ mask, i); +} + +async function reason(p: Promise): Promise { + try { + await p; + return '(no error)'; + } catch (err) { + expect(err).toBeInstanceOf(EnvelopeError); + return (err as EnvelopeError).reason; + } +} + +/** Re-frame an envelope with a mutated header (JSON-level tampering; AAD stays the parsed ctx). */ +function rewriteHeader( + envelope: Buffer, + mutate: (header: Record) => void, +): Buffer { + const headerLen = envelope.readUInt32BE(4); + const header = JSON.parse( + envelope.subarray(8, 8 + headerLen).toString('utf8'), + ) as Record; + mutate(header); + const next = Buffer.from(JSON.stringify(header), 'utf8'); + const lenBuf = Buffer.alloc(4); + lenBuf.writeUInt32BE(next.length, 0); + return Buffer.concat([ + envelope.subarray(0, 4), + lenBuf, + next, + envelope.subarray(8 + headerLen), + ]); +} + +/** Counting provider so tests can assert "no key operation happened before the context assert". */ +class CountingProvider implements KeyProvider { + generated = 0; + unwrapped = 0; + constructor(private readonly inner: KeyProvider) {} + generateDataKey(ctx: EnvelopeContext): Promise { + this.generated++; + return this.inner.generateDataKey(ctx); + } + unwrapDataKey( + wdk: Buffer, + kid: string, + ctx: EnvelopeContext, + ): Promise { + this.unwrapped++; + return this.inner.unwrapDataKey(wdk, kid, ctx); + } + clear(): void {} +} + +describe('envelope codec', () => { + it.each([ + ['html document', { ...CTX }, 'text/html; charset=utf-8'], + [ + 'vendored asset', + { ...CTX, path: '__vendor/abc123.js' }, + 'text/javascript; charset=utf-8', + ], + ['storage meta', { ...CTX, path: '__meta.json' }, 'application/json'], + [ + 'object-store object', + { + scope: 'upload:u-1', + artifactId: null, + version: null, + path: '_staging/u-1', + }, + 'text/html', + ], + ] as const)( + 'round-trips %s with content type', + async (_name, ctx, contentType) => { + const plain = randomBytes(257); + const sealed = await seal(plain, ctx, provider, contentType); + expect(isEnvelope(sealed)).toBe(true); + expect(sealed.subarray(0, 4).toString()).toBe('CAE1'); + expect(sealed.includes(plain)).toBe(false); + const out = await open(sealed, { ...ctx }, provider); + expect(out.plain.equals(plain)).toBe(true); + expect(out.contentType).toBe(contentType); + expect(out.legacy).toBe(false); + }, + ); + + it('writes the newest supported version and only that', async () => { + expect(ENVELOPE_WRITE_VERSION).toBe( + Math.max(...ENVELOPE_SUPPORTED_VERSIONS), + ); + const sealed = await seal(Buffer.from('x'), CTX, provider); + const headerLen = sealed.readUInt32BE(4); + const header = JSON.parse( + sealed.subarray(8, 8 + headerLen).toString('utf8'), + ) as { v: number }; + expect(header.v).toBe(ENVELOPE_WRITE_VERSION); + }); + + it('fails closed on a flipped ciphertext byte', async () => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + flip(sealed, 8 + sealed.readUInt32BE(4)); + expect(await reason(open(sealed, EXPECT, provider))).toBe('auth-failed'); + }); + + it('fails closed on a flipped auth-tag byte', async () => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + flip(sealed, sealed.length - 1); + expect(await reason(open(sealed, EXPECT, provider))).toBe('auth-failed'); + }); + + it('never succeeds on raw header-byte tampering', async () => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + // Flip every header byte one at a time; each must fail with a fail-closed reason. + const headerLen = sealed.readUInt32BE(4); + for (let i = 8; i < 8 + headerLen; i += 7) { + const copy = Buffer.from(sealed); + flip(copy, i, 0x20); + const r = await reason(open(copy, EXPECT, provider)); + expect([ + 'malformed-header', + 'context-mismatch', + 'key-unavailable', + 'auth-failed', + 'unsupported-version', + 'unsupported-alg', + ]).toContain(r); + } + }); + + it.each([[4], [8], [20]])( + 'fails closed when truncated to %i bytes', + async (len) => { + const sealed = await seal(Buffer.from('tiny'), CTX, provider); + expect( + await reason(open(sealed.subarray(0, len), EXPECT, provider)), + ).toBe('truncated'); + }, + ); + + it('fails closed on tail truncation, never success', async () => { + const sealed = await seal(Buffer.from('tiny'), CTX, provider); + for (const cut of [1, 17]) { + const r = await reason( + open(sealed.subarray(0, sealed.length - cut), EXPECT, provider), + ); + expect(['truncated', 'auth-failed']).toContain(r); + } + }); + + it.each([ + ['artifactId', { ...EXPECT, artifactId: 'art-2' }], + ['path', { ...EXPECT, path: 'other.html' }], + ['version', { ...EXPECT, version: 'v9' }], + ['scope', { ...EXPECT, scope: 'org:other' }], + ] as const)( + 'asserts %s before any key operation', + async (_field, expectCtx) => { + const counting = new CountingProvider(provider); + const sealed = await seal(Buffer.from('payload'), CTX, provider); + expect(await reason(open(sealed, expectCtx, counting))).toBe( + 'context-mismatch', + ); + expect(counting.unwrapped).toBe(0); + }, + ); + + it('rejects a forged header scope under SCOPE_FROM_HEADER via the wrap binding', async () => { + // The sandbox case: the reader cannot assert scope, so a forged one must die at unwrap. + const sealed = await seal(Buffer.from('payload'), CTX, provider); + const forged = rewriteHeader(sealed, (h) => { + (h.ctx as Record).scope = 'org:attacker'; + }); + const r = await reason( + open(forged, { ...EXPECT, scope: SCOPE_FROM_HEADER }, provider), + ); + expect(r).toBe('key-unavailable'); + }); + + it('accepts the true header scope under SCOPE_FROM_HEADER', async () => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + const out = await open( + sealed, + { ...EXPECT, scope: SCOPE_FROM_HEADER }, + provider, + ); + expect(out.plain.toString()).toBe('payload'); + expect(out.ctx.scope).toBe(CTX.scope); + }); + + it.each([[0], [2]])('rejects header version %i', async (v) => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + const mutated = rewriteHeader(sealed, (h) => { + h.v = v; + }); + expect(await reason(open(mutated, EXPECT, provider))).toBe( + 'unsupported-version', + ); + }); + + it('rejects an unsupported algorithm', async () => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + const mutated = rewriteHeader(sealed, (h) => { + h.alg = 'A128GCM'; + }); + expect(await reason(open(mutated, EXPECT, provider))).toBe( + 'unsupported-alg', + ); + }); + + it('authenticates the content type (ct rides the AAD)', async () => { + const sealed = await seal( + Buffer.from('payload'), + CTX, + provider, + 'image/png', + ); + const mutated = rewriteHeader(sealed, (h) => { + h.ct = 'text/html'; + }); + expect(await reason(open(mutated, EXPECT, provider))).toBe('auth-failed'); + // Stripping ct entirely must fail too, not fall back to an unauthenticated default. + const stripped = rewriteHeader(sealed, (h) => { + delete h.ct; + }); + expect(await reason(open(stripped, EXPECT, provider))).toBe('auth-failed'); + }); + + it('rejects an empty content type instead of treating it as unauthenticated metadata', async () => { + await expect( + seal(Buffer.from('payload'), CTX, provider, ''), + ).rejects.toThrow(/content type must be non-empty/); + + const noContentType = await seal(Buffer.from('payload'), CTX, provider); + const injected = rewriteHeader(noContentType, (header) => { + header.ct = ''; + }); + expect(await reason(open(injected, EXPECT, provider))).toBe( + 'malformed-header', + ); + }); + + it('never emits a header larger than the reader accepts', async () => { + const counting = new CountingProvider(provider); + await expect( + seal( + Buffer.from('payload'), + CTX, + counting, + `text/plain; x=${'a'.repeat(70_000)}`, + ), + ).rejects.toThrow(/header exceeds/); + expect(counting.generated).toBe(0); + }); + + it('rejects a writer context the reader would reject before generating a key', async () => { + const counting = new CountingProvider(provider); + const invalid = { ...CTX, extra: 'smuggled' } as EnvelopeContext; + await expect( + seal(Buffer.from('payload'), invalid, counting), + ).rejects.toThrow(/ctx key set/); + expect(counting.generated).toBe(0); + }); + + it.each(['kid', 'wdk', 'dek'] as const)( + 'rejects an invalid %s from a custom key provider and zeroes its DEK', + async (field) => { + const dek = Buffer.alloc(field === 'dek' ? 31 : 32, 7); + const invalidProvider: KeyProvider = { + generateDataKey: () => + Promise.resolve({ + dek, + kid: field === 'kid' ? '' : 'custom:key', + wdk: field === 'wdk' ? Buffer.alloc(0) : Buffer.from('wrapped'), + }), + unwrapDataKey: () => Promise.reject(new Error('unused')), + clear: () => {}, + }; + + await expect( + seal(Buffer.from('payload'), CTX, invalidProvider), + ).rejects.toThrow(/key provider returned/); + expect(dek.equals(Buffer.alloc(dek.length))).toBe(true); + }, + ); + + it('maps a wrong-sized unwrapped DEK to key-unavailable', async () => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + const invalidDek = Buffer.alloc(31, 7); + const invalidProvider: KeyProvider = { + generateDataKey: () => Promise.reject(new Error('unused')), + unwrapDataKey: () => Promise.resolve(invalidDek), + clear: () => {}, + }; + expect(await reason(open(sealed, EXPECT, invalidProvider))).toBe( + 'key-unavailable', + ); + expect(invalidDek.equals(Buffer.alloc(31))).toBe(true); + }); + + it('rejects a surplus ctx key', async () => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + const mutated = rewriteHeader(sealed, (h) => { + (h.ctx as Record).extra = 'smuggled'; + }); + expect(await reason(open(mutated, EXPECT, provider))).toBe( + 'malformed-header', + ); + }); + + it('rejects a surplus header key', async () => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + const mutated = rewriteHeader(sealed, (h) => { + h.note = 'smuggled'; + }); + expect(await reason(open(mutated, EXPECT, provider))).toBe( + 'malformed-header', + ); + }); + + it('maps a provider outage to key-unavailable', async () => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + const down: KeyProvider = { + generateDataKey: () => Promise.reject(new Error('kms down')), + unwrapDataKey: () => Promise.reject(new Error('kms down')), + clear: () => {}, + }; + expect(await reason(open(sealed, EXPECT, down))).toBe('key-unavailable'); + }); + + it('rejects plaintext when the legacy flag is off', async () => { + expect( + await reason(open(Buffer.from('legacy'), EXPECT, provider)), + ).toBe('bad-magic'); + }); + + it('passes plaintext through verbatim when the legacy flag is on, marked legacy', async () => { + const plain = Buffer.from('legacy'); + const out = await open(plain, EXPECT, provider, { + allowLegacyPlaintext: true, + }); + expect(out.plain.equals(plain)).toBe(true); + expect(out.legacy).toBe(true); + }); + + it('never downgrades a valid-magic envelope via the legacy flag', async () => { + const sealed = await seal(Buffer.from('payload'), CTX, provider); + flip(sealed, 8 + sealed.readUInt32BE(4)); + const r = await reason( + open(sealed, EXPECT, provider, { allowLegacyPlaintext: true }), + ); + expect(r).toBe('auth-failed'); + }); + + it('rejects an oversized header length without allocating', async () => { + const bogus = Buffer.concat([Buffer.from('CAE1'), Buffer.alloc(4 + 32)]); + bogus.writeUInt32BE(0xffffffff, 4); + expect(await reason(open(bogus, EXPECT, provider))).toBe( + 'malformed-header', + ); + }); +}); diff --git a/src/artifacts/crypto/codec.ts b/src/artifacts/crypto/codec.ts new file mode 100644 index 0000000..de6330e --- /dev/null +++ b/src/artifacts/crypto/codec.ts @@ -0,0 +1,349 @@ +import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto'; + +import { + canonicalJson, + SCOPE_FROM_HEADER, + type EnvelopeContext, + type ExpectedContext, +} from './context.js'; +import type { KeyProvider } from './key-provider.js'; + +/** + * Envelope v1 framing (ADR-0001 D2, normative): + * + * bytes 0-3 magic "CAE1" + * bytes 4-7 u32 BE headerLen + * bytes 8-… headerJSON (UTF-8, exactly headerLen bytes) + * rest AES-256-GCM ciphertext ‖ 16-byte auth tag + * + * The GCM AAD is canonicalJson(ctx); ctx is duplicated in the header for operability (rewrap + * jobs, debugging) but the BINDING is the AAD — a tampered header fails the tag check. + */ +const MAGIC = Buffer.from('CAE1'); +const TAG_LEN = 16; +const IV_LEN = 12; +/** Headers are small (~400 bytes); anything larger is hostile or corrupt. */ +const MAX_HEADER_LEN = 64 * 1024; +/** Maximum bytes added around plaintext by a valid CAE1 envelope. Useful for bounded raw reads. */ +export const ENVELOPE_MAX_OVERHEAD_BYTES = 8 + MAX_HEADER_LEN + TAG_LEN; + +export const ENVELOPE_WRITE_VERSION = 1; +export const ENVELOPE_SUPPORTED_VERSIONS: readonly number[] = [ + ENVELOPE_WRITE_VERSION, +]; + +export type EnvelopeFailureReason = + | 'bad-magic' + | 'truncated' + | 'malformed-header' + | 'unsupported-version' + | 'unsupported-alg' + | 'context-mismatch' + | 'key-unavailable' + | 'auth-failed'; + +/** + * Structured decrypt failure. Carries only address metadata, never key material or plaintext — + * safe to log as-is (the failure-reason counter in the runbook keys off `reason`). + */ +export class EnvelopeError extends Error { + constructor( + readonly reason: EnvelopeFailureReason, + readonly artifactId: string | null, + readonly path: string, + detail?: string, + ) { + super( + `envelope ${reason} (artifact=${artifactId ?? '-'} path=${path})${detail ? `: ${detail}` : ''}`, + ); + this.name = 'EnvelopeError'; + } +} + +interface EnvelopeHeader { + v: number; + alg: string; + kid: string; + wdk: string; + iv: string; + ctx: EnvelopeContext; + ct?: string; +} + +export interface OpenResult { + plain: Buffer; + ctx: EnvelopeContext; + contentType?: string; + /** True only when legacy plaintext passed through under the explicit migration flag. */ + legacy: boolean; +} + +export interface OpenOptions { + /** + * SEC-04's migration-window escape hatch, default off. Consulted ONLY when the magic bytes + * are absent (a pre-envelope plaintext object) — never for a valid-magic object that fails + * validation, so the flag cannot downgrade a tampered envelope to a plaintext read. + */ + allowLegacyPlaintext?: boolean; +} + +/** + * Payload AAD. ADR-0001 D2 specifies canonicalJson(ctx); we additionally bind `ct` (deliberate + * strengthening, called out in the PR): the content type steers how bytes are SERVED, and it is + * stable across SEC-04's rewrap (which rewrites only kid/wdk — those stay unbound so rewrap + * never re-encrypts payloads; their integrity comes from the context-bound wrap itself). If a + * future v2 coexists with v1, bind `v` too — two live versions make downgrade flips meaningful. + */ +function aadFor(ctx: EnvelopeContext, contentType?: string): Buffer { + // NUL separator: canonicalJson is self-delimiting JSON and NUL cannot appear in a JSON string + // or a content type, so distinct (ctx, ct) pairs can never collide into one AAD. FROZEN — the + // golden-vector spec pins these exact bytes; changing them bricks every stored object. + return Buffer.from( + contentType === undefined + ? canonicalJson(ctx) + : `${canonicalJson(ctx)}\0${contentType}`, + ); +} + +/** Encrypt one object. Mints a fresh DEK per write; the DEK never outlives the call. */ +export async function seal( + plain: Buffer, + ctx: EnvelopeContext, + provider: KeyProvider, + contentType?: string, +): Promise { + validateContextShape(ctx, (detail) => { + throw new Error(`seal: invalid context (${detail})`); + }); + if ( + contentType !== undefined && + (typeof contentType !== 'string' || contentType.length === 0) + ) { + throw new Error('seal: content type must be non-empty when provided'); + } + // Reject obviously impossible user metadata before asking KMS to mint a data key. The exact + // serialized header is checked again once provider-owned kid/wdk fields are available. + const userMetadataLength = Buffer.byteLength( + JSON.stringify({ ctx, ...(contentType ? { ct: contentType } : {}) }), + 'utf8', + ); + if (userMetadataLength > MAX_HEADER_LEN) { + throw new Error(`seal: envelope header exceeds ${MAX_HEADER_LEN} bytes`); + } + const generated = await provider.generateDataKey(ctx); + const { dek, wdk, kid } = generated; + try { + if (!Buffer.isBuffer(dek) || dek.length !== 32) { + throw new Error('seal: key provider returned an invalid 32-byte DEK'); + } + if (!Buffer.isBuffer(wdk) || wdk.length === 0) { + throw new Error('seal: key provider returned an empty wrapped data key'); + } + if (typeof kid !== 'string' || kid.length === 0) { + throw new Error('seal: key provider returned an empty key id'); + } + const iv = randomBytes(IV_LEN); + const header: EnvelopeHeader = { + v: ENVELOPE_WRITE_VERSION, + alg: 'A256GCM', + kid, + wdk: wdk.toString('base64'), + iv: iv.toString('base64'), + ctx, + ...(contentType ? { ct: contentType } : {}), + }; + const headerJson = Buffer.from(JSON.stringify(header), 'utf8'); + if (headerJson.length > MAX_HEADER_LEN) { + throw new Error(`seal: envelope header exceeds ${MAX_HEADER_LEN} bytes`); + } + const cipher = createCipheriv('aes-256-gcm', dek, iv); + cipher.setAAD(aadFor(ctx, contentType)); + const body = Buffer.concat([ + cipher.update(plain), + cipher.final(), + cipher.getAuthTag(), + ]); + const lenBuf = Buffer.alloc(4); + lenBuf.writeUInt32BE(headerJson.length, 0); + return Buffer.concat([MAGIC, lenBuf, headerJson, body]); + } finally { + if (Buffer.isBuffer(dek)) dek.fill(0); + } +} + +/** True when the bytes carry the envelope magic (cheap pre-check; open() revalidates). */ +export function isEnvelope(bytes: Buffer): boolean { + return ( + bytes.length >= MAGIC.length && + bytes.subarray(0, MAGIC.length).equals(MAGIC) + ); +} + +/** + * Decrypt one object, fail-closed (ADR-0001 D2/D6). Order matters and is normative: + * frame → header shape → CONTEXT ASSERTION → key unwrap → tag check. The context assertion runs + * before any key operation so a swapped object costs no KMS call and leaks no oracle; `scope` + * alone may come from the header (SCOPE_FROM_HEADER) because the wrap layer binds it. + */ +export async function open( + bytes: Buffer, + expect: ExpectedContext, + provider: KeyProvider, + opts: OpenOptions = {}, +): Promise { + const fail = (reason: EnvelopeFailureReason, detail?: string): never => { + throw new EnvelopeError(reason, expect.artifactId, expect.path, detail); + }; + + if (!isEnvelope(bytes)) { + if (opts.allowLegacyPlaintext) + return { plain: bytes, ctx: legacyCtx(expect), legacy: true }; + fail('bad-magic'); + } + if (bytes.length < 8) fail('truncated', 'no header length'); + const headerLen = bytes.readUInt32BE(4); + if (headerLen > MAX_HEADER_LEN) + fail('malformed-header', 'header length out of range'); + if (8 + headerLen + TAG_LEN > bytes.length) + fail('truncated', 'body shorter than header + tag'); + + let header: EnvelopeHeader; + try { + header = JSON.parse( + bytes.subarray(8, 8 + headerLen).toString('utf8'), + ) as EnvelopeHeader; + } catch { + return fail('malformed-header', 'header is not valid JSON'); + } + validateHeaderShape(header, fail); + if (header.v !== ENVELOPE_WRITE_VERSION) { + // Reads must support every version in ENVELOPE_SUPPORTED_VERSIONS; with only v1 defined, + // anything else is unsupported. A future v2 branches here — writes stay on the newest + // version and never silently downgrade. + fail('unsupported-version', `v=${String(header.v)}`); + } + if (header.alg !== 'A256GCM') fail('unsupported-alg', header.alg); + + const ctx = header.ctx; + // The anti-swap assertion: every field the reader independently knows must match the header + // BEFORE any key operation. Only `scope` may be delegated to the header (sandbox), where the + // wrap binding enforces it instead. + if (ctx.artifactId !== expect.artifactId) + fail('context-mismatch', 'artifactId'); + if (ctx.path !== expect.path) fail('context-mismatch', 'path'); + if (ctx.version !== expect.version) fail('context-mismatch', 'version'); + if (expect.scope !== SCOPE_FROM_HEADER && ctx.scope !== expect.scope) { + fail('context-mismatch', 'scope'); + } + + const iv = Buffer.from(header.iv, 'base64'); + if (iv.length !== IV_LEN) fail('malformed-header', 'iv must be 12 bytes'); + const wdk = Buffer.from(header.wdk, 'base64'); + if (wdk.length === 0) fail('malformed-header', 'empty wrapped data key'); + + let unwrapped: unknown; + try { + unwrapped = await provider.unwrapDataKey(wdk, header.kid, ctx); + } catch { + // Wrong wrap context, foreign kid, or an unavailable key service — indistinguishable by + // design; serving plaintext because the key layer failed is the vulnerability (ADR D7). + return fail('key-unavailable'); + } + if (!Buffer.isBuffer(unwrapped) || unwrapped.length !== 32) { + if (Buffer.isBuffer(unwrapped)) unwrapped.fill(0); + return fail('key-unavailable'); + } + const dek = unwrapped; + try { + const body = bytes.subarray(8 + headerLen); + const tag = body.subarray(body.length - TAG_LEN); + const decipher = createDecipheriv('aes-256-gcm', dek, iv); + decipher.setAAD(aadFor(ctx, header.ct)); + decipher.setAuthTag(tag); + let plain: Buffer; + try { + plain = Buffer.concat([ + decipher.update(body.subarray(0, body.length - TAG_LEN)), + decipher.final(), + ]); + } catch { + return fail('auth-failed'); + } + return { + plain, + ctx, + ...(header.ct === undefined ? {} : { contentType: header.ct }), + legacy: false, + }; + } finally { + dek.fill(0); + } +} + +/** Header-shape validation: exact key sets, exact types. Any surplus key is hostile (ADR D2). */ +function validateHeaderShape( + header: EnvelopeHeader, + fail: (reason: EnvelopeFailureReason, detail?: string) => never, +): void { + if (typeof header !== 'object' || header === null) + fail('malformed-header', 'not an object'); + const allowed = new Set(['v', 'alg', 'kid', 'wdk', 'iv', 'ctx', 'ct']); + for (const key of Object.keys(header)) { + if (!allowed.has(key)) + fail('malformed-header', `unexpected header key "${key}"`); + } + if (typeof header.v !== 'number') fail('malformed-header', 'v'); + if (typeof header.alg !== 'string') fail('malformed-header', 'alg'); + if (typeof header.kid !== 'string' || header.kid === '') + fail('malformed-header', 'kid'); + if (typeof header.wdk !== 'string' || header.wdk === '') + fail('malformed-header', 'wdk'); + if (typeof header.iv !== 'string') fail('malformed-header', 'iv'); + if ( + header.ct !== undefined && + (typeof header.ct !== 'string' || header.ct.length === 0) + ) + fail('malformed-header', 'ct'); + validateContextShape(header.ctx, (detail) => + fail('malformed-header', detail), + ); +} + +function validateContextShape( + ctx: unknown, + fail: (detail: string) => never, +): asserts ctx is EnvelopeContext { + if (typeof ctx !== 'object' || ctx === null) fail('ctx'); + const ctxKeys = Object.keys(ctx); + const expected = ['scope', 'artifactId', 'version', 'path']; + if ( + ctxKeys.length !== expected.length || + expected.some((k) => !ctxKeys.includes(k)) + ) { + // Exactly these four keys: a surplus field could otherwise ride the AAD unexamined. + fail('ctx key set'); + } + const c = ctx as Record; + if ( + typeof c.scope !== 'string' || + c.scope === '' || + c.scope === SCOPE_FROM_HEADER + ) { + fail('ctx.scope'); + } + if (c.artifactId !== null && typeof c.artifactId !== 'string') + fail('ctx.artifactId'); + if (c.version !== null && typeof c.version !== 'string') fail('ctx.version'); + if (typeof c.path !== 'string') fail('ctx.path'); +} + +/** Descriptive ctx for a legacy plaintext pass-through (nothing was verified). */ +function legacyCtx(expect: ExpectedContext): EnvelopeContext { + return { + scope: + expect.scope === SCOPE_FROM_HEADER ? 'legacy:unverified' : expect.scope, + artifactId: expect.artifactId, + version: expect.version, + path: expect.path, + }; +} diff --git a/src/artifacts/crypto/context.ts b/src/artifacts/crypto/context.ts new file mode 100644 index 0000000..2a7d619 --- /dev/null +++ b/src/artifacts/crypto/context.ts @@ -0,0 +1,71 @@ +/** + * The encryption context that binds every stored object to its address (ADR-0001 D2). It is the + * GCM AAD of the payload AND the wrap context of the data key, so a ciphertext moved to another + * scope/artifact/path fails closed at decrypt — object swapping and cross-artifact replay are + * structurally impossible, not policy-checked. + */ +export interface EnvelopeContext { + /** Owning tenant: `org:` | `user:` | `upload:`. */ + scope: string; + /** The artifact the object belongs to; `null` for non-artifact ObjectStore objects. */ + artifactId: string | null; + /** Immutable version id; `null` until VER-02 introduces versions. */ + version: string | null; + /** Storage-relative path AS READ (e.g. `index.html` on the fs directory rewrite — ADR D6.1). */ + path: string; +} + +/** + * Read-side sentinel for callers that cannot independently know the owning scope — the sandbox, + * which has only the artifactId from the URL and no DB. The reader still asserts every field it + * DOES know (artifactId, path, version) against the cleartext header before any key operation; + * `scope` is then taken from the header and enforced cryptographically by the wrap layer: the + * KMS EncryptionContext / local wrap-AAD includes it, so a forged header scope fails at + * `unwrapDataKey`, not silently. + */ +export const SCOPE_FROM_HEADER = '@from-header'; + +/** What a reader knows independently about the object it expects (the anti-swap assertion). */ +export interface ExpectedContext { + // `string` absorbs the literal for the checker, but the sentinel is domain vocabulary + // here — collapsing the union would erase what @from-header means to a reader. + // eslint-disable-next-line typescript/no-redundant-type-constituents + scope: string | typeof SCOPE_FROM_HEADER; + artifactId: string | null; + version: string | null; + path: string; +} + +/** Scope for an artifact: the owning org when there is one, else the owning user. */ +export function artifactScope(owner: { + organizationId?: string | null; + ownerUserId: string; +}): string { + return owner.organizationId + ? `org:${owner.organizationId}` + : `user:${owner.ownerUserId}`; +} + +/** Scope for organization-owned non-artifact objects (logos). */ +export function orgScope(organizationId: string): string { + return `org:${organizationId}`; +} + +/** Scope for staged upload objects (`_staging/*`) — short-lived but still customer plaintext. */ +export function uploadScope(uploadId: string): string { + return `upload:${uploadId}`; +} + +/** + * Canonical serialization used as the payload AAD and the local wrap AAD: fixed sorted key + * order, no whitespace. Rebuilt from parsed values so header key ORDER never matters, while the + * key SET is pinned by the codec's strict header validation. + */ +export function canonicalJson(ctx: EnvelopeContext): string { + return JSON.stringify({ + artifactId: ctx.artifactId, + path: ctx.path, + scope: ctx.scope, + version: ctx.version, + }); +} diff --git a/src/artifacts/crypto/dek-cache.spec.ts b/src/artifacts/crypto/dek-cache.spec.ts new file mode 100644 index 0000000..25f2fc7 --- /dev/null +++ b/src/artifacts/crypto/dek-cache.spec.ts @@ -0,0 +1,215 @@ +import { randomBytes } from 'node:crypto'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { type EnvelopeContext } from './context.js'; +import { + CachingKeyProvider, + MAX_DEK_CACHE_ENTRIES, + MAX_DEK_CACHE_TTL_MS, +} from './dek-cache.js'; +import type { DataKey, KeyProvider } from './key-provider.js'; + +const CTX: EnvelopeContext = { + scope: 'org:o1', + artifactId: 'a1', + version: null, + path: 'index.html', +}; + +/** Deterministic inner provider that counts calls and hands out fresh DEK copies. */ +function fakeInner() { + const calls = { generate: 0, unwrap: 0, cleared: 0 }; + const inner: KeyProvider = { + generateDataKey(_ctx): Promise { + calls.generate++; + const dek = randomBytes(32); + return Promise.resolve({ dek, wdk: Buffer.from(dek), kid: 'fake' }); + }, + unwrapDataKey(wdk, _kid, _ctx): Promise { + calls.unwrap++; + return Promise.resolve(Buffer.from(wdk)); + }, + clear(): void { + calls.cleared++; + }, + }; + return { inner, calls }; +} + +describe('CachingKeyProvider', () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it('caches unwraps for an identical (kid, wdk, ctx) triple', async () => { + const { inner, calls } = fakeInner(); + const cache = new CachingKeyProvider(inner); + const wdk = randomBytes(32); + const a = await cache.unwrapDataKey(wdk, 'fake', CTX); + const b = await cache.unwrapDataKey(wdk, 'fake', CTX); + expect(calls.unwrap).toBe(1); + expect(a.equals(b)).toBe(true); + // Callers own their copies: zeroing one must not poison the cache or other callers. + a.fill(0); + const c = await cache.unwrapDataKey(wdk, 'fake', CTX); + expect(calls.unwrap).toBe(1); + expect(c.equals(b)).toBe(true); + }); + + it('misses for the same wdk under a different context (the scope-enforcement property)', async () => { + const { inner, calls } = fakeInner(); + const cache = new CachingKeyProvider(inner); + const wdk = randomBytes(32); + await cache.unwrapDataKey(wdk, 'fake', CTX); + await cache.unwrapDataKey(wdk, 'fake', { ...CTX, scope: 'org:other' }); + // A wdk-only key would have served the second call from cache, bypassing the wrap-layer + // context check — the exact bypass the sha256(kid‖wdk‖ctx) key exists to prevent. + expect(calls.unwrap).toBe(2); + }); + + it('length-prefixes cache-key fields so NUL boundaries cannot collide', async () => { + let unwraps = 0; + const inner: KeyProvider = { + generateDataKey: () => Promise.reject(new Error('unused')), + unwrapDataKey: () => Promise.resolve(Buffer.alloc(32, ++unwraps)), + clear: () => {}, + }; + const cache = new CachingKeyProvider(inner); + const suffix = randomBytes(31); + + const legitimate = await cache.unwrapDataKey( + Buffer.concat([Buffer.from([0]), suffix]), + 'K', + CTX, + ); + const forgedBoundary = await cache.unwrapDataKey(suffix, 'K\0', CTX); + + expect(unwraps).toBe(2); + expect(legitimate.equals(forgedBoundary)).toBe(false); + }); + + it('expires entries after the TTL', async () => { + const { inner, calls } = fakeInner(); + const cache = new CachingKeyProvider(inner, { ttlMs: 1000 }); + const wdk = randomBytes(32); + await cache.unwrapDataKey(wdk, 'fake', CTX); + vi.advanceTimersByTime(1500); + await cache.unwrapDataKey(wdk, 'fake', CTX); + expect(calls.unwrap).toBe(2); + }); + + it('zeroes an idle cached DEK when its TTL elapses', async () => { + const cachedDek = Buffer.alloc(32, 7); + const inner: KeyProvider = { + generateDataKey: () => Promise.reject(new Error('unused')), + unwrapDataKey: () => Promise.resolve(cachedDek), + clear: () => {}, + }; + const cache = new CachingKeyProvider(inner, { ttlMs: 1000 }); + const callerCopy = await cache.unwrapDataKey(randomBytes(32), 'fake', CTX); + expect(callerCopy.equals(cachedDek)).toBe(true); + + vi.advanceTimersByTime(1000); + + expect(cachedDek.equals(Buffer.alloc(32))).toBe(true); + expect(callerCopy.equals(Buffer.alloc(32, 7))).toBe(true); + }); + + it('single-flights concurrent unwraps for the same cache key', async () => { + let resolveUnwrap: ((dek: Buffer) => void) | undefined; + const calls = { unwrap: 0 }; + const inner: KeyProvider = { + generateDataKey: () => Promise.reject(new Error('unused')), + unwrapDataKey: () => { + calls.unwrap++; + return new Promise((resolve) => { + resolveUnwrap = resolve; + }); + }, + clear: () => {}, + }; + const cache = new CachingKeyProvider(inner); + const wdk = randomBytes(32); + const first = cache.unwrapDataKey(wdk, 'fake', CTX); + const second = cache.unwrapDataKey(wdk, 'fake', CTX); + expect(calls.unwrap).toBe(1); + + resolveUnwrap?.(Buffer.alloc(32, 9)); + + const [a, b] = await Promise.all([first, second]); + expect(a).toEqual(b); + expect(a).not.toBe(b); + }); + + it('never repopulates the cache from an unwrap that raced clear()', async () => { + let resolveUnwrap: ((dek: Buffer) => void) | undefined; + const inner: KeyProvider = { + generateDataKey: () => Promise.reject(new Error('unused')), + unwrapDataKey: () => + new Promise((resolve) => { + resolveUnwrap = resolve; + }), + clear: () => {}, + }; + const cache = new CachingKeyProvider(inner); + const pending = cache.unwrapDataKey(randomBytes(32), 'fake', CTX); + cache.clear(); + const staleDek = Buffer.alloc(32, 5); + resolveUnwrap?.(staleDek); + + await expect(pending).rejects.toThrow(/cleared during unwrap/); + expect(staleDek.equals(Buffer.alloc(32))).toBe(true); + }); + + it.each([ + 0, + -1, + Number.NaN, + Number.POSITIVE_INFINITY, + MAX_DEK_CACHE_TTL_MS + 1, + ])('rejects an unsafe cache TTL: %s', (ttlMs) => { + const { inner } = fakeInner(); + expect(() => new CachingKeyProvider(inner, { ttlMs })).toThrow(/ttlMs/); + }); + + it.each([0, -1, 1.5, Number.POSITIVE_INFINITY, MAX_DEK_CACHE_ENTRIES + 1])( + 'rejects an unsafe cache capacity: %s', + (max) => { + const { inner } = fakeInner(); + expect(() => new CachingKeyProvider(inner, { max })).toThrow(/max/); + }, + ); + + it('evicts the least recently used entry at capacity', async () => { + const { inner, calls } = fakeInner(); + const cache = new CachingKeyProvider(inner, { max: 2 }); + const [w1, w2, w3] = [randomBytes(32), randomBytes(32), randomBytes(32)]; + await cache.unwrapDataKey(w1, 'fake', CTX); + await cache.unwrapDataKey(w2, 'fake', CTX); + await cache.unwrapDataKey(w1, 'fake', CTX); // refresh w1 — w2 becomes LRU + await cache.unwrapDataKey(w3, 'fake', CTX); // evicts w2 + calls.unwrap = 0; + await cache.unwrapDataKey(w1, 'fake', CTX); + expect(calls.unwrap).toBe(0); + await cache.unwrapDataKey(w2, 'fake', CTX); + expect(calls.unwrap).toBe(1); + }); + + it('never caches generateDataKey (fresh DEK per write)', async () => { + const { inner, calls } = fakeInner(); + const cache = new CachingKeyProvider(inner); + await cache.generateDataKey(CTX); + await cache.generateDataKey(CTX); + expect(calls.generate).toBe(2); + }); + + it('clear() empties the cache and cascades to the inner provider', async () => { + const { inner, calls } = fakeInner(); + const cache = new CachingKeyProvider(inner); + const wdk = randomBytes(32); + await cache.unwrapDataKey(wdk, 'fake', CTX); + cache.clear(); + expect(calls.cleared).toBe(1); + await cache.unwrapDataKey(wdk, 'fake', CTX); + expect(calls.unwrap).toBe(2); + }); +}); diff --git a/src/artifacts/crypto/dek-cache.ts b/src/artifacts/crypto/dek-cache.ts new file mode 100644 index 0000000..8c464cc --- /dev/null +++ b/src/artifacts/crypto/dek-cache.ts @@ -0,0 +1,182 @@ +import { createHash } from 'node:crypto'; + +import { canonicalJson, type EnvelopeContext } from './context.js'; +import type { DataKey, KeyProvider } from './key-provider.js'; + +export interface DekCacheOptions { + /** Max cached unwrapped DEKs (default 1024). */ + max?: number; + /** Entry lifetime; ADR-0001 D6.2 caps this at 5 minutes (the default). */ + ttlMs?: number; +} + +/** Hard policy bounds for cached plaintext data-encryption keys. */ +export const MAX_DEK_CACHE_ENTRIES = 65_536; +export const MAX_DEK_CACHE_TTL_MS = 5 * 60_000; + +/** + * Bounded in-memory LRU of unwrapped DEKs so the sandbox's per-request reads don't turn into a + * KMS Decrypt per object (ADR-0001 D6.2). Never persisted; cleared on shutdown. Deliberate + * T4-shaped trade: a compromised runtime could read this cache, but it can already call unwrap. + * + * Deviation from the ADR's literal wording, on purpose: the key is a length-prefixed hash of + * (kid, wdk, ctx), not sha256(wdk) — keying on wdk alone would let a cache hit bypass the very + * context check (KMS EncryptionContext / local wrap-AAD) that makes header-sourced scope + * trustworthy. Length prefixes are required because kid may contain NUL and wdk is arbitrary + * binary; delimiter concatenation would let distinct tuples collide before hashing. + */ +export class CachingKeyProvider implements KeyProvider { + private readonly cache = new Map< + string, + { + dek: Buffer; + expiresAt: number; + timer: ReturnType; + token: number; + } + >(); + private readonly inFlight = new Map< + string, + { generation: number; promise: Promise } + >(); + private readonly max: number; + private readonly ttlMs: number; + private generation = 0; + private nextToken = 0; + + constructor( + private readonly inner: KeyProvider, + opts: DekCacheOptions = {}, + ) { + this.max = opts.max ?? 1024; + this.ttlMs = opts.ttlMs ?? MAX_DEK_CACHE_TTL_MS; + if ( + !Number.isSafeInteger(this.max) || + this.max < 1 || + this.max > MAX_DEK_CACHE_ENTRIES + ) { + throw new RangeError( + `DEK cache max must be an integer between 1 and ${MAX_DEK_CACHE_ENTRIES}`, + ); + } + if ( + !Number.isFinite(this.ttlMs) || + this.ttlMs <= 0 || + this.ttlMs > MAX_DEK_CACHE_TTL_MS + ) { + throw new RangeError( + `DEK cache ttlMs must be greater than 0 and at most ${MAX_DEK_CACHE_TTL_MS}`, + ); + } + } + + generateDataKey(ctx: EnvelopeContext): Promise { + // Writes always mint a fresh DEK (one per object write) — nothing to cache. + return this.inner.generateDataKey(ctx); + } + + async unwrapDataKey( + wdk: Buffer, + kid: string, + ctx: EnvelopeContext, + ): Promise { + const hash = createHash('sha256'); + for (const part of [ + Buffer.from(kid, 'utf8'), + wdk, + Buffer.from(canonicalJson(ctx), 'utf8'), + ]) { + const length = Buffer.allocUnsafe(8); + length.writeBigUInt64BE(BigInt(part.byteLength)); + hash.update(length).update(part); + } + const key = hash.digest('hex'); + const hit = this.cache.get(key); + if (hit && hit.expiresAt > Date.now()) { + // Refresh recency (Map preserves insertion order — oldest entry is the first key). + this.cache.delete(key); + this.cache.set(key, hit); + return Buffer.from(hit.dek); + } + if (hit) this.evict(key, hit); + + const existingFlight = this.inFlight.get(key); + if ( + existingFlight !== undefined && + existingFlight.generation === this.generation + ) { + const dek = await existingFlight.promise; + if (existingFlight.generation !== this.generation) { + throw new Error('DEK cache was cleared during unwrap'); + } + return Buffer.from(dek); + } + + const generation = this.generation; + const flight = { + generation, + promise: this.loadAndCache(key, wdk, kid, ctx, generation), + }; + this.inFlight.set(key, flight); + try { + const dek = await flight.promise; + if (generation !== this.generation) { + throw new Error('DEK cache was cleared during unwrap'); + } + return Buffer.from(dek); + } finally { + if (this.inFlight.get(key) === flight) this.inFlight.delete(key); + } + } + + private async loadAndCache( + key: string, + wdk: Buffer, + kid: string, + ctx: EnvelopeContext, + generation: number, + ): Promise { + const dek = await this.inner.unwrapDataKey(wdk, kid, ctx); + if (!Buffer.isBuffer(dek) || dek.length !== 32) { + if (Buffer.isBuffer(dek)) dek.fill(0); + throw new Error('inner key provider returned an invalid 32-byte DEK'); + } + if (generation !== this.generation) { + dek.fill(0); + throw new Error('DEK cache was cleared during unwrap'); + } + if (this.cache.size >= this.max) { + const oldest = this.cache.keys().next().value; + if (oldest !== undefined) this.evict(oldest, this.cache.get(oldest)); + } + const token = ++this.nextToken; + const timer = setTimeout(() => { + const entry = this.cache.get(key); + if (entry?.token === token) this.evict(key, entry); + }, this.ttlMs); + timer.unref?.(); + this.cache.set(key, { + dek, + expiresAt: Date.now() + this.ttlMs, + timer, + token, + }); + return dek; + } + + clear(): void { + for (const [key, entry] of this.cache) this.evict(key, entry); + this.generation++; + this.inFlight.clear(); + this.inner.clear(); + } + + private evict( + key: string, + entry?: { dek: Buffer; timer: ReturnType }, + ): void { + if (entry !== undefined) clearTimeout(entry.timer); + entry?.dek.fill(0); + this.cache.delete(key); + } +} diff --git a/src/artifacts/crypto/env-config.spec.ts b/src/artifacts/crypto/env-config.spec.ts new file mode 100644 index 0000000..8332136 --- /dev/null +++ b/src/artifacts/crypto/env-config.spec.ts @@ -0,0 +1,208 @@ +import { randomBytes } from 'node:crypto'; +import { describe, expect, it } from 'vitest'; + +import { artifactStorageConfigFromEnv } from '../env-config.js'; +import { defaultFsRoot } from '../storage-driver.js'; +import { storageCryptoFromEnv } from './index.js'; +import { keyProviderConfigFromEnv } from './key-provider.js'; + +const KEK = randomBytes(32).toString('base64'); + +// These validators are the fail-closed boot gates for BOTH the api and the sandbox — a hole here +// is a service that boots without encryption or with drifting S3 rules. +describe('keyProviderConfigFromEnv', () => { + it('resolves a valid local provider with the default name', () => { + const cfg = keyProviderConfigFromEnv({ + ARTIFACT_KEY_PROVIDER: 'local', + ARTIFACT_KEK: KEK, + }); + expect(cfg).toMatchObject({ kind: 'local', name: 'default' }); + expect((cfg as { kek: Buffer }).kek.length).toBe(32); + }); + + it('honors ARTIFACT_KEK_NAME', () => { + const cfg = keyProviderConfigFromEnv({ + ARTIFACT_KEY_PROVIDER: 'local', + ARTIFACT_KEK: KEK, + ARTIFACT_KEK_NAME: 'staging-1', + }); + expect(cfg).toMatchObject({ kind: 'local', name: 'staging-1' }); + }); + + it.each([ + ['missing', undefined], + ['empty', ''], + ['not base64 32 bytes', 'dG9vLXNob3J0'], + ['garbage that base64-decodes lossily', '!'.repeat(44)], + ])('rejects a local provider with a %s KEK', (_name, kek) => { + expect(() => + keyProviderConfigFromEnv({ + ARTIFACT_KEY_PROVIDER: 'local', + ARTIFACT_KEK: kek, + }), + ).toThrow(/ARTIFACT_KEK/); + }); + + it('resolves kms with the explicit region winning over S3_REGION', () => { + expect( + keyProviderConfigFromEnv({ + ARTIFACT_KEY_PROVIDER: 'kms', + ARTIFACT_KMS_KEY_ID: 'alias/a', + ARTIFACT_KMS_REGION: 'eu-west-1', + S3_REGION: 'us-east-1', + }), + ).toEqual({ kind: 'kms', keyId: 'alias/a', region: 'eu-west-1' }); + expect( + keyProviderConfigFromEnv({ + ARTIFACT_KEY_PROVIDER: 'kms', + ARTIFACT_KMS_KEY_ID: 'alias/a', + S3_REGION: 'us-east-1', + }), + ).toEqual({ kind: 'kms', keyId: 'alias/a', region: 'us-east-1' }); + // No region anywhere → undefined, so the KMS client falls to the SDK default chain (parity + // between api and sandbox — neither may invent a region the other doesn't see). + expect( + keyProviderConfigFromEnv({ + ARTIFACT_KEY_PROVIDER: 'kms', + ARTIFACT_KMS_KEY_ID: 'alias/a', + }), + ).toEqual({ kind: 'kms', keyId: 'alias/a', region: undefined }); + }); + + it('rejects kms without a key id', () => { + expect(() => + keyProviderConfigFromEnv({ ARTIFACT_KEY_PROVIDER: 'kms' }), + ).toThrow(/ARTIFACT_KMS_KEY_ID/); + }); + + it.each([[undefined], [''], ['none'], ['plaintext']])( + 'rejects provider %j — there is no plaintext mode', + (provider) => { + expect(() => + keyProviderConfigFromEnv({ ARTIFACT_KEY_PROVIDER: provider }), + ).toThrow(/ARTIFACT_KEY_PROVIDER/); + }, + ); +}); + +describe('storageCryptoFromEnv', () => { + it('builds a working provider and parses the legacy flag strictly', async () => { + const on = storageCryptoFromEnv({ + ARTIFACT_KEY_PROVIDER: 'local', + ARTIFACT_KEK: KEK, + ARTIFACT_ENCRYPTION_READ_LEGACY: 'true', + }); + expect(on.allowLegacyPlaintext).toBe(true); + const ctx = { scope: 'org:o', artifactId: 'a', version: null, path: 'p' }; + const { dek, wdk, kid } = await on.keyProvider.generateDataKey(ctx); + expect( + (await on.keyProvider.unwrapDataKey(wdk, kid, ctx)).equals(dek), + ).toBe(true); + + for (const value of [undefined, '', 'TRUE', '1', 'yes']) { + const crypto = storageCryptoFromEnv({ + ARTIFACT_KEY_PROVIDER: 'local', + ARTIFACT_KEK: KEK, + ARTIFACT_ENCRYPTION_READ_LEGACY: value, + }); + expect(crypto.allowLegacyPlaintext).toBe(false); // only the literal "true" opens the window + } + }); +}); + +describe('artifactStorageConfigFromEnv', () => { + it('defaults to the filesystem provider with a root', () => { + expect(artifactStorageConfigFromEnv({})).toEqual({ + provider: 'fs', + config: { root: defaultFsRoot() }, + }); + expect(artifactStorageConfigFromEnv({ STORAGE_ROOT: '/data' })).toEqual({ + provider: 'fs', + config: { root: '/data' }, + }); + expect(artifactStorageConfigFromEnv({ STORAGE_ROOT: '' })).toEqual({ + provider: 'fs', + config: { root: defaultFsRoot() }, + }); + }); + + it('still honors the retired ARTIFACT_STORAGE and ARTIFACTS_DIR names', () => { + expect(artifactStorageConfigFromEnv({ ARTIFACTS_DIR: '/legacy' })).toEqual({ + provider: 'fs', + config: { root: '/legacy' }, + }); + expect( + artifactStorageConfigFromEnv({ ARTIFACT_STORAGE: 's3', S3_BUCKET: 'b' }), + ).toMatchObject({ provider: 's3', config: { bucket: 'b' } }); + }); + + it('rejects a provider it cannot build, naming the ones it can', () => { + expect(() => + artifactStorageConfigFromEnv({ STORAGE_PROVIDER: 'dropbox-ish' }), + ).toThrow(/Unknown storage provider/); + }); + + it('rejects a provider configured with nowhere to write', () => { + expect(() => + artifactStorageConfigFromEnv({ STORAGE_PROVIDER: 's3' }), + ).toThrow(/needs somewhere to write/); + }); + + it('accepts any provider the module ships, not just s3', () => { + expect( + artifactStorageConfigFromEnv({ + STORAGE_PROVIDER: 'gcs', + STORAGE_BUCKET: 'b', + }), + ).toEqual({ provider: 'gcs', config: { bucket: 'b' } }); + expect( + artifactStorageConfigFromEnv({ + STORAGE_PROVIDER: 'azure', + STORAGE_CONTAINER: 'c', + STORAGE_ACCOUNT_NAME: 'acct', + }), + ).toEqual({ + provider: 'azure', + config: { container: 'c', accountName: 'acct' }, + }); + }); + + it('enforces both-or-neither credentials', () => { + const base = { STORAGE_PROVIDER: 's3', STORAGE_BUCKET: 'b' }; + expect(() => + artifactStorageConfigFromEnv({ ...base, STORAGE_ACCESS_KEY_ID: 'k' }), + ).toThrow(/set together/); + expect(() => + artifactStorageConfigFromEnv({ ...base, STORAGE_SECRET_ACCESS_KEY: 's' }), + ).toThrow(/set together/); + // Neither set: the config carries no credentials at all, so the provider SDK's own chain + // (the ECS task role) resolves them. + expect(artifactStorageConfigFromEnv(base).config).toEqual({ bucket: 'b' }); + }); + + it('maps the full object-store config', () => { + expect( + artifactStorageConfigFromEnv({ + STORAGE_PROVIDER: 's3', + STORAGE_ENDPOINT: 'https://s3', + STORAGE_BUCKET: 'b', + STORAGE_ACCESS_KEY_ID: 'k', + STORAGE_SECRET_ACCESS_KEY: 's', + STORAGE_REGION: 'eu-west-1', + STORAGE_PREFIX: 'apps/a', + STORAGE_FORCE_PATH_STYLE: 'true', + }), + ).toEqual({ + provider: 's3', + prefix: 'apps/a', + config: { + endpoint: 'https://s3', + bucket: 'b', + accessKeyId: 'k', + secretAccessKey: 's', + region: 'eu-west-1', + forcePathStyle: true, + }, + }); + }); +}); diff --git a/src/artifacts/crypto/golden.spec.ts b/src/artifacts/crypto/golden.spec.ts new file mode 100644 index 0000000..2c55916 --- /dev/null +++ b/src/artifacts/crypto/golden.spec.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'vitest'; +import { + GOLDEN_CTX, + GOLDEN_ENVELOPE, + GOLDEN_KEK, + GOLDEN_PLAINTEXT, +} from '../../../test/fixtures/cae1-golden.js'; + +import { open } from './codec.js'; +import { canonicalJson, SCOPE_FROM_HEADER } from './context.js'; +import { LocalKeyProvider } from './local-provider.js'; + +/** + * FORMAT-STABILITY GUARD. This envelope was sealed once and is FROZEN: it pins the CAE1 framing, + * the header key names, the canonicalJson key order, the AAD construction (ctx + content type), + * and the local wrap layout. If any of these tests fail after a code change, that change silently + * bricks every already-stored object (the CredentialCipher failure mode ADR-0001 exists to + * prevent) — fix the code, do NOT regenerate the vector. A deliberate format change is a new + * envelope version `v`, written alongside continued v1 read support. + */ +describe('CAE1 format stability (golden vector)', () => { + it('opens the frozen v1 envelope byte-for-byte', async () => { + const provider = new LocalKeyProvider('golden', GOLDEN_KEK); + const out = await open(GOLDEN_ENVELOPE, GOLDEN_CTX, provider); + expect(out.plain.toString('utf8')).toBe(GOLDEN_PLAINTEXT); + expect(out.contentType).toBe('text/html; charset=utf-8'); + expect(out.legacy).toBe(false); + }); + + it("opens the frozen envelope under the sandbox's header-scope contract too", async () => { + const provider = new LocalKeyProvider('golden', GOLDEN_KEK); + const out = await open( + GOLDEN_ENVELOPE, + { ...GOLDEN_CTX, scope: SCOPE_FROM_HEADER }, + provider, + ); + expect(out.ctx.scope).toBe('org:golden-org'); + }); + + it('pins the canonical AAD serialization exactly', () => { + // Sorted key order, no whitespace — SEC-04's rewrap tooling and every stored tag depend on it. + expect(canonicalJson(GOLDEN_CTX)).toBe( + '{"artifactId":"golden-artifact","path":"index.html","scope":"org:golden-org","version":null}', + ); + }); + + it('pins the frame layout and header key set', () => { + expect(GOLDEN_ENVELOPE.subarray(0, 4).toString()).toBe('CAE1'); + const headerLen = GOLDEN_ENVELOPE.readUInt32BE(4); + const header = JSON.parse( + GOLDEN_ENVELOPE.subarray(8, 8 + headerLen).toString('utf8'), + ) as Record; + expect(Object.keys(header)).toEqual([ + 'v', + 'alg', + 'kid', + 'wdk', + 'iv', + 'ctx', + 'ct', + ]); + expect(header.v).toBe(1); + expect(header.alg).toBe('A256GCM'); + expect(header.kid).toBe('local:golden'); + }); +}); diff --git a/src/artifacts/crypto/index.ts b/src/artifacts/crypto/index.ts new file mode 100644 index 0000000..02d3408 --- /dev/null +++ b/src/artifacts/crypto/index.ts @@ -0,0 +1,71 @@ +import { CachingKeyProvider, type DekCacheOptions } from './dek-cache.js'; +import { + keyProviderConfigFromEnv, + type KeyProvider, + type KeyProviderConfig, +} from './key-provider.js'; +import { KmsKeyProvider } from './kms-provider.js'; +import { LocalKeyProvider } from './local-provider.js'; + +export { + artifactScope, + canonicalJson, + orgScope, + SCOPE_FROM_HEADER, + uploadScope, + type EnvelopeContext, + type ExpectedContext, +} from './context.js'; +export { + ENVELOPE_SUPPORTED_VERSIONS, + ENVELOPE_WRITE_VERSION, + ENVELOPE_MAX_OVERHEAD_BYTES, + EnvelopeError, + isEnvelope, + open, + seal, + type EnvelopeFailureReason, + type OpenOptions, + type OpenResult, +} from './codec.js'; +export { CachingKeyProvider, type DekCacheOptions } from './dek-cache.js'; +export { + KEK_GENERATE_HINT, + keyProviderConfigFromEnv, + type DataKey, + type KeyProvider, + type KeyProviderConfig, +} from './key-provider.js'; +export { KmsKeyProvider } from './kms-provider.js'; +export { LocalKeyProvider } from './local-provider.js'; + +/** Everything a storage backend needs to encrypt/decrypt. Required — there is no plaintext mode. */ +export interface StorageCrypto { + keyProvider: KeyProvider; + /** SEC-04's migration-window flag; see OpenOptions.allowLegacyPlaintext. Default off. */ + allowLegacyPlaintext?: boolean; +} + +/** Build the configured provider wrapped in the process-wide DEK cache (ADR-0001 D6.2). */ +export function createKeyProvider( + cfg: KeyProviderConfig, + opts?: { cache?: DekCacheOptions }, +): KeyProvider { + const inner = + cfg.kind === 'kms' + ? new KmsKeyProvider(cfg.keyId, cfg.region) + : new LocalKeyProvider(cfg.name, cfg.kek); + return new CachingKeyProvider(inner, opts?.cache); +} + +/** + * One-call env→crypto resolution shared by the api and the sandbox (each constructs storage from + * env separately; this keeps their validation identical). Throws on misconfiguration — callers + * fail boot, following the SEC-00 gating pattern. + */ +export function storageCryptoFromEnv(env: NodeJS.ProcessEnv): StorageCrypto { + return { + keyProvider: createKeyProvider(keyProviderConfigFromEnv(env)), + allowLegacyPlaintext: env.ARTIFACT_ENCRYPTION_READ_LEGACY === 'true', + }; +} diff --git a/src/artifacts/crypto/key-provider.ts b/src/artifacts/crypto/key-provider.ts new file mode 100644 index 0000000..fed8394 --- /dev/null +++ b/src/artifacts/crypto/key-provider.ts @@ -0,0 +1,86 @@ +import type { EnvelopeContext } from './context.js'; + +/** A fresh per-object data key: the plaintext DEK plus its wrapped form and the wrapping key id. */ +export interface DataKey { + /** 32-byte AES-256 key. Callers own this buffer and must zero it after use. */ + dek: Buffer; + /** The DEK wrapped by the provider's KEK/CMK; safe to persist in the envelope header. */ + wdk: Buffer; + /** Wrapping key identifier (`local:` or the KMS key ARN). */ + kid: string; +} + +/** + * Pluggable wrap/unwrap of per-object DEKs (ADR-0001 D4). Both operations take the full + * EnvelopeContext and MUST bind it to the wrap (KMS EncryptionContext / local wrap-AAD): that + * binding is what makes the sandbox's header-sourced `scope` trustworthy — see SCOPE_FROM_HEADER. + * There is deliberately no `none` implementation; unenveloped writes must not exist. + */ +export interface KeyProvider { + generateDataKey(ctx: EnvelopeContext): Promise; + /** + * Unwrap a stored DEK. Must reject a `kid` that does not belong to this provider's configured + * key — a hostile header can never select the decrypting key. Returns a caller-owned copy. + */ + unwrapDataKey( + wdk: Buffer, + kid: string, + ctx: EnvelopeContext, + ): Promise; + /** Drop and zero any cached key material (shutdown hook). */ + clear(): void; +} + +export type KeyProviderConfig = + | { kind: 'local'; name: string; kek: Buffer } + | { kind: 'kms'; keyId: string; region?: string }; + +/** One-liner shown wherever a missing/invalid KEK aborts boot. */ +export const KEK_GENERATE_HINT = + "generate one with: node -e \"console.log(require('node:crypto').randomBytes(32).toString('base64'))\""; + +/** + * Shared env→config validation for the api and the sandbox (both construct storage from env + * separately; this is the single source of truth, same role as the S3 both-or-neither rule). + * Throws on any misconfiguration — boot must fail, never degrade to plaintext (ADR D4). + */ +export function keyProviderConfigFromEnv( + env: NodeJS.ProcessEnv, +): KeyProviderConfig { + const provider = env.ARTIFACT_KEY_PROVIDER; + if (provider === 'local') { + if (!env.ARTIFACT_KEK) { + throw new Error( + `ARTIFACT_KEY_PROVIDER=local requires ARTIFACT_KEK (base64, 32 bytes); ${KEK_GENERATE_HINT}`, + ); + } + const kek = Buffer.from(env.ARTIFACT_KEK, 'base64'); + // Round-trip the decode: Buffer.from(_, "base64") silently tolerates garbage. + if ( + kek.length !== 32 || + kek.toString('base64').replace(/=+$/, '') !== + env.ARTIFACT_KEK.replace(/=+$/, '') + ) { + throw new Error( + `ARTIFACT_KEK must be exactly 32 random bytes, base64-encoded; ${KEK_GENERATE_HINT}`, + ); + } + return { kind: 'local', name: env.ARTIFACT_KEK_NAME || 'default', kek }; + } + if (provider === 'kms') { + if (!env.ARTIFACT_KMS_KEY_ID) { + throw new Error( + 'ARTIFACT_KEY_PROVIDER=kms requires ARTIFACT_KMS_KEY_ID (KMS key id, ARN, or alias)', + ); + } + const region = env.ARTIFACT_KMS_REGION || env.S3_REGION; + return { + kind: 'kms', + keyId: env.ARTIFACT_KMS_KEY_ID, + ...(region === undefined ? {} : { region }), + }; + } + throw new Error( + 'ARTIFACT_KEY_PROVIDER must be "local" or "kms" — artifact storage is always encrypted and has no plaintext mode (ADR-0001)', + ); +} diff --git a/src/artifacts/crypto/kms-provider.spec.ts b/src/artifacts/crypto/kms-provider.spec.ts new file mode 100644 index 0000000..db160b2 --- /dev/null +++ b/src/artifacts/crypto/kms-provider.spec.ts @@ -0,0 +1,119 @@ +import { + DecryptCommand, + GenerateDataKeyCommand, + KMSClient, +} from '@aws-sdk/client-kms'; +import { mockClient } from 'aws-sdk-client-mock'; +import { randomBytes } from 'node:crypto'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { type EnvelopeContext } from './context.js'; +import { KmsKeyProvider } from './kms-provider.js'; + +const CTX: EnvelopeContext = { + scope: 'org:o1', + artifactId: 'a1', + version: null, + path: 'index.html', +}; +const KEY_ARN = 'arn:aws:kms:us-east-1:111122223333:key/abc'; + +const kms = mockClient(KMSClient); + +afterEach(() => kms.reset()); + +describe('KmsKeyProvider', () => { + it('generates under the configured key with the scope+artifactId EncryptionContext', async () => { + const dek = randomBytes(32); + const expectedDek = Buffer.from(dek); + kms.on(GenerateDataKeyCommand).resolves({ + Plaintext: dek, + CiphertextBlob: randomBytes(60), + KeyId: KEY_ARN, + }); + const p = new KmsKeyProvider('alias/concepta-artifacts', 'us-east-1'); + const out = await p.generateDataKey(CTX); + expect(out.kid).toBe(KEY_ARN); + expect(out.dek.equals(expectedDek)).toBe(true); + expect(dek.equals(Buffer.alloc(32))).toBe(true); + const input = kms.commandCalls(GenerateDataKeyCommand)[0]!.args[0].input; + expect(input.KeyId).toBe('alias/concepta-artifacts'); + expect(input.KeySpec).toBe('AES_256'); + expect(input.EncryptionContext).toEqual({ + scope: 'org:o1', + artifactId: 'a1', + }); + }); + + it('omits a null artifactId from the EncryptionContext (KMS values must be strings)', async () => { + kms.on(GenerateDataKeyCommand).resolves({ + Plaintext: randomBytes(32), + CiphertextBlob: randomBytes(60), + KeyId: KEY_ARN, + }); + const p = new KmsKeyProvider(KEY_ARN); + await p.generateDataKey({ + scope: 'upload:u1', + artifactId: null, + version: null, + path: '_staging/u1', + }); + const input = kms.commandCalls(GenerateDataKeyCommand)[0]!.args[0].input; + expect(input.EncryptionContext).toEqual({ scope: 'upload:u1' }); + }); + + it('pins Decrypt to the configured key and verifies the resolved key against kid', async () => { + const dek = randomBytes(32); + const expectedDek = Buffer.from(dek); + kms.on(DecryptCommand).resolves({ Plaintext: dek, KeyId: KEY_ARN }); + const p = new KmsKeyProvider('alias/concepta-artifacts'); + const out = await p.unwrapDataKey(randomBytes(60), KEY_ARN, CTX); + expect(out.equals(expectedDek)).toBe(true); + expect(dek.equals(Buffer.alloc(32))).toBe(true); + const input = kms.commandCalls(DecryptCommand)[0]!.args[0].input; + // The attacker-influenced header kid must never reach KMS as the key selector. + expect(input.KeyId).toBe('alias/concepta-artifacts'); + expect(input.EncryptionContext).toEqual({ + scope: 'org:o1', + artifactId: 'a1', + }); + }); + + it('rejects a tampered header kid after pinned KMS decryption', async () => { + const dek = randomBytes(32); + kms.on(DecryptCommand).resolves({ + Plaintext: dek, + KeyId: KEY_ARN, + }); + const p = new KmsKeyProvider('alias/concepta-artifacts'); + await expect( + p.unwrapDataKey( + randomBytes(60), + 'arn:aws:kms:us-east-1:999:key/attacker', + CTX, + ), + ).rejects.toThrow(/does not match/); + expect(dek.equals(Buffer.alloc(32))).toBe(true); + }); + + it('accepts the same multi-Region key replica in another region', async () => { + const east = 'arn:aws:kms:us-east-1:111122223333:key/mrk-shared'; + const west = 'arn:aws:kms:us-west-2:111122223333:key/mrk-shared'; + kms.on(DecryptCommand).resolves({ + Plaintext: randomBytes(32), + KeyId: west, + }); + const p = new KmsKeyProvider(west); + await expect( + p.unwrapDataKey(randomBytes(60), east, CTX), + ).resolves.toHaveLength(32); + }); + + it('propagates a KMS failure (wrong context / unavailable key)', async () => { + kms.on(DecryptCommand).rejects(new Error('InvalidCiphertextException')); + const p = new KmsKeyProvider(KEY_ARN); + await expect( + p.unwrapDataKey(randomBytes(60), KEY_ARN, CTX), + ).rejects.toThrow(); + }); +}); diff --git a/src/artifacts/crypto/kms-provider.ts b/src/artifacts/crypto/kms-provider.ts new file mode 100644 index 0000000..de7a4ed --- /dev/null +++ b/src/artifacts/crypto/kms-provider.ts @@ -0,0 +1,115 @@ +import { + DecryptCommand, + GenerateDataKeyCommand, + KMSClient, +} from '@aws-sdk/client-kms'; + +import type { EnvelopeContext } from './context.js'; +import type { DataKey, KeyProvider } from './key-provider.js'; + +/** + * KMS EncryptionContext for the wrap (ADR-0001 D3): `{scope, artifactId}` — so CloudTrail + * records which tenant's which artifact every decrypt serves, and a wrapped DEK cannot be + * unwrapped under a different claimed context. KMS context values must be strings, so a null + * artifactId (ObjectStore objects) is omitted rather than stringified. + */ +function encryptionContext(ctx: EnvelopeContext): Record { + return ctx.artifactId === null + ? { scope: ctx.scope } + : { scope: ctx.scope, artifactId: ctx.artifactId }; +} + +function equivalentKmsKeyId(headerKid: string, resolvedKeyId: string): boolean { + if (headerKid === resolvedKeyId) return true; + + // Multi-Region replicas share the same `mrk-…` resource id but have region-specific ARNs. + // Treat those replicas as one logical CMK only when partition and account also match. + const parseMrk = (value: string): [string, string, string] | null => { + const match = + /^arn:([^:]+):kms:[^:]+:([^:]+):key\/(mrk-[A-Za-z0-9-]+)$/.exec(value); + return match === null ? null : [match[1]!, match[2]!, match[3]!]; + }; + const header = parseMrk(headerKid); + const resolved = parseMrk(resolvedKeyId); + return ( + header !== null && + resolved !== null && + header.every((part, index) => part === resolved[index]) + ); +} + +/** + * AWS KMS provider for deployed environments (ADR-0001 D4). Uses the SDK default credential + * chain (ECS task role), like the S3 backend. Requires SEC-02's CMK and task-role grants: + * api — GenerateDataKey/Decrypt/ReEncrypt*; sandbox — Decrypt only. + */ +export class KmsKeyProvider implements KeyProvider { + private readonly client: KMSClient; + + constructor( + private readonly keyId: string, + region?: string, + ) { + this.client = new KMSClient(region ? { region } : {}); + } + + async generateDataKey(ctx: EnvelopeContext): Promise { + const out = await this.client.send( + new GenerateDataKeyCommand({ + KeyId: this.keyId, + KeySpec: 'AES_256', + EncryptionContext: encryptionContext(ctx), + }), + ); + if (!out.Plaintext || !out.CiphertextBlob || !out.KeyId) { + out.Plaintext?.fill(0); + throw new Error('KMS GenerateDataKey returned an incomplete response'); + } + // kid is the resolved key ARN (not the configured alias) so envelopes are self-describing + // for SEC-04's rewrap tooling. + try { + return { + dek: Buffer.from(out.Plaintext), + wdk: Buffer.from(out.CiphertextBlob), + kid: out.KeyId, + }; + } finally { + out.Plaintext.fill(0); + } + } + + async unwrapDataKey( + wdk: Buffer, + kid: string, + ctx: EnvelopeContext, + ): Promise { + // The header kid is attacker-influenced and is deliberately NOT passed to KMS: Decrypt is + // pinned to the configured key, so KMS itself rejects ciphertext wrapped under any other key + // (a stricter guarantee than string-comparing kid, which may be an alias vs the ARN). + const out = await this.client.send( + new DecryptCommand({ + KeyId: this.keyId, + CiphertextBlob: wdk, + EncryptionContext: encryptionContext(ctx), + }), + ); + if (!out.Plaintext || !out.KeyId) { + out.Plaintext?.fill(0); + throw new Error('KMS Decrypt returned an incomplete response'); + } + try { + if (!equivalentKmsKeyId(kid, out.KeyId)) { + throw new Error( + 'KMS Decrypt resolved a key that does not match the envelope', + ); + } + return Buffer.from(out.Plaintext); + } finally { + out.Plaintext.fill(0); + } + } + + clear(): void { + this.client.destroy(); + } +} diff --git a/src/artifacts/crypto/local-provider.spec.ts b/src/artifacts/crypto/local-provider.spec.ts new file mode 100644 index 0000000..f3d6b73 --- /dev/null +++ b/src/artifacts/crypto/local-provider.spec.ts @@ -0,0 +1,52 @@ +import { randomBytes } from 'node:crypto'; +import { describe, expect, it } from 'vitest'; + +import { type EnvelopeContext } from './context.js'; +import { LocalKeyProvider } from './local-provider.js'; + +const CTX: EnvelopeContext = { + scope: 'org:o1', + artifactId: 'a1', + version: null, + path: 'index.html', +}; + +describe('LocalKeyProvider', () => { + it('round-trips a data key under the same context', async () => { + const p = new LocalKeyProvider('test', randomBytes(32)); + const { dek, wdk, kid } = await p.generateDataKey(CTX); + expect(kid).toBe('local:test'); + expect(wdk.includes(dek)).toBe(false); + const out = await p.unwrapDataKey(wdk, kid, CTX); + expect(out.equals(dek)).toBe(true); + }); + + it('binds the wrap to the full context — a different scope fails to unwrap', async () => { + const p = new LocalKeyProvider('test', randomBytes(32)); + const { wdk, kid } = await p.generateDataKey(CTX); + await expect( + p.unwrapDataKey(wdk, kid, { ...CTX, scope: 'org:attacker' }), + ).rejects.toThrow(); + }); + + it('rejects a foreign kid without attempting the unwrap', async () => { + const p = new LocalKeyProvider('test', randomBytes(32)); + const { wdk } = await p.generateDataKey(CTX); + await expect(p.unwrapDataKey(wdk, 'local:other', CTX)).rejects.toThrow( + /unknown key id/, + ); + }); + + it('rejects a wdk wrapped under a different KEK', async () => { + const a = new LocalKeyProvider('test', randomBytes(32)); + const b = new LocalKeyProvider('test', randomBytes(32)); + const { wdk, kid } = await a.generateDataKey(CTX); + await expect(b.unwrapDataKey(wdk, kid, CTX)).rejects.toThrow(); + }); + + it('requires a 32-byte KEK', () => { + expect(() => new LocalKeyProvider('test', randomBytes(16))).toThrow( + /32 bytes/, + ); + }); +}); diff --git a/src/artifacts/crypto/local-provider.ts b/src/artifacts/crypto/local-provider.ts new file mode 100644 index 0000000..e2e8848 --- /dev/null +++ b/src/artifacts/crypto/local-provider.ts @@ -0,0 +1,75 @@ +import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto'; + +import { canonicalJson, type EnvelopeContext } from './context.js'; +import type { DataKey, KeyProvider } from './key-provider.js'; + +// wdk layout: 12-byte IV ‖ 16-byte GCM tag ‖ 32-byte wrapped DEK. +const IV_LEN = 12; +const TAG_LEN = 16; +const DEK_LEN = 32; + +/** + * Static-KEK provider for dev, CI, and self-hosted fs deployments (ADR-0001 D4). The wrap is + * AES-256-GCM with the canonical context as AAD, so a wdk replayed under a different + * scope/artifact/path fails to unwrap — the same property KMS EncryptionContext gives the `kms` + * provider, and the property the sandbox's header-sourced scope relies on. + */ +export class LocalKeyProvider implements KeyProvider { + private readonly kid: string; + + constructor( + name: string, + private readonly kek: Buffer, + ) { + if (kek.length !== 32) + throw new Error('local KEK must be exactly 32 bytes'); + this.kid = `local:${name}`; + } + + generateDataKey(ctx: EnvelopeContext): Promise { + const dek = randomBytes(DEK_LEN); + const iv = randomBytes(IV_LEN); + const cipher = createCipheriv('aes-256-gcm', this.kek, iv); + cipher.setAAD(Buffer.from(canonicalJson(ctx))); + const wrapped = Buffer.concat([cipher.update(dek), cipher.final()]); + const wdk = Buffer.concat([iv, cipher.getAuthTag(), wrapped]); + return Promise.resolve({ dek, wdk, kid: this.kid }); + } + + unwrapDataKey( + wdk: Buffer, + kid: string, + ctx: EnvelopeContext, + ): Promise { + // A hostile header must never select the key: only our own kid is accepted. + if (kid !== this.kid) { + return Promise.reject( + new Error(`unknown key id "${kid}" (configured: "${this.kid}")`), + ); + } + if (wdk.length !== IV_LEN + TAG_LEN + DEK_LEN) { + return Promise.reject(new Error('malformed wrapped data key')); + } + const iv = wdk.subarray(0, IV_LEN); + const tag = wdk.subarray(IV_LEN, IV_LEN + TAG_LEN); + const wrapped = wdk.subarray(IV_LEN + TAG_LEN); + const decipher = createDecipheriv('aes-256-gcm', this.kek, iv); + decipher.setAAD(Buffer.from(canonicalJson(ctx))); + decipher.setAuthTag(tag); + try { + return Promise.resolve( + Buffer.concat([decipher.update(wrapped), decipher.final()]), + ); + } catch { + // GCM cannot distinguish wrong-context from corruption; both are "this key is not + // available for this context" — fail closed without detail. + return Promise.reject( + new Error('wrapped data key failed to unwrap for this context'), + ); + } + } + + clear(): void { + // Stateless: the KEK itself must outlive clear() (it is the provider's configuration). + } +} diff --git a/src/artifacts/env-config.ts b/src/artifacts/env-config.ts new file mode 100644 index 0000000..3ba46cb --- /dev/null +++ b/src/artifacts/env-config.ts @@ -0,0 +1,124 @@ +import type { StorageProviderConfig } from '../files-sdk/provider/index.js'; + +import { + assertStorageProvider, + defaultFsRoot, + type ArtifactStorageConfig, +} from './storage-driver.js'; + +/** + * Shared env→storage-config validation for every service that constructs storage (the api's + * zod-validated env delegates here with normalized values; the sandbox passes process.env + * directly). One source of truth for the rules — the two services once drifted by + * re-implementing them independently. + * + * `STORAGE_PROVIDER` names any provider `@nestm/storage` can build (`fs`, `s3`, `gcs`, `azure`, + * `r2`, `minio`, `supabase`, …); the `STORAGE_*` values below are one flat bag that each provider + * reads what it needs from. The platform is not S3-only: it used to be, because storage shipped + * exactly two hand-written drivers, and the old `ARTIFACT_STORAGE=fs|s3` switch chose between + * them. Those names are still accepted so a deployment can migrate without a coordinated restart. + */ +export function artifactStorageConfigFromEnv( + env: NodeJS.ProcessEnv, +): ArtifactStorageConfig { + const provider = assertStorageProvider( + env.STORAGE_PROVIDER || legacyProvider(env) || 'fs', + ); + const prefix = env.STORAGE_PREFIX || env.S3_PREFIX || undefined; + const config = flatConfig(env); + + // Both-or-neither: both set → static credentials (an IAM user, or any S3-compatible store); + // both unset → the provider SDK's own chain (the ECS task role, Application Default + // Credentials, a shared profile) so no long-lived keys sit in env. + if (!config.accessKeyId !== !config.secretAccessKey) { + throw new Error( + "STORAGE_ACCESS_KEY_ID and STORAGE_SECRET_ACCESS_KEY must be set together (omit both to use the provider's default credential chain)", + ); + } + + if (provider === 'fs') { + return { + provider, + ...(prefix ? { prefix } : {}), + config: { root: config.root ?? defaultFsRoot() }, + }; + } + + // `root` is filesystem-only, and the api's schema always defaults one. Carrying it onto an + // object-store provider would both litter the config and make the guard below think a bucketless + // deployment was configured. + const { root: _fsOnly, ...objectStoreConfig } = config; + + // Generic "you configured nothing" guard. Which field a provider actually needs is the + // adapter's business — it fails closed on its own — but an empty config is always a + // deployment mistake, and catching it here names the variable to set. + if (!identifiesAStore(objectStoreConfig)) { + throw new Error( + `STORAGE_PROVIDER=${provider} needs somewhere to write: set STORAGE_BUCKET (or the container/store field that provider uses)`, + ); + } + + return { provider, ...(prefix ? { prefix } : {}), config: objectStoreConfig }; +} + +/** The retired `ARTIFACT_STORAGE=fs|s3` switch, mapped onto a provider slug. */ +function legacyProvider(env: NodeJS.ProcessEnv): string | undefined { + return env.ARTIFACT_STORAGE || undefined; +} + +function identifiesAStore(config: StorageProviderConfig): boolean { + return [ + config.bucket, + config.container, + config.connectionString, + config.storeName, + config.token, + config.url, + ].some((value) => typeof value === 'string' && value.length > 0); +} + +/** + * One flat bag of provider settings, assembled from `STORAGE_*` with the retired `S3_*` and + * `ARTIFACTS_DIR` names as fallbacks. Absent values are omitted rather than set to `undefined`, + * so a provider's own credential chain stays reachable. + */ +function flatConfig(env: NodeJS.ProcessEnv): StorageProviderConfig { + const entries: Array<[string, string | boolean | undefined]> = [ + ['accessKeyId', env.STORAGE_ACCESS_KEY_ID || env.S3_ACCESS_KEY_ID], + ['accountId', env.STORAGE_ACCOUNT_ID], + ['accountKey', env.STORAGE_ACCOUNT_KEY], + ['accountName', env.STORAGE_ACCOUNT_NAME], + ['bucket', env.STORAGE_BUCKET || env.S3_BUCKET], + ['connectionString', env.STORAGE_CONNECTION_STRING], + ['container', env.STORAGE_CONTAINER], + ['endpoint', env.STORAGE_ENDPOINT || env.S3_ENDPOINT], + ['keyFilename', env.STORAGE_KEY_FILENAME], + ['projectId', env.STORAGE_PROJECT_ID], + ['publicBaseUrl', env.STORAGE_PUBLIC_BASE_URL], + ['region', env.STORAGE_REGION || env.S3_REGION], + ['root', env.STORAGE_ROOT || env.ARTIFACTS_DIR], + [ + 'secretAccessKey', + env.STORAGE_SECRET_ACCESS_KEY || env.S3_SECRET_ACCESS_KEY, + ], + ['serviceRoleKey', env.STORAGE_SERVICE_ROLE_KEY], + ['sessionToken', env.STORAGE_SESSION_TOKEN], + ['siteId', env.STORAGE_SITE_ID], + ['storeName', env.STORAGE_STORE_NAME], + ['token', env.STORAGE_TOKEN], + ['url', env.STORAGE_URL], + ]; + + const forcePathStyle = + env.STORAGE_FORCE_PATH_STYLE || env.S3_FORCE_PATH_STYLE; + if (forcePathStyle) + entries.push(['forcePathStyle', forcePathStyle === 'true']); + + // One dynamic bag by design: the fields are provider-specific and the union is wider than any + // one deployment uses, so it is assembled by name rather than spelled out per provider. + return Object.fromEntries( + entries.flatMap(([key, value]) => + value === undefined || value === '' ? [] : [[key, value]], + ), + ) as StorageProviderConfig; +} diff --git a/src/artifacts/index.spec.ts b/src/artifacts/index.spec.ts new file mode 100644 index 0000000..0ca44d6 --- /dev/null +++ b/src/artifacts/index.spec.ts @@ -0,0 +1,824 @@ +import { + DeleteObjectCommand, + GetObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { mockClient } from 'aws-sdk-client-mock'; +import { randomBytes } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + ArtifactBundleLimitError, + ArtifactReadLimitError, + createArtifactStorage, + createArtifactStorageWithClient, + createObjectStore, + type ArtifactStorage, +} from './index.js'; +import { + EnvelopeError, + LocalKeyProvider, + seal, + SCOPE_FROM_HEADER, + type StorageCrypto, +} from './crypto/index.js'; + +const crypto: StorageCrypto = { + keyProvider: new LocalKeyProvider('spec', randomBytes(32)), +}; +const REF = { scope: 'org:org-1' }; +const ARTIFACT_ID = '11111111-1111-4111-8111-111111111111'; +const OTHER_ARTIFACT_ID = '22222222-2222-4222-8222-222222222222'; + +// The `fs` backend against a real temp dir: covers the write→read round-trip, ciphertext-at-rest, +// nested-parent creation, the containment guard, and the ADR D6.1 path-as-read contract. +describe('FsArtifactStorage', () => { + let root: string; + let storage: ArtifactStorage; + + beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), 'storage-spec-')); + storage = await createArtifactStorage( + { provider: 'fs', config: { root } }, + crypto, + ); + }); + + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + it('round-trips a written file through read()', async () => { + const body = Buffer.from('{"vendored":true}', 'utf8'); + await storage.writeFile(ARTIFACT_ID, '__meta.json', body, REF); + expect(await storage.read(ARTIFACT_ID, '__meta.json', REF)).toEqual(body); + }); + + it('bounds the opaque envelope download and decrypted plaintext', async () => { + const body = Buffer.alloc(257, 0x61); + await storage.writeFile(ARTIFACT_ID, 'surface.a2ui.json', body, REF); + + await expect( + storage.read(ARTIFACT_ID, 'surface.a2ui.json', REF, { + maxPlainBytes: 256, + }), + ).rejects.toEqual(new ArtifactReadLimitError(256)); + await expect( + storage.read(ARTIFACT_ID, 'surface.a2ui.json', REF, { + maxPlainBytes: 257, + }), + ).resolves.toEqual(body); + }); + + it('rejects an oversized object from metadata without downloading its body', async () => { + const head = vi.fn().mockResolvedValue({ size: 10_000_000 }); + const downloadBytes = vi.fn(); + const bounded = createArtifactStorageWithClient( + { provider: 's3', config: { bucket: 'unused' } }, + crypto, + { head, downloadBytes } as never, + ); + + await expect( + bounded.read(ARTIFACT_ID, 'surface.a2ui.json', REF, { + maxPlainBytes: 256, + }), + ).rejects.toBeInstanceOf(ArtifactReadLimitError); + expect(head).toHaveBeenCalledWith(`${ARTIFACT_ID}/surface.a2ui.json`); + expect(downloadBytes).not.toHaveBeenCalled(); + }); + + it('never masks a direct object-provider error with an index fallback', async () => { + const fallback = await seal( + Buffer.from('fallback'), + { + scope: REF.scope, + artifactId: ARTIFACT_ID, + version: null, + path: 'docs/index.html', + }, + crypto.keyProvider, + 'text/html; charset=utf-8', + ); + const downloadBytes = vi.fn(async (key: string) => { + if (key === `${ARTIFACT_ID}/docs`) throw new Error('access denied'); + return fallback; + }); + const guarded = createArtifactStorageWithClient( + { provider: 's3' }, + crypto, + { downloadBytes } as never, + ); + + await expect(guarded.read(ARTIFACT_ID, 'docs', REF)).rejects.toThrow( + 'access denied', + ); + expect(downloadBytes).toHaveBeenCalledOnce(); + }); + + it('stores ciphertext, not plaintext, and nothing on disk but the object and its metadata', async () => { + const secret = Buffer.from( + 'customer dashboard 4d61726b6572', + 'utf8', + ); + await storage.writeHtml(ARTIFACT_ID, secret, REF); + + // The filesystem driver keeps a metadata sidecar beside each body — a filesystem has nowhere + // else to put an ETag or a content type, where an object store has native metadata. It is + // driver bookkeeping, never a key: `list` and `search` skip it. No legacy `.ct` sidecars and + // no temp files. + const files = readdirSync(join(root, ARTIFACT_ID), { + recursive: true, + }) as string[]; + expect(files.sort()).toEqual(['index.html', 'index.html.meta.json']); + + const onDisk = readFileSync(join(root, ARTIFACT_ID, 'index.html')); + expect(onDisk.subarray(0, 4).toString()).toBe('CAE1'); + expect(onDisk.includes(secret)).toBe(false); + + // The sidecar must not become a plaintext side channel: the real content type stays inside + // the authenticated envelope (ADR-0001), so what lands here is the opaque default. + const sidecar = readFileSync( + join(root, ARTIFACT_ID, 'index.html.meta.json'), + 'utf8', + ); + expect(sidecar.includes(secret.toString())).toBe(false); + expect(sidecar).not.toContain('text/html'); + }); + + it('creates parent directories for a nested relPath', async () => { + const body = Buffer.from('export const x = 1;', 'utf8'); + await storage.writeFile(ARTIFACT_ID, '__vendor/abc123.js', body, REF); + expect(existsSync(join(root, ARTIFACT_ID, '__vendor', 'abc123.js'))).toBe( + true, + ); + expect(await storage.read(ARTIFACT_ID, '__vendor/abc123.js', REF)).toEqual( + body, + ); + }); + + it.each(['../escape', 'a/../../escape', '/etc/passwd'])( + 'rejects a path escaping containment: %s', + async (relPath) => { + await expect( + storage.writeFile(ARTIFACT_ID, relPath, Buffer.from('x'), REF), + ).rejects.toThrow(); + // The `..` cases would resolve to `/escape` (outside the artifact dir) if unguarded. + expect(existsSync(join(root, 'escape'))).toBe(false); + }, + ); + + it.each([ + '', + '.', + '..', + '../escape', + 'nested/id', + 'nested\\id', + '_objects', + '_OBJECTS', + '_staging', + 'ORG-LOGOS', + ])( + 'rejects an artifact id that can cross storage namespaces: %j', + async (artifactId) => { + await expect( + storage.writeHtml(artifactId, Buffer.from('x'), REF), + ).rejects.toThrow(/artifact id|reserved/); + await expect(storage.remove(artifactId)).rejects.toThrow( + /artifact id|reserved/, + ); + await expect(storage.listFiles(artifactId)).rejects.toThrow( + /artifact id|reserved/, + ); + }, + ); + + it('keeps the filesystem ObjectStore namespace isolated from artifact operations', async () => { + const objects = await createObjectStore( + { provider: 'fs', config: { root } }, + crypto, + ); + await objects.putObject( + 'org-logos/victim', + Buffer.from('logo'), + 'image/png', + { scope: 'org:victim' }, + ); + + await expect(storage.remove('_objects')).rejects.toThrow(/reserved/); + await expect(storage.listFiles('_objects')).rejects.toThrow(/reserved/); + expect( + (await objects.getObject('org-logos/victim', { scope: 'org:victim' })) + ?.body, + ).toEqual(Buffer.from('logo')); + }); + + it("asserts the reader's scope (api-style read)", async () => { + await storage.writeHtml(ARTIFACT_ID, Buffer.from('x'), REF); + await expect( + storage.read(ARTIFACT_ID, 'index.html', { scope: 'org:other' }), + ).rejects.toThrow(EnvelopeError); + }); + + it('decrypts under SCOPE_FROM_HEADER (sandbox-style read)', async () => { + await storage.writeHtml(ARTIFACT_ID, Buffer.from('doc'), REF); + const out = await storage.read(ARTIFACT_ID, 'index.html', { + scope: SCOPE_FROM_HEADER, + }); + expect(out?.toString()).toBe('doc'); + }); + + it('returns null for a missing SPA route (fallback stays a miss, not a decrypt error)', async () => { + await storage.writeHtml(ARTIFACT_ID, Buffer.from('doc'), REF); + expect(await storage.read(ARTIFACT_ID, 'some/spa/route', REF)).toBeNull(); + expect( + (await storage.read(ARTIFACT_ID, 'index.html', REF))?.toString(), + ).toBe('doc'); + }); + + it("binds the AAD to the path actually read: '' and directory rewrites (ADR D6.1)", async () => { + await storage.writeHtml(ARTIFACT_ID, Buffer.from('root-doc'), REF); + await storage.writeFile( + ARTIFACT_ID, + 'app/index.html', + Buffer.from('app-doc'), + REF, + ); + // "" resolves to index.html; "app" (a directory) resolves to app/index.html — both must + // decrypt because the write bound the same effective path the read resolves to. + expect((await storage.read(ARTIFACT_ID, '', REF))?.toString()).toBe( + 'root-doc', + ); + expect((await storage.read(ARTIFACT_ID, 'app', REF))?.toString()).toBe( + 'app-doc', + ); + expect( + ( + await storage.read(ARTIFACT_ID, 'app', REF, { + maxPlainBytes: Buffer.byteLength('app-doc'), + }) + )?.toString(), + ).toBe('app-doc'); + }); + + it('throws on tampered stored bytes — never serves them and never masks as 404', async () => { + await storage.writeHtml(ARTIFACT_ID, Buffer.from('payload'), REF); + const file = join(root, ARTIFACT_ID, 'index.html'); + const bytes = readFileSync(file); + bytes.writeUInt8( + bytes.readUInt8(bytes.length - 1) ^ 0x01, + bytes.length - 1, + ); + writeFileSync(file, bytes); + await expect(storage.read(ARTIFACT_ID, 'index.html', REF)).rejects.toThrow( + EnvelopeError, + ); + }); + + it("swapped objects fail closed: another artifact's file cannot be replayed", async () => { + await storage.writeHtml(ARTIFACT_ID, Buffer.from('a1'), REF); + await storage.writeHtml(OTHER_ARTIFACT_ID, Buffer.from('a2'), REF); + writeFileSync( + join(root, OTHER_ARTIFACT_ID, 'index.html'), + readFileSync(join(root, ARTIFACT_ID, 'index.html')), + ); + await expect( + storage.read(OTHER_ARTIFACT_ID, 'index.html', REF), + ).rejects.toThrow(EnvelopeError); + }); + + it('reads legacy plaintext only under the explicit migration flag', async () => { + const legacyStorage = await createArtifactStorage( + { provider: 'fs', config: { root } }, + { ...crypto, allowLegacyPlaintext: true }, + ); + const legacy = Buffer.from('pre-envelope'); + mkdirSync(join(root, ARTIFACT_ID), { recursive: true }); + writeFileSync(join(root, ARTIFACT_ID, 'index.html'), legacy); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + try { + expect(await legacyStorage.read(ARTIFACT_ID, 'index.html', REF)).toEqual( + legacy, + ); + expect(warn).toHaveBeenCalledOnce(); + await expect( + storage.read(ARTIFACT_ID, 'index.html', REF), + ).rejects.toThrow(EnvelopeError); + } finally { + warn.mockRestore(); + } + }); + + it('expands bundles into per-entry envelopes', async () => { + const AdmZip = (await import('adm-zip')).default; + const zip = new AdmZip(); + zip.addFile('index.html', Buffer.from('bundle')); + zip.addFile('assets/app.js', Buffer.from('console.log(1)')); + await storage.writeBundle(ARTIFACT_ID, zip.toBuffer(), REF); + expect( + (await storage.read(ARTIFACT_ID, 'index.html', REF))?.toString(), + ).toBe('bundle'); + expect( + (await storage.read(ARTIFACT_ID, 'assets/app.js', REF))?.toString(), + ).toBe('console.log(1)'); + const raw = readFileSync(join(root, ARTIFACT_ID, 'assets', 'app.js')); + expect(raw.subarray(0, 4).toString()).toBe('CAE1'); + }); + + it('preflights bundle expansion limits before writing any entry', async () => { + const AdmZip = (await import('adm-zip')).default; + const zip = new AdmZip(); + zip.addFile('first.txt', Buffer.from('ok')); + zip.addFile('oversized.txt', Buffer.from('four')); + + await expect( + storage.writeBundle(ARTIFACT_ID, zip.toBuffer(), REF, { + limits: { maxEntryBytes: 3 }, + }), + ).rejects.toEqual(new ArtifactBundleLimitError('entry-bytes')); + expect(await storage.listFiles(ARTIFACT_ID)).toEqual([]); + }); + + it('bounds bundle entry count and total expanded bytes', async () => { + const AdmZip = (await import('adm-zip')).default; + const zip = new AdmZip(); + zip.addFile('one.txt', Buffer.from('12')); + zip.addFile('two.txt', Buffer.from('34')); + + await expect( + storage.writeBundle(ARTIFACT_ID, zip.toBuffer(), REF, { + limits: { maxEntries: 1 }, + }), + ).rejects.toEqual(new ArtifactBundleLimitError('entries')); + await expect( + storage.writeBundle(ARTIFACT_ID, zip.toBuffer(), REF, { + limits: { maxTotalBytes: 3 }, + }), + ).rejects.toEqual(new ArtifactBundleLimitError('total-bytes')); + expect(await storage.listFiles(ARTIFACT_ID)).toEqual([]); + }); + + it('rejects case-folded duplicate bundle paths before writing', async () => { + const AdmZip = (await import('adm-zip')).default; + const zip = new AdmZip(); + zip.addFile('assets/App.js', Buffer.from('first')); + zip.addFile('assets/app.js', Buffer.from('second')); + + await expect( + storage.writeBundle(ARTIFACT_ID, zip.toBuffer(), REF), + ).rejects.toThrow(/duplicate entry/); + expect(await storage.listFiles(ARTIFACT_ID)).toEqual([]); + }); + + it('rejects an escaping version id before it can address the object-store namespace', async () => { + const AdmZip = (await import('adm-zip')).default; + const zip = new AdmZip(); + zip.addFile('victim', Buffer.from('replacement')); + const objects = await createObjectStore( + { provider: 'fs', config: { root } }, + crypto, + ); + await objects.putObject( + '_staging/victim', + Buffer.from('original'), + 'text/plain', + { scope: 'upload:victim' }, + ); + + await expect( + storage.writeBundle(ARTIFACT_ID, zip.toBuffer(), REF, { + versionId: '../../_objects/_staging', + }), + ).rejects.toThrow(/version id/); + expect( + ( + await objects.getObject('_staging/victim', { + scope: 'upload:victim', + }) + )?.body, + ).toEqual(Buffer.from('original')); + }); + + it('rejects traversal in list prefixes and direct version references', async () => { + await expect( + storage.listFiles(ARTIFACT_ID, '../_objects/'), + ).rejects.toThrow(/list prefix/); + await expect( + storage.writeFile( + ARTIFACT_ID, + 'v/../../_objects/victim', + Buffer.from('x'), + { + scope: REF.scope, + version: '../../_objects', + }, + ), + ).rejects.toThrow(/version id|artifact path|path escapes/); + }); + + // ── AP-002 rule 2: path ⇔ version, asserted at the ctx chokepoint ───────────── + describe('version addressing (VER-02)', () => { + const VID = '0b3adf87-2c1a-4e9e-9f30-1c6a3a1f0000'; + const VREF = { scope: 'org:org-1', version: VID }; + + it('round-trips a versioned object under v// with ctx.version = the vid', async () => { + const body = Buffer.from('immutable v1'); + await storage.writeFile(ARTIFACT_ID, `v/${VID}/index.html`, body, VREF); + expect( + await storage.read(ARTIFACT_ID, `v/${VID}/index.html`, VREF), + ).toEqual(body); + }); + + it('a versioned object can never be read as a root object (AAD binds the mode)', async () => { + await storage.writeFile( + ARTIFACT_ID, + `v/${VID}/index.html`, + Buffer.from('x'), + VREF, + ); + // Root-mode read of the same bytes at the same key is refused BEFORE any key op — the + // path/version assertion throws (v/ path with version null is structurally invalid). + await expect( + storage.read(ARTIFACT_ID, `v/${VID}/index.html`, REF), + ).rejects.toThrow(/reserved v\//); + }); + + it('refuses a versioned write outside its own v// prefix', async () => { + await expect( + storage.writeFile(ARTIFACT_ID, 'index.html', Buffer.from('x'), VREF), + ).rejects.toThrow(/must live under/); + await expect( + storage.writeFile( + ARTIFACT_ID, + 'v/other-vid/index.html', + Buffer.from('x'), + VREF, + ), + ).rejects.toThrow(/must live under/); + // writeHtml hardcodes the root path — passing a version through it must fail, not + // silently write a mis-addressed root object. + await expect( + storage.writeHtml(ARTIFACT_ID, Buffer.from('x'), VREF), + ).rejects.toThrow(/must live under/); + }); + + it('refuses a root write into the reserved v/ namespace', async () => { + await expect( + storage.writeFile( + ARTIFACT_ID, + `v/${VID}/index.html`, + Buffer.from('x'), + REF, + ), + ).rejects.toThrow(/reserved v\//); + await expect( + storage.writeFile( + ARTIFACT_ID, + `V/${VID}/index.html`, + Buffer.from('x'), + REF, + ), + ).rejects.toThrow(/reserved v\//); + }); + + it('writeBundle SKIPS reserved-namespace zip entries instead of throwing mid-write', async () => { + const AdmZip = (await import('adm-zip')).default; + const zip = new AdmZip(); + zip.addFile('index.html', Buffer.from('b')); + zip.addFile('v/legacy-asset.js', Buffer.from('evil or just unlucky')); + zip.addFile('__meta.json', Buffer.from('{"cspTier":"locked"}')); // planted tier marker + zip.addFile('assets/__hidden.js', Buffer.from('x')); + zip.addFile('__vendor/abc123.js', Buffer.from('pool-shaped')); // the serving exemption + await storage.writeBundle(ARTIFACT_ID, zip.toBuffer(), REF); // must NOT throw + expect( + (await storage.read(ARTIFACT_ID, 'index.html', REF))?.toString(), + ).toBe('b'); + const stored = (await storage.listFiles(ARTIFACT_ID)).map((f) => f.path); + expect(stored).toEqual(['__vendor/abc123.js', 'index.html']); + }); + + it('listFiles enumerates paths + mtimes, optionally under a version prefix', async () => { + await storage.writeHtml(ARTIFACT_ID, Buffer.from('root'), REF); + await storage.writeFile( + ARTIFACT_ID, + '__meta.json', + Buffer.from('{}'), + REF, + ); + await storage.writeFile( + ARTIFACT_ID, + `v/${VID}/index.html`, + Buffer.from('v1'), + VREF, + ); + const all = await storage.listFiles(ARTIFACT_ID); + expect(all.map((f) => f.path)).toEqual([ + '__meta.json', + 'index.html', + `v/${VID}/index.html`, + ]); + expect(all.every((f) => f.lastModified instanceof Date)).toBe(true); + const versioned = await storage.listFiles(ARTIFACT_ID, `v/${VID}/`); + expect(versioned.map((f) => f.path)).toEqual([`v/${VID}/index.html`]); + expect(await storage.listFiles('missing-artifact')).toEqual([]); + }); + + // ── VER-03 primitives: the publish copy loop's read/delete/versioned-bundle surface ──────── + it('readWithInfo returns the AUTHENTICATED content type alongside the plaintext', async () => { + await storage.writeFile( + ARTIFACT_ID, + 'assets/app.js', + Buffer.from('console.log(1)'), + REF, + ); + const js = await storage.readWithInfo(ARTIFACT_ID, 'assets/app.js', REF); + expect(js?.plain.toString()).toBe('console.log(1)'); + expect(js?.contentType).toBe('text/javascript; charset=utf-8'); + // Explicit content types survive the round-trip (writeFile's override rides the envelope). + await storage.writeFile( + ARTIFACT_ID, + 'data.bin', + Buffer.from('x'), + REF, + 'application/x-custom', + ); + expect( + (await storage.readWithInfo(ARTIFACT_ID, 'data.bin', REF))?.contentType, + ).toBe('application/x-custom'); + expect( + await storage.readWithInfo(ARTIFACT_ID, 'missing.js', REF), + ).toBeNull(); + }); + + it('readWithInfo round-trips a versioned object (the publish copy read)', async () => { + await storage.writeFile( + ARTIFACT_ID, + `v/${VID}/style.css`, + Buffer.from('body{}'), + VREF, + ); + const out = await storage.readWithInfo( + ARTIFACT_ID, + `v/${VID}/style.css`, + VREF, + ); + expect(out?.plain.toString()).toBe('body{}'); + expect(out?.contentType).toBe('text/css; charset=utf-8'); + }); + + it('deleteFiles removes exactly the named files, ignoring misses and traversal', async () => { + await storage.writeHtml(ARTIFACT_ID, Buffer.from('doc'), REF); + await storage.writeFile( + ARTIFACT_ID, + 'assets/app.js', + Buffer.from('x'), + REF, + ); + await storage.writeFile( + ARTIFACT_ID, + 'assets/old.js', + Buffer.from('y'), + REF, + ); + await storage.deleteFiles(ARTIFACT_ID, [ + 'assets/old.js', + 'assets/never-existed.js', // miss → no-op (publish repair re-runs the same prune) + '../escape', // traversal → skipped, never deletes outside the artifact dir + '/etc/passwd', + '', + ]); + expect((await storage.listFiles(ARTIFACT_ID)).map((f) => f.path)).toEqual( + ['assets/app.js', 'index.html'], + ); + }); + + it('writeBundle with versionId lands entries under v// sealed as versioned objects', async () => { + const AdmZip = (await import('adm-zip')).default; + const zip = new AdmZip(); + zip.addFile('index.html', Buffer.from('v-bundle')); + zip.addFile('assets/app.js', Buffer.from('console.log(2)')); + zip.addFile( + 'v/planted.js', + Buffer.from('reserved entry — must be skipped, not nested'), + ); + zip.addFile('__meta.json', Buffer.from('{"cspTier":"locked"}')); + await storage.writeBundle(ARTIFACT_ID, zip.toBuffer(), REF, { + versionId: VID, + }); + expect( + ( + await storage.read(ARTIFACT_ID, `v/${VID}/index.html`, VREF) + )?.toString(), + ).toBe('v-bundle'); + expect( + ( + await storage.read(ARTIFACT_ID, `v/${VID}/assets/app.js`, VREF) + )?.toString(), + ).toBe('console.log(2)'); + // Reserved entry names are screened on the ENTRY, so nothing lands at v//v/… or + // v//__meta.json — and nothing leaks to the root. + expect((await storage.listFiles(ARTIFACT_ID)).map((f) => f.path)).toEqual( + [`v/${VID}/assets/app.js`, `v/${VID}/index.html`], + ); + // The versioned entries are unreadable as root objects (AAD binds the mode). + await expect( + storage.read(ARTIFACT_ID, `v/${VID}/index.html`, REF), + ).rejects.toThrow(/reserved v\//); + }); + }); +}); + +// The `s3` backend through an in-memory mocked bucket: the same chokepoints (put/read) must +// produce and require envelopes, with the real content type kept out of S3 object metadata. +describe('S3ArtifactStorage', () => { + const s3 = mockClient(S3Client); + const bucket = new Map(); + + beforeEach(() => { + bucket.clear(); + s3.reset(); + s3.on(PutObjectCommand).callsFake( + (input: { Key: string; Body: Buffer; ContentType?: string }) => { + bucket.set(input.Key, { + body: Buffer.from(input.Body), + ...(input.ContentType === undefined + ? {} + : { contentType: input.ContentType }), + }); + return {}; + }, + ); + s3.on(GetObjectCommand).callsFake((input: { Key: string }) => { + const hit = bucket.get(input.Key); + if (!hit) { + const err = new Error('NoSuchKey'); + err.name = 'NoSuchKey'; + throw err; + } + return { + // The driver streams rather than buffering, so the stub answers the same way a real + // GetObject response does. `transformToByteArray` stays for any buffered caller. + Body: { + transformToByteArray: () => Promise.resolve(new Uint8Array(hit.body)), + transformToWebStream: () => + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(hit.body)); + controller.close(); + }, + }), + }, + ContentLength: hit.body.byteLength, + ContentType: hit.contentType, + }; + }); + }); + + function s3Storage(): Promise { + return createArtifactStorage( + { + provider: 's3', + config: { + endpoint: 'http://127.0.0.1:1', + bucket: 'dummy', + accessKeyId: 'k', + secretAccessKey: 's', + region: 'us-east-1', + }, + }, + crypto, + ); + } + + it('round-trips through the mocked bucket as an opaque envelope', async () => { + const storage = await s3Storage(); + const secret = Buffer.from('s3 doc'); + await storage.writeHtml(ARTIFACT_ID, secret, REF); + const stored = bucket.get(`${ARTIFACT_ID}/index.html`)!; + expect(stored.body.subarray(0, 4).toString()).toBe('CAE1'); + expect(stored.body.includes(secret)).toBe(false); + // The served content type lives in the authenticated header, not S3 metadata (ADR D2). + expect(stored.contentType).toBe('application/octet-stream'); + expect(await storage.read(ARTIFACT_ID, 'index.html', REF)).toEqual(secret); + expect(await storage.read(ARTIFACT_ID, '', REF)).toEqual(secret); + }); + + it('keeps configured object namespaces disjoint on a shared object-store root', async () => { + const config = { + provider: 's3' as const, + config: { + endpoint: 'http://127.0.0.1:1', + bucket: 'dummy', + accessKeyId: 'k', + secretAccessKey: 's', + region: 'us-east-1', + }, + }; + const storage = await createArtifactStorage(config, crypto); + const objects = await createObjectStore(config, crypto); + await objects.putObject( + 'org-logos/victim', + Buffer.from('logo'), + 'image/png', + { scope: 'org:victim' }, + ); + + await expect(storage.remove('org-logos')).rejects.toThrow(/reserved/); + await expect(storage.listFiles('ORG-LOGOS')).rejects.toThrow(/reserved/); + expect( + ( + await objects.getObject('org-logos/victim', { + scope: 'org:victim', + }) + )?.body, + ).toEqual(Buffer.from('logo')); + }); + + it('throws on tampered bucket bytes', async () => { + const storage = await s3Storage(); + await storage.writeHtml(ARTIFACT_ID, Buffer.from('payload'), REF); + const stored = bucket.get(`${ARTIFACT_ID}/index.html`)!; + stored.body.writeUInt8( + stored.body.readUInt8(stored.body.length - 1) ^ 0x01, + stored.body.length - 1, + ); + await expect(storage.read(ARTIFACT_ID, 'index.html', REF)).rejects.toThrow( + EnvelopeError, + ); + }); + + it('returns null for a miss and rejects traversal before any network call', async () => { + const storage = await s3Storage(); + expect(await storage.read(ARTIFACT_ID, 'missing.js', REF)).toBeNull(); + await expect( + storage.writeFile('id', '../x', Buffer.from('x'), REF), + ).rejects.toThrow(); + expect(await storage.read('id', '../x', REF)).toBeNull(); + }); + + it('readWithInfo surfaces the authenticated content type (VER-03)', async () => { + const storage = await s3Storage(); + await storage.writeFile(ARTIFACT_ID, 'app.css', Buffer.from('body{}'), REF); + const out = await storage.readWithInfo(ARTIFACT_ID, 'app.css', REF); + expect(out?.plain.toString()).toBe('body{}'); + expect(out?.contentType).toBe('text/css; charset=utf-8'); + expect( + await storage.readWithInfo(ARTIFACT_ID, 'missing.css', REF), + ).toBeNull(); + }); + + it('deleteFiles issues key-level deletes, skipping traversal (VER-03)', async () => { + const storage = await s3Storage(); + const deleted: string[] = []; + s3.on(DeleteObjectCommand).callsFake((input: { Key: string }) => { + deleted.push(input.Key); + bucket.delete(input.Key); + return {}; + }); + await storage.writeFile( + ARTIFACT_ID, + 'assets/old.js', + Buffer.from('y'), + REF, + ); + await storage.deleteFiles(ARTIFACT_ID, [ + 'assets/old.js', + '../escape', + '/abs', + '', + ]); + expect(deleted).toEqual([`${ARTIFACT_ID}/assets/old.js`]); + expect(await storage.read(ARTIFACT_ID, 'assets/old.js', REF)).toBeNull(); + }); + + it('writeBundle with versionId prefixes keys and seals with ctx.version (VER-03)', async () => { + const storage = await s3Storage(); + const VID = '0b3adf87-2c1a-4e9e-9f30-1c6a3a1f0000'; + const AdmZip = (await import('adm-zip')).default; + const zip = new AdmZip(); + zip.addFile('index.html', Buffer.from('v')); + zip.addFile('v/planted.js', Buffer.from('reserved — skipped')); + await storage.writeBundle(ARTIFACT_ID, zip.toBuffer(), REF, { + versionId: VID, + }); + expect([...bucket.keys()]).toEqual([`${ARTIFACT_ID}/v/${VID}/index.html`]); + const out = await storage.read(ARTIFACT_ID, `v/${VID}/index.html`, { + ...REF, + version: VID, + }); + expect(out?.toString()).toBe('v'); + }); +}); diff --git a/src/artifacts/index.ts b/src/artifacts/index.ts new file mode 100644 index 0000000..1a29fd0 --- /dev/null +++ b/src/artifacts/index.ts @@ -0,0 +1,15 @@ +export * from './artifact-storage.js'; +export * from './crypto/index.js'; +export * from './env-config.js'; +export * from './object-store.js'; +export { + ARTIFACT_STORAGE_CLIENT_NAME, + DEFAULT_OBJECT_NAMESPACES, + OBJECT_STORE_PREFIX, + OBJECT_STORAGE_CLIENT_NAME, + createArtifactStorageClient, + createArtifactStorageDriver, + createObjectStorageClient, + createObjectStorageDriver, + resolveObjectNamespaces, +} from './storage-driver.js'; diff --git a/src/artifacts/nest/index.ts b/src/artifacts/nest/index.ts new file mode 100644 index 0000000..05ee576 --- /dev/null +++ b/src/artifacts/nest/index.ts @@ -0,0 +1,194 @@ +import { + Inject, + Module, + type DynamicModule, + type FactoryProvider, + type ModuleMetadata, + type OnApplicationShutdown, + type Provider, +} from '@nestjs/common'; +import type { StorageClient } from '../../storage.client.js'; +import { StorageModule } from '../../storage.module.js'; +import { getStorageToken } from '../../storage.tokens.js'; + +import { + createArtifactStorageWithClient, + type ArtifactStorage, +} from '../artifact-storage.js'; +import type { StorageCrypto } from '../crypto/index.js'; +import { + createObjectStoreWithClient, + type ObjectStore, +} from '../object-store.js'; +import { + ARTIFACT_STORAGE_CLIENT_NAME, + OBJECT_STORAGE_CLIENT_NAME, + createArtifactStorageDriver, + createObjectStorageDriver, + type ArtifactStorageConfig, +} from '../storage-driver.js'; + +export const ARTIFACT_STORAGE = Symbol.for( + '@nestm/storage/artifacts/artifact-storage', +); +export const OBJECT_STORE = Symbol.for('@nestm/storage/artifacts/object-store'); +export const ARTIFACT_STORAGE_MODULE_OPTIONS = Symbol.for( + '@nestm/storage/artifacts/module-options', +); + +export interface ArtifactStorageModuleOptions { + config: ArtifactStorageConfig; + crypto: StorageCrypto; + /** Makes the product adapters and @nestm/storage clients globally injectable. */ + isGlobal?: boolean; +} + +interface ResolvedArtifactStorageModuleOptions { + config: ArtifactStorageConfig; + crypto: StorageCrypto; +} + +export interface ArtifactStorageModuleAsyncOptions extends Pick< + ModuleMetadata, + 'imports' +> { + inject?: FactoryProvider['inject']; + isGlobal?: boolean; + useFactory: ( + ...dependencies: never[] + ) => + | ResolvedArtifactStorageModuleOptions + | Promise; +} + +export function InjectArtifactStorage(): ParameterDecorator & + PropertyDecorator { + return Inject(ARTIFACT_STORAGE); +} + +export function InjectObjectStore(): ParameterDecorator & PropertyDecorator { + return Inject(OBJECT_STORE); +} + +@Module({}) +class ArtifactStorageOptionsHostModule {} + +function syncOptionsHost( + options: ResolvedArtifactStorageModuleOptions, +): DynamicModule { + return { + module: ArtifactStorageOptionsHostModule, + providers: [ + { provide: ARTIFACT_STORAGE_MODULE_OPTIONS, useValue: options }, + ], + exports: [ARTIFACT_STORAGE_MODULE_OPTIONS], + }; +} + +function asyncOptionsHost( + options: ArtifactStorageModuleAsyncOptions, +): DynamicModule { + return { + module: ArtifactStorageOptionsHostModule, + imports: options.imports ?? [], + providers: [ + { + provide: ARTIFACT_STORAGE_MODULE_OPTIONS, + inject: options.inject ?? [], + useFactory: options.useFactory, + }, + ], + exports: [ARTIFACT_STORAGE_MODULE_OPTIONS], + }; +} + +function domainProviders(): Provider[] { + return [ + { + provide: ARTIFACT_STORAGE, + inject: [ + ARTIFACT_STORAGE_MODULE_OPTIONS, + getStorageToken(ARTIFACT_STORAGE_CLIENT_NAME), + ], + useFactory: ( + options: ResolvedArtifactStorageModuleOptions, + client: StorageClient, + ): ArtifactStorage => + createArtifactStorageWithClient(options.config, options.crypto, client), + }, + { + provide: OBJECT_STORE, + inject: [ + ARTIFACT_STORAGE_MODULE_OPTIONS, + getStorageToken(OBJECT_STORAGE_CLIENT_NAME), + ], + useFactory: ( + options: ResolvedArtifactStorageModuleOptions, + client: StorageClient, + ): ObjectStore => + createObjectStoreWithClient(options.config, options.crypto, client), + }, + ArtifactStorageCryptoShutdown, + ]; +} + +class ArtifactStorageCryptoShutdown implements OnApplicationShutdown { + constructor( + @Inject(ARTIFACT_STORAGE_MODULE_OPTIONS) + private readonly options: ResolvedArtifactStorageModuleOptions, + ) {} + + onApplicationShutdown(): void { + this.options.crypto.keyProvider.clear(); + } +} + +function composeModule( + optionsHost: DynamicModule, + isGlobal: boolean, +): DynamicModule { + const rawStorageModule = StorageModule.forRootAsync({ + imports: [optionsHost], + default: ARTIFACT_STORAGE_CLIENT_NAME, + isGlobal, + stores: [ + { + name: ARTIFACT_STORAGE_CLIENT_NAME, + inject: [ARTIFACT_STORAGE_MODULE_OPTIONS], + useFactory: (options: ResolvedArtifactStorageModuleOptions) => + createArtifactStorageDriver(options.config), + }, + { + name: OBJECT_STORAGE_CLIENT_NAME, + inject: [ARTIFACT_STORAGE_MODULE_OPTIONS], + useFactory: (options: ResolvedArtifactStorageModuleOptions) => + createObjectStorageDriver(options.config), + }, + ], + }); + + return { + module: ArtifactStorageModule, + global: isGlobal, + imports: [optionsHost, rawStorageModule], + providers: domainProviders(), + exports: [ARTIFACT_STORAGE, OBJECT_STORE, StorageModule], + }; +} + +@Module({}) +export class ArtifactStorageModule { + static forRoot(options: ArtifactStorageModuleOptions): DynamicModule { + const { config, crypto } = options; + return composeModule( + syncOptionsHost({ config, crypto }), + options.isGlobal === true, + ); + } + + static forRootAsync( + options: ArtifactStorageModuleAsyncOptions, + ): DynamicModule { + return composeModule(asyncOptionsHost(options), options.isGlobal === true); + } +} diff --git a/src/artifacts/nest/storage.module.spec.ts b/src/artifacts/nest/storage.module.spec.ts new file mode 100644 index 0000000..1f51e6a --- /dev/null +++ b/src/artifacts/nest/storage.module.spec.ts @@ -0,0 +1,121 @@ +import 'reflect-metadata'; + +import { Module } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import type { StorageClient } from '../../storage.client.js'; +import { getStorageToken } from '../../storage.tokens.js'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import type { ArtifactStorage } from '../artifact-storage.js'; +import { LocalKeyProvider } from '../crypto/index.js'; +import type { ObjectStore } from '../object-store.js'; +import { + ARTIFACT_STORAGE_CLIENT_NAME, + OBJECT_STORAGE_CLIENT_NAME, +} from '../storage-driver.js'; +import { + ARTIFACT_STORAGE, + OBJECT_STORE, + ArtifactStorageModule, + type ArtifactStorageModuleOptions, +} from './index.js'; + +const OPTIONS = Symbol('storage-test-options'); + +@Module({}) +class StorageTestOptionsModule {} + +describe('ArtifactStorageModule', () => { + let root = ''; + + afterEach(() => { + if (root) rmSync(root, { recursive: true, force: true }); + }); + + function options(): ArtifactStorageModuleOptions { + root = mkdtempSync(join(tmpdir(), 'storage-nest-module-')); + return { + config: { provider: 'fs', config: { root } }, + crypto: { + keyProvider: new LocalKeyProvider('nest', Buffer.alloc(32, 7)), + }, + }; + } + + it('composes product adapters over named @nestm StorageClient providers', async () => { + const testingModule = await Test.createTestingModule({ + imports: [ArtifactStorageModule.forRoot(options())], + }).compile(); + + const artifactStorage = + testingModule.get(ARTIFACT_STORAGE); + const objectStore = testingModule.get(OBJECT_STORE); + const artifactClient = testingModule.get( + getStorageToken(ARTIFACT_STORAGE_CLIENT_NAME), + ); + const objectClient = testingModule.get( + getStorageToken(OBJECT_STORAGE_CLIENT_NAME), + ); + + await artifactStorage.writeHtml( + 'artifact', + Buffer.from('nest'), + { + scope: 'org:nest', + }, + ); + await objectStore.putObject( + 'org-logos/nest', + Buffer.from('logo'), + 'image/png', + { + scope: 'org:nest', + }, + ); + + expect( + Buffer.from(await artifactClient.downloadBytes('artifact/index.html')) + .subarray(0, 4) + .toString(), + ).toBe('CAE1'); + expect( + Buffer.from(await objectClient.downloadBytes('org-logos/nest')) + .subarray(0, 4) + .toString(), + ).toBe('CAE1'); + expect( + ( + await artifactStorage.read('artifact', 'index.html', { + scope: 'org:nest', + }) + )?.toString(), + ).toBe('nest'); + + await testingModule.close(); + }); + + it('supports async composition without importing the application config package', async () => { + const resolved = options(); + const optionsModule = { + module: StorageTestOptionsModule, + providers: [{ provide: OPTIONS, useValue: resolved }], + exports: [OPTIONS], + }; + const testingModule = await Test.createTestingModule({ + imports: [ + ArtifactStorageModule.forRootAsync({ + imports: [optionsModule], + inject: [OPTIONS], + useFactory: (value: ArtifactStorageModuleOptions) => value, + }), + ], + }).compile(); + + expect(testingModule.get(ARTIFACT_STORAGE)).toBeDefined(); + expect(testingModule.get(OBJECT_STORE)).toBeDefined(); + await testingModule.close(); + }); +}); diff --git a/src/artifacts/object-store.spec.ts b/src/artifacts/object-store.spec.ts new file mode 100644 index 0000000..1523b57 --- /dev/null +++ b/src/artifacts/object-store.spec.ts @@ -0,0 +1,344 @@ +import { + GetObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { mockClient } from 'aws-sdk-client-mock'; +import { randomBytes } from 'node:crypto'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + utimesSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + EnvelopeError, + LocalKeyProvider, + type StorageCrypto, +} from './crypto/index.js'; +import { + createObjectStore, + createObjectStoreWithClient, +} from './object-store.js'; + +const crypto: StorageCrypto = { + keyProvider: new LocalKeyProvider('spec', randomBytes(32)), +}; +const REF = { scope: 'upload:u1' }; + +let root: string; +afterEach(() => rmSync(root, { recursive: true, force: true })); + +async function fsStore(overrides?: Partial) { + root = mkdtempSync(join(tmpdir(), 'object-store-spec-')); + return await createObjectStore( + { provider: 'fs', config: { root } }, + { ...crypto, ...overrides }, + ); +} + +describe('ObjectStore round-trip (fs backend)', () => { + it('stores an envelope (no plaintext, no sidecar) and returns the header content type', async () => { + const store = await fsStore(); + const body = Buffer.from('staged'); + await store.putObject('_staging/u1', body, 'text/html', REF); + const onDisk = readFileSync(join(root, '_objects', '_staging', 'u1')); + expect(onDisk.subarray(0, 4).toString()).toBe('CAE1'); + expect(onDisk.includes(body)).toBe(false); + const out = await store.getObject('_staging/u1', REF); + expect(out?.body).toEqual(body); + expect(out?.contentType).toBe('text/html'); + }); + + it("asserts the reader's scope and fails closed on tamper", async () => { + const store = await fsStore(); + await store.putObject('_staging/u1', Buffer.from('x'), 'text/html', REF); + await expect( + store.getObject('_staging/u1', { scope: 'upload:other' }), + ).rejects.toThrow(EnvelopeError); + const p = join(root, '_objects', '_staging', 'u1'); + const bytes = readFileSync(p); + bytes.writeUInt8( + bytes.readUInt8(bytes.length - 1) ^ 0x01, + bytes.length - 1, + ); + writeFileSync(p, bytes); + await expect(store.getObject('_staging/u1', REF)).rejects.toThrow( + EnvelopeError, + ); + }); + + it('reads a pre-envelope object with its .ct sidecar only under the migration flag', async () => { + const store = await fsStore({ allowLegacyPlaintext: true }); + const p = join(root, '_objects', 'org-logos', 'o1'); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, Buffer.from('png-bytes')); + writeFileSync(`${p}.ct`, 'image/png'); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + try { + const out = await store.getObject('org-logos/o1', { scope: 'org:o1' }); + expect(out?.body.toString()).toBe('png-bytes'); + expect(out?.contentType).toBe('image/png'); + expect(warn).toHaveBeenCalledOnce(); + } finally { + warn.mockRestore(); + } + const strict = await createObjectStore( + { provider: 'fs', config: { root } }, + crypto, + ); + await expect( + strict.getObject('org-logos/o1', { scope: 'org:o1' }), + ).rejects.toThrow(EnvelopeError); + }); + + it('re-putting over a legacy object removes the stale sidecar', async () => { + const store = await fsStore(); + const p = join(root, '_objects', 'org-logos', 'o1'); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, Buffer.from('old')); + writeFileSync(`${p}.ct`, 'image/png'); + await store.putObject('org-logos/o1', Buffer.from('new'), 'image/webp', { + scope: 'org:o1', + }); + const out = await store.getObject('org-logos/o1', { scope: 'org:o1' }); + expect(out?.contentType).toBe('image/webp'); + expect(await store.listObjects('org-logos/o1')).toEqual(['org-logos/o1']); // no phantom .ct key + }); + + it('reserves the legacy .ct sidecar suffix as a non-addressable key', async () => { + const store = await fsStore(); + await expect( + store.putObject('org-logos/logo.ct', Buffer.from('x'), 'text/plain', REF), + ).rejects.toThrow(/invalid object key/); + await expect(store.getObject('org-logos/logo.ct', REF)).rejects.toThrow( + /invalid object key/, + ); + await expect(store.deleteObject('org-logos/logo.ct')).rejects.toThrow( + /invalid object key/, + ); + await expect( + store.putObject( + 'org-logos/logo.CT /child', + Buffer.from('x'), + 'text/plain', + REF, + ), + ).rejects.toThrow(/invalid object key/); + }); +}); + +describe('ObjectStore.sweepObjects (fs backend)', () => { + it('deletes only objects under the prefix older than the cutoff', async () => { + const store = await fsStore(); + await store.putObject('_staging/old', Buffer.from('old'), 'text/html', REF); + await store.putObject( + '_staging/fresh', + Buffer.from('fresh'), + 'text/html', + REF, + ); + await store.putObject( + 'org-logos/old', + Buffer.from('keep'), + 'text/html', + REF, + ); + + // Backdate the "old" objects beyond the cutoff. The driver reports `lastModified` from the + // object's own metadata rather than the body file's mtime, so the sidecar is what has to move + // — touching the body alone would leave the sweep looking at the write time. + const past = new Date(Date.now() - 60 * 60 * 1000); + for (const rel of ['_objects/_staging/old', '_objects/org-logos/old']) { + const body = join(root, rel); + utimesSync(body, past, past); + const sidecar = `${body}.meta.json`; + writeFileSync( + sidecar, + JSON.stringify({ + ...JSON.parse(readFileSync(sidecar, 'utf8')), + lastModified: past.getTime(), + }), + ); + } + + const deleted = await store.sweepObjects('_staging/', 30 * 60 * 1000); + expect(deleted).toBe(1); + expect(await store.getObject('_staging/old', REF)).toBeNull(); + expect( + (await store.getObject('_staging/fresh', REF))?.body.toString(), + ).toBe('fresh'); + expect((await store.getObject('org-logos/old', REF))?.body.toString()).toBe( + 'keep', + ); + }); + + it('returns 0 when the prefix directory does not exist', async () => { + const store = await fsStore(); + expect(await store.sweepObjects('_staging/', 0)).toBe(0); + }); + + it('refuses traversal prefixes', async () => { + const store = await fsStore(); + await expect(store.sweepObjects('../escape/', 0)).rejects.toThrow( + /invalid/i, + ); + }); + + it.each([-1, Number.NaN, Number.POSITIVE_INFINITY])( + 'rejects an unsafe age: %s', + async (olderThanMs) => { + const store = await fsStore(); + await expect( + store.sweepObjects('_staging/', olderThanMs), + ).rejects.toThrow(/olderThanMs/); + }, + ); + + it('never deletes objects whose provider timestamp is missing or invalid', async () => { + const remove = vi.fn().mockResolvedValue(undefined); + const listAll = vi.fn(() => + (async function* () { + yield { key: '_staging/missing-date', size: 1 }; + yield { + key: '_staging/invalid-date', + size: 1, + lastModified: new Date(Number.NaN), + }; + })(), + ); + const store = createObjectStoreWithClient({ provider: 's3' }, crypto, { + delete: remove, + listAll, + } as never); + + await expect(store.sweepObjects('_staging/', 1)).resolves.toBe(0); + expect(remove).not.toHaveBeenCalled(); + }); +}); + +describe('ObjectStore.listObjects (fs backend)', () => { + it('lists only keys under the prefix, sorted', async () => { + const store = await fsStore(); + await store.putObject( + '_staging/u1.part-00001', + Buffer.from('b'), + 'text/html', + REF, + ); + await store.putObject( + '_staging/u1.part-00000', + Buffer.from('a'), + 'text/html', + REF, + ); + await store.putObject('_staging/u2', Buffer.from('z'), 'text/html', REF); + expect(await store.listObjects('_staging/u1.part-')).toEqual([ + '_staging/u1.part-00000', + '_staging/u1.part-00001', + ]); + }); + + it('returns [] for an empty prefix dir and excludes legacy .ct sidecars', async () => { + const store = await fsStore(); + expect(await store.listObjects('_staging/none-')).toEqual([]); + await store.putObject( + '_staging/u3.part-00000', + Buffer.from('a'), + 'text/html', + REF, + ); + // A leftover pre-envelope sidecar must not surface as a phantom key. + writeFileSync( + join(root, '_objects', '_staging', 'u3.part-00000.ct'), + 'text/html', + ); + expect(await store.listObjects('_staging/u3.part-')).toEqual([ + '_staging/u3.part-00000', + ]); + }); + + it('refuses traversal prefixes', async () => { + const store = await fsStore(); + await expect(store.listObjects('../escape')).rejects.toThrow(/invalid/i); + }); +}); + +describe('ObjectStore (s3 backend)', () => { + const s3 = mockClient(S3Client); + const bucket = new Map(); + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'object-store-s3-spec-')); // unused; satisfies afterEach + bucket.clear(); + s3.reset(); + s3.on(PutObjectCommand).callsFake( + (input: { Key: string; Body: Buffer; ContentType?: string }) => { + bucket.set(input.Key, { + body: Buffer.from(input.Body), + ...(input.ContentType === undefined + ? {} + : { contentType: input.ContentType }), + }); + return {}; + }, + ); + s3.on(GetObjectCommand).callsFake((input: { Key: string }) => { + const hit = bucket.get(input.Key); + if (!hit) { + const err = new Error('NoSuchKey'); + err.name = 'NoSuchKey'; + throw err; + } + return { + Body: { + transformToByteArray: () => Promise.resolve(new Uint8Array(hit.body)), + transformToWebStream: () => + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(hit.body)); + controller.close(); + }, + }), + }, + ContentLength: hit.body.byteLength, + ContentType: hit.contentType, + }; + }); + }); + + it('round-trips an envelope and keeps the real content type out of S3 metadata', async () => { + const store = await createObjectStore( + { + provider: 's3', + config: { + endpoint: 'http://127.0.0.1:1', + bucket: 'dummy', + accessKeyId: 'k', + secretAccessKey: 's', + region: 'us-east-1', + }, + }, + crypto, + ); + const body = Buffer.from('logo-bytes'); + await store.putObject('org-logos/o1', body, 'image/png', { + scope: 'org:o1', + }); + const stored = bucket.get('org-logos/o1')!; + expect(stored.body.subarray(0, 4).toString()).toBe('CAE1'); + expect(stored.contentType).toBe('application/octet-stream'); + const out = await store.getObject('org-logos/o1', { scope: 'org:o1' }); + expect(out?.body).toEqual(body); + expect(out?.contentType).toBe('image/png'); + expect( + await store.getObject('org-logos/missing', { scope: 'org:o1' }), + ).toBeNull(); + }); +}); diff --git a/src/artifacts/object-store.ts b/src/artifacts/object-store.ts new file mode 100644 index 0000000..e9efb8a --- /dev/null +++ b/src/artifacts/object-store.ts @@ -0,0 +1,307 @@ +import { + StorageErrorCode, + isStorageError, + type StorageClient, + type StorageObject, +} from '../core/index.js'; + +import { + warnLegacyRead, + type ReadRef, + type ScopeRef, +} from './artifact-storage.js'; +import { + open, + seal, + type EnvelopeContext, + type StorageCrypto, +} from './crypto/index.js'; +import { + createObjectStorageClient, + DEFAULT_OBJECT_NAMESPACES, + resolveObjectNamespaces, + type ArtifactStorageConfig, +} from './storage-driver.js'; + +/** + * Generic key→bytes object store (not artifact-shaped). Backs org-logo uploads and staged + * artifact uploads. Objects are CAE1 envelopes like artifact files; the content type lives in + * the authenticated envelope header on every provider. Filesystem `.ct` sidecars remain read-only + * compatibility data and are never emitted. + */ +export interface ObjectStore { + putObject( + key: string, + body: Buffer, + contentType: string, + ref: ScopeRef, + ): Promise; + /** `null` if absent; throws `EnvelopeError` on decrypt failure (never masked as a miss). */ + getObject( + key: string, + ref: ReadRef, + ): Promise<{ body: Buffer; contentType?: string } | null>; + deleteObject(key: string): Promise; + /** Delete every object under `prefix` last modified more than `olderThanMs` ago. Returns the count deleted. */ + sweepObjects(prefix: string, olderThanMs: number): Promise; + /** Keys beginning with `prefix` (store namespace), sorted lexicographically. */ + listObjects(prefix: string): Promise; +} + +export async function createObjectStore( + cfg: ArtifactStorageConfig, + crypto: StorageCrypto, +): Promise { + const client = await createObjectStorageClient(cfg); + return createObjectStoreWithClient(cfg, crypto, client); +} + +/** Framework-neutral domain adapter used by both direct consumers and the Nest composition entry. */ +export function createObjectStoreWithClient( + cfg: ArtifactStorageConfig, + crypto: StorageCrypto, + client: StorageClient, +): ObjectStore { + return new ClientObjectStore( + client, + crypto, + cfg.provider === 'fs', + cfg.objectNamespaces ?? DEFAULT_OBJECT_NAMESPACES, + ); +} + +/** + * Envelope context for a store object. `path` is the store-relative KEY on every provider — + * the fs `_objects/` nesting is a driver-level prefix and must never reach the AAD, or two + * providers would bind ciphertext to different addresses for the same logical object. + */ +function objectCtx(key: string, ref: ScopeRef | ReadRef): EnvelopeContext { + return { + scope: ref.scope as string, + artifactId: null, + version: ref.version ?? null, + path: key, + }; +} + +function aliasesLegacyFilesystemSidecar(segment: string): boolean { + let end = segment.length; + while ( + end > 0 && + (segment.charCodeAt(end - 1) === 0x2e || + segment.charCodeAt(end - 1) === 0x20) + ) { + end--; + } + return segment.slice(0, end).toLowerCase().endsWith('.ct'); +} + +function assertObjectKey( + key: string, + namespaces: ReadonlySet, + filesystemLayout: boolean, + allowEmpty = false, +): void { + const firstSegment = key.split(/[/\\]/, 1)[0] ?? ''; + const segments = key.split(/[/\\]/); + if (allowEmpty && segments.at(-1) === '') segments.pop(); + if ( + (!allowEmpty && key === '') || + key.startsWith('/') || + key.includes('\0') || + (filesystemLayout && segments.some(aliasesLegacyFilesystemSidecar)) || + segments.some( + (segment) => segment === '' || segment === '.' || segment === '..', + ) || + (key !== '' && !namespaces.has(firstSegment.toLowerCase())) + ) { + throw new Error(`invalid object key: ${key}`); + } +} + +async function collectRawObject(object: StorageObject): Promise { + const chunks: Buffer[] = []; + const reader = object.body.getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(Buffer.from(value)); + } + } finally { + reader.releaseLock(); + } + return Buffer.concat(chunks); +} + +interface RawObject { + raw: Buffer; + contentType?: string; +} + +function isObjectKeyAllowed( + key: string, + namespaces: ReadonlySet, + filesystemLayout: boolean, +): boolean { + try { + assertObjectKey(key, namespaces, filesystemLayout); + return true; + } catch { + return false; + } +} + +class ClientObjectStore implements ObjectStore { + private readonly namespaces: ReadonlySet; + + constructor( + private readonly client: StorageClient, + private readonly crypto: StorageCrypto, + private readonly filesystemLayout: boolean, + namespaces: readonly string[], + ) { + this.namespaces = resolveObjectNamespaces(namespaces); + } + + async putObject( + key: string, + body: Buffer, + contentType: string, + ref: ScopeRef, + ): Promise { + assertObjectKey(key, this.namespaces, this.filesystemLayout); + const sealed = await seal( + body, + objectCtx(key, ref), + this.crypto.keyProvider, + contentType, + ); + await this.client.upload(key, sealed, { + // Real content type rides the authenticated envelope; raw storage remains opaque. + contentType: 'application/octet-stream', + }); + if (this.filesystemLayout) await this.client.delete(`${key}.ct`); + } + + async getObject( + key: string, + ref: ReadRef, + ): Promise<{ body: Buffer; contentType?: string } | null> { + assertObjectKey(key, this.namespaces, this.filesystemLayout); + const stored = await this.downloadRaw(this.client, key); + if (stored === null) return null; + const out = await open( + stored.raw, + objectCtx(key, ref), + this.crypto.keyProvider, + this.crypto.allowLegacyPlaintext === undefined + ? {} + : { allowLegacyPlaintext: this.crypto.allowLegacyPlaintext }, + ); + let contentType = out.contentType; + if (out.legacy) { + warnLegacyRead(this.filesystemLayout ? 'fs-object' : 'object', key); + if (this.filesystemLayout) { + contentType = await this.readLegacyFilesystemContentType(key); + } else { + contentType = stored.contentType; + } + } + return { + body: out.plain, + ...(contentType === undefined ? {} : { contentType }), + }; + } + + async deleteObject(key: string): Promise { + assertObjectKey(key, this.namespaces, this.filesystemLayout); + await this.client.delete(key); + if (this.filesystemLayout) await this.client.delete(`${key}.ct`); + } + + async sweepObjects(prefix: string, olderThanMs: number): Promise { + assertObjectKey(prefix, this.namespaces, this.filesystemLayout, true); + if (!Number.isFinite(olderThanMs) || olderThanMs < 0) { + throw new RangeError('olderThanMs must be a finite non-negative number'); + } + const cutoff = Date.now() - olderThanMs; + let deleted = 0; + for await (const object of this.client.listAll({ prefix })) { + if ( + !isObjectKeyAllowed(object.key, this.namespaces, this.filesystemLayout) + ) { + continue; + } + if (this.filesystemLayout && object.key.toLowerCase().endsWith('.ct')) + continue; + if (!isExpired(object.lastModified, cutoff)) continue; + await this.client.delete(object.key); + if (this.filesystemLayout) await this.client.delete(`${object.key}.ct`); + deleted++; + } + return deleted; + } + + async listObjects(prefix: string): Promise { + assertObjectKey(prefix, this.namespaces, this.filesystemLayout, true); + const keys = new Set(); + for await (const object of this.client.listAll({ prefix })) { + if ( + !isObjectKeyAllowed(object.key, this.namespaces, this.filesystemLayout) + ) { + continue; + } + if (this.filesystemLayout && object.key.toLowerCase().endsWith('.ct')) + continue; + keys.add(object.key); + } + return [...keys].sort(); + } + + private async downloadRaw( + client: StorageClient, + key: string, + ): Promise { + try { + const object = await client.downloadStream(key); + const raw = await collectRawObject(object); + return { + raw, + ...(object.contentType === undefined + ? {} + : { contentType: object.contentType }), + }; + } catch (error) { + if (isStorageError(error) && error.code === StorageErrorCode.NOT_FOUND) { + return null; + } + throw error; + } + } + + private async readLegacyFilesystemContentType( + key: string, + ): Promise { + try { + return await this.client.downloadText(`${key}.ct`, { + maxBytes: 64 * 1024, + }); + } catch (error) { + if (isStorageError(error) && error.code === StorageErrorCode.NOT_FOUND) { + return undefined; + } + throw error; + } + } +} + +function isExpired(lastModified: Date | undefined, cutoff: number): boolean { + const modifiedAt = lastModified?.getTime(); + // A provider that cannot supply a trustworthy timestamp cannot prove an object is expired. + return ( + modifiedAt !== undefined && + Number.isFinite(modifiedAt) && + modifiedAt < cutoff + ); +} diff --git a/src/artifacts/storage-driver.ts b/src/artifacts/storage-driver.ts new file mode 100644 index 0000000..e0529e3 --- /dev/null +++ b/src/artifacts/storage-driver.ts @@ -0,0 +1,158 @@ +import { StorageClient, type StorageDriver } from '../core/index.js'; +import { + createProviderStorageDriver, + isStorageProvider, + listStorageProviders, + type StorageProviderConfig, + type StorageProviderName, +} from '../files-sdk/provider/index.js'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +export const ARTIFACT_STORAGE_CLIENT_NAME = 'artifacts'; +export const OBJECT_STORAGE_CLIENT_NAME = 'objects'; +export const DEFAULT_OBJECT_NAMESPACES = ['_staging', 'org-logos'] as const; + +/** + * Where the generic object store lives relative to the artifact store on a filesystem. Artifact + * keys are `/`, so an unprefixed object key like `org-logos/x.png` would read + * back as an artifact directory named `org-logos` — listing, sweeping, and removal would all + * cross the two. Filesystems therefore retain the historical `_objects` directory. Object-store + * providers retain their historical unprefixed keys; the configured top-level namespace + * allowlist keeps those keys disjoint from artifact ids without a rolling-deploy layout change. + */ +export const OBJECT_STORE_PREFIX = '_objects'; + +/** + * Which store backs artifacts, and how to reach it. `provider` is any slug `@nestm/storage` + * can build — `fs`, `s3`, `gcs`, `azure`, `r2`, `minio`, `supabase`, and ~40 more — so a + * deployment picks its store from the environment instead of the platform shipping a driver + * per backend. Only the selected provider's SDK is ever imported. + */ +export interface ArtifactStorageConfig { + provider: StorageProviderName; + /** Key prefix so artifacts can share a bucket (or a directory tree) with other apps. */ + prefix?: string; + /** + * Flat provider settings: `bucket`/`region`/`endpoint` for an object store, `root` for the + * filesystem, `accountName`/`container` for Azure. Each provider reads what it needs. + * Credentials may be omitted wherever the provider's SDK resolves its own chain — on AWS that + * is the ECS task role, so S3 needs no long-lived keys in env. + */ + config?: StorageProviderConfig; + /** + * Allowed top-level ObjectStore namespaces. Artifact ids matching one of these values are + * rejected, keeping the two public facades disjoint on providers where they share a key root. + */ + objectNamespaces?: readonly string[]; +} + +/** Default filesystem root, shared by the api and the sandbox so zero-config dev lines up. */ +export function defaultFsRoot(): string { + return join(tmpdir(), 'concepta-artifacts-artifacts'); +} + +function trimSlashes(value: string): string { + return value.replace(/^\/+|\/+$/g, ''); +} + +/** Resolve and validate the case-insensitive top-level ObjectStore namespace allowlist. */ +export function resolveObjectNamespaces( + namespaces: readonly string[] = DEFAULT_OBJECT_NAMESPACES, +): ReadonlySet { + const resolved = new Set(); + for (const namespace of namespaces) { + const normalized = namespace.toLowerCase(); + if ( + namespace === '' || + namespace === '.' || + namespace === '..' || + namespace.includes('/') || + namespace.includes('\\') || + namespace.includes('\0') || + Buffer.byteLength(namespace, 'utf8') > 255 || + normalized === OBJECT_STORE_PREFIX || + resolved.has(normalized) + ) { + throw new Error( + `invalid or duplicate object namespace: ${JSON.stringify(namespace)}`, + ); + } + resolved.add(normalized); + } + if (resolved.size === 0) { + throw new Error('at least one object namespace is required'); + } + return resolved; +} + +function joinPrefix(...parts: Array): string | undefined { + const joined = parts + .flatMap((part) => (part === undefined ? [] : [trimSlashes(part)])) + .filter((part) => part.length > 0) + .join('/'); + return joined.length === 0 ? undefined : joined; +} + +/** Fail fast on an unknown slug with the full list, rather than at the first upload. */ +export function assertStorageProvider(provider: string): StorageProviderName { + if (!isStorageProvider(provider)) { + const known = listStorageProviders() + .map((candidate) => candidate.slug) + .join(', '); + throw new Error( + `Unknown storage provider ${JSON.stringify(provider)}. Known providers: ${known}`, + ); + } + return provider; +} + +function providerConfig(cfg: ArtifactStorageConfig): StorageProviderConfig { + if (cfg.provider !== 'fs') return cfg.config ?? {}; + // The filesystem adapter needs a root; every other provider names its store some other way. + return { root: defaultFsRoot(), ...cfg.config }; +} + +function driver( + cfg: ArtifactStorageConfig, + prefix: string | undefined, +): Promise { + return createProviderStorageDriver({ + provider: cfg.provider, + config: providerConfig(cfg), + ...(prefix === undefined ? {} : { prefix }), + }); +} + +export function createArtifactStorageDriver( + cfg: ArtifactStorageConfig, +): Promise { + return driver(cfg, joinPrefix(cfg.prefix)); +} + +export function createObjectStorageDriver( + cfg: ArtifactStorageConfig, +): Promise { + return driver( + cfg, + joinPrefix(cfg.prefix, cfg.provider === 'fs' ? OBJECT_STORE_PREFIX : ''), + ); +} + +export async function createArtifactStorageClient( + cfg: ArtifactStorageConfig, +): Promise { + return new StorageClient( + ARTIFACT_STORAGE_CLIENT_NAME, + await createArtifactStorageDriver(cfg), + ); +} + +export async function createObjectStorageClient( + cfg: ArtifactStorageConfig, +): Promise { + return new StorageClient( + OBJECT_STORAGE_CLIENT_NAME, + await createObjectStorageDriver(cfg), + ); +} diff --git a/test/fixtures/cae1-golden.ts b/test/fixtures/cae1-golden.ts new file mode 100644 index 0000000..65222ea --- /dev/null +++ b/test/fixtures/cae1-golden.ts @@ -0,0 +1,39 @@ +/** Frozen CAE1 v1 vector produced by the pre-consolidation Concepta storage package. */ +export const GOLDEN_KEK = Buffer.alloc(32, 42); +export const GOLDEN_CTX = { + scope: 'org:golden-org', + artifactId: 'golden-artifact', + version: null, + path: 'index.html', +} as const; +export const GOLDEN_PLAINTEXT = 'golden vector v1 — do not regenerate'; +export const GOLDEN_ENVELOPE = Buffer.from( + '43414531000001207b2276223a312c22616c67223a224132353647434d222c226b6964223a226c6f63616c3a67' + + '6f6c64656e222c2277646b223a22706a576e3938555375666c7a6d35424a4d4d4a5152762b6e717550536f2f2f' + + '30497530315866494d42453254737733426a614c714c56777764724f6b4f7247575567747459645377306a625a' + + '4d644e4d222c226976223a225651754e6c415574465a4c6e6e753159222c22637478223a7b2273636f7065223a' + + '226f72673a676f6c64656e2d6f7267222c2261727469666163744964223a22676f6c64656e2d6172746966616374' + + '222c2276657273696f6e223a6e756c6c2c2270617468223a22696e6465782e68746d6c227d2c226374223a2274' + + '6578742f68746d6c3b20636861727365743d7574662d38227d045a814daef4d754b7025df042f920d86e5e188e' + + '8968f4347ef418c257debbff5678687d3cca0f1ccb727d38ef634af285f9e3d0c3ed', + 'hex', +); + +/** Frozen pre-consolidation ObjectStore envelope; notably pins artifactId:null AAD. */ +export const GOLDEN_OBJECT_CTX = { + scope: 'org:compat-org', + artifactId: null, + version: null, + path: 'org-logos/compat-org', +} as const; +export const GOLDEN_OBJECT_PLAINTEXT = 'legacy-logo'; +export const GOLDEN_OBJECT_ENVELOPE = Buffer.from( + '434145310000010e7b2276223a312c22616c67223a224132353647434d222c226b6964223a226c6f63616c3a636f' + + '6d706174222c2277646b223a22676872454d392b4e7651675764583272754f6635456968762f354452553877784f4f' + + '5a533344544a69386e30356478456e52335a66784f5756574f35794a69715532356f414a736d5465565a5878772f22' + + '2c226976223a22315731684d695639417a694261613052222c22637478223a7b2273636f7065223a226f72673a636f' + + '6d7061742d6f7267222c2261727469666163744964223a6e756c6c2c2276657273696f6e223a6e756c6c2c22706174' + + '68223a226f72672d6c6f676f732f636f6d7061742d6f7267227d2c226374223a22696d6167652f706e67227dd19b7e' + + '61e8e604c610375a322c282f88b8b28ac6cdc677c9dbd718', + 'hex', +);