diff --git a/.changeset/add-agent-workspaces.md b/.changeset/add-agent-workspaces.md new file mode 100644 index 0000000..974154a --- /dev/null +++ b/.changeset/add-agent-workspaces.md @@ -0,0 +1,17 @@ +--- +'@nestm/storage': minor +--- + +Add a backend-neutral `StorageWorkspace` capability and optional AI SDK 7 tool +adapter. Workspaces expose only canonical mount-relative paths, enforce +permissions and resource limits, hide provider coordinates and cursors, and use +atomic create/ETag mutation preconditions. S3 now advertises and implements the +conditional mutation primitives used by writable workspaces. + +Harden local filesystem workspace reads and conditional mutations against +symlink aliases. Moves retain their create-only destination whenever source +deletion cannot be confirmed, avoiding data loss after provider or +post-operation hook ambiguity. + +Fix cross-store sync so `destinationPrefix` is applied to uploaded keys as well +as pruning, keeping every mutation inside the selected destination scope. diff --git a/.prettierignore b/.prettierignore index f278820..6474773 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,5 +1,6 @@ dist/ coverage/ node_modules/ +references/ pnpm-lock.yaml .changeset/pre.json diff --git a/README.md b/README.md index 2856f48..28f280e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # @nestm/storage Framework-neutral storage clients with NestJS 12 integration, named stores, -explicit streaming I/O, cross-store workflows, and an optional guarded HTTP -gateway. +explicit streaming I/O, capability-scoped agent workspaces, cross-store +workflows, and an optional guarded HTTP gateway. The package uses [`files-sdk`](https://github.com/haydenbleasel/files-sdk) as its provider engine, but owns the API injected into Nest applications. Provider @@ -88,6 +88,186 @@ storage errors and operation types, and `StorageUploadControl`. It has no NestJS runtime or declaration imports. Provider adapters remain available through `@nestm/storage/files-sdk`. +## Mounted agent workspaces + +`@nestm/storage/workspace` turns a `StorageClient` into a narrow capability for +one logical directory. The mount is virtual: the same API works over S3, a +filesystem driver, or another storage backend without exposing the provider, +bucket, filesystem root, raw cursor, or internal prefix to its caller. + +```mermaid +flowchart LR + A["Trusted application context"] -->|"store + opaque prefix + policy"| W["StorageWorkspace"] + W --> C["StorageClient"] + C --> D["S3 / filesystem / other driver"] + W --> T["AI SDK workspace tools"] + T --> G["ToolLoopAgent"] +``` + +Only trusted application code chooses the mount prefix. Every path accepted by +the workspace is a canonical, relative POSIX path. Absolute paths, backslashes, +control characters, repeated separators, and `.` or `..` segments are rejected +rather than normalized. Keys and provider cursors returned by a driver are also +checked before they are converted back to logical paths. + +```ts +import { mountStorageWorkspace } from '@nestm/storage/workspace'; + +const workspace = mountStorageWorkspace(agentFiles, { + // Use an opaque server-derived run id, never a value selected by the model. + prefix: `workspaces/${runId}`, + permissions: [ + 'list', + 'read', + 'search', + 'create', + 'replace', + 'copy', + 'move', + 'delete', + ], + limits: { + maxReadBytes: 1024 * 1024, + maxWriteBytes: 1024 * 1024, + maxPageSize: 100, + maxSearchResults: 100, + maxSearchScan: 1000, + }, +}); + +const created = await workspace.writeFile( + 'src/main.ts', + 'export const ready = true;\n', + { mode: 'create', contentType: 'text/typescript' }, +); + +if (created.etag === undefined) { + throw new Error('This backend cannot safely replace the object.'); +} + +await workspace.writeFile('src/main.ts', 'export const ready = false;\n', { + mode: 'replace', + etag: created.etag, + contentType: 'text/typescript', +}); +``` + +Create, replace, and delete are conditional operations. A driver that cannot +enforce the requested not-exists or ETag precondition fails with +`NOT_SUPPORTED`; the workspace never substitutes an `exists()`/`head()` check +followed by an unconditional mutation. Reads enforce their byte ceiling while +consuming the stream, and list/search results are bounded. Search supports +exact, substring, and workspace-coordinate glob matching, but no caller-supplied +regular expressions. + +Move is implemented as create-only copy followed by ETag-conditional source +delete. If source deletion cannot be confirmed, the destination is retained and +the call returns `CONFLICT`; inspect both logical paths before retrying. This +preserves at least one copy across provider timeouts and post-operation hook +failures, but does not pretend a multi-object move is transactionally atomic. + +A child mount may further restrict a directory, permissions, or limits, but it +cannot widen any of them: + +```ts +const readOnlySource = workspace.mount('src', { + permissions: ['list', 'read', 'search'], + limits: { maxReadBytes: 256 * 1024 }, +}); +``` + +### AI SDK and NestJS composition + +Install AI SDK 7 and Zod only in applications that use the optional adapter: + +```sh +pnpm add ai@^7 zod@^4 +``` + +`files-sdk` 2.2.x still declares an optional `ai@^6` peer for its own adapter, +so some package managers may print a peer warning when AI SDK 7 is installed. +This package does not import that adapter; `@nestm/storage/ai-sdk` targets AI +SDK 7 directly. + +`@nestm/storage/ai-sdk` converts an already-mounted workspace to an ordinary +upstream `ToolSet`. It does not import NestJS or `@nestm/ai-sdk`; the application +composes the tool set through the AI module's existing named-toolset factory. +For a tenant or run selected per request, make both factories request-scoped and +derive the mount coordinate from authenticated host context: + +```ts +import { Module, Scope } from '@nestjs/common'; +import { AiSdkModule, AiSdkService, getAiToolsetToken } from '@nestm/ai-sdk'; +import { getStorageToken, type StorageClient } from '@nestm/storage'; +import { createAiSdkWorkspaceTools } from '@nestm/storage/ai-sdk'; +import { mountStorageWorkspace } from '@nestm/storage/workspace'; +import type { ToolSet } from 'ai'; + +@Module({ + imports: [ + AppStorageModule, + WorkspaceContextModule, + AiSdkModule.forFeature({ + imports: [AppStorageModule, WorkspaceContextModule], + toolsets: [ + { + name: 'workspace', + scope: Scope.REQUEST, + inject: [getStorageToken('agent-files'), WorkspaceContext], + useFactory: (storage: StorageClient, context: WorkspaceContext) => + createAiSdkWorkspaceTools({ + workspace: mountStorageWorkspace(storage, { + // A validated, opaque coordinate from trusted auth/run state. + // It is never accepted from a prompt or tool input. + prefix: context.storagePrefix, + permissions: [ + 'list', + 'read', + 'search', + 'create', + 'replace', + 'copy', + 'move', + 'delete', + ], + }), + }), + }, + ], + agents: [ + { + name: 'workspace-agent', + scope: Scope.REQUEST, + inject: [AiSdkService, getAiToolsetToken('workspace')], + useFactory: (ai: AiSdkService, tools: ToolSet) => ({ + model: ai.languageModel(), + instructions: + 'Use only the mounted workspace tools for file operations.', + tools, + }), + }, + ], + }), + ], +}) +export class WorkspaceAgentModule {} +``` + +The generated set contains only tools allowed by the workspace permissions: +bounded list, stat, UTF-8 read, and search tools plus conditional create, +replace, copy, move, and delete tools when granted. Mutation tools require AI +SDK user approval by default; approval can be configured per tool, but the +workspace capability remains the authorization boundary even when approval is +disabled. The module's `AiSdkService.files()` API is the model provider's file +upload facility and is unrelated to storage workspaces. + +This logical confinement is sufficient for a `ToolLoopAgent` whose only file +capabilities are these tools. It cannot constrain a coding harness that already +has shell, `node:fs`, or subprocess access. For Codex/Claude-style harnesses, +materialize the workspace into a per-session container or VM, mount only that +directory, run the harness there, and synchronize reviewed changes back through +`StorageWorkspace`. A working directory alone is not a sandbox. + ## Configure named stores Use the package-owned S3 factory when applicable. For other providers, create a diff --git a/SECURITY.md b/SECURITY.md index dc2ceff..d1f059f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -46,3 +46,43 @@ the validated `head` or with an immutable provider version. Ordinary `copy` does not protect against a staging-key replay between validation and copy. Conditional promotion keeps the staging source; delete it only after the application's metadata transaction commits. + +## Workspace security boundary + +`StorageWorkspace` is a narrowing capability for storage operations. Construct +it from trusted application context, keep the underlying `StorageClient` and +mount prefix private, and pass only the workspace or its AI tool set to +untrusted agent code. A tenant id, run id, prefix, provider cursor, snapshot id, +or fork id supplied by a model is not a safe mount coordinate. + +The workspace accepts only canonical mount-relative POSIX paths and rechecks +every key returned by a driver before unscoping it. Its cursors are opaque and +bound to the mount and query. Permissions, byte limits, result limits, and +conditional mutation preconditions are enforced inside the capability; tool +omission and user approval are additional workflow controls, not the +authorization boundary. + +Conditional mutations can still have an ambiguous outcome when a remote +provider commits and then loses or violates its response, or when a configured +post-operation plugin fails after the driver has committed. The API fails +closed in that case: inspect the logical destination and reconcile it before +retrying. Create-only and ETag preconditions prevent a blind retry from +silently overwriting a different object, but they cannot make a multi-object +move transactionally atomic. + +This guarantee covers calls made through `StorageWorkspace`. It does not +confine arbitrary `node:fs`, shell, subprocess, or native-code access in the +same process. A coding harness with built-in shell or filesystem tools must run +inside an OS sandbox (container, VM, or equivalent) that exposes only a +materialized workspace. Setting `cwd` to a workspace directory is not +isolation. + +For local filesystem storage, use a dedicated service-owned root and do not let +another untrusted process mutate its directory tree concurrently. High-level +Node filesystem checks reject symlinks and hard-linked object files in existing +workspace read and mutation paths, including metadata sidecars, but cannot +provide a race-proof boundary against an actor that can replace path components +between validation and use. For that threat model, mount only the workspace +into a separate UID/container/VM and synchronize approved results back through +storage. The local adapter also commits the body and metadata sidecar as two +files; a process crash between their atomic renames can require reconciliation. diff --git a/package.json b/package.json index 872b9f6..07720ef 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@nestm/storage", "version": "0.1.0-alpha.6", - "description": "Framework-neutral storage clients with NestJS 12 integration, named stores, streaming I/O, and an optional guarded HTTP gateway.", + "description": "Framework-neutral storage clients, capability-scoped agent workspaces, NestJS 12 integration, and guarded HTTP access.", "license": "MIT", "author": "Kauan Guesser", "type": "module", @@ -23,6 +23,14 @@ "types": "./dist/core/index.d.ts", "import": "./dist/core/index.js" }, + "./workspace": { + "types": "./dist/workspace/index.d.ts", + "import": "./dist/workspace/index.js" + }, + "./ai-sdk": { + "types": "./dist/ai-sdk/index.d.ts", + "import": "./dist/ai-sdk/index.js" + }, "./files-sdk": { "types": "./dist/files-sdk/index.d.ts", "import": "./dist/files-sdk/index.js" @@ -71,6 +79,7 @@ "nestjs-12", "storage", "object-storage", + "agent-workspace", "s3", "gcs", "azure", @@ -107,8 +116,10 @@ "@aws-sdk/s3-request-presigner": "^3.700.0", "@nestjs/common": "^12.0.0-alpha.5", "@nestjs/core": "^12.0.0-alpha.5", + "ai": ">=7.0.0 <8.0.0", "reflect-metadata": "^0.2.2", - "rxjs": "^7.8.1" + "rxjs": "^7.8.1", + "zod": "^4.1.8" }, "peerDependenciesMeta": { "@aws-sdk/client-s3": { @@ -129,11 +140,17 @@ "@nestjs/core": { "optional": true }, + "ai": { + "optional": true + }, "reflect-metadata": { "optional": true }, "rxjs": { "optional": true + }, + "zod": { + "optional": true } }, "devDependencies": { @@ -150,6 +167,7 @@ "@types/node": "26.1.2", "@types/supertest": "7.2.1", "@vitest/coverage-v8": "4.1.10", + "ai": "7.0.52", "fastify": "5.11.2", "oxlint": "1.77.0", "prettier": "3.9.6", @@ -158,6 +176,7 @@ "rxjs": "7.8.2", "supertest": "7.2.2", "typescript": "7.0.2", - "vitest": "4.1.10" + "vitest": "4.1.10", + "zod": "4.4.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09c3514..09561a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,7 +13,7 @@ importers: dependencies: files-sdk: specifier: 2.2.3 - version: 2.2.3(@aws-sdk/client-s3@3.1103.0)(@aws-sdk/lib-storage@3.1103.0(@aws-sdk/client-s3@3.1103.0))(@aws-sdk/s3-presigned-post@3.1103.0)(@aws-sdk/s3-request-presigner@3.1103.0)(@nestjs/common@12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0))(fastify@5.11.2)(hono@4.13.0)(supports-color@7.2.0)(zod@4.4.3) + version: 2.2.3(@aws-sdk/client-s3@3.1103.0)(@aws-sdk/lib-storage@3.1103.0(@aws-sdk/client-s3@3.1103.0))(@aws-sdk/s3-presigned-post@3.1103.0)(@aws-sdk/s3-request-presigner@3.1103.0)(@nestjs/common@12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0))(ai@7.0.52(zod@4.4.3))(fastify@5.11.2)(hono@4.13.0)(supports-color@7.2.0)(zod@4.4.3) devDependencies: '@aws-sdk/client-s3': specifier: 3.1103.0 @@ -54,6 +54,9 @@ importers: '@vitest/coverage-v8': specifier: 4.1.10 version: 4.1.10(vitest@4.1.10) + ai: + specifier: 7.0.52 + version: 7.0.52(zod@4.4.3) fastify: specifier: 5.11.2 version: 5.11.2 @@ -81,9 +84,28 @@ importers: vitest: specifier: 4.1.10 version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(vite@8.1.5(@types/node@26.1.2)) + zod: + specifier: 4.4.3 + version: 4.4.3 packages: + '@ai-sdk/gateway@4.0.41': + resolution: {integrity: sha512-DPkDmmA336kqmIp62on550/6Q6JHvldsnvN1SmkwU8QGrVt1PzFxqyKUn0C5EGGmSBpvF1ohZNtvWNl8gEVugA==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/provider-utils@5.0.21': + resolution: {integrity: sha512-Q4z27c5vqKuXZN65QuF7FVm+uvm3qxEioiQ+8c3s5xXlhbE1Y/SLGBB4BuF+UWia4KaDu2HmNpdR1RGW02/eGg==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/provider@4.0.5': + resolution: {integrity: sha512-9WAerTaPU2jcVi8dh8x27tySEuLur1Kfg7Go74GuCmsC/2MDSvrTkrK8ZrXWjryEevKfcPPmO1enveE7eqlzoA==} + engines: {node: '>=22'} + '@aws-sdk/checksums@3.1000.26': resolution: {integrity: sha512-CGznePoL+1oWCSzqmkvlYMpWQEZohPR3LntjKXXfVH+oh6Kd8d+yHLkjpiAGwPIHAxFe4OAq3aKfRnv/wqWIyA==} engines: {node: '>=20.0.0'} @@ -839,6 +861,10 @@ packages: cpu: [x64] os: [win32] + '@vercel/oidc@3.2.0': + resolution: {integrity: sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==} + engines: {node: '>= 20'} + '@vitest/coverage-v8@4.1.10': resolution: {integrity: sha512-IM49HmthevbgAO4anp1hwtoT9wYe59w0LR00gr+eagHE+ZJ5lK4sLPeO0ubgoJcwLk6dehU3R24N+FbEEKDc8g==} peerDependencies: @@ -877,6 +903,9 @@ packages: '@vitest/utils@4.1.10': resolution: {integrity: sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==} + '@workflow/serde@4.1.0': + resolution: {integrity: sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==} + abstract-logging@2.0.1: resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} @@ -884,6 +913,12 @@ packages: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} + ai@7.0.52: + resolution: {integrity: sha512-sEgRnYA+ddJ1rJSz1uKBjkGbOXhXdon1nvno49fVk3tabnlf8v5SaceLBpPDMtmDuLKQFztc/YbRsoCQoPzKZw==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} peerDependencies: @@ -1551,6 +1586,9 @@ packages: json-schema-typed@8.0.2: resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + jsonfile@4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} @@ -2144,6 +2182,10 @@ packages: undici-types@8.3.0: resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + universalify@0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} @@ -2266,6 +2308,26 @@ packages: snapshots: + '@ai-sdk/gateway@4.0.41(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 4.0.5 + '@ai-sdk/provider-utils': 5.0.21(zod@4.4.3) + '@vercel/oidc': 3.2.0 + zod: 4.4.3 + + '@ai-sdk/provider-utils@5.0.21(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 4.0.5 + '@standard-schema/spec': 1.1.0 + '@workflow/serde': 4.1.0 + eventsource-parser: 3.1.0 + undici: 7.29.0 + zod: 4.4.3 + + '@ai-sdk/provider@4.0.5': + dependencies: + json-schema: 0.4.0 + '@aws-sdk/checksums@3.1000.26': dependencies: '@aws-sdk/core': 3.977.6 @@ -3079,6 +3141,8 @@ snapshots: '@typescript/typescript-win32-x64@7.0.2': optional: true + '@vercel/oidc@3.2.0': {} + '@vitest/coverage-v8@4.1.10(vitest@4.1.10)': dependencies: '@bcoe/v8-coverage': 1.0.2 @@ -3134,6 +3198,8 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.1.1 + '@workflow/serde@4.1.0': {} + abstract-logging@2.0.1: {} accepts@2.0.0: @@ -3141,6 +3207,13 @@ snapshots: mime-types: 3.0.2 negotiator: 1.0.0 + ai@7.0.52(zod@4.4.3): + dependencies: + '@ai-sdk/gateway': 4.0.41(zod@4.4.3) + '@ai-sdk/provider': 4.0.5 + '@ai-sdk/provider-utils': 5.0.21(zod@4.4.3) + zod: 4.4.3 + ajv-formats@3.0.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -3353,8 +3426,7 @@ snapshots: events@3.3.0: {} - eventsource-parser@3.1.0: - optional: true + eventsource-parser@3.1.0: {} eventsource@3.0.7: dependencies: @@ -3502,7 +3574,7 @@ snapshots: transitivePeerDependencies: - supports-color - files-sdk@2.2.3(@aws-sdk/client-s3@3.1103.0)(@aws-sdk/lib-storage@3.1103.0(@aws-sdk/client-s3@3.1103.0))(@aws-sdk/s3-presigned-post@3.1103.0)(@aws-sdk/s3-request-presigner@3.1103.0)(@nestjs/common@12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0))(fastify@5.11.2)(hono@4.13.0)(supports-color@7.2.0)(zod@4.4.3): + files-sdk@2.2.3(@aws-sdk/client-s3@3.1103.0)(@aws-sdk/lib-storage@3.1103.0(@aws-sdk/client-s3@3.1103.0))(@aws-sdk/s3-presigned-post@3.1103.0)(@aws-sdk/s3-request-presigner@3.1103.0)(@nestjs/common@12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0))(ai@7.0.52(zod@4.4.3))(fastify@5.11.2)(hono@4.13.0)(supports-color@7.2.0)(zod@4.4.3): dependencies: aws4fetch: 1.0.20 commander: 15.0.0 @@ -3515,6 +3587,7 @@ snapshots: '@aws-sdk/s3-request-presigner': 3.1103.0 '@modelcontextprotocol/sdk': 1.30.0(supports-color@7.2.0)(zod@4.4.3) '@nestjs/common': 12.0.0-alpha.5(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@7.2.0) + ai: 7.0.52(zod@4.4.3) fastify: 5.11.2 hono: 4.13.0 zod: 4.4.3 @@ -3724,6 +3797,8 @@ snapshots: json-schema-typed@8.0.2: optional: true + json-schema@0.4.0: {} + jsonfile@4.0.0: optionalDependencies: graceful-fs: 4.2.11 @@ -4302,6 +4377,8 @@ snapshots: undici-types@8.3.0: {} + undici@7.29.0: {} + universalify@0.1.2: {} unpipe@1.0.0: {} @@ -4365,5 +4442,4 @@ snapshots: zod: 4.4.3 optional: true - zod@4.4.3: - optional: true + zod@4.4.3: {} diff --git a/src/ai-sdk/ai-sdk-workspace-tools.spec.ts b/src/ai-sdk/ai-sdk-workspace-tools.spec.ts new file mode 100644 index 0000000..7c18b7c --- /dev/null +++ b/src/ai-sdk/ai-sdk-workspace-tools.spec.ts @@ -0,0 +1,609 @@ +import type { ToolSet } from 'ai'; +import { z } from 'zod'; + +import { StorageErrorCode } from '../storage.error.js'; +import { + StorageWorkspaceError, + type StorageWorkspace, + type StorageWorkspaceEntry, + type StorageWorkspaceFile, + type StorageWorkspacePermission, + type StorageWorkspaceTextFile, +} from '../workspace/index.js'; +import { + AiSdkWorkspaceToolError, + createAiSdkWorkspaceTools, +} from './ai-sdk-workspace-tools.js'; + +interface ToolView { + execute?: ( + input: unknown, + options: { + toolCallId: string; + messages: []; + context: undefined; + abortSignal?: AbortSignal; + }, + ) => unknown; + inputSchema: z.ZodType; + needsApproval?: unknown; + strict?: boolean; +} + +interface WorkspaceDouble { + workspace: StorageWorkspace; + list: ReturnType; + stat: ReturnType; + readText: ReturnType; + search: ReturnType; + writeFile: ReturnType; + copyFile: ReturnType; + moveFile: ReturnType; + deleteFile: ReturnType; +} + +const FILE: StorageWorkspaceFile = { + kind: 'file', + path: 'docs/readme.md', + name: 'readme.md', + size: 5, + contentType: 'text/markdown', + etag: 'etag-1', + lastModified: new Date('2026-08-12T12:00:00.000Z'), +}; + +const TEXT_FILE: StorageWorkspaceTextFile = { + ...FILE, + text: 'hello', +}; + +function createWorkspaceDouble( + permissions: readonly StorageWorkspacePermission[], +): WorkspaceDouble { + const permissionSet = new Set(permissions); + const list = vi.fn(async () => ({ entries: [] as StorageWorkspaceEntry[] })); + const stat = vi.fn(async () => FILE); + const readText = vi.fn(async () => TEXT_FILE); + const search = vi.fn(async () => ({ + entries: [] as StorageWorkspaceEntry[], + })); + const writeFile = vi.fn(async () => FILE); + const copyFile = vi.fn(async () => FILE); + const moveFile = vi.fn(async () => FILE); + const deleteFile = vi.fn(async () => undefined); + + const workspace = { + permissions: permissionSet, + limits: { + maxPathBytes: 1_024, + maxReadBytes: 100, + maxWriteBytes: 200, + maxPageSize: 25, + maxSearchResults: 20, + maxSearchScan: 100, + cursorTtlMs: 60_000, + }, + allows: (permission: StorageWorkspacePermission) => + permissionSet.has(permission), + list, + stat, + readText, + search, + writeFile, + copyFile, + moveFile, + deleteFile, + } as unknown as StorageWorkspace; + + return { + workspace, + list, + stat, + readText, + search, + writeFile, + copyFile, + moveFile, + deleteFile, + }; +} + +function viewTool(tools: ToolSet, name: string): ToolView { + const candidate = tools[name] as unknown as ToolView | undefined; + if (candidate === undefined) { + throw new Error(`Missing tool: ${name}`); + } + return candidate; +} + +async function executeTool( + tools: ToolSet, + name: string, + input: unknown, + abortSignal?: AbortSignal, +): Promise { + const execute = viewTool(tools, name).execute; + if (execute === undefined) { + throw new Error(`Tool is not executable: ${name}`); + } + return await execute(input, { + toolCallId: 'test-call', + messages: [], + context: undefined, + ...(abortSignal === undefined ? {} : { abortSignal }), + }); +} + +describe('createAiSdkWorkspaceTools', () => { + it('returns only tools permitted by the mounted workspace', () => { + const readOnly = createWorkspaceDouble(['list', 'read', 'search']); + const tools = createAiSdkWorkspaceTools({ workspace: readOnly.workspace }); + + expect(Object.keys(tools).toSorted()).toEqual([ + 'workspace_list', + 'workspace_read_file', + 'workspace_search', + 'workspace_stat', + ]); + expect('workspace_write_file' in tools).toBe(false); + expect('workspace_delete_file' in tools).toBe(false); + }); + + it('maps every mutation permission to its tool', () => { + const fixture = createWorkspaceDouble([ + 'read', + 'create', + 'replace', + 'copy', + 'move', + 'delete', + ]); + const tools = createAiSdkWorkspaceTools({ workspace: fixture.workspace }); + + expect(Object.keys(tools).toSorted()).toEqual([ + 'workspace_copy_file', + 'workspace_delete_file', + 'workspace_move_file', + 'workspace_read_file', + 'workspace_stat', + 'workspace_write_file', + ]); + }); + + it('omits compound mutations until all enforcing permissions are present', () => { + const missingPrerequisites = createWorkspaceDouble(['copy', 'move']); + expect( + Object.keys( + createAiSdkWorkspaceTools({ + workspace: missingPrerequisites.workspace, + }), + ), + ).toEqual([]); + + const copyOnly = createWorkspaceDouble(['read', 'create', 'copy', 'move']); + const tools = createAiSdkWorkspaceTools({ workspace: copyOnly.workspace }); + expect('workspace_copy_file' in tools).toBe(true); + expect('workspace_move_file' in tools).toBe(false); + }); + + it('requires approval for mutations by default and supports granular overrides', () => { + const fixture = createWorkspaceDouble([ + 'read', + 'create', + 'copy', + 'move', + 'delete', + ]); + const defaults = createAiSdkWorkspaceTools({ + workspace: fixture.workspace, + }); + + expect(viewTool(defaults, 'workspace_write_file').needsApproval).toBe(true); + expect(viewTool(defaults, 'workspace_copy_file').needsApproval).toBe(true); + expect(viewTool(defaults, 'workspace_move_file').needsApproval).toBe(true); + expect(viewTool(defaults, 'workspace_delete_file').needsApproval).toBe( + true, + ); + + const configured = createAiSdkWorkspaceTools({ + workspace: fixture.workspace, + requireApproval: { + workspace_write_file: false, + workspace_delete_file: false, + }, + }); + expect(viewTool(configured, 'workspace_write_file').needsApproval).toBe( + false, + ); + expect(viewTool(configured, 'workspace_copy_file').needsApproval).toBe( + true, + ); + expect(viewTool(configured, 'workspace_delete_file').needsApproval).toBe( + false, + ); + }); + + it('uses strict schemas and accepts only relative logical paths', () => { + const fixture = createWorkspaceDouble(['read', 'search', 'create', 'copy']); + const tools = createAiSdkWorkspaceTools({ workspace: fixture.workspace }); + const statSchema = viewTool(tools, 'workspace_stat').inputSchema; + const copySchema = viewTool(tools, 'workspace_copy_file').inputSchema; + const searchSchema = viewTool(tools, 'workspace_search').inputSchema; + + for (const path of [ + 'src/index.ts', + 'docs/caf\u00e9.md', + 'docs/COM10.txt', + 'docs/console.txt', + ]) { + expect(statSchema.safeParse({ path }).success, path).toBe(true); + } + for (const path of [ + '/etc/passwd', + '../secret', + 'a/../secret', + 'C:\\secret', + 'C:/secret', + 'docs/name:stream.txt', + 'docs/report.', + 'docs/report ', + 'CON', + 'docs/aux.txt', + 'docs/COM1.log', + 'docs/lPt9', + 'docs/cafe\u0301.md', + 'docs/zero\u200bwidth.txt', + 'docs/private-\ue000.txt', + ]) { + expect(statSchema.safeParse({ path }).success, path).toBe(false); + } + expect( + statSchema.safeParse({ path: 'src/index.ts', providerKey: 'raw/key' }) + .success, + ).toBe(false); + expect( + copySchema.safeParse({ + source: 'from.txt', + destination: 'to.txt', + overwrite: true, + }).success, + ).toBe(false); + expect( + searchSchema.safeParse({ query: '.*', match: 'regex' }).success, + ).toBe(false); + expect(searchSchema.safeParse({ query: 'name:stream' }).success).toBe(true); + expect(searchSchema.safeParse({ query: 'dir\\*.txt' }).success).toBe(false); + expect(searchSchema.safeParse({ query: 'zero\u200bwidth' }).success).toBe( + false, + ); + expect(searchSchema.safeParse({ query: 'x'.repeat(1_025) }).success).toBe( + false, + ); + expect(viewTool(tools, 'workspace_stat').strict).toBe(true); + }); + + it('describes cursor continuation with the bound query options', () => { + const fixture = createWorkspaceDouble(['list', 'search']); + const tools = createAiSdkWorkspaceTools({ workspace: fixture.workspace }); + const listSchema = z.toJSONSchema( + viewTool(tools, 'workspace_list').inputSchema, + ); + const searchSchema = z.toJSONSchema( + viewTool(tools, 'workspace_search').inputSchema, + ); + const cursor = 'a'.repeat(32); + + expect( + viewTool(tools, 'workspace_list').inputSchema.safeParse({ cursor }) + .success, + ).toBe(true); + expect( + viewTool(tools, 'workspace_list').inputSchema.safeParse({ + cursor: 'a'.repeat(31), + }).success, + ).toBe(false); + expect( + viewTool(tools, 'workspace_search').inputSchema.safeParse({ + query: '*.ts', + cursor: `${'a'.repeat(31)}+`, + }).success, + ).toBe(false); + + expect(listSchema).toMatchObject({ + properties: { + cursor: { + description: + 'Opaque one-use cursor returned by the preceding list call. Repeat the same directory, recursive, and limit options when continuing.', + }, + }, + }); + expect(searchSchema).toMatchObject({ + properties: { + cursor: { + description: + 'Opaque one-use cursor returned by the preceding search call. Repeat the same query, directory, match, caseInsensitive, and limit options when continuing.', + }, + }, + }); + }); + + it('narrows the write schema to the granted create or replace mode', () => { + const createFixture = createWorkspaceDouble(['create']); + const createTools = createAiSdkWorkspaceTools({ + workspace: createFixture.workspace, + }); + const createSchema = viewTool( + createTools, + 'workspace_write_file', + ).inputSchema; + expect( + createSchema.safeParse({ + path: 'new.txt', + content: 'new', + mode: 'create', + }).success, + ).toBe(true); + expect( + createSchema.safeParse({ + path: 'too-large.txt', + content: 'x'.repeat(201), + mode: 'create', + }).success, + ).toBe(false); + expect( + createSchema.safeParse({ + path: 'old.txt', + content: 'new', + mode: 'replace', + etag: 'old-etag', + }).success, + ).toBe(false); + + const replaceFixture = createWorkspaceDouble(['replace']); + const replaceTools = createAiSdkWorkspaceTools({ + workspace: replaceFixture.workspace, + }); + const replaceSchema = viewTool( + replaceTools, + 'workspace_write_file', + ).inputSchema; + expect( + replaceSchema.safeParse({ + path: 'old.txt', + content: 'new', + mode: 'replace', + }).success, + ).toBe(false); + expect( + replaceSchema.safeParse({ + path: 'old.txt', + content: 'new', + mode: 'replace', + etag: 'old-etag', + }).success, + ).toBe(true); + expect( + replaceSchema.safeParse({ + path: 'old.txt', + content: 'new', + mode: 'replace', + etag: 'unsafe\nvalue', + }).success, + ).toBe(false); + expect( + replaceSchema.safeParse({ + path: 'old.txt', + content: 'new', + mode: 'replace', + etag: 'x'.repeat(1_025), + }).success, + ).toBe(false); + }); + + it('clamps bounded text reads and serializes file metadata', async () => { + const fixture = createWorkspaceDouble(['read']); + const controller = new AbortController(); + const tools = createAiSdkWorkspaceTools({ + workspace: fixture.workspace, + maxReadBytes: 1_000, + }); + + await expect( + executeTool( + tools, + 'workspace_read_file', + { path: 'docs/readme.md' }, + controller.signal, + ), + ).resolves.toEqual({ + kind: 'file', + path: 'docs/readme.md', + name: 'readme.md', + size: 5, + contentType: 'text/markdown', + etag: 'etag-1', + lastModified: '2026-08-12T12:00:00.000Z', + text: 'hello', + }); + expect(fixture.readText).toHaveBeenCalledWith('docs/readme.md', { + maxBytes: 100, + signal: controller.signal, + }); + }); + + it('forwards bounded list and search calls and serializes entries', async () => { + const fixture = createWorkspaceDouble(['list', 'search']); + fixture.list.mockResolvedValueOnce({ + entries: [{ kind: 'directory', path: 'docs', name: 'docs' }, FILE], + cursor: 'next-list', + }); + fixture.search.mockResolvedValueOnce({ + entries: [FILE], + cursor: 'next-search', + }); + const tools = createAiSdkWorkspaceTools({ workspace: fixture.workspace }); + + await expect( + executeTool(tools, 'workspace_list', { + directory: 'docs', + recursive: true, + limit: 10, + cursor: 'previous', + }), + ).resolves.toMatchObject({ + entries: [ + { kind: 'directory', path: 'docs', name: 'docs' }, + { kind: 'file', path: 'docs/readme.md' }, + ], + cursor: 'next-list', + }); + expect(fixture.list).toHaveBeenCalledWith({ + directory: 'docs', + recursive: true, + limit: 10, + cursor: 'previous', + }); + + await expect( + executeTool(tools, 'workspace_search', { + query: '*.md', + directory: 'docs', + match: 'glob', + caseInsensitive: true, + limit: 5, + }), + ).resolves.toMatchObject({ cursor: 'next-search' }); + expect(fixture.search).toHaveBeenCalledWith('*.md', { + directory: 'docs', + match: 'glob', + caseInsensitive: true, + limit: 5, + }); + }); + + it('preserves create-only destinations and required ETag preconditions', async () => { + const fixture = createWorkspaceDouble([ + 'read', + 'create', + 'replace', + 'copy', + 'move', + 'delete', + ]); + const tools = createAiSdkWorkspaceTools({ workspace: fixture.workspace }); + + await executeTool(tools, 'workspace_write_file', { + path: 'created.txt', + content: 'created', + mode: 'create', + }); + expect(fixture.writeFile).toHaveBeenCalledWith('created.txt', 'created', { + mode: 'create', + }); + + await executeTool(tools, 'workspace_write_file', { + path: 'updated.txt', + content: 'updated', + mode: 'replace', + etag: 'replace-etag', + }); + expect(fixture.writeFile).toHaveBeenCalledWith('updated.txt', 'updated', { + mode: 'replace', + etag: 'replace-etag', + }); + + await executeTool(tools, 'workspace_copy_file', { + source: 'from.txt', + destination: 'copied.txt', + }); + expect(fixture.copyFile).toHaveBeenCalledWith('from.txt', 'copied.txt', {}); + + await executeTool(tools, 'workspace_move_file', { + source: 'from.txt', + destination: 'moved.txt', + etag: 'source-etag', + }); + expect(fixture.moveFile).toHaveBeenCalledWith('from.txt', 'moved.txt', { + etag: 'source-etag', + }); + + await expect( + executeTool(tools, 'workspace_delete_file', { + path: 'old.txt', + etag: 'delete-etag', + }), + ).resolves.toEqual({ deleted: true, path: 'old.txt' }); + expect(fixture.deleteFile).toHaveBeenCalledWith('old.txt', { + etag: 'delete-etag', + }); + }); + + it('sanitizes workspace and unknown failures without retaining their cause', async () => { + const fixture = createWorkspaceDouble(['read']); + fixture.stat.mockRejectedValueOnce( + new StorageWorkspaceError( + 'provider bucket secret-bucket/raw/prefix/private.txt was missing', + { + code: StorageErrorCode.NOT_FOUND, + operation: 'stat', + path: 'private.txt', + permanent: true, + }, + ), + ); + const tools = createAiSdkWorkspaceTools({ workspace: fixture.workspace }); + + const notFound = await executeTool(tools, 'workspace_stat', { + path: 'private.txt', + }).catch((error: unknown) => error); + expect(notFound).toBeInstanceOf(AiSdkWorkspaceToolError); + expect(notFound).toMatchObject({ + code: StorageErrorCode.NOT_FOUND, + message: 'The requested workspace path was not found.', + }); + expect((notFound as Error & { cause?: unknown }).cause).toBeUndefined(); + expect(JSON.stringify(notFound)).not.toContain('secret'); + + fixture.stat.mockRejectedValueOnce( + new Error('SDK request leaked https://provider.invalid/private-key'), + ); + const provider = await executeTool(tools, 'workspace_stat', { + path: 'private.txt', + }).catch((error: unknown) => error); + expect(provider).toMatchObject({ + code: StorageErrorCode.PROVIDER, + message: 'The workspace operation failed.', + }); + expect(String(provider)).not.toContain('provider.invalid'); + }); + + it('reports an aborted call without exposing the underlying failure', async () => { + const fixture = createWorkspaceDouble(['read']); + fixture.stat.mockRejectedValueOnce(new Error('private provider failure')); + const tools = createAiSdkWorkspaceTools({ workspace: fixture.workspace }); + const controller = new AbortController(); + controller.abort(); + + const result = await executeTool( + tools, + 'workspace_stat', + { path: 'file.txt' }, + controller.signal, + ).catch((error: unknown) => error); + expect(result).toMatchObject({ + code: StorageErrorCode.ABORTED, + message: 'The workspace operation was aborted.', + }); + }); + + it('rejects invalid factory read limits before creating tools', () => { + const fixture = createWorkspaceDouble(['read']); + + expect(() => + createAiSdkWorkspaceTools({ + workspace: fixture.workspace, + maxReadBytes: 0, + }), + ).toThrow('maxReadBytes must be a positive safe integer.'); + }); +}); diff --git a/src/ai-sdk/ai-sdk-workspace-tools.ts b/src/ai-sdk/ai-sdk-workspace-tools.ts new file mode 100644 index 0000000..7443b14 --- /dev/null +++ b/src/ai-sdk/ai-sdk-workspace-tools.ts @@ -0,0 +1,623 @@ +import { tool, type ToolSet } from 'ai'; +import { z } from 'zod'; + +import { + isStorageWorkspaceError, + type StorageWorkspace, + type StorageWorkspaceEntry, + type StorageWorkspaceFile, + type StorageWorkspaceTextFile, +} from '../workspace/index.js'; +import { + StorageErrorCode, + isStorageError, + type StorageErrorCode as StorageErrorCodeValue, +} from '../storage.error.js'; + +export const AI_SDK_WORKSPACE_TOOL_NAMES = [ + 'workspace_list', + 'workspace_stat', + 'workspace_read_file', + 'workspace_search', + 'workspace_write_file', + 'workspace_copy_file', + 'workspace_move_file', + 'workspace_delete_file', +] as const; + +export type AiSdkWorkspaceToolName = + (typeof AI_SDK_WORKSPACE_TOOL_NAMES)[number]; + +export type AiSdkWorkspaceMutationToolName = Extract< + AiSdkWorkspaceToolName, + | 'workspace_write_file' + | 'workspace_copy_file' + | 'workspace_move_file' + | 'workspace_delete_file' +>; + +/** + * Approval policy for mutation tools. An omitted entry defaults to requiring + * approval, matching an omitted policy. + */ +export type AiSdkWorkspaceApprovalConfig = + boolean | Partial>; + +export interface CreateAiSdkWorkspaceToolsOptions { + /** The already-mounted, policy-enforcing workspace exposed to the tools. */ + workspace: StorageWorkspace; + /** + * Optional tighter read ceiling. Values above the workspace ceiling are + * clamped; the model cannot choose or raise this value. + */ + maxReadBytes?: number; + /** Mutation tools require approval by default. */ + requireApproval?: AiSdkWorkspaceApprovalConfig; +} + +export type AiSdkWorkspaceToolErrorCode = StorageErrorCodeValue; + +const SAFE_ERROR_MESSAGES: Readonly< + Record +> = { + [StorageErrorCode.NOT_FOUND]: 'The requested workspace path was not found.', + [StorageErrorCode.UNAUTHORIZED]: + 'This operation is not permitted in the workspace.', + [StorageErrorCode.CONFLICT]: + 'The operation conflicts with current workspace state. Refresh metadata and retry with the current ETag or a new destination.', + [StorageErrorCode.READ_ONLY]: + 'This operation is not permitted in the workspace.', + [StorageErrorCode.INVALID_ARGUMENT]: 'The workspace tool input was rejected.', + [StorageErrorCode.NOT_SUPPORTED]: + 'This workspace operation is not supported by the configured storage.', + [StorageErrorCode.ABORTED]: 'The workspace operation was aborted.', + [StorageErrorCode.TIMEOUT]: 'The workspace operation timed out.', + [StorageErrorCode.LIMIT_EXCEEDED]: + 'The workspace operation exceeded a configured limit.', + [StorageErrorCode.PROVIDER]: 'The workspace operation failed.', +}; + +/** + * Safe error exposed at the AI tool boundary. It deliberately carries no + * provider error, storage key, mount prefix, store name, or cause. + */ +export class AiSdkWorkspaceToolError extends Error { + readonly code: AiSdkWorkspaceToolErrorCode; + + constructor(code: AiSdkWorkspaceToolErrorCode) { + super(SAFE_ERROR_MESSAGES[code]); + this.name = 'AiSdkWorkspaceToolError'; + this.code = code; + } +} + +export interface AiSdkWorkspaceFileResult { + kind: 'file'; + path: string; + name: string; + size: number; + contentType: string; + etag?: string; + lastModified?: string; +} + +export interface AiSdkWorkspaceDirectoryResult { + kind: 'directory'; + path: string; + name: string; +} + +export type AiSdkWorkspaceEntryResult = + AiSdkWorkspaceFileResult | AiSdkWorkspaceDirectoryResult; + +export interface AiSdkWorkspaceTextFileResult extends AiSdkWorkspaceFileResult { + text: string; +} + +export interface AiSdkWorkspacePageResult { + entries: AiSdkWorkspaceEntryResult[]; + cursor?: string; +} + +const mutationToolNames = new Set([ + 'workspace_write_file', + 'workspace_copy_file', + 'workspace_move_file', + 'workspace_delete_file', +]); +const utf8Encoder = new TextEncoder(); +const forbiddenUnicodeCharacter = /\p{C}/u; +const workspaceCursor = /^[A-Za-z0-9_-]{32}$/u; +const windowsDeviceName = /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$/iu; + +function isLogicalPath(value: string): boolean { + if ( + value.length === 0 || + value.startsWith('/') || + value.endsWith('/') || + value !== value.normalize('NFC') || + value.includes('\\') || + forbiddenUnicodeCharacter.test(value) + ) { + return false; + } + + return value + .split('/') + .every( + (segment) => + segment.length > 0 && + segment !== '.' && + segment !== '..' && + !segment.includes(':') && + !segment.endsWith('.') && + !segment.endsWith(' ') && + !windowsDeviceName.test(segment), + ); +} + +function logicalPath(label: string, maxBytes: number) { + return z + .string() + .min(1) + .refine(isLogicalPath, { + message: `${label} must be a portable, NFC-normalized, relative POSIX workspace path.`, + }) + .refine((value) => utf8Encoder.encode(value).byteLength <= maxBytes, { + message: `${label} exceeds the ${maxBytes}-byte workspace path limit.`, + }) + .describe( + `${label}. Use an NFC-normalized relative POSIX path inside the mounted workspace. Empty, dot, parent, colon, trailing dot or space, and Windows reserved device segments are forbidden.`, + ); +} + +function searchQuery(maxBytes: number) { + return z + .string() + .min(1) + .refine( + (value) => + !value.includes('\\') && !forbiddenUnicodeCharacter.test(value), + { message: 'Search query contains forbidden characters.' }, + ) + .refine((value) => utf8Encoder.encode(value).byteLength <= maxBytes, { + message: `Search query exceeds the ${maxBytes}-byte workspace limit.`, + }) + .describe('Path pattern or text to find inside the mounted workspace.'); +} + +function boundedEtag(maxBytes: number) { + return z + .string() + .min(1) + .refine((value) => !forbiddenUnicodeCharacter.test(value), { + message: 'ETag contains forbidden characters.', + }) + .refine((value) => utf8Encoder.encode(value).byteLength <= maxBytes, { + message: `ETag exceeds the ${maxBytes}-byte workspace limit.`, + }) + .describe('Exact ETag returned by the latest workspace stat or read.'); +} + +function continuationCursor(description: string) { + return z + .string() + .regex( + workspaceCursor, + 'Cursor must be the opaque base64url token returned by the workspace.', + ) + .optional() + .describe(description); +} + +function positiveIntegerAtMost(maximum: number, description: string) { + return z + .number() + .int() + .positive() + .max(maximum) + .optional() + .describe(description); +} + +function operationOptions(signal: AbortSignal | undefined): { + signal?: AbortSignal; +} { + return signal === undefined ? {} : { signal }; +} + +function resolveApproval( + toolName: AiSdkWorkspaceMutationToolName, + config: AiSdkWorkspaceApprovalConfig, +): boolean { + return typeof config === 'boolean' ? config : (config[toolName] ?? true); +} + +function resolveReadLimit( + workspace: StorageWorkspace, + requested: number | undefined, +): number { + const workspaceLimit = workspace.limits.maxReadBytes; + if (!Number.isSafeInteger(workspaceLimit) || workspaceLimit <= 0) { + throw new RangeError( + 'workspace.limits.maxReadBytes must be a positive safe integer.', + ); + } + if ( + requested !== undefined && + (!Number.isSafeInteger(requested) || requested <= 0) + ) { + throw new RangeError('maxReadBytes must be a positive safe integer.'); + } + return Math.min(requested ?? workspaceLimit, workspaceLimit); +} + +function sanitizeToolError( + error: unknown, + signal: AbortSignal | undefined, +): AiSdkWorkspaceToolError { + if (error instanceof AiSdkWorkspaceToolError) { + return error; + } + if (signal?.aborted === true) { + return new AiSdkWorkspaceToolError(StorageErrorCode.ABORTED); + } + if (isStorageWorkspaceError(error)) { + return new AiSdkWorkspaceToolError(error.code); + } + if (isStorageError(error)) { + return new AiSdkWorkspaceToolError(error.code); + } + return new AiSdkWorkspaceToolError(StorageErrorCode.PROVIDER); +} + +async function executeSafely( + signal: AbortSignal | undefined, + operation: () => Promise, +): Promise { + try { + return await operation(); + } catch (error) { + throw sanitizeToolError(error, signal); + } +} + +function serializeFile(file: StorageWorkspaceFile): AiSdkWorkspaceFileResult { + return { + kind: 'file', + path: file.path, + name: file.name, + size: file.size, + contentType: file.contentType, + ...(file.etag === undefined ? {} : { etag: file.etag }), + ...(file.lastModified === undefined + ? {} + : { lastModified: file.lastModified.toISOString() }), + }; +} + +function serializeEntry( + entry: StorageWorkspaceEntry, +): AiSdkWorkspaceEntryResult { + return entry.kind === 'file' + ? serializeFile(entry) + : { kind: 'directory', path: entry.path, name: entry.name }; +} + +function serializeTextFile( + file: StorageWorkspaceTextFile, +): AiSdkWorkspaceTextFileResult { + return { ...serializeFile(file), text: file.text }; +} + +function serializePage(page: { + entries: StorageWorkspaceEntry[]; + cursor?: string; +}): AiSdkWorkspacePageResult { + return { + entries: page.entries.map(serializeEntry), + ...(page.cursor === undefined ? {} : { cursor: page.cursor }), + }; +} + +/** + * Creates Vercel AI SDK tools backed only by a mounted StorageWorkspace. + * + * A tool is omitted unless the workspace grants its required permission. + * The workspace remains the enforcing boundary if a retained tool reference + * is invoked after further narrowing. + */ +export function createAiSdkWorkspaceTools({ + workspace, + maxReadBytes: requestedMaxReadBytes, + requireApproval = true, +}: CreateAiSdkWorkspaceToolsOptions): ToolSet { + const maxReadBytes = resolveReadLimit(workspace, requestedMaxReadBytes); + const tools: ToolSet = {}; + const pathSchema = logicalPath('File path', workspace.limits.maxPathBytes); + const directorySchema = logicalPath( + 'Directory path', + workspace.limits.maxPathBytes, + ); + const etagSchema = boundedEtag(workspace.limits.maxPathBytes); + + if (workspace.allows('list')) { + tools.workspace_list = tool({ + description: + 'List files and directories inside the mounted workspace. Omit directory to list the workspace root.', + strict: true, + inputSchema: z + .object({ + directory: directorySchema.optional(), + recursive: z + .boolean() + .optional() + .describe('Whether to include descendants recursively.'), + limit: positiveIntegerAtMost( + workspace.limits.maxPageSize, + `Maximum entries to return, up to ${workspace.limits.maxPageSize}.`, + ), + cursor: continuationCursor( + 'Opaque one-use cursor returned by the preceding list call. Repeat the same directory, recursive, and limit options when continuing.', + ), + }) + .strict(), + execute: (input, { abortSignal }) => + executeSafely(abortSignal, async () => + serializePage( + await workspace.list({ + ...(input.directory === undefined + ? {} + : { directory: input.directory }), + ...(input.recursive === undefined + ? {} + : { recursive: input.recursive }), + ...(input.limit === undefined ? {} : { limit: input.limit }), + ...(input.cursor === undefined ? {} : { cursor: input.cursor }), + ...operationOptions(abortSignal), + }), + ), + ), + }); + } + + if (workspace.allows('read')) { + tools.workspace_stat = tool({ + description: + 'Inspect a file inside the mounted workspace without reading its contents. Returns the ETag required for safe replace, move, and delete operations.', + strict: true, + inputSchema: z.object({ path: pathSchema }).strict(), + execute: ({ path }, { abortSignal }) => + executeSafely(abortSignal, async () => + serializeFile( + await workspace.stat(path, operationOptions(abortSignal)), + ), + ), + }); + + tools.workspace_read_file = tool({ + description: `Read a UTF-8 text file inside the mounted workspace. The result is bounded to ${maxReadBytes} bytes.`, + strict: true, + inputSchema: z.object({ path: pathSchema }).strict(), + execute: ({ path }, { abortSignal }) => + executeSafely(abortSignal, async () => + serializeTextFile( + await workspace.readText(path, { + maxBytes: maxReadBytes, + ...operationOptions(abortSignal), + }), + ), + ), + }); + } + + if (workspace.allows('search')) { + tools.workspace_search = tool({ + description: + 'Search logical paths inside the mounted workspace using a bounded glob, substring, or exact match. Omit directory to search from the workspace root.', + strict: true, + inputSchema: z + .object({ + query: searchQuery(workspace.limits.maxPathBytes), + directory: directorySchema.optional(), + match: z + .enum(['glob', 'substring', 'exact']) + .optional() + .describe('Matching strategy; defaults to glob.'), + caseInsensitive: z.boolean().optional(), + limit: positiveIntegerAtMost( + workspace.limits.maxSearchResults, + `Maximum matches to return, up to ${workspace.limits.maxSearchResults}.`, + ), + cursor: continuationCursor( + 'Opaque one-use cursor returned by the preceding search call. Repeat the same query, directory, match, caseInsensitive, and limit options when continuing.', + ), + }) + .strict(), + execute: ( + { query, directory, match, caseInsensitive, limit, cursor }, + { abortSignal }, + ) => + executeSafely(abortSignal, async () => + serializePage( + await workspace.search(query, { + ...(directory === undefined ? {} : { directory }), + ...(match === undefined ? {} : { match }), + ...(caseInsensitive === undefined ? {} : { caseInsensitive }), + ...(limit === undefined ? {} : { limit }), + ...(cursor === undefined ? {} : { cursor }), + ...operationOptions(abortSignal), + }), + ), + ), + }); + } + + const canCreate = workspace.allows('create'); + const canReplace = workspace.allows('replace'); + if (canCreate || canReplace) { + const commonWriteShape = { + path: pathSchema, + content: z + .string() + .refine( + (value) => + utf8Encoder.encode(value).byteLength <= + workspace.limits.maxWriteBytes, + { + message: `Content exceeds the ${workspace.limits.maxWriteBytes}-byte workspace write limit.`, + }, + ) + .describe( + `UTF-8 text to write. The workspace enforces its ${workspace.limits.maxWriteBytes}-byte write limit.`, + ), + }; + const createSchema = z + .object({ ...commonWriteShape, mode: z.literal('create') }) + .strict(); + const replaceSchema = z + .object({ + ...commonWriteShape, + mode: z.literal('replace'), + etag: etagSchema, + }) + .strict(); + const inputSchema = + canCreate && canReplace + ? z.discriminatedUnion('mode', [createSchema, replaceSchema]) + : canCreate + ? createSchema + : replaceSchema; + + tools.workspace_write_file = tool({ + description: + canCreate && canReplace + ? 'Create a new UTF-8 text file or replace an existing file inside the mounted workspace. Create fails if the destination exists; replace requires its current ETag.' + : canCreate + ? 'Create a new UTF-8 text file inside the mounted workspace. The operation fails if the destination already exists.' + : 'Replace an existing UTF-8 text file inside the mounted workspace using its current ETag.', + strict: true, + inputSchema, + needsApproval: resolveApproval('workspace_write_file', requireApproval), + execute: (input, { abortSignal }) => + executeSafely(abortSignal, async () => + serializeFile( + input.mode === 'create' + ? await workspace.writeFile(input.path, input.content, { + mode: 'create', + ...operationOptions(abortSignal), + }) + : await workspace.writeFile(input.path, input.content, { + mode: 'replace', + etag: input.etag, + ...operationOptions(abortSignal), + }), + ), + ), + }); + } + + const canCopy = + workspace.allows('copy') && + workspace.allows('read') && + workspace.allows('create'); + if (canCopy) { + tools.workspace_copy_file = tool({ + description: + 'Copy a file inside the mounted workspace. The source remains intact, and the operation fails if the destination already exists.', + strict: true, + inputSchema: z + .object({ + source: logicalPath( + 'Source file path', + workspace.limits.maxPathBytes, + ), + destination: logicalPath( + 'Destination file path', + workspace.limits.maxPathBytes, + ), + }) + .strict(), + needsApproval: resolveApproval('workspace_copy_file', requireApproval), + execute: ({ source, destination }, { abortSignal }) => + executeSafely(abortSignal, async () => + serializeFile( + await workspace.copyFile( + source, + destination, + operationOptions(abortSignal), + ), + ), + ), + }); + } + + const canMove = + workspace.allows('move') && + workspace.allows('read') && + workspace.allows('create') && + workspace.allows('delete'); + if (canMove) { + tools.workspace_move_file = tool({ + description: + "Move a file inside the mounted workspace using the source's current ETag. The operation fails if the destination already exists. If source deletion cannot be confirmed, the destination is retained and the tool reports a conflict; inspect both paths before retrying.", + strict: true, + inputSchema: z + .object({ + source: logicalPath( + 'Source file path', + workspace.limits.maxPathBytes, + ), + destination: logicalPath( + 'Destination file path', + workspace.limits.maxPathBytes, + ), + etag: etagSchema.describe( + 'Exact ETag of the source returned by the latest workspace stat or read.', + ), + }) + .strict(), + needsApproval: resolveApproval('workspace_move_file', requireApproval), + execute: ({ source, destination, etag }, { abortSignal }) => + executeSafely(abortSignal, async () => + serializeFile( + await workspace.moveFile(source, destination, { + etag, + ...operationOptions(abortSignal), + }), + ), + ), + }); + } + + if (workspace.allows('delete')) { + tools.workspace_delete_file = tool({ + description: + 'Delete a file inside the mounted workspace using its current ETag.', + strict: true, + inputSchema: z + .object({ + path: pathSchema, + etag: etagSchema, + }) + .strict(), + needsApproval: resolveApproval('workspace_delete_file', requireApproval), + execute: ({ path, etag }, { abortSignal }) => + executeSafely(abortSignal, async () => { + await workspace.deleteFile(path, { + etag, + ...operationOptions(abortSignal), + }); + return { deleted: true as const, path }; + }), + }); + } + + return tools; +} + +export function isAiSdkWorkspaceMutationToolName( + value: string, +): value is AiSdkWorkspaceMutationToolName { + return mutationToolNames.has(value as AiSdkWorkspaceMutationToolName); +} diff --git a/src/ai-sdk/index.ts b/src/ai-sdk/index.ts new file mode 100644 index 0000000..5a5f667 --- /dev/null +++ b/src/ai-sdk/index.ts @@ -0,0 +1,16 @@ +export { + AI_SDK_WORKSPACE_TOOL_NAMES, + AiSdkWorkspaceToolError, + createAiSdkWorkspaceTools, + isAiSdkWorkspaceMutationToolName, + type AiSdkWorkspaceApprovalConfig, + type AiSdkWorkspaceDirectoryResult, + type AiSdkWorkspaceEntryResult, + type AiSdkWorkspaceFileResult, + type AiSdkWorkspaceMutationToolName, + type AiSdkWorkspacePageResult, + type AiSdkWorkspaceTextFileResult, + type AiSdkWorkspaceToolErrorCode, + type AiSdkWorkspaceToolName, + type CreateAiSdkWorkspaceToolsOptions, +} from './ai-sdk-workspace-tools.js'; diff --git a/src/files-sdk/files-sdk.driver.spec.ts b/src/files-sdk/files-sdk.driver.spec.ts index a7c75c8..bdd176a 100644 --- a/src/files-sdk/files-sdk.driver.spec.ts +++ b/src/files-sdk/files-sdk.driver.spec.ts @@ -89,4 +89,37 @@ describe('FilesSdkStorageDriver', () => { message: 'late provider failure', }); }); + + it('rejects a conditional adapter result from the wrong physical key', async () => { + const adapter = Object.assign(memory(), { + conditionalMutation: { + create: true, + delete: true, + etag: true, + replace: true, + }, + deleteConditional: vi.fn(async () => undefined), + uploadConditional: vi.fn(async () => ({ + contentType: 'text/plain', + etag: 'etag', + key: 'scope/other.txt', + size: 4, + })), + }); + const driver = createFilesSdkDriver({ adapter, prefix: 'scope' }); + + await expect( + driver.uploadConditional('requested.txt', 'body', { + condition: { type: 'create' }, + }), + ).rejects.toMatchObject({ + code: StorageErrorCode.PROVIDER, + key: 'requested.txt', + }); + expect(adapter.uploadConditional).toHaveBeenCalledWith( + 'scope/requested.txt', + 'body', + { condition: { type: 'create' } }, + ); + }); }); diff --git a/src/files-sdk/files-sdk.driver.ts b/src/files-sdk/files-sdk.driver.ts index 32ce57c..358b3b1 100644 --- a/src/files-sdk/files-sdk.driver.ts +++ b/src/files-sdk/files-sdk.driver.ts @@ -26,6 +26,9 @@ import { import type { StorageDriver } from '../storage.driver.js'; import type { StorageBody, + StorageConditionalDeleteOptions, + StorageConditionalMutationCapability, + StorageConditionalUploadOptions, StorageDownloadOptions, StorageListOptions, StorageListResult, @@ -62,6 +65,20 @@ export interface FilesSdkConditionalCopyAdapter { ): Promise; } +/** Optional adapter extension for native compare-and-set mutations. */ +export interface FilesSdkConditionalMutationAdapter { + readonly conditionalMutation: StorageConditionalMutationCapability; + uploadConditional( + key: string, + body: Body, + options: StorageConditionalUploadOptions, + ): Promise; + deleteConditional( + key: string, + options: StorageConditionalDeleteOptions, + ): Promise; +} + export interface FilesSdkSignedUploadPolicyAdapter { readonly signedUploadPolicy: StorageSignedUploadPolicyCapability; } @@ -96,6 +113,35 @@ function conditionalCopyAdapterOf( return candidate as Adapter & FilesSdkConditionalCopyAdapter; } +function conditionalMutationAdapterOf( + adapter: Adapter, +): FilesSdkConditionalMutationAdapter | undefined { + if ( + typeof adapter !== 'object' || + adapter === null || + !('conditionalMutation' in adapter) || + !('uploadConditional' in adapter) || + !('deleteConditional' in adapter) + ) { + return undefined; + } + const candidate = adapter as Adapter & + Partial; + const capability = candidate.conditionalMutation; + if ( + capability === undefined || + typeof capability.create !== 'boolean' || + typeof capability.replace !== 'boolean' || + typeof capability.delete !== 'boolean' || + typeof capability.etag !== 'boolean' || + typeof candidate.uploadConditional !== 'function' || + typeof candidate.deleteConditional !== 'function' + ) { + return undefined; + } + return candidate as Adapter & FilesSdkConditionalMutationAdapter; +} + function signedUploadPolicyAdapterOf( adapter: Adapter, ): FilesSdkSignedUploadPolicyAdapter | undefined { @@ -281,6 +327,41 @@ function mapRetryOptions( }; } +function storageRetryOptions( + retries: RetryOptions | undefined, +): StorageRetryOptions | undefined { + if (retries === undefined || typeof retries === 'number') { + return retries; + } + return { + max: retries.max, + ...(retries.backoff !== undefined && { + backoff: ({ attempt, error }) => + retries.backoff?.({ + attempt, + error: new FilesError( + error.code === StorageErrorCode.NOT_FOUND + ? 'NotFound' + : error.code === StorageErrorCode.UNAUTHORIZED + ? 'Unauthorized' + : error.code === StorageErrorCode.CONFLICT + ? 'Conflict' + : error.code === StorageErrorCode.READ_ONLY + ? 'ReadOnly' + : 'Provider', + error.message, + error, + { + aborted: error.aborted, + permanent: error.permanent, + timedOut: error.timedOut, + }, + ), + }) ?? 0, + }), + }; +} + function operationOptions( options?: StorageOperationOptions, ): OperationOptions | undefined { @@ -507,6 +588,12 @@ export class FilesSdkStorageDriver< readonly #files: Files; readonly #name: string; readonly #conditionalCopy: FilesSdkConditionalCopyAdapter | undefined; + readonly #conditionalMutation: FilesSdkConditionalMutationAdapter | undefined; + readonly #prefix: string; + readonly #readOnly: boolean; + readonly #retries: StorageRetryOptions | undefined; + readonly #signal: AbortSignal | undefined; + readonly #timeout: number | undefined; readonly #signedUploadPolicy: FilesSdkSignedUploadPolicyAdapter | undefined; readonly #signedDownloadPolicy: FilesSdkSignedDownloadPolicyAdapter | undefined; @@ -515,6 +602,12 @@ export class FilesSdkStorageDriver< this.#files = new Files(options); this.#name = options.adapter.name; this.#conditionalCopy = conditionalCopyAdapterOf(options.adapter); + this.#conditionalMutation = conditionalMutationAdapterOf(options.adapter); + this.#prefix = this.#files.prefix; + this.#readOnly = options.readonly === true; + this.#retries = storageRetryOptions(options.retries); + this.#signal = options.signal; + this.#timeout = options.timeout; this.#signedUploadPolicy = signedUploadPolicyAdapterOf(options.adapter); this.#signedDownloadPolicy = signedDownloadPolicyAdapterOf(options.adapter); } @@ -535,6 +628,12 @@ export class FilesSdkStorageDriver< ...(this.#conditionalCopy !== undefined && { conditionalCopy: { ...this.#conditionalCopy.conditionalCopy }, }), + ...(this.#conditionalMutation !== undefined && + !this.#readOnly && { + conditionalMutation: { + ...this.#conditionalMutation.conditionalMutation, + }, + }), signedDownload: { ...capabilities.signedUrl }, ...(this.#signedDownloadPolicy !== undefined && { signedDownloadPolicy: { @@ -561,6 +660,61 @@ export class FilesSdkStorageDriver< ); } + uploadConditional( + key: string, + body: StorageBody, + options: StorageConditionalUploadOptions, + ): Promise { + if (this.#readOnly) { + return Promise.reject( + new StorageError( + `Cannot call uploadConditional() on a read-only storage adapter.`, + { + code: StorageErrorCode.READ_ONLY, + key, + operation: 'upload', + permanent: true, + }, + ), + ); + } + const adapter = this.#conditionalMutation; + if (adapter === undefined) { + return Promise.reject( + new StorageError( + `Storage adapter "${this.#name}" does not support conditional upload.`, + { + code: StorageErrorCode.NOT_SUPPORTED, + key, + operation: 'upload', + permanent: true, + }, + ), + ); + } + const mergedOptions = this.#conditionalOptions(options); + return this.#call(async () => { + const physicalKey = this.#path(key); + const result = await adapter.uploadConditional( + physicalKey, + mapBody(body), + mergedOptions, + ); + if (result.key !== physicalKey) { + throw new StorageError( + 'Storage adapter returned an unexpected conditional upload key.', + { + code: StorageErrorCode.PROVIDER, + key, + operation: 'upload', + permanent: true, + }, + ); + } + return { ...result, key }; + }); + } + async download( key: string, options?: StorageDownloadOptions, @@ -593,6 +747,43 @@ export class FilesSdkStorageDriver< }); } + deleteConditional( + key: string, + options: StorageConditionalDeleteOptions, + ): Promise { + if (this.#readOnly) { + return Promise.reject( + new StorageError( + `Cannot call deleteConditional() on a read-only storage adapter.`, + { + code: StorageErrorCode.READ_ONLY, + key, + operation: 'delete', + permanent: true, + }, + ), + ); + } + const adapter = this.#conditionalMutation; + if (adapter === undefined) { + return Promise.reject( + new StorageError( + `Storage adapter "${this.#name}" does not support conditional delete.`, + { + code: StorageErrorCode.NOT_SUPPORTED, + key, + operation: 'delete', + permanent: true, + }, + ), + ); + } + const mergedOptions = this.#conditionalOptions(options); + return this.#call(() => + adapter.deleteConditional(this.#path(key), mergedOptions), + ); + } + copy( sourceKey: string, destinationKey: string, @@ -698,6 +889,53 @@ export class FilesSdkStorageDriver< throw mapFilesSdkError(error); } } + + #path(key: string): string { + if (typeof key !== 'string' || key.length === 0) { + throw new StorageError('key must be a non-empty string.', { + code: StorageErrorCode.INVALID_ARGUMENT, + permanent: true, + }); + } + if (key.includes('\0')) { + throw new StorageError('key must not contain null bytes.', { + code: StorageErrorCode.INVALID_ARGUMENT, + key, + permanent: true, + }); + } + if (this.#prefix.length === 0) { + return key; + } + const normalized = key.replace(/^\/+/u, ''); + if ( + normalized + .split('/') + .some((segment) => segment === '.' || segment === '..') + ) { + throw new StorageError('key must not contain . or .. path segments.', { + code: StorageErrorCode.INVALID_ARGUMENT, + key, + permanent: true, + }); + } + return `${this.#prefix}/${normalized}`; + } + + #conditionalOptions< + Options extends + StorageConditionalUploadOptions | StorageConditionalDeleteOptions, + >(options: Options): Options { + return { + ...options, + ...(options.retries === undefined && + this.#retries !== undefined && { retries: this.#retries }), + ...(options.signal === undefined && + this.#signal !== undefined && { signal: this.#signal }), + ...(options.timeout === undefined && + this.#timeout !== undefined && { timeout: this.#timeout }), + }; + } } export function createFilesSdkDriver( diff --git a/src/files-sdk/fs/fs.driver.spec.ts b/src/files-sdk/fs/fs.driver.spec.ts index f544945..8c84d86 100644 --- a/src/files-sdk/fs/fs.driver.spec.ts +++ b/src/files-sdk/fs/fs.driver.spec.ts @@ -1,4 +1,13 @@ -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -75,4 +84,203 @@ describe('createFsStorageDriver', () => { code: StorageErrorCode.NOT_FOUND, }); }); + + it('rejects reads through a symlink that aliases another mounted prefix', async () => { + mkdirSync(join(root, 'scope'), { recursive: true }); + mkdirSync(join(root, 'outside'), { recursive: true }); + writeFileSync(join(root, 'outside/secret.txt'), 'secret'); + symlinkSync( + join(root, 'outside/secret.txt'), + join(root, 'scope/link.txt'), + 'file', + ); + const client = new StorageClient( + 'artifacts', + createFsStorageDriver({ adapter: { root } }), + ); + + await expect(client.downloadText('scope/link.txt')).rejects.toMatchObject({ + code: StorageErrorCode.INVALID_ARGUMENT, + }); + await expect(client.head('scope/link.txt')).rejects.toMatchObject({ + code: StorageErrorCode.INVALID_ARGUMENT, + }); + await expect(client.exists('scope/link.txt')).rejects.toMatchObject({ + code: StorageErrorCode.INVALID_ARGUMENT, + }); + }); + + it('rejects reads through a hard link that aliases another mounted prefix', async () => { + mkdirSync(join(root, 'scope'), { recursive: true }); + mkdirSync(join(root, 'outside'), { recursive: true }); + writeFileSync(join(root, 'outside/secret.txt'), 'secret'); + linkSync(join(root, 'outside/secret.txt'), join(root, 'scope/link.txt')); + const client = new StorageClient( + 'artifacts', + createFsStorageDriver({ adapter: { root } }), + ); + + await expect(client.downloadText('scope/link.txt')).rejects.toMatchObject({ + code: StorageErrorCode.INVALID_ARGUMENT, + }); + await expect(client.head('scope/link.txt')).rejects.toMatchObject({ + code: StorageErrorCode.INVALID_ARGUMENT, + }); + }); + + it('supports create, replace, and delete with exact ETag preconditions', async () => { + const client = new StorageClient( + 'artifacts', + createFsStorageDriver({ adapter: { root } }), + ); + + expect(client.capabilities.conditionalMutation).toEqual({ + create: true, + delete: true, + etag: true, + replace: true, + }); + const created = await client.uploadConditional('note.txt', 'first', { + condition: { type: 'create' }, + }); + + await expect( + client.uploadConditional('note.txt', 'duplicate', { + condition: { type: 'create' }, + }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + await expect( + client.uploadConditional('note.txt', 'wrong', { + condition: { etag: 'wrong-etag', type: 'replace' }, + }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + await expect(client.downloadText('note.txt')).resolves.toBe('first'); + + const replaced = await client.uploadConditional('note.txt', 'second', { + condition: { etag: created.etag ?? '', type: 'replace' }, + }); + await expect(client.downloadText('note.txt')).resolves.toBe('second'); + await expect( + client.deleteConditional('note.txt', { + condition: { etag: created.etag ?? '' }, + }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + + await client.deleteConditional('note.txt', { + condition: { etag: replaced.etag ?? '' }, + }); + await expect(client.exists('note.txt')).resolves.toBe(false); + }); + + it('serializes conditional creates across drivers for the same root', async () => { + const first = new StorageClient( + 'first', + createFsStorageDriver({ adapter: { root } }), + ); + const second = new StorageClient( + 'second', + createFsStorageDriver({ adapter: { root } }), + ); + + const settled = await Promise.allSettled([ + first.uploadConditional('race.txt', 'first', { + condition: { type: 'create' }, + }), + second.uploadConditional('race.txt', 'second', { + condition: { type: 'create' }, + }), + ]); + + expect( + settled.filter((result) => result.status === 'fulfilled'), + ).toHaveLength(1); + expect( + settled.filter((result) => result.status === 'rejected'), + ).toMatchObject([{ reason: { code: StorageErrorCode.CONFLICT } }]); + }); + + it('does not advertise conditional mutation from a readonly filesystem driver', () => { + const driver = createFsStorageDriver({ + adapter: { root }, + readonly: true, + }); + + expect(driver.capabilities.conditionalMutation).toBeUndefined(); + }); + + it('rejects a parent symlink during conditional create', async () => { + const outside = mkdtempSync(join(tmpdir(), 'nestm-storage-outside-')); + try { + symlinkSync(outside, join(root, 'link'), 'dir'); + const client = new StorageClient( + 'artifacts', + createFsStorageDriver({ adapter: { root } }), + ); + + await expect( + client.uploadConditional('link/new.txt', 'escaped', { + condition: { type: 'create' }, + }), + ).rejects.toMatchObject({ code: StorageErrorCode.INVALID_ARGUMENT }); + expect(existsSync(join(outside, 'new.txt'))).toBe(false); + } finally { + rmSync(outside, { force: true, recursive: true }); + } + }); + + it('rejects a body symlink during conditional replace', async () => { + const outside = mkdtempSync(join(tmpdir(), 'nestm-storage-outside-')); + try { + const outsideFile = join(outside, 'target.txt'); + writeFileSync(outsideFile, 'outside'); + symlinkSync(outsideFile, join(root, 'note.txt'), 'file'); + writeFileSync( + join(root, 'note.txt.meta.json'), + JSON.stringify({ + contentType: 'text/plain', + etag: 'outside-etag', + lastModified: Date.now(), + }), + ); + const client = new StorageClient( + 'artifacts', + createFsStorageDriver({ adapter: { root } }), + ); + + await expect( + client.uploadConditional('note.txt', 'escaped', { + condition: { etag: 'outside-etag', type: 'replace' }, + }), + ).rejects.toMatchObject({ code: StorageErrorCode.INVALID_ARGUMENT }); + expect(readFileSync(outsideFile, 'utf8')).toBe('outside'); + } finally { + rmSync(outside, { force: true, recursive: true }); + } + }); + + it('rejects a sidecar symlink during conditional delete', async () => { + const outside = mkdtempSync(join(tmpdir(), 'nestm-storage-outside-')); + try { + const outsideSidecar = join(outside, 'metadata.json'); + writeFileSync(outsideSidecar, '{"etag":"outside-etag"}'); + writeFileSync(join(root, 'note.txt'), 'inside'); + symlinkSync(outsideSidecar, join(root, 'note.txt.meta.json'), 'file'); + const client = new StorageClient( + 'artifacts', + createFsStorageDriver({ adapter: { root } }), + ); + + await expect( + client.deleteConditional('note.txt', { + condition: { etag: 'outside-etag' }, + }), + ).rejects.toMatchObject({ code: StorageErrorCode.INVALID_ARGUMENT }); + expect(readFileSync(join(root, 'note.txt'), 'utf8')).toBe('inside'); + expect(readFileSync(outsideSidecar, 'utf8')).toBe( + '{"etag":"outside-etag"}', + ); + } finally { + rmSync(outside, { force: true, recursive: true }); + } + }); }); diff --git a/src/files-sdk/fs/index.ts b/src/files-sdk/fs/index.ts index 8ad6706..14360c1 100644 --- a/src/files-sdk/fs/index.ts +++ b/src/files-sdk/fs/index.ts @@ -1,7 +1,28 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { constants as fsConstants, type Stats } from 'node:fs'; +import * as fsp from 'node:fs/promises'; +import path from 'node:path'; + +import type { + Body, + DownloadOptions, + ListOptions, + ListResult, + OperationOptions, + StoredFile, +} from 'files-sdk'; import { fs, type FsAdapter, type FsAdapterOptions } from 'files-sdk/fs'; +import { StorageError, StorageErrorCode } from '../../storage.error.js'; +import type { + StorageConditionalDeleteOptions, + StorageConditionalUploadOptions, + StorageOperationOptions, + StorageUploadResult, +} from '../../storage.types.js'; import { createFilesSdkDriver, + type FilesSdkConditionalMutationAdapter, type FilesSdkDriverOptions, type FilesSdkStorageDriver, } from '../files-sdk.driver.js'; @@ -13,6 +34,740 @@ export interface FsStorageDriverOptions extends Omit< adapter: FsAdapterOptions; } +export type FsStorageAdapter = FsAdapter & FilesSdkConditionalMutationAdapter; + +const SIDECAR_SUFFIX = '.meta.json'; +const TEMP_SUFFIX = '.fls-part'; +const MAX_SIDECAR_BYTES = 1024 * 1024; + +interface FsSidecar { + contentType: string; + cacheControl?: string; + metadata?: Record; + etag: string; + lastModified: number; +} + +interface OperationRuntime { + signal: AbortSignal | undefined; + timeoutSignal: AbortSignal | undefined; + callerSignal: AbortSignal | undefined; +} + +function fsErrorCode(error: unknown): string | undefined { + if (typeof error === 'object' && error !== null && 'code' in error) { + return typeof error.code === 'string' ? error.code : undefined; + } + return undefined; +} + +function storageFsError( + error: unknown, + key: string, + operation: 'upload' | 'delete', +): StorageError { + if (error instanceof StorageError) { + return error; + } + const systemCode = fsErrorCode(error); + const code = + systemCode === 'ENOENT' || systemCode === 'ENOTDIR' + ? StorageErrorCode.NOT_FOUND + : systemCode === 'EEXIST' + ? StorageErrorCode.CONFLICT + : systemCode === 'EACCES' || systemCode === 'EPERM' + ? StorageErrorCode.UNAUTHORIZED + : StorageErrorCode.PROVIDER; + return new StorageError(`Filesystem ${operation} failed for "${key}".`, { + cause: error, + code, + key, + operation, + permanent: + code === StorageErrorCode.NOT_FOUND || + code === StorageErrorCode.CONFLICT || + code === StorageErrorCode.UNAUTHORIZED, + }); +} + +function invalidFsPath(key: string, message: string): never { + throw new StorageError(`Invalid filesystem storage key: ${message}.`, { + code: StorageErrorCode.INVALID_ARGUMENT, + key, + permanent: true, + }); +} + +function strictSegments(key: string): string[] { + if (key.length === 0 || key.includes('\0')) { + invalidFsPath(key, 'the key is empty or contains a null byte'); + } + if (path.isAbsolute(key) || key.includes('\\')) { + invalidFsPath(key, 'only relative POSIX paths are accepted'); + } + const segments = key.split('/'); + if ( + segments.some( + (segment) => segment.length === 0 || segment === '.' || segment === '..', + ) + ) { + invalidFsPath(key, 'empty, dot, and parent segments are not accepted'); + } + if (segments.some((segment) => /[. ]$/u.test(segment))) { + invalidFsPath(key, 'segments ending in a dot or space are not accepted'); + } + const leaf = segments.at(-1)?.toLowerCase() ?? ''; + if (leaf.endsWith(SIDECAR_SUFFIX) || leaf.endsWith(TEMP_SUFFIX)) { + invalidFsPath(key, 'the final segment uses a reserved adapter suffix'); + } + return segments; +} + +function symlinkError(key: string): StorageError { + return new StorageError( + `Filesystem storage key "${key}" crosses a symbolic link.`, + { + code: StorageErrorCode.INVALID_ARGUMENT, + key, + permanent: true, + }, + ); +} + +function unsupportedUpload(key: string, message: string): never { + throw new StorageError(message, { + code: StorageErrorCode.NOT_SUPPORTED, + key, + operation: 'upload', + permanent: true, + }); +} + +function conflict(key: string): never { + throw new StorageError(`Conditional mutation conflicted for "${key}".`, { + code: StorageErrorCode.CONFLICT, + key, + permanent: true, + }); +} + +function notFound(key: string): never { + throw new StorageError(`Storage object "${key}" was not found.`, { + code: StorageErrorCode.NOT_FOUND, + key, + permanent: true, + }); +} + +function runtimeOf(options: StorageOperationOptions): OperationRuntime { + const timeoutSignal = + options.timeout === undefined || options.timeout <= 0 + ? undefined + : AbortSignal.timeout(options.timeout); + return { + callerSignal: options.signal, + signal: + options.signal === undefined + ? timeoutSignal + : timeoutSignal === undefined + ? options.signal + : AbortSignal.any([options.signal, timeoutSignal]), + timeoutSignal, + }; +} + +function assertActive(runtime: OperationRuntime, key: string): void { + if (runtime.signal?.aborted !== true) { + return; + } + const timedOut = + runtime.timeoutSignal?.aborted === true && + runtime.callerSignal?.aborted !== true; + throw new StorageError( + timedOut + ? `Filesystem mutation timed out for "${key}".` + : `Filesystem mutation was aborted for "${key}".`, + { + aborted: true, + cause: runtime.signal.reason, + code: timedOut ? StorageErrorCode.TIMEOUT : StorageErrorCode.ABORTED, + key, + permanent: true, + timedOut, + }, + ); +} + +async function ensureRoot(root: string, key: string): Promise { + await fsp.mkdir(root, { mode: 0o700, recursive: true }); + const stat = await fsp.lstat(root); + if (stat.isSymbolicLink()) { + throw symlinkError(key); + } + if (!stat.isDirectory()) { + invalidFsPath(key, 'the configured root is not a directory'); + } +} + +async function ensureParents( + root: string, + segments: readonly string[], + key: string, + create: boolean, + runtime: OperationRuntime, +): Promise { + await ensureRoot(root, key); + let current = root; + for (const segment of segments.slice(0, -1)) { + assertActive(runtime, key); + current = path.join(current, segment); + let stat: Stats; + try { + stat = await fsp.lstat(current); + } catch (error) { + if (fsErrorCode(error) !== 'ENOENT') { + throw error; + } + if (!create) { + notFound(key); + } + try { + await fsp.mkdir(current, { mode: 0o700 }); + } catch (mkdirError) { + if (fsErrorCode(mkdirError) !== 'EEXIST') { + throw mkdirError; + } + } + stat = await fsp.lstat(current); + } + if (stat.isSymbolicLink()) { + throw symlinkError(key); + } + if (!stat.isDirectory()) { + invalidFsPath(key, 'a parent segment is not a directory'); + } + } + return path.join(root, ...segments); +} + +async function regularFileOrMissing( + target: string, + key: string, +): Promise { + try { + const stat = await fsp.lstat(target); + if (stat.isSymbolicLink()) { + throw symlinkError(key); + } + if (!stat.isFile()) { + invalidFsPath(key, 'an object path is not a regular file'); + } + if (stat.nlink !== 1) { + invalidFsPath(key, 'hard-linked object files are not accepted'); + } + return stat; + } catch (error) { + if (fsErrorCode(error) === 'ENOENT') { + return undefined; + } + throw error; + } +} + +async function assertSymlinkFreeReadPath( + root: string, + key: string, +): Promise { + const segments = strictSegments(key); + let rootStat: Stats; + try { + rootStat = await fsp.lstat(root); + } catch (error) { + if (fsErrorCode(error) === 'ENOENT') { + return; + } + throw error; + } + if (rootStat.isSymbolicLink()) { + throw symlinkError(key); + } + if (!rootStat.isDirectory()) { + invalidFsPath(key, 'the configured root is not a directory'); + } + + let current = root; + for (const [index, segment] of segments.entries()) { + current = path.join(current, segment); + let stat: Stats; + try { + stat = await fsp.lstat(current); + } catch (error) { + if (fsErrorCode(error) === 'ENOENT') { + return; + } + throw error; + } + if (stat.isSymbolicLink()) { + throw symlinkError(key); + } + const leaf = index === segments.length - 1; + if ((!leaf && !stat.isDirectory()) || (leaf && !stat.isFile())) { + invalidFsPath( + key, + leaf + ? 'an object path is not a regular file' + : 'a parent segment is not a directory', + ); + } + if (leaf && stat.nlink !== 1) { + invalidFsPath(key, 'hard-linked object files are not accepted'); + } + } + + const sidecar = await regularFileOrMissing(current + SIDECAR_SUFFIX, key); + if (sidecar?.isSymbolicLink()) { + throw symlinkError(key); + } +} + +async function readSidecar(target: string, key: string): Promise { + const handle = await fsp.open( + target, + fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW, + ); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size > MAX_SIDECAR_BYTES) { + throw new StorageError(`Filesystem metadata is invalid for "${key}".`, { + code: StorageErrorCode.PROVIDER, + key, + permanent: true, + }); + } + const parsed: unknown = JSON.parse(await handle.readFile('utf8')); + if ( + typeof parsed !== 'object' || + parsed === null || + !('etag' in parsed) || + typeof parsed.etag !== 'string' || + !('contentType' in parsed) || + typeof parsed.contentType !== 'string' || + !('lastModified' in parsed) || + typeof parsed.lastModified !== 'number' + ) { + throw new StorageError(`Filesystem metadata is invalid for "${key}".`, { + code: StorageErrorCode.PROVIDER, + key, + permanent: true, + }); + } + return parsed as FsSidecar; + } finally { + await handle.close(); + } +} + +function bodyBytes(body: Body): Uint8Array | undefined { + if (typeof body === 'string') { + return new TextEncoder().encode(body); + } + if (body instanceof Uint8Array) { + return body; + } + if (body instanceof ArrayBuffer) { + return new Uint8Array(body); + } + if (ArrayBuffer.isView(body)) { + return new Uint8Array(body.buffer, body.byteOffset, body.byteLength); + } + return undefined; +} + +function defaultContentType(body: Body, override: string | undefined): string { + if (override !== undefined) { + return override; + } + if (typeof body === 'string') { + return 'text/plain; charset=utf-8'; + } + if (body instanceof Blob && body.type.length > 0) { + return body.type; + } + return 'application/octet-stream'; +} + +async function writeAll( + handle: fsp.FileHandle, + bytes: Uint8Array, +): Promise { + let offset = 0; + while (offset < bytes.byteLength) { + const { bytesWritten } = await handle.write( + bytes, + offset, + bytes.byteLength - offset, + ); + offset += bytesWritten; + } +} + +function tempPath(directory: string): string { + return path.join(directory, `.nestm-${randomUUID()}${TEMP_SUFFIX}`); +} + +async function openTemp(directory: string): Promise<{ + handle: fsp.FileHandle; + path: string; +}> { + const target = tempPath(directory); + const handle = await fsp.open( + target, + fsConstants.O_CREAT | + fsConstants.O_EXCL | + fsConstants.O_WRONLY | + fsConstants.O_NOFOLLOW, + 0o600, + ); + return { handle, path: target }; +} + +async function removeTemp(target: string | undefined): Promise { + if (target === undefined) { + return; + } + try { + await fsp.unlink(target); + } catch (error) { + if (fsErrorCode(error) !== 'ENOENT') { + throw error; + } + } +} + +async function stageBody( + directory: string, + key: string, + body: Body, + options: StorageConditionalUploadOptions, + runtime: OperationRuntime, +): Promise<{ etag: string; path: string; size: number }> { + const staged = await openTemp(directory); + const hash = createHash('sha256'); + let size = 0; + const knownBytes = bodyBytes(body); + const knownSize = + knownBytes?.byteLength ?? (body instanceof Blob ? body.size : undefined); + options.onProgress?.({ + loaded: 0, + ...(knownSize !== undefined && { total: knownSize }), + }); + try { + if (knownBytes !== undefined) { + assertActive(runtime, key); + await writeAll(staged.handle, knownBytes); + hash.update(knownBytes); + size = knownBytes.byteLength; + options.onProgress?.({ loaded: size, total: size }); + } else if (body instanceof Blob) { + const bytes = new Uint8Array(await body.arrayBuffer()); + assertActive(runtime, key); + await writeAll(staged.handle, bytes); + hash.update(bytes); + size = bytes.byteLength; + options.onProgress?.({ loaded: size, total: size }); + } else { + const reader = (body as ReadableStream).getReader(); + try { + while (true) { + assertActive(runtime, key); + const chunk = await reader.read(); + if (chunk.done) { + break; + } + await writeAll(staged.handle, chunk.value); + hash.update(chunk.value); + size += chunk.value.byteLength; + options.onProgress?.({ loaded: size }); + } + } catch (error) { + await reader.cancel(error).catch(() => undefined); + throw error; + } finally { + reader.releaseLock(); + } + } + await staged.handle.sync(); + await staged.handle.close(); + return { + etag: `"${hash.digest('hex')}"`, + path: staged.path, + size, + }; + } catch (error) { + await staged.handle.close().catch(() => undefined); + await removeTemp(staged.path).catch(() => undefined); + throw error; + } +} + +async function stageSidecar( + directory: string, + sidecar: FsSidecar, +): Promise { + const staged = await openTemp(directory); + try { + await writeAll( + staged.handle, + new TextEncoder().encode(JSON.stringify(sidecar)), + ); + await staged.handle.sync(); + await staged.handle.close(); + return staged.path; + } catch (error) { + await staged.handle.close().catch(() => undefined); + await removeTemp(staged.path).catch(() => undefined); + throw error; + } +} + +const CONDITIONAL_FS_TAILS = new Map>(); + +async function serializeFsKey( + lockKey: string, + operation: () => Promise, +): Promise { + const previous = CONDITIONAL_FS_TAILS.get(lockKey) ?? Promise.resolve(); + let release = (): void => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + const tail = previous.then(() => gate); + CONDITIONAL_FS_TAILS.set(lockKey, tail); + await previous; + try { + return await operation(); + } finally { + release(); + if (CONDITIONAL_FS_TAILS.get(lockKey) === tail) { + CONDITIONAL_FS_TAILS.delete(lockKey); + } + } +} + +/** + * Adds symlink-defended reads and process-local conditional mutations to a + * files-sdk fs adapter. + * + * The configured root must be dedicated to this driver: no other process and + * no unconditional filesystem writer may mutate it concurrently. Each call + * rejects symlinks in the existing key path and uses exclusive same-directory + * temporary files plus atomic renames, but Node has no portable `openat2` + * equivalent that could make those checks safe against a hostile concurrent + * tree rewrite. This is a CAS implementation for an exclusively-owned local + * workspace, not a host-filesystem sandbox. Body and metadata sidecars are two + * files, so a process crash between their renames can still require cleanup. + */ +export function withFsConditionalMutation(base: FsAdapter): FsStorageAdapter { + const root = path.resolve(base.root); + const download = base.download.bind(base); + const exists = base.exists.bind(base); + const head = base.head.bind(base); + const list = base.list.bind(base); + const serialize = ( + key: string, + operation: () => Promise, + ): Promise => + serializeFsKey(`${root}\0${key.normalize('NFC').toLowerCase()}`, operation); + + return Object.assign(base, { + conditionalMutation: Object.freeze({ + create: true, + delete: true, + etag: true, + replace: true, + }), + async download( + key: string, + options?: DownloadOptions, + ): Promise { + await assertSymlinkFreeReadPath(root, key); + return download(key, options); + }, + async exists(key: string, options?: OperationOptions): Promise { + await assertSymlinkFreeReadPath(root, key); + return exists(key, options); + }, + async head(key: string, options?: OperationOptions): Promise { + await assertSymlinkFreeReadPath(root, key); + return head(key, options); + }, + async list(options?: ListOptions): Promise { + const result = await list(options); + for (const item of result.items) { + await assertSymlinkFreeReadPath(root, item.key); + } + return result; + }, + async deleteConditional( + key: string, + options: StorageConditionalDeleteOptions, + ): Promise { + return serialize(key, async () => { + const runtime = runtimeOf(options); + try { + const segments = strictSegments(key); + const bodyPath = await ensureParents( + root, + segments, + key, + false, + runtime, + ); + const sidecarPath = bodyPath + SIDECAR_SUFFIX; + if ( + (await regularFileOrMissing(bodyPath, key)) === undefined || + (await regularFileOrMissing(sidecarPath, key)) === undefined + ) { + notFound(key); + } + const sidecar = await readSidecar(sidecarPath, key); + if (sidecar.etag !== options.condition.etag) { + conflict(key); + } + + assertActive(runtime, key); + await ensureParents(root, segments, key, false, runtime); + if ( + (await regularFileOrMissing(bodyPath, key)) === undefined || + (await regularFileOrMissing(sidecarPath, key)) === undefined || + (await readSidecar(sidecarPath, key)).etag !== + options.condition.etag + ) { + conflict(key); + } + await fsp.unlink(bodyPath); + try { + await fsp.unlink(sidecarPath); + } catch (error) { + throw storageFsError(error, key, 'delete'); + } + } catch (error) { + throw storageFsError(error, key, 'delete'); + } + }); + }, + async uploadConditional( + key: string, + body: Body, + options: StorageConditionalUploadOptions, + ): Promise { + return serialize(key, async () => { + let bodyTemp: string | undefined; + let sidecarTemp: string | undefined; + const runtime = runtimeOf(options); + try { + if (options.multipart !== undefined && options.multipart !== false) { + unsupportedUpload( + key, + 'Conditional filesystem uploads do not support multipart mode.', + ); + } + if (options.control !== undefined) { + unsupportedUpload( + key, + 'Conditional filesystem uploads do not support resumable control.', + ); + } + + const segments = strictSegments(key); + const bodyPath = await ensureParents( + root, + segments, + key, + true, + runtime, + ); + const sidecarPath = bodyPath + SIDECAR_SUFFIX; + const existingBody = await regularFileOrMissing(bodyPath, key); + const existingSidecar = await regularFileOrMissing(sidecarPath, key); + if (options.condition.type === 'create') { + if (existingBody !== undefined || existingSidecar !== undefined) { + conflict(key); + } + } else { + if (existingBody === undefined || existingSidecar === undefined) { + notFound(key); + } + if ( + (await readSidecar(sidecarPath, key)).etag !== + options.condition.etag + ) { + conflict(key); + } + } + + const staged = await stageBody( + path.dirname(bodyPath), + key, + body, + options, + runtime, + ); + bodyTemp = staged.path; + const lastModified = Date.now(); + const contentType = defaultContentType(body, options.contentType); + sidecarTemp = await stageSidecar(path.dirname(bodyPath), { + contentType, + etag: staged.etag, + lastModified, + ...(options.cacheControl !== undefined && { + cacheControl: options.cacheControl, + }), + ...(options.metadata !== undefined && { + metadata: options.metadata, + }), + }); + + assertActive(runtime, key); + await ensureParents(root, segments, key, false, runtime); + const currentBody = await regularFileOrMissing(bodyPath, key); + const currentSidecar = await regularFileOrMissing(sidecarPath, key); + if (options.condition.type === 'create') { + if (currentBody !== undefined || currentSidecar !== undefined) { + conflict(key); + } + } else if ( + currentBody === undefined || + currentSidecar === undefined || + (await readSidecar(sidecarPath, key)).etag !== + options.condition.etag + ) { + conflict(key); + } + + await fsp.rename(bodyTemp, bodyPath); + bodyTemp = undefined; + await fsp.rename(sidecarTemp, sidecarPath); + sidecarTemp = undefined; + return { + contentType, + etag: staged.etag, + key, + lastModified: new Date(lastModified), + size: staged.size, + }; + } catch (error) { + throw storageFsError(error, key, 'upload'); + } finally { + await removeTemp(bodyTemp).catch(() => undefined); + await removeTemp(sidecarTemp).catch(() => undefined); + } + }); + }, + } satisfies FilesSdkConditionalMutationAdapter & + Pick); +} + /** * Creates the files-sdk filesystem driver from the storage package's own * dependency context. The adapter reaches only `node:fs`, so this entry point @@ -24,12 +779,19 @@ export interface FsStorageDriverOptions extends Omit< * put. Sidecars never surface as keys: `list` and `search` skip them, and * uploading a key that ends in `.meta.json` fails closed rather than colliding * with one. + * + * Conditional mutations additionally require an exclusively-owned root. See + * {@link withFsConditionalMutation}; use an OS sandbox as well when running an + * agent that has direct shell or filesystem tools. */ export function createFsStorageDriver( options: FsStorageDriverOptions, -): FilesSdkStorageDriver { +): FilesSdkStorageDriver { const { adapter: adapterOptions, ...filesOptions } = options; - return createFilesSdkDriver({ ...filesOptions, adapter: fs(adapterOptions) }); + return createFilesSdkDriver({ + ...filesOptions, + adapter: withFsConditionalMutation(fs(adapterOptions)), + }); } export { fs, mapFsError } from 'files-sdk/fs'; diff --git a/src/files-sdk/index.ts b/src/files-sdk/index.ts index b75c051..aa4387f 100644 --- a/src/files-sdk/index.ts +++ b/src/files-sdk/index.ts @@ -2,6 +2,7 @@ export { FilesSdkStorageDriver, createFilesSdkDriver, type FilesSdkConditionalCopyAdapter, + type FilesSdkConditionalMutationAdapter, type FilesSdkSignedUploadPolicyAdapter, type FilesSdkSignedDownloadPolicyAdapter, type FilesSdkDriverOptions, diff --git a/src/files-sdk/provider/index.ts b/src/files-sdk/provider/index.ts index 1fb2ef2..b5a5f4b 100644 --- a/src/files-sdk/provider/index.ts +++ b/src/files-sdk/provider/index.ts @@ -83,6 +83,12 @@ async function resolveAdapter( throw mapFilesSdkError(error); } const { adapter } = resolved.files; + if (provider === 'fs') { + const { withFsConditionalMutation } = await import('../fs/index.js'); + return withFsConditionalMutation( + adapter as Parameters[0], + ); + } if (provider !== 's3') { return adapter; } @@ -92,6 +98,9 @@ async function resolveAdapter( return withS3Capabilities( adapter as Parameters[0], { + ...(typeof config?.endpoint === 'string' && { + endpoint: config.endpoint, + }), ...(config?.publicBaseUrl !== undefined && { publicBaseUrl: config.publicBaseUrl, }), diff --git a/src/files-sdk/provider/provider.driver.spec.ts b/src/files-sdk/provider/provider.driver.spec.ts index 127e2d8..1ac5078 100644 --- a/src/files-sdk/provider/provider.driver.spec.ts +++ b/src/files-sdk/provider/provider.driver.spec.ts @@ -88,12 +88,33 @@ describe('createProviderStorageDriver', () => { supported: true, version: true, }); + expect(driver.capabilities.conditionalMutation).toEqual({ + create: true, + delete: true, + etag: true, + replace: true, + }); expect(driver.capabilities.signedUploadPolicy).toEqual({ contentType: true, sizeRange: true, }); }); + it('does not advertise conditional mutation for an unverified S3-compatible endpoint', async () => { + const driver = await createProviderStorageDriver({ + config: { + accessKeyId: 'test', + bucket: 'artifacts', + endpoint: 'https://objects.example.test', + region: 'us-east-1', + secretAccessKey: 'test', + }, + provider: 's3', + }); + + expect(driver.capabilities.conditionalMutation).toBeUndefined(); + }); + it('claims no conditional copy for a provider that does not declare it', async () => { const driver = await createProviderStorageDriver({ config: { root }, @@ -101,6 +122,12 @@ describe('createProviderStorageDriver', () => { }); expect(driver.capabilities.conditionalCopy).toBeUndefined(); + expect(driver.capabilities.conditionalMutation).toEqual({ + create: true, + delete: true, + etag: true, + replace: true, + }); }); it('rejects an unknown slug before importing anything', async () => { diff --git a/src/files-sdk/s3/index.ts b/src/files-sdk/s3/index.ts index bd014ae..efc5293 100644 --- a/src/files-sdk/s3/index.ts +++ b/src/files-sdk/s3/index.ts @@ -1,4 +1,9 @@ -import { CopyObjectCommand } from '@aws-sdk/client-s3'; +import { + CopyObjectCommand, + DeleteObjectCommand, + PutObjectCommand, +} from '@aws-sdk/client-s3'; +import { Readable } from 'node:stream'; import { mapS3Error, s3, @@ -6,10 +11,19 @@ import { type S3AdapterOptions, } from 'files-sdk/s3'; -import type { StoragePromotionOptions } from '../../storage.types.js'; +import type { + StorageBody, + StorageConditionalDeleteOptions, + StorageConditionalUploadOptions, + StorageOperationOptions, + StoragePromotionOptions, + StorageUploadResult, +} from '../../storage.types.js'; +import { StorageError, StorageErrorCode } from '../../storage.error.js'; import { createFilesSdkDriver, type FilesSdkConditionalCopyAdapter, + type FilesSdkConditionalMutationAdapter, type FilesSdkDriverOptions, type FilesSdkSignedUploadPolicyAdapter, type FilesSdkSignedDownloadPolicyAdapter, @@ -22,13 +36,22 @@ export interface S3StorageDriverOptions extends Omit< 'adapter' > { adapter: S3AdapterOptions; + /** + * Opt in or out of conditional writes for an S3-compatible endpoint whose + * support is known. Native AWS S3 enables them by default; custom endpoints + * default to false because compatibility varies. + */ + conditionalMutation?: boolean; } -export type S3StorageAdapter = S3Adapter & +type S3StorageAdapterBase = S3Adapter & FilesSdkConditionalCopyAdapter & FilesSdkSignedDownloadPolicyAdapter & FilesSdkSignedUploadPolicyAdapter; +export type S3StorageAdapter = S3StorageAdapterBase & + Partial; + function copySource( bucket: string, key: string, @@ -41,7 +64,7 @@ function copySource( } function operationSignal( - options: StoragePromotionOptions, + options: StorageOperationOptions, ): AbortSignal | undefined { const timeoutSignal = options.timeout === undefined || options.timeout <= 0 @@ -55,7 +78,7 @@ function operationSignal( : AbortSignal.any([options.signal, timeoutSignal]); } -function maxRetries(options: StoragePromotionOptions): number { +function maxRetries(options: StorageOperationOptions): number { const configured = typeof options.retries === 'number' ? options.retries @@ -63,6 +86,124 @@ function maxRetries(options: StoragePromotionOptions): number { return Math.max(0, Math.floor(configured ?? 0)); } +function etagHeader(etag: string): string { + return etag.startsWith('"') && etag.endsWith('"') ? etag : `"${etag}"`; +} + +function stripEtag(etag: string | undefined): string | undefined { + return etag?.replace(/^"+|"+$/gu, ''); +} + +function contentTypeOf( + body: StorageBody, + override: string | undefined, +): string { + if (override !== undefined) { + return override; + } + if (typeof body === 'string') { + return 'text/plain; charset=utf-8'; + } + if (body instanceof Blob && body.type.length > 0) { + return body.type; + } + return 'application/octet-stream'; +} + +async function normalizeConditionalBody( + body: StorageBody, + onProgress: StorageConditionalUploadOptions['onProgress'], +): Promise<{ + body: Uint8Array | ReadableStream; + size: () => number; + contentLength?: number; +}> { + let bytes: Uint8Array | undefined; + if (typeof body === 'string') { + bytes = new TextEncoder().encode(body); + } else if (body instanceof Uint8Array) { + bytes = body; + } else if (body instanceof ArrayBuffer) { + bytes = new Uint8Array(body); + } else if (ArrayBuffer.isView(body)) { + bytes = new Uint8Array(body.buffer, body.byteOffset, body.byteLength); + } else if (body instanceof Blob) { + bytes = new Uint8Array(await body.arrayBuffer()); + } + + if (bytes !== undefined) { + const total = bytes.byteLength; + onProgress?.({ loaded: 0, total }); + return { body: bytes, contentLength: total, size: () => total }; + } + + const stream = + body instanceof Readable + ? (Readable.toWeb(body) as ReadableStream) + : body; + if (!(stream instanceof ReadableStream)) { + throw new StorageError('Unsupported conditional upload body.', { + code: StorageErrorCode.INVALID_ARGUMENT, + operation: 'upload', + permanent: true, + }); + } + const reader = stream.getReader(); + let loaded = 0; + return { + body: new ReadableStream({ + cancel(reason) { + return reader.cancel(reason); + }, + async pull(controller) { + const result = await reader.read(); + if (result.done) { + controller.close(); + return; + } + loaded += result.value.byteLength; + controller.enqueue(result.value); + onProgress?.({ loaded }); + }, + }), + size: () => loaded, + }; +} + +async function withS3Retry( + options: StorageOperationOptions, + operation: (signal: AbortSignal | undefined) => Promise, +): Promise { + const retries = maxRetries(options); + for (let attempt = 0; ; attempt += 1) { + const signal = operationSignal(options); + try { + return await operation(signal); + } catch (error) { + const mapped = mapS3Error(error); + if ( + attempt >= retries || + mapped.code !== 'Provider' || + mapped.aborted || + mapped.permanent || + signal?.aborted === true + ) { + throw mapped; + } + const storageError = mapFilesSdkError(mapped); + const delay = + typeof options.retries === 'object' && + options.retries.backoff !== undefined + ? options.retries.backoff({ + attempt: attempt + 1, + error: storageError, + }) + : Math.min(1000, 100 * 2 ** attempt); + await waitForRetry(delay, options.signal); + } + } +} + async function waitForRetry( milliseconds: number, signal: AbortSignal | undefined, @@ -100,9 +241,13 @@ async function waitForRetry( */ export function withS3Capabilities( base: S3Adapter, - options: Pick = {}, + options: Pick & { + conditionalMutation?: boolean; + } = {}, ): S3StorageAdapter { - return Object.assign(base, { + const supportsConditionalMutation = + options.conditionalMutation ?? options.endpoint === undefined; + const common = Object.assign(base, { conditionalCopy: Object.freeze({ etag: true, supported: true, @@ -120,53 +265,142 @@ export function withS3Capabilities( destinationKey: string, promotion: StoragePromotionOptions, ): Promise { - const retries = maxRetries(promotion); - for (let attempt = 0; ; attempt += 1) { - const signal = operationSignal(promotion); - try { - await base.raw.send( - new CopyObjectCommand({ - Bucket: base.bucket, - CopySource: copySource( - base.bucket, - sourceKey, - promotion.sourceVersion, - ), - ...(promotion.sourceEtag !== undefined && { - CopySourceIfMatch: promotion.sourceEtag, - }), - Key: destinationKey, + await withS3Retry(promotion, async (signal) => { + await base.raw.send( + new CopyObjectCommand({ + Bucket: base.bucket, + CopySource: copySource( + base.bucket, + sourceKey, + promotion.sourceVersion, + ), + ...(promotion.sourceEtag !== undefined && { + CopySourceIfMatch: promotion.sourceEtag, }), - signal === undefined ? undefined : { abortSignal: signal }, - ); - return; - } catch (error) { - const mapped = mapS3Error(error); - if ( - attempt >= retries || - mapped.code !== 'Provider' || - mapped.aborted || - mapped.permanent || - signal?.aborted === true - ) { - throw mapped; - } - const storageError = mapFilesSdkError(mapped); - const delay = - typeof promotion.retries === 'object' && - promotion.retries.backoff !== undefined - ? promotion.retries.backoff({ - attempt: attempt + 1, - error: storageError, - }) - : Math.min(1000, 100 * 2 ** attempt); - await waitForRetry(delay, promotion.signal); - } - } + Key: destinationKey, + }), + signal === undefined ? undefined : { abortSignal: signal }, + ); + }); }, } satisfies FilesSdkConditionalCopyAdapter & FilesSdkSignedDownloadPolicyAdapter & FilesSdkSignedUploadPolicyAdapter); + if (!supportsConditionalMutation) { + return common; + } + return Object.assign(common, { + conditionalMutation: Object.freeze({ + create: true, + delete: true, + etag: true, + replace: true, + }), + async deleteConditional( + key: string, + options: StorageConditionalDeleteOptions, + ): Promise { + await withS3Retry(options, async (signal) => { + await base.raw.send( + new DeleteObjectCommand({ + Bucket: base.bucket, + IfMatch: etagHeader(options.condition.etag), + Key: key, + }), + signal === undefined ? undefined : { abortSignal: signal }, + ); + }); + }, + async uploadConditional( + key: string, + body: StorageBody, + conditional: StorageConditionalUploadOptions, + ): Promise { + if ( + conditional.multipart !== undefined && + conditional.multipart !== false + ) { + throw new StorageError( + 'Conditional S3 uploads do not support multipart mode.', + { + code: StorageErrorCode.NOT_SUPPORTED, + key, + operation: 'upload', + permanent: true, + }, + ); + } + if (conditional.control !== undefined) { + throw new StorageError( + 'Conditional S3 uploads do not support resumable control.', + { + code: StorageErrorCode.NOT_SUPPORTED, + key, + operation: 'upload', + permanent: true, + }, + ); + } + + const normalized = await normalizeConditionalBody( + body, + conditional.onProgress, + ); + const contentType = contentTypeOf(body, conditional.contentType); + const retryOptions: StorageOperationOptions = + normalized.contentLength === undefined + ? { ...conditional, retries: 0 } + : conditional; + const result = await withS3Retry(retryOptions, (signal) => + base.raw.send( + new PutObjectCommand({ + Body: normalized.body, + Bucket: base.bucket, + ...(conditional.cacheControl !== undefined && { + CacheControl: conditional.cacheControl, + }), + ...(normalized.contentLength !== undefined && { + ContentLength: normalized.contentLength, + }), + ContentType: contentType, + ...(conditional.condition.type === 'create' + ? { IfNoneMatch: '*' } + : { IfMatch: etagHeader(conditional.condition.etag) }), + Key: key, + ...(conditional.metadata !== undefined && { + Metadata: conditional.metadata, + }), + }), + signal === undefined ? undefined : { abortSignal: signal }, + ), + ); + const size = normalized.size(); + conditional.onProgress?.({ + loaded: size, + ...(normalized.contentLength !== undefined && { + total: normalized.contentLength, + }), + }); + const etag = stripEtag(result.ETag); + if (etag === undefined || etag.length === 0) { + throw new StorageError( + 'S3 committed a conditional upload without returning the ETag required to identify its result; reconcile the destination before retrying.', + { + code: StorageErrorCode.PROVIDER, + key, + operation: 'upload', + permanent: true, + }, + ); + } + return { + contentType, + etag, + key, + size, + }; + }, + } satisfies FilesSdkConditionalMutationAdapter); } /** @@ -177,9 +411,13 @@ export function createS3StorageDriver( options: S3StorageDriverOptions, ): FilesSdkStorageDriver { const { adapter: adapterOptions, ...filesOptions } = options; + const { conditionalMutation, ...plainFilesOptions } = filesOptions; return createFilesSdkDriver({ - ...filesOptions, - adapter: withS3Capabilities(s3(adapterOptions), adapterOptions), + ...plainFilesOptions, + adapter: withS3Capabilities(s3(adapterOptions), { + ...adapterOptions, + ...(conditionalMutation !== undefined && { conditionalMutation }), + }), }); } diff --git a/src/files-sdk/s3/s3.driver.spec.ts b/src/files-sdk/s3/s3.driver.spec.ts index 3cde238..f4570ac 100644 --- a/src/files-sdk/s3/s3.driver.spec.ts +++ b/src/files-sdk/s3/s3.driver.spec.ts @@ -1,4 +1,9 @@ -import { CopyObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { + CopyObjectCommand, + DeleteObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; import { StorageClient } from '../../storage.client.js'; import { StorageErrorCode } from '../../storage.error.js'; @@ -28,6 +33,12 @@ describe('createS3StorageDriver', () => { supported: true, version: true, }); + expect(client.capabilities.conditionalMutation).toEqual({ + create: true, + delete: true, + etag: true, + replace: true, + }); expect(client.capabilities.signedUploadPolicy).toEqual({ contentType: true, sizeRange: true, @@ -52,6 +63,138 @@ describe('createS3StorageDriver', () => { }); }); + it('maps prefixed conditional mutations to atomic S3 commands', async () => { + const send = vi + .spyOn(S3Client.prototype, 'send') + .mockResolvedValue({ ETag: '"next-etag"' } as never); + const client = new StorageClient( + 'objects', + createS3StorageDriver({ + adapter: { + bucket: 'private-bucket', + credentials: { + accessKeyId: 'test', + secretAccessKey: 'test', + }, + region: 'us-east-1', + }, + prefix: 'tenant-a', + }), + ); + + await expect( + client.uploadConditional('notes/a.txt', 'next', { + condition: { type: 'create' }, + metadata: { owner: 'agent' }, + }), + ).resolves.toMatchObject({ + etag: 'next-etag', + key: 'notes/a.txt', + size: 4, + }); + await client.uploadConditional('notes/a.txt', 'newer', { + condition: { etag: 'next-etag', type: 'replace' }, + }); + await client.deleteConditional('notes/a.txt', { + condition: { etag: 'newer-etag' }, + }); + + const [create, replace, remove] = send.mock.calls.map((call) => call[0]); + expect(create).toBeInstanceOf(PutObjectCommand); + expect((create as PutObjectCommand).input).toMatchObject({ + Bucket: 'private-bucket', + IfNoneMatch: '*', + Key: 'tenant-a/notes/a.txt', + Metadata: { owner: 'agent' }, + }); + expect(replace).toBeInstanceOf(PutObjectCommand); + expect((replace as PutObjectCommand).input).toMatchObject({ + IfMatch: '"next-etag"', + Key: 'tenant-a/notes/a.txt', + }); + expect(remove).toBeInstanceOf(DeleteObjectCommand); + expect((remove as DeleteObjectCommand).input).toEqual({ + Bucket: 'private-bucket', + IfMatch: '"newer-etag"', + Key: 'tenant-a/notes/a.txt', + }); + }); + + it('fails closed when S3 violates the advertised ETag guarantee', async () => { + vi.spyOn(S3Client.prototype, 'send').mockResolvedValue({} as never); + const client = new StorageClient( + 'objects', + createS3StorageDriver({ + adapter: { + bucket: 'private-bucket', + credentials: { accessKeyId: 'test', secretAccessKey: 'test' }, + region: 'us-east-1', + }, + }), + ); + + await expect( + client.uploadConditional('ambiguous.txt', 'body', { + condition: { type: 'create' }, + }), + ).rejects.toMatchObject({ + code: StorageErrorCode.PROVIDER, + key: 'ambiguous.txt', + permanent: true, + }); + }); + + it('fails closed for unknown S3-compatible endpoints and readonly stores', () => { + const compatible = createS3StorageDriver({ + adapter: { + bucket: 'objects', + credentials: { accessKeyId: 'test', secretAccessKey: 'test' }, + endpoint: 'https://objects.example.test', + region: 'us-east-1', + }, + }); + const readOnly = createS3StorageDriver({ + adapter: { + bucket: 'objects', + credentials: { accessKeyId: 'test', secretAccessKey: 'test' }, + region: 'us-east-1', + }, + readonly: true, + }); + + expect(compatible.capabilities.conditionalMutation).toBeUndefined(); + expect(readOnly.capabilities.conditionalMutation).toBeUndefined(); + expect(() => + new StorageClient('readonly', readOnly).uploadConditional('a.txt', 'a', { + condition: { type: 'create' }, + }), + ).toThrow( + expect.objectContaining({ code: StorageErrorCode.NOT_SUPPORTED }), + ); + }); + + it('rejects conditional multipart uploads without issuing a request', async () => { + const send = vi.spyOn(S3Client.prototype, 'send'); + const client = new StorageClient( + 'objects', + createS3StorageDriver({ + adapter: { + bucket: 'private-bucket', + credentials: { accessKeyId: 'test', secretAccessKey: 'test' }, + region: 'us-east-1', + }, + }), + ); + + await expect( + client.uploadConditional('a.txt', 'a', { + condition: { type: 'create' }, + multipart: true, + }), + ).rejects.toMatchObject({ code: StorageErrorCode.NOT_SUPPORTED }); + expect(send).not.toHaveBeenCalled(); + }); + it('does not claim expiring downloads for a permanent public base URL', () => { const driver = createS3StorageDriver({ adapter: { @@ -97,4 +240,34 @@ describe('createS3StorageDriver', () => { }), ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); }); + + it('maps failed conditional-mutation preconditions to a storage conflict', async () => { + vi.spyOn(S3Client.prototype, 'send').mockRejectedValue( + Object.assign(new Error('object changed'), { + $metadata: { httpStatusCode: 412 }, + name: 'PreconditionFailed', + }), + ); + const client = new StorageClient( + 'objects', + createS3StorageDriver({ + adapter: { + bucket: 'private-bucket', + credentials: { accessKeyId: 'test', secretAccessKey: 'test' }, + region: 'us-east-1', + }, + }), + ); + + await expect( + client.uploadConditional('image.png', 'changed', { + condition: { etag: 'old-etag', type: 'replace' }, + }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + await expect( + client.deleteConditional('image.png', { + condition: { etag: 'old-etag' }, + }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + }); }); diff --git a/src/storage.client.spec.ts b/src/storage.client.spec.ts index ae91d31..01e0f1b 100644 --- a/src/storage.client.spec.ts +++ b/src/storage.client.spec.ts @@ -133,6 +133,95 @@ describe('StorageClient', () => { ); }); + it('fails conditional mutations closed unless the exact primitive is declared', () => { + const client = new StorageClient('media', createMemoryStorageDriver()); + + expect(() => + client.uploadConditional('new.txt', 'new', { + condition: { type: 'create' }, + }), + ).toThrow( + expect.objectContaining({ code: StorageErrorCode.NOT_SUPPORTED }), + ); + expect(() => + client.deleteConditional('old.txt', { + condition: { etag: 'old-etag' }, + }), + ).toThrow( + expect.objectContaining({ code: StorageErrorCode.NOT_SUPPORTED }), + ); + }); + + it('delegates only driver-declared conditional mutations', async () => { + const driver = createMemoryStorageDriver(); + const uploadConditional = vi.fn(async (key: string) => ({ + contentType: 'text/plain', + etag: 'next-etag', + key, + size: 4, + })); + const deleteConditional = vi.fn(async () => undefined); + Object.defineProperty(driver, 'capabilities', { + value: { + ...driver.capabilities, + conditionalMutation: { + create: true, + delete: true, + etag: true, + replace: true, + }, + }, + }); + driver.uploadConditional = uploadConditional; + driver.deleteConditional = deleteConditional; + const client = new StorageClient('media', driver); + + await client.file('note.txt').uploadConditional('next', { + condition: { etag: 'old-etag', type: 'replace' }, + }); + await client.file('note.txt').deleteConditional({ + condition: { etag: 'next-etag' }, + }); + + expect(uploadConditional).toHaveBeenCalledWith('note.txt', 'next', { + condition: { etag: 'old-etag', type: 'replace' }, + }); + expect(deleteConditional).toHaveBeenCalledWith('note.txt', { + condition: { etag: 'next-etag' }, + }); + }); + + it('rejects invalid conditional ETags before calling a driver', () => { + const driver = createMemoryStorageDriver(); + Object.defineProperty(driver, 'capabilities', { + value: { + ...driver.capabilities, + conditionalMutation: { + create: true, + delete: true, + etag: true, + replace: true, + }, + }, + }); + driver.uploadConditional = vi.fn(); + driver.deleteConditional = vi.fn(); + const client = new StorageClient('media', driver); + + expect(() => + client.uploadConditional('note.txt', 'next', { + condition: { etag: '', type: 'replace' }, + }), + ).toThrow( + expect.objectContaining({ code: StorageErrorCode.INVALID_ARGUMENT }), + ); + expect(() => + client.deleteConditional('note.txt', { condition: { etag: '' } }), + ).toThrow( + expect.objectContaining({ code: StorageErrorCode.INVALID_ARGUMENT }), + ); + }); + it('classifies invalid owned options before calling the provider', async () => { const client = new StorageClient('media', createMemoryStorageDriver()); diff --git a/src/storage.client.ts b/src/storage.client.ts index 392646a..8dc0980 100644 --- a/src/storage.client.ts +++ b/src/storage.client.ts @@ -10,6 +10,8 @@ import type { StorageBufferedDownloadOptions, StorageBulkOptions, StorageCapabilities, + StorageConditionalDeleteOptions, + StorageConditionalUploadOptions, StorageDeleteManyResult, StorageDownloadManyResult, StorageDownloadOptions, @@ -208,12 +210,17 @@ export interface StorageFileHandle { body: StorageBody, options?: StorageUploadOptions, ): Promise; + uploadConditional( + body: StorageBody, + options: StorageConditionalUploadOptions, + ): Promise; download(options?: StorageDownloadOptions): Promise; bytes(options?: StorageBufferedDownloadOptions): Promise; text(options?: StorageBufferedDownloadOptions): Promise; head(options?: StorageOperationOptions): Promise; exists(options?: StorageOperationOptions): Promise; delete(options?: StorageOperationOptions): Promise; + deleteConditional(options: StorageConditionalDeleteOptions): Promise; signDownload(options?: StorageSignedDownloadOptions): Promise; signUpload(options: StorageSignedUploadOptions): Promise; copyTo( @@ -260,6 +267,7 @@ export class StorageClient { copyTo: (destinationKey, options) => this.copy(key, destinationKey, options), delete: (options) => this.delete(key, options), + deleteConditional: (options) => this.deleteConditional(key, options), download: (options) => this.downloadStream(key, options), exists: (options) => this.exists(key, options), head: (options) => this.head(key, options), @@ -273,6 +281,8 @@ export class StorageClient { bytes: (options) => this.downloadBytes(key, options), text: (options) => this.downloadText(key, options), upload: (body, options) => this.upload(key, body, options), + uploadConditional: (body, options) => + this.uploadConditional(key, body, options), }; } @@ -287,6 +297,50 @@ export class StorageClient { ); } + uploadConditional( + key: string, + body: StorageBody, + options: StorageConditionalUploadOptions, + ): Promise { + assertKey(key); + const condition = options.condition; + if ( + condition === undefined || + (condition.type !== 'create' && condition.type !== 'replace') + ) { + invalidArgument('condition.type must be "create" or "replace".'); + } + if ( + condition.type === 'replace' && + (typeof condition.etag !== 'string' || condition.etag.length === 0) + ) { + invalidArgument('condition.etag must be a non-empty string.'); + } + + const capability = this.#driver.capabilities.conditionalMutation; + const uploadConditional = this.#driver.uploadConditional; + const supported = + condition.type === 'create' + ? capability?.create === true + : capability?.replace === true; + if (!supported || uploadConditional === undefined) { + throw new StorageError( + `Store "${this.name}" does not support the requested conditional upload.`, + { + code: StorageErrorCode.NOT_SUPPORTED, + key, + operation: 'upload', + permanent: true, + store: this.name, + }, + ); + } + + return this.#execute({ key, operation: 'upload', store: this.name }, () => + uploadConditional.call(this.#driver, key, body, options), + ); + } + downloadStream( key: string, options?: StorageDownloadOptions, @@ -357,6 +411,39 @@ export class StorageClient { ); } + deleteConditional( + key: string, + options: StorageConditionalDeleteOptions, + ): Promise { + assertKey(key); + if ( + options.condition === undefined || + typeof options.condition.etag !== 'string' || + options.condition.etag.length === 0 + ) { + invalidArgument('condition.etag must be a non-empty string.'); + } + + const capability = this.#driver.capabilities.conditionalMutation; + const deleteConditional = this.#driver.deleteConditional; + if (capability?.delete !== true || deleteConditional === undefined) { + throw new StorageError( + `Store "${this.name}" does not support conditional delete.`, + { + code: StorageErrorCode.NOT_SUPPORTED, + key, + operation: 'delete', + permanent: true, + store: this.name, + }, + ); + } + + return this.#execute({ key, operation: 'delete', store: this.name }, () => + deleteConditional.call(this.#driver, key, options), + ); + } + copy( sourceKey: string, destinationKey: string, diff --git a/src/storage.driver.ts b/src/storage.driver.ts index c596f01..7935e26 100644 --- a/src/storage.driver.ts +++ b/src/storage.driver.ts @@ -1,6 +1,8 @@ import type { StorageBody, StorageCapabilities, + StorageConditionalDeleteOptions, + StorageConditionalUploadOptions, StorageDownloadOptions, StorageListOptions, StorageListResult, @@ -25,6 +27,11 @@ export interface StorageDriver { body: StorageBody, options?: StorageUploadOptions, ): Promise; + uploadConditional?( + key: string, + body: StorageBody, + options: StorageConditionalUploadOptions, + ): Promise; download( key: string, options?: StorageDownloadOptions, @@ -35,6 +42,10 @@ export interface StorageDriver { ): Promise; exists(key: string, options?: StorageOperationOptions): Promise; delete(key: string, options?: StorageOperationOptions): Promise; + deleteConditional?( + key: string, + options: StorageConditionalDeleteOptions, + ): Promise; copy( sourceKey: string, destinationKey: string, diff --git a/src/storage.service.spec.ts b/src/storage.service.spec.ts index 5bfdf6a..764e2b3 100644 --- a/src/storage.service.spec.ts +++ b/src/storage.service.spec.ts @@ -113,6 +113,88 @@ describe('StorageService cross-store operations', () => { await module.close(); }); + it('writes, compares, and prunes inside destinationPrefix', async () => { + const module = await Test.createTestingModule({ + imports: [ + StorageModule.forRoot({ + stores: [ + { + driver: createMemoryStorageDriver({ + adapter: { initial: { 'current.txt': 'new' } }, + }), + name: 'source', + }, + { + driver: createMemoryStorageDriver({ + adapter: { initial: { 'workspace/stale.txt': 'stale' } }, + }), + name: 'destination', + }, + ], + }), + ], + }).compile(); + const storage = module.get(StorageService); + + const result = await storage.sync({ + destinationPrefix: 'workspace/', + from: 'source', + prune: true, + to: 'destination', + transformKey: (key) => `nested/${key}`, + }); + + expect(result).toEqual({ + deleted: ['workspace/stale.txt'], + skipped: [], + uploaded: ['current.txt'], + }); + await expect( + storage.use('destination').downloadText('workspace/nested/current.txt'), + ).resolves.toBe('new'); + await expect( + storage.use('destination').exists('nested/current.txt'), + ).resolves.toBe(false); + await module.close(); + }); + + it('rejects transformed keys that could escape destinationPrefix', async () => { + const module = await Test.createTestingModule({ + imports: [ + StorageModule.forRoot({ + stores: [ + { + driver: createMemoryStorageDriver({ + adapter: { initial: { 'source.txt': 'source' } }, + }), + name: 'source', + }, + { + driver: createMemoryStorageDriver(), + name: 'destination', + }, + ], + }), + ], + }).compile(); + const storage = module.get(StorageService); + + await expect( + storage.sync({ + destinationPrefix: 'workspace/', + from: 'source', + to: 'destination', + transformKey: () => '../outside.txt', + }), + ).rejects.toMatchObject({ + code: StorageErrorCode.INVALID_ARGUMENT, + }); + await expect( + storage.use('destination').exists('outside.txt'), + ).resolves.toBe(false); + await module.close(); + }); + it('rejects unsafe same-store transfer', async () => { const module = await Test.createTestingModule({ imports: [ diff --git a/src/storage.service.ts b/src/storage.service.ts index 46b0ad3..8241192 100644 --- a/src/storage.service.ts +++ b/src/storage.service.ts @@ -16,6 +16,39 @@ import type { const identity = (key: string): string => key; +function joinStorageKey(prefix: string | undefined, key: string): string { + if (prefix === undefined || prefix.length === 0) { + return key; + } + const normalizedPrefix = prefix.replace(/\/+$/u, ''); + for (const [label, value] of [ + ['destinationPrefix', normalizedPrefix], + ['transformed destination key', key], + ] as const) { + if ( + value.length === 0 || + value.startsWith('/') || + value.includes('\\') || + value.includes('\0') || + value + .split('/') + .some( + (segment) => + segment.length === 0 || segment === '.' || segment === '..', + ) + ) { + throw new StorageError( + `${label} must be a canonical relative POSIX storage key.`, + { + code: StorageErrorCode.INVALID_ARGUMENT, + permanent: true, + }, + ); + } + } + return `${normalizedPrefix}/${key}`; +} + function assertDifferentStores(from: string, to: string): void { if (from === to) { throw new StorageError( @@ -200,6 +233,8 @@ export class StorageService { const source = this.use(fromName); const destination = this.use(toName); const transformKey = options.transformKey ?? identity; + const destinationKeyOf = (key: string): string => + joinStorageKey(options.destinationPrefix, transformKey(key)); const compare = options.compare ?? 'etag'; const sourceObjects = await walk(source, options); const destinationObjects = await walk(destination, { @@ -215,7 +250,7 @@ export class StorageService { destinationObjects.map((object) => [object.key, object]), ); const desiredDestinationKeys = new Set( - sourceObjects.map((object) => transformKey(object.key)), + sourceObjects.map((object) => destinationKeyOf(object.key)), ); const deleteKeys = (options.prune ?? false) @@ -227,7 +262,7 @@ export class StorageService { const plannedUploads: string[] = []; const plannedSkips: string[] = []; for (const object of sourceObjects) { - const existing = destinationIndex.get(transformKey(object.key)); + const existing = destinationIndex.get(destinationKeyOf(object.key)); (existing && unchanged(object, existing, compare) ? plannedSkips : plannedUploads @@ -258,7 +293,7 @@ export class StorageService { sourceObjects, (object) => object.key, async (object) => { - const destinationKey = transformKey(object.key); + const destinationKey = destinationKeyOf(object.key); const existing = destinationIndex.get(destinationKey); if (existing && unchanged(object, existing, compare)) { skipped.add(object.key); diff --git a/src/storage.types.ts b/src/storage.types.ts index 4055279..ee71a81 100644 --- a/src/storage.types.ts +++ b/src/storage.types.ts @@ -61,6 +61,23 @@ export interface StorageUploadOptions extends StorageOperationOptions { control?: StorageUploadControl; } +export type StorageConditionalUploadOptions = StorageUploadOptions & + ( + | { + /** Atomically create the object only when its key does not exist. */ + condition: { type: 'create' }; + } + | { + /** Atomically replace the object only when its current ETag matches. */ + condition: { type: 'replace'; etag: string }; + } + ); + +export interface StorageConditionalDeleteOptions extends StorageOperationOptions { + /** Atomically delete the object only when its current ETag matches. */ + condition: { etag: string }; +} + export interface StorageUploadResult { key: string; size: number; @@ -158,6 +175,20 @@ export interface StorageConditionalCopyCapability { version: boolean; } +export interface StorageConditionalMutationCapability { + /** Native create-if-absent support. */ + create: boolean; + /** Native replace-if-current-ETag-matches support. */ + replace: boolean; + /** Native delete-if-current-ETag-matches support. */ + delete: boolean; + /** + * Successful conditional uploads return an ETag usable for CAS. A provider + * that commits without returning one must fail rather than report success. + */ + etag: boolean; +} + export interface StorageSignedUploadPolicyCapability { /** The signed request fixes the exact declared content type. */ contentType: boolean; @@ -185,6 +216,11 @@ export interface StorageCapabilities { * compatibility with drivers built against earlier package versions. */ conditionalCopy?: StorageConditionalCopyCapability; + /** + * Native conditional mutations. Absent means unsupported; callers must not + * emulate these operations with a separate `exists()` or `head()` request. + */ + conditionalMutation?: StorageConditionalMutationCapability; signedDownload: StorageSignedUrlCapability; /** Expiry guarantees enforced by the provider adapter. */ signedDownloadPolicy?: StorageSignedDownloadPolicyCapability; diff --git a/src/workspace/index.ts b/src/workspace/index.ts new file mode 100644 index 0000000..02d594f --- /dev/null +++ b/src/workspace/index.ts @@ -0,0 +1,29 @@ +export { + StorageWorkspaceError, + isStorageWorkspaceError, +} from './storage-workspace.error.js'; +export { + createStorageWorkspace, + mountStorageWorkspace, +} from './storage-workspace.js'; +export { + DEFAULT_STORAGE_WORKSPACE_LIMITS, + STORAGE_WORKSPACE_PERMISSIONS, + type MountStorageWorkspaceOptions, + type StorageWorkspaceBody, + type StorageWorkspace, + type StorageWorkspaceDirectory, + type StorageWorkspaceEntry, + type StorageWorkspaceFile, + type StorageWorkspaceLimits, + type StorageWorkspaceListOptions, + type StorageWorkspaceMountOptions, + type StorageWorkspaceMutationOptions, + type StorageWorkspacePage, + type StorageWorkspacePermission, + type StorageWorkspaceReadOptions, + type StorageWorkspaceSearchMatch, + type StorageWorkspaceSearchOptions, + type StorageWorkspaceTextFile, + type StorageWorkspaceWriteOptions, +} from './storage-workspace.types.js'; diff --git a/src/workspace/storage-workspace.cursor.ts b/src/workspace/storage-workspace.cursor.ts new file mode 100644 index 0000000..b52839d --- /dev/null +++ b/src/workspace/storage-workspace.cursor.ts @@ -0,0 +1,71 @@ +import { randomBytes } from 'node:crypto'; + +import { StorageErrorCode } from '../storage.error.js'; + +import { workspaceError } from './storage-workspace.error.js'; + +interface CursorRecord { + readonly binding: string; + readonly expiresAt: number; + readonly state: State; +} + +const MAX_ACTIVE_CURSORS = 1024; + +export class StorageWorkspaceCursorStore { + readonly #records = new Map>(); + + issue(binding: string, state: State, ttlMs: number): string { + this.#prune(); + if (this.#records.size >= MAX_ACTIVE_CURSORS) { + throw workspaceError( + StorageErrorCode.LIMIT_EXCEEDED, + 'Workspace has too many active cursors.', + { permanent: true }, + ); + } + let cursor: string; + do { + cursor = randomBytes(24).toString('base64url'); + } while (this.#records.has(cursor)); + this.#records.set(cursor, { + binding, + expiresAt: Date.now() + ttlMs, + state, + }); + return cursor; + } + + consume(cursor: string, binding: string): State { + if (!/^[A-Za-z0-9_-]{32}$/u.test(cursor)) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + 'Workspace cursor has an invalid format.', + { permanent: true }, + ); + } + const record = this.#records.get(cursor); + this.#records.delete(cursor); + if ( + record === undefined || + record.expiresAt <= Date.now() || + record.binding !== binding + ) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + 'Workspace cursor is invalid, expired, or belongs to another query.', + { permanent: true }, + ); + } + return record.state as State; + } + + #prune(): void { + const now = Date.now(); + for (const [cursor, record] of this.#records) { + if (record.expiresAt <= now) { + this.#records.delete(cursor); + } + } + } +} diff --git a/src/workspace/storage-workspace.error.ts b/src/workspace/storage-workspace.error.ts new file mode 100644 index 0000000..243ef0b --- /dev/null +++ b/src/workspace/storage-workspace.error.ts @@ -0,0 +1,110 @@ +import { + StorageError, + StorageErrorCode, + isStorageError, +} from '../storage.error.js'; +import type { StorageErrorCode as StorageErrorCodeValue } from '../storage.error.js'; + +const STORAGE_WORKSPACE_ERROR_BRAND = Symbol('StorageWorkspaceError'); + +export class StorageWorkspaceError extends StorageError { + declare readonly [STORAGE_WORKSPACE_ERROR_BRAND]: true; + override readonly code: StorageErrorCodeValue; + override readonly operation: string | undefined; + readonly path: string | undefined; + override readonly permanent: boolean; + + constructor( + message: string, + options: { + code: StorageErrorCodeValue; + operation?: string; + path?: string; + permanent?: boolean; + }, + ) { + super(message, { + code: options.code, + ...(options.path !== undefined && { key: options.path }), + ...(options.operation !== undefined && { + operation: options.operation, + }), + permanent: options.permanent === true, + }); + Object.defineProperty(this, STORAGE_WORKSPACE_ERROR_BRAND, { + configurable: false, + enumerable: false, + value: true, + writable: false, + }); + this.name = 'StorageWorkspaceError'; + this.code = options.code; + this.operation = options.operation; + this.path = options.path; + this.permanent = options.permanent === true; + } +} + +export function isStorageWorkspaceError( + error: unknown, +): error is StorageWorkspaceError { + if (error instanceof StorageWorkspaceError) { + return true; + } + if (!(error instanceof Error) || error.name !== 'StorageWorkspaceError') { + return false; + } + try { + const candidate = error as Error & { + readonly [STORAGE_WORKSPACE_ERROR_BRAND]?: unknown; + readonly code?: unknown; + readonly operation?: unknown; + readonly path?: unknown; + readonly permanent?: unknown; + }; + return ( + candidate[STORAGE_WORKSPACE_ERROR_BRAND] === true && + Object.values(StorageErrorCode).includes( + candidate.code as StorageErrorCodeValue, + ) && + (candidate.operation === undefined || + typeof candidate.operation === 'string') && + (candidate.path === undefined || typeof candidate.path === 'string') && + typeof candidate.permanent === 'boolean' + ); + } catch { + return false; + } +} + +export function workspaceError( + code: StorageErrorCodeValue, + message: string, + options: { operation?: string; path?: string; permanent?: boolean } = {}, +): StorageWorkspaceError { + return new StorageWorkspaceError(message, { code, ...options }); +} + +export function sanitizeWorkspaceError( + error: unknown, + options: { operation: string; path?: string }, +): StorageWorkspaceError { + if (isStorageWorkspaceError(error)) { + return error; + } + + const code = isStorageError(error) ? error.code : StorageErrorCode.PROVIDER; + const permanent = isStorageError(error) ? error.permanent : false; + const logicalPath = options.path; + const target = logicalPath === undefined ? '' : ` for "${logicalPath}"`; + + return workspaceError( + code, + `Workspace ${options.operation} failed${target}.`, + { + operation: options.operation, + ...(logicalPath !== undefined && { path: logicalPath }), + permanent, + }, + ); +} diff --git a/src/workspace/storage-workspace.path.ts b/src/workspace/storage-workspace.path.ts new file mode 100644 index 0000000..591dec4 --- /dev/null +++ b/src/workspace/storage-workspace.path.ts @@ -0,0 +1,85 @@ +import { StorageErrorCode } from '../storage.error.js'; + +import { workspaceError } from './storage-workspace.error.js'; + +const FORBIDDEN_UNICODE_CHARACTER = /\p{C}/u; +const WINDOWS_DEVICE_NAME = + /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$/iu; +const encoder = new TextEncoder(); + +export function containsControlCharacter(value: string): boolean { + return FORBIDDEN_UNICODE_CHARACTER.test(value); +} + +function invalidPath(label: string, reason: string): never { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + `${label} ${reason}.`, + { permanent: true }, + ); +} + +export function assertWorkspacePath( + value: string, + maxBytes: number, + options: { allowRoot: boolean; label?: string }, +): string { + const label = options.label ?? 'path'; + if (typeof value !== 'string') { + invalidPath(label, 'must be a string'); + } + if (value.length === 0) { + if (options.allowRoot) { + return ''; + } + invalidPath(label, 'must not be empty'); + } + if (value.startsWith('/')) { + invalidPath(label, 'must be relative'); + } + if (value !== value.normalize('NFC')) { + invalidPath(label, 'must use NFC Unicode normalization'); + } + if (value.includes('\\')) { + invalidPath(label, 'must use POSIX separators'); + } + if (containsControlCharacter(value)) { + invalidPath(label, 'must not contain control characters'); + } + if (encoder.encode(value).byteLength > maxBytes) { + invalidPath(label, `exceeds the ${maxBytes}-byte path limit`); + } + + const segments = value.split('/'); + for (const segment of segments) { + if (segment.length === 0) { + invalidPath(label, 'must not contain empty path segments'); + } + if (segment === '.' || segment === '..') { + invalidPath(label, 'must not contain . or .. segments'); + } + if (segment.includes(':')) { + invalidPath(label, 'must not contain colon characters'); + } + if (segment.endsWith('.') || segment.endsWith(' ')) { + invalidPath(label, 'segments must not end with a dot or space'); + } + if (WINDOWS_DEVICE_NAME.test(segment)) { + invalidPath(label, 'must not contain a reserved device segment'); + } + } + return value; +} + +export function workspaceBasename(path: string): string { + const separator = path.lastIndexOf('/'); + return separator === -1 ? path : path.slice(separator + 1); +} + +export function joinWorkspacePath(directory: string, path: string): string { + return directory.length === 0 ? path : `${directory}/${path}`; +} + +export function isPathInside(directory: string, path: string): boolean { + return directory.length === 0 || path.startsWith(`${directory}/`); +} diff --git a/src/workspace/storage-workspace.spec.ts b/src/workspace/storage-workspace.spec.ts new file mode 100644 index 0000000..bf271de --- /dev/null +++ b/src/workspace/storage-workspace.spec.ts @@ -0,0 +1,495 @@ +import { + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { createFsStorageDriver } from '../files-sdk/fs/index.js'; +import { StorageClient } from '../storage.client.js'; +import { StorageError, StorageErrorCode } from '../storage.error.js'; +import { createMemoryStorageDriver } from '../testing/index.js'; +import type { StorageDriver } from '../storage.driver.js'; +import type { StorageObject } from '../storage.types.js'; + +import { + isStorageWorkspaceError, + mountStorageWorkspace, + type StorageWorkspacePermission, +} from './index.js'; + +const ALL_PERMISSIONS: readonly StorageWorkspacePermission[] = [ + 'list', + 'read', + 'search', + 'create', + 'replace', + 'copy', + 'move', + 'delete', +]; + +function mountedFs(root: string) { + const client = new StorageClient( + 'workspace-tests', + createFsStorageDriver({ adapter: { root } }), + ); + return { + client, + workspace: mountStorageWorkspace(client, { + permissions: ALL_PERMISSIONS, + prefix: 'runs/run-1', + }), + }; +} + +describe('StorageWorkspace', () => { + let root = ''; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'nestm-workspace-')); + }); + + afterEach(() => { + rmSync(root, { force: true, recursive: true }); + }); + + it('defaults to read-only permissions and child mounts only narrow authority', () => { + const client = new StorageClient('memory', createMemoryStorageDriver()); + const workspace = mountStorageWorkspace(client, { prefix: 'runs/one' }); + + expect([...workspace.permissions]).toEqual(['list', 'read', 'search']); + const exposed = workspace.permissions as Set; + exposed.add('delete'); + expect(workspace.allows('delete')).toBe(false); + + const child = workspace.mount('src', { + limits: { maxReadBytes: 10 }, + permissions: ['read'], + }); + expect([...child.permissions]).toEqual(['read']); + expect(child.limits.maxReadBytes).toBe(10); + expect(() => child.mount('nested', { permissions: ['create'] })).toThrow( + expect.objectContaining({ code: StorageErrorCode.UNAUTHORIZED }), + ); + expect(() => + child.mount('nested', { limits: { maxReadBytes: 11 } }), + ).toThrow( + expect.objectContaining({ code: StorageErrorCode.INVALID_ARGUMENT }), + ); + }); + + it.each([ + '', + '/absolute', + 'a//b', + 'a/./b', + 'a/../b', + 'a\\b', + 'file:stream', + 'CON', + 'aux.txt', + 'dir/name.', + 'dir/name ', + 'dir/\u0000name', + 'e\u0301.txt', + ])('rejects non-portable file path %j', async (path) => { + const { workspace } = mountedFs(root); + await expect(workspace.stat(path)).rejects.toMatchObject({ + code: StorageErrorCode.INVALID_ARGUMENT, + }); + }); + + it('requires a non-empty strict trusted prefix', () => { + const client = new StorageClient('memory', createMemoryStorageDriver()); + for (const prefix of ['', '/root', 'root\\child', 'root/../child']) { + expect(() => mountStorageWorkspace(client, { prefix })).toThrow( + expect.objectContaining({ code: StorageErrorCode.INVALID_ARGUMENT }), + ); + } + }); + + it('cannot read through a filesystem symlink into a sibling prefix', async () => { + mkdirSync(join(root, 'runs/run-1'), { recursive: true }); + mkdirSync(join(root, 'runs/other'), { recursive: true }); + writeFileSync(join(root, 'runs/other/secret.txt'), 'secret'); + symlinkSync( + join(root, 'runs/other/secret.txt'), + join(root, 'runs/run-1/link.txt'), + 'file', + ); + const { workspace } = mountedFs(root); + + await expect(workspace.readText('link.txt')).rejects.toMatchObject({ + code: StorageErrorCode.INVALID_ARGUMENT, + }); + await expect( + workspace.copyFile('link.txt', 'copied.txt'), + ).rejects.toMatchObject({ code: StorageErrorCode.INVALID_ARGUMENT }); + }); + + it('creates, conditionally replaces, reads, and conditionally deletes files', async () => { + const { workspace } = mountedFs(root); + const created = await workspace.writeFile('src/a.ts', 'export {};', { + contentType: 'text/typescript', + metadata: { owner: 'agent' }, + mode: 'create', + }); + expect(created).toMatchObject({ + contentType: 'text/typescript', + kind: 'file', + path: 'src/a.ts', + size: 10, + }); + expect(created.etag).toBeTypeOf('string'); + + await expect( + workspace.writeFile('src/a.ts', 'collision', { mode: 'create' }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + await expect( + workspace.writeFile('src/a.ts', 'changed', { + etag: 'stale', + mode: 'replace', + }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + + const replaced = await workspace.writeFile('src/a.ts', 'changed', { + etag: created.etag ?? '', + mode: 'replace', + }); + await expect(workspace.readText('src/a.ts')).resolves.toMatchObject({ + path: 'src/a.ts', + text: 'changed', + }); + await workspace.deleteFile('src/a.ts', { etag: replaced.etag ?? '' }); + await expect(workspace.stat('src/a.ts')).rejects.toMatchObject({ + code: StorageErrorCode.NOT_FOUND, + }); + }); + + it('bounds reads by streamed bytes even when provider metadata lies', async () => { + const driver = createMemoryStorageDriver(); + driver.download = async (key): Promise => ({ + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('123')); + controller.enqueue(new TextEncoder().encode('456')); + controller.close(); + }, + }), + contentType: 'text/plain', + key, + name: key, + size: 1, + }); + const workspace = mountStorageWorkspace( + new StorageClient('lying', driver), + { limits: { maxReadBytes: 5 }, prefix: 'scope' }, + ); + + await expect(workspace.readText('large.txt')).rejects.toMatchObject({ + code: StorageErrorCode.LIMIT_EXCEEDED, + key: 'large.txt', + }); + }); + + it('lists and searches relative to a selected directory with opaque cursors', async () => { + const { workspace } = mountedFs(root); + for (const path of [ + 'src/a.ts', + 'src/b.ts', + 'src/readme.md', + 'other/c.ts', + ]) { + await workspace.writeFile(path, path, { mode: 'create' }); + } + + const first = await workspace.list({ + directory: 'src', + limit: 1, + recursive: true, + }); + expect(first.entries).toHaveLength(1); + expect(first.cursor).toMatch(/^[A-Za-z0-9_-]{32}$/u); + const second = await workspace.list({ cursor: first.cursor }); + expect(second.entries).toHaveLength(1); + await expect( + workspace.list({ cursor: first.cursor }), + ).rejects.toMatchObject({ code: StorageErrorCode.INVALID_ARGUMENT }); + + const searched = await workspace.search('*.ts', { + directory: 'src', + limit: 1, + }); + expect(searched.entries[0]?.path).toMatch(/^src\/[ab]\.ts$/u); + expect(searched.cursor).toBeTypeOf('string'); + const continued = await workspace.search('', { cursor: searched.cursor }); + expect(continued.entries[0]?.path).toMatch(/^src\/[ab]\.ts$/u); + expect(continued.entries[0]?.path).not.toBe(searched.entries[0]?.path); + }); + + it('binds cursors to one workspace, operation, and query', async () => { + const { workspace } = mountedFs(root); + await workspace.writeFile('a.txt', 'a', { mode: 'create' }); + await workspace.writeFile('b.txt', 'b', { mode: 'create' }); + const listed = await workspace.list({ limit: 1, recursive: true }); + expect(listed.cursor).toBeTypeOf('string'); + + await expect( + workspace.search('', { cursor: listed.cursor }), + ).rejects.toMatchObject({ code: StorageErrorCode.INVALID_ARGUMENT }); + + const next = await workspace.list({ limit: 1, recursive: true }); + const child = workspace.mount('child'); + await expect(child.list({ cursor: next.cursor })).rejects.toMatchObject({ + code: StorageErrorCode.INVALID_ARGUMENT, + }); + await expect( + workspace.list({ cursor: '../not-a-token' }), + ).rejects.toMatchObject({ code: StorageErrorCode.INVALID_ARGUMENT }); + }); + + it('fails closed on out-of-scope and wrong-coordinate provider results', async () => { + const driver = createMemoryStorageDriver({ + adapter: { initial: { 'scope/requested.txt': 'ok' } }, + }); + driver.head = async () => ({ + contentType: 'text/plain', + key: 'scope/other.txt', + name: 'scope/other.txt', + size: 2, + }); + const workspace = mountStorageWorkspace( + new StorageClient('wrong-key', driver), + { prefix: 'scope' }, + ); + await expect(workspace.stat('requested.txt')).rejects.toMatchObject({ + code: StorageErrorCode.PROVIDER, + }); + + driver.list = async () => ({ + items: [ + { + contentType: 'text/plain', + key: 'outside/secret.txt', + name: 'outside/secret.txt', + size: 6, + }, + ], + }); + await expect(workspace.list()).rejects.toMatchObject({ + code: StorageErrorCode.PROVIDER, + }); + }); + + it('sanitizes spoofed workspace/provider errors without leaking prefix or cause', async () => { + const driver = createMemoryStorageDriver(); + const secret = new Error('credential secret'); + driver.head = async () => { + const spoof = new Error('raw provider path scope/private.txt', { + cause: secret, + }); + spoof.name = 'StorageWorkspaceError'; + Object.assign(spoof, { + code: StorageErrorCode.PROVIDER, + operation: 'stat', + path: 'scope/private.txt', + permanent: false, + }); + throw spoof; + }; + const workspace = mountStorageWorkspace( + new StorageClient('provider', driver), + { prefix: 'scope' }, + ); + + const error = await workspace + .stat('file.txt') + .catch((caught: unknown) => caught); + expect(isStorageWorkspaceError(error)).toBe(true); + expect(error).toMatchObject({ + code: StorageErrorCode.PROVIDER, + key: 'file.txt', + message: 'Workspace stat failed for "file.txt".', + path: 'file.txt', + }); + expect((error as Error).cause).toBeUndefined(); + expect((error as Error).message).not.toContain('scope'); + }); + + it('copies create-only with metadata and retains a destination after an unconfirmed source delete', async () => { + const driver = createFsStorageDriver({ adapter: { root } }); + const originalDelete = driver.deleteConditional.bind(driver); + let rejectSourceDelete = true; + driver.deleteConditional = vi.fn(async (key, options) => { + if (key === 'scope/source.txt' && rejectSourceDelete) { + rejectSourceDelete = false; + throw new StorageError('simulated source conflict', { + code: StorageErrorCode.CONFLICT, + key, + permanent: true, + }); + } + return originalDelete(key, options); + }); + const client = new StorageClient('move', driver); + const workspace = mountStorageWorkspace(client, { + permissions: ALL_PERMISSIONS, + prefix: 'scope', + }); + const source = await workspace.writeFile('source.txt', 'body', { + contentType: 'text/plain', + metadata: { copied: 'yes' }, + mode: 'create', + }); + + const copied = await workspace.copyFile('source.txt', 'copy.txt'); + expect(copied.contentType).toBe('text/plain'); + await expect( + workspace.copyFile('source.txt', 'copy.txt'), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + await expect( + workspace.moveFile('source.txt', 'moved.txt', { + etag: source.etag ?? '', + }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + await expect(workspace.stat('source.txt')).resolves.toBeDefined(); + await expect(workspace.stat('moved.txt')).resolves.toBeDefined(); + + const storedCopy = await client.head('scope/copy.txt'); + expect(storedCopy.metadata).toEqual({ copied: 'yes' }); + }); + + it('retains a destination when caller cancellation makes source deletion ambiguous', async () => { + const driver = createFsStorageDriver({ adapter: { root } }); + const originalDelete = driver.deleteConditional.bind(driver); + const controller = new AbortController(); + driver.deleteConditional = vi.fn(async (key, options) => { + if (key === 'scope/source.txt') { + controller.abort(new Error('agent stopped')); + throw new StorageError('source delete aborted', { + aborted: true, + code: StorageErrorCode.ABORTED, + key, + permanent: true, + }); + } + return originalDelete(key, options); + }); + const workspace = mountStorageWorkspace( + new StorageClient('cancelled-move', driver), + { permissions: ALL_PERMISSIONS, prefix: 'scope' }, + ); + const source = await workspace.writeFile('source.txt', 'body', { + mode: 'create', + }); + + await expect( + workspace.moveFile('source.txt', 'moved.txt', { + etag: source.etag ?? '', + signal: controller.signal, + }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + expect(controller.signal.aborted).toBe(true); + await expect(workspace.stat('moved.txt')).resolves.toBeDefined(); + }); + + it('never deletes both copies when a post-delete plugin reports failure', async () => { + const driver = createFsStorageDriver({ adapter: { root } }); + const client = new StorageClient('hooked-move', driver, [ + { + afterOperation(context) { + if (context.operation === 'delete') { + throw new Error('mandatory audit sink unavailable'); + } + }, + }, + ]); + const workspace = mountStorageWorkspace(client, { + permissions: ALL_PERMISSIONS, + prefix: 'scope', + }); + const source = await workspace.writeFile('source.txt', 'body', { + mode: 'create', + }); + + await expect( + workspace.moveFile('source.txt', 'moved.txt', { + etag: source.etag ?? '', + }), + ).rejects.toMatchObject({ code: StorageErrorCode.CONFLICT }); + await expect(workspace.stat('source.txt')).rejects.toMatchObject({ + code: StorageErrorCode.NOT_FOUND, + }); + await expect(workspace.stat('moved.txt')).resolves.toMatchObject({ + path: 'moved.txt', + }); + }); + + it('rejects malformed over-returning and repeating backend pages', async () => { + const driver = createMemoryStorageDriver(); + driver.list = vi.fn(async (options) => ({ + cursor: options?.cursor ?? 'same', + items: [ + { + contentType: 'text/plain', + key: 'scope/a.txt', + name: 'scope/a.txt', + size: 1, + }, + { + contentType: 'text/plain', + key: 'scope/b.txt', + name: 'scope/b.txt', + size: 1, + }, + ], + })); + const workspace = mountStorageWorkspace( + new StorageClient('malformed-list', driver), + { prefix: 'scope' }, + ); + + await expect( + workspace.list({ limit: 1, recursive: true }), + ).rejects.toMatchObject({ code: StorageErrorCode.PROVIDER }); + + driver.list = vi.fn(async (options) => ({ + cursor: options?.cursor ?? 'repeat', + items: [ + { + contentType: 'text/plain', + key: 'scope/a.txt', + name: 'scope/a.txt', + size: 1, + }, + ], + })); + const first = await workspace.list({ limit: 1, recursive: true }); + await expect( + workspace.list({ cursor: first.cursor }), + ).rejects.toMatchObject({ code: StorageErrorCode.PROVIDER }); + }); + + it('preflights copy and move when atomic capabilities are absent', async () => { + const driver: StorageDriver = createMemoryStorageDriver({ + adapter: { initial: { 'scope/source.txt': 'body' } }, + }); + const download = vi.spyOn(driver, 'download'); + const workspace = mountStorageWorkspace( + new StorageClient('unsupported', driver), + { permissions: ALL_PERMISSIONS, prefix: 'scope' }, + ); + + await expect( + workspace.copyFile('source.txt', 'copy.txt'), + ).rejects.toMatchObject({ code: StorageErrorCode.NOT_SUPPORTED }); + await expect( + workspace.moveFile('source.txt', 'moved.txt', { etag: 'etag' }), + ).rejects.toMatchObject({ code: StorageErrorCode.NOT_SUPPORTED }); + expect(download).not.toHaveBeenCalled(); + }); +}); diff --git a/src/workspace/storage-workspace.ts b/src/workspace/storage-workspace.ts new file mode 100644 index 0000000..701cad9 --- /dev/null +++ b/src/workspace/storage-workspace.ts @@ -0,0 +1,1161 @@ +import { StorageClient } from '../storage.client.js'; +import { StorageErrorCode } from '../storage.error.js'; +import type { + StorageObjectMetadata, + StorageOperationOptions, + StorageUploadResult, +} from '../storage.types.js'; + +import { StorageWorkspaceCursorStore } from './storage-workspace.cursor.js'; +import { + sanitizeWorkspaceError, + workspaceError, +} from './storage-workspace.error.js'; +import { + assertWorkspacePath, + containsControlCharacter, + isPathInside, + joinWorkspacePath, + workspaceBasename, +} from './storage-workspace.path.js'; +import { + DEFAULT_STORAGE_WORKSPACE_LIMITS, + STORAGE_WORKSPACE_PERMISSIONS, +} from './storage-workspace.types.js'; +import type { + MountStorageWorkspaceOptions, + StorageWorkspace as StorageWorkspaceContract, + StorageWorkspaceBody, + StorageWorkspaceDirectory, + StorageWorkspaceEntry, + StorageWorkspaceFile, + StorageWorkspaceLimits, + StorageWorkspaceListOptions, + StorageWorkspaceMountOptions, + StorageWorkspaceMutationOptions, + StorageWorkspacePage, + StorageWorkspacePermission, + StorageWorkspaceReadOptions, + StorageWorkspaceSearchMatch, + StorageWorkspaceSearchOptions, + StorageWorkspaceTextFile, + StorageWorkspaceWriteOptions, +} from './storage-workspace.types.js'; + +const DEFAULT_PERMISSIONS: ReadonlySet = new Set([ + 'list', + 'read', + 'search', +]); +const PERMISSION_VALUES = new Set( + STORAGE_WORKSPACE_PERMISSIONS, +); +const encoder = new TextEncoder(); +const decoder = new TextDecoder('utf-8', { fatal: true }); +const WORKSPACE_CONSTRUCTOR = Symbol('StorageWorkspace.constructor'); + +interface WorkspaceState { + readonly client: StorageClient; + readonly cursorStore: StorageWorkspaceCursorStore; + readonly id: string; + readonly prefix: string; +} + +interface ListCursorState { + readonly backendCursor: string; + readonly directory: string; + readonly limit: number; + readonly recursive: boolean; +} + +interface SearchCursorState { + readonly backendCursor: string | undefined; + readonly caseInsensitive: boolean; + readonly directory: string; + readonly limit: number; + readonly match: StorageWorkspaceSearchMatch; + readonly query: string; + readonly scanned: number; +} + +function operationOptions( + options?: StorageOperationOptions, +): StorageOperationOptions | undefined { + if (options === undefined) { + return undefined; + } + return { + ...(options.retries !== undefined && { retries: options.retries }), + ...(options.signal !== undefined && { signal: options.signal }), + ...(options.timeout !== undefined && { timeout: options.timeout }), + }; +} + +function positiveSafeInteger(value: number, label: string): number { + if (!Number.isSafeInteger(value) || value <= 0) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + `${label} must be a positive safe integer.`, + { permanent: true }, + ); + } + return value; +} + +function assertEtag(etag: string, maxBytes: number, operation: string): void { + if ( + typeof etag !== 'string' || + etag.length === 0 || + containsControlCharacter(etag) || + encoder.encode(etag).byteLength > maxBytes + ) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + `${operation} requires a valid non-empty etag.`, + { permanent: true }, + ); + } +} + +function resolveLimits( + requested: Partial | undefined, + parent?: Readonly, +): Readonly { + const baseline = parent ?? DEFAULT_STORAGE_WORKSPACE_LIMITS; + const next: StorageWorkspaceLimits = { + cursorTtlMs: requested?.cursorTtlMs ?? baseline.cursorTtlMs, + maxPageSize: requested?.maxPageSize ?? baseline.maxPageSize, + maxPathBytes: requested?.maxPathBytes ?? baseline.maxPathBytes, + maxReadBytes: requested?.maxReadBytes ?? baseline.maxReadBytes, + maxSearchResults: requested?.maxSearchResults ?? baseline.maxSearchResults, + maxSearchScan: requested?.maxSearchScan ?? baseline.maxSearchScan, + maxWriteBytes: requested?.maxWriteBytes ?? baseline.maxWriteBytes, + }; + for (const [name, value] of Object.entries(next)) { + positiveSafeInteger(value, `limits.${name}`); + const parentValue = baseline[name as keyof StorageWorkspaceLimits]; + if (parent !== undefined && value > parentValue) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + `Child workspace limit "${name}" cannot exceed its parent limit.`, + { permanent: true }, + ); + } + } + return Object.freeze(next); +} + +function resolvePermissions( + requested: Iterable | undefined, + parent?: ReadonlySet, +): ReadonlySet { + const baseline = parent ?? DEFAULT_PERMISSIONS; + const values = requested === undefined ? [...baseline] : [...requested]; + const permissions = new Set(); + for (const permission of values) { + if (!PERMISSION_VALUES.has(permission)) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + `Unknown workspace permission "${String(permission)}".`, + { permanent: true }, + ); + } + if (parent !== undefined && !parent.has(permission)) { + throw workspaceError( + StorageErrorCode.UNAUTHORIZED, + `Child workspace cannot add the "${permission}" permission.`, + { permanent: true }, + ); + } + permissions.add(permission); + } + return permissions; +} + +function logicalFile( + metadata: StorageObjectMetadata | StorageUploadResult, + path: string, +): StorageWorkspaceFile { + if ( + !Number.isSafeInteger(metadata.size) || + metadata.size < 0 || + typeof metadata.contentType !== 'string' || + metadata.contentType.length === 0 || + (metadata.etag !== undefined && + (typeof metadata.etag !== 'string' || metadata.etag.length === 0)) || + (metadata.lastModified !== undefined && + !Number.isFinite(new Date(metadata.lastModified).getTime())) + ) { + throw workspaceError( + StorageErrorCode.PROVIDER, + 'Storage provider returned malformed file metadata.', + { permanent: true }, + ); + } + return { + contentType: metadata.contentType, + ...(metadata.etag !== undefined && { etag: metadata.etag }), + kind: 'file', + ...(metadata.lastModified !== undefined && { + lastModified: new Date(metadata.lastModified), + }), + name: workspaceBasename(path), + path, + size: metadata.size, + }; +} + +function logicalDirectory(path: string): StorageWorkspaceDirectory { + const normalized = path.endsWith('/') ? path.slice(0, -1) : path; + return { + kind: 'directory', + name: normalized.length === 0 ? '' : workspaceBasename(normalized), + path: normalized, + }; +} + +function segmentGlobMatches(pattern: string, value: string): boolean { + let patternIndex = 0; + let valueIndex = 0; + let starIndex = -1; + let starValueIndex = -1; + while (valueIndex < value.length) { + if ( + patternIndex < pattern.length && + (pattern[patternIndex] === '?' || + pattern[patternIndex] === value[valueIndex]) + ) { + patternIndex += 1; + valueIndex += 1; + } else if (pattern[patternIndex] === '*') { + starIndex = patternIndex; + patternIndex += 1; + starValueIndex = valueIndex; + } else if (starIndex !== -1) { + patternIndex = starIndex + 1; + starValueIndex += 1; + valueIndex = starValueIndex; + } else { + return false; + } + } + while (pattern[patternIndex] === '*') { + patternIndex += 1; + } + return patternIndex === pattern.length; +} + +function globMatches(pattern: string, value: string): boolean { + const patternSegments = pattern.split('/'); + const valueSegments = value.split('/'); + let patternIndex = 0; + let valueIndex = 0; + let globstarIndex = -1; + let globstarValueIndex = -1; + while (valueIndex < valueSegments.length) { + const patternSegment = patternSegments[patternIndex]; + const valueSegment = valueSegments[valueIndex]; + if (patternSegment === '**') { + globstarIndex = patternIndex; + globstarValueIndex = valueIndex; + patternIndex += 1; + } else if ( + patternSegment !== undefined && + valueSegment !== undefined && + segmentGlobMatches(patternSegment, valueSegment) + ) { + patternIndex += 1; + valueIndex += 1; + } else if (globstarIndex !== -1) { + patternIndex = globstarIndex + 1; + globstarValueIndex += 1; + valueIndex = globstarValueIndex; + } else { + return false; + } + } + while (patternSegments[patternIndex] === '**') { + patternIndex += 1; + } + return patternIndex === patternSegments.length; +} + +function searchMatches( + query: string, + path: string, + match: StorageWorkspaceSearchMatch, + caseInsensitive: boolean, +): boolean { + const candidate = caseInsensitive ? path.toLowerCase() : path; + const needle = caseInsensitive ? query.toLowerCase() : query; + if (match === 'exact') { + return candidate === needle; + } + if (match === 'substring') { + return candidate.includes(needle); + } + return globMatches(needle, candidate); +} + +async function collectBoundedText( + stream: ReadableStream, + announcedSize: number, + maxBytes: number, + path: string, +): Promise { + if (announcedSize > maxBytes) { + await stream.cancel().catch(() => undefined); + throw workspaceError( + StorageErrorCode.LIMIT_EXCEEDED, + `Workspace file "${path}" exceeds the ${maxBytes}-byte read limit.`, + { path, permanent: true }, + ); + } + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const result = await reader.read(); + if (result.done) { + break; + } + if (result.value.byteLength === 0) { + throw workspaceError( + StorageErrorCode.PROVIDER, + 'Storage provider returned an empty non-terminal body chunk.', + { path, permanent: true }, + ); + } + size += result.value.byteLength; + if (size > maxBytes) { + await reader.cancel().catch(() => undefined); + throw workspaceError( + StorageErrorCode.LIMIT_EXCEEDED, + `Workspace file "${path}" exceeded the ${maxBytes}-byte read limit.`, + { path, permanent: true }, + ); + } + chunks.push(result.value); + } + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + try { + return decoder.decode(bytes); + } catch { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + `Workspace file "${path}" is not valid UTF-8 text.`, + { path, permanent: true }, + ); + } +} + +async function collectBoundedBytes( + stream: ReadableStream, + announcedSize: number, + maxBytes: number, + path: string, +): Promise { + if (announcedSize > maxBytes) { + await stream.cancel().catch(() => undefined); + throw workspaceError( + StorageErrorCode.LIMIT_EXCEEDED, + `Workspace file "${path}" exceeds the ${maxBytes}-byte copy limit.`, + { path, permanent: true }, + ); + } + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const result = await reader.read(); + if (result.done) { + break; + } + if (result.value.byteLength === 0) { + throw workspaceError( + StorageErrorCode.PROVIDER, + 'Storage provider returned an empty non-terminal body chunk.', + { path, permanent: true }, + ); + } + size += result.value.byteLength; + if (size > maxBytes) { + await reader.cancel().catch(() => undefined); + throw workspaceError( + StorageErrorCode.LIMIT_EXCEEDED, + `Workspace file "${path}" exceeded the ${maxBytes}-byte copy limit.`, + { path, permanent: true }, + ); + } + chunks.push(result.value); + } + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes; +} + +class StorageWorkspaceImplementation implements StorageWorkspaceContract { + readonly #state: WorkspaceState; + readonly #permissions: ReadonlySet; + readonly #limits: Readonly; + + /** @internal Construct workspaces through {@link mountStorageWorkspace}. */ + constructor( + constructorToken: symbol, + state: WorkspaceState, + permissions: ReadonlySet, + limits: Readonly, + ) { + if (constructorToken !== WORKSPACE_CONSTRUCTOR) { + throw workspaceError( + StorageErrorCode.UNAUTHORIZED, + 'StorageWorkspace must be created by mountStorageWorkspace().', + { permanent: true }, + ); + } + this.#state = state; + this.#permissions = permissions; + this.#limits = limits; + } + + get permissions(): ReadonlySet { + return new Set(this.#permissions); + } + + get limits(): Readonly { + return Object.freeze({ ...this.#limits }); + } + + allows(permission: StorageWorkspacePermission): boolean { + return this.#permissions.has(permission); + } + + async stat( + path: string, + options?: StorageOperationOptions, + ): Promise { + this.#require('read'); + const logicalPath = this.#filePath(path); + try { + const metadata = await this.#state.client.head( + this.#scope(logicalPath), + options, + ); + this.#assertResultPath(metadata.key, logicalPath); + return logicalFile(metadata, logicalPath); + } catch (error) { + throw sanitizeWorkspaceError(error, { + operation: 'stat', + path: logicalPath, + }); + } + } + + async readText( + path: string, + options?: StorageWorkspaceReadOptions, + ): Promise { + this.#require('read'); + const logicalPath = this.#filePath(path); + const maxBytes = options?.maxBytes ?? this.#limits.maxReadBytes; + positiveSafeInteger(maxBytes, 'maxBytes'); + if (maxBytes > this.#limits.maxReadBytes) { + throw workspaceError( + StorageErrorCode.LIMIT_EXCEEDED, + `maxBytes cannot exceed the ${this.#limits.maxReadBytes}-byte workspace read limit.`, + { path: logicalPath, permanent: true }, + ); + } + try { + const object = await this.#state.client.downloadStream( + this.#scope(logicalPath), + operationOptions(options), + ); + this.#assertResultPath(object.key, logicalPath); + const text = await collectBoundedText( + object.body, + object.size, + maxBytes, + logicalPath, + ); + return { ...logicalFile(object, logicalPath), text }; + } catch (error) { + throw sanitizeWorkspaceError(error, { + operation: 'read', + path: logicalPath, + }); + } + } + + async list( + options: StorageWorkspaceListOptions = {}, + ): Promise { + this.#require('list'); + const continuation = options.cursor + ? this.#state.cursorStore.consume( + options.cursor, + this.#binding('list'), + ) + : undefined; + if ( + continuation !== undefined && + ((options.directory !== undefined && + options.directory !== continuation.directory) || + (options.limit !== undefined && options.limit !== continuation.limit) || + (options.recursive !== undefined && + options.recursive !== continuation.recursive)) + ) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + 'Workspace cursor conflicts with the supplied list query.', + { permanent: true }, + ); + } + const directory = this.#directoryPath( + continuation?.directory ?? options.directory ?? '', + ); + const limit = this.#pageLimit( + continuation?.limit ?? options.limit, + this.#limits.maxPageSize, + ); + const recursive = continuation?.recursive ?? options.recursive === true; + const binding = this.#binding('list'); + const scopedDirectory = this.#scopeDirectory(directory); + try { + const page = await this.#state.client.list({ + ...(continuation !== undefined && { + cursor: continuation.backendCursor, + }), + ...(!recursive && { delimiter: '/' }), + limit, + ...operationOptions(options), + prefix: scopedDirectory, + }); + if (page.items.length + (page.prefixes?.length ?? 0) > limit) { + this.#malformedPage(); + } + if (page.cursor !== undefined && page.cursor.length === 0) { + this.#malformedPage(); + } + if ( + page.cursor !== undefined && + (page.items.length + (page.prefixes?.length ?? 0) === 0 || + page.cursor === continuation?.backendCursor) + ) { + this.#malformedPage(); + } + const entries: StorageWorkspaceEntry[] = []; + for (const item of page.items) { + const path = this.#unscoped(item.key); + if (!isPathInside(directory, path)) { + this.#providerScopeFailure(); + } + if (!recursive) { + const relative = + directory.length === 0 ? path : path.slice(directory.length + 1); + if (relative.includes('/')) { + this.#malformedPage(); + } + } + entries.push(logicalFile(item, path)); + } + for (const prefix of page.prefixes ?? []) { + const path = this.#unscoped( + prefix.endsWith('/') ? prefix.slice(0, -1) : prefix, + ); + if (!isPathInside(directory, path)) { + this.#providerScopeFailure(); + } + if (!recursive) { + const relative = + directory.length === 0 ? path : path.slice(directory.length + 1); + if (relative.includes('/')) { + this.#malformedPage(); + } + } + entries.push(logicalDirectory(path)); + } + return { + ...(page.cursor !== undefined && { + cursor: this.#state.cursorStore.issue( + binding, + { + backendCursor: page.cursor, + directory, + limit, + recursive, + } satisfies ListCursorState, + this.#limits.cursorTtlMs, + ), + }), + entries, + }; + } catch (error) { + throw sanitizeWorkspaceError(error, { + operation: 'list', + ...(directory.length > 0 && { path: directory }), + }); + } + } + + async search( + query: string, + options: StorageWorkspaceSearchOptions = {}, + ): Promise { + this.#require('search'); + const continuation = options.cursor + ? this.#state.cursorStore.consume( + options.cursor, + this.#binding('search'), + ) + : undefined; + if ( + continuation !== undefined && + ((query.length > 0 && query !== continuation.query) || + (options.directory !== undefined && + options.directory !== continuation.directory) || + (options.match !== undefined && options.match !== continuation.match) || + (options.caseInsensitive !== undefined && + options.caseInsensitive !== continuation.caseInsensitive) || + (options.limit !== undefined && options.limit !== continuation.limit)) + ) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + 'Workspace cursor conflicts with the supplied search query.', + { permanent: true }, + ); + } + query = continuation?.query ?? query; + if (typeof query !== 'string' || query.length === 0) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + 'Search query must be a non-empty string.', + { permanent: true }, + ); + } + if (query.includes('\\') || containsControlCharacter(query)) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + 'Search query contains forbidden characters.', + { permanent: true }, + ); + } + if (encoder.encode(query).byteLength > this.#limits.maxPathBytes) { + throw workspaceError( + StorageErrorCode.LIMIT_EXCEEDED, + 'Search query exceeds the workspace path-byte limit.', + { permanent: true }, + ); + } + const directory = this.#directoryPath( + continuation?.directory ?? options.directory ?? '', + ); + const match = continuation?.match ?? options.match ?? 'glob'; + if (match !== 'glob' && match !== 'substring' && match !== 'exact') { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + 'Search match must be glob, substring, or exact.', + { permanent: true }, + ); + } + const caseInsensitive = + continuation?.caseInsensitive ?? options.caseInsensitive === true; + const limit = this.#pageLimit( + continuation?.limit ?? options.limit, + this.#limits.maxSearchResults, + ); + const binding = this.#binding('search'); + const prior = continuation ?? { + backendCursor: undefined, + caseInsensitive, + directory, + limit, + match, + query, + scanned: 0, + }; + return this.#searchPage(query, { + binding, + caseInsensitive, + directory, + limit, + match, + operation: operationOptions(options), + state: prior, + }); + } + + async writeFile( + path: string, + body: StorageWorkspaceBody, + options: StorageWorkspaceWriteOptions, + ): Promise { + this.#require(options.mode); + const logicalPath = this.#filePath(path); + if (typeof body !== 'string' && !(body instanceof Uint8Array)) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + 'Workspace writes accept only strings or Uint8Array bodies.', + { path: logicalPath, permanent: true }, + ); + } + const size = + typeof body === 'string' + ? encoder.encode(body).byteLength + : body.byteLength; + if (size > this.#limits.maxWriteBytes) { + throw workspaceError( + StorageErrorCode.LIMIT_EXCEEDED, + `Workspace write exceeds the ${this.#limits.maxWriteBytes}-byte limit.`, + { path: logicalPath, permanent: true }, + ); + } + if (options.mode === 'replace') { + assertEtag(options.etag, this.#limits.maxPathBytes, 'Replace'); + } + try { + const common = { + ...(options.contentType !== undefined && { + contentType: options.contentType, + }), + ...(options.metadata !== undefined && { metadata: options.metadata }), + ...operationOptions(options), + }; + const result = + options.mode === 'create' + ? await this.#state.client.uploadConditional( + this.#scope(logicalPath), + body, + { ...common, condition: { type: 'create' } }, + ) + : await this.#state.client.uploadConditional( + this.#scope(logicalPath), + body, + { + ...common, + condition: { etag: options.etag, type: 'replace' }, + }, + ); + this.#assertResultPath(result.key, logicalPath); + return logicalFile(result, logicalPath); + } catch (error) { + throw sanitizeWorkspaceError(error, { + operation: options.mode, + path: logicalPath, + }); + } + } + + async copyFile( + source: string, + destination: string, + options?: StorageOperationOptions, + ): Promise { + this.#require('copy'); + this.#require('read'); + this.#require('create'); + const sourcePath = this.#filePath(source, 'source path'); + const destinationPath = this.#filePath(destination, 'destination path'); + if (sourcePath === destinationPath) { + throw workspaceError( + StorageErrorCode.CONFLICT, + 'Copy destination must differ from its source.', + { path: destinationPath, permanent: true }, + ); + } + if (this.#state.client.capabilities.conditionalMutation?.create !== true) { + throw workspaceError( + StorageErrorCode.NOT_SUPPORTED, + 'Safe copy requires create-only conditional upload support.', + { operation: 'copy', path: destinationPath, permanent: true }, + ); + } + return this.#copyCreate(sourcePath, destinationPath, options); + } + + async moveFile( + source: string, + destination: string, + options: StorageWorkspaceMutationOptions, + ): Promise { + this.#require('move'); + this.#require('read'); + this.#require('create'); + this.#require('delete'); + const conditional = this.#state.client.capabilities.conditionalMutation; + if ( + conditional?.create !== true || + conditional.delete !== true || + conditional.etag !== true + ) { + throw workspaceError( + StorageErrorCode.NOT_SUPPORTED, + 'Safe move requires create-only upload, conditional delete, and upload ETag support.', + { operation: 'move', permanent: true }, + ); + } + const sourcePath = this.#filePath(source, 'source path'); + const destinationPath = this.#filePath(destination, 'destination path'); + if (sourcePath === destinationPath) { + throw workspaceError( + StorageErrorCode.CONFLICT, + 'Move destination must differ from its source.', + { path: destinationPath, permanent: true }, + ); + } + assertEtag(options.etag, this.#limits.maxPathBytes, 'Move'); + const copied = await this.#copyCreate(sourcePath, destinationPath, options); + if (copied.etag === undefined) { + throw workspaceError( + StorageErrorCode.NOT_SUPPORTED, + 'Safe move requires the destination provider to return an etag.', + { path: destinationPath, permanent: true }, + ); + } + try { + await this.#state.client.deleteConditional(this.#scope(sourcePath), { + condition: { etag: options.etag }, + ...operationOptions(options), + }); + return copied; + } catch { + throw workspaceError( + StorageErrorCode.CONFLICT, + `Workspace move could not confirm source deletion; destination "${destinationPath}" was retained. Inspect both paths before retrying.`, + { + operation: 'move', + path: destinationPath, + permanent: true, + }, + ); + } + } + + async deleteFile( + path: string, + options: StorageWorkspaceMutationOptions, + ): Promise { + this.#require('delete'); + const logicalPath = this.#filePath(path); + assertEtag(options.etag, this.#limits.maxPathBytes, 'Delete'); + try { + await this.#state.client.deleteConditional(this.#scope(logicalPath), { + condition: { etag: options.etag }, + ...operationOptions(options), + }); + } catch (error) { + throw sanitizeWorkspaceError(error, { + operation: 'delete', + path: logicalPath, + }); + } + } + + mount( + directory: string, + options: StorageWorkspaceMountOptions = {}, + ): StorageWorkspaceContract { + const path = this.#directoryPath(directory); + if (path.length === 0) { + throw workspaceError( + StorageErrorCode.INVALID_ARGUMENT, + 'A child workspace mount directory must not be empty.', + { permanent: true }, + ); + } + return new StorageWorkspaceImplementation( + WORKSPACE_CONSTRUCTOR, + { + ...this.#state, + id: `${this.#state.id}/${path}`, + prefix: joinWorkspacePath(this.#state.prefix, path), + }, + resolvePermissions(options.permissions, this.#permissions), + resolveLimits(options.limits, this.#limits), + ); + } + + async #copyCreate( + sourcePath: string, + destinationPath: string, + options?: StorageOperationOptions, + ): Promise { + try { + const source = await this.#state.client.downloadStream( + this.#scope(sourcePath), + operationOptions(options), + ); + this.#assertResultPath(source.key, sourcePath); + const bytes = await collectBoundedBytes( + source.body, + source.size, + this.#limits.maxWriteBytes, + sourcePath, + ); + const result = await this.#state.client.uploadConditional( + this.#scope(destinationPath), + bytes, + { + condition: { type: 'create' }, + contentType: source.contentType, + ...(source.metadata !== undefined && { metadata: source.metadata }), + ...operationOptions(options), + }, + ); + this.#assertResultPath(result.key, destinationPath); + return logicalFile(result, destinationPath); + } catch (error) { + throw sanitizeWorkspaceError(error, { + operation: 'copy', + path: destinationPath, + }); + } + } + + async #searchPage( + query: string, + context: { + binding: string; + caseInsensitive: boolean; + directory: string; + limit: number; + match: StorageWorkspaceSearchMatch; + operation: StorageOperationOptions | undefined; + state: SearchCursorState; + }, + ): Promise { + const entries: StorageWorkspaceEntry[] = []; + let backendCursor = context.state.backendCursor; + let scanned = context.state.scanned; + try { + while ( + entries.length < context.limit && + scanned < this.#limits.maxSearchScan + ) { + const remaining = this.#limits.maxSearchScan - scanned; + const remainingResults = context.limit - entries.length; + const page = await this.#state.client.list({ + ...(backendCursor !== undefined && { cursor: backendCursor }), + limit: Math.min( + this.#limits.maxPageSize, + remaining, + remainingResults, + ), + ...context.operation, + prefix: this.#scopeDirectory(context.directory), + }); + const requested = Math.min( + this.#limits.maxPageSize, + remaining, + remainingResults, + ); + if (page.items.length > requested) { + this.#malformedPage(); + } + if ((page.prefixes?.length ?? 0) > 0) { + this.#malformedPage(); + } + if (page.cursor !== undefined && page.cursor.length === 0) { + this.#malformedPage(); + } + if ( + page.cursor !== undefined && + (page.items.length === 0 || page.cursor === backendCursor) + ) { + this.#malformedPage(); + } + for (let index = 0; index < page.items.length; index += 1) { + const item = page.items[index]; + if (item === undefined) { + continue; + } + scanned += 1; + const path = this.#unscoped(item.key); + if (!isPathInside(context.directory, path)) { + this.#providerScopeFailure(); + } + if ( + searchMatches( + query, + context.directory.length === 0 + ? path + : path.slice(context.directory.length + 1), + context.match, + context.caseInsensitive, + ) + ) { + entries.push(logicalFile(item, path)); + } + if ( + entries.length === context.limit || + scanned === this.#limits.maxSearchScan + ) { + break; + } + } + backendCursor = page.cursor; + if (backendCursor === undefined) { + break; + } + } + const hasMore = backendCursor !== undefined; + if (scanned >= this.#limits.maxSearchScan && hasMore) { + throw workspaceError( + StorageErrorCode.LIMIT_EXCEEDED, + `Search exceeded the ${this.#limits.maxSearchScan}-object scan limit.`, + { permanent: true }, + ); + } + return { + ...(hasMore && { + cursor: this.#state.cursorStore.issue( + context.binding, + { + backendCursor, + caseInsensitive: context.caseInsensitive, + directory: context.directory, + limit: context.limit, + match: context.match, + query, + scanned, + } satisfies SearchCursorState, + this.#limits.cursorTtlMs, + ), + }), + entries, + }; + } catch (error) { + throw sanitizeWorkspaceError(error, { + operation: 'search', + ...(context.directory.length > 0 && { path: context.directory }), + }); + } + } + + #require(permission: StorageWorkspacePermission): void { + if (!this.#permissions.has(permission)) { + throw workspaceError( + StorageErrorCode.UNAUTHORIZED, + `Workspace does not allow ${permission} operations.`, + { permanent: true }, + ); + } + } + + #filePath(path: string, label = 'path'): string { + return assertWorkspacePath(path, this.#limits.maxPathBytes, { + allowRoot: false, + label, + }); + } + + #directoryPath(path: string): string { + return assertWorkspacePath(path, this.#limits.maxPathBytes, { + allowRoot: true, + label: 'directory', + }); + } + + #scope(path: string): string { + return `${this.#state.prefix}/${path}`; + } + + #scopeDirectory(directory: string): string { + return directory.length === 0 + ? `${this.#state.prefix}/` + : `${this.#state.prefix}/${directory}/`; + } + + #unscoped(key: string): string { + const scoped = `${this.#state.prefix}/`; + if (!key.startsWith(scoped)) { + this.#providerScopeFailure(); + } + const path = key.slice(scoped.length); + return assertWorkspacePath(path, this.#limits.maxPathBytes, { + allowRoot: false, + label: 'backend result path', + }); + } + + #assertResultPath(key: string, expected: string): void { + if (this.#unscoped(key) !== expected) { + this.#providerScopeFailure(); + } + } + + #providerScopeFailure(): never { + throw workspaceError( + StorageErrorCode.PROVIDER, + 'Storage provider returned data outside the mounted workspace.', + { permanent: true }, + ); + } + + #malformedPage(): never { + throw workspaceError( + StorageErrorCode.PROVIDER, + 'Storage provider returned a malformed workspace page.', + { permanent: true }, + ); + } + + #pageLimit(value: number | undefined, maximum: number): number { + const limit = value ?? maximum; + positiveSafeInteger(limit, 'limit'); + if (limit > maximum) { + throw workspaceError( + StorageErrorCode.LIMIT_EXCEEDED, + `limit cannot exceed the workspace maximum of ${maximum}.`, + { permanent: true }, + ); + } + return limit; + } + + #binding(operation: string): string { + return `${this.#state.id}:${operation}`; + } +} + +export function mountStorageWorkspace( + client: StorageClient, + options: MountStorageWorkspaceOptions, +): StorageWorkspaceContract { + const limits = resolveLimits(options.limits); + const prefix = assertWorkspacePath(options.prefix, limits.maxPathBytes, { + allowRoot: false, + label: 'workspace prefix', + }); + return new StorageWorkspaceImplementation( + WORKSPACE_CONSTRUCTOR, + { + client, + cursorStore: new StorageWorkspaceCursorStore(), + id: `workspace:${crypto.randomUUID()}`, + prefix, + }, + resolvePermissions(options.permissions), + limits, + ); +} + +/** @deprecated Prefer {@link mountStorageWorkspace}. */ +export const createStorageWorkspace = mountStorageWorkspace; diff --git a/src/workspace/storage-workspace.types.ts b/src/workspace/storage-workspace.types.ts new file mode 100644 index 0000000..a62fdaf --- /dev/null +++ b/src/workspace/storage-workspace.types.ts @@ -0,0 +1,164 @@ +import type { StorageBody, StorageOperationOptions } from '../storage.types.js'; + +export const STORAGE_WORKSPACE_PERMISSIONS = [ + 'list', + 'read', + 'search', + 'create', + 'replace', + 'copy', + 'move', + 'delete', +] as const; + +export type StorageWorkspacePermission = + (typeof STORAGE_WORKSPACE_PERMISSIONS)[number]; + +export interface StorageWorkspaceLimits { + /** Maximum UTF-8 byte length of a workspace-relative path. */ + maxPathBytes: number; + /** Maximum bytes returned by a buffered text read. */ + maxReadBytes: number; + /** Maximum bytes accepted by one write. */ + maxWriteBytes: number; + /** Maximum entries returned by one list page. */ + maxPageSize: number; + /** Maximum entries returned by one search page. */ + maxSearchResults: number; + /** Maximum objects inspected by one search query across all pages. */ + maxSearchScan: number; + /** Lifetime of an opaque in-memory continuation cursor. */ + cursorTtlMs: number; +} + +export const DEFAULT_STORAGE_WORKSPACE_LIMITS: Readonly = + Object.freeze({ + cursorTtlMs: 5 * 60 * 1000, + maxPageSize: 100, + maxPathBytes: 1024, + maxReadBytes: 1024 * 1024, + maxSearchResults: 100, + maxSearchScan: 1000, + maxWriteBytes: 1024 * 1024, + }); + +export interface StorageWorkspaceFile { + kind: 'file'; + path: string; + name: string; + size: number; + contentType: string; + etag?: string; + lastModified?: Date; +} + +export interface StorageWorkspaceDirectory { + kind: 'directory'; + path: string; + name: string; +} + +export type StorageWorkspaceEntry = + StorageWorkspaceFile | StorageWorkspaceDirectory; + +export interface StorageWorkspaceTextFile extends StorageWorkspaceFile { + text: string; +} + +export interface StorageWorkspacePage { + entries: StorageWorkspaceEntry[]; + cursor?: string; +} + +export interface StorageWorkspaceMountOptions { + permissions?: Iterable; + limits?: Partial; +} + +export interface MountStorageWorkspaceOptions extends StorageWorkspaceMountOptions { + /** Trusted backend namespace. It is never exposed through the workspace. */ + prefix: string; +} + +export interface StorageWorkspaceReadOptions extends StorageOperationOptions { + maxBytes?: number; +} + +export interface StorageWorkspaceListOptions extends StorageOperationOptions { + directory?: string | undefined; + recursive?: boolean | undefined; + limit?: number | undefined; + cursor?: string | undefined; +} + +export type StorageWorkspaceSearchMatch = 'glob' | 'substring' | 'exact'; + +export interface StorageWorkspaceSearchOptions extends StorageOperationOptions { + directory?: string | undefined; + match?: StorageWorkspaceSearchMatch | undefined; + caseInsensitive?: boolean | undefined; + limit?: number | undefined; + cursor?: string | undefined; +} + +interface StorageWorkspaceWriteCommon extends StorageOperationOptions { + contentType?: string; + metadata?: Record; +} + +export type StorageWorkspaceWriteOptions = StorageWorkspaceWriteCommon & + ( + | { mode: 'create' } + | { + mode: 'replace'; + etag: string; + } + ); + +export interface StorageWorkspaceMutationOptions extends StorageOperationOptions { + etag: string; +} + +export type StorageWorkspaceBody = Extract; + +export interface StorageWorkspace { + readonly permissions: ReadonlySet; + readonly limits: Readonly; + allows(permission: StorageWorkspacePermission): boolean; + stat( + path: string, + options?: StorageOperationOptions, + ): Promise; + readText( + path: string, + options?: StorageWorkspaceReadOptions, + ): Promise; + list(options?: StorageWorkspaceListOptions): Promise; + search( + query: string, + options?: StorageWorkspaceSearchOptions, + ): Promise; + writeFile( + path: string, + body: StorageWorkspaceBody, + options: StorageWorkspaceWriteOptions, + ): Promise; + copyFile( + source: string, + destination: string, + options?: StorageOperationOptions, + ): Promise; + moveFile( + source: string, + destination: string, + options: StorageWorkspaceMutationOptions, + ): Promise; + deleteFile( + path: string, + options: StorageWorkspaceMutationOptions, + ): Promise; + mount( + directory: string, + options?: StorageWorkspaceMountOptions, + ): StorageWorkspace; +}