diff --git a/.changeset/runtime-provider-selection.md b/.changeset/runtime-provider-selection.md new file mode 100644 index 0000000..469ed2a --- /dev/null +++ b/.changeset/runtime-provider-selection.md @@ -0,0 +1,28 @@ +--- +'@nestm/storage': minor +--- + +Add runtime provider selection and a package-owned filesystem driver. + +`@nestm/storage/files-sdk/provider` builds a driver from a provider slug carried +as data — `createProviderStorageDriver({ provider: 's3' | 'gcs' | 'azure' | 'r2' +| 'fs' | … })` — importing that provider's adapter, and only that one, on +demand. A deployment now selects its store with an environment variable and +installs a single native SDK instead of the application hard-coding a driver per +backend. The same entry point exposes the provider catalog (`listStorageProviders`, +`getStorageProvider`, `listStorageProviderEnvVars`, +`listStorageProviderSecretEnvVars`, `isStorageProvider`) as pure data, so config +validation and health checks can read a provider's env contract without loading +an adapter. An unknown slug fails closed with `INVALID_ARGUMENT` before anything +is imported. + +`@nestm/storage/files-sdk/fs` adds `createFsStorageDriver` for local filesystem +storage, mirroring the S3 factory. Its adapter reaches only `node:fs`, so it adds +no native SDK to an install. + +`@nestm/storage/files-sdk/s3` additionally exports `withS3Capabilities`, which +applies S3's conditional-copy promotion and signed-policy declarations to an +adapter built by `s3(...)`. The provider factory uses it so the `s3` slug keeps +those capabilities without re-deriving `S3AdapterOptions` from flat config. +`EnhancedS3Adapter` is now `S3StorageAdapter`; the type was not previously +exported. diff --git a/README.md b/README.md index 3fc7e3b..2856f48 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,71 @@ modules retain their own DI scope rather than mutating an application-wide registry. Names are case-sensitive and cannot contain leading or trailing whitespace. +### Select the provider at runtime + +An application that ships to more than one environment usually cannot name its +provider at build time. `createProviderStorageDriver` takes the slug as data and +imports that provider's adapter — and only that one — on demand, so a deployment +picks its store with an environment variable and installs one native SDK: + +```ts +import { createProviderStorageDriver } from '@nestm/storage/files-sdk/provider'; + +StorageModule.forRootAsync({ + imports: [ConfigModule], + stores: [ + { + name: 'media', + inject: [ConfigService], + useFactory: (config: ConfigService) => + createProviderStorageDriver({ + provider: config.getOrThrow('STORAGE_PROVIDER'), + prefix: config.get('STORAGE_PREFIX'), + config: { + bucket: config.get('STORAGE_BUCKET'), + region: config.get('STORAGE_REGION'), + root: config.get('STORAGE_ROOT'), + }, + }), + }, + ], +}); +``` + +`config` is one flat bag of provider settings — `bucket` and `region` for an +object store, `root` for the filesystem, `accountName` and `container` for +Azure. Each provider reads what it needs and ignores the rest, so the same shape +survives a provider change. Credentials may be omitted wherever the provider's +SDK resolves its own chain (an IAM role, Application Default Credentials, a +shared profile). + +An unknown slug fails closed with `INVALID_ARGUMENT` before anything is +imported. Validate untrusted input up front with `isStorageProvider`, and drive +config validation from the catalog rather than a hand-kept list: + +```ts +import { + getStorageProvider, + isStorageProvider, + listStorageProviders, + listStorageProviderSecretEnvVars, +} from '@nestm/storage/files-sdk/provider'; + +listStorageProviders().map((provider) => provider.slug); // 'akamai', 'alibaba', … +getStorageProvider('gcs')?.peerDeps; // ['@google-cloud/storage', …] +listStorageProviderSecretEnvVars('s3').map((variable) => variable.key); +``` + +The catalog is pure data and pulls in no adapter, so it is safe in config UIs, +health checks, and startup validation. + +The `s3` slug additionally carries the conditional-promotion and signed-policy +capabilities described under +[Race-free staged-object promotion](#race-free-staged-object-promotion); every +other provider exposes exactly what its adapter declares. When the provider _is_ +known at build time, import `@nestm/storage/files-sdk/s3` or +`@nestm/storage/files-sdk/fs` directly and skip the indirection. + ## Storage API `StorageClient` exposes: @@ -510,8 +575,26 @@ StorageModule.forRoot({ }); ``` -For local filesystem storage, import `fs` from `files-sdk/fs` and pass it to -`createFilesSdkDriver` exactly like a cloud adapter. +For local filesystem storage, use the package-owned factory. The adapter reaches +only `node:fs`, so it needs no native SDK: + +```ts +import { createFsStorageDriver } from '@nestm/storage/files-sdk/fs'; + +StorageModule.forRoot({ + stores: [ + { + name: 'artifacts', + driver: createFsStorageDriver({ adapter: { root: './var/artifacts' } }), + }, + ], +}); +``` + +Bodies are written verbatim at `/`. A `.meta.json` sidecar beside +each one carries the content type, ETag, and custom metadata a filesystem has +nowhere else to put; sidecars never surface as keys, and uploading a key ending +in `.meta.json` fails closed rather than colliding with one. ## License diff --git a/package.json b/package.json index 307b0c7..03f389a 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,14 @@ "types": "./dist/files-sdk/index.d.ts", "import": "./dist/files-sdk/index.js" }, + "./files-sdk/fs": { + "types": "./dist/files-sdk/fs/index.d.ts", + "import": "./dist/files-sdk/fs/index.js" + }, + "./files-sdk/provider": { + "types": "./dist/files-sdk/provider/index.d.ts", + "import": "./dist/files-sdk/provider/index.js" + }, "./files-sdk/s3": { "types": "./dist/files-sdk/s3/index.d.ts", "import": "./dist/files-sdk/s3/index.js" diff --git a/src/files-sdk/fs/fs.driver.spec.ts b/src/files-sdk/fs/fs.driver.spec.ts new file mode 100644 index 0000000..f544945 --- /dev/null +++ b/src/files-sdk/fs/fs.driver.spec.ts @@ -0,0 +1,78 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { StorageClient } from '../../storage.client.js'; +import { StorageErrorCode } from '../../storage.error.js'; +import { createFsStorageDriver } from './index.js'; + +describe('createFsStorageDriver', () => { + let root = ''; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'nestm-storage-fs-')); + }); + + afterEach(() => { + rmSync(root, { force: true, recursive: true }); + }); + + it('stores the body verbatim at the key path under the root', async () => { + const client = new StorageClient( + 'artifacts', + createFsStorageDriver({ adapter: { root } }), + ); + + await client.upload('nested/page.html', 'hi', { + contentType: 'text/html; charset=utf-8', + }); + + expect(readFileSync(join(root, 'nested/page.html'), 'utf8')).toBe( + 'hi', + ); + const head = await client.head('nested/page.html'); + expect(head.contentType).toBe('text/html; charset=utf-8'); + }); + + it('keeps sidecars out of listings', async () => { + const client = new StorageClient( + 'artifacts', + createFsStorageDriver({ adapter: { root } }), + ); + await client.upload('a.txt', 'a'); + await client.upload('b.txt', 'b'); + + const listed = await client.list(); + + expect(listed.items.map((item) => item.key).sort()).toEqual([ + 'a.txt', + 'b.txt', + ]); + }); + + it('scopes every key under the configured prefix', async () => { + const client = new StorageClient( + 'artifacts', + createFsStorageDriver({ adapter: { root }, prefix: 'tenant-a' }), + ); + + await client.upload('report.txt', 'scoped'); + + expect(readFileSync(join(root, 'tenant-a/report.txt'), 'utf8')).toBe( + 'scoped', + ); + const listed = await client.list(); + expect(listed.items.map((item) => item.key)).toEqual(['report.txt']); + }); + + it('reports a missing object as NOT_FOUND', async () => { + const client = new StorageClient( + 'artifacts', + createFsStorageDriver({ adapter: { root } }), + ); + + await expect(client.downloadBytes('absent.bin')).rejects.toMatchObject({ + code: StorageErrorCode.NOT_FOUND, + }); + }); +}); diff --git a/src/files-sdk/fs/index.ts b/src/files-sdk/fs/index.ts new file mode 100644 index 0000000..8ad6706 --- /dev/null +++ b/src/files-sdk/fs/index.ts @@ -0,0 +1,36 @@ +import { fs, type FsAdapter, type FsAdapterOptions } from 'files-sdk/fs'; + +import { + createFilesSdkDriver, + type FilesSdkDriverOptions, + type FilesSdkStorageDriver, +} from '../files-sdk.driver.js'; + +export interface FsStorageDriverOptions extends Omit< + FilesSdkDriverOptions, + 'adapter' +> { + adapter: FsAdapterOptions; +} + +/** + * Creates the files-sdk filesystem driver from the storage package's own + * dependency context. The adapter reaches only `node:fs`, so this entry point + * adds no native SDK to the install — unlike every object-store provider, it is + * always available. + * + * The adapter keeps a `.meta.json` sidecar beside each body to carry the + * content type, ETag, and custom metadata that a filesystem has nowhere else to + * put. Sidecars never surface as keys: `list` and `search` skip them, and + * uploading a key that ends in `.meta.json` fails closed rather than colliding + * with one. + */ +export function createFsStorageDriver( + options: FsStorageDriverOptions, +): FilesSdkStorageDriver { + const { adapter: adapterOptions, ...filesOptions } = options; + return createFilesSdkDriver({ ...filesOptions, adapter: fs(adapterOptions) }); +} + +export { fs, mapFsError } from 'files-sdk/fs'; +export type { FsAdapter, FsAdapterOptions } from 'files-sdk/fs'; diff --git a/src/files-sdk/provider/index.ts b/src/files-sdk/provider/index.ts new file mode 100644 index 0000000..1fb2ef2 --- /dev/null +++ b/src/files-sdk/provider/index.ts @@ -0,0 +1,141 @@ +import type { Adapter } from 'files-sdk'; +import { loadFiles, type LoadFilesOptions } from 'files-sdk/loader'; +import { + PROVIDER_NAMES, + getProvider, + getSecretEnvVars, + listEnvVars, + type EnvVar, + type Provider, + type ProviderSlug, +} from 'files-sdk/providers'; + +import { StorageError, StorageErrorCode } from '../../storage.error.js'; +import { + createFilesSdkDriver, + mapFilesSdkError, + type FilesSdkDriverOptions, + type FilesSdkStorageDriver, +} from '../files-sdk.driver.js'; + +/** Slug of a storage provider this package can build a driver for. */ +export type StorageProviderName = ProviderSlug; + +/** + * Flat, provider-specific settings — `bucket` and `region` for an object store, + * `root` for the filesystem, `accountName` and `container` for Azure, and so + * on. Every provider reads what it needs and ignores the rest, so one config + * shape serves a deployment that switches providers by name. + * + * Credentials may be omitted for any provider whose SDK resolves its own chain + * (an IAM role, Application Default Credentials, a shared profile). + */ +export type StorageProviderConfig = Omit; + +export interface ProviderStorageDriverOptions extends Omit< + FilesSdkDriverOptions, + 'adapter' +> { + /** Which provider to build. Validate untrusted input with {@link isStorageProvider}. */ + provider: StorageProviderName; + config?: StorageProviderConfig; +} + +/** + * Builds a {@link FilesSdkStorageDriver} for a provider named at runtime, + * importing that provider's adapter — and only that one — on demand. A + * deployment selects its store with a string (`'s3'`, `'gcs'`, `'azure'`, + * `'r2'`, `'fs'`, …) and installs one native SDK, instead of the application + * hard-coding a driver per backend. + * + * The `s3` slug additionally gets the conditional-promotion and signed-policy + * capabilities {@link createS3StorageDriver} attaches; every other provider + * exposes exactly what its adapter declares. Import + * `@nestm/storage/files-sdk/s3` directly when the provider is known at build + * time and the extra indirection buys nothing. + * + * @throws StorageError `INVALID_ARGUMENT` for an unknown slug, and whatever the + * adapter reports (mapped) when required config or credentials are missing. + */ +export async function createProviderStorageDriver( + options: ProviderStorageDriverOptions, +): Promise { + const { provider, config, ...filesOptions } = options; + if (!isStorageProvider(provider)) { + throw unknownProvider(provider); + } + + // `loadFiles` owns the slug → adapter mapping and keeps the import lazy; the + // client it returns is discarded so the caller's own driver options (prefix, + // hooks, plugins, readonly, retries) apply to the instance the bridge builds. + const adapter = await resolveAdapter(provider, config); + return createFilesSdkDriver({ ...filesOptions, adapter }); +} + +async function resolveAdapter( + provider: StorageProviderName, + config: StorageProviderConfig | undefined, +): Promise { + let resolved; + try { + resolved = await loadFiles({ ...config, provider }); + } catch (error) { + throw mapFilesSdkError(error); + } + const { adapter } = resolved.files; + if (provider !== 's3') { + return adapter; + } + // Built by `s3(...)`, so its `raw` is an S3Client and the promotion path + // holds. The S3-compatible wrappers keep only what they declare themselves. + const { withS3Capabilities } = await import('../s3/index.js'); + return withS3Capabilities( + adapter as Parameters[0], + { + ...(config?.publicBaseUrl !== undefined && { + publicBaseUrl: config.publicBaseUrl, + }), + }, + ); +} + +function unknownProvider(provider: string): StorageError { + return new StorageError( + `Unknown storage provider: ${JSON.stringify(provider)}. Known providers: ${PROVIDER_NAMES.join(', ')}.`, + { code: StorageErrorCode.INVALID_ARGUMENT, permanent: true }, + ); +} + +/** Narrows an untrusted string — an env var, a config file — to a known slug. */ +export function isStorageProvider(value: string): value is StorageProviderName { + return getProvider(value) !== undefined; +} + +/** Every provider that can be named, sorted by slug. */ +export function listStorageProviders(): readonly Provider[] { + return PROVIDER_NAMES.flatMap((slug) => getProvider(slug) ?? []); +} + +/** One provider's display name, description, native SDKs, and env contract. */ +export function getStorageProvider(slug: string): Provider | undefined { + return getProvider(slug); +} + +/** + * Every environment variable a provider reads, flattened across required, all + * credential modes, and optional. Useful for validating a deployment's config + * before the first upload rather than at it. + */ +export function listStorageProviderEnvVars(slug: string): EnvVar[] { + return listEnvVars(slug); +} + +/** The subset of {@link listStorageProviderEnvVars} that carries secrets. */ +export function listStorageProviderSecretEnvVars(slug: string): EnvVar[] { + return getSecretEnvVars(slug); +} + +export type { + EnvVar as StorageProviderEnvVar, + Provider as StorageProviderInfo, +} from 'files-sdk/providers'; diff --git a/src/files-sdk/provider/provider.driver.spec.ts b/src/files-sdk/provider/provider.driver.spec.ts new file mode 100644 index 0000000..127e2d8 --- /dev/null +++ b/src/files-sdk/provider/provider.driver.spec.ts @@ -0,0 +1,152 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { StorageClient } from '../../storage.client.js'; +import { StorageErrorCode } from '../../storage.error.js'; +import { + createProviderStorageDriver, + getStorageProvider, + isStorageProvider, + listStorageProviderEnvVars, + listStorageProviderSecretEnvVars, + listStorageProviders, +} from './index.js'; + +describe('createProviderStorageDriver', () => { + let root = ''; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'nestm-storage-provider-')); + }); + + afterEach(() => { + rmSync(root, { force: true, recursive: true }); + }); + + it('builds a working driver from a provider named at runtime', async () => { + const client = new StorageClient( + 'artifacts', + await createProviderStorageDriver({ + config: { root }, + provider: 'fs', + }), + ); + + await client.upload('index.html', 'named'); + + expect(readFileSync(join(root, 'index.html'), 'utf8')).toBe( + 'named', + ); + }); + + it('applies the caller driver options to the resolved adapter', async () => { + const client = new StorageClient( + 'artifacts', + await createProviderStorageDriver({ + config: { root }, + prefix: 'tenant-b', + provider: 'fs', + }), + ); + + await client.upload('report.txt', 'scoped'); + + expect(readFileSync(join(root, 'tenant-b/report.txt'), 'utf8')).toBe( + 'scoped', + ); + }); + + it('honors a readonly driver regardless of provider', async () => { + const client = new StorageClient( + 'artifacts', + await createProviderStorageDriver({ + config: { root }, + provider: 'fs', + readonly: true, + }), + ); + + await expect(client.upload('blocked.txt', 'nope')).rejects.toMatchObject({ + code: StorageErrorCode.READ_ONLY, + }); + }); + + it('adds the S3-only capabilities for the s3 slug', async () => { + const driver = await createProviderStorageDriver({ + config: { + accessKeyId: 'test', + bucket: 'artifacts', + region: 'us-east-1', + secretAccessKey: 'test', + }, + provider: 's3', + }); + + expect(driver.capabilities.conditionalCopy).toEqual({ + etag: true, + supported: true, + version: true, + }); + expect(driver.capabilities.signedUploadPolicy).toEqual({ + contentType: true, + sizeRange: true, + }); + }); + + it('claims no conditional copy for a provider that does not declare it', async () => { + const driver = await createProviderStorageDriver({ + config: { root }, + provider: 'fs', + }); + + expect(driver.capabilities.conditionalCopy).toBeUndefined(); + }); + + it('rejects an unknown slug before importing anything', async () => { + await expect( + createProviderStorageDriver({ + provider: 'not-a-provider' as never, + }), + ).rejects.toMatchObject({ + code: StorageErrorCode.INVALID_ARGUMENT, + permanent: true, + }); + }); +}); + +describe('storage provider catalog', () => { + it('narrows an untrusted string to a known slug', () => { + expect(isStorageProvider('gcs')).toBe(true); + expect(isStorageProvider('not-a-provider')).toBe(false); + }); + + it('lists the providers a deployment can choose between', () => { + const slugs = listStorageProviders().map((provider) => provider.slug); + + expect(slugs).toEqual([...slugs].sort()); + expect(slugs).toEqual(expect.arrayContaining(['azure', 'fs', 'gcs', 's3'])); + }); + + it('describes the native SDKs a provider needs', () => { + expect(getStorageProvider('gcs')?.peerDeps).toContain( + '@google-cloud/storage', + ); + expect(getStorageProvider('fs')?.peerDeps).toEqual([]); + expect(getStorageProvider('not-a-provider')).toBeUndefined(); + }); + + it('reports the env contract and which of it is secret', () => { + const keys = listStorageProviderEnvVars('s3').map( + (variable) => variable.key, + ); + const secrets = listStorageProviderSecretEnvVars('s3').map( + (variable) => variable.key, + ); + + expect(keys).toContain('AWS_REGION'); + expect(secrets).toContain('AWS_SECRET_ACCESS_KEY'); + expect(secrets).not.toContain('AWS_REGION'); + expect(secrets.every((key) => keys.includes(key))).toBe(true); + }); +}); diff --git a/src/files-sdk/s3/index.ts b/src/files-sdk/s3/index.ts index f9b07fd..bd014ae 100644 --- a/src/files-sdk/s3/index.ts +++ b/src/files-sdk/s3/index.ts @@ -24,7 +24,7 @@ export interface S3StorageDriverOptions extends Omit< adapter: S3AdapterOptions; } -type EnhancedS3Adapter = S3Adapter & +export type S3StorageAdapter = S3Adapter & FilesSdkConditionalCopyAdapter & FilesSdkSignedDownloadPolicyAdapter & FilesSdkSignedUploadPolicyAdapter; @@ -89,15 +89,20 @@ async function waitForRetry( } /** - * Creates the files-sdk S3 driver from the storage package's own dependency - * context and adds an ETag/version-conditional server-side promotion. + * Adds the S3-only capabilities to an adapter already built by `s3(...)`: an + * ETag/version-conditional server-side promotion, and the signed upload and + * download policies the bridge advertises through `capabilities`. + * + * Exported so the provider factory can apply them to the adapter `loadFiles` + * resolved for the `s3` slug instead of re-deriving {@link S3AdapterOptions} + * from flat provider config. It expects an adapter whose `raw` is an `S3Client` + * — pass one built by `s3(...)`, not an S3-compatible wrapper. */ -export function createS3StorageDriver( - options: S3StorageDriverOptions, -): FilesSdkStorageDriver { - const { adapter: adapterOptions, ...filesOptions } = options; - const base = s3(adapterOptions); - const adapter: EnhancedS3Adapter = Object.assign(base, { +export function withS3Capabilities( + base: S3Adapter, + options: Pick = {}, +): S3StorageAdapter { + return Object.assign(base, { conditionalCopy: Object.freeze({ etag: true, supported: true, @@ -108,7 +113,7 @@ export function createS3StorageDriver( sizeRange: true, }), signedDownloadPolicy: Object.freeze({ - expiresIn: adapterOptions.publicBaseUrl === undefined, + expiresIn: options.publicBaseUrl === undefined, }), async promote( sourceKey: string, @@ -162,10 +167,19 @@ export function createS3StorageDriver( } satisfies FilesSdkConditionalCopyAdapter & FilesSdkSignedDownloadPolicyAdapter & FilesSdkSignedUploadPolicyAdapter); +} +/** + * Creates the files-sdk S3 driver from the storage package's own dependency + * context and adds an ETag/version-conditional server-side promotion. + */ +export function createS3StorageDriver( + options: S3StorageDriverOptions, +): FilesSdkStorageDriver { + const { adapter: adapterOptions, ...filesOptions } = options; return createFilesSdkDriver({ ...filesOptions, - adapter, + adapter: withS3Capabilities(s3(adapterOptions), adapterOptions), }); }