From 0bcc24cf2e8c57b995967cd11ff52ef6fda5daa1 Mon Sep 17 00:00:00 2001 From: lijunwang2024-ship-it Date: Sun, 9 Aug 2026 17:33:47 +0800 Subject: [PATCH] feat: support path-style S3 endpoints --- cli/src/backends/fly.ts | 1 + cli/test/fly-sandbox.test.ts | 1 + src/config.ts | 3 +++ src/deploy/aws-deploy-provider.ts | 9 ++++++- src/files/durable-byte-store.ts | 8 +++++- src/persistence/blob-transfer.ts | 8 +++++- src/persistence/s3.ts | 6 ++--- src/sandbox/aws-sandbox.ts | 9 ++++++- src/wiring.ts | 5 ++++ test/config.test.ts | 15 ++++++++++- test/s3.test.ts | 44 +++++++++++++++++++++++++++++++ 11 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 test/s3.test.ts diff --git a/cli/src/backends/fly.ts b/cli/src/backends/fly.ts index 0940eb03..8268f113 100644 --- a/cli/src/backends/fly.ts +++ b/cli/src/backends/fly.ts @@ -374,6 +374,7 @@ const key = "qm-health/" + randomUUID(); const client = new S3Client({ region: process.env.S3_REGION || "auto", ...(process.env.AWS_ENDPOINT_URL_S3 ? { endpoint: process.env.AWS_ENDPOINT_URL_S3 } : {}), + forcePathStyle: ["1", "true", "yes", "on"].includes((process.env.S3_FORCE_PATH_STYLE || "").trim().toLowerCase()), }); try { await client.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body })); diff --git a/cli/test/fly-sandbox.test.ts b/cli/test/fly-sandbox.test.ts index 4d1d7f28..379bfcc9 100644 --- a/cli/test/fly-sandbox.test.ts +++ b/cli/test/fly-sandbox.test.ts @@ -216,6 +216,7 @@ test("the Fly S3 probe is valid CommonJS that reports async failures", () => { assert.doesNotMatch(source, /^\s*import\s/m, "eval never receives a static ESM import"); assert.match(source, /\(async \(\) => \{/); assert.match(source, /\.catch\(\(error\) => \{/); + assert.match(source, /forcePathStyle:.*S3_FORCE_PATH_STYLE/); const path = join(dir, "probe.cjs"); writeFileSync(path, source); const checked = spawnSync(process.execPath, ["--check", path], { encoding: "utf8" }); diff --git a/src/config.ts b/src/config.ts index 16e85a7b..27f7af83 100644 --- a/src/config.ts +++ b/src/config.ts @@ -101,6 +101,7 @@ export interface Config { s3Bucket?: string; s3Region?: string; s3Prefix?: string; + s3ForcePathStyle: boolean; deployIdleTtlMs?: number; deployGitDir: string; deployDialTimeoutMs: number; @@ -395,6 +396,7 @@ export const CONFIG_DEFAULTS = { backgroundJobTtlSec: 1800, backgroundJobTtlMaxSec: 3600, backgroundWorkEnabled: true, + s3ForcePathStyle: false, monitorPollMs: 10_000, skillSyncPollMs: 0, deployDialTimeoutMs: 20_000, @@ -815,6 +817,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { ...(env.S3_BUCKET ? { s3Bucket: env.S3_BUCKET } : {}), ...(env.S3_REGION ? { s3Region: env.S3_REGION } : {}), ...(env.S3_PREFIX ? { s3Prefix: env.S3_PREFIX } : {}), + s3ForcePathStyle: boolEnvStrict("S3_FORCE_PATH_STYLE", env.S3_FORCE_PATH_STYLE) ?? CONFIG_DEFAULTS.s3ForcePathStyle, ...(numEnvStrict("DEPLOY_IDLE_TTL_MS", env.DEPLOY_IDLE_TTL_MS) !== undefined ? { deployIdleTtlMs: numEnvStrict("DEPLOY_IDLE_TTL_MS", env.DEPLOY_IDLE_TTL_MS) } : {}), diff --git a/src/deploy/aws-deploy-provider.ts b/src/deploy/aws-deploy-provider.ts index 1f3b328a..e5bdc2c2 100644 --- a/src/deploy/aws-deploy-provider.ts +++ b/src/deploy/aws-deploy-provider.ts @@ -7,6 +7,7 @@ import { bytes, normalizeRelPath, posixJoin, readTree } from "./deploy-fs.ts"; import { AwsApiError, createMicrovmApi, createMicrovmClient, type AwsMicrovmApi } from "../sandbox/aws-microvm-api.ts"; import { createMemoryMap, type DurableMap } from "../persistence/durable-map.ts"; import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts"; +import { s3Client } from "../persistence/s3.ts"; import { createKeyedQueue, sleep } from "../util/async.ts"; import { shq } from "../util/shell.ts"; import { swallow } from "../util/errors.ts"; @@ -78,6 +79,7 @@ export interface AwsDeployProviderOptions { tokenTtlMinutes?: number; dataBucket?: string; dataPrefix?: string; + forcePathStyle?: boolean; snapshotIntervalMs?: number; dataRoleArn?: string; store?: DurableMap; @@ -131,7 +133,12 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP const dataPrefix = (opts.dataPrefix ?? "deploy-data").replace(/\/+$/, ""); const snapshotIntervalMs = opts.snapshotIntervalMs ?? 5 * 60_000; const s3: Pick | undefined = dataBucket - ? (opts.s3 ?? new S3Client({ region, ...(opts.profile ? { profile: opts.profile } : {}) })) + ? (opts.s3 ?? + s3Client({ + region, + ...(opts.profile ? { profile: opts.profile } : {}), + forcePathStyle: opts.forcePathStyle, + })) : undefined; const dataRoleArn = opts.dataRoleArn; const litestream = !!(dataBucket && dataRoleArn); diff --git a/src/files/durable-byte-store.ts b/src/files/durable-byte-store.ts index 7e2a8627..e629d22b 100644 --- a/src/files/durable-byte-store.ts +++ b/src/files/durable-byte-store.ts @@ -126,6 +126,7 @@ export interface S3DurableByteOptions { bucket: string; region?: string; prefix?: string; + forcePathStyle?: boolean; _client?: S3Send; } @@ -133,7 +134,12 @@ export function createS3DurableByteStore(options: S3DurableByteOptions): Durable const bucket = options.bucket; const prefix = options.prefix ?? ""; const s3Key = (blobKey: string): string => prefix + blobKey; - const client = options._client ?? s3Client(options.region); + const client = + options._client ?? + s3Client({ + ...(options.region ? { region: options.region } : {}), + forcePathStyle: options.forcePathStyle, + }); return { async put(source, opts) { diff --git a/src/persistence/blob-transfer.ts b/src/persistence/blob-transfer.ts index 64a23276..24a2345a 100644 --- a/src/persistence/blob-transfer.ts +++ b/src/persistence/blob-transfer.ts @@ -163,6 +163,7 @@ export interface S3BlobTransferOptions { bucket: string; region?: string; prefix?: string; + forcePathStyle?: boolean; _client?: S3Send; } @@ -170,7 +171,12 @@ export function createS3BlobTransferStore(options: S3BlobTransferOptions): BlobT const bucket = options.bucket; const prefix = (options.prefix ?? "") + "transfer/"; const keyFor = (blobId: string): string => prefix + blobId; - const client = options._client ?? s3Client(options.region); + const client = + options._client ?? + s3Client({ + ...(options.region ? { region: options.region } : {}), + forcePathStyle: options.forcePathStyle, + }); return { async put(source, opts) { diff --git a/src/persistence/s3.ts b/src/persistence/s3.ts index e82df5ee..f15fb65f 100644 --- a/src/persistence/s3.ts +++ b/src/persistence/s3.ts @@ -1,12 +1,12 @@ -import { S3Client } from "@aws-sdk/client-s3"; +import { S3Client, type S3ClientConfig } from "@aws-sdk/client-s3"; import { Readable } from "node:stream"; export interface S3Send { send(command: unknown): Promise; } -export function s3Client(region?: string): S3Send { - return new S3Client(region ? { region } : {}) as S3Send; +export function s3Client(config: S3ClientConfig = {}): S3Send { + return new S3Client({ ...config, forcePathStyle: config.forcePathStyle ?? false }) as S3Send; } export function bodyToReadable(body: unknown): Readable { diff --git a/src/sandbox/aws-sandbox.ts b/src/sandbox/aws-sandbox.ts index 55a1270a..66804d7f 100644 --- a/src/sandbox/aws-sandbox.ts +++ b/src/sandbox/aws-sandbox.ts @@ -4,6 +4,7 @@ import type { WorkspaceLayer } from "../types.ts"; import type { WorkspaceStore } from "../workspace/workspace-store.ts"; import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts"; import { createMemoryMap, type DurableMap } from "../persistence/durable-map.ts"; +import { s3Client } from "../persistence/s3.ts"; import { createKeyedQueue } from "../util/async.ts"; import { scopeStorageKey } from "../util/scope-storage-key.ts"; import { swallow, swallowAs, errMessage } from "../util/errors.ts"; @@ -72,6 +73,7 @@ export interface AwsSandboxOptions { egressConnectorArns?: string[]; s3Bucket: string; s3Prefix?: string; + forcePathStyle?: boolean; agentPort?: number; maxIdleDurationSeconds?: number; suspendedDurationSeconds?: number; @@ -107,7 +109,12 @@ export function createAwsSandbox(workspace: WorkspaceStore, opts: AwsSandboxOpti ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}), }); const s3: Pick = - opts.s3 ?? new S3Client({ region, ...(opts.profile ? { profile: opts.profile } : {}) }); + opts.s3 ?? + s3Client({ + region, + ...(opts.profile ? { profile: opts.profile } : {}), + forcePathStyle: opts.forcePathStyle, + }); const store = opts.store ?? createMemoryMap(); const advisoryLock = opts.advisoryLock ?? createNoopAdvisoryLock(); const provisionQueue = createKeyedQueue(); diff --git a/src/wiring.ts b/src/wiring.ts index 83540e3d..e9d8a301 100644 --- a/src/wiring.ts +++ b/src/wiring.ts @@ -543,6 +543,7 @@ export function buildApp( bucket: config.s3Bucket, ...(config.s3Region ? { region: config.s3Region } : {}), ...(config.s3Prefix ? { prefix: config.s3Prefix } : {}), + forcePathStyle: config.s3ForcePathStyle, }) : createLocalBlobTransferStore(join(config.dataDir, "transfer")); const fileBytes: DurableByteStore = @@ -551,6 +552,7 @@ export function buildApp( bucket: config.s3Bucket, ...(config.s3Region ? { region: config.s3Region } : {}), ...(config.s3Prefix ? { prefix: config.s3Prefix } : {}), + forcePathStyle: config.s3ForcePathStyle, }) : createLocalDurableByteStore(join(config.dataDir, "docstore")); const files: FileArtifactStore = config.databaseUrl @@ -588,6 +590,7 @@ export function buildApp( return createAwsSandbox(workspace, { ...config.awsSandbox, s3Bucket: config.awsSandbox.s3Bucket, + forcePathStyle: config.s3ForcePathStyle, advisoryLock, extraTools: deploymentLayer.advertisedTools, credentialPaths: deploymentLayer.credentialPaths, @@ -825,6 +828,7 @@ export function buildApp( bucket: config.s3Bucket, ...(config.s3Region ? { region: config.s3Region } : {}), prefix: `${config.s3Prefix ?? ""}deploy-git/`, + forcePathStyle: config.s3ForcePathStyle, }), } : {}), @@ -837,6 +841,7 @@ export function buildApp( ...(!config.awsDeploy.dataBucket && config.awsSandbox.s3Bucket ? { dataBucket: config.awsSandbox.s3Bucket } : {}), + forcePathStyle: config.s3ForcePathStyle, advisoryLock, store: artifactMap("aws_deploy_bodies"), }) diff --git a/test/config.test.ts b/test/config.test.ts index ee5ba307..faf90cfa 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -171,12 +171,19 @@ test("boolEnv: one vocabulary for every boolean env knob", () => { }); test("every boolean knob accepts the shared vocabulary (off means off)", () => { - const off = loadConfig({ SEED_SKILLS: "off", EXECUTE_SCRATCH: "off", REACH_EXEC: "off", PI_CAPTURE_REQUESTS: "off" }); + const off = loadConfig({ + SEED_SKILLS: "off", + EXECUTE_SCRATCH: "off", + REACH_EXEC: "off", + PI_CAPTURE_REQUESTS: "off", + S3_FORCE_PATH_STYLE: "off", + }); assert.equal(off.seedSkills, false); assert.equal(off.scratchExecEnabled, false); assert.equal(off.reachExecEnabled, false); assert.equal(off.sharedOwnerAuthIsolation, false); assert.equal(off.piCaptureRequests, false); + assert.equal(off.s3ForcePathStyle, false); const on = loadConfig({ SEED_SKILLS: "yes", @@ -184,12 +191,14 @@ test("every boolean knob accepts the shared vocabulary (off means off)", () => { REACH_EXEC: "1", SHARED_OWNER_AUTH_ISOLATION: "yes", PI_SYSTEM_CACHE_SPLIT: "on", + S3_FORCE_PATH_STYLE: "on", }); assert.equal(on.seedSkills, true); assert.equal(on.scratchExecEnabled, true); assert.equal(on.reachExecEnabled, true); assert.equal(on.sharedOwnerAuthIsolation, true); assert.equal(on.piSystemCacheSplit, true); + assert.equal(on.s3ForcePathStyle, true); const unset = loadConfig({}); assert.equal(unset.piCaptureRequests, true, "capture defaults on"); @@ -207,6 +216,10 @@ test("a set-but-unparseable env value refuses to boot instead of silently taking assert.throws(() => loadConfig({ WORKERS: "not-a-number" }), /WORKERS="not-a-number" is not a number/); assert.throws(() => loadConfig({ BUDGET_USD_PER_WINDOW: "10$" }), /BUDGET_USD_PER_WINDOW="10\$" is not a number/); assert.throws(() => loadConfig({ EXECUTE_SCRATCH: "2" }), /EXECUTE_SCRATCH="2" is not a recognized boolean/); + assert.throws( + () => loadConfig({ S3_FORCE_PATH_STYLE: "enabled" }), + /S3_FORCE_PATH_STYLE="enabled" is not a recognized boolean/, + ); assert.throws(() => loadConfig({ SANDBOX_BACKEND: "docker" }), /SANDBOX_BACKEND="docker" is not recognized/); assert.equal(loadConfig({ WORKERS: " " }).workers, CONFIG_DEFAULTS.workers); assert.equal(loadConfig({ EXECUTE_SCRATCH: "" }).scratchExecEnabled, false); diff --git a/test/s3.test.ts b/test/s3.test.ts new file mode 100644 index 00000000..63c1bc33 --- /dev/null +++ b/test/s3.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import type { HttpRequest } from "@smithy/types"; +import { s3Client } from "../src/persistence/s3.ts"; + +test("s3Client enables path-style addressing for S3-compatible storage", () => { + const defaultClient = s3Client({ region: "auto" }) as S3Client; + const pathStyleClient = s3Client({ region: "auto", forcePathStyle: true }) as S3Client; + try { + assert.equal(defaultClient.config.forcePathStyle, false); + assert.equal(pathStyleClient.config.forcePathStyle, true); + } finally { + defaultClient.destroy(); + pathStyleClient.destroy(); + } +}); + +async function putRequestUrl(forcePathStyle: boolean): Promise { + let requestUrl = ""; + const client = s3Client({ + region: "us-east-1", + endpoint: "https://minio.example.test", + credentials: { accessKeyId: "test", secretAccessKey: "test" }, + forcePathStyle, + requestHandler: { + async handle(request: HttpRequest) { + requestUrl = `${request.protocol}//${request.hostname}${request.path}`; + return { response: { statusCode: 200, headers: {} } }; + }, + }, + }) as S3Client; + try { + await client.send(new PutObjectCommand({ Bucket: "qm-data", Key: "files/test", Body: "test" })); + return requestUrl; + } finally { + client.destroy(); + } +} + +test("s3Client sends path-style requests to S3-compatible endpoints", async () => { + assert.equal(await putRequestUrl(false), "https://qm-data.minio.example.test/files/test"); + assert.equal(await putRequestUrl(true), "https://minio.example.test/qm-data/files/test"); +});