Support path-style S3 endpoints

cli/src/backends/fly.ts

@@ -374,6 +374,7 @@ const key = "qm-health/" + randomUUID();
 const client = new S3Client({
   region: process.env.S3_REGION || "auto",
   ...(process.env.AWS_ENDPOINT_URL_S3 ? { endpoint: process.env.AWS_ENDPOINT_URL_S3 } : {}),
+  forcePathStyle: ["1", "true", "yes", "on"].includes((process.env.S3_FORCE_PATH_STYLE || "").trim().toLowerCase()),
 });
 try {
   await client.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body }));

cli/test/fly-sandbox.test.ts

@@ -216,6 +216,7 @@ test("the Fly S3 probe is valid CommonJS that reports async failures", () => {
     assert.doesNotMatch(source, /^\s*import\s/m, "eval never receives a static ESM import");
     assert.match(source, /\(async \(\) => \{/);
     assert.match(source, /\.catch\(\(error\) => \{/);
+    assert.match(source, /forcePathStyle:.*S3_FORCE_PATH_STYLE/);
     const path = join(dir, "probe.cjs");
     writeFileSync(path, source);
     const checked = spawnSync(process.execPath, ["--check", path], { encoding: "utf8" });

src/config.ts

@@ -101,6 +101,7 @@ export interface Config {
   s3Bucket?: string;
   s3Region?: string;
   s3Prefix?: string;
+  s3ForcePathStyle: boolean;
   deployIdleTtlMs?: number;
   deployGitDir: string;
   deployDialTimeoutMs: number;
@@ -395,6 +396,7 @@ export const CONFIG_DEFAULTS = {
   backgroundJobTtlSec: 1800,
   backgroundJobTtlMaxSec: 3600,
   backgroundWorkEnabled: true,
+  s3ForcePathStyle: false,
   monitorPollMs: 10_000,
   skillSyncPollMs: 0,
   deployDialTimeoutMs: 20_000,
@@ -815,6 +817,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
     ...(env.S3_BUCKET ? { s3Bucket: env.S3_BUCKET } : {}),
     ...(env.S3_REGION ? { s3Region: env.S3_REGION } : {}),
     ...(env.S3_PREFIX ? { s3Prefix: env.S3_PREFIX } : {}),
+    s3ForcePathStyle: boolEnvStrict("S3_FORCE_PATH_STYLE", env.S3_FORCE_PATH_STYLE) ?? CONFIG_DEFAULTS.s3ForcePathStyle,
     ...(numEnvStrict("DEPLOY_IDLE_TTL_MS", env.DEPLOY_IDLE_TTL_MS) !== undefined
       ? { deployIdleTtlMs: numEnvStrict("DEPLOY_IDLE_TTL_MS", env.DEPLOY_IDLE_TTL_MS) }
       : {}),

src/deploy/aws-deploy-provider.ts

@@ -7,6 +7,7 @@ import { bytes, normalizeRelPath, posixJoin, readTree } from "./deploy-fs.ts";
 import { AwsApiError, createMicrovmApi, createMicrovmClient, type AwsMicrovmApi } from "../sandbox/aws-microvm-api.ts";
 import { createMemoryMap, type DurableMap } from "../persistence/durable-map.ts";
 import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";
+import { s3Client } from "../persistence/s3.ts";
 import { createKeyedQueue, sleep } from "../util/async.ts";
 import { shq } from "../util/shell.ts";
 import { swallow } from "../util/errors.ts";
@@ -78,6 +79,7 @@ export interface AwsDeployProviderOptions {
   tokenTtlMinutes?: number;
   dataBucket?: string;
   dataPrefix?: string;
+  forcePathStyle?: boolean;
   snapshotIntervalMs?: number;
   dataRoleArn?: string;
   store?: DurableMap<StoredDeployBody>;
@@ -131,7 +133,12 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP
   const dataPrefix = (opts.dataPrefix ?? "deploy-data").replace(/\/+$/, "");
   const snapshotIntervalMs = opts.snapshotIntervalMs ?? 5 * 60_000;
   const s3: Pick<S3Client, "send"> | undefined = dataBucket
-    ? (opts.s3 ?? new S3Client({ region, ...(opts.profile ? { profile: opts.profile } : {}) }))
+    ? (opts.s3 ??
+      s3Client({
+        region,
+        ...(opts.profile ? { profile: opts.profile } : {}),
+        forcePathStyle: opts.forcePathStyle,
+      }))
     : undefined;
   const dataRoleArn = opts.dataRoleArn;
   const litestream = !!(dataBucket && dataRoleArn);

src/files/durable-byte-store.ts

@@ -126,14 +126,20 @@ export interface S3DurableByteOptions {
   bucket: string;
   region?: string;
   prefix?: string;
+  forcePathStyle?: boolean;
   _client?: S3Send;
 }
 
 export function createS3DurableByteStore(options: S3DurableByteOptions): DurableByteStore {
   const bucket = options.bucket;
   const prefix = options.prefix ?? "";
   const s3Key = (blobKey: string): string => prefix + blobKey;
-  const client = options._client ?? s3Client(options.region);
+  const client =
+    options._client ??
+    s3Client({
+      ...(options.region ? { region: options.region } : {}),
+      forcePathStyle: options.forcePathStyle,
+    });
 
   return {
     async put(source, opts) {

src/persistence/blob-transfer.ts

@@ -163,14 +163,20 @@ export interface S3BlobTransferOptions {
   bucket: string;
   region?: string;
   prefix?: string;
+  forcePathStyle?: boolean;
   _client?: S3Send;
 }
 
 export function createS3BlobTransferStore(options: S3BlobTransferOptions): BlobTransferStore {
   const bucket = options.bucket;
   const prefix = (options.prefix ?? "") + "transfer/";
   const keyFor = (blobId: string): string => prefix + blobId;
-  const client = options._client ?? s3Client(options.region);
+  const client =
+    options._client ??
+    s3Client({
+      ...(options.region ? { region: options.region } : {}),
+      forcePathStyle: options.forcePathStyle,
+    });
 
   return {
     async put(source, opts) {

src/persistence/s3.ts

@@ -1,12 +1,12 @@
-import { S3Client } from "@aws-sdk/client-s3";
+import { S3Client, type S3ClientConfig } from "@aws-sdk/client-s3";
 import { Readable } from "node:stream";
 
 export interface S3Send {
   send(command: unknown): Promise<unknown>;
 }
 
-export function s3Client(region?: string): S3Send {
-  return new S3Client(region ? { region } : {}) as S3Send;
+export function s3Client(config: S3ClientConfig = {}): S3Send {
+  return new S3Client({ ...config, forcePathStyle: config.forcePathStyle ?? false }) as S3Send;
 }
 
 export function bodyToReadable(body: unknown): Readable {

src/sandbox/aws-sandbox.ts

@@ -4,6 +4,7 @@ import type { WorkspaceLayer } from "../types.ts";
 import type { WorkspaceStore } from "../workspace/workspace-store.ts";
 import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";
 import { createMemoryMap, type DurableMap } from "../persistence/durable-map.ts";
+import { s3Client } from "../persistence/s3.ts";
 import { createKeyedQueue } from "../util/async.ts";
 import { scopeStorageKey } from "../util/scope-storage-key.ts";
 import { swallow, swallowAs, errMessage } from "../util/errors.ts";
@@ -72,6 +73,7 @@ export interface AwsSandboxOptions {
   egressConnectorArns?: string[];
   s3Bucket: string;
   s3Prefix?: string;
+  forcePathStyle?: boolean;
   agentPort?: number;
   maxIdleDurationSeconds?: number;
   suspendedDurationSeconds?: number;
@@ -107,7 +109,12 @@ export function createAwsSandbox(workspace: WorkspaceStore, opts: AwsSandboxOpti
       ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
     });
   const s3: Pick<S3Client, "send"> =
-    opts.s3 ?? new S3Client({ region, ...(opts.profile ? { profile: opts.profile } : {}) });
+    opts.s3 ??
+    s3Client({
+      region,
+      ...(opts.profile ? { profile: opts.profile } : {}),
+      forcePathStyle: opts.forcePathStyle,
+    });
   const store = opts.store ?? createMemoryMap<StoredMicrovm>();
   const advisoryLock = opts.advisoryLock ?? createNoopAdvisoryLock();
   const provisionQueue = createKeyedQueue<string>();

src/wiring.ts

@@ -543,6 +543,7 @@ export function buildApp(
           bucket: config.s3Bucket,
           ...(config.s3Region ? { region: config.s3Region } : {}),
           ...(config.s3Prefix ? { prefix: config.s3Prefix } : {}),
+          forcePathStyle: config.s3ForcePathStyle,
         })
       : createLocalBlobTransferStore(join(config.dataDir, "transfer"));
   const fileBytes: DurableByteStore =
@@ -551,6 +552,7 @@ export function buildApp(
           bucket: config.s3Bucket,
           ...(config.s3Region ? { region: config.s3Region } : {}),
           ...(config.s3Prefix ? { prefix: config.s3Prefix } : {}),
+          forcePathStyle: config.s3ForcePathStyle,
         })
       : createLocalDurableByteStore(join(config.dataDir, "docstore"));
   const files: FileArtifactStore = config.databaseUrl
@@ -588,6 +590,7 @@ export function buildApp(
     return createAwsSandbox(workspace, {
       ...config.awsSandbox,
       s3Bucket: config.awsSandbox.s3Bucket,
+      forcePathStyle: config.s3ForcePathStyle,
       advisoryLock,
       extraTools: deploymentLayer.advertisedTools,
       credentialPaths: deploymentLayer.credentialPaths,
@@ -825,6 +828,7 @@ export function buildApp(
               bucket: config.s3Bucket,
               ...(config.s3Region ? { region: config.s3Region } : {}),
               prefix: `${config.s3Prefix ?? ""}deploy-git/`,
+              forcePathStyle: config.s3ForcePathStyle,
             }),
           }
         : {}),
@@ -837,6 +841,7 @@ export function buildApp(
           ...(!config.awsDeploy.dataBucket && config.awsSandbox.s3Bucket
             ? { dataBucket: config.awsSandbox.s3Bucket }
             : {}),
+          forcePathStyle: config.s3ForcePathStyle,
           advisoryLock,
           store: artifactMap<StoredDeployBody>("aws_deploy_bodies"),
         })

test/config.test.ts

@@ -171,25 +171,34 @@ test("boolEnv: one vocabulary for every boolean env knob", () => {
 });
 
 test("every boolean knob accepts the shared vocabulary (off means off)", () => {
-  const off = loadConfig({ SEED_SKILLS: "off", EXECUTE_SCRATCH: "off", REACH_EXEC: "off", PI_CAPTURE_REQUESTS: "off" });
+  const off = loadConfig({
+    SEED_SKILLS: "off",
+    EXECUTE_SCRATCH: "off",
+    REACH_EXEC: "off",
+    PI_CAPTURE_REQUESTS: "off",
+    S3_FORCE_PATH_STYLE: "off",
+  });
   assert.equal(off.seedSkills, false);
   assert.equal(off.scratchExecEnabled, false);
   assert.equal(off.reachExecEnabled, false);
   assert.equal(off.sharedOwnerAuthIsolation, false);
   assert.equal(off.piCaptureRequests, false);
+  assert.equal(off.s3ForcePathStyle, false);
 
   const on = loadConfig({
     SEED_SKILLS: "yes",
     EXECUTE_SCRATCH: "on",
     REACH_EXEC: "1",
     SHARED_OWNER_AUTH_ISOLATION: "yes",
     PI_SYSTEM_CACHE_SPLIT: "on",
+    S3_FORCE_PATH_STYLE: "on",
   });
   assert.equal(on.seedSkills, true);
   assert.equal(on.scratchExecEnabled, true);
   assert.equal(on.reachExecEnabled, true);
   assert.equal(on.sharedOwnerAuthIsolation, true);
   assert.equal(on.piSystemCacheSplit, true);
+  assert.equal(on.s3ForcePathStyle, true);
 
   const unset = loadConfig({});
   assert.equal(unset.piCaptureRequests, true, "capture defaults on");
@@ -207,6 +216,10 @@ test("a set-but-unparseable env value refuses to boot instead of silently taking
   assert.throws(() => loadConfig({ WORKERS: "not-a-number" }), /WORKERS="not-a-number" is not a number/);
   assert.throws(() => loadConfig({ BUDGET_USD_PER_WINDOW: "10$" }), /BUDGET_USD_PER_WINDOW="10\$" is not a number/);
   assert.throws(() => loadConfig({ EXECUTE_SCRATCH: "2" }), /EXECUTE_SCRATCH="2" is not a recognized boolean/);
+  assert.throws(
+    () => loadConfig({ S3_FORCE_PATH_STYLE: "enabled" }),
+    /S3_FORCE_PATH_STYLE="enabled" is not a recognized boolean/,
+  );
   assert.throws(() => loadConfig({ SANDBOX_BACKEND: "docker" }), /SANDBOX_BACKEND="docker" is not recognized/);
   assert.equal(loadConfig({ WORKERS: "  " }).workers, CONFIG_DEFAULTS.workers);
   assert.equal(loadConfig({ EXECUTE_SCRATCH: "" }).scratchExecEnabled, false);

test/s3.test.ts

@@ -0,0 +1,44 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import type { HttpRequest } from "@smithy/types";
+import { s3Client } from "../src/persistence/s3.ts";
+
+test("s3Client enables path-style addressing for S3-compatible storage", () => {
+  const defaultClient = s3Client({ region: "auto" }) as S3Client;
+  const pathStyleClient = s3Client({ region: "auto", forcePathStyle: true }) as S3Client;
+  try {
+    assert.equal(defaultClient.config.forcePathStyle, false);
+    assert.equal(pathStyleClient.config.forcePathStyle, true);
+  } finally {
+    defaultClient.destroy();
+    pathStyleClient.destroy();
+  }
+});
+
+async function putRequestUrl(forcePathStyle: boolean): Promise<string> {
+  let requestUrl = "";
+  const client = s3Client({
+    region: "us-east-1",
+    endpoint: "https://minio.example.test",
+    credentials: { accessKeyId: "test", secretAccessKey: "test" },
+    forcePathStyle,
+    requestHandler: {
+      async handle(request: HttpRequest) {
+        requestUrl = `${request.protocol}//${request.hostname}${request.path}`;
+        return { response: { statusCode: 200, headers: {} } };
+      },
+    },
+  }) as S3Client;
+  try {
+    await client.send(new PutObjectCommand({ Bucket: "qm-data", Key: "files/test", Body: "test" }));
+    return requestUrl;
+  } finally {
+    client.destroy();
+  }
+}
+
+test("s3Client sends path-style requests to S3-compatible endpoints", async () => {
+  assert.equal(await putRequestUrl(false), "https://qm-data.minio.example.test/files/test");
+  assert.equal(await putRequestUrl(true), "https://minio.example.test/qm-data/files/test");
+});