fix(eve): instrumentation - preload before server dependencies

.changeset/olive-donkeys-shave.md

@@ -0,0 +1,7 @@
+---
+"eve": patch
+---
+
+Run authored instrumentation before the rest of the server bundle, so OpenTelemetry auto-instrumentations can patch dependencies such as `pg`.
+
+`agent/instrumentation.ts`'s `setup` callback previously ran from a Nitro plugin. Plugin bodies are inlined into the bundled entry, while the entry's dependencies — including externals — are hoisted static imports above them, and ESM evaluates every import before any body code. A driver was therefore already loaded by the time `registerOTel` installed its require hook, so no spans were produced for it. The instrumentation module is now emitted as its own chunk and imported on the entry's first line, ahead of every hoisted import.

docs/guides/instrumentation.md

@@ -58,6 +58,23 @@ You are responsible for ensuring any observability or eval provider is approved
 
 The third configurable surface, [runtime context events](#runtime-context), attaches per-model-call values to these spans.
 
+### Auto-instrumenting a dependency
+
+Auto-instrumentations such as `@opentelemetry/auto-instrumentations-node` patch a package as the module loader hands it over, so the package has to reach the loader at runtime. eve runs `setup` before the rest of the server bundle, which covers the timing half of that requirement; the remaining half is yours: a package that eve bundles never passes through the loader at all and cannot be patched.
+
+Keep any package you want auto-instrumented external, so eve traces it into the hosted output and imports it at runtime:
+
+```ts title="agent/agent.ts"
+export default defineAgent({
+  model: "anthropic/claude-sonnet-5",
+  build: {
+    externalDependencies: ["pg"],
+  },
+});
+```
+
+Without this, an instrumentation registers cleanly and simply never reports spans for that package.
+
 ## Runtime context
 
 _Runtime context_ is an [AI SDK concept](https://ai-sdk.dev/docs/reference/ai-sdk-core/stream-text): a user-defined object that flows through a generation lifecycle. eve exposes it through `events["step.started"]`, a callback that runs once eve has assembled the model input for an attempt and returns `{ runtimeContext }`. Because eve registers the AI SDK's OpenTelemetry integration with runtime context enabled, those returned values ride onto the model-call span and its children. The field is named `runtimeContext`, not `metadata`, because AI SDK v7 carries per-call attributes on runtime context rather than a dedicated metadata field.

packages/eve/src/harness/instrumentation-config.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it, vi } from "vitest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
 
 /**
  * Regression coverage for the instrumentation-config chunk-isolation failure.
@@ -21,6 +21,14 @@ import { describe, expect, it, vi } from "vitest";
  * module loaders holding their own copies of the file.
  */
 describe("instrumentation-config chunk-isolation regression", () => {
+  // The setup-invoked latch lives on `globalThis`, so it outlives
+  // `vi.resetModules()` and would otherwise leak between tests.
+  beforeEach(() => {
+    delete (globalThis as Record<symbol, unknown>)[
+      Symbol.for("eve.harness-instrumentation-setup-invoked")
+    ];
+  });
+
   it("a config registered in one module evaluation is visible from another", async () => {
     vi.resetModules();
     const moduleA = await import("#harness/instrumentation-config.js");
@@ -70,4 +78,21 @@ describe("instrumentation-config chunk-isolation regression", () => {
 
     expect(setup).toHaveBeenCalledExactlyOnceWith({ agentName: "weather-agent" });
   });
+
+  it("invokes the setup callback once across separate module evaluations", async () => {
+    // The bundler reaches this module from both the entry's instrumentation
+    // preload chunk and the Nitro plugin, so registration can run twice.
+    // Registering OTel providers twice would double-instrument the process.
+    const setup = vi.fn();
+
+    vi.resetModules();
+    const moduleA = await import("#harness/instrumentation-config.js");
+    moduleA.registerInstrumentationConfig({ setup }, { agentName: "weather-agent" });
+
+    vi.resetModules();
+    const moduleB = await import("#harness/instrumentation-config.js");
+    moduleB.registerInstrumentationConfig({ setup }, { agentName: "weather-agent" });
+
+    expect(setup).toHaveBeenCalledOnce();
+  });
 });

packages/eve/src/harness/instrumentation-config.ts

@@ -6,23 +6,34 @@ import type {
 /**
  * Process-global store for the authored instrumentation config.
  *
- * Populated at server startup by the generated Nitro instrumentation plugin
- * when the user's `agent/instrumentation.ts` has a default export produced
- * by `defineInstrumentation()`. The harness reads from this at turn time
- * to decide whether telemetry is enabled and which settings to pass to the
+ * Populated at server startup by the generated instrumentation module when
+ * the user's `agent/instrumentation.ts` has a default export produced by
+ * `defineInstrumentation()`. The harness reads from this at turn time to
+ * decide whether telemetry is enabled and which settings to pass to the
  * AI SDK.
  *
- * Rooted on `globalThis` so the generated Nitro instrumentation plugin
- * (which Nitro keeps external by `file://` URL) and the bundled harness
- * chunk (which Nitro inlines via the package's `#harness/*` import alias)
- * share one source of truth, even though they resolve to two distinct ESM
- * module instances. See `context/key.ts` and
+ * Rooted on `globalThis` so every copy of that module shares one source of
+ * truth: the bundler emits it as the entry's instrumentation preload chunk
+ * and as a Nitro plugin (which Nitro keeps external by `file://` URL), and
+ * the bundled harness chunk (which Nitro inlines via the package's
+ * `#harness/*` import alias) is a third instance. See `context/key.ts` and
  * `runtime/sessions/runtime-session.ts` for the established pattern.
  */
 const INSTRUMENTATION_CONFIG_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-config");
 
+/**
+ * Tracks whether `setup` already ran in this process. Rooted on
+ * `globalThis` for the same reason as the config itself: the instrumentation
+ * preload chunk and the Nitro plugin can resolve to two distinct module
+ * instances, and `setup` must still run only once.
+ */
+const INSTRUMENTATION_SETUP_INVOKED_GLOBAL_KEY = Symbol.for(
+  "eve.harness-instrumentation-setup-invoked",
+);
+
 interface InstrumentationConfigGlobal {
   [INSTRUMENTATION_CONFIG_GLOBAL_KEY]?: InstrumentationDefinition;
+  [INSTRUMENTATION_SETUP_INVOKED_GLOBAL_KEY]?: boolean;
 }
 
 const globalContainer = globalThis as typeof globalThis & InstrumentationConfigGlobal;
@@ -31,16 +42,22 @@ const globalContainer = globalThis as typeof globalThis & InstrumentationConfigG
  * Registers the authored instrumentation config and invokes its `setup`
  * callback with the resolved agent name.
  *
- * Called once by the generated instrumentation Nitro plugin at server
- * startup. Subsequent calls overwrite the previous value.
+ * Called at server startup by the generated instrumentation module, which
+ * the bundler emits both as the entry's preload chunk and as a Nitro
+ * plugin. Subsequent calls overwrite the stored config, but `setup` runs
+ * only on the first.
  *
  * @internal — not part of the public API.
  */
 export function registerInstrumentationConfig(
   config: InstrumentationDefinition,
   context: InstrumentationSetupContext,
 ): void {
-  if (config.setup !== undefined) {
+  if (
+    config.setup !== undefined &&
+    globalContainer[INSTRUMENTATION_SETUP_INVOKED_GLOBAL_KEY] !== true
+  ) {
+    globalContainer[INSTRUMENTATION_SETUP_INVOKED_GLOBAL_KEY] = true;
     config.setup(context);
   }
   globalContainer[INSTRUMENTATION_CONFIG_GLOBAL_KEY] = config;

packages/eve/src/internal/bundler/instrumentation-preload-plugin.scenario.test.ts

@@ -0,0 +1,87 @@
+import { execFile } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { promisify } from "node:util";
+
+import { describe, expect, it } from "vitest";
+
+import { buildWithNitroRolldown } from "#internal/bundler/nitro-rolldown.js";
+import { createInstrumentationPreloadPlugin } from "#internal/bundler/instrumentation-preload-plugin.js";
+import { useTemporaryDirectories } from "#internal/testing/use-temporary-app-roots.js";
+
+const execFileAsync = promisify(execFile);
+const createScratchDirectory = useTemporaryDirectories();
+
+const EXTERNAL_PACKAGE_NAME = "fake-db-driver";
+
+/**
+ * Writes an app whose entry imports an external package, mirroring how a
+ * bundled Nitro server reaches a driver such as `pg`. The external records
+ * whether instrumentation already ran when it was loaded, which is the
+ * property the preload has to guarantee.
+ */
+function createScratchApp(directory: string): { entryPath: string; instrumentationPath: string } {
+  const packageDirectory = join(directory, "node_modules", EXTERNAL_PACKAGE_NAME);
+  mkdirSync(packageDirectory, { recursive: true });
+  writeFileSync(
+    join(packageDirectory, "package.json"),
+    JSON.stringify({ name: EXTERNAL_PACKAGE_NAME, version: "1.0.0", type: "module" }),
+  );
+  writeFileSync(
+    join(packageDirectory, "index.js"),
+    "globalThis.__loadOrder.push(`external:${globalThis.__instrumented === true}`);\nexport const connect = () => undefined;\n",
+  );
+
+  const entryPath = join(directory, "entry.mjs");
+  writeFileSync(
+    entryPath,
+    `import { connect } from "${EXTERNAL_PACKAGE_NAME}";\nglobalThis.__loadOrder.push("entry-body");\nexport default connect;\n`,
+  );
+
+  const instrumentationPath = join(directory, "instrumentation.mjs");
+  writeFileSync(
+    instrumentationPath,
+    'globalThis.__instrumented = true;\nglobalThis.__loadOrder.push("instrumentation");\nexport default function installInstrumentationPlugin() {}\n',
+  );
+
+  return { entryPath, instrumentationPath };
+}
+
+async function bundleWithPreload(directory: string): Promise<string> {
+  const { entryPath, instrumentationPath } = createScratchApp(directory);
+  const outDir = join(directory, "out");
+
+  await buildWithNitroRolldown({
+    cwd: directory,
+    input: entryPath,
+    platform: "node",
+    external: [EXTERNAL_PACKAGE_NAME],
+    plugins: [createInstrumentationPreloadPlugin(instrumentationPath)],
+    output: { dir: outDir, entryFileNames: "index.mjs", format: "esm", sourcemap: false },
+  });
+
+  return outDir;
+}
+
+describe("instrumentation preload (bundled)", () => {
+  it("evaluates instrumentation before the entry's external imports", async () => {
+    const directory = await createScratchDirectory("eve-instrumentation-preload-");
+    const outDir = await bundleWithPreload(directory);
+
+    // Node resolves the external from the scratch app's node_modules, so the
+    // built entry runs the same way the hosted server would.
+    const { stdout } = await execFileAsync(
+      process.execPath,
+      [
+        "--input-type=module",
+        "-e",
+        `globalThis.__loadOrder = [];
+         await import(${JSON.stringify(join(outDir, "index.mjs"))});
+         console.log(JSON.stringify(globalThis.__loadOrder));`,
+      ],
+      { cwd: directory },
+    );
+
+    expect(JSON.parse(stdout.trim())).toEqual(["instrumentation", "external:true", "entry-body"]);
+  });
+});

packages/eve/src/internal/bundler/instrumentation-preload-plugin.test.ts

@@ -0,0 +1,110 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  createInstrumentationPreloadPlugin,
+  INSTRUMENTATION_PRELOAD_CHUNK_FILE_NAME,
+} from "#internal/bundler/instrumentation-preload-plugin.js";
+
+const INSTRUMENTATION_MODULE_PATH = "/app/.eve/host/compiled-artifacts-instrumentation.mjs";
+
+interface EmittedChunk {
+  readonly type: "chunk";
+  readonly id: string;
+  readonly fileName: string;
+}
+
+/** Minimal stand-in for the Rollup/Rolldown plugin context. */
+function createBundlerContext() {
+  const emitted: EmittedChunk[] = [];
+
+  return {
+    emitted,
+    emitFile(file: EmittedChunk): string {
+      emitted.push(file);
+      return `ref-${emitted.length}`;
+    },
+  };
+}
+
+function renderEntry(input: {
+  readonly code?: string;
+  readonly fileName: string;
+  readonly isEntry?: boolean;
+}) {
+  const plugin = createInstrumentationPreloadPlugin(INSTRUMENTATION_MODULE_PATH);
+  const context = createBundlerContext();
+  plugin.buildStart.call(context);
+
+  return plugin.renderChunk.call(context, input.code ?? "console.log('entry');", {
+    fileName: input.fileName,
+    isEntry: input.isEntry ?? true,
+  });
+}
+
+describe("createInstrumentationPreloadPlugin", () => {
+  it("emits the instrumentation module as its own chunk", () => {
+    const plugin = createInstrumentationPreloadPlugin(INSTRUMENTATION_MODULE_PATH);
+    const context = createBundlerContext();
+
+    plugin.buildStart.call(context);
+
+    expect(context.emitted).toEqual([
+      {
+        type: "chunk",
+        id: INSTRUMENTATION_MODULE_PATH,
+        fileName: INSTRUMENTATION_PRELOAD_CHUNK_FILE_NAME,
+      },
+    ]);
+  });
+
+  it("imports the preload on the entry's first line, ahead of hoisted imports", () => {
+    const result = renderEntry({
+      code: 'import { Pool } from "pg";\nconsole.log(Pool);',
+      fileName: "index.mjs",
+    });
+
+    expect(result?.code.split("\n")[0]).toBe(
+      `import "./${INSTRUMENTATION_PRELOAD_CHUNK_FILE_NAME}";`,
+    );
+    expect(result?.code).toContain('import { Pool } from "pg";');
+  });
+
+  it("maps the shifted chunk back to its original line", () => {
+    const result = renderEntry({ fileName: "index.mjs" });
+
+    expect(result?.map.mappings.startsWith(";")).toBe(true);
+    expect(result?.map.sources).toEqual(["index.mjs"]);
+  });
+
+  it("resolves the preload relative to a nested entry chunk", () => {
+    const result = renderEntry({ fileName: "functions/__server.func/index.mjs" });
+
+    expect(result?.code.split("\n")[0]).toBe(
+      `import "../../${INSTRUMENTATION_PRELOAD_CHUNK_FILE_NAME}";`,
+    );
+  });
+
+  it("does not prepend the import twice when rendered again", () => {
+    const plugin = createInstrumentationPreloadPlugin(INSTRUMENTATION_MODULE_PATH);
+    const context = createBundlerContext();
+    plugin.buildStart.call(context);
+    const chunk = { fileName: "index.mjs", isEntry: true };
+
+    const first = plugin.renderChunk.call(context, "console.log('entry');", chunk);
+    const second = plugin.renderChunk.call(context, first?.code ?? "", chunk);
+
+    expect(second).toBeNull();
+  });
+
+  it("leaves shared chunks alone", () => {
+    const result = renderEntry({ fileName: "_libs/drizzle-orm.mjs", isEntry: false });
+
+    expect(result).toBeNull();
+  });
+
+  it("does not make the preload chunk import itself", () => {
+    const result = renderEntry({ fileName: INSTRUMENTATION_PRELOAD_CHUNK_FILE_NAME });
+
+    expect(result).toBeNull();
+  });
+});