fix: exclude unmeasurable LangSmith runs from outliers

.changeset/measured-langsmith-outliers.md

@@ -0,0 +1,5 @@
+---
+"openwiki": patch
+---
+
+fix: exclude unmeasurable LangSmith runs from latency outliers

src/connectors/sources/langsmith/runs.ts

@@ -21,12 +21,12 @@ export interface SampleCaps {
 /**
  * Selects the sample from two lean root-run pools within the window, biased
  * toward anomalies: errors first (up to errorCap), then latency outliers among
- * the non-errored runs (up to outlierCap, at most a quarter of the non-errored
- * pool so the bucket stays a genuine tail rather than swallowing a small pull,
- * and the remaining budget), then the most-recent non-errored runs to backfill
- * to `total`. With no errors/outliers it degrades to all-baseline — the same
- * recency behavior as before. Runs are deduped by id; `nonErrorRuns` is assumed
- * most-recent-first.
+ * the measurable non-errored runs (up to outlierCap, at most a quarter of the
+ * measurable pool so the bucket stays a genuine tail rather than swallowing a
+ * small pull, and the remaining budget), then the most-recent non-errored runs
+ * to backfill to `total`. With no errors/outliers it degrades to all-baseline —
+ * the same recency behavior as before. Runs are deduped by id; `nonErrorRuns` is
+ * assumed most-recent-first.
  */
 export function selectSampleBuckets(
   errorRuns: Run[],
@@ -51,14 +51,19 @@ export function selectSampleBuckets(
   // Keep outliers a genuine tail: never more than the flat cap, the remaining
   // budget, or a quarter of the non-errored pool (so a small pull stays mostly
   // baseline instead of being relabeled as outliers).
+  const measuredNonErrorRuns = nonErrorRuns
+    .filter((run) => !usedIds.has(run.id))
+    .map((run) => ({ latency: latencyMs(run), run }))
+    .filter(
+      (item): item is { latency: number; run: Run } =>
+        item.latency !== undefined,
+    );
   const outlierBudget = Math.min(
     caps.outlierCap,
     caps.total - selected.length,
-    Math.floor(nonErrorRuns.length / 4),
+    Math.floor(measuredNonErrorRuns.length / 4),
   );
-  const byLatencyDesc = nonErrorRuns
-    .filter((run) => !usedIds.has(run.id))
-    .map((run) => ({ latency: latencyMs(run) ?? -1, run }))
+  const byLatencyDesc = measuredNonErrorRuns
     .sort((left, right) => right.latency - left.latency)
     .slice(0, outlierBudget);
   for (const { run } of byLatencyDesc) {

test/langsmith-runs.test.ts

@@ -210,6 +210,49 @@ describe("selectSampleBuckets", () => {
     expect(outliers.map((s) => s.run.id).sort()).toEqual(["n6", "n7"]);
     expect(selected).toHaveLength(8);
   });
+
+  test("never labels runs without measurable latency as outliers", () => {
+    const nonErrorRuns = [
+      run({ id: "n1", start_time: BASE }),
+      run({ id: "n2", start_time: BASE }),
+      run({ id: "n3", start_time: BASE }),
+      run({ id: "n4", start_time: BASE }),
+    ];
+
+    const selected = selectSampleBuckets([], nonErrorRuns, {
+      errorCap: 0,
+      outlierCap: 5,
+      total: 4,
+    });
+
+    expect(selected.every((item) => item.bucket !== "outlier")).toBe(true);
+    expect(selected.every((item) => item.bucket === "baseline")).toBe(true);
+  });
+
+  test("fills outlier slots only with measured runs", () => {
+    const nonErrorRuns = [
+      run({ id: "missing-1", start_time: BASE }),
+      root("measured", 900),
+      root("measured-2", 100),
+      root("measured-3", 200),
+      root("measured-4", 300),
+      run({ id: "missing-2", start_time: BASE }),
+      run({ id: "missing-3", start_time: BASE }),
+      run({ id: "missing-4", start_time: BASE }),
+      run({ id: "missing-5", start_time: BASE }),
+      run({ id: "missing-6", start_time: BASE }),
+    ];
+
+    const selected = selectSampleBuckets([], nonErrorRuns, {
+      errorCap: 0,
+      outlierCap: 5,
+      total: 8,
+    });
+    const outliers = selected.filter((item) => item.bucket === "outlier");
+
+    expect(outliers).toHaveLength(1);
+    expect(outliers[0]?.run.id).toBe("measured");
+  });
 });
 
 describe("summarizeSample", () => {