No measurement results in this run.
diff --git a/viewer/src/lib/server/artifacts.ts b/viewer/src/lib/server/artifacts.ts
index ea2da1dc9..707233cec 100644
--- a/viewer/src/lib/server/artifacts.ts
+++ b/viewer/src/lib/server/artifacts.ts
@@ -13,6 +13,7 @@ export const RUN_INFERENCE_SET_FILE = 'inference_set.jsonl';
export const RUN_SCORES_FILE = 'scores.jsonl';
export const RUN_CONFIG_FILE = 'config.yaml';
export const RUN_MANIFEST_FILE = 'manifest.json';
+export const RUN_METRICS_FILE = 'metrics.json';
export const VIEWER_CACHE_DIR = '.viewer';
export const VIEWER_RUN_MANIFEST_FILE = 'viewer_run_manifest.json';
export const VIEWER_PROMPT_ROWS_FILE = 'viewer_prompt_rows.json';
diff --git a/viewer/src/lib/server/data.ts b/viewer/src/lib/server/data.ts
index 5bc614c1a..5fbf40f47 100644
--- a/viewer/src/lib/server/data.ts
+++ b/viewer/src/lib/server/data.ts
@@ -7,6 +7,7 @@ import { loadDimensions } from './dimensions.js';
import {
RUN_CONFIG_FILE,
RUN_MANIFEST_FILE,
+ RUN_METRICS_FILE,
ViewerReadModelError,
loadIndexedRunScoreRow,
loadIndexedRunTranscriptRow,
@@ -61,6 +62,11 @@ import type {
Suite,
SuiteListItem,
SuiteStatus,
+ TokenActualUsageView,
+ TokenEstimateAccuracyView,
+ TokenEstimateView,
+ TokenStageEstimateView,
+ TokenUsageView,
Behavior,
ViewerResultItem
} from '$lib/types.js';
@@ -145,6 +151,149 @@ function readObject(value: unknown): Record
| null {
: null;
}
+function readFiniteNumber(value: unknown): number | null {
+ return typeof value === 'number' && Number.isFinite(value) ? value : null;
+}
+
+function readNonNegativeNumber(value: unknown): number | null {
+ const parsed = readFiniteNumber(value);
+ return parsed !== null && parsed >= 0 ? parsed : null;
+}
+
+function readInteger(value: unknown): number | null {
+ const parsed = readFiniteNumber(value);
+ return parsed === null ? null : Math.trunc(parsed);
+}
+
+function readNonNegativeInteger(value: unknown): number | null {
+ const parsed = readInteger(value);
+ return parsed !== null && parsed >= 0 ? parsed : null;
+}
+
+function normalizeTokenStageEstimate(value: unknown): TokenStageEstimateView | null {
+ const record = readObject(value);
+ if (!record) return null;
+ const calls = readNonNegativeInteger(record.calls) ?? 0;
+ const inputTokens = readNonNegativeInteger(record.input_tokens) ?? 0;
+ const outputTokens = readNonNegativeInteger(record.output_tokens) ?? 0;
+ const totalTokens = readNonNegativeInteger(record.total_tokens) ?? inputTokens + outputTokens;
+ if (calls === 0 && inputTokens === 0 && outputTokens === 0 && totalTokens === 0) return null;
+ return { calls, inputTokens, outputTokens, totalTokens };
+}
+
+function normalizeTokenEstimate(value: unknown): TokenEstimateView | null {
+ const record = readObject(value);
+ if (!record) return null;
+ const aggregate = normalizeTokenStageEstimate(record);
+ if (!aggregate) return null;
+
+ const stages: Record = {};
+ for (const [name, stageValue] of Object.entries(readObject(record.stages) ?? {})) {
+ const stage = normalizeTokenStageEstimate(stageValue);
+ if (stage) stages[name] = stage;
+ }
+
+ return {
+ ...aggregate,
+ lowerBoundTokens:
+ readNonNegativeInteger(record.lower_bound_tokens) ?? aggregate.totalTokens,
+ upperBoundTokens:
+ readNonNegativeInteger(record.upper_bound_tokens) ?? aggregate.totalTokens,
+ stages,
+ notes: Array.isArray(record.notes)
+ ? record.notes.filter((note): note is string => typeof note === 'string' && note.length > 0)
+ : []
+ };
+}
+
+function normalizeActualTokenUsage(value: unknown): TokenActualUsageView | null {
+ const record = readObject(value);
+ if (!record) return null;
+ const requests = readNonNegativeInteger(record.requests) ?? 0;
+ const calls = readNonNegativeInteger(record.calls) ?? 0;
+ const missingUsageCalls = readNonNegativeInteger(record.missing_usage_calls) ?? 0;
+ const inputTokens = readNonNegativeInteger(record.input_tokens) ?? 0;
+ const outputTokens = readNonNegativeInteger(record.output_tokens) ?? 0;
+ const totalTokens = readNonNegativeInteger(record.total_tokens) ?? inputTokens + outputTokens;
+ const cachedInputTokens = readNonNegativeInteger(record.cached_input_tokens) ?? 0;
+ const cacheCreationInputTokens =
+ readNonNegativeInteger(record.cache_creation_input_tokens) ?? 0;
+ if (
+ requests === 0 &&
+ calls === 0 &&
+ inputTokens === 0 &&
+ outputTokens === 0 &&
+ totalTokens === 0
+ ) {
+ return null;
+ }
+ return {
+ requests,
+ calls,
+ missingUsageCalls,
+ inputTokens,
+ outputTokens,
+ totalTokens,
+ cachedInputTokens,
+ cacheCreationInputTokens,
+ cacheHitRate:
+ readNonNegativeNumber(record.cache_hit_rate) ??
+ (inputTokens > 0 ? cachedInputTokens / inputTokens : 0),
+ usageCoverage:
+ readNonNegativeNumber(record.usage_coverage) ??
+ (requests > 0 ? calls / requests : 0)
+ };
+}
+
+function normalizeTokenEstimateAccuracy(value: unknown): TokenEstimateAccuracyView | null {
+ const record = readObject(value);
+ if (!record) return null;
+ if (record.status === 'available' || record.status === undefined) {
+ const actualTotalTokens = readNonNegativeInteger(record.actual_total_tokens);
+ const estimatedTotalTokens = readNonNegativeInteger(record.estimated_total_tokens);
+ const differenceTokens = readInteger(record.difference_tokens);
+ const differenceRatio = readFiniteNumber(record.difference_ratio);
+ const absolutePercentageError = readNonNegativeNumber(record.absolute_percentage_error);
+ if (
+ actualTotalTokens === null ||
+ estimatedTotalTokens === null ||
+ differenceTokens === null ||
+ differenceRatio === null ||
+ absolutePercentageError === null
+ ) {
+ return null;
+ }
+ return {
+ status: 'available',
+ actualTotalTokens,
+ estimatedTotalTokens,
+ differenceTokens,
+ differenceRatio,
+ absolutePercentageError
+ };
+ }
+ if (record.status === 'unavailable') {
+ return {
+ status: 'unavailable',
+ reason: typeof record.reason === 'string' ? record.reason : 'unknown',
+ usageCoverage: readNonNegativeNumber(record.usage_coverage)
+ };
+ }
+ return null;
+}
+
+function loadRunTokenUsage(suiteId: string, runId: string): TokenUsageView | null {
+ const payload = readJsonFile>(
+ `${runDirPath(suiteId, runId)}/${RUN_METRICS_FILE}`,
+ { missingOk: true }
+ );
+ if (!payload) return null;
+ const estimate = normalizeTokenEstimate(payload.token_estimate);
+ const actual = normalizeActualTokenUsage(payload.totals);
+ const accuracy = normalizeTokenEstimateAccuracy(payload.token_estimate_accuracy);
+ return estimate || actual ? { estimate, actual, accuracy } : null;
+}
+
function readSeedPayload(row: UnifiedSeedRow | undefined): Record | null {
return readObject(row?.seed);
}
@@ -1351,6 +1500,7 @@ function loadCompletedRunPageData(
const scenarioSeeds = buildScenarioSeeds(suiteSnapshot);
const promptMetrics = resolvedTab === 'prompts' ? computeRunMetrics(samples, behaviors) : null;
const auditMetrics = resolvedTab === 'audit' ? computeAuditRunMetrics(auditScores, behaviors) : null;
+ const tokenUsage = loadRunTokenUsage(suiteId, runId);
return {
suite_id: suiteId,
@@ -1371,7 +1521,8 @@ function loadCompletedRunPageData(
dimensionDefs: loadDimensions(),
multiJudgeStats: buildMultiJudgeStats(samples, auditScores),
metrics: toPromptMetricView(promptMetrics),
- auditMetrics: toAuditMetricView(auditMetrics)
+ auditMetrics: toAuditMetricView(auditMetrics),
+ tokenUsage
};
}
@@ -1415,8 +1566,15 @@ export function loadRunPageData(suiteId: string, runId: string, activeTab: 'prom
resolvedTab === 'audit' && auditScores.length === 0
? buildInferencePreviewRowsFromSnapshot(runSnapshot)
: [];
-
- if (!runSnapshot.manifest && promptCount === 0 && auditCount === 0 && inferencePreviewRows.length === 0) {
+ const tokenUsage = loadRunTokenUsage(suiteId, runId);
+
+ if (
+ !runSnapshot.manifest &&
+ promptCount === 0 &&
+ auditCount === 0 &&
+ inferencePreviewRows.length === 0 &&
+ !tokenUsage
+ ) {
return null;
}
@@ -1445,7 +1603,8 @@ export function loadRunPageData(suiteId: string, runId: string, activeTab: 'prom
dimensionDefs: loadDimensions(),
multiJudgeStats: buildMultiJudgeStats(samples, auditScores),
metrics: toPromptMetricView(promptMetrics),
- auditMetrics: toAuditMetricView(auditMetrics)
+ auditMetrics: toAuditMetricView(auditMetrics),
+ tokenUsage
};
}
diff --git a/viewer/src/lib/server/run-spawn.ts b/viewer/src/lib/server/run-spawn.ts
index 255b38c9f..f1517574c 100644
--- a/viewer/src/lib/server/run-spawn.ts
+++ b/viewer/src/lib/server/run-spawn.ts
@@ -41,7 +41,7 @@ import {
runDirPath,
suiteDirPath
} from './artifacts.js';
-import { MEASUREMENTS_ROOT } from './config.js';
+import { ARTIFACTS_ROOT, MEASUREMENTS_ROOT } from './config.js';
// ─── Errors ────────────────────────────────────────────────────────────
@@ -70,6 +70,13 @@ export class SpawnError extends Error {
}
}
+export class EstimateError extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = 'EstimateError';
+ }
+}
+
// ─── Wizard payload (mirrors the wizard's local state shape) ───────────
//
// All keys use post-PR#23 terminology (systematize / test_set / inference /
@@ -194,6 +201,10 @@ const DEFAULT_BEHAVIOR_CATEGORY_COUNT = 6;
const RUN_EVAL_CONFIG_FILE = 'eval_config.yaml';
const RUN_LOG_FILE = 'runner.log';
const RUN_PID_FILE = 'runner.pid';
+const ESTIMATE_TIMEOUT_MS = 45_000;
+const ESTIMATE_TERMINATION_GRACE_MS = 2_000;
+const ESTIMATE_PIPE_CLOSE_GRACE_MS = 250;
+const MAX_ESTIMATE_OUTPUT_BYTES = 1024 * 1024;
// Server-decided filenames for uploaded tool artifacts. Both are resolved by the
// runner relative to the config directory (the run dir).
@@ -647,6 +658,15 @@ export interface WrittenRun {
pidPath: string;
}
+function writeExtraFiles(directory: string, files: NormalizedRun['extraFiles']) {
+ for (const file of files) {
+ if (!file.name || file.name.includes('/') || file.name.includes('\\') || file.name.includes('..')) {
+ throw new Error(`Refusing to write tool artifact with unsafe name: ${file.name}`);
+ }
+ fs.writeFileSync(path.join(directory, file.name), file.content, { encoding: 'utf-8' });
+ }
+}
+
/**
* Atomically reserves the run directory and writes eval_config.yaml. The mkdir
* is the lock: if the directory already exists we refuse rather than overwrite.
@@ -679,15 +699,9 @@ export function writeRunConfigFiles(normalized: NormalizedRun): WrittenRun {
const yamlText = stringifyYaml(normalized.configObject, { lineWidth: 0 });
fs.writeFileSync(configPath, yamlText, { encoding: 'utf-8' });
- // Write uploaded tool artifacts (toolset YAML / Python tool backend) next to
- // the config. Names are server-decided constants; reject anything path-like as
+ // Names are server-decided constants; reject anything path-like as
// defense-in-depth so a future caller can't smuggle in a traversal.
- for (const file of normalized.extraFiles) {
- if (!file.name || file.name.includes('/') || file.name.includes('\\') || file.name.includes('..')) {
- throw new Error(`Refusing to write tool artifact with unsafe name: ${file.name}`);
- }
- fs.writeFileSync(path.join(runDir, file.name), file.content, { encoding: 'utf-8' });
- }
+ writeExtraFiles(runDir, normalized.extraFiles);
return { runDir, configPath, logPath, pidPath };
}
@@ -739,8 +753,7 @@ function candidateVenvDirs(): string[] {
return dirs;
}
-function resolveAssertAiCommand(configPath: string): ResolvedCommand {
- const cliArgs = ['run', '--config', configPath];
+function resolveAssertAiCommand(cliArgs: string[]): ResolvedCommand {
// Module invocation is the reliable form: it works even when the `assert-ai`
// console script was never (re)generated for a venv — e.g. after the package
// was renamed and only an older console script remains on disk.
@@ -778,7 +791,17 @@ function resolveAssertAiCommand(configPath: string): ResolvedCommand {
return { command: 'assert-ai', args: cliArgs, source: 'PATH (assert-ai)' };
}
- // 4. Last resort: a Python on PATH running the CLI as a module.
+ // 4. Windows Python launcher. It is commonly available even when python.exe
+ // itself is not on PATH, and imports the checkout from MEASUREMENTS_ROOT.
+ if (os.platform() === 'win32' && commandExistsOnPath('py.exe')) {
+ return {
+ command: 'py',
+ args: ['-3', '-m', 'assert_ai.cli', ...cliArgs],
+ source: 'PATH (py -3 -m assert_ai.cli)'
+ };
+ }
+
+ // 5. Last resort: a Python on PATH running the CLI as a module.
const pathPython = os.platform() === 'win32' ? 'python.exe' : 'python3';
if (commandExistsOnPath(pathPython) || commandExistsOnPath('python')) {
const python = commandExistsOnPath(pathPython) ? pathPython : 'python';
@@ -816,13 +839,213 @@ function commandExistsOnPath(command: string): boolean {
return false;
}
+export interface TokenEstimatePayload {
+ schema_version: number;
+ calls: number;
+ input_tokens: number;
+ output_tokens: number;
+ total_tokens: number;
+ lower_bound_tokens: number;
+ upper_bound_tokens: number;
+ stages: Record<
+ string,
+ {
+ calls: number;
+ input_tokens: number;
+ output_tokens: number;
+ total_tokens: number;
+ }
+ >;
+ notes: string[];
+}
+
+function parseTokenEstimate(stdout: string): TokenEstimatePayload {
+ const lines = stdout
+ .split(/\r?\n/)
+ .map((line) => line.trim())
+ .filter(Boolean)
+ .reverse();
+ for (const line of lines) {
+ let value: unknown;
+ try {
+ value = JSON.parse(line);
+ } catch {
+ continue;
+ }
+ if (
+ isRecord(value) &&
+ Number.isFinite(value.total_tokens) &&
+ Number.isFinite(value.lower_bound_tokens) &&
+ Number.isFinite(value.upper_bound_tokens)
+ ) {
+ return value as unknown as TokenEstimatePayload;
+ }
+ }
+ throw new EstimateError('assert-ai estimate did not return a valid token estimate.');
+}
+
+function runTokenEstimate(
+ configPath: string,
+ signal?: AbortSignal
+): Promise {
+ const resolved = resolveAssertAiCommand([
+ 'estimate',
+ '--config',
+ configPath,
+ '--output',
+ 'json'
+ ]);
+
+ return new Promise((resolve, reject) => {
+ let stdout = '';
+ let stderr = '';
+ let settled = false;
+ let terminationError: EstimateError | null = null;
+ let child: ChildProcess;
+ try {
+ child = spawn(resolved.command, resolved.args, {
+ cwd: MEASUREMENTS_ROOT,
+ env: process.env,
+ detached: os.platform() !== 'win32',
+ stdio: ['ignore', 'pipe', 'pipe'],
+ windowsHide: true
+ });
+ } catch (err) {
+ reject(
+ new EstimateError(
+ `Failed to start assert-ai estimate via ${resolved.source}: ${(err as Error).message ?? String(err)}`
+ )
+ );
+ return;
+ }
+
+ let timeout: ReturnType;
+ let forceKillTimeout: ReturnType | undefined;
+ let pipeCloseTimeout: ReturnType | undefined;
+ const onAbort = () => {
+ requestTermination(new EstimateError('Token estimation was cancelled.'));
+ };
+ const cleanup = () => {
+ clearTimeout(timeout);
+ if (forceKillTimeout) clearTimeout(forceKillTimeout);
+ if (pipeCloseTimeout) clearTimeout(pipeCloseTimeout);
+ signal?.removeEventListener('abort', onAbort);
+ };
+ const finish = (error: EstimateError | null, payload?: TokenEstimatePayload) => {
+ if (settled) return;
+ settled = true;
+ cleanup();
+ if (error) reject(error);
+ else if (payload) resolve(payload);
+ };
+ const killChild = (killSignal: 'SIGTERM' | 'SIGKILL') => {
+ if (os.platform() !== 'win32' && child.pid !== undefined) {
+ try {
+ process.kill(-child.pid, killSignal);
+ return;
+ } catch {
+ // Fall back to the direct child if its process group is gone.
+ }
+ }
+ try {
+ child.kill(killSignal);
+ } catch {
+ // A concurrent process exit will still deliver `close`.
+ }
+ };
+ const requestTermination = (error: EstimateError) => {
+ if (settled || terminationError) return;
+ terminationError = error;
+ killChild('SIGTERM');
+ forceKillTimeout = setTimeout(() => {
+ killChild('SIGKILL');
+ pipeCloseTimeout = setTimeout(() => {
+ child.stdout?.destroy();
+ child.stderr?.destroy();
+ }, ESTIMATE_PIPE_CLOSE_GRACE_MS);
+ }, ESTIMATE_TERMINATION_GRACE_MS);
+ };
+ const append = (current: string, chunk: Buffer): string => {
+ if (terminationError) return current;
+ const next = current + chunk.toString('utf-8');
+ if (Buffer.byteLength(next, 'utf-8') > MAX_ESTIMATE_OUTPUT_BYTES) {
+ requestTermination(new EstimateError('assert-ai estimate produced too much output.'));
+ return current;
+ }
+ return next;
+ };
+ child.stdout?.on('data', (chunk: Buffer) => {
+ stdout = append(stdout, chunk);
+ });
+ child.stderr?.on('data', (chunk: Buffer) => {
+ stderr = append(stderr, chunk);
+ });
+ child.on('error', (err: Error) => {
+ terminationError =
+ terminationError ??
+ new EstimateError(
+ `assert-ai estimate failed to start via ${resolved.source}: ${err.message}`
+ );
+ });
+ child.on('close', (code) => {
+ if (settled) return;
+ if (terminationError) {
+ finish(terminationError);
+ return;
+ }
+ if (code !== 0) {
+ const detail = stderr.trim().slice(-2000);
+ finish(
+ new EstimateError(
+ `assert-ai estimate exited with code ${code ?? 'unknown'}${detail ? `: ${detail}` : ''}`
+ )
+ );
+ return;
+ }
+ try {
+ finish(null, parseTokenEstimate(stdout));
+ } catch (err) {
+ finish(err instanceof EstimateError ? err : new EstimateError(String(err)));
+ }
+ });
+ timeout = setTimeout(() => {
+ requestTermination(new EstimateError('Token estimation timed out after 45 seconds.'));
+ }, ESTIMATE_TIMEOUT_MS);
+ signal?.addEventListener('abort', onAbort, { once: true });
+ if (signal?.aborted) onAbort();
+ });
+}
+
+/**
+ * Estimate a normalized wizard payload from a temporary config. This never
+ * reserves a run directory and always removes its temporary files.
+ */
+export async function estimateAssertAiRun(
+ normalized: NormalizedRun,
+ signal?: AbortSignal
+): Promise {
+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'assert-ai-estimate-'));
+ const configPath = path.join(tempDir, RUN_EVAL_CONFIG_FILE);
+ try {
+ const configObject = cloneRecord(normalized.configObject);
+ const resultsRoot = path.resolve(ARTIFACTS_ROOT);
+ configObject.artifacts_root = path.dirname(resultsRoot);
+ configObject.results_dir = resultsRoot;
+ fs.writeFileSync(configPath, stringifyYaml(configObject, { lineWidth: 0 }), 'utf-8');
+ writeExtraFiles(tempDir, normalized.extraFiles);
+ return await runTokenEstimate(configPath, signal);
+ } finally {
+ fs.rmSync(tempDir, { recursive: true, force: true });
+ }
+}
+
/**
* Spawn assert-ai detached, wait for the OS to confirm the spawn (or fail).
* Only after we hear back do we resolve — that way a missing `assert-ai`
* binary surfaces as a 500 instead of a 200 followed by a forever-pending monitor.
*/
export function spawnAssertAiRun(written: WrittenRun): Promise {
- const resolved = resolveAssertAiCommand(written.configPath);
+ const resolved = resolveAssertAiCommand(['run', '--config', written.configPath]);
let logFd: number;
try {
diff --git a/viewer/src/lib/token-usage.ts b/viewer/src/lib/token-usage.ts
new file mode 100644
index 000000000..ff27aa078
--- /dev/null
+++ b/viewer/src/lib/token-usage.ts
@@ -0,0 +1,64 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+type EstimateRange = {
+ lowerBoundTokens: number;
+ upperBoundTokens: number;
+};
+
+const compactTokenFormatter = new Intl.NumberFormat('en-US', {
+ notation: 'compact',
+ maximumFractionDigits: 1
+});
+
+const TOKEN_STAGE_LABELS: Record = {
+ systematize: 'Behavior categories',
+ test_set: 'Test set',
+ inference: 'Inference',
+ judge: 'Scoring'
+};
+
+export function formatTokenCount(value: number): string {
+ const rounded = Math.max(0, Math.round(value));
+ return rounded < 1000 ? rounded.toLocaleString('en-US') : compactTokenFormatter.format(rounded);
+}
+
+export function formatTokenPercent(value: number): string {
+ return `${(Math.max(0, value) * 100).toFixed(1)}%`;
+}
+
+export function formatActualVsEstimate(differenceRatio: number): string {
+ if (Math.abs(differenceRatio) < 0.0005) return 'Matched estimate';
+ return `${Math.abs(differenceRatio * 100).toFixed(1)}% ${differenceRatio > 0 ? 'higher' : 'lower'}`;
+}
+
+export function actualVsEstimateSentence(differenceRatio: number): string {
+ if (Math.abs(differenceRatio) < 0.0005) return 'Actual usage matched the pre-run estimate.';
+ return `Actual usage was ${differenceRatio > 0 ? 'above' : 'below'} the pre-run estimate.`;
+}
+
+export function actualIsWithinEstimate(
+ actualTokens: number,
+ estimate: EstimateRange
+): boolean {
+ return actualTokens >= estimate.lowerBoundTokens && actualTokens <= estimate.upperBoundTokens;
+}
+
+export function tokenAccuracyUnavailableMessage(
+ reason: string,
+ usageCoverage: number | null
+): string {
+ if (reason === 'pipeline_incomplete') return 'The pipeline did not complete.';
+ if (reason === 'pipeline_partial') return 'The pipeline returned a partial result.';
+ if (reason === 'no_usage_reported') return 'The provider did not report token usage.';
+ if (reason === 'provider_usage_incomplete') {
+ return usageCoverage === null
+ ? 'Some provider calls did not report complete usage.'
+ : `Complete usage was reported for ${formatTokenPercent(usageCoverage)} of calls.`;
+ }
+ return 'A complete comparison is not available for this run.';
+}
+
+export function tokenStageLabel(stage: string): string {
+ return TOKEN_STAGE_LABELS[stage] ?? stage.replace(/_/g, ' ');
+}
diff --git a/viewer/src/lib/types.ts b/viewer/src/lib/types.ts
index 370b1ec19..8a6adf098 100644
--- a/viewer/src/lib/types.ts
+++ b/viewer/src/lib/types.ts
@@ -310,6 +310,54 @@ export interface RunMetrics {
dimensions: Record;
}
+export interface TokenStageEstimateView {
+ calls: number;
+ inputTokens: number;
+ outputTokens: number;
+ totalTokens: number;
+}
+
+export interface TokenEstimateView extends TokenStageEstimateView {
+ lowerBoundTokens: number;
+ upperBoundTokens: number;
+ stages: Record;
+ notes: string[];
+}
+
+export interface TokenActualUsageView {
+ requests: number;
+ calls: number;
+ missingUsageCalls: number;
+ inputTokens: number;
+ outputTokens: number;
+ totalTokens: number;
+ cachedInputTokens: number;
+ cacheCreationInputTokens: number;
+ cacheHitRate: number;
+ usageCoverage: number;
+}
+
+export type TokenEstimateAccuracyView =
+ | {
+ status: 'available';
+ actualTotalTokens: number;
+ estimatedTotalTokens: number;
+ differenceTokens: number;
+ differenceRatio: number;
+ absolutePercentageError: number;
+ }
+ | {
+ status: 'unavailable';
+ reason: string;
+ usageCoverage: number | null;
+ };
+
+export interface TokenUsageView {
+ estimate: TokenEstimateView | null;
+ actual: TokenActualUsageView | null;
+ accuracy: TokenEstimateAccuracyView | null;
+}
+
export interface RunListItem {
run_id: string;
has_judged: boolean;
diff --git a/viewer/src/routes/api/runs/estimate/+server.ts b/viewer/src/routes/api/runs/estimate/+server.ts
new file mode 100644
index 000000000..bb2fb7428
--- /dev/null
+++ b/viewer/src/routes/api/runs/estimate/+server.ts
@@ -0,0 +1,57 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { json } from '@sveltejs/kit';
+import {
+ estimateAssertAiRun,
+ EstimateError,
+ normalizeWizardPayload,
+ WizardValidationError
+} from '$lib/server/run-spawn.js';
+import type { RequestHandler } from './$types.js';
+
+/**
+ * POST /api/runs/estimate
+ *
+ * Validate the same payload used to create a run, then execute the local,
+ * read-only token estimator against a temporary config. No provider calls are
+ * made and no run directory is reserved.
+ */
+export const POST: RequestHandler = async ({ request }) => {
+ let raw: unknown;
+ try {
+ raw = await request.json();
+ } catch (err) {
+ return json(
+ { error: 'Request body must be valid JSON.', details: [(err as Error).message] },
+ { status: 400 }
+ );
+ }
+
+ let normalized;
+ try {
+ normalized = normalizeWizardPayload(raw);
+ } catch (err) {
+ if (err instanceof WizardValidationError) {
+ return json(
+ { error: 'Wizard payload validation failed.', details: err.details },
+ { status: 400 }
+ );
+ }
+ throw err;
+ }
+
+ try {
+ const estimate = await estimateAssertAiRun(normalized, request.signal);
+ return json({ estimate, warnings: normalized.warnings });
+ } catch (err) {
+ if (request.signal.aborted) {
+ return new Response(null, { status: 499 });
+ }
+ const message = err instanceof EstimateError ? err.message : (err as Error).message ?? String(err);
+ return json(
+ { error: 'Token estimate unavailable.', details: [message] },
+ { status: 500 }
+ );
+ }
+};
diff --git a/viewer/src/routes/new/+page.svelte b/viewer/src/routes/new/+page.svelte
index 047fdc74b..e634eaef0 100644
--- a/viewer/src/routes/new/+page.svelte
+++ b/viewer/src/routes/new/+page.svelte
@@ -17,6 +17,7 @@
import { onMount } from 'svelte';
import { goto } from '$app/navigation';
import InfoTooltip from '$lib/components/InfoTooltip.svelte';
+ import { formatTokenCount } from '$lib/token-usage.js';
// ── Constants ───────────────────────────────────────────────────
const STEPS = [
@@ -42,6 +43,19 @@
interface KnownSuite { suite_id: string; behavior_name: string; behavior_category_count: number }
interface JudgeDimension { name: string; description: string; rubric: string }
interface EvalDimension { name: string; levels: string[] }
+ interface PreRunTokenEstimate {
+ calls: number;
+ input_tokens: number;
+ output_tokens: number;
+ total_tokens: number;
+ lower_bound_tokens: number;
+ upper_bound_tokens: number;
+ }
+ interface TokenEstimateResponse {
+ estimate?: PreRunTokenEstimate;
+ error?: string;
+ details?: string[];
+ }
// ── Catalog data ────────────────────────────────────────────────
let knownBehaviors = $state([]);
@@ -131,6 +145,9 @@
let runId = $state('v1');
let submitting = $state(false);
let submitError = $state('');
+ let tokenEstimate = $state(null);
+ let tokenEstimateLoading = $state(false);
+ let tokenEstimateError = $state('');
let showDiscardModal = $state(false);
let isDirty = $state(false);
@@ -424,6 +441,30 @@
});
let step3Valid = $derived(runId.trim().length > 0);
+ $effect(() => {
+ const hasEstimateInputs =
+ step1BehaviorValid && step1ContextValid && step1ToolsValid && step2Valid && step3Valid;
+ if (currentStep !== 3 || !hasEstimateInputs) {
+ tokenEstimate = null;
+ tokenEstimateLoading = false;
+ tokenEstimateError = '';
+ return;
+ }
+
+ const payload = buildRunPayload();
+ const controller = new AbortController();
+ tokenEstimate = null;
+ tokenEstimateLoading = true;
+ tokenEstimateError = '';
+ const timer = window.setTimeout(() => {
+ void loadTokenEstimate(payload, controller.signal);
+ }, 250);
+ return () => {
+ window.clearTimeout(timer);
+ controller.abort();
+ };
+ });
+
function stepValid(s: number) {
return s === 1 ? step1Valid : s === 2 ? step2Valid : s === 3 ? step3Valid : false;
}
@@ -507,12 +548,8 @@
markDirty();
}
- async function handleSubmit() {
- if (submitting) return;
- submitting = true;
- submitError = '';
-
- const payload = {
+ function buildRunPayload() {
+ return {
behavior:
step1Mode === 'select'
? { mode: 'existing', name: selectedBehavior?.name, suiteId: selectedBehavior?.suiteId }
@@ -571,7 +608,41 @@
}
: {})
};
+ }
+
+ async function loadTokenEstimate(
+ payload: ReturnType,
+ signal: AbortSignal
+ ) {
+ try {
+ const response = await fetch('/api/runs/estimate', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(payload),
+ signal
+ });
+ const body = (await response.json()) as TokenEstimateResponse;
+ if (!response.ok || !body.estimate) {
+ const details = body.details?.length ? ` ${body.details.join(' ')}` : '';
+ throw new Error(`${body.error ?? `HTTP ${response.status}`}${details}`);
+ }
+ if (!signal.aborted) tokenEstimate = body.estimate;
+ } catch (err) {
+ if (!signal.aborted) {
+ tokenEstimate = null;
+ tokenEstimateError = (err as Error).message ?? String(err);
+ }
+ } finally {
+ if (!signal.aborted) tokenEstimateLoading = false;
+ }
+ }
+ async function handleSubmit() {
+ if (submitting) return;
+ submitting = true;
+ submitError = '';
+
+ const payload = buildRunPayload();
let response: Response;
try {
response = await fetch('/api/runs', {
@@ -1588,6 +1659,28 @@
Summary & submit
Review your configuration and submit the evaluation run.
+
+
+
Estimated token usage
+
Conservative local estimate; no provider call.
+
+ {#if tokenEstimateLoading}
+
+ {:else if tokenEstimate}
+
+ ~{formatTokenCount(tokenEstimate.total_tokens)}
+ Likely {formatTokenCount(tokenEstimate.lower_bound_tokens)}–{formatTokenCount(tokenEstimate.upper_bound_tokens)} · {tokenEstimate.calls} {tokenEstimate.calls === 1 ? 'call' : 'calls'}
+
+ {:else if tokenEstimateError}
+
Estimate unavailable
+ {:else}
+
Complete required fields to estimate
+ {/if}
+
Summary
@@ -1670,10 +1763,12 @@
{#if currentStep < 3}
{:else}
-
+{#if data.tokenUsage}
+
+{/if}
+
{#if !hasPromptEval && !hasAuditContent}