Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 22 additions & 24 deletions packages/client/src/__tests__/judges.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -386,44 +386,42 @@ describe('runJudge result trackData', () => {
vi.clearAllMocks();
});

it('does not inherit the parent modelKey/modelVersion when the judge trackData has none', async () => {
it('returns the evaluated (parent) config identity, identical to the inline track payload', async () => {
// The judge's own trackData differs on every identity key; none of it may leak into the result.
mockExecuteAndTrack.mockResolvedValue({
usage: { input: 1, output: 1, total: 2 },
response: '{"score":0.9,"reasoning":"good"}',
trackData: {
runId: 'judge-run',
configKey: 'judge-key',
variationKey: 'j1',
version: 1,
version: 3,
modelName: 'claude',
providerName: 'Anthropic',
modelKey: 'judge-model',
modelVersion: 2,
},
});
const result = await runJudge(makeTask(parentTrackData), [makeHandler()]);
expect(result).not.toBeNull();
expect('modelKey' in result!.trackData).toBe(false);
expect('modelVersion' in result!.trackData).toBe(false);
expect(result!.trackData.judgeConfigKey).toBe('judge-key');
expect(result!.trackData.graphKey).toBe('g1');
});
mockExtractVariation.mockResolvedValue({ config: mockJudgeConfig, meta: mockJudgeMeta });
const track = vi.fn();
mockGetClient.mockReturnValue({ track });

it('keeps the judge own modelKey/modelVersion over the parent values', async () => {
mockExecuteAndTrack.mockResolvedValue({
usage: { input: 1, output: 1, total: 2 },
response: '{"score":0.9,"reasoning":"good"}',
trackData: {
runId: 'judge-run',
configKey: 'judge-key',
variationKey: 'j1',
version: 1,
modelName: 'claude',
providerName: 'Anthropic',
modelKey: 'judge-model',
modelVersion: 2,
await runJudges({
config: {
model: { name: 'gpt-4o' },
provider: { name: 'OpenAI' },
instructions: 'Be helpful.',
judgeConfiguration: { judges: [{ key: 'judge-key', samplingRate: 1 }] },
},
userContext: mockContext,
handler: makeHandler(),
llmResponse: 'response',
baseTrackData: parentTrackData,
});
const inlineTrackData = track.mock.calls[0][2];

const result = await runJudge(makeTask(parentTrackData), [makeHandler()]);
expect(result!.trackData.modelKey).toBe('judge-model');
expect(result!.trackData.modelVersion).toBe(2);
expect(result!.trackData).toEqual({ ...parentTrackData, judgeConfigKey: 'judge-key' });
expect(result!.trackData).toEqual(inlineTrackData);
});
});
18 changes: 4 additions & 14 deletions packages/client/src/judges.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,7 @@ import type {
TrackData,
VariationMeta,
} from './types.js';
import {
collapseMessagesToInstructions,
normalizeMode,
omitModelStamps,
parseJSONWithPossibleFences,
} from './utils.js';
import { collapseMessagesToInstructions, normalizeMode, parseJSONWithPossibleFences } from './utils.js';

/**
* Resolves a judge's own AI Config, returning `null` instead of throwing when it
Expand Down Expand Up @@ -341,11 +336,7 @@ export const runJudge = async (task: JudgeTask, handlers: ProviderHandler[]): Pr
const messageHistory = [actualOutput, FORMATTING_INSTRUCTIONS].join('\n\n');

return withJudgeEvaluation(configKey, async (recordEvaluation) => {
const {
usage,
response: rawResponse,
trackData,
} = await executeAndTrack({
const { usage, response: rawResponse } = await executeAndTrack({
configKey,
config: effectiveConfig,
meta: judgeMeta,
Expand All @@ -371,9 +362,8 @@ export const runJudge = async (task: JudgeTask, handlers: ProviderHandler[]): Pr
score,
response: reasoning,
usage,
// A judge without a pinned model config must not inherit the parent's
// modelKey / modelVersion; other parent-only keys (graphKey, ...) still carry over.
trackData: { ...omitModelStamps(parentTrackData), ...trackData, judgeConfigKey: configKey },
// The score belongs to the evaluated (parent) config, exactly as in the inline path.
trackData: { ...parentTrackData, judgeConfigKey: configKey },
};
});
};
Loading