From 66901342b9aa53ca8bafa2b684c6d0e22a942319 Mon Sep 17 00:00:00 2001
From: Jeff Dupont <jdupont@launchdarkly.com>
Date: Tue, 20 Oct 2026 10:33:46 -0700
Subject: [PATCH] fix(AIC-3576): record deferred judge scores under the
 evaluated config

RunJudge built its result trackData from the judge's own executeAndTrack
trackData, copying over only the parent's GraphKey, so a deferred judge score
was tracked under the judge config's configKey, variationKey, version, runId
and model stamps instead of the config it evaluated. It now returns the
parent's trackData plus JudgeConfigKey, the same payload the inline path
tracks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 aiserver/judges.go      |  4 +--
 aiserver/judges_test.go | 54 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/aiserver/judges.go b/aiserver/judges.go
index 3d9dad5..1ecce15 100644
--- a/aiserver/judges.go
+++ b/aiserver/judges.go
@@ -258,8 +258,8 @@ func RunJudge(ctx context.Context, task JudgeTask, handlers []*ProviderHandler)
 			explanation = reasoning
 		}
 		record(score, explanation)
-		td := res.TrackData
-		td.GraphKey = task.ParentTrackData.GraphKey
+		// The score belongs to the evaluated (parent) config, exactly as in the inline path.
+		td := task.ParentTrackData
 		td.JudgeConfigKey = task.ConfigKey
 		out = &JudgeRunResult{Score: score, Response: reasoning, Usage: res.Usage, TrackData: td}
 		return nil
diff --git a/aiserver/judges_test.go b/aiserver/judges_test.go
index f1ee9a9..01ca9c8 100644
--- a/aiserver/judges_test.go
+++ b/aiserver/judges_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -198,3 +199,56 @@ func TestPickJudgeHandlerCollapse(t *testing.T) {
 		t.Errorf("collapse = %+v", got)
 	}
 }
+
+// A deferred judge result is tracked under the evaluated config, with exactly
+// the payload the inline path sends (AIC-3576).
+func TestDeferredJudgeTrackDataMatchesInline(t *testing.T) {
+	parent := enabledConfig("P", "", judgedConfig(nil))
+	parent["_ldMeta"] = map[string]any{"enabled": true, "variationKey": "v1", "version": 3, "modelKey": "parent-model", "modelVersion": 7}
+	judge := enabledConfig("P", "judge", map[string]any{"evaluationMetricKey": "quality"})
+	judge["_ldMeta"] = map[string]any{"enabled": true, "variationKey": "j1", "version": 5, "mode": "judge", "modelKey": "judge-model", "modelVersion": 2}
+	f := withFake(t, map[string]any{"cfg": parent, "judge": judge})
+	h := stubHandler("P", ModeMessages, `{"score":0.8,"reasoning":"good"}`, nil, nil)
+	args := ConfigArgs{Key: "cfg", Handlers: []*ProviderHandler{h}}
+
+	if _, err := Config(args).Invoke(bg, "question", userCtx); err != nil {
+		t.Fatal(err)
+	}
+	q := f.events("quality")
+	if len(q) != 1 {
+		t.Fatalf("metric events = %v", q)
+	}
+	inline := q[0].Data
+
+	args.SkipJudges = true
+	resp, err := Config(args).Invoke(bg, "question", userCtx)
+	if err != nil || len(resp.JudgeTasks) != 1 {
+		t.Fatalf("tasks=%v err=%v", resp.JudgeTasks, err)
+	}
+	res, err := RunJudge(bg, resp.JudgeTasks[0], []*ProviderHandler{h})
+	if err != nil || res == nil {
+		t.Fatalf("res=%v err=%v", res, err)
+	}
+	want := resp.TrackData
+	want.JudgeConfigKey = "judge"
+	if !reflect.DeepEqual(res.TrackData, want) {
+		t.Errorf("deferred trackData = %+v, want parent's %+v", res.TrackData, want)
+	}
+
+	// Same keys and values as the inline payload; only the per-invocation runId differs.
+	b, _ := json.Marshal(res.TrackData)
+	var deferred map[string]any
+	_ = json.Unmarshal(b, &deferred)
+	delete(deferred, "runId")
+	delete(inline, "runId")
+	if !reflect.DeepEqual(deferred, inline) {
+		t.Errorf("deferred %v != inline %v", deferred, inline)
+	}
+
+	// A graph judge's task carries the graphKey in its parent trackData; it survives.
+	task := resp.JudgeTasks[0]
+	task.ParentTrackData.GraphKey = "g1"
+	if res, _ := RunJudge(bg, task, []*ProviderHandler{h}); res == nil || res.TrackData.GraphKey != "g1" {
+		t.Errorf("graphKey lost: %+v", res)
+	}
+}