Skip to content

Commit 7231e0c

Browse files
fengting124fengting124
authored
Add evaluation batch execution framework
Adds the backend batch execution layer for evaluation runs, including model-call boundaries, retry state, failure persistence, HTTP run/retry endpoints, tests, and documentation.
1 parent b1045e4 commit 7231e0c

19 files changed

Lines changed: 862 additions & 10 deletions

‎backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClient.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package com.fengting.aigcforensics.evaluation.client;
2+
3+
import java.nio.charset.StandardCharsets;
4+
import java.util.zip.CRC32;
5+
6+
import org.springframework.stereotype.Component;
7+
8+
import com.fengting.aigcforensics.domain.ModelLabel;
9+
10+
@Component
11+
public class DeterministicEvaluationModelClient implements EvaluationModelClient {
12+
13+
@Override
14+
public EvaluationModelResult predict(EvaluationModelRequest request) {
15+
double score = stableScore(request.modelId() + ":" + request.filename());
16+
ModelLabel label = score >= 0.5 ? ModelLabel.SYNTHETIC : ModelLabel.AUTHENTIC;
17+
int latencyMs = 10 + (int) Math.round(score * 20);
18+
return new EvaluationModelResult(label, score, latencyMs);
19+
}
20+
21+
private double stableScore(String value) {
22+
CRC32 crc32 = new CRC32();
23+
crc32.update(value.getBytes(StandardCharsets.UTF_8));
24+
return crc32.getValue() / (double) 0xffffffffL;
25+
}
26+
}
27+

‎backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelClient.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package com.fengting.aigcforensics.evaluation.client;
2+
3+
public interface EvaluationModelClient {
4+
5+
EvaluationModelResult predict(EvaluationModelRequest request);
6+
}
7+

‎backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelRequest.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package com.fengting.aigcforensics.evaluation.client;
2+
3+
public record EvaluationModelRequest(
4+
String evaluationId,
5+
String sampleId,
6+
String modelId,
7+
String filename) {
8+
}
9+

‎backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelResult.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package com.fengting.aigcforensics.evaluation.client;
2+
3+
import com.fengting.aigcforensics.domain.ModelLabel;
4+
5+
public record EvaluationModelResult(
6+
ModelLabel predictedLabel,
7+
double score,
8+
int latencyMs) {
9+
}
10+

‎backend-java/src/main/java/com/fengting/aigcforensics/evaluation/controller/EvaluationController.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.fengting.aigcforensics.evaluation.dto.EvaluationDetailResponse;
1717
import com.fengting.aigcforensics.evaluation.dto.EvaluationRunResponse;
1818
import com.fengting.aigcforensics.evaluation.dto.EvaluationSampleResponse;
19+
import com.fengting.aigcforensics.evaluation.service.EvaluationExecutionService;
1920
import com.fengting.aigcforensics.evaluation.service.EvaluationService;
2021

2122
import jakarta.validation.Valid;
@@ -25,9 +26,13 @@
2526
public class EvaluationController {
2627

2728
private final EvaluationService evaluationService;
29+
private final EvaluationExecutionService evaluationExecutionService;
2830

29-
public EvaluationController(EvaluationService evaluationService) {
31+
public EvaluationController(
32+
EvaluationService evaluationService,
33+
EvaluationExecutionService evaluationExecutionService) {
3034
this.evaluationService = evaluationService;
35+
this.evaluationExecutionService = evaluationExecutionService;
3136
}
3237

3338
@PostMapping
@@ -46,6 +51,16 @@ public EvaluationDetailResponse getEvaluation(@PathVariable String evaluationId)
4651
return evaluationService.getEvaluation(evaluationId);
4752
}
4853

54+
@PostMapping("/{evaluationId}/run")
55+
public EvaluationDetailResponse runEvaluation(@PathVariable String evaluationId) {
56+
return evaluationExecutionService.runEvaluation(evaluationId);
57+
}
58+
59+
@PostMapping("/{evaluationId}/retry")
60+
public EvaluationDetailResponse retryEvaluation(@PathVariable String evaluationId) {
61+
return evaluationExecutionService.runEvaluation(evaluationId);
62+
}
63+
4964
@GetMapping("/{evaluationId}/samples")
5065
public List<EvaluationSampleResponse> listSamples(
5166
@PathVariable String evaluationId,

‎backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationRun.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ public class EvaluationRun {
5353
@Column(name = "f1_score")
5454
private Double f1;
5555

56+
@Column(name = "attempt_count", nullable = false)
57+
private int attemptCount;
58+
59+
@Column(name = "max_attempts", nullable = false)
60+
private int maxAttempts;
61+
5662
@Column(name = "created_at", nullable = false)
5763
private Instant createdAt;
5864

@@ -80,6 +86,8 @@ public EvaluationRun(
8086
Double precision,
8187
Double recall,
8288
Double f1,
89+
int attemptCount,
90+
int maxAttempts,
8391
Instant createdAt,
8492
Instant startedAt,
8593
Instant completedAt,
@@ -95,6 +103,8 @@ public EvaluationRun(
95103
this.precision = precision;
96104
this.recall = recall;
97105
this.f1 = f1;
106+
this.attemptCount = attemptCount;
107+
this.maxAttempts = maxAttempts;
98108
this.createdAt = createdAt;
99109
this.startedAt = startedAt;
100110
this.completedAt = completedAt;
@@ -149,6 +159,14 @@ public Double getF1() {
149159
return f1;
150160
}
151161

162+
public int getAttemptCount() {
163+
return attemptCount;
164+
}
165+
166+
public int getMaxAttempts() {
167+
return maxAttempts;
168+
}
169+
152170
public Instant getCreatedAt() {
153171
return createdAt;
154172
}
@@ -164,4 +182,40 @@ public Instant getCompletedAt() {
164182
public String getFailureReason() {
165183
return failureReason;
166184
}
185+
186+
public boolean canRetry() {
187+
return attemptCount < maxAttempts;
188+
}
189+
190+
public void markStarted(Instant startedAt) {
191+
this.status = EvaluationStatus.RUNNING;
192+
this.attemptCount++;
193+
this.startedAt = startedAt;
194+
this.completedAt = null;
195+
this.failureReason = null;
196+
}
197+
198+
public void markCompleted(
199+
int completedSamples,
200+
Double accuracy,
201+
Double precision,
202+
Double recall,
203+
Double f1,
204+
Instant completedAt) {
205+
this.status = EvaluationStatus.COMPLETED;
206+
this.completedSamples = completedSamples;
207+
this.accuracy = accuracy;
208+
this.precision = precision;
209+
this.recall = recall;
210+
this.f1 = f1;
211+
this.completedAt = completedAt;
212+
this.failureReason = null;
213+
}
214+
215+
public void markFailed(String failureReason, int completedSamples, Instant completedAt) {
216+
this.status = EvaluationStatus.FAILED;
217+
this.completedSamples = completedSamples;
218+
this.completedAt = completedAt;
219+
this.failureReason = failureReason;
220+
}
167221
}

‎backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationSample.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,4 +122,16 @@ public String getFailureReason() {
122122
public Instant getCreatedAt() {
123123
return createdAt;
124124
}
125+
126+
public void markPredicted(ModelLabel predictedLabel, Double score, Integer latencyMs) {
127+
this.predictedLabel = predictedLabel;
128+
this.score = score;
129+
this.latencyMs = latencyMs;
130+
this.correct = predictedLabel == groundTruthLabel;
131+
this.failureReason = null;
132+
}
133+
134+
public void markFailed(String failureReason) {
135+
this.failureReason = failureReason;
136+
}
125137
}

‎backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationDetailResponse.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ public record EvaluationDetailResponse(
1717
Double precision,
1818
Double recall,
1919
Double f1,
20+
int attemptCount,
21+
int maxAttempts,
2022
Instant createdAt,
2123
Instant startedAt,
2224
Instant completedAt,

‎backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationRunResponse.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ public record EvaluationRunResponse(
1616
Double precision,
1717
Double recall,
1818
Double f1,
19+
int attemptCount,
20+
int maxAttempts,
1921
Instant createdAt,
2022
Instant startedAt,
2123
Instant completedAt,

‎backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionService.java‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
package com.fengting.aigcforensics.evaluation.service;
2+
3+
import java.time.Clock;
4+
import java.time.Instant;
5+
import java.util.List;
6+
7+
import org.springframework.beans.factory.annotation.Autowired;
8+
import org.springframework.stereotype.Service;
9+
import org.springframework.transaction.annotation.Transactional;
10+
11+
import com.fengting.aigcforensics.evaluation.client.EvaluationModelClient;
12+
import com.fengting.aigcforensics.evaluation.client.EvaluationModelRequest;
13+
import com.fengting.aigcforensics.evaluation.client.EvaluationModelResult;
14+
import com.fengting.aigcforensics.evaluation.domain.EvaluationRun;
15+
import com.fengting.aigcforensics.evaluation.domain.EvaluationSample;
16+
import com.fengting.aigcforensics.evaluation.domain.EvaluationStatus;
17+
import com.fengting.aigcforensics.evaluation.dto.EvaluationDetailResponse;
18+
import com.fengting.aigcforensics.evaluation.dto.EvaluationSampleResponse;
19+
import com.fengting.aigcforensics.evaluation.repository.EvaluationRunRepository;
20+
import com.fengting.aigcforensics.evaluation.repository.EvaluationSampleRepository;
21+
import com.fengting.aigcforensics.service.ResourceNotFoundException;
22+
23+
@Service
24+
public class EvaluationExecutionService {
25+
26+
private final EvaluationRunRepository evaluationRunRepository;
27+
private final EvaluationSampleRepository evaluationSampleRepository;
28+
private final EvaluationMetricsCalculator metricsCalculator;
29+
private final EvaluationModelClient modelClient;
30+
private final Clock clock;
31+
32+
@Autowired
33+
public EvaluationExecutionService(
34+
EvaluationRunRepository evaluationRunRepository,
35+
EvaluationSampleRepository evaluationSampleRepository,
36+
EvaluationMetricsCalculator metricsCalculator,
37+
EvaluationModelClient modelClient) {
38+
this(
39+
evaluationRunRepository,
40+
evaluationSampleRepository,
41+
metricsCalculator,
42+
modelClient,
43+
Clock.systemUTC());
44+
}
45+
46+
EvaluationExecutionService(
47+
EvaluationRunRepository evaluationRunRepository,
48+
EvaluationSampleRepository evaluationSampleRepository,
49+
EvaluationMetricsCalculator metricsCalculator,
50+
EvaluationModelClient modelClient,
51+
Clock clock) {
52+
this.evaluationRunRepository = evaluationRunRepository;
53+
this.evaluationSampleRepository = evaluationSampleRepository;
54+
this.metricsCalculator = metricsCalculator;
55+
this.modelClient = modelClient;
56+
this.clock = clock;
57+
}
58+
59+
@Transactional
60+
public EvaluationDetailResponse runEvaluation(String evaluationId) {
61+
EvaluationRun run = findRun(evaluationId);
62+
List<EvaluationSample> samples = evaluationSampleRepository.findByEvaluationIdOrderByCreatedAtAsc(evaluationId);
63+
if (run.getStatus() == EvaluationStatus.COMPLETED) {
64+
return toDetailResponse(run, samples);
65+
}
66+
if (run.getStatus() == EvaluationStatus.FAILED && !run.canRetry()) {
67+
throw new IllegalStateException("Evaluation retry attempts exhausted: " + evaluationId);
68+
}
69+
70+
run.markStarted(Instant.now(clock));
71+
for (EvaluationSample sample : samples) {
72+
if (sample.getPredictedLabel() != null) {
73+
continue;
74+
}
75+
try {
76+
EvaluationModelResult result = modelClient.predict(new EvaluationModelRequest(
77+
run.getEvaluationId(),
78+
sample.getSampleId(),
79+
run.getModelId(),
80+
sample.getFilename()));
81+
sample.markPredicted(result.predictedLabel(), result.score(), result.latencyMs());
82+
} catch (RuntimeException exception) {
83+
String failureReason = failureMessage(exception);
84+
sample.markFailed(failureReason);
85+
run.markFailed(failureReason, countCompleted(samples), Instant.now(clock));
86+
evaluationSampleRepository.saveAll(samples);
87+
evaluationRunRepository.save(run);
88+
return toDetailResponse(run, samples);
89+
}
90+
}
91+
92+
EvaluationMetrics metrics = metricsCalculator.calculate(samples.stream()
93+
.map(sample -> new EvaluationPredictionCase(sample.getGroundTruthLabel(), sample.getPredictedLabel()))
94+
.toList());
95+
run.markCompleted(
96+
samples.size(),
97+
metrics.accuracy(),
98+
metrics.precision(),
99+
metrics.recall(),
100+
metrics.f1(),
101+
Instant.now(clock));
102+
evaluationSampleRepository.saveAll(samples);
103+
evaluationRunRepository.save(run);
104+
return toDetailResponse(run, samples);
105+
}
106+
107+
private int countCompleted(List<EvaluationSample> samples) {
108+
return (int) samples.stream().filter(sample -> sample.getPredictedLabel() != null).count();
109+
}
110+
111+
private String failureMessage(RuntimeException exception) {
112+
if (exception.getMessage() == null || exception.getMessage().isBlank()) {
113+
return exception.getClass().getSimpleName();
114+
}
115+
return exception.getMessage();
116+
}
117+
118+
private EvaluationRun findRun(String evaluationId) {
119+
return evaluationRunRepository.findByEvaluationId(evaluationId)
120+
.orElseThrow(() -> new ResourceNotFoundException("Evaluation not found: " + evaluationId));
121+
}
122+
123+
private EvaluationDetailResponse toDetailResponse(EvaluationRun run, List<EvaluationSample> samples) {
124+
return new EvaluationDetailResponse(
125+
run.getEvaluationId(),
126+
run.getName(),
127+
run.getDatasetName(),
128+
run.getModelId(),
129+
run.getStatus(),
130+
run.getTotalSamples(),
131+
run.getCompletedSamples(),
132+
run.getAccuracy(),
133+
run.getPrecision(),
134+
run.getRecall(),
135+
run.getF1(),
136+
run.getAttemptCount(),
137+
run.getMaxAttempts(),
138+
run.getCreatedAt(),
139+
run.getStartedAt(),
140+
run.getCompletedAt(),
141+
run.getFailureReason(),
142+
samples.stream().map(this::toSampleResponse).toList());
143+
}
144+
145+
private EvaluationSampleResponse toSampleResponse(EvaluationSample sample) {
146+
return new EvaluationSampleResponse(
147+
sample.getSampleId(),
148+
sample.getEvaluationId(),
149+
sample.getFilename(),
150+
sample.getGroundTruthLabel(),
151+
sample.getPredictedLabel(),
152+
sample.getScore(),
153+
sample.getLatencyMs(),
154+
sample.getCorrect(),
155+
sample.getFailureReason(),
156+
sample.getCreatedAt());
157+
}
158+
}

0 commit comments

Comments
 (0)