Skip to content

fix: accumulate ChatUsage across Anthropic streaming events #2467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
waterWang wants to merge 1 commit into
base: main
Choose a base branch
from
+42 −3
Open

fix: accumulate ChatUsage across Anthropic streaming events #2467

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 42 additions & 3 deletions ...main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,17 +102,50 @@ public static ChatResponse parseMessage(Message message, Instant startTime) {
*/
public static Flux<ChatResponse> parseStreamEvents(
Flux<RawMessageStreamEvent> eventFlux, Instant startTime) {
ChatUsage[] accumulatedUsage = new ChatUsage[1];
return eventFlux
.flatMap(
event -> {
try {
return Flux.just(parseStreamEvent(event, startTime));
ChatResponse response = parseStreamEvent(event, startTime);
// Accumulate usage across events: message_start gives inputTokens,
// message_delta gives outputTokens
if (response.getUsage() != null) {
ChatUsage newUsage = response.getUsage();
ChatUsage existing = accumulatedUsage[0];
if (existing == null) {
accumulatedUsage[0] = newUsage;
} else {
int inputTokens = newUsage.getInputTokens() > 0
? newUsage.getInputTokens()
: existing.getInputTokens();
int outputTokens = newUsage.getOutputTokens() > 0
? newUsage.getOutputTokens()
: existing.getOutputTokens();
accumulatedUsage[0] = ChatUsage.builder()
.inputTokens(inputTokens)
.outputTokens(outputTokens)
.time(newUsage.getTime())
.build();
}
}
// Attach accumulated usage to content events
if (!response.getContent().isEmpty()
&& accumulatedUsage[0] != null) {
return Flux.just(ChatResponse.builder()
.id(response.getId())
.content(response.getContent())
.usage(accumulatedUsage[0])
.build());
}
return Flux.just(response);
} catch (Exception e) {
log.warn("Error parsing stream event: {}", e.getMessage());
return Flux.empty();
}
})
.filter(response -> response != null && !response.getContent().isEmpty());
.filter(
response -> response != null && !response.getContent().isEmpty());
}

/**
Expand All @@ -125,7 +158,13 @@ private static ChatResponse parseStreamEvent(RawMessageStreamEvent event, Instan

// Message start
if (event.isMessageStart()) {
messageId = event.asMessageStart().message().id();
var msgStart = event.asMessageStart();
messageId = msgStart.message().id();
// Parse input tokens from message_start event
usage = ChatUsage.builder()
.inputTokens((int) msgStart.message().usage().inputTokens())
.time(Duration.between(startTime, Instant.now()).toMillis() / 1000.0)
.build();
}

// Content block delta - text
Expand Down
Loading