diff --git a/docs/embedding.md b/docs/embedding.md index 0a2da1a..0bfb0ed 100644 --- a/docs/embedding.md +++ b/docs/embedding.md @@ -228,6 +228,7 @@ A response with no `subtype` at all is treated as success. Both are bounded by | `result` | the final assistant text; for an error, `"Error: "` | | `num_turns` | model calls in this turn (one per tool round-trip) | | `stop_reason` | the last model call's stop reason | +| `trail_off_nudges` | optional; present when non-zero. Replies the turn refused to end on because they only announced a next step, each answered with a "do it now" nudge and counted in `num_turns` | | `usage` | tokens for **this turn**, summed over its model calls, not cumulative for the session | | `total_cost_usd` | priced from Klaudia's model table; `0` for an unpriced model (every OpenAI-compatible one). 0 means unpriced, not free; `usage` is the reliable figure | diff --git a/internal/agent/loop.go b/internal/agent/loop.go index dec44f1..43c7900 100644 --- a/internal/agent/loop.go +++ b/internal/agent/loop.go @@ -174,6 +174,15 @@ type Options struct { // model-aware MaxTokens default comes from Claude's table only on // Anthropic; any other provider gets the conservative unknown-model cap. ProviderName string + // ContinueOnTrailOff keeps going when a turn ends with no tool call and + // the text only announces the next step ("I'll read X next."). The loop + // appends a nudge and requests again, up to maxTrailOffNudges times, and + // counts each one on Result.TrailOffNudges. It is for runs where nobody + // is there to re-prompt — headless and the goal loop — and stays off for + // an interactive session, where ending the turn is how the user gets to + // answer. A question, and a reply carrying a completion marker, still end + // the turn. + ContinueOnTrailOff bool // PartialMessages, if set, receives raw model stream events during the main // answer turn (not compaction summaries). The CLI wires this to a // stream_event emitter when --include-partial-messages is set. Nil by @@ -231,6 +240,11 @@ type Result struct { // by delegating — and the breakdown is here so /cost and the stream-json // result can show it rather than only the total. Children []ChildUsage + // TrailOffNudges counts the times this run refused to end on a reply that + // only announced the next step (see trailoff.go). It is a session-economics + // figure: each one is a model call the announcement would otherwise have + // cost a whole re-prompt to recover. + TrailOffNudges int // Messages is the full conversation after the run (initial + this turn's // exchanges), so a caller can carry it forward as InitialMessages for the // next turn (used by the stream-json embedding frontend). @@ -433,6 +447,10 @@ func (l *Loop) Run(ctx context.Context, opts Options, emit Emitter) (Result, err // most once per Run, so a genuinely oversized request still surfaces. var calib compaction.Calibration overflowRecovered := false + // trailOffNudges counts how many times this run has already pushed back + // on a reply that announced its next step instead of taking it. It caps + // at maxTrailOffNudges, after which the announcement stands. + trailOffNudges := 0 for { res.NumTurns++ @@ -595,6 +613,23 @@ func (l *Loop) Run(ctx context.Context, opts Options, emit Emitter) (Result, err // Collect tool_use blocks from this turn. toolUses := toolUseBlocks(assistant) if len(toolUses) == 0 { + // A reply that only promises the next step has done nothing, and + // in a run with nobody watching it costs a whole re-prompt to + // recover. Nudge inside this loop instead, up to the cap. The + // assistant turn is recorded first, exactly as a real final + // answer is, so the transcript stays paired. + if opts.ContinueOnTrailOff && !halted && trailOff(finalText) && trailOffNudges < maxTrailOffNudges && withinLimits(opts, res) { + rec("assistant", assistant) + nudge := anthropic.NewBetaUserMessage(anthropic.NewBetaTextBlock(trailOffNudge)) + messages = append(messages, nudge) + rec("user", nudge) + trailOffNudges++ + res.TrailOffNudges = trailOffNudges + if emit != nil { + emit(Event{Type: "notice", Content: fmt.Sprintf("trail-off: the reply announced a next step without doing it — continuing (%d/%d)", trailOffNudges, maxTrailOffNudges)}) + } + continue + } // Final (tool-less) answer: structurally fine on its own, record now. rec("assistant", assistant) res.Messages = messages @@ -706,6 +741,16 @@ func (l *Loop) Run(ctx context.Context, opts Options, emit Emitter) (Result, err } } +// withinLimits reports whether the run may make another request under its +// turn cap and budget. A trail-off nudge is another request, so it is bound by +// the same limits as a tool turn; past them the announcement is the answer. +func withinLimits(opts Options, res Result) bool { + if opts.MaxTurns > 0 && res.NumTurns >= opts.MaxTurns { + return false + } + return opts.MaxBudgetUSD <= 0 || res.CostUSD < opts.MaxBudgetUSD +} + // elisionSpiller gives microcompact somewhere to put the tool results it // removes, so an elision stays recoverable: the placeholder names the file and // the model can read it. It uses the same store as the per-result cap, under diff --git a/internal/agent/trailoff.go b/internal/agent/trailoff.go new file mode 100644 index 0000000..9d43026 --- /dev/null +++ b/internal/agent/trailoff.go @@ -0,0 +1,109 @@ +package agent + +import ( + "regexp" + "strings" +) + +// maxTrailOffNudges is how many times one Run will push a model that keeps +// ending its turn by announcing the next step instead of taking it. Each nudge +// is another model call, so the bound is small: past it the announcement is +// the answer and the turn ends. +const maxTrailOffNudges = 2 + +// trailOffNudge is the user message appended when a turn trails off. It is a +// user message rather than a system one because it has to land between two +// assistant turns, and it names the behaviour rather than the topic so it +// cannot be mistaken for new work. +const trailOffNudge = "You announced what you would do next but made no tool call, so nothing happened. Do that step now, in this turn, by calling the tool. If the work is actually finished, give the final answer instead of describing a next step — and if you are waiting on the user, ask them plainly." + +// announcement patterns, matched against the reply's LAST sentence only: a +// trailing-off turn ends on its promise, while "I'll" earlier in a real +// answer ("I'll note the fix is in loop.go. Everything passes now.") is +// narration. The first pattern wants the sentence to open with the promise, +// allowing one lead word ("OK, I'll…", "Next, let me…"); the second catches +// the promise anywhere in the sentence when it ends on "next" or "now". +var trailOffPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?i)^(?:[\w']+[,:]?\s+)?(?:i(?:'ll|’ll| will)|let me|i'm going to|i am going to)\s+\S`), + regexp.MustCompile(`(?i)\b(?:i(?:'ll|’ll| will)|let me|i'm going to|i am going to)\s+\S.*\b(?:next|now)\W*$`), +} + +// notNowPatterns mark a last sentence that promises something other than an +// immediate step: a conditional offer ("If you approve, I'll push") waits on +// the user, and a deferral ("in a follow-up PR") is out of this run's scope. +// A sign-off ("Let me know what you think", "I'll stop here", "I'll leave +// the rest to you") hands the turn back on purpose, so it is an ending too. +// Nudging any of these would tell the model to do now what it rightly did not — +// under -p, to push without the approval it asked for. +var notNowPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?i)\b(?:if|once|when|whenever|unless|after|as soon as|should you|provided)\b`), + regexp.MustCompile(`(?i)\b(?:follow-?up|later|separate(?:ly)?|another (?:pr|change|commit|session)|next time|future|tomorrow|eventually)\b`), + regexp.MustCompile(`(?i)\blet me know\b|\b(?:stop|stopping|pause|pausing|end|ending|wrap(?:ping)? up|leave|leaving|hand(?:ing)? (?:it |this |the rest )?(?:back|over))\b`), +} + +// doneMarkers are lines a reply uses to say it is finished. A reply carrying +// one is a final answer however it is phrased, so it is never a trail-off. +var doneMarkers = []string{ + "autopilot: done", + "", +} + +// trailOff reports whether text is a reply that only announces a next step. +// A question is the model waiting on the user, and a completion marker is the +// model declaring itself finished; neither is nudged. +func trailOff(text string) bool { + text = strings.TrimSpace(text) + if text == "" || hasDoneMarker(text) || endsWithQuestion(text) { + return false + } + last := strings.TrimLeft(lastSentence(text), "-*#>• \t") + for _, re := range notNowPatterns { + if re.MatchString(last) { + return false + } + } + for _, re := range trailOffPatterns { + if re.MatchString(last) { + return true + } + } + return false +} + +func hasDoneMarker(text string) bool { + low := strings.ToLower(text) + for _, m := range doneMarkers { + if strings.Contains(low, m) { + return true + } + } + return false +} + +// endsWithQuestion reports whether the reply's last sentence asks something. +// An earlier question followed by an announcement ("Shall I? I'll read it +// next.") is still an announcement. +func endsWithQuestion(text string) bool { + return strings.ContainsRune(lastSentence(text), '?') +} + +// lastSentence returns the final non-empty sentence of text. Sentences break +// on a newline or after ". ", "! " or "? ", so a "?" ends its own sentence. +func lastSentence(text string) string { + last := strings.TrimSpace(text) + for { + cut := -1 + for _, sep := range []string{"\n", ". ", "! ", "? "} { + if i := strings.LastIndex(last, sep); i >= 0 && i+len(sep) > cut { + cut = i + len(sep) + } + } + if cut < 0 { + return last + } + if tail := strings.TrimSpace(last[cut:]); tail != "" { + return tail + } + last = strings.TrimSpace(last[:cut]) + } +} diff --git a/internal/agent/trailoff_loop_test.go b/internal/agent/trailoff_loop_test.go new file mode 100644 index 0000000..2738f81 --- /dev/null +++ b/internal/agent/trailoff_loop_test.go @@ -0,0 +1,161 @@ +package agent + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/anthropics/anthropic-sdk-go" + + "github.com/greenthread-ai/klaudia/internal/api" + "github.com/greenthread-ai/klaudia/internal/tools" +) + +// scripted is a Provider that returns canned assistant messages in order and +// records the text of every user message it was shown, so a test can see the +// nudge land. +type scripted struct { + replies []string + seen []string +} + +func (s *scripted) StreamTurn(_ context.Context, params anthropic.BetaMessageNewParams, sink api.StreamSink) (anthropic.BetaMessage, error) { + for _, m := range params.Messages { + if m.Role != "user" { + continue + } + for _, b := range m.Content { + if b.OfText != nil { + s.seen = append(s.seen, b.OfText.Text) + } + } + } + text := "(script exhausted)" + if len(s.replies) > 0 { + text, s.replies = s.replies[0], s.replies[1:] + } + sink.OnText(text) + raw := fmt.Sprintf(`{"id":"m","type":"message","role":"assistant","model":"x","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1},"content":[{"type":"text","text":%q}]}`, text) + var msg anthropic.BetaMessage + if err := json.Unmarshal([]byte(raw), &msg); err != nil { + return msg, err + } + return msg, nil +} + +func runScript(t *testing.T, replies []string, cont bool) (Result, *scripted) { + t.Helper() + return runScriptOpts(t, replies, Options{ContinueOnTrailOff: cont}) +} + +func runScriptOpts(t *testing.T, replies []string, opts Options) (Result, *scripted) { + t.Helper() + s := &scripted{replies: replies} + opts.Prompt = "do the thing" + res, err := New(s, tools.NewRegistry()).Run(context.Background(), opts, nil) + if err != nil { + t.Fatalf("Run: %v", err) + } + return res, s +} + +func TestTrailOffNudgeThenDone(t *testing.T) { + res, s := runScript(t, []string{"I'll read loop.go next.", "Read it. The exit is at the empty tool list."}, true) + if res.TrailOffNudges != 1 { + t.Fatalf("TrailOffNudges = %d, want 1", res.TrailOffNudges) + } + if res.NumTurns != 2 { + t.Fatalf("NumTurns = %d, want 2", res.NumTurns) + } + if !strings.Contains(res.Text, "Read it") { + t.Fatalf("final text = %q, want the second reply", res.Text) + } + if !strings.Contains(strings.Join(s.seen, "\n"), "made no tool call") { + t.Fatalf("the nudge never reached the model; seen %q", s.seen) + } +} + +func TestTrailOffBounded(t *testing.T) { + res, _ := runScript(t, []string{ + "I'll read loop.go next.", + "Next I'll run the tests.", + "Let me check the output now.", + }, true) + if res.TrailOffNudges != maxTrailOffNudges { + t.Fatalf("TrailOffNudges = %d, want the cap %d", res.TrailOffNudges, maxTrailOffNudges) + } + // The cap stops the loop: one turn per nudge plus the turn that stands. + if res.NumTurns != maxTrailOffNudges+1 { + t.Fatalf("NumTurns = %d, want %d", res.NumTurns, maxTrailOffNudges+1) + } + if res.Text != "Let me check the output now." { + t.Fatalf("the turn past the cap should stand, got %q", res.Text) + } +} + +func TestTrailOffOffByDefault(t *testing.T) { + res, s := runScript(t, []string{"I'll read loop.go next."}, false) + if res.TrailOffNudges != 0 || res.NumTurns != 1 { + t.Fatalf("with the guard off: nudges %d turns %d, want 0 and 1", res.TrailOffNudges, res.NumTurns) + } + if strings.Contains(strings.Join(s.seen, "\n"), "made no tool call") { + t.Fatal("nudged a turn the guard was not asked to watch") + } +} + +func TestTrailOffIgnoresARealEnding(t *testing.T) { + for _, reply := range []string{ + "All done.\nAUTOPILOT: DONE", + "Which file should I read?", + "The tests pass.", + } { + res, _ := runScript(t, []string{reply}, true) + if res.TrailOffNudges != 0 || res.NumTurns != 1 { + t.Errorf("%q: nudges %d turns %d, want a plain ending", reply, res.TrailOffNudges, res.NumTurns) + } + } +} + +// A nudge is another request, so it is bound by --max-turns like a tool turn. +// Before the fix MaxTurns=1 with three announcements gave three turns. +func TestTrailOffRespectsMaxTurns(t *testing.T) { + replies := []string{"I'll read loop.go next.", "Next I'll run the tests.", "Let me check the output now."} + res, _ := runScriptOpts(t, replies, Options{ContinueOnTrailOff: true, MaxTurns: 1}) + if res.NumTurns != 1 || res.TrailOffNudges != 0 { + t.Fatalf("MaxTurns=1: turns %d nudges %d, want 1 and 0", res.NumTurns, res.TrailOffNudges) + } + res, _ = runScriptOpts(t, replies, Options{ContinueOnTrailOff: true, MaxTurns: 2}) + if res.NumTurns != 2 || res.TrailOffNudges != 1 { + t.Fatalf("MaxTurns=2: turns %d nudges %d, want 2 and 1", res.NumTurns, res.TrailOffNudges) + } +} + +func TestWithinLimits(t *testing.T) { + cases := []struct { + opts Options + res Result + want bool + }{ + {Options{}, Result{NumTurns: 50, CostUSD: 9}, true}, + {Options{MaxTurns: 3}, Result{NumTurns: 2}, true}, + {Options{MaxTurns: 3}, Result{NumTurns: 3}, false}, + {Options{MaxBudgetUSD: 1}, Result{CostUSD: 0.5}, true}, + {Options{MaxBudgetUSD: 1}, Result{CostUSD: 1}, false}, + } + for _, c := range cases { + if got := withinLimits(c.opts, c.res); got != c.want { + t.Errorf("withinLimits(%+v, turns %d cost %v) = %v, want %v", c.opts, c.res.NumTurns, c.res.CostUSD, got, c.want) + } + } +} + +// The review's conditional offer must end the run: nudging it under -p would +// tell the model to push without the approval it asked for. +func TestTrailOffLeavesAConditionalOffer(t *testing.T) { + res, _ := runScript(t, []string{"Done. If you approve, I'll push the branch."}, true) + if res.TrailOffNudges != 0 || res.NumTurns != 1 { + t.Fatalf("nudges %d turns %d, want a plain ending", res.TrailOffNudges, res.NumTurns) + } +} diff --git a/internal/agent/trailoff_test.go b/internal/agent/trailoff_test.go new file mode 100644 index 0000000..11cb459 --- /dev/null +++ b/internal/agent/trailoff_test.go @@ -0,0 +1,63 @@ +package agent + +import "testing" + +func TestTrailOff(t *testing.T) { + positives := []string{ + "I'll read loop.go next.", + "Next I'll run the tests.", + "Let me check the result now.", + "I will look at the diff next", + "Done with that part.\n\nNext, I'll run go test.", + "I'll grep for StopReason.", + "Let me read the guide.", + "I'm going to verify the build next.", + } + for _, s := range positives { + if !trailOff(s) { + t.Errorf("trailOff(%q) = false, want true", s) + } + } + + negatives := []string{ + "", + " ", + "The build passes and the tests are green.", + "I fixed the nil check in loop.go; both packages test clean.", + "AUTOPILOT: DONE", + "All three items are done.\nAUTOPILOT: DONE", + "The goal is satisfied.\n", + "Which file should I read?", + "I've read it. Want me to change it?", + "I read loop.go and the exit is at line 597.", + "Earlier I said I'd read it. I did, and here is what it says.", + // From the WI-1301 review: an "I'll" that is not the last sentence, + // a conditional offer, and a deferral are real endings. + "Done. If you approve, I'll push the branch.", + "I'll note the fix is in loop.go. Everything passes now.", + "I've updated the docs; I will update the changelog in a follow-up PR.", + "Once CI is green I'll mark it ready.", + "When you're back, I'll rebase it.", + "I'll handle the rename separately.", + "I'll revisit the cap later.", + // From the WI-1301 re-review: sign-offs hand the turn back. + "Let me know what you think.", + "Summary above. Let me know.", + "Done. I'll leave the rest to you.", + "I'm going to stop here.", + "I will now stop.", + } + for _, s := range negatives { + if trailOff(s) { + t.Errorf("trailOff(%q) = true, want false", s) + } + } +} + +// A question followed by the announcement is still an announcement: the +// question is not the last thing the reply does. +func TestTrailOffQuestionThenAnnouncement(t *testing.T) { + if !trailOff("Should I stop here? No. I'll read the tests next.") { + t.Fatal("an announcement after a question is still a trail-off") + } +} diff --git a/internal/cli/background_drain.go b/internal/cli/background_drain.go index 4abe159..b7f2fc8 100644 --- a/internal/cli/background_drain.go +++ b/internal/cli/background_drain.go @@ -181,6 +181,7 @@ func remainingBudget(limit, spent float64) float64 { // describes the end of the run comes from the latest turn. func sumResults(sofar, next agent.Result) agent.Result { next.NumTurns += sofar.NumTurns + next.TrailOffNudges += sofar.TrailOffNudges next.InputTokens += sofar.InputTokens next.OutputTokens += sofar.OutputTokens next.CacheReadInputTokens += sofar.CacheReadInputTokens diff --git a/internal/cli/background_drain_test.go b/internal/cli/background_drain_test.go index fe6f0b7..b6f32c4 100644 --- a/internal/cli/background_drain_test.go +++ b/internal/cli/background_drain_test.go @@ -57,7 +57,7 @@ func noFollowUp(t *testing.T) followUpFunc { // are summed over both turns. func TestDrainBackgroundRunsFollowUpWithinCaps(t *testing.T) { reg := launch(t, childProvider{}) - first := agent.Result{NumTurns: 2, CostUSD: 0.25, InputTokens: 10, Text: "launched", StopReason: "end_turn"} + first := agent.Result{NumTurns: 2, CostUSD: 0.25, InputTokens: 10, TrailOffNudges: 1, Text: "launched", StopReason: "end_turn"} calls := 0 run := func(_ context.Context, _ []anthropic.BetaMessageParam, maxTurns int, budget float64) (agent.Result, error) { @@ -69,7 +69,7 @@ func TestDrainBackgroundRunsFollowUpWithinCaps(t *testing.T) { if report := reg.PendingReport(); !strings.Contains(report, "child done") { t.Errorf("report = %q", report) } - return agent.Result{NumTurns: 1, CostUSD: 0.5, InputTokens: 5, Text: "final", StopReason: "end_turn"}, nil + return agent.Result{NumTurns: 1, CostUSD: 0.5, InputTokens: 5, TrailOffNudges: 1, Text: "final", StopReason: "end_turn"}, nil } var warned []string res, err := drainBackground(context.Background(), reg, first, nil, @@ -81,7 +81,7 @@ func TestDrainBackgroundRunsFollowUpWithinCaps(t *testing.T) { if calls != 1 { t.Fatalf("follow-up ran %d times, want 1", calls) } - if res.Text != "final" || res.NumTurns != 3 || res.CostUSD != 0.75 || res.InputTokens != 15 { + if res.Text != "final" || res.NumTurns != 3 || res.CostUSD != 0.75 || res.InputTokens != 15 || res.TrailOffNudges != 2 { t.Errorf("summed result = %+v", res) } if len(warned) != 0 { diff --git a/internal/cli/headless.go b/internal/cli/headless.go index 47f160a..8194818 100644 --- a/internal/cli/headless.go +++ b/internal/cli/headless.go @@ -37,18 +37,21 @@ func ParseOutputFormat(s string) (OutputFormat, error) { // and JSON tags match the JS bundle (07-app-features.js:33746) so JSON output // is byte-compatible after timestamp/UUID normalization. type ResultMessage struct { - Type string `json:"type"` // always "result" - Subtype string `json:"subtype"` - IsError bool `json:"is_error"` - DurationMS int64 `json:"duration_ms"` - DurationAPIMS int64 `json:"duration_api_ms"` - NumTurns int `json:"num_turns"` - Result string `json:"result"` - StopReason string `json:"stop_reason,omitempty"` - SessionID string `json:"session_id"` - TotalCostUSD float64 `json:"total_cost_usd"` - Usage map[string]any `json:"usage,omitempty"` - UUID string `json:"uuid"` + Type string `json:"type"` // always "result" + Subtype string `json:"subtype"` + IsError bool `json:"is_error"` + DurationMS int64 `json:"duration_ms"` + DurationAPIMS int64 `json:"duration_api_ms"` + NumTurns int `json:"num_turns"` + Result string `json:"result"` + StopReason string `json:"stop_reason,omitempty"` + // TrailOffNudges counts the replies the run refused to end on because + // they only announced a next step (agent.Result.TrailOffNudges). + TrailOffNudges int `json:"trail_off_nudges,omitempty"` + SessionID string `json:"session_id"` + TotalCostUSD float64 `json:"total_cost_usd"` + Usage map[string]any `json:"usage,omitempty"` + UUID string `json:"uuid"` } // Renderer writes headless output in the configured format. diff --git a/internal/cli/loop.go b/internal/cli/loop.go index c4103ad..d08f6a8 100644 --- a/internal/cli/loop.go +++ b/internal/cli/loop.go @@ -195,6 +195,9 @@ func runGoalLoop(ctx context.Context, cmd *cobra.Command, p loopRun) error { Diagnostics: p.diagnostics, Hooks: p.hooks, CommandGuard: baseline.Guard(), + // An iteration that ends by announcing the next step has not + // finished the iteration. Continue inside it. + ContinueOnTrailOff: true, }, emit) } diff --git a/internal/cli/root.go b/internal/cli/root.go index a59b179..5563c4f 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -2158,6 +2158,11 @@ func runFormat(cmd *cobra.Command, opts *options, format OutputFormat, st *runSt // The transcript on disk and the peer's envelope stream are fed // from the same Record calls, as in the -p path. o.Recorder = multiRecorder{recorder, turn.Recorder} + // A stream-json peer is a driver, not a person at a keyboard: an + // announcement with no tool call would otherwise end the turn and + // wait for the peer to re-prompt. An interactive TUI turn leaves + // this off, because there the user is who answers. + o.ContinueOnTrailOff = true return loop.Run(ctx, o, turn.Emit) } // initialMessages, not nil: an explicit --resume/--continue was being @@ -2267,6 +2272,9 @@ func runFormat(cmd *cobra.Command, opts *options, format OutputFormat, st *runSt headlessOpts.Recorder = runRecorder headlessOpts.PartialMessages = partial headlessOpts.DeferredTools = deferredTools + // Nobody is watching a -p run, so a reply that only announces its next + // step would end the process. Keep going inside the turn instead. + headlessOpts.ContinueOnTrailOff = true res, err := loop.Run(ctx, headlessOpts, emit) // Background sub-agents the turn launched are owed a later turn, and a -p // run has no other: wait for them and deliver their results before the @@ -2294,16 +2302,17 @@ func runFormat(cmd *cobra.Command, opts *options, format OutputFormat, st *runSt }) out := ResultMessage{ - Type: "result", - Subtype: "success", - IsError: err != nil, - DurationMS: time.Since(st.start).Milliseconds(), - DurationAPIMS: res.APIDuration.Milliseconds(), - NumTurns: res.NumTurns, - Result: res.Text, - StopReason: res.StopReason, - SessionID: sessionID, - TotalCostUSD: res.CostUSD, // derived from usage + api pricing table; 0 for unpriced models + Type: "result", + Subtype: "success", + IsError: err != nil, + DurationMS: time.Since(st.start).Milliseconds(), + DurationAPIMS: res.APIDuration.Milliseconds(), + NumTurns: res.NumTurns, + Result: res.Text, + StopReason: res.StopReason, + TrailOffNudges: res.TrailOffNudges, + SessionID: sessionID, + TotalCostUSD: res.CostUSD, // derived from usage + api pricing table; 0 for unpriced models Usage: map[string]any{ "input_tokens": res.InputTokens, "output_tokens": res.OutputTokens, diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index bc00989..635b85c 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -74,6 +74,24 @@ func TestRunJSONOutputIsOneResultObject(t *testing.T) { } } +// A -p run that trails off is nudged, and the JSON result says how often: +// without trail_off_nudges neither the run nor session economics can see it. +func TestRunJSONReportsTrailOffNudges(t *testing.T) { + m := newFakeModel(t, say("I'll read loop.go next."), say("Read it; the exit is at line 597.")) + e := newCLIEnv(t, m) + r := e.run(nil, "-p", "q", "--output-format", "json") + if r.Err != nil { + t.Fatal(r.dump()) + } + res := r.lines()[0] + if res["trail_off_nudges"] != float64(1) || res["num_turns"] != float64(2) { + t.Errorf("trail_off_nudges = %v, num_turns = %v; want 1 and 2\n%s", res["trail_off_nudges"], res["num_turns"], r.dump()) + } + if res["result"] != "Read it; the exit is at line 597." { + t.Errorf("result = %v", res["result"]) + } +} + // stream-json output is the conversation as message envelopes, stamped with // the session id, and ends with the result line; the tool ran on disk. func TestRunStreamJSONEmitsEnvelopesThenResult(t *testing.T) { diff --git a/internal/streamjson/driver.go b/internal/streamjson/driver.go index 95f14d4..3111107 100644 --- a/internal/streamjson/driver.go +++ b/internal/streamjson/driver.go @@ -805,6 +805,9 @@ func (d *Driver) resultEvent(res agent.Result, err error, dur time.Duration, int "cache_creation_input_tokens": res.CacheCreationInputTokens, }, } + if res.TrailOffNudges > 0 { + m["trail_off_nudges"] = res.TrailOffNudges + } if len(res.Children) > 0 { children := make([]map[string]any, 0, len(res.Children)) for _, c := range res.Children { diff --git a/internal/streamjson/turncontrol_test.go b/internal/streamjson/turncontrol_test.go index edccf8d..9371868 100644 --- a/internal/streamjson/turncontrol_test.go +++ b/internal/streamjson/turncontrol_test.go @@ -360,3 +360,15 @@ func TestResultEventSubtype(t *testing.T) { }) } } + +// trail_off_nudges appears on the result line when the turn was nudged, and +// is absent otherwise so an un-nudged result is unchanged. +func TestResultEventReportsTrailOffNudges(t *testing.T) { + d := NewDriver(io.Discard) + if m := d.resultEvent(agent.Result{Text: "ok", TrailOffNudges: 2}, nil, 0, false); m["trail_off_nudges"] != 2 { + t.Errorf("trail_off_nudges = %v, want 2", m["trail_off_nudges"]) + } + if m := d.resultEvent(agent.Result{Text: "ok"}, nil, 0, false); m["trail_off_nudges"] != nil { + t.Errorf("trail_off_nudges = %v on an un-nudged turn, want absent", m["trail_off_nudges"]) + } +}