Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions .github/workflows/research.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@ on:
workflow_dispatch:
inputs:
focus_area:
description: "Optional focus area for this manual run. Leave blank for a randomly chosen area. Constrained free-text: lowercase letters, digits, and hyphens only; must start with a letter; 1–32 characters. See the predefined list in research.md §12 for the conventional values."
description: "Optional focus area for this manual run. Leave blank for a
randomly chosen area. Constrained free-text: lowercase letters,
digits, and hyphens only; must start with a letter; 1–32 characters.
See the predefined list in research.md §12 for the conventional
values."
required: false
type: string
default: ""
Expand Down Expand Up @@ -112,7 +116,7 @@ jobs:
allowed_bots: "*"
claude_args: |
--model opus
--max-turns 80
--max-turns 200
Comment thread
chrisleekr marked this conversation as resolved.
--allowedTools "WebSearch,WebFetch,Read,Glob,Grep,Bash(gh issue create:*),Bash(gh issue list:*),Bash(gh label create:*),Bash(git log:*),Bash(cat:*),Bash(date:*),Bash(mmdc:*),Bash(awk:*),Bash(grep:*)"
--disallowedTools ""
prompt: |
Expand Down Expand Up @@ -157,7 +161,7 @@ jobs:

## Strict Workflow — Follow In Order

You have a LIMITED turn budget (~80 turns). Do NOT read every file.
You have a LIMITED turn budget (~200 turns). Do NOT read every file.

### Step 1 — Architecture overview (2-3 turns max)

Expand Down
18 changes: 9 additions & 9 deletions docs/use/workflows/triage.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,15 @@ The agent classifies the issue (bug, feature, refactor, docs, unclear). For bugs

## Outputs

| Field | Type | Notes |
| ----------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
| `state.valid` | boolean | Verdict. |
| `state.confidence` | float `[0, 1]` | Agent's self-assessment. |
| `state.summary` | string | Verdict rationale; uncapped. Embedded into the failed-cascade reason when `valid = false`. |
| `state.recommendedNext` | `'plan'` \| `'stop'` | — |
| `state.evidence` | `Array<{file, line?, note?}>` | — |
| `state.reproduction` | `{attempted, reproduced, details}` | `attempted=false` for non-bug class. `reproduced=null` is allowed only after the harness ladder is walked AND an invariant test is ruled out. |
| `state.report` | markdown | The full `TRIAGE.md`. Embedded verbatim in the tracking comment. |
| Field | Type | Notes |
| ----------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `state.valid` | boolean | Verdict. |
| `state.confidence` | float `[0, 1]` | Agent's self-assessment. |
| `state.summary` | string | Verdict rationale; uncapped. Embedded into the failed-cascade reason when `valid = false`. |
| `state.recommendedNext` | `'plan'` \| `'stop'` | — |
| `state.evidence` | `Array<{file?, line?, note?}>` | Each entry must have at least one of `file` or `note`. Prefer `file`+`line` citations; `note`-only is for cross-cutting evidence (e.g. a negative grep result that has no single-file pointer). |
| `state.reproduction` | `{attempted, reproduced, details}` | `attempted=false` for non-bug class. `reproduced=null` is allowed only after the harness ladder is walked AND an invariant test is ruled out. |
| `state.report` | markdown | The full `TRIAGE.md`. Embedded verbatim in the tracking comment. |

## Stop conditions

Expand Down
19 changes: 12 additions & 7 deletions src/workflows/handlers/triage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,15 @@ const verdictSchema = z
recommendedNext: z.enum(["plan", "stop"]),
evidence: z
.array(
z.object({
file: z.string().min(1),
line: z.number().int().nonnegative().optional(),
note: z.string().min(1).optional(),
}),
z
.object({
file: z.string().min(1).optional(),
line: z.number().int().nonnegative().optional(),
note: z.string().min(1).optional(),
})
.refine((e) => e.file !== undefined || e.note !== undefined, {
message: "evidence entry must have at least one of `file` or `note`",
}),
)
.default([]),
reproduction: reproductionSchema,
Expand Down Expand Up @@ -337,9 +341,10 @@ function buildTriagePrompt(input: {
` "summary": "<as long as needed to faithfully convey the verdict>",`,
` "recommendedNext": "plan" | "stop",`,
` "evidence": [`,
` { "file": "<path>", "line": <int|omit>, "note": "<short>" },`,
` { "file": "<path|omit>", "line": <int|omit>, "note": "<short|omit>" },`,
` ...`,
` ],`,
` (every evidence entry MUST have at least one of \`file\` or \`note\`. Prefer \`file\`+\`line\` citations; use \`note\`-only for cross-cutting observations like negative grep results.)`,
Comment thread
chrisleekr marked this conversation as resolved.
` "reproduction": {`,
` "attempted": true | false,`,
` "reproduced": true | false | null,`,
Expand All @@ -348,7 +353,7 @@ function buildTriagePrompt(input: {
` }`,
``,
`Rules:`,
`- Be ruthless about evidence. A claim without a file:line citation is a guess.`,
`- Be ruthless about evidence. Prefer \`file:line\` citations; a claim with neither a citation nor a concrete cross-cutting observation (e.g. a negative grep result) is a guess.`,
`- For bug issues, a verdict without an honest reproduction attempt is a failure of your job.`,
`- It is OK to report \`reproduced=null\` if the bug genuinely can't be reproduced in this environment — but you MUST explain WHY honestly. Never lie about reproduction status.`,
`- Do NOT modify any source files in the repo (writing temporary scripts under /tmp is fine; running tests is fine; do not stage or commit anything).`,
Expand Down
46 changes: 46 additions & 0 deletions test/workflows/handlers/triage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,52 @@ describe("triage handler (SDK-driven)", () => {
}
});

it("accepts evidence entries with note but no file (cross-cutting observations)", async () => {
triageVerdict = JSON.stringify({
valid: true,
confidence: 0.9,
summary: "Reproduced — non-constant-time bearer comparison.",
recommendedNext: "plan",
evidence: [
{ file: "src/orchestrator/ws-server.ts", line: 52, note: "plain !== check" },
{ note: "grep timingSafeEqual src/ returns 0 hits" },
],
reproduction: {
attempted: true,
reproduced: true,
details: "Code inspection confirms structural defect at cited line.",
},
});
const ctx = buildCtx();

const result = await triageHandler(ctx);

expect(result.status).toBe("succeeded");
if (result.status === "succeeded") {
const state = result.state as { evidence: unknown[] };
expect(state.evidence).toHaveLength(2);
}
});

it("rejects evidence entries that have neither file nor note", async () => {
triageVerdict = JSON.stringify({
valid: true,
confidence: 0.9,
summary: "x",
recommendedNext: "plan",
evidence: [{ line: 10 }],
reproduction: { attempted: false, reproduced: null, details: "skipped" },
});
const ctx = buildCtx();

const result = await triageHandler(ctx);

expect(result.status).toBe("failed");
if (result.status === "failed") {
expect(result.reason).toContain("TRIAGE_VERDICT.json failed validation");
}
});

it("fails when target is a PR rather than an issue", async () => {
const ctx = buildCtx();
const prCtx: WorkflowRunContext = { ...ctx, target: { ...ctx.target, type: "pr" } };
Expand Down
Loading