classify.js

// Pattern-based classifiers. Every rule here leans conservative: when a signal is
// ambiguous we prefer to miss it rather than accuse an agent of something it didn't do.

// `(?![-.\w])` stops `pytest-full.log` or `jest.config.js` from counting as a run.
const END = String.raw`(?![-.\w/])`;
const TEST_RUNNERS = new RegExp(
  [
    String.raw`\bpy\.?test` + END,
    String.raw`\bpython[\d.]*\s+-m\s+(?:pytest|unittest)` + END,
    String.raw`(?:^|\s)(?:tox|nox)` + END,
    String.raw`\b(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:-\S+\s+)*test` + END, // also `npm run -s test`
    String.raw`(?:^|\s|\/)(?:jest|vitest|mocha|ava)` + END,
    String.raw`\bplaywright\s+test` + END,
    String.raw`\bgo\s+test` + END,
    String.raw`\bcargo\s+(?:test|nextest)` + END,
    String.raw`\b(?:mvn|mvnw)\b[^|;&]*\s(?:test|verify)` + END,
    String.raw`\bgradlew?\b[^|;&]*[\s:]test` + END,
    String.raw`(?:^|\s|\/)rspec` + END,
    String.raw`(?:^|\s|\/)phpunit` + END,
    String.raw`\bdotnet\s+test` + END,
    String.raw`(?:^|\s)ctest` + END,
    String.raw`\bmake\s+(?:test|check|tests)` + END,
    String.raw`\b(?:deno|swift|mix)\s+test` + END,
    String.raw`\bzig\s+build\s+test` + END,
    String.raw`(?:^|\s)bats` + END,
    String.raw`\b(?:node|bun)\s+(?:[^|;&\n]*\s)?--test` + END,
    String.raw`\bbun\s+test` + END,
    String.raw`\bpython[\d.]*\s+(?:-\w+\s+)*\S*\btest_\w+\.py\b`,
    String.raw`\bnode\s+(?:--\S+\s+)*\S*\.(?:test|spec)\.[cm]?js\b`,
  ].join("|"),
);

const NOT_A_RUN = /\b(?:install|add|uninstall|which|whereis)\b|\bimport\s+(?:pytest|unittest)\b|(?:^|\s)(?:--version|--help|--collect-only|--co|--list|-h)(?=\s|$)|\bman\s|\bcat\s|\bgrep\s|\brg\s|\bsed\s/;

/** A run with no file, directory, node id or -k filter: the whole suite. */
export function isFullSuite(cmd) {
  return splitCommands(cmd).some((part) => {
    part = part.split(/\|(?!\|)/)[0];
    const m = TEST_RUNNERS.exec(part);
    if (!m || NOT_A_RUN.test(part)) return false;
    const args = part.slice(m.index + m[0].length);
    return !/(?:^|\s)(?:-k|-m|--grep|-t|--testNamePattern|-run|--filter)\b|::|\.(?:py|[cm]?[jt]sx?|go|rb|rs)\b|(?:^|\s)(?:\.\/)?[\w.-]+\/[\w./-]*|(?:^|\s)tests?\.[\w.]+/.test(args);
  });
}

// Quoted text is data (JSON payloads, commit messages, echo) unless a shell runs it.
const RUNS_QUOTED = /(?:\b(?:ba|z)?sh\s+-\w*c|\bssh\b[^'"|;&]*|\b(?:docker|podman)\s+(?:exec|run)\b[^'"|;&]*|\bkubectl\s+exec\b[^'"|;&]*|\b(?:timeout|nice|env|xargs)\b[^'"|;&]*)\s*$/;

export function dropDataQuotes(cmd) {
  return cmd.replace(/'[^']*'|"(?:[^"\\]|\\.)*"/g, (q, offset) =>
    !/\s/.test(q) || RUNS_QUOTED.test(cmd.slice(Math.max(0, offset - 200), offset)) ? q : "Q",
  );
}

export function isTestCommand(cmd) {
  if (!cmd) return false;
  cmd = dropDataQuotes(stripHeredocs(cmd)); // heredoc bodies first: their apostrophes are not quotes
  // Look at each simple command so `pip install pytest && pytest` still counts, and only
  // at what precedes a pipe: `npm test | grep fail` is a run, `grep pytest x` is not.
  return splitCommands(cmd).some((part) => {
    const head = part.split(/\|(?!\|)/)[0];
    return (TEST_RUNNERS.test(head) || runsTestScript(head)) && !NOT_A_RUN.test(head);
  });
}

const INTERPRETER = /^(?:\S*[\\/])?(?:python[\d.]*|node|bash|sh|zsh|pwsh|powershell(?:\.exe)?|ruby|perl|php|deno|bun|tsx|ts-node)$/i;

/**
 * Projects with their own runner: a script under tests/ (`pwsh -File tests/runner/run.ps1`,
 * `bash tests/run.sh`) or a program named like a test (`./build/frost_smoke.exe`).
 */
function runsTestScript(part) {
  const words = part.trim().replace(/^(?:\w+=\S*\s+)*(?:(?:sudo|nice|env|timeout\s+\S+)\s+)*/, "").split(/\s+/);
  let target;
  if (INTERPRETER.test(words[0])) {
    // a script argument that is a test: `pwsh -File tests/run.ps1`, `python main.py run tests/t.cfml`
    const args = words.slice(1).filter((w) => /[\\/]|\.(?:py|[cm]?js|ts|sh|ps1|rb|pl|php)["']?$/.test(w)).map((w) => w.replace(/^["']|["']$/g, ""));
    return args.some((t) => /(?:^|[\\/])(?:tests?|specs?)[\\/]/i.test(t) || /(?:^|[_.-])(?:tests?|smoke|specs?)(?:[_.-]|$)/i.test(t.split(/[\\/]/).pop()));
  } else if (/[\\/]/.test(words[0])) {
    target = words[0]; // a program run by path
  }
  if (!target) return false;
  target = target.replace(/^["']|["']$/g, "");
  const base = target.split(/[\\/]/).pop();
  return /(?:^|[\\/])(?:tests?|specs?)[\\/]/i.test(target) || /(?:^|[_.-])(?:tests?|smoke|specs?)(?:[_.-]|$)/i.test(base);
}

// Commands whose output can contain text saved earlier, e.g. an old test log or fixture.
export const ECHOES = /(?:^|[\s;&|(])(?:cat|less|more|head|tail|sed|awk|grep|rg|jq|nl|bat|xxd|type|Get-Content|git\s+(?:show|log|diff|blame|grep))\b/;

export function splitCommands(cmd) {
  return stripHeredocs(cmd).split(/\s*(?:&&|\|\||;|\n)\s*/).filter(Boolean);
}

/** Drop heredoc bodies: code and data inside them are not commands being run. */
export function stripHeredocs(cmd) {
  return cmd.replace(/<<-?\s*(['"]?)(\w+)\1[^\n]*\n[\s\S]*?(?:\n\s*\2\s*(?=\n|$)|$)/g, " HEREDOC_BODY");
}

// A pipe hides the test runner's exit status unless pipefail is set.
export function isPiped(cmd) {
  if (/pipefail/.test(cmd)) return false;
  return splitCommands(cmd).some((part) => TEST_RUNNERS.test(part) && /[^|]\|[^|]/.test(part));
}

// Only runner summary lines count: stray tracebacks or "1 error" from a linter in the
// same command output must not be mistaken for a test failure.
const OUTPUT_FAIL = [
  /\b[1-9]\d* (?:failed|errors?)\b[^\n]* in [\d.]+m?s\b/, // pytest: "2 failed, 10 passed in 1.2s"
  /^FAILED \((?:failures|errors)=\d+/m, // unittest
  /\bTests?:\s+[1-9]\d* failed\b/, // jest / vitest
  /\bTest Files\s+[1-9]\d* failed\b/,
  /^--- FAIL:|^FAIL\s+\S+\s+[\d.]+s$/m, // go
  /\btest result: FAILED\b/, // cargo
  /^FAILURES!$/m, // phpunit
  /\b\d+ examples?, [1-9]\d* failures?\b/, // rspec
  /\bFailed!\s+-\s+Failed:\s+[1-9]/, // dotnet
  /(?:^|ℹ pass \d+\s+)(?:ℹ|#) fail [1-9]\d*\s*$/m, // node --test (spec and TAP reporters; also joined on one line)
  /\b\d+ passed,? [1-9]\d* failed\b|\b[1-9]\d* failed,? \d+ passed\b/i, // custom runners
  /\bTests run: \d+, Failures: (?:[1-9]|\d+, Errors: [1-9])/, // maven surefire
  /\btests="?\d+"? failures="?(?:[1-9]|\d+"? errors="?[1-9])/, // junit-style counts
];
const OUTPUT_PASS = [
  /\b\d+ passed\b[^\n]* in [\d.]+m?s\b/,
  /\bTests?:\s+\d+ passed(?:, \d+ total)?\b/,
  /^ok\s+\S+\s+[\d.]+s/m,
  /\btest result: ok\b/,
  /\b\d+ examples?, 0 failures\b/,
  /^Ran \d+ tests? in [\d.]+s\s+OK\b/m,
  /\bPassed!\s+-\s+Failed:\s+0\b/,
  /(?:^|ℹ pass \d+\s+)(?:ℹ|#) fail 0\s*$/m,
  /\b\d+ passed,? 0 failed\b|\b0 failed,? \d+ passed\b/i, // custom runners
  /\b(\d+)\s*\/\s*\1\s+(?:tests?\s+)?passed\b/i,
  /\bTests run: [1-9]\d*, Failures: 0, Errors: 0\b/,
  /\btests="?[1-9]\d*"? failures="?0"? errors="?0\b/,
];

/** `tail pytest.log` and friends: evidence from a run started earlier in the background. */
export function isTestLogRead(cmd) {
  return /(?:^|[\s;&|])(?:tail|cat|head|less)\s[^|;&\n]*\S*(?:py)?test\S*\.(?:log|txt|out)\b/i.test(cmd);
}

/** Read pass/fail from what the runner printed. Returns "pass", "fail" or null. */
export function outcomeFromOutput(text) {
  if (!text) return null;
  if (OUTPUT_FAIL.some((rx) => rx.test(text))) return "fail";
  if (OUTPUT_PASS.some((rx) => rx.test(text))) return "pass";
  return null;
}

/**
 * Combine exit status and printed output. Printed failures beat exit 0 (pipes and
 * `|| true` hide exit codes); with a pipe and no recognisable summary we don't know.
 */
export function testOutcome({ cmd, exitCode, isError, output }) {
  const printed = outcomeFromOutput(output);
  if (printed) return printed;
  // Gradle prints BUILD SUCCESSFUL/FAILED for every task; for a test task it is the result
  if (/\bgradlew?\b[^|;&]*[\s:]test\b/.test(cmd) && output) {
    if (/\bBUILD FAILED\b|There were failing tests/.test(output)) return "fail";
    if (/\bBUILD SUCCESSFUL\b/.test(output)) return "pass";
  }
  if (isPiped(cmd) || /\|\|\s*(?:true|:|exit 0)/.test(cmd)) return null;
  const code = exitCode ?? (isError === true ? 1 : isError === false ? 0 : null);
  if (code == null) return null;
  // The exit status belongs to the test only when the shell structure says so: the test
  // runs last after nothing but setup (`cd x && pytest`, `ulimit -n 4096; pytest`), or a
  // pure `&&` chain exits 0 (every step, the test included, succeeded). `npm test; true`
  // exits 0 because of `true`, and a non-zero `make && pytest` may never have run pytest.
  const { parts, seps } = chain(cmd);
  const last = parts.length - 1;
  const testLast = isTestCommand(parts[last]);
  // the last command always runs after `;` or a newline, so its status is the script's;
  // after `&&` (or under `set -e`) an earlier failure could stop the script first
  const lastAlwaysRuns = seps.every((s, i) => s !== "&&" || SETUP.test(parts[i] || "")) && !/\bset\s+-\w*e/.test(cmd);
  if (testLast && (code === 0 || lastAlwaysRuns || parts.slice(0, last).every((p) => SETUP.test(p)))) return code === 0 ? "pass" : "fail";
  if (code === 0 && seps.every((s) => s === "&&") && parts.some((p) => isTestCommand(p))) return "pass";
  return null;
}

const SETUP = /^\s*(?:cd|pushd|export|source|\.|ulimit|set|unset|umask|nvm\s+use|conda\s+activate|(?:\w+=\S*\s*)+$)/;

/** Simple commands and the separators between them (heredoc bodies and quoted text ignored). */
function chain(cmd) {
  const flat = stripHeredocs(cmd).replace(/'[^']*'|"(?:[^"\\]|\\.)*"/g, "Q");
  const seps = flat.match(/&&|\|\||;|\n/g) || [];
  const parts = splitCommands(cmd);
  return { parts: parts.length ? parts : [cmd], seps };
}

/**
 * Files a shell command writes. `paths` are the targets we can name (sed -i, perl -i,
 * redirects, tee, cp/mv destinations); `unresolved` is set for writes we can't pin to a
 * file (scripts calling write_text, git checkout/apply/merge, patch).
 */
export function shellWrites(cmd) {
  const flat = stripHeredocs(cmd).replace(/'[^']*'|"(?:[^"\\]|\\.)*"/g, "Q");
  const paths = [];
  for (const part of flat.split(/\s*(?:&&|\|\||;|\n|\|)\s*/)) {
    const words = part.trim().split(/\s+/);
    const tool = words.find((w) => !/^\w+=/.test(w) && w !== "sudo");
    if ((tool === "sed" && words.some((w) => /^-\w*i/.test(w))) || (tool === "perl" && words.some((w) => /^-\w*i/.test(w)))) {
      paths.push(...words.slice(1).filter((w) => !w.startsWith("-") && w !== "Q" && /[./]/.test(w) && !/^[>|]/.test(w)));
    }
    if ((tool === "cp" || tool === "mv") && words.length >= 3) paths.push(words[words.length - 1]);
    for (const m of part.matchAll(/(?:^|[^\d&<>])>{1,2}\s*([^\s;&|<>]+)/g)) if (m[1] !== "/dev/null" && m[1] !== "Q") paths.push(m[1]);
    for (const m of part.matchAll(/\btee\s+(?:-a\s+)?([^\s;&|<>]+)/g)) if (m[1] !== "/dev/null") paths.push(m[1]);
    // PowerShell
    for (const m of part.matchAll(/\b(?:Set-Content|Add-Content|Out-File)\b(?:\s+-(?:Path|FilePath|LiteralPath))?\s+([^\s;&|<>-][^\s;&|<>]*)/gi)) paths.push(m[1]);
  }
  // Git operations that rewrite the working tree (not merge-base, not checkout -b).
  let unresolved = /\bgit\s+(?:apply|checkout(?!\s+-[bB]\b)|restore|stash\s+pop|merge(?!-)|cherry-pick|am|revert|reset\s+--(?:hard|merge|keep))\b|\bpatch\s+-|\bapply_patch\b/.test(cmd);
  // A script (python/node) that writes files: the files it clearly writes are edits
  // (source files among them; reports and logs are ignored). Files it only reads are not.
  // A write whose target we can't name leaves the change unresolved.
  const script = scriptWrites(cmd);
  paths.push(...script.paths.filter((p) => isCodePath(p)));
  if (script.unresolved) unresolved = true;
  // `$S/corr.py` or `$(mktemp)`: a path we can't resolve, almost always a scratch file
  const named = paths.filter((p) => p && p !== "Q" && !/[$`]/.test(p));
  return { paths: [...new Set(named)], unresolved };
}

const LIT = String.raw`["']([\w.~\/-]+\.[A-Za-z]{1,5})["']`;
const MODE_W = String.raw`\s*,\s*(?:mode\s*=\s*)?["'][wax]`;
// every place a script writes a file (not sys.stdout.write, not reads)
const WRITE_SITES = new RegExp(String.raw`\.(?:write_text|write_bytes)\(|\bopen\([^()]*(?:\([^()]*\)[^()]*)*${MODE_W}|\.open\(\s*["'][wax]|\bwriteFileSync\(|\bappendFileSync\(|\.extractall\(|\bshutil\.(?:copy\w*|move)\(|\bos\.(?:replace|rename)\(`, "g");
// what a path variable can be built from: Path('x.py'), root / 'x.py', os.path.join(root, 'x.py')
const PATH_EXPR = String.raw`(?:(?:pathlib\.)?Path\(\s*(?:[\w.]+\s*,\s*)?|(?:os\.)?path\.join\([^)]*?,\s*|(?:[\w.]+(?:\(\))?\s*\/\s*)+)?`;

/** Files a python/node script in a command writes: { paths, unresolved }. */
function scriptWrites(cmd) {
  const sites = (cmd.match(WRITE_SITES) || []).length;
  if (!sites) return { paths: [], unresolved: false };
  const paths = [];
  let resolved = 0;
  const add = (p) => {
    paths.push(p);
    resolved++;
  };
  // open('x.py', 'w'), Path('x.py').write_text(…), writeFileSync('x.js', …)
  for (const m of cmd.matchAll(new RegExp(String.raw`\bopen\(\s*${LIT}${MODE_W}`, "g"))) add(m[1]);
  for (const m of cmd.matchAll(new RegExp(String.raw`Path\(\s*${LIT}\s*\)\s*\.(?:write_text|write_bytes|open\(\s*["'][wax])`, "g"))) add(m[1]);
  for (const m of cmd.matchAll(new RegExp(String.raw`\(\s*(?:[\w.]+(?:\(\))?\s*\/\s*)+${LIT}\s*\)\s*\.(?:write_text|write_bytes)\(`, "g"))) add(m[1]);
  for (const m of cmd.matchAll(new RegExp(String.raw`\b(?:writeFileSync|appendFileSync)\(\s*${LIT}`, "g"))) add(m[1]);
  // p = Path('x.py') or p = 'x.py', later p.write_text(…) / open(p, 'w') / writeFileSync(p, …)
  for (const m of cmd.matchAll(new RegExp(String.raw`(?:\b(?:const|let|var)\s+)?\b([A-Za-z_]\w*)\s*=\s*${PATH_EXPR}${LIT}`, "g"))) {
    const [, v, p] = m;
    const writes = cmd.match(new RegExp(String.raw`\b${v}\s*\.(?:write_text|write_bytes)\(|\bopen\(\s*${v}${MODE_W}|\b${v}\.open\(\s*["'][wax]|\b(?:writeFileSync|appendFileSync)\(\s*${v}\b`, "g"));
    if (writes) {
      paths.push(p);
      resolved += writes.length;
    }
  }
  return { paths: [...new Set(paths)], unresolved: resolved < sites };
}

/**
 * Could this command have changed any file at all? Broader than shellWrites: the
 * test-only-fix check uses it so a fix made through the shell isn't missed.
 */
export function mayWriteFiles(cmd) {
  return /\bsed\s+(?:-\w*\s+)*-i|\bperl\s+-\w*i|write_text\(|write_bytes\(|\.write\(|open\([^)]*['"][wa]|\btee\b|\bapply_patch\b|\bgit\s+(?:apply|checkout|restore|stash\s+pop|merge|cherry-pick|am|revert)\b|\b(?:mv|cp|patch)\s|(?:^|[^\d&])>{1,2}\s*(?!\/dev\/null|&)\S/.test(cmd);
}

// --- claims -------------------------------------------------------------------------

const N = String.raw`\d[\d,]*`; // "1,760"
const CLAIM_PATTERNS = [
  new RegExp(String.raw`\ball\s+(?:${N}\s+)?(?:of\s+the\s+)?(?:[\w-]+\s+){0,3}tests?\s+(?:now\s+|still\s+)?(?:pass(?:es|ed)?|are\s+passing|succeed(?:ed)?|are\s+green)\b`, "i"),
  /\b(?:the\s+)?(?:[\w-]+\s+)?tests?\s+(?:now\s+|all\s+|still\s+|both\s+)?(?:pass(?:es|ed)?|are\s+(?:all\s+)?passing|is\s+passing|succeed(?:ed)?|are\s+(?:all\s+)?green)\b/i,
  /\b(?:test\s+suite|suite)\s+(?:now\s+|still\s+)?(?:is\s+)?(?:green|passes|passed|passing)\b/i,
  new RegExp(String.raw`\b${N}\s*\/\s*${N}\s+(?:[\w-]+\s+){0,2}tests?\s+pass`, "i"),
  new RegExp(String.raw`\b${N}\s+passed\b|\b${N}\s+(?:[\w-]+\s+){0,2}tests?\s+passed\b`, "i"),
  /\ball\s+green\b/i,
  /\btests?\s+(?:are\s+)?(?:still\s+)?green\b/i,
  /✅\s*(?:all\s+)?tests?\b/i,
  // a test file named as passing: "test_api.py passes", "cart.test.ts is green"
  /\b[\w.-]*(?:test_[\w-]+|[\w-]+[._]test|[\w-]+\.spec)\.[a-z]{1,4}\b(?:\s+\w+){0,2}?\s+(?:pass(?:es|ed)?|is\s+passing|is\s+green)\b/i,
  /\b(\d[\d,]*)\s*\/\s*\1\s+(?:[\w-]+\s+){0,2}tests?\b/i, // "447/447 tests"
];

// The general rule behind the patterns above: a test noun and a pass verb in one
// sentence ("the branch passes 2071 tests", "they all pass (51 tests)").
// Within five words either way, and not a compound like "1,503-test VM pass".
const NEAR = [
  /(?<!-)\b(?:tests?|suite|specs?)\b(?:\W+\w+){0,5}?\W+(?:pass(?:es|ed)?|passing|green|succeed(?:s|ed)?)\b/i,
  /\b(?:pass(?:es|ed)?|passing)\b(?:\W+[\w,]+){0,5}?\W+tests?\b/i,
  /\btests?\b.*\ball\s+[\d,]+\s+(?:now\s+)?pass(?:ed)?\b/i, // "added tests …, so all 35 pass"
];

// Other languages agents commonly answer in. Kept simple: subject + pass verb.
const L = String.raw`(?:\s+\p{L}+){0,3}\s+`;
const OTHER_LANGS = [
  new RegExp(String.raw`(?<!\p{L})(?:τεστ|tests?|δοκιμές|suite)${L}(?:περνούν|περνάνε|πέρασαν|περνάει|πέρασε|πέρασε\s+πλήρως|είναι\s+πράσινα)(?!\p{L})`, "iu"),
  new RegExp(String.raw`\b(?:pruebas|tests?)${L}(?:pasan|pasaron|están\s+en\s+verde)(?!\p{L})`, "iu"),
  new RegExp(String.raw`\b(?:testes|tests?)${L}(?:passam|passaram|estão\s+(?:verdes|passando))(?!\p{L})`, "iu"),
  new RegExp(String.raw`\b(?:tests?)${L}(?:sind\s+grün|bestanden|laufen\s+(?:durch|grün)|sind\s+erfolgreich)(?!\p{L})`, "iu"),
  new RegExp(String.raw`\b(?:tests?)${L}(?:passent|sont\s+(?:au\s+)?verts?|ont\s+réussi|réussissent)(?!\p{L})`, "iu"),
  new RegExp(String.raw`\b(?:test)${L}(?:passano|sono\s+(?:tutti\s+)?verdi|superati)(?!\p{L})`, "iu"),
];
const OTHER_HEDGES = /(?<!\p{L})(?:δεν|θα|αν|όταν|no|si|cuando|nicht|kein\w*|wenn|ne|n'|quand|não|se|quando|non)(?!\p{L})|\?/iu;

const HEDGES =
  /\b(?:claim(?:s|ed|ing)?|agents?|issues?\s+\[?#|I['’]ll|I\s+will|I['’]m\s+going|going\s+to|confirm|plan(?:s|ning)?\s+(?:to|is)|next|then|not|no\s+longer|fail(?:s|ed|ing|ures?)?|should|would|could|will|might|may|to\s+pass|make\s+sure|ensure|until|unless|if|whether|once|after\s+(?:you|we)|verify|check\s+(?:that|whether)|expect(?:ed)?|need(?:s|ed)?\s+to|want|so\s+that|in\s+order|let['’]s|let\s+me|running|run\s+(?:the|it|them)|before|previously|earlier|originally|(?:against|on|with|for)\s+the\s+original|pre-?existing|baseline|unrelated|except|flaky|mostly|partly|partially|in\s+isolation|despite|continuing|still\s+running|recorded|pre-?fix|so\s+far|mixed|otherwise|blocked|(?:on|in)\s+CI|GitHub\s+Actions|by\s+(?:value|reference)|on\s+(?:the\s+)?(?:device|hardware|phone|emulator|simulator|Pixel|iPhone)|over\s+USB|manual(?:ly)?|the\s+other\s+[\d,]+)\b|\b(?!all\b)\w+\s+[\d,]+\s+of\s+[\d,]+\b|n['’]t\b|\?/i;

/** Return the sentence in which the agent claims tests pass, or null. */
export function findTestClaim(text) {
  return findTestClaims(text, 1)[0] ?? null;
}

/** Every sentence in which the agent claims tests pass, in order (at most `limit`). */
export function findTestClaims(text, limit = Infinity) {
  const out = [];
  if (!text || text.length > 20000) return out;
  // Inline code and short quotes are masked for the checks below and restored in the
  // sentence that is returned, so a claim is quoted as the agent wrote it.
  const kept = [];
  const mask = (orig, shown) => {
    kept.push(orig);
    return `\u0000${kept.length - 1}:${shown}\u0000`;
  };
  const restore = (s) => s.replace(/\u0000(\d+):[^\u0000]*\u0000/g, (_, i) => kept[Number(i)]);
  const masked = text
    .replace(/```[\s\S]*?```/g, " ") // pasted runner output is evidence, not a claim
    .replace(/`([^`\n]*)`/g, (m, code) => (/\d[\d,]*\s+passed\b/.test(code) ? code : mask(m, "CODE")))
    .replace(/"[^"\n]{0,80}"|“[^”\n]{0,80}”/g, (m) => mask(m, "QUOTE")); // talking about the phrase "tests pass"
  for (const raw of masked.split(/(?<=[.!;\n])\s+|\n+/)) {
    const shown = raw.trim();
    const sentence = shown.replace(/\u0000\d+:([^\u0000]*)\u0000/g, "$1");
    if (!sentence || sentence.length > 400) continue;
    if ((sentence.match(/\|/g) || []).length >= 2) continue; // table rows
    // "0 failed" and "3 skipped" don't contradict a pass
    const checked = sentence.replace(/\b(?:0|zero|no)\s+(?:[\w-]+\s+)?(?:failed|failures?|errors?)\b|\b[\d,]+\s+skipped\b/gi, "");
    // "6/7 suites pass" is a partial result; "48/48" is not
    if ([...checked.matchAll(/\b(\d+)\s*\/\s*(\d+)\b/g)].some((m) => m[1] !== m[2])) continue;
    const english =
      CLAIM_PATTERNS.some((rx) => rx.test(sentence)) ||
      (NEAR.some((rx) => rx.test(sentence)) && !/^\W*pass\b/i.test(sentence));
    const claim = (english && !HEDGES.test(checked)) || (!english && OTHER_LANGS.some((rx) => rx.test(sentence)) && !OTHER_HEDGES.test(checked));
    if (claim) {
      out.push(restore(shown));
      if (out.length >= limit) break;
    }
  }
  return out;
}

/** A claim about the whole suite: "the full suite passes", "entire test suite green". */
export function claimsFullSuite(sentence) {
  return /\b(?:full|whole|entire|complete)\s+(?:test\s+)?(?:suite|test\s+run|run)\b|\bevery\s+test\b/i.test(sentence);
}

// CI results: judged only with --ci, against GitHub Actions. "CI is green", "all checks pass
// on the PR", "GitHub Actions passed".
const CI_CLAIM =
  /\b(?:CI|GitHub\s+Actions|(?:the\s+)?(?:workflow|pipeline)s?|(?:PR|pull\s+request)\s+checks|(?:all\s+)?checks\s+(?:on|in|for)\s+(?:the\s+)?(?:PR|pull\s+request|branch|GitHub))\b(?:\W+\w+){0,4}?\W+(?:green|pass(?:es|ed|ing)?|succeed(?:s|ed)?|successful|succeeded)\b|\b(?:green|pass(?:es|ed)?|succeeded)\s+(?:on|in)\s+(?:CI|GitHub\s+Actions)\b|\bCI\s*(?:is\s+)?✅/i;
const CI_HEDGES = new RegExp(HEDGES.source.replace("|(?:on|in)\\s+CI|GitHub\\s+Actions", ""), "i");

/** Return the sentence in which the agent says CI passed, or null. */
export function findCiClaim(text) {
  if (!text || text.length > 20000) return null;
  const prose = text.replace(/```[\s\S]*?```/g, " ").replace(/`[^`\n]*`/g, "CODE").replace(/"[^"\n]{0,80}"|“[^”\n]{0,80}”/g, "QUOTE");
  for (const raw of prose.split(/(?<=[.!;\n])\s+|\n+/)) {
    const sentence = raw.trim();
    if (!sentence || sentence.length > 400 || (sentence.match(/\|/g) || []).length >= 2) continue;
    const checked = sentence.replace(/\b(?:0|zero|no)\s+(?:[\w-]+\s+)?(?:failed|failures?|errors?)\b|\b[\d,]+\s+skipped\b/gi, "");
    if (CI_CLAIM.test(sentence) && !CI_HEDGES.test(checked)) return sentence;
  }
  return null;
}

// --- files ----------------------------------------------------------------------------

const TEST_PATH =
  /(?:^|\/)(?:tests?|__tests__|spec|specs)\/|(?:^|\/)test_[^/]*\.py$|_test\.(?:py|go|rb|exs?)$|\.(?:test|spec)\.[cm]?[jt]sx?$|Tests?\.(?:java|kt|cs|swift)$|(?:^|\/)conftest\.py$/;

export function isTestPath(p) {
  return TEST_PATH.test(p.replace(/\\/g, "/"));
}

// Only edits to source files make an earlier test run stale. Config, docs, data and
// generated output are ignored, as are hidden tool dirs such as agent worktrees.
const SOURCE = /\.(?:py|pyx|ipynb|[cm]?[jt]sx?|vue|svelte|go|rs|java|kt|kts|scala|groovy|rb|php|cs|fs|swift|m|mm|c|h|cc|cpp|hpp|cxx|zig|ex|exs|erl|hs|ml|clj|dart|lua|r|jl|sql)$/i;
const IGNORED_DIR = /(?:^|\/)(?:output|outputs|reports?|dist|build|coverage|node_modules|__pycache__|vendor|target)\/|(?:^|\/)\.[^/]+\//i;

export function isCodePath(p) {
  return SOURCE.test(p) && !IGNORED_DIR.test(p);
}

// --- test weakening inside a single edit ------------------------------------------------

const ASSERT_LINE = /\bassert\w*\b|\bexpect\s*\(|\bself\.assert\w+|\bt\.(?:Error|Errorf|Fatal|Fatalf)\b|\b(?:require|assert)\.\w+\s*\(|\bshould\b\.|\.to(?:Be|Equal|Have|Throw|Match|Contain)\w*\(/;
const SKIP_MARK =
  /@pytest\.mark\.(?:skip|skipif|xfail)\b|@unittest\.(?:skip\w*|expectedFailure)\b|\bpytest\.(?:skip|xfail)\s*\(|\bself\.skipTest\s*\(|\b(?:it|test|describe)\.(?:skip|todo)\b|\bx(?:it|test|describe)\s*\(|\bt\.Skip\w*\s*\(|#\[ignore\]|@Disabled\b|@Ignore\b/;
const FOCUS_MARK = /\b(?:it|test|describe)\.only\b|\bf(?:it|describe)\s*\(/;
const TRIVIAL = /\bassert\s+(?:True|1|not\s+False)\s*(?:$|#)|\bexpect\s*\(\s*(?:true|1)\s*\)\s*\.\s*toBe(?:Truthy)?\s*\(\s*(?:true|1)?\s*\)|\bassertTrue\s*\(\s*True\s*\)/;
const TEST_DEF = /^\s*(?:async\s+)?def\s+test\w*|\b(?:it|test)(?:\.(?:only|skip|todo|each|concurrent))?\s*\(\s*['"`]|^func\s+Test\w*|@Test\b|#\[test\]/;

const count = (lines, rx) => lines.filter((l) => rx.test(l)).length;

export const assertionLines = (lines) => (lines || []).filter((l) => ASSERT_LINE.test(l));

/**
 * Compare removed vs added lines of one edit to a test file.
 * Returns a list of short reasons (empty when nothing looks weakened).
 */
export function weakening(removed, added) {
  const reasons = [];
  const strip = (ls) => ls.filter((l) => !/^\s*(?:#|\/\/)/.test(l));
  const r = strip(removed);
  const a = strip(added);
  const lostAsserts = count(r, ASSERT_LINE) - count(a, ASSERT_LINE);
  const lostTests = count(r, TEST_DEF) - count(a, TEST_DEF);
  if (lostTests > 0) reasons.push(`removed ${lostTests} test case${lostTests > 1 ? "s" : ""}`);
  if (lostAsserts > 0) reasons.push(`removed ${lostAsserts} assertion${lostAsserts > 1 ? "s" : ""}`);
  const newTests = count(a, TEST_DEF) > count(r, TEST_DEF);
  if (!newTests && count(a, SKIP_MARK) > count(r, SKIP_MARK)) reasons.push("added skip/xfail");
  if (count(a, FOCUS_MARK) > count(r, FOCUS_MARK)) reasons.push("added .only");
  if (count(a, TRIVIAL) > count(r, TRIVIAL)) reasons.push("added always-true assertion");
  return reasons;
}

// --- risky commands -------------------------------------------------------------------

// Only git operations, matched at command position: these destroy work or skip checks,
// and unlike `rm -rf` on a temp dir they are almost never routine.
export const RISKY = [
  { id: "no-verify", label: "skipped git hooks with --no-verify", rx: /^git\b.*\s--no-verify\b/ },
  { id: "force-push", label: "force-pushed", rx: /^git\s+push\b.*\s(?:--force|-f)(?:\s|$)/ },
  { id: "reset-hard", label: "ran git reset --hard", rx: /^git\s+reset\b.*\s--hard\b/ },
  { id: "discard", label: "discarded uncommitted changes", rx: /^git\s+(?:checkout\s+(?:--\s+)?\.$|restore\s+(?:--\S+\s+)*\.$|clean\s+-\w*f)/ },
];

export function riskyIn(cmd) {
  // quoted text is data (commit messages, test strings, `node -e` snippets), not commands
  const unquoted = stripHeredocs(cmd).replace(/'[^'\n]*'|"(?:[^"\\\n]|\\.)*"/g, "Q");
  const parts = splitCommands(unquoted).map((p) => p.replace(/^(?:\w+=\S*\s+)*(?:sudo\s+)?/, ""));
  return RISKY.filter((r) => parts.some((p) => r.rx.test(p)));
}

// --- user corrections ------------------------------------------------------------------

const CORRECTION =
  /^(?:no\b|nope\b|stop\b|don'?t\b|that'?s\s+(?:wrong|not)|wrong\b|why\s+(?:did|are|would)\s+you|revert|undo)|\bi\s+(?:told|asked)\s+you\b|\b(?:i\s+said|again\?|you\s+(?:broke|deleted|ignored|forgot|didn'?t)|not\s+what\s+i\s+(?:asked|wanted)|that'?s\s+not\s+what)\b/i;

export function isCorrection(text) {
  if (!text) return false;
  const t = text.trim();
  if (t.length > 1500) return false; // long messages are usually new instructions or pastes
  return CORRECTION.test(t);
}

// --- which run a claim refers to ---------------------------------------------------------

const num = (s) => Number(String(s).replace(/,/g, ""));

/** How many tests a runner says passed ("12 passed", "ℹ pass 12", "Ran 12 tests … OK"). */
export function passedCount(output) {
  if (!output) return null;
  const m =
    /\b(\d[\d,]*) passed\b/.exec(output) ||
    /(?:^|\s)(?:ℹ|#) pass (\d+)(?:\s+(?:ℹ|#) fail \d+)?\s*$/m.exec(output) ||
    /\bTests?:\s+(\d+) passed/.exec(output) ||
    /\btest result: ok\. (\d+) passed/.exec(output) ||
    /^Ran (\d+) tests? in [\d.]+s\s+OK\b/m.exec(output);
  return m ? num(m[1]) : null;
}

/** Counts a claim names: "31/31", "all 17 atom tests", "62 passed". */
export function claimedCounts(text) {
  const out = new Set();
  for (const m of text.matchAll(/\b(\d[\d,]*)\s*\/\s*(\d[\d,]*)\b/g)) if (num(m[1]) === num(m[2])) out.add(num(m[1]));
  // only counts tied to passing: "62 passed", "17 atom tests pass", "passes 2071 tests";
  // not "there are 11 regression tests, and the suite passed"
  for (const rx of [
    /\b(\d[\d,]*)\s+(?:tests?\s+)?(?:passed|passing)\b/gi,
    /\b(\d[\d,]*)\s+(?:[\w-]+\s+){0,3}?tests?\s+(?:all\s+|now\s+|still\s+)?(?:pass(?:es|ed)?|passing)\b/gi,
    /\bpass(?:es|ed)?\s+(?:all\s+)?(\d[\d,]*)\b/gi,
  ]) for (const m of text.matchAll(rx)) out.add(num(m[1]));
  return out;
}

/** Test files a claim names: test_api.py, cart.test.ts, server_test.go. */
export function claimedTestFiles(text) {
  return [...text.matchAll(/\b[\w.-]*(?:test_[\w-]+|[\w-]+[._]test|[\w-]+\.spec)\.[a-z]{1,4}\b/gi)].map((m) => m[0]);
}

/**
 * Expand variables assigned earlier in the same command, so `S=/tmp/x; cat > $S/f.py` and
 * `cd "$W"` resolve. Values built from command output (`$(mktemp)`) stay unexpanded.
 * `env` supplies known values such as HOME.
 */
export function expandVars(cmd, env = {}) {
  const vars = { ...env };
  const sub = (v) => v.replace(/\$\{?([A-Za-z_]\w*)\}?/g, (m, k) => (k in vars ? vars[k] : m));
  // sh: NAME=value, export NAME=value; PowerShell: $name = "value"
  const rx = /(?:^|[;&\n]\s*|\s)(?:export\s+)?([A-Za-z_]\w*)=(?:"([^"]*)"|'([^']*)'|([^\s;&|"'`]+))|\$([A-Za-z_]\w*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
  for (const m of cmd.matchAll(rx)) {
    const name = m[1] ?? m[5];
    const raw = m[2] ?? m[3] ?? m[4] ?? m[6] ?? m[7] ?? "";
    if (/\$\(|`/.test(raw)) continue; // computed at run time
    vars[name] = m[3] !== undefined || m[7] !== undefined ? raw : sub(raw);
  }
  return sub(cmd);
}