From 1fca78235bac7d8588778cb72614426e1b6f463f Mon Sep 17 00:00:00 2001 From: imkp1 Date: Mon, 20 Jul 2026 23:53:31 +0530 Subject: [PATCH 1/3] fix(repo-finder): grade maintainer signal from comment text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The approval bonus was set from comment association alone, so any comment by an OWNER/MEMBER/COLLABORATOR earned the full +3 and rendered as "Maintainer approved: yes" — refutations and bare links included. Replace the boolean with a signal graded from the comment text: invites_pr +5, confirms +3, neutral +1, none 0. Association stays a prerequisite and stops being sufficient. Close two claims the association and assignee tests cannot see: - a maintainer who confirms and claims in one sentence; the claim test now runs before the tiering, which would otherwise score it as approval - a non-maintainer claim, windowed at 14 days so an abandoned one does not fence the issue off permanently Strip links and mentions before grading and require prose in what remains. A comment that is only a pointer states no position. Reads the bulk issue payload already in hand; no additional API calls. --- agents/repo-finder.md | 32 +++++++++---- schemas/repo_shortlist.schema.json | 5 ++- scripts/orchestrator/triage_filter.sh | 51 ++++++++++++++++++++- tests/scripts/test_triage_filter.sh | 65 ++++++++++++++++++++++++++- 4 files changed, 142 insertions(+), 11 deletions(-) diff --git a/agents/repo-finder.md b/agents/repo-finder.md index a2f6242..abde717 100644 --- a/agents/repo-finder.md +++ b/agents/repo-finder.md @@ -575,7 +575,7 @@ fi 2. A `good first issue` or `help wanted` label (explicit triage-accepted signal). 3. A `kind:*` / `type:*` / `priority:*` / `severity:*` / `area:*` label (maintainer-applied taxonomy — NOT just `needs-triage` / `triage` / `status:need-triage` alone). - Why this gate exists: a `needs-triage`-only issue with zero maintainer comments is an untriaged feature request or unconfirmed bug report. Shipping a PR against it at Apache-/Airflow-class repos triggers a lazy-consensus or devlist redirect that structurally caps merge probability below 95% regardless of code quality. The prior `has_maintainer_approval` field was additive (+3 bonus), so issues without maintainer approval could still reach rank 1 via the other bonuses. This gate converts it to a prerequisite. + Why this gate exists: a `needs-triage`-only issue with zero maintainer comments is an untriaged feature request or unconfirmed bug report. Shipping a PR against it at Apache-/Airflow-class repos triggers a lazy-consensus or devlist redirect that structurally caps merge probability below 95% regardless of code quality. The maintainer-signal bonus is additive, so an issue no maintainer engaged with could otherwise reach rank 1 on the other bonuses alone. This gate converts engagement to a prerequisite. A bot is not a maintainer — exclude bot accounts from the comment test. @@ -726,7 +726,7 @@ jq -r --rawfile m "$MAINTAINERS" \ elif ((now - ($engaged | max)) / 86400 | floor) > $window then "SKIP \(.number): no maintainer engagement in \(((now - ($engaged | max)) / 86400 | floor))d (window \($window)d) — abandoned" else - "KEEP \(.number) engaged_days=\(((now - ($engaged | max)) / 86400 | floor)) maintainer_commented=\(.maintainer_commented) approval=\(.maintainer_comment_assoc)" + "KEEP \(.number) engaged_days=\(((now - ($engaged | max)) / 86400 | floor)) maintainer_commented=\(.maintainer_commented) signal=\(.maintainer_signal)" end ' "$SCRATCH/stage_a.jsonl" ``` @@ -751,14 +751,30 @@ visibly drop one. Apply the prose here, then say in `notes` that it did. - **Is a bug** (not a feature request): +3 - **Has reproduction steps**: +2 - **Labeled good-first-issue or help-wanted**: +2 -- **Maintainer commented with approval signal** ("PRs welcome", "happy to review"): +3. Requires `maintainer_comment_assoc` — a union-only commenter does not earn this bonus. +- **Maintainer signal**, graded from `maintainer_signal`: `invites_pr` **+5** · `confirms` **+3** · `neutral` **+1** · `none` **0**. Association is the prerequisite, never the grade — a union-only commenter earns nothing here. - **Maintainer engaged within the last 28 days**: +2 - **Issue age**, graded: 2–30d **+1** · 30–90d **0** · 90–365d **−1** · >365d **−2**. Scored, never hard-skipped — old is riskier, not worthless. - **Scope is small** (likely < 100 lines, single-file fix): +2 -Set `best_issue.has_maintainer_approval` from `maintainer_comment_assoc` — true only when GitHub reported a commenter on this issue as `OWNER`/`MEMBER`/`COLLABORATOR`. Never set it from `maintainer_commented`: that flag includes the wide 3c maintainer union, which is correct for passing the gate but too loose to call approval. - -Label-only triage and union-only comments still pass the gate; they just record `has_maintainer_approval=false`. The field answers one question — "did an account GitHub vouches for as a maintainer speak on this issue?" — and the +3 approval bonus is the only thing that reads it. +Set `best_issue.maintainer_signal` from the `maintainer_signal` the triage filter +emitted. Do not recompute it from `maintainer_comment_assoc`: association answers +"did an account GitHub vouches for speak here", which is a prerequisite for being +graded and not a grade. Never derive it from `maintainer_commented` either — that +flag includes the wide 3c union, correct for passing the gate and too loose to +read as endorsement. + +The filter grades on comment text, and only on text that is text: it strips links +and bare mentions first, so a maintainer comment that is only a pointer somewhere +else grades `none`. A pointer states no position on this issue. + +Read the graded comment yourself before accepting the tier. The filter matches +prose with regexes and will misgrade: a refutation phrased in the project's own +domain terms disputes the premise while matching none of the decline patterns. +**If the maintainer disputes the premise, redirects it, or calls it +working-as-intended, skip the issue outright** rather than scoring it down — +zeroing the bonus still leaves bug + repro + small scope at 7, and labels plus +recency clear the 8 threshold on their own. Record any tier you overrode in +`notes`, so the regex gaps stay visible. If no issue scores 8+ after applying the hard filter, mark the repo as "no clear opportunity right now" and exclude from the final list. @@ -854,7 +870,7 @@ needs that number to know the list is partial rather than the field being thin. "title": "issue title", "type": "bug", "labels": ["bug", "good first issue"], - "has_maintainer_approval": true, + "maintainer_signal": "invites_pr", "issue_score": 12, "url": "https://github.com/OWNER/REPO/issues/1234" }, @@ -880,7 +896,7 @@ Found N repos worth contributing to (searched M candidates). ## Top Pick **owner/repo#1234** — [issue title] -- Maintainer approved: yes +- Maintainer signal: invites a PR - Type: bug - Estimated scope: small (single file) - Median PR merge time: 2 days diff --git a/schemas/repo_shortlist.schema.json b/schemas/repo_shortlist.schema.json index e6a727e..1264272 100644 --- a/schemas/repo_shortlist.schema.json +++ b/schemas/repo_shortlist.schema.json @@ -80,7 +80,10 @@ "title": {"type": "string"}, "type": {"type": "string"}, "labels": {"type": "array", "items": {"type": "string"}}, - "has_maintainer_approval": {"type": "boolean"}, + "maintainer_signal": { + "type": "string", + "enum": ["invites_pr", "confirms", "neutral", "none"] + }, "issue_score": {"type": "number"}, "url": {"type": "string"} } diff --git a/scripts/orchestrator/triage_filter.sh b/scripts/orchestrator/triage_filter.sh index efddb43..5e0bbf7 100755 --- a/scripts/orchestrator/triage_filter.sh +++ b/scripts/orchestrator/triage_filter.sh @@ -75,7 +75,29 @@ jq -c --rawfile m "$MAINTAINERS" --argjson now "$NOW" ' + "|i.m working on|working on (a fix|this)" + "|(already |should have )?fixed (it |this )?in [a-z0-9.@/#-]" + "|(fix|patch) (has )?landed|landed in|resolved (in|by) (#|[a-z0-9])" - + "|(fix|pr) (is )?incoming") as $claimed + + "|(fix|pr) (is )?incoming" + # A claim is often worded as an intention to act rather than a patch in hand. + # "this is valid, we will look into it" confirms the defect and takes it in one + # sentence — which is why the claim test runs before the signal tiering below. + + "|we.ll look into|will look into|looking into (this|it) on our" + + "|noted (it )?(on our side|internally)|on our (side|end)|taking (this|it) on") as $claimed + + # An outsider saying "I would like to work on this" is not an assignee, so the + # assignee test cannot see the claim. Windowed, unlike the maintainer test: a + # drive-by claim nobody acted on must not fence the issue off forever. + | ("i.d like to work on|i would like to work on|can i (take|work on|pick up)" + + "|i.ll (take|work on|pick) (this|it)|assign (this|it) to me" + + "|i.m (going to |gonna )?work(ing)? on (this|it)" + + "|(may|could) i (take|work on) (this|it)") as $outsider_claim + | 1209600 as $claim_ttl + + # Association makes a comment gradeable; it does not make it approval. The text + # sets the grade. + | ("prs? (are )?welcome|welcome a (pr|patch)|happy to (review|take a pr|accept)" + + "|feel free to (submit|open|send)|would (accept|welcome) a (pr|patch)" + + "|go ahead and (open|submit)|contributions? welcome") as $invites + | ("reproduc|confirmed|i see the same|can confirm|this is (valid|a bug|indeed)" + + "|good catch|you.re right|nice find|makes sense to me") as $confirms # Announcements and containers wear defect labels. A pinned "the project moved" # notice earns a maintainer taxonomy label and every engagement signal the gate @@ -104,6 +126,29 @@ jq -c --rawfile m "$MAINTAINERS" --argjson now "$NOW" ' | ($i.labels | map(.name | ascii_downcase)) as $L | ($mc | map(.body // "") | join("\n") | ascii_downcase) as $mbody + # Grade only comments that carry prose. Strip links and bare @mentions first: a + # comment that is only a pointer to somewhere else states no position on this + # issue, and no sentiment tier can read one. The 15-char floor is what is left + # after stripping — "cc @someone for the quant path" survives it, a bare URL + # does not. + | ($mca | map((.body // "") + | ascii_downcase + | gsub("https?://\\S+"; " ") + | gsub("@[a-z0-9_-]+"; " ") + | gsub("\\s+"; " ") | sub("^ +"; "") | sub(" +$"; "")) + | map(select(length >= 15))) as $gradeable + | ($gradeable | join("\n")) as $gbody + | (if ($gradeable | length) == 0 then "none" + elif ($gbody | test($invites)) then "invites_pr" + elif ($gbody | test($confirms)) then "confirms" + else "neutral" end) as $signal + + # Non-maintainer claims, still inside the window. + | ($human | map(select((.authorAssociation | IN("OWNER","MEMBER","COLLABORATOR")) | not)) + | map(select(((.body // "") | ascii_downcase | test($outsider_claim)) + and ((.createdAt | fromdateiso8601) > ($now - $claim_ttl)))) + | length) as $outsider_live + # (24h rule) an issue younger than a day has not been triaged; racing to it # produces noise PRs. | select(($i.createdAt | fromdateiso8601) <= ($now - 86400)) @@ -139,6 +184,9 @@ jq -c --rawfile m "$MAINTAINERS" --argjson now "$NOW" ' elif ($mbody | test($claimed)) then {verdict: "SKIP", number: $i.number, reason: "maintainer is already fixing it"} + elif $outsider_live > 0 then + {verdict: "SKIP", number: $i.number, + reason: "claimed — another contributor said they are taking it"} else # Carry the issue payload, not just a verdict on a number. The Step-4 rubric # scores title, labels, body and createdAt, and the agent contract forbids @@ -150,6 +198,7 @@ jq -c --rawfile m "$MAINTAINERS" --argjson now "$NOW" ' createdAt: $i.createdAt, maintainer_commented: ($mc | length > 0), maintainer_comment_assoc: ($mca | length > 0), + maintainer_signal: $signal, last_maintainer_comment: ($mc | map(.createdAt) | max)} end ' "$ISSUES" diff --git a/tests/scripts/test_triage_filter.sh b/tests/scripts/test_triage_filter.sh index e3d9a13..9181621 100755 --- a/tests/scripts/test_triage_filter.sh +++ b/tests/scripts/test_triage_filter.sh @@ -13,6 +13,8 @@ trap 'rm -rf "$tmpdir"' EXIT NOW=1783814400 OLD="2026-07-01T00:00:00Z" # 11 days old — clears the 24h rule FRESH="2026-07-11T18:00:00Z" # 6 hours old — must be dropped +RECENT="2026-07-10T00:00:00Z" # 2 days old — a claim this fresh is still live +STALE="2026-05-20T00:00:00Z" # 53 days old — a claim this old is abandoned printf 'unionlead\n' > "$tmpdir/maintainers.txt" @@ -106,7 +108,41 @@ cat > "$tmpdir/issues.json" < Date: Tue, 21 Jul 2026 00:01:54 +0530 Subject: [PATCH 2/3] fix(repo-finder): harden signal grading against three defects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match before measuring. The prose floor ran ahead of the pattern match, so "PRs welcome!" — 12 characters, the strongest signal there is — graded as no signal. The floor now separates a remark from a pointer and decides neutral-or-none only. Narrow the claim test. Bare "on our side" also matches "the regression is on our side", which owns the defect and invites a fix rather than claiming it. Require an intent to act. Guard the claim timestamp. A comment can serialize without createdAt, and null | fromdateiso8601 throws, aborting the batch and taking every other issue with it. Undated now reads as expired rather than as a live claim. --- scripts/orchestrator/triage_filter.sh | 32 +++++++++++++++---------- tests/scripts/test_triage_filter.sh | 34 ++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scripts/orchestrator/triage_filter.sh b/scripts/orchestrator/triage_filter.sh index 5e0bbf7..01b3b95 100755 --- a/scripts/orchestrator/triage_filter.sh +++ b/scripts/orchestrator/triage_filter.sh @@ -79,8 +79,10 @@ jq -c --rawfile m "$MAINTAINERS" --argjson now "$NOW" ' # A claim is often worded as an intention to act rather than a patch in hand. # "this is valid, we will look into it" confirms the defect and takes it in one # sentence — which is why the claim test runs before the signal tiering below. + # "on our side" only claims the work when it is attached to an intent to act: + # "the regression is on our side" owns the defect and invites a fix. + "|we.ll look into|will look into|looking into (this|it) on our" - + "|noted (it )?(on our side|internally)|on our (side|end)|taking (this|it) on") as $claimed + + "|noted (it )?(on our side|internally)|taking (this|it) on") as $claimed # An outsider saying "I would like to work on this" is not an assignee, so the # assignee test cannot see the claim. Windowed, unlike the maintainer test: a @@ -126,26 +128,32 @@ jq -c --rawfile m "$MAINTAINERS" --argjson now "$NOW" ' | ($i.labels | map(.name | ascii_downcase)) as $L | ($mc | map(.body // "") | join("\n") | ascii_downcase) as $mbody - # Grade only comments that carry prose. Strip links and bare @mentions first: a - # comment that is only a pointer to somewhere else states no position on this - # issue, and no sentiment tier can read one. The 15-char floor is what is left - # after stripping — "cc @someone for the quant path" survives it, a bare URL - # does not. + # Grade only what is left after stripping links and bare @mentions: a comment + # that is only a pointer somewhere else states no position on this issue, and no + # sentiment tier can read one. | ($mca | map((.body // "") | ascii_downcase | gsub("https?://\\S+"; " ") | gsub("@[a-z0-9_-]+"; " ") | gsub("\\s+"; " ") | sub("^ +"; "") | sub(" +$"; "")) - | map(select(length >= 15))) as $gradeable - | ($gradeable | join("\n")) as $gbody - | (if ($gradeable | length) == 0 then "none" - elif ($gbody | test($invites)) then "invites_pr" - elif ($gbody | test($confirms)) then "confirms" - else "neutral" end) as $signal + | map(select(length > 0))) as $stripped + | ($stripped | join("\n")) as $gbody + # Match before measuring. The floor separates a remark from a pointer, so it + # decides neutral-or-none only — applied first it would grade "PRs welcome!" + # (12 chars, the strongest signal there is) as no signal at all. + | (if ($stripped | length) == 0 then "none" + elif ($gbody | test($invites)) then "invites_pr" + elif ($gbody | test($confirms)) then "confirms" + elif ($stripped | any(length >= 15)) then "neutral" + else "none" end) as $signal # Non-maintainer claims, still inside the window. | ($human | map(select((.authorAssociation | IN("OWNER","MEMBER","COLLABORATOR")) | not)) + # `// ""` is not enough: an undated comment must not reach + # fromdateiso8601, which throws on null and aborts the whole batch. + # Undated reads as old, so the claim expires rather than fencing off. | map(select(((.body // "") | ascii_downcase | test($outsider_claim)) + and (((.createdAt // "") | length) > 0) and ((.createdAt | fromdateiso8601) > ($now - $claim_ttl)))) | length) as $outsider_live diff --git a/tests/scripts/test_triage_filter.sh b/tests/scripts/test_triage_filter.sh index 9181621..7f14372 100755 --- a/tests/scripts/test_triage_filter.sh +++ b/tests/scripts/test_triage_filter.sh @@ -142,7 +142,25 @@ cat > "$tmpdir/issues.json" < Date: Tue, 21 Jul 2026 00:24:11 +0530 Subject: [PATCH 3/3] fix(repo-finder): anchor the claim regex, discount negated reproduction Two regexes credited or skipped on text they had not actually matched. - `reproduc` matched its own negation, so a maintainer who could not reproduce graded `confirms` (+3). Negated phrases are now deleted from the graded text before the tier test, not short-circuited on, so a second maintainer who did reproduce it still earns the tier. Not a decline: the report may still be sound, so it falls to `neutral`. - `can i (take|work on|pick up)` had no object anchor, so "take a look at this" read as a claim and skipped the issue. Anchored on `(this|it)` like every other alternative. Regression tests for both, plus guards that neither over-corrects. --- scripts/orchestrator/triage_filter.sh | 16 ++++++++-- tests/scripts/test_triage_filter.sh | 46 ++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/scripts/orchestrator/triage_filter.sh b/scripts/orchestrator/triage_filter.sh index 01b3b95..cb5fc14 100755 --- a/scripts/orchestrator/triage_filter.sh +++ b/scripts/orchestrator/triage_filter.sh @@ -87,7 +87,10 @@ jq -c --rawfile m "$MAINTAINERS" --argjson now "$NOW" ' # An outsider saying "I would like to work on this" is not an assignee, so the # assignee test cannot see the claim. Windowed, unlike the maintainer test: a # drive-by claim nobody acted on must not fence the issue off forever. - | ("i.d like to work on|i would like to work on|can i (take|work on|pick up)" + # Every alternative anchors on its object. Unanchored, `take` swallows "take a + # look at this" and a question about the issue reads as a claim on it. + | ("i.d like to work on|i would like to work on" + + "|can i (take|work on|pick up) (this|it)" + "|i.ll (take|work on|pick) (this|it)|assign (this|it) to me" + "|i.m (going to |gonna )?work(ing)? on (this|it)" + "|(may|could) i (take|work on) (this|it)") as $outsider_claim @@ -100,6 +103,14 @@ jq -c --rawfile m "$MAINTAINERS" --argjson now "$NOW" ' + "|go ahead and (open|submit)|contributions? welcome") as $invites | ("reproduc|confirmed|i see the same|can confirm|this is (valid|a bug|indeed)" + "|good catch|you.re right|nice find|makes sense to me") as $confirms + # A maintainer who could not reproduce is stating the opposite of a confirmation, + # and the `reproduc` stem cannot tell the two apart. Not a decline either — the + # report may still be sound — so these phrases are deleted from the graded text + # rather than skipped on, leaving the comment to fall to the neutral floor. + # Deleted, not short-circuited: a second maintainer who did reproduce it must + # still be able to earn `confirms` on the same issue. + | ("(can.t|cannot|can not|could ?n.t|could not|unable to|failed to|fails to)" + + " reproduc\\w*|not reproducible|irreproducible|no repro\\b") as $no_repro # Announcements and containers wear defect labels. A pinned "the project moved" # notice earns a maintainer taxonomy label and every engagement signal the gate @@ -138,12 +149,13 @@ jq -c --rawfile m "$MAINTAINERS" --argjson now "$NOW" ' | gsub("\\s+"; " ") | sub("^ +"; "") | sub(" +$"; "")) | map(select(length > 0))) as $stripped | ($stripped | join("\n")) as $gbody + | ($gbody | gsub($no_repro; " ")) as $gbody_pos # Match before measuring. The floor separates a remark from a pointer, so it # decides neutral-or-none only — applied first it would grade "PRs welcome!" # (12 chars, the strongest signal there is) as no signal at all. | (if ($stripped | length) == 0 then "none" elif ($gbody | test($invites)) then "invites_pr" - elif ($gbody | test($confirms)) then "confirms" + elif ($gbody_pos | test($confirms)) then "confirms" elif ($stripped | any(length >= 15)) then "neutral" else "none" end) as $signal diff --git a/tests/scripts/test_triage_filter.sh b/tests/scripts/test_triage_filter.sh index 7f14372..c51901b 100755 --- a/tests/scripts/test_triage_filter.sh +++ b/tests/scripts/test_triage_filter.sh @@ -160,7 +160,33 @@ cat > "$tmpdir/issues.json" <