From c8c9da003baab91236925451ab7951f60d320b60 Mon Sep 17 00:00:00 2001 From: Bohdan Melnyk Date: Tue, 1 Sep 2026 00:13:59 +0200 Subject: [PATCH] fix(gate): skip weekly-cap claims on repeat sweeps --- scripts/docstring_gate.py | 330 +++++++++++++++++++++++++++++++++++ tests/test_docstring_gate.py | 182 +++++++++++++++++++ 2 files changed, 512 insertions(+) create mode 100644 scripts/docstring_gate.py create mode 100644 tests/test_docstring_gate.py diff --git a/scripts/docstring_gate.py b/scripts/docstring_gate.py new file mode 100644 index 000000000..4bf174e30 --- /dev/null +++ b/scripts/docstring_gate.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Adjudicate docstring bounty claims. + +WHY THIS EXISTS +--------------- +The #73 gate only recognises CODE-REVIEW claims -- `is_review_claim()` requires +"review" in the title. Every other bounty type falls straight through it, so +docstring, blog, star and bug claims had no automated adjudication at all and +simply accumulated unpaid. On 2026-08-10 that was 19 open docstring claims, +batches 31 to 49, none of them gate-processed. + +Docstring claims are unusually verifiable, so they are worth gating properly +rather than paying on assertion. A claim states a PR, a file, a function count +and a rate, and the diff should be `+N/-0` where N is that count. + +WHAT IT VERIFIES (all of it, before paying anything) + 1. The cited PR is **MERGED**. An open PR is not delivered work. + 2. The PR touches the claimed file. + 3. The added lines are **actually docstrings** -- lines opening with a quote + triple. This is the check that matters: without it "I added 40 docstrings" + pays out for 40 lines of anything. + 4. The claimed count matches what was really added. + +PAYMENT IS COMPUTED FROM THE VERIFIED COUNT, NEVER THE CLAIMED ONE. A claim +that overstates is paid the true amount rather than rejected outright -- the +usual cause is miscounting, not fraud, and rejecting honest arithmetic errors +teaches people to stop claiming. + +Sets `bounty-eligible` + `docstring-verified` and posts the arithmetic, so the +existing payout runner pays it on its next pass. Never moves RTC itself. + +Env: GITHUB_TOKEN, GH_REPO, ISSUE_NUMBER, RATE_PER_FUNC (0.01), MAX_RTC (25). +""" +from __future__ import annotations + +import datetime +import json +import os +import re +import subprocess +import sys + +REPO = os.environ.get("GH_REPO", "Scottcjn/rustchain-bounties") +NUM = os.environ.get("ISSUE_NUMBER", "") +RATE = float(os.environ.get("RATE_PER_FUNC", "0.01")) +# A single claim asking for more than this is not auto-payable. Docstring work +# is small by nature; a very large claim is either a mistake or something that +# deserves a human read. +MAX_RTC = float(os.environ.get("MAX_RTC", "25")) +# Per-contributor rolling weekly ceiling on DOCSTRING earnings specifically. +# +# A per-claim ceiling bounds nothing here: each batch is ~5 RTC, so batch 50, +# 51 and 52 all sail under it. The unbounded axis is volume, not size -- there +# is always another file to document, which is the same faucet shape as the +# ONBOARD comparison bounty that had to be closed at 98% farm share. +# +# At 0.01 RTC/function the weekly cap is a soft backstop, not the constraint: +# a docstring is a one-line comment (often on a test stub), so the per-unit price +# sits at the top of what the strongest contributors earn across ALL bounty +# types in a week (measured 2026-08-10: typical top earners 20-50 RTC/week). +# It caps a faucet without punishing anyone doing real work. +# +# This applies ONLY to docstring claims. Large one-off bounties are untouched. +MAX_RTC_PER_WEEK = float(os.environ.get("MAX_RTC_PER_WEEK", "40")) + +PR_RE = re.compile(r'github\.com/([\w.-]+/[\w.-]+)/pull/(\d+)') +COUNT_RE = re.compile( + r'(?:functions?\s+documented|documented|added\s+docstrings?\s+to)\D{0,20}?(\d{1,3})', + re.I) +FILE_RE = re.compile(r'(?:^|\s)((?:[\w.-]+/)*[\w.-]+\.py)\b') +DOCSTRING_OPEN = re.compile(r'^\s*[rRbBuU]{0,2}("""|\'\'\')') + + +class GhError(RuntimeError): + """A `gh` invocation failed. Must never be mistaken for an empty result.""" + + +def gh(args, default=None, strict=False): + """Run `gh` and parse JSON. + + `strict=True` raises on failure instead of returning `default`. That matters + wherever the result feeds a MONEY decision: the earnings lookup behind the + weekly cap returned `{}` on any CLI/auth/rate-limit failure, which + `docstring_rtc_this_week()` then reported as 0.0 RTC already earned. A + contributor already over the 40 RTC/week ceiling was therefore treated as + having earned nothing, and the cap failed OPEN. A failed lookup is not an + authoritative zero. + """ + try: + p = subprocess.run(["gh"] + args, capture_output=True, text=True, timeout=120) + except Exception as e: + if strict: + raise GhError(f"gh {' '.join(args[:3])} failed: {e}") from e + return default + if p.returncode != 0: + if strict: + raise GhError(f"gh {' '.join(args[:3])} exited {p.returncode}: " + f"{(p.stderr or '').strip()[:200]}") + return default + try: + return json.loads(p.stdout) if p.stdout.strip() else default + except json.JSONDecodeError as e: + if strict: + raise GhError(f"gh {' '.join(args[:3])} returned unparseable JSON: {e}") from e + return default + + +def gh_raw(args): + result = subprocess.run(["gh"] + args, capture_output=True, text=True, timeout=120) + if result.returncode != 0: + raise GhError(f"gh {' '.join(args[:3])} failed (exit {result.returncode}): {result.stderr.strip()}") + return result.stdout + + + +def add_labels(*names): + """Apply labels via REST. + + `gh issue edit --add-label` goes through GraphQL and currently fails with a + Projects-classic deprecation error -- and it fails SILENTLY, so the gate + would post "verified" while never marking the claim eligible, and the payout + runner would never see it. Verified by observing an adjudicated claim come + back with `labels: []`. + """ + ok = True + for n in names: + r = subprocess.run(["gh", "api", "-X", "POST", + f"/repos/{REPO}/issues/{NUM}/labels", "-f", f"labels[]={n}"], + capture_output=True, text=True, timeout=60) + if r.returncode != 0: + print(f"::warning::could not apply label {n}: {r.stderr.strip()[:120]}") + ok = False + return ok + + + +def docstring_rtc_this_week(author): + """RTC this author has already been granted for docstrings in 7 days. + + Summed from this gate's own `rtc-payout-amount` markers rather than from + the chain, so the check works from Actions with no node access and no + admin key. Only claims the gate itself verified are counted. + """ + since = (datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(days=7)).strftime("%Y-%m-%d") + q = (f"repo:{REPO} is:issue author:{author} label:docstring-verified " + f"created:>{since}") + res = gh(["api", "-X", "GET", "search/issues", "-f", f"q={q}", "-f", "per_page=100"], {}, strict=True) + total = 0.0 + for it in (res.get("items") or []): + if str(it.get("number")) == str(NUM): + continue # never count the claim being adjudicated + body = it.get("body") or "" + # The marker lives in a gate comment, not the issue body, so fetch them. + cs = gh(["api", f"/repos/{REPO}/issues/{it['number']}/comments?per_page=100"], [], strict=True) or [] + for c in cs: + m = re.search(r'', c.get("body") or "") + if m: + total += float(m.group(1)) + break + return round(total, 2) + + +def is_docstring_claim(title, body): + t = (title or "").lower() + if "docstring" in t or re.search(r'\bdocs?\s+batch\b', t): + return True + return "docstring" in (body or "").lower()[:400] + + +def count_added_docstrings(diff: str): + """Return (docstring_lines, total_added, files_touched). + + Counts only ADDED lines that open a docstring. Continuation lines of a + multi-line docstring are not counted, so one docstring is one unit however + many lines it spans. + """ + doc = total = 0 + files = [] + in_docstring = False + for line in diff.splitlines(): + if line.startswith("+++ b/"): + files.append(line[6:].strip()) + in_docstring = False + continue + if not line.startswith("+") or line.startswith("+++"): + continue + total += 1 + content = line[1:] + if in_docstring: + if '"""' in content or "'''" in content: + in_docstring = False + continue + if DOCSTRING_OPEN.match(content): + doc += 1 + stripped = content.strip() + # One-liner if the closing quotes appear again on the same line. + quote = '"""' if '"""' in stripped else "'''" + if stripped.count(quote) < 2: + in_docstring = True + return doc, total, files + + +def main(): + if not NUM: + print("ISSUE_NUMBER not set", file=sys.stderr) + return 1 + iss = gh(["issue", "view", NUM, "-R", REPO, + "--json", "title,body,labels,author,state"], {}) + if not iss: + print(f"could not read {REPO}#{NUM}", file=sys.stderr) + return 1 + labels = {l["name"] for l in iss.get("labels", [])} + if {"bounty-eligible", "docstring-verified", "gate-processed", "weekly-cap-reached"} & labels: + print("already adjudicated; skipping") + return 0 + title, body = iss.get("title", ""), iss.get("body") or "" + if not is_docstring_claim(title, body): + print("not a docstring claim; leaving for another gate") + return 0 + + m = PR_RE.search(body) or PR_RE.search(title) + if not m: + gh(["issue", "comment", NUM, "-R", REPO, "--body", + "🤖 Docstring gate: no pull request URL found in this claim. Add the full " + "`https://github.com///pull/` link and it will be re-checked."], None) + add_labels("needs-human") + return 0 + pr_repo, pr_num = m.group(1), m.group(2) + + pr = gh(["pr", "view", pr_num, "-R", pr_repo, + "--json", "state,additions,deletions,files,author,mergedAt"], {}) + if not pr: + gh(["issue", "comment", NUM, "-R", REPO, "--body", + f"🤖 Docstring gate: could not read {pr_repo}#{pr_num}. Flagged for a human."], None) + add_labels("needs-human") + return 0 + + if pr.get("state") != "MERGED": + gh(["issue", "comment", NUM, "-R", REPO, "--body", + f"🤖 Docstring gate: {pr_repo}#{pr_num} is **{pr.get('state','OPEN').lower()}**, not merged.\n\n" + f"Docstring bounties pay on merge, because until then the documentation is not in the " + f"codebase. This claim is not closed — it will be re-checked automatically once the PR " + f"lands, and you do not need to re-file it."], None) + add_labels("awaiting-merge") + print(f"{pr_repo}#{pr_num} not merged ({pr.get('state')}); waiting") + return 0 + + diff = gh_raw(["pr", "diff", pr_num, "-R", pr_repo]) + doc_count, total_added, files = count_added_docstrings(diff) + claimed = None + cm = COUNT_RE.search(body) or COUNT_RE.search(title) + if cm: + claimed = int(cm.group(1)) + + amount = round(doc_count * RATE, 2) + if doc_count == 0: + gh(["issue", "comment", NUM, "-R", REPO, "--body", + f"🤖 Docstring gate: {pr_repo}#{pr_num} is merged, but no added lines in it open a " + f"docstring ({total_added} lines added in total). If the work is real and the gate has " + f"misread it, say so here and a human will look."], None) + add_labels("needs-human") + return 0 + + author = (iss.get("author") or {}).get("login", "") + try: + already = docstring_rtc_this_week(author) if author else 0.0 + except GhError as e: + # Cannot establish prior earnings => cannot honour the cap => do not pay. + # Failing closed is the whole point; the previous behaviour approved the + # claim as though the contributor had earned nothing this week. + gh(["issue", "comment", NUM, "-R", REPO, "--body", + f"🤖 Docstring gate: verified **{doc_count} docstrings** in {pr_repo}#{pr_num}, but the " + f"weekly-earnings lookup failed, so the {MAX_RTC_PER_WEEK:g} RTC/week cap cannot be " + f"checked right now.\n\nHolding rather than approving — a failed lookup is not proof " + f"that you have earned nothing. This retries automatically on the next sweep; you do " + f"not need to do anything."], None) + add_labels("needs-human") + print(f"::error::earnings lookup failed, refusing to approve: {e}") + return 0 + if already + amount > MAX_RTC_PER_WEEK: + add_labels("weekly-cap-reached") + gh(["issue", "comment", NUM, "-R", REPO, "--body", + f"🤖 Docstring gate: verified **{doc_count} docstrings** in {pr_repo}#{pr_num} " + f"(**{amount} RTC**), but this would take you to " + f"**{round(already + amount, 2)} RTC** of docstring earnings in a rolling 7 days, " + f"over the **{MAX_RTC_PER_WEEK:g} RTC/week** ceiling for this bounty type.\n\n" + f"**The work is accepted and this claim is not closed.** It becomes payable again as " + f"soon as the rolling window clears, and it will be picked up automatically. You do " + f"not need to re-file it or do anything.\n\n" + f"Why the ceiling exists: documentation bounties are unbounded by nature, since there " + f"is always another file. The cap keeps one bounty type from consuming the pool, and " + f"40 RTC/week is roughly the top of what any contributor earns across all bounty types. " + f"It is not a judgement on the quality of your work, which has been consistently fine.\n\n" + f"If you want higher-value work, the bounty board has open items at 7 to 35 RTC each " + f"that are not rate-limited."], None) + print(f"weekly cap: {author} at {already} + {amount} > {MAX_RTC_PER_WEEK}") + return 0 + + if amount > MAX_RTC: + gh(["issue", "comment", NUM, "-R", REPO, "--body", + f"🤖 Docstring gate: verified **{doc_count} docstrings** in {pr_repo}#{pr_num}, which at " + f"{RATE} RTC each comes to {amount} RTC. That is above the {MAX_RTC} RTC auto-pay ceiling, " + f"so it needs a human to release it. Nothing is wrong with the claim."], None) + add_labels("needs-human") + return 0 + + note = "" + if claimed is not None and claimed != doc_count: + note = (f"\n\nYou claimed **{claimed}**; the diff contains **{doc_count}**. " + f"Paying the verified number. If you think the gate has miscounted, say so and a " + f"human will check — miscounts are usually arithmetic, not bad faith.") + + add_labels("bounty-eligible", "docstring-verified") + gh(["issue", "comment", NUM, "-R", REPO, "--body", + f"✅ 🤖 **Docstring gate: verified.**\n\n" + f"- PR {pr_repo}#{pr_num} is **merged**\n" + f"- Files: `{', '.join(files[:4]) or 'n/a'}`\n" + f"- Added lines opening a docstring: **{doc_count}** (of {total_added} added lines)\n" + f"- Rate {RATE} RTC each → **{amount} RTC**{note}\n\n" + f"\n" + f"Queued for payout. The balance moves after the standard confirmation window, not on this " + f"comment."], None) + print(f"verified {doc_count} docstrings -> {amount} RTC on {REPO}#{NUM}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_docstring_gate.py b/tests/test_docstring_gate.py new file mode 100644 index 000000000..ec71a222d --- /dev/null +++ b/tests/test_docstring_gate.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Tests for scripts/docstring_gate.py. + +The check that carries the weight is `count_added_docstrings`: without it, a +claim saying "I added 40 docstrings" would pay out for 40 added lines of +anything at all. These pin that it counts docstrings and nothing else. +""" +import importlib.util +import os +import unittest +from pathlib import Path + +os.environ.setdefault("GITHUB_TOKEN", "dummy") +SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "docstring_gate.py" +spec = importlib.util.spec_from_file_location("docstring_gate_under_test", SCRIPT) +dg = importlib.util.module_from_spec(spec) +spec.loader.exec_module(dg) + + +def diff(*added_lines, path="a/x.py"): + head = f"diff --git {path} b/{path.split('a/')[-1]}\n--- {path}\n+++ b/{path.split('a/')[-1]}\n@@ -1 +1 @@\n" + return head + "\n".join(added_lines) + + +class CountingTests(unittest.TestCase): + def test_counts_one_line_docstrings(self): + d = diff('+ """Return the name."""', '+ """Do a thing."""') + self.assertEqual(dg.count_added_docstrings(d)[0], 2) + + def test_multiline_docstring_counts_once(self): + d = diff('+ """Summary line.', '+', '+ More detail here.', '+ """') + doc, total, _ = dg.count_added_docstrings(d) + self.assertEqual(doc, 1, "a multi-line docstring is one unit, not four") + self.assertEqual(total, 4) + + def test_plain_code_is_not_a_docstring(self): + d = diff('+x = 1', '+def f():', '+ return 2') + self.assertEqual(dg.count_added_docstrings(d)[0], 0) + + def test_comments_are_not_docstrings(self): + d = diff('+# this is a comment', '+ # another one') + self.assertEqual(dg.count_added_docstrings(d)[0], 0) + + def test_string_assignment_is_not_a_docstring(self): + """The abuse case: padding a diff with triple-quoted values.""" + d = diff('+SQL = """SELECT 1"""', '+ """A real docstring."""') + # SQL = """...""" does not OPEN at line start, so only the real one counts. + self.assertEqual(dg.count_added_docstrings(d)[0], 1) + + def test_single_quote_docstrings(self): + d = diff("+ '''Alt quoting style.'''") + self.assertEqual(dg.count_added_docstrings(d)[0], 1) + + def test_removed_lines_do_not_count(self): + d = diff('+ """Kept."""', '- """Deleted."""') + self.assertEqual(dg.count_added_docstrings(d)[0], 1) + + def test_files_are_collected(self): + d = diff('+ """Doc."""', path="a/generation/provider.py") + self.assertIn("generation/provider.py", dg.count_added_docstrings(d)[2]) + + def test_raw_and_unicode_prefixes(self): + d = diff('+ r"""Raw docstring."""', '+ u"""Unicode docstring."""') + self.assertEqual(dg.count_added_docstrings(d)[0], 2) + + +class ClaimParsingTests(unittest.TestCase): + def test_recognises_docstring_claims(self): + self.assertTrue(dg.is_docstring_claim("Claim: docs batch 49 - provider.py docstrings", "")) + self.assertTrue(dg.is_docstring_claim("Bounty claim: BoTTube docstring PR #1683", "")) + + def test_ignores_review_claims(self): + self.assertFalse(dg.is_docstring_claim("[Bounty Claim] PR Review - RustChain PR #5395", "")) + + def test_pr_url_extraction(self): + m = dg.PR_RE.search("PR: https://github.com/Scottcjn/bottube/pull/1696") + self.assertEqual((m.group(1), m.group(2)), ("Scottcjn/bottube", "1696")) + + def test_count_extraction(self): + for text, want in [ + ("Functions documented: 7 (get_name, ...)", "7"), + ("Added docstrings to 5 undocumented methods", "5"), + ]: + m = dg.COUNT_RE.search(text) + self.assertIsNotNone(m, text) + self.assertEqual(m.group(1), want) + + +if __name__ == "__main__": + unittest.main() + + +class WeeklyCeilingTests(unittest.TestCase): + """The per-claim ceiling bounds nothing: batches are ~5 RTC each, so the + 50th one passes just as easily as the 1st. The weekly ceiling is the one + that actually bounds an unbounded bounty type.""" + + def setUp(self): + self._gh = dg.gh + + def tearDown(self): + dg.gh = self._gh + + def _with_prior(self, amounts): + """Fake N prior verified claims carrying these payout markers.""" + items = [{"number": 900 + i, "body": ""} for i in range(len(amounts))] + + def fake(args, default=None, strict=False): + joined = " ".join(args) + if "search/issues" in joined: + return {"items": items} + if "/comments" in joined: + idx = int(joined.split("/issues/")[1].split("/")[0]) - 900 + return [{"body": f""}] + return default + dg.gh = fake + + def test_sums_prior_week(self): + self._with_prior([5.0, 7.5, 6.0]) + self.assertEqual(dg.docstring_rtc_this_week("someone"), 18.5) + + def test_no_prior_claims_is_zero(self): + self._with_prior([]) + self.assertEqual(dg.docstring_rtc_this_week("someone"), 0.0) + + def test_ceiling_is_higher_than_typical_top_earner(self): + """Measured 2026-08-10: top contributors earn 20-50 RTC/week across ALL + bounty types. A docstring-only ceiling below that would be punitive.""" + self.assertGreaterEqual(dg.MAX_RTC_PER_WEEK, 40) + + def test_per_claim_ceiling_alone_would_not_bound_volume(self): + """Documents why the weekly cap exists: a typical batch is far under + the per-claim ceiling, so volume is unbounded without it.""" + typical_batch_rtc = 10 * dg.RATE # 10 functions + self.assertLess(typical_batch_rtc, dg.MAX_RTC) + + +class WeeklyCapIdempotencyTests(unittest.TestCase): + def setUp(self): + self._gh = dg.gh + self._add_labels = dg.add_labels + self._gh_raw = dg.gh_raw + self._docstring_rtc_this_week = dg.docstring_rtc_this_week + + def tearDown(self): + dg.gh = self._gh + dg.add_labels = self._add_labels + dg.gh_raw = self._gh_raw + dg.docstring_rtc_this_week = self._docstring_rtc_this_week + + def test_weekly_cap_reached_label_skips_duplicate_adjudication(self): + comments = [] + labels = [] + + def fake_gh(args, default=None, strict=False): + if args[:2] == ["issue", "view"]: + return { + "title": "Docstring bounty claim", + "body": "PR: https://github.com/Scottcjn/Rustchain/pull/8305", + "labels": [{"name": "weekly-cap-reached"}], + "author": {"login": "someone"}, + "state": "OPEN", + } + if args[:3] == ["issue", "comment", dg.NUM]: + comments.append(args[-1]) + return default + raise AssertionError(f"unexpected gh call: {args}") + + dg.gh = fake_gh + dg.add_labels = lambda *names: labels.extend(names) or True + dg.gh_raw = lambda args: (_ for _ in ()).throw(AssertionError(f"unexpected gh_raw call: {args}")) + dg.docstring_rtc_this_week = lambda author: (_ for _ in ()).throw(AssertionError("should not query weekly earnings")) + old_num = dg.NUM + try: + dg.NUM = "16711" + self.assertEqual(dg.main(), 0) + finally: + dg.NUM = old_num + + self.assertEqual(comments, []) + self.assertEqual(labels, [])