fix(ci): classify agent authorship with python3 instead of Ruby (#155)

* fix(ci): classify agent authorship with python3 instead of Ruby

The reusable agent-authorship-label workflow fails fleet-wide with
`ruby: command not found` (exit 127) on every pull_request_target event:
callers pin it by SHA at a revision whose classify step runs
`ruby org-defaults/.github/scripts/classify-agent-authorship.rb`, and the
resolved PUBLIC_PR_VALIDATION_RUNNER image no longer ships Ruby.

Provisioning Ruby at runtime (ruby/setup-ruby) is a live tool install,
which the repo rail reserves against and which #153 already removed from
the other org reusable workflows. Follow that established pattern instead:
port the classifier to python3 (standard library only), which is proven to
exist on every runner image we operate, and assert the toolchain up front
with a preflight so a missing interpreter fails loudly instead of after a
silent skip.

The Python port matches the Ruby original on 65 differential cases
(plus identical file-arg and --github-output behavior) covering complete,
incomplete, mixed, case-varied, whitespace-varied, unicode, nested
commit.message, and empty inputs.

Consumers pin this repo by SHA and must bump their `uses:` ref to pick up
this fix; the SHA pinned in evalops/mono#5205 cannot be retrofixed.

Refs evalops/mono#5205

* fix: split commit records and messages on newline only

Python str.splitlines() also splits on \v \f \x1c-\x1e \x85 U+2028 U+2029,
which jq emits literally inside JSON strings — a commit message containing
one tore a JSONL record into unparseable fragments and crashed the classify
step. Ruby's each_line splits on "\n" only; match it exactly.

Author: haasonsaas

.github/scripts/classify-agent-authorship.rb

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Classify PR commit authorship from a JSONL stream of commits.
+
+Port of classify-agent-authorship.rb. Ruby is not present on every runner
+image this reusable workflow runs on, so the classifier uses python3, which
+is proven to exist on all of them. Standard library only.
+"""
+
+import argparse
+import json
+import re
+import sys
+
+# Ruby \s / \S semantics are ASCII-only; keep re.ASCII so unicode whitespace
+# does not change trailer matching relative to the Ruby original.
+FLAGS = re.IGNORECASE | re.ASCII
+
+REQUIRED_PATTERNS = {
+    "co_author": re.compile(r"^Co-Authored-By:\s*Maestro\s+<maestro@evalops\.dev>\s*$", FLAGS),
+    "version": re.compile(r"^Maestro-Version:\s*\S.*$", FLAGS),
+    "prompt_id": re.compile(r"^Maestro-Prompt-Id:\s*\S.*$", FLAGS),
+    "approvals_id": re.compile(r"^Maestro-Approvals-Id:\s*\S.*$", FLAGS),
+}
+
+MARKER_PATTERN = re.compile(
+    r"^Co-Authored-By:\s*Maestro\s+<maestro@evalops\.dev>\s*$"
+    r"|^Maestro-(?:Version|Prompt-Id|Approvals-Id):",
+    FLAGS,
+)
+
+
+def read_input(paths):
+    if paths:
+        chunks = []
+        for path in paths:
+            with open(path, encoding="utf-8") as handle:
+                chunks.append(handle.read())
+        return "".join(chunks)
+    return sys.stdin.read()
+
+
+def extract_messages(text):
+    messages = []
+    # Split only on "\n" to match Ruby's each_line: jq emits characters like
+    # U+2028/U+2029 literally inside JSON strings, and str.splitlines() would
+    # tear one JSON record into unparseable fragments.
+    for line in text.split("\n"):
+        if not line.strip():
+            continue
+        parsed = json.loads(line)
+        if isinstance(parsed, dict):
+            commit = parsed.get("commit")
+            message = commit.get("message") if isinstance(commit, dict) else None
+            if message is None:
+                message = parsed.get("message")
+            if message is not None:
+                messages.append(message)
+    return messages
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--github-output",
+        metavar="PATH",
+        help="Append key=value outputs for GitHub Actions",
+    )
+    parser.add_argument("files", nargs="*")
+    args = parser.parse_args()
+
+    messages = extract_messages(read_input(args.files))
+
+    agent_commits = 0
+    untrailered_commits = 0
+    incomplete_commits = 0
+
+    for message in messages:
+        # Ruby each_line splits on "\n" only; keep that exact segmentation.
+        parts = message.split("\n")
+        lines = [part + "\n" for part in parts[:-1]]
+        if parts[-1]:
+            lines.append(parts[-1])
+        has_marker = any(MARKER_PATTERN.search(line) for line in lines)
+
+        if not has_marker:
+            untrailered_commits += 1
+            continue
+
+        agent_commits += 1
+        missing_required = any(
+            not any(pattern.search(line) for line in lines)
+            for pattern in REQUIRED_PATTERNS.values()
+        )
+        if missing_required:
+            incomplete_commits += 1
+
+    if agent_commits > 0 and untrailered_commits > 0:
+        label = "mixed-authorship"
+    elif agent_commits > 0:
+        label = "agent-authored"
+    else:
+        label = "agent-assisted"
+
+    outputs = {
+        "label": label,
+        "total_commits": len(messages),
+        "agent_commits": agent_commits,
+        "untrailered_commits": untrailered_commits,
+        "human_commits": untrailered_commits,
+        "incomplete_agent_commits": incomplete_commits,
+    }
+
+    for key, value in outputs.items():
+        print(f"{key}={value}")
+
+    if args.github_output:
+        with open(args.github_output, "a", encoding="utf-8") as handle:
+            for key, value in outputs.items():
+                handle.write(f"{key}={value}\n")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/classify_agent_authorship.py

@@ -1,5 +1,12 @@
 name: agent-authorship-label
 
+# Reusable org authorship labeler, called by repositories across the
+# organisation. It runs on whatever runner the caller names, so it may only
+# depend on tools proven to exist on every runner image we operate. Ruby is
+# absent from newer images (fleet-wide `ruby: command not found`, see
+# evalops/platform#5205), so the classifier runs on python3, asserted up
+# front by the preflight.
+
 on:
   workflow_call:
     inputs:
@@ -36,10 +43,16 @@ jobs:
           ref: ${{ inputs.helper_ref }}
           path: org-defaults
 
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
-        with:
-          ruby-version: "3.3.9"
+      - name: Preflight the classification toolchain
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if ! command -v python3 >/dev/null 2>&1; then
+            echo "::error::python3 is not on this runner (${RUNNER_NAME:-unknown}). agent-authorship-label cannot classify commits without it."
+            exit 1
+          fi
+          python3 -V
 
       - name: Resolve pull request
         id: pr
@@ -70,7 +83,7 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          ruby org-defaults/.github/scripts/classify-agent-authorship.rb \
+          python3 org-defaults/.github/scripts/classify_agent_authorship.py \
             --github-output "${GITHUB_OUTPUT}" \
             commits.jsonl
 

.github/workflows/agent-authorship-label.yml

@@ -0,0 +1,93 @@
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+SCRIPT = ROOT / ".github" / "scripts" / "classify_agent_authorship.py"
+
+
+def parse_outputs(text):
+    return dict(line.split("=", 1) for line in text.splitlines())
+
+
+def classify(commits, github_output=None):
+    input_data = "\n".join(json.dumps(commit) for commit in commits)
+    args = [sys.executable, str(SCRIPT)]
+    if github_output:
+        args += ["--github-output", github_output]
+    result = subprocess.run(
+        args, input=input_data, capture_output=True, text=True
+    )
+    assert result.returncode == 0, result.stderr
+    return parse_outputs(result.stdout)
+
+
+class ClassifyAgentAuthorshipTest(unittest.TestCase):
+    def test_untrailered_commits_are_agent_assisted(self):
+        outputs = classify([{"sha": "abc", "message": "fix: regular change"}])
+
+        self.assertEqual("agent-assisted", outputs["label"])
+        self.assertEqual("1", outputs["total_commits"])
+        self.assertEqual("0", outputs["agent_commits"])
+        self.assertEqual("1", outputs["untrailered_commits"])
+        self.assertEqual("0", outputs["incomplete_agent_commits"])
+
+    def test_complete_maestro_trailers_are_agent_authored(self):
+        outputs = classify(
+            [
+                {
+                    "sha": "abc",
+                    "message": (
+                        "feat: ship change\n"
+                        "\n"
+                        "Co-Authored-By: Maestro <maestro@evalops.dev>\n"
+                        "Maestro-Version: 2026.04.28 / gpt-5\n"
+                        "Maestro-Prompt-Id: prompt-123\n"
+                        "Maestro-Approvals-Id: approval-456\n"
+                    ),
+                }
+            ]
+        )
+
+        self.assertEqual("agent-authored", outputs["label"])
+        self.assertEqual("1", outputs["agent_commits"])
+        self.assertEqual("0", outputs["untrailered_commits"])
+        self.assertEqual("0", outputs["incomplete_agent_commits"])
+
+    def test_mixed_authorship_and_incomplete_trailers_are_reported(self):
+        outputs = classify(
+            [
+                {
+                    "sha": "abc",
+                    "message": (
+                        "feat: partial agent change\n"
+                        "\n"
+                        "Co-Authored-By: Maestro <maestro@evalops.dev>\n"
+                        "Maestro-Version: 2026.04.28 / gpt-5\n"
+                    ),
+                },
+                {"sha": "def", "message": "docs: human follow-up"},
+            ]
+        )
+
+        self.assertEqual("mixed-authorship", outputs["label"])
+        self.assertEqual("1", outputs["agent_commits"])
+        self.assertEqual("1", outputs["untrailered_commits"])
+        self.assertEqual("1", outputs["incomplete_agent_commits"])
+
+    def test_github_output_file_gets_same_outputs(self):
+        with tempfile.NamedTemporaryFile(mode="r", suffix="github-output") as handle:
+            outputs = classify(
+                [{"sha": "abc", "message": "fix: regular change"}],
+                github_output=handle.name,
+            )
+            file_outputs = parse_outputs(handle.read())
+
+        self.assertEqual(outputs, file_outputs)
+
+
+if __name__ == "__main__":
+    unittest.main()