fix: publish generated research data through protected PRs

.github/workflows/rss_source_pipeline.yml

@@ -36,6 +36,7 @@ on:
 
 permissions:
   contents: write
+  pull-requests: write
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref_name }}
@@ -75,7 +76,7 @@ jobs:
             --watchlist "${WATCHLIST_PATH}" \
             --events data/output/rss_source_pipeline/source_events.csv \
             --output data/output/rss_source_pipeline/source_tracker.csv
-      - name: Publish live CSV outputs to repository
+      - name: Stage live CSV outputs for protected publication
         env:
           COMMIT_OUTPUTS: ${{ github.event_name == 'schedule' && 'true' || github.event.inputs.commit_outputs || 'false' }}
         run: |
@@ -98,15 +99,16 @@ jobs:
             data/live/source_events.csv \
             data/live/political_events.csv \
             data/live/source_tracker.csv
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git add data/live/source_items.csv data/live/source_events.csv data/live/political_events.csv data/live/source_tracker.csv data/live/source_fetch_status.json data/live/source_manifest.json
-          if git diff --cached --quiet; then
-            echo "No live RSS output changes to commit."
-          else
-            git commit -m "Update live RSS source events [skip ci]"
-            git push
-          fi
+      - name: Create or update protected live-output PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          python scripts/publish_live_outputs_pr.py \
+            --branch automation/live-rss-source-events \
+            --title "Update generated live RSS source events" \
+            --body "Automated research-data update. Generated files are reviewed through the normal protected-branch CI path; this workflow never pushes to main directly." \
+            --commit-message "Update generated live RSS source events"
       - name: Upload RSS source artifact
         uses: actions/upload-artifact@v7
         with:

.github/workflows/source_event_pipeline.yml

@@ -31,6 +31,7 @@ on:
 
 permissions:
   contents: write
+  pull-requests: write
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref_name }}
@@ -63,7 +64,7 @@ jobs:
             --watchlist "${WATCHLIST_PATH}" \
             --events data/output/source_event_pipeline/source_events.csv \
             --output data/output/source_event_pipeline/source_tracker.csv
-      - name: Publish live CSV outputs to repository
+      - name: Stage live CSV outputs for protected publication
         env:
           COMMIT_OUTPUTS: ${{ github.event_name == 'schedule' && 'true' || github.event.inputs.commit_outputs || 'false' }}
         run: |
@@ -89,15 +90,16 @@ jobs:
             --base-dir . \
             --output data/live/source_manifest.json \
             "${MANIFEST_PATHS[@]}"
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git add data/live/source_events.csv data/live/political_events.csv data/live/source_tracker.csv data/live/source_manifest.json
-          if git diff --cached --quiet; then
-            echo "No live source output changes to commit."
-          else
-            git commit -m "Update live source events [skip ci]"
-            git push
-          fi
+      - name: Create or update protected live-output PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          python scripts/publish_live_outputs_pr.py \
+            --branch automation/live-source-events \
+            --title "Update generated live source events" \
+            --body "Automated research-data update. Generated files are reviewed through the normal protected-branch CI path; this workflow never pushes to main directly." \
+            --commit-message "Update generated live source events"
       - name: Upload source event artifact
         uses: actions/upload-artifact@v7
         with:

scripts/publish_live_outputs_pr.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Publish generated research data through a protected-branch pull request.
+
+Scheduled workflows must not push generated data directly to ``main``.  This
+small, repository-local adapter commits only the paths staged by its caller to
+an automation branch and creates (or reuses) a pull request for the normal CI
+and branch-protection path.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+from collections.abc import Sequence
+
+
+class PublishError(RuntimeError):
+    """Raised when a generated-data PR cannot be created safely."""
+
+
+def run(command: Sequence[str], *, capture: bool = False) -> str:
+    completed = subprocess.run(
+        command,
+        check=False,
+        text=True,
+        stdout=subprocess.PIPE if capture else None,
+        stderr=subprocess.PIPE if capture else None,
+    )
+    if completed.returncode:
+        detail = (completed.stderr or completed.stdout or "command failed").strip()
+        raise PublishError(f"{' '.join(command)}: {detail}")
+    return (completed.stdout or "").strip()
+
+
+def staged_changes_present() -> bool:
+    return subprocess.run(["git", "diff", "--cached", "--quiet"], check=False).returncode != 0
+
+
+def existing_pull_request(branch: str) -> str:
+    return run(
+        [
+            "gh",
+            "pr",
+            "list",
+            "--repo",
+            os.environ["GITHUB_REPOSITORY"],
+            "--state",
+            "open",
+            "--head",
+            branch,
+            "--json",
+            "url",
+            "--jq",
+            ".[0].url // \"\"",
+        ],
+        capture=True,
+    )
+
+
+def publish(branch: str, title: str, body: str, commit_message: str) -> str:
+    if not staged_changes_present():
+        return "No generated data changes to publish."
+
+    run(["git", "config", "user.name", "github-actions[bot]"])
+    run(["git", "config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com"])
+    run(["git", "switch", "-C", branch])
+    run(["git", "commit", "-m", commit_message])
+    run(["git", "push", "--force-with-lease", "origin", f"HEAD:refs/heads/{branch}"])
+
+    existing_url = existing_pull_request(branch)
+    if existing_url:
+        return f"Updated generated-data PR: {existing_url}"
+
+    url = run(
+        [
+            "gh",
+            "pr",
+            "create",
+            "--repo",
+            os.environ["GITHUB_REPOSITORY"],
+            "--head",
+            branch,
+            "--base",
+            "main",
+            "--title",
+            title,
+            "--body",
+            body,
+        ],
+        capture=True,
+    )
+    return f"Created generated-data PR: {url}"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--branch", required=True)
+    parser.add_argument("--title", required=True)
+    parser.add_argument("--body", required=True)
+    parser.add_argument("--commit-message", required=True)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    print(publish(args.branch, args.title, args.body, args.commit_message))
+
+
+if __name__ == "__main__":
+    main()

tests/test_publish_live_outputs_pr.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+def load_module():
+    path = Path(__file__).parents[1] / "scripts" / "publish_live_outputs_pr.py"
+    spec = importlib.util.spec_from_file_location("publish_live_outputs_pr", path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_publish_does_not_create_branch_or_pr_without_staged_changes(monkeypatch) -> None:
+    module = load_module()
+    monkeypatch.setattr(module, "staged_changes_present", lambda: False)
+
+    assert module.publish("automation/generated", "Generated", "body", "commit") == "No generated data changes to publish."
+
+
+def test_publish_uses_an_automation_branch_and_reuses_existing_pr(monkeypatch) -> None:
+    module = load_module()
+    commands: list[list[str]] = []
+    monkeypatch.setattr(module, "staged_changes_present", lambda: True)
+    monkeypatch.setattr(module, "existing_pull_request", lambda branch: "https://example.test/pr/1")
+
+    def fake_run(command, *, capture=False):
+        commands.append(list(command))
+        return ""
+
+    monkeypatch.setattr(module, "run", fake_run)
+
+    result = module.publish("automation/generated", "Generated", "body", "commit")
+
+    assert result == "Updated generated-data PR: https://example.test/pr/1"
+    assert ["git", "switch", "-C", "automation/generated"] in commands
+    assert ["git", "push", "--force-with-lease", "origin", "HEAD:refs/heads/automation/generated"] in commands
+    assert all("main" not in command[-1:] for command in commands if command[:2] == ["git", "push"])
+
+
+def test_publish_creates_pull_request_after_pushing_branch(monkeypatch) -> None:
+    module = load_module()
+    commands: list[list[str]] = []
+    monkeypatch.setenv("GITHUB_REPOSITORY", "QuantStrategyLab/example")
+    monkeypatch.setattr(module, "staged_changes_present", lambda: True)
+    monkeypatch.setattr(module, "existing_pull_request", lambda branch: "")
+
+    def fake_run(command, *, capture=False):
+        commands.append(list(command))
+        return "https://example.test/pr/2" if command[:3] == ["gh", "pr", "create"] else ""
+
+    monkeypatch.setattr(module, "run", fake_run)
+
+    result = module.publish("automation/generated", "Generated", "body", "commit")
+
+    assert result == "Created generated-data PR: https://example.test/pr/2"
+    create = next(command for command in commands if command[:3] == ["gh", "pr", "create"])
+    assert ["--head", "automation/generated"] == create[create.index("--head") : create.index("--head") + 2]
+    assert ["--base", "main"] == create[create.index("--base") : create.index("--base") + 2]