From 9af76e359d4b7aa3962262db1661a570a200e257 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:18:47 +0800 Subject: [PATCH] fix: publish generated research data through protected PRs Co-Authored-By: Codex --- .github/workflows/rss_source_pipeline.yml | 20 ++-- .github/workflows/source_event_pipeline.yml | 20 ++-- scripts/publish_live_outputs_pr.py | 111 ++++++++++++++++++++ tests/test_publish_live_outputs_pr.py | 61 +++++++++++ 4 files changed, 194 insertions(+), 18 deletions(-) create mode 100644 scripts/publish_live_outputs_pr.py create mode 100644 tests/test_publish_live_outputs_pr.py diff --git a/.github/workflows/rss_source_pipeline.yml b/.github/workflows/rss_source_pipeline.yml index 7bd252b..83d541c 100644 --- a/.github/workflows/rss_source_pipeline.yml +++ b/.github/workflows/rss_source_pipeline.yml @@ -36,6 +36,7 @@ on: permissions: contents: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} @@ -75,7 +76,7 @@ jobs: --watchlist "${WATCHLIST_PATH}" \ --events data/output/rss_source_pipeline/source_events.csv \ --output data/output/rss_source_pipeline/source_tracker.csv - - name: Publish live CSV outputs to repository + - name: Stage live CSV outputs for protected publication env: COMMIT_OUTPUTS: ${{ github.event_name == 'schedule' && 'true' || github.event.inputs.commit_outputs || 'false' }} run: | @@ -98,15 +99,16 @@ jobs: data/live/source_events.csv \ data/live/political_events.csv \ data/live/source_tracker.csv - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add data/live/source_items.csv data/live/source_events.csv data/live/political_events.csv data/live/source_tracker.csv data/live/source_fetch_status.json data/live/source_manifest.json - if git diff --cached --quiet; then - echo "No live RSS output changes to commit." - else - git commit -m "Update live RSS source events [skip ci]" - git push - fi + - name: Create or update protected live-output PR + env: + GH_TOKEN: ${{ github.token }} + run: | + python scripts/publish_live_outputs_pr.py \ + --branch automation/live-rss-source-events \ + --title "Update generated live RSS source events" \ + --body "Automated research-data update. Generated files are reviewed through the normal protected-branch CI path; this workflow never pushes to main directly." \ + --commit-message "Update generated live RSS source events" - name: Upload RSS source artifact uses: actions/upload-artifact@v7 with: diff --git a/.github/workflows/source_event_pipeline.yml b/.github/workflows/source_event_pipeline.yml index fe66790..60d9457 100644 --- a/.github/workflows/source_event_pipeline.yml +++ b/.github/workflows/source_event_pipeline.yml @@ -31,6 +31,7 @@ on: permissions: contents: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} @@ -63,7 +64,7 @@ jobs: --watchlist "${WATCHLIST_PATH}" \ --events data/output/source_event_pipeline/source_events.csv \ --output data/output/source_event_pipeline/source_tracker.csv - - name: Publish live CSV outputs to repository + - name: Stage live CSV outputs for protected publication env: COMMIT_OUTPUTS: ${{ github.event_name == 'schedule' && 'true' || github.event.inputs.commit_outputs || 'false' }} run: | @@ -89,15 +90,16 @@ jobs: --base-dir . \ --output data/live/source_manifest.json \ "${MANIFEST_PATHS[@]}" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add data/live/source_events.csv data/live/political_events.csv data/live/source_tracker.csv data/live/source_manifest.json - if git diff --cached --quiet; then - echo "No live source output changes to commit." - else - git commit -m "Update live source events [skip ci]" - git push - fi + - name: Create or update protected live-output PR + env: + GH_TOKEN: ${{ github.token }} + run: | + python scripts/publish_live_outputs_pr.py \ + --branch automation/live-source-events \ + --title "Update generated live source events" \ + --body "Automated research-data update. Generated files are reviewed through the normal protected-branch CI path; this workflow never pushes to main directly." \ + --commit-message "Update generated live source events" - name: Upload source event artifact uses: actions/upload-artifact@v7 with: diff --git a/scripts/publish_live_outputs_pr.py b/scripts/publish_live_outputs_pr.py new file mode 100644 index 0000000..76fc3ec --- /dev/null +++ b/scripts/publish_live_outputs_pr.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Publish generated research data through a protected-branch pull request. + +Scheduled workflows must not push generated data directly to ``main``. This +small, repository-local adapter commits only the paths staged by its caller to +an automation branch and creates (or reuses) a pull request for the normal CI +and branch-protection path. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +from collections.abc import Sequence + + +class PublishError(RuntimeError): + """Raised when a generated-data PR cannot be created safely.""" + + +def run(command: Sequence[str], *, capture: bool = False) -> str: + completed = subprocess.run( + command, + check=False, + text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None, + ) + if completed.returncode: + detail = (completed.stderr or completed.stdout or "command failed").strip() + raise PublishError(f"{' '.join(command)}: {detail}") + return (completed.stdout or "").strip() + + +def staged_changes_present() -> bool: + return subprocess.run(["git", "diff", "--cached", "--quiet"], check=False).returncode != 0 + + +def existing_pull_request(branch: str) -> str: + return run( + [ + "gh", + "pr", + "list", + "--repo", + os.environ["GITHUB_REPOSITORY"], + "--state", + "open", + "--head", + branch, + "--json", + "url", + "--jq", + ".[0].url // \"\"", + ], + capture=True, + ) + + +def publish(branch: str, title: str, body: str, commit_message: str) -> str: + if not staged_changes_present(): + return "No generated data changes to publish." + + run(["git", "config", "user.name", "github-actions[bot]"]) + run(["git", "config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com"]) + run(["git", "switch", "-C", branch]) + run(["git", "commit", "-m", commit_message]) + run(["git", "push", "--force-with-lease", "origin", f"HEAD:refs/heads/{branch}"]) + + existing_url = existing_pull_request(branch) + if existing_url: + return f"Updated generated-data PR: {existing_url}" + + url = run( + [ + "gh", + "pr", + "create", + "--repo", + os.environ["GITHUB_REPOSITORY"], + "--head", + branch, + "--base", + "main", + "--title", + title, + "--body", + body, + ], + capture=True, + ) + return f"Created generated-data PR: {url}" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--branch", required=True) + parser.add_argument("--title", required=True) + parser.add_argument("--body", required=True) + parser.add_argument("--commit-message", required=True) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + print(publish(args.branch, args.title, args.body, args.commit_message)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_publish_live_outputs_pr.py b/tests/test_publish_live_outputs_pr.py new file mode 100644 index 0000000..f055e11 --- /dev/null +++ b/tests/test_publish_live_outputs_pr.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +def load_module(): + path = Path(__file__).parents[1] / "scripts" / "publish_live_outputs_pr.py" + spec = importlib.util.spec_from_file_location("publish_live_outputs_pr", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_publish_does_not_create_branch_or_pr_without_staged_changes(monkeypatch) -> None: + module = load_module() + monkeypatch.setattr(module, "staged_changes_present", lambda: False) + + assert module.publish("automation/generated", "Generated", "body", "commit") == "No generated data changes to publish." + + +def test_publish_uses_an_automation_branch_and_reuses_existing_pr(monkeypatch) -> None: + module = load_module() + commands: list[list[str]] = [] + monkeypatch.setattr(module, "staged_changes_present", lambda: True) + monkeypatch.setattr(module, "existing_pull_request", lambda branch: "https://example.test/pr/1") + + def fake_run(command, *, capture=False): + commands.append(list(command)) + return "" + + monkeypatch.setattr(module, "run", fake_run) + + result = module.publish("automation/generated", "Generated", "body", "commit") + + assert result == "Updated generated-data PR: https://example.test/pr/1" + assert ["git", "switch", "-C", "automation/generated"] in commands + assert ["git", "push", "--force-with-lease", "origin", "HEAD:refs/heads/automation/generated"] in commands + assert all("main" not in command[-1:] for command in commands if command[:2] == ["git", "push"]) + + +def test_publish_creates_pull_request_after_pushing_branch(monkeypatch) -> None: + module = load_module() + commands: list[list[str]] = [] + monkeypatch.setenv("GITHUB_REPOSITORY", "QuantStrategyLab/example") + monkeypatch.setattr(module, "staged_changes_present", lambda: True) + monkeypatch.setattr(module, "existing_pull_request", lambda branch: "") + + def fake_run(command, *, capture=False): + commands.append(list(command)) + return "https://example.test/pr/2" if command[:3] == ["gh", "pr", "create"] else "" + + monkeypatch.setattr(module, "run", fake_run) + + result = module.publish("automation/generated", "Generated", "body", "commit") + + assert result == "Created generated-data PR: https://example.test/pr/2" + create = next(command for command in commands if command[:3] == ["gh", "pr", "create"]) + assert ["--head", "automation/generated"] == create[create.index("--head") : create.index("--head") + 2] + assert ["--base", "main"] == create[create.index("--base") : create.index("--base") + 2]