PoliticalEventTrackingResearch/.github/workflows/source_event_pipeline.yml at main · QuantStrategyLab/PoliticalEventTrackingResearch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
name: Source Event Pipeline

on:
  workflow_dispatch:
    inputs:
      raw_items_path:
        description: "Raw post/news CSV path."
        required: false
        default: "data/live/source_items.csv"
        type: string
      aliases_path:
        description: "Symbol alias CSV path."
        required: false
        default: "config/core_us_equity_aliases.csv"
        type: string
      watchlist_path:
        description: "Watchlist CSV path."
        required: false
        default: "data/live/political_watchlist.csv"
        type: string
      commit_outputs:
        description: "Commit generated live CSV outputs back to data/live."
        required: false
        default: "false"
        type: choice
        options:
          - "false"
          - "true"
  schedule:
    - cron: "45 12 * * 6"

permissions:
  contents: write
  pull-requests: write

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}
  cancel-in-progress: false

jobs:
  build-source-events:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"
      - name: Install package
        run: python -m pip install -e .
      - name: Extract source mentions
        env:
          RAW_ITEMS_PATH: ${{ github.event.inputs.raw_items_path || 'data/live/source_items.csv' }}
          ALIASES_PATH: ${{ github.event.inputs.aliases_path || 'config/core_us_equity_aliases.csv' }}
          WATCHLIST_PATH: ${{ github.event.inputs.watchlist_path || 'data/live/political_watchlist.csv' }}
        run: |
          set -euo pipefail
          mkdir -p data/output/source_event_pipeline
          python scripts/extract_source_mentions.py \
            --raw-items "${RAW_ITEMS_PATH}" \
            --aliases "${ALIASES_PATH}" \
            --output data/output/source_event_pipeline/source_events.csv
          python scripts/build_tracker.py \
            --watchlist "${WATCHLIST_PATH}" \
            --events data/output/source_event_pipeline/source_events.csv \
            --output data/output/source_event_pipeline/source_tracker.csv
      - name: Stage live CSV outputs for protected publication
        env:
          COMMIT_OUTPUTS: ${{ github.event_name == 'schedule' && 'true' || github.event.inputs.commit_outputs || 'false' }}
        run: |
          set -euo pipefail
          if [ "${COMMIT_OUTPUTS}" != "true" ]; then
            echo "Live output commit disabled."
            exit 0
          fi
          mkdir -p data/live
          cp data/output/source_event_pipeline/source_events.csv data/live/source_events.csv
          cp data/output/source_event_pipeline/source_events.csv data/live/political_events.csv
          cp data/output/source_event_pipeline/source_tracker.csv data/live/source_tracker.csv
          MANIFEST_PATHS=(
            data/live/source_items.csv
            data/live/source_events.csv
            data/live/political_events.csv
            data/live/source_tracker.csv
          )
          if [ -f data/live/source_fetch_status.json ]; then
            MANIFEST_PATHS+=(data/live/source_fetch_status.json)
          fi
          python scripts/write_live_manifest.py \
            --base-dir . \
            --output data/live/source_manifest.json \
            "${MANIFEST_PATHS[@]}"
          git add data/live/source_events.csv data/live/political_events.csv data/live/source_tracker.csv data/live/source_manifest.json
      - name: Create or update protected live-output PR
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          python scripts/publish_live_outputs_pr.py \
            --branch automation/live-source-events \
            --title "Update generated live source events" \
            --body "Automated research-data update. Generated files are reviewed through the normal protected-branch CI path; this workflow never pushes to main directly." \
            --commit-message "Update generated live source events"
      - name: Upload source event artifact
        uses: actions/upload-artifact@v7
        with:
          name: source-event-pipeline
          path: data/output/source_event_pipeline/
          if-no-files-found: error