Rebuild Registry #21
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rebuild Registry | |
| # Weekly rebuild of src/findata/data/registry.sqlite from upstream sources. | |
| # If the new content hash differs from the current commit, the rebuilt file | |
| # is validated against the registry tests, then opened as a PR and squash- | |
| # merged automatically — fully hands-off, but every change still lands as a | |
| # PR in the history and only merges on a green test run. | |
| # | |
| # Manual trigger via "Run workflow" button is also supported for ad-hoc | |
| # regeneration (e.g. after a data-quality fix in scripts/build_registry.py). | |
| on: | |
| schedule: | |
| # Mondays at 06:00 UTC (~ 03:00 BRT). Most BR gov sources publish | |
| # on weekday mornings, so by Monday morning we have a fresh week. | |
| - cron: "0 6 * * 1" | |
| workflow_dispatch: {} | |
| env: | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" | |
| # Only one rebuild at a time — never overlap with a previous run. | |
| concurrency: | |
| group: rebuild-registry | |
| cancel-in-progress: false | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| rebuild: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| - name: Install | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e '.[dev]' | |
| - name: Capture current registry sha | |
| id: before | |
| run: | | |
| if [ -f src/findata/data/registry.sqlite ]; then | |
| python - <<'PY' >> "$GITHUB_OUTPUT" | |
| import asyncio, sys | |
| sys.path.insert(0, "src") | |
| from findata.registry import get_meta | |
| meta = asyncio.run(get_meta()) | |
| print(f"sha={meta.get('content_sha256','')}") | |
| print(f"built_at={meta.get('built_at','')}") | |
| PY | |
| else | |
| echo "sha=" >> "$GITHUB_OUTPUT" | |
| echo "built_at=" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Rebuild registry | |
| run: | | |
| python scripts/build_registry.py | |
| - name: Capture new registry sha | |
| id: after | |
| run: | | |
| python - <<'PY' >> "$GITHUB_OUTPUT" | |
| import asyncio, sys | |
| sys.path.insert(0, "src") | |
| from findata.registry import get_meta | |
| meta = asyncio.run(get_meta()) | |
| print(f"sha={meta.get('content_sha256','')}") | |
| print(f"sources={meta.get('sources_json','')}") | |
| PY | |
| # Gate the merge on the rebuilt data actually working. test_registry_router | |
| # runs against the *real* embedded registry.sqlite we just regenerated, so | |
| # a broken/empty rebuild fails here and never reaches main. | |
| - name: Validate rebuilt registry | |
| if: success() && steps.before.outputs.sha != steps.after.outputs.sha | |
| run: | | |
| python -m pytest tests/test_registry_router.py -q | |
| - name: Open PR | |
| id: cpr | |
| if: success() && steps.before.outputs.sha != steps.after.outputs.sha | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: registry-update/${{ github.run_id }} | |
| base: main | |
| delete-branch: true | |
| title: "chore(registry): weekly rebuild" | |
| body: | | |
| Automated weekly rebuild of `src/findata/data/registry.sqlite`, | |
| validated against the registry tests and squash-merged automatically. | |
| **Before** | |
| - sha256: `${{ steps.before.outputs.sha }}` | |
| - built_at: ${{ steps.before.outputs.built_at }} | |
| **After** | |
| - sha256: `${{ steps.after.outputs.sha }}` | |
| - sources: `${{ steps.after.outputs.sources }}` | |
| Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| labels: registry-update,automated | |
| add-paths: src/findata/data/registry.sqlite | |
| # PRs opened by the built-in token don't trigger CI (anti-recursion), so | |
| # there's nothing for native --auto to wait on. The data is already | |
| # validated above, so squash-merge it directly; retry to ride out the | |
| # brief window where GitHub is still computing mergeability post-create. | |
| - name: Auto-merge PR | |
| if: success() && steps.cpr.outputs.pull-request-number != '' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| pr="${{ steps.cpr.outputs.pull-request-number }}" | |
| for attempt in 1 2 3 4 5 6; do | |
| if gh pr merge "$pr" --squash --delete-branch; then | |
| echo "Merged PR #$pr" | |
| exit 0 | |
| fi | |
| echo "PR #$pr not mergeable yet (attempt $attempt); waiting…" | |
| sleep 5 | |
| done | |
| echo "::error::Failed to auto-merge PR #$pr after retries" | |
| exit 1 | |
| - name: Report no-op | |
| if: success() && steps.before.outputs.sha == steps.after.outputs.sha | |
| run: | | |
| echo "Registry content hash unchanged (${{ steps.after.outputs.sha }})." | |
| echo "Nothing to merge." |