Data Ingestion #69
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Data Ingestion | |
| on: | |
| # Legacy Hansard search schedule: | |
| # - 12:00 UTC is 08:00 EDT, the primary morning run. | |
| # - 03:00 UTC is 23:00 EDT, the overnight run for late Friday publications. | |
| # Legacy lobbying schedule: | |
| # - Monday 12:00 UTC is 08:00 EDT after quarterly OCL drops. | |
| # - 1st of every third month at 12:00 UTC is the quarterly calendar anchor. | |
| schedule: | |
| - cron: '0 12 * * *' | |
| - cron: '0 3 * * *' | |
| - cron: '0 12 * * 1' | |
| - cron: '0 12 1 */3 *' | |
| workflow_dispatch: | |
| # Each toggle runs one ingestor; leave a toggle off to skip it. Multiple | |
| # toggles can run together in a single dispatch. | |
| # | |
| # Bills and Members each have two pipelines that produce different outputs: | |
| # - "artifacts" (run_bills, run_members) publish legacy flat-JSON files to S3. | |
| # - "SQLite index" (run_bills_indexer, run_members_indexer) publish a SQLite | |
| # .db to S3. The live Bills and Members Lambda APIs query the SQLite index | |
| # after the EPAC-2260 re-architecture, so to refresh what the public API | |
| # serves you must run the indexer toggle (the "artifacts" toggle does not). | |
| # The two pipelines are independent: neither depends on the other to run first. | |
| inputs: | |
| environment: | |
| description: Environment to ingest into | |
| required: true | |
| type: choice | |
| options: | |
| - staging | |
| - production | |
| run_search: | |
| description: 'Hansard: SQLite FTS5 search index (.db) -> S3, read by the live Hansard search API' | |
| required: true | |
| type: boolean | |
| default: false | |
| run_lobbying: | |
| description: 'Lobbying: SQLite index (.db) -> S3, read by the live Lobbying API' | |
| required: true | |
| type: boolean | |
| default: false | |
| run_bills: | |
| description: 'Bills: legacy flat-JSON artifacts -> S3 (does not refresh the live Bills API; use run_bills_indexer for that)' | |
| required: true | |
| type: boolean | |
| default: false | |
| run_bills_indexer: | |
| description: 'Bills: SQLite index (.db) -> S3, read by the live Bills API' | |
| required: true | |
| type: boolean | |
| default: false | |
| run_members: | |
| description: 'Members: legacy flat-JSON artifacts -> S3 (does not refresh the live Members API; use run_members_indexer for that)' | |
| required: true | |
| type: boolean | |
| default: false | |
| run_members_indexer: | |
| description: 'Members: SQLite index (.db) -> S3, read by the live Members API' | |
| required: true | |
| type: boolean | |
| default: false | |
| run_votes: | |
| description: 'Member votes: per-member recorded-vote JSON artifacts -> S3' | |
| required: true | |
| type: boolean | |
| default: false | |
| run_senators_indexer: | |
| description: 'Senators: fetch and generate senators JSON -> S3' | |
| required: true | |
| type: boolean | |
| default: false | |
| parliament_number: | |
| description: Optional parliament number override for search, lobbying, and bills | |
| required: false | |
| type: string | |
| session_number: | |
| description: Optional session number override for search, lobbying, and bills | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| id-token: write | |
| jobs: | |
| plan: | |
| name: Plan selected ingestors | |
| runs-on: ubuntu-latest | |
| outputs: | |
| environment: ${{ steps.plan.outputs.environment }} | |
| matrix: ${{ steps.plan.outputs.matrix }} | |
| should_run: ${{ steps.plan.outputs.should_run }} | |
| steps: | |
| - name: Build ingestion matrix | |
| id: plan | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| EVENT_SCHEDULE: ${{ github.event.schedule }} | |
| INPUT_ENVIRONMENT: ${{ inputs.environment }} | |
| RUN_SEARCH: ${{ inputs.run_search }} | |
| RUN_LOBBYING: ${{ inputs.run_lobbying }} | |
| RUN_BILLS: ${{ inputs.run_bills }} | |
| RUN_BILLS_INDEXER: ${{ inputs.run_bills_indexer }} | |
| RUN_MEMBERS: ${{ inputs.run_members }} | |
| RUN_MEMBERS_INDEXER: ${{ inputs.run_members_indexer }} | |
| RUN_VOTES: ${{ inputs.run_votes }} | |
| RUN_SENATORS_INDEXER: ${{ inputs.run_senators_indexer }} | |
| run: | | |
| set -euo pipefail | |
| environment="${INPUT_ENVIRONMENT:-production}" | |
| items='[]' | |
| add_ingestor() { | |
| local ingestor="$1" | |
| local label="$2" | |
| local module="$3" | |
| local requires_database="$4" | |
| items=$(jq -c \ | |
| --arg ingestor "$ingestor" \ | |
| --arg label "$label" \ | |
| --arg module "$module" \ | |
| --arg requires_database "$requires_database" \ | |
| '. + [{ | |
| ingestor: $ingestor, | |
| label: $label, | |
| module: $module, | |
| requires_database: $requires_database | |
| }]' <<< "$items") | |
| } | |
| if [ "$EVENT_NAME" = "schedule" ]; then | |
| case "$EVENT_SCHEDULE" in | |
| "0 12 * * *"|"0 3 * * *") | |
| add_ingestor "search" "Hansard search index" "hansard-search-index" "false" | |
| add_ingestor "senators-indexer" "Senators indexer" "senators-indexer" "false" | |
| ;; | |
| "0 12 * * 1"|"0 12 1 */3 *") | |
| add_ingestor "lobbying" "Lobbying index" "lobbying-index" "false" | |
| ;; | |
| *) | |
| echo "No ingestors mapped for schedule: ${EVENT_SCHEDULE}" | |
| ;; | |
| esac | |
| else | |
| [ "$RUN_SEARCH" = "true" ] && add_ingestor "search" "Hansard search index" "hansard-search-index" "false" | |
| [ "$RUN_LOBBYING" = "true" ] && add_ingestor "lobbying" "Lobbying index" "lobbying-index" "false" | |
| [ "$RUN_BILLS" = "true" ] && add_ingestor "bills" "Bills artifacts" "bills-publisher" "false" | |
| [ "$RUN_BILLS_INDEXER" = "true" ] && add_ingestor "bills-indexer" "Bills SQLite index" "bills-indexer" "false" | |
| [ "$RUN_MEMBERS" = "true" ] && add_ingestor "members" "Members artifacts" "members-publisher" "true" | |
| [ "$RUN_MEMBERS_INDEXER" = "true" ] && add_ingestor "members-indexer" "Members SQLite index" "members-indexer" "false" | |
| [ "$RUN_VOTES" = "true" ] && add_ingestor "votes" "Member votes artifacts" "member-votes-publisher" "true" | |
| [ "$RUN_SENATORS_INDEXER" = "true" ] && add_ingestor "senators-indexer" "Senators indexer" "senators-indexer" "false" | |
| fi | |
| if [ "$(jq 'length' <<< "$items")" -gt 0 ]; then | |
| should_run=true | |
| else | |
| should_run=false | |
| fi | |
| { | |
| echo "environment=$environment" | |
| echo "should_run=$should_run" | |
| echo "matrix=$(jq -c '{include: .}' <<< "$items")" | |
| } >> "$GITHUB_OUTPUT" | |
| ingest: | |
| name: Run ${{ matrix.label }} | |
| runs-on: ubuntu-latest | |
| needs: plan | |
| if: ${{ needs.plan.outputs.should_run == 'true' }} | |
| timeout-minutes: 360 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.plan.outputs.matrix) }} | |
| env: | |
| ENVIRONMENT: ${{ needs.plan.outputs.environment }} | |
| AWS_REGION: us-east-1 | |
| EPAC_ARTIFACT_BUCKET: ${{ needs.plan.outputs.environment == 'production' && (vars.EPAC_ARTIFACT_BUCKET_PRODUCTION || vars.EPAC_ARTIFACT_BUCKET || vars.ARTIFACTS_BUCKET || 'epac-artifacts-227530433709') || (vars.EPAC_ARTIFACT_BUCKET_STAGING || vars.EPAC_ARTIFACT_BUCKET || vars.ARTIFACTS_BUCKET || 'epac-artifacts-227530433709') }} | |
| EPAC_ARTIFACT_PREFIX_DEFAULT: ${{ vars.EPAC_ARTIFACT_PREFIX }} | |
| EPAC_ARTIFACT_PREFIX_PRODUCTION: ${{ vars.EPAC_ARTIFACT_PREFIX_PRODUCTION }} | |
| EPAC_ARTIFACT_PREFIX_STAGING: ${{ vars.EPAC_ARTIFACT_PREFIX_STAGING }} | |
| EPAC_HANSARD_SEARCH_PREFIX: ${{ vars.EPAC_HANSARD_SEARCH_PREFIX || 'hansard-search/v1' }} | |
| LOBBYING_INDEX_PREFIX: ${{ vars.EPAC_LOBBYING_INDEX_PREFIX || 'lobbying-index/v1' }} | |
| BILLS_INDEX_PREFIX: ${{ vars.EPAC_BILLS_INDEX_PREFIX || 'bills/v1' }} | |
| MEMBERS_INDEX_PREFIX: ${{ vars.EPAC_MEMBERS_INDEX_PREFIX || 'members/v1' }} | |
| PARLIAMENT_NUMBER: ${{ inputs.parliament_number || vars.EPAC_HANSARD_PARLIAMENT_NUMBER || '45' }} | |
| SESSION_NUMBER: ${{ inputs.session_number || vars.EPAC_HANSARD_SESSION_NUMBER || '1' }} | |
| PARLIAMENT_NUM: ${{ inputs.parliament_number || vars.EPAC_HANSARD_PARLIAMENT_NUMBER || '45' }} | |
| SESSION_NUM: ${{ inputs.session_number || vars.EPAC_HANSARD_SESSION_NUMBER || '1' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: backend/go.work | |
| cache-dependency-path: backend/go.work.sum | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ needs.plan.outputs.environment == 'production' && secrets.AWS_BACKEND_PRODUCTION_ROLE_ARN || secrets.AWS_BACKEND_STAGING_ROLE_ARN }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Resolve artifact prefix | |
| run: | | |
| set -euo pipefail | |
| prefix="$EPAC_ARTIFACT_PREFIX_DEFAULT" | |
| if [ "$ENVIRONMENT" = "production" ] && [ -n "$EPAC_ARTIFACT_PREFIX_PRODUCTION" ]; then | |
| prefix="$EPAC_ARTIFACT_PREFIX_PRODUCTION" | |
| elif [ "$ENVIRONMENT" = "staging" ] && [ -n "$EPAC_ARTIFACT_PREFIX_STAGING" ]; then | |
| prefix="$EPAC_ARTIFACT_PREFIX_STAGING" | |
| fi | |
| echo "EPAC_ARTIFACT_PREFIX=${prefix#/}" >> "$GITHUB_ENV" | |
| - name: Resolve database URL | |
| if: ${{ matrix.requires_database == 'true' }} | |
| env: | |
| DATABASE_URL_PARAMETER_NAME: ${{ vars.EPAC_DATABASE_URL_PARAMETER_NAME || '/epac/database-url' }} | |
| run: | | |
| set -euo pipefail | |
| database_url=$(aws ssm get-parameter \ | |
| --name "$DATABASE_URL_PARAMETER_NAME" \ | |
| --with-decryption \ | |
| --query Parameter.Value \ | |
| --output text) | |
| if printf '%s' "$database_url" | jq -e 'type == "object" and has("DATABASE_URL")' >/dev/null 2>&1; then | |
| database_url=$(printf '%s' "$database_url" | jq -r '.DATABASE_URL') | |
| fi | |
| if [ -z "$database_url" ] || [ "$database_url" = "None" ]; then | |
| echo "Parameter ${DATABASE_URL_PARAMETER_NAME} did not contain a DATABASE_URL value" | |
| exit 1 | |
| fi | |
| echo "::add-mask::$database_url" | |
| echo "DATABASE_URL=$database_url" >> "$GITHUB_ENV" | |
| - name: Run ingestor | |
| env: | |
| INGESTOR: ${{ matrix.ingestor }} | |
| run: | | |
| set -euo pipefail | |
| if [ -z "$EPAC_ARTIFACT_BUCKET" ]; then | |
| echo "EPAC_ARTIFACT_BUCKET is required" | |
| exit 1 | |
| fi | |
| artifact_uri() { | |
| local key="${1#/}" | |
| local prefix="${EPAC_ARTIFACT_PREFIX:-}" | |
| prefix="${prefix#/}" | |
| prefix="${prefix%/}" | |
| if [ -n "$prefix" ]; then | |
| echo "s3://$EPAC_ARTIFACT_BUCKET/$prefix/$key" | |
| else | |
| echo "s3://$EPAC_ARTIFACT_BUCKET/$key" | |
| fi | |
| } | |
| case "$INGESTOR" in | |
| search) | |
| (cd backend/hansard-search-index && go run .) | |
| ;; | |
| lobbying) | |
| (cd backend/lobbying-index && go run .) | |
| ;; | |
| bills) | |
| output_dir="$RUNNER_TEMP/artifacts/bills" | |
| (cd backend/bills-publisher && go run . --output "$output_dir") | |
| aws s3 sync "$output_dir" "$(artifact_uri bills)" | |
| ;; | |
| bills-indexer) | |
| (cd backend/bills-indexer && DB_PATH="$RUNNER_TEMP/bills.db" go run .) | |
| ;; | |
| members) | |
| output_dir="$RUNNER_TEMP/artifacts/members" | |
| (cd backend/members-publisher && go run . --output "$output_dir") | |
| aws s3 sync "$output_dir" "$(artifact_uri members)" | |
| ;; | |
| members-indexer) | |
| (cd backend/members-indexer && DB_PATH="$RUNNER_TEMP/members.db" go run .) | |
| ;; | |
| votes) | |
| (cd backend/member-votes-publisher && go run .) | |
| ;; | |
| senators-indexer) | |
| output_dir="$RUNNER_TEMP/artifacts/senators" | |
| python3 scripts/artifacts/fetch_senators.py --output "$output_dir/v1/all.json" | |
| aws s3 sync "$output_dir" "$(artifact_uri senators)" | |
| ;; | |
| *) | |
| echo "Unknown ingestor: $INGESTOR" | |
| exit 1 | |
| ;; | |
| esac | |
| - name: Summarize ingestor | |
| if: ${{ always() }} | |
| env: | |
| INGESTOR: ${{ matrix.ingestor }} | |
| LABEL: ${{ matrix.label }} | |
| run: | | |
| { | |
| echo "## ${LABEL}" | |
| echo "" | |
| echo "- Environment: \`${ENVIRONMENT}\`" | |
| echo "- Ingestor: \`${INGESTOR}\`" | |
| echo "- Artifact bucket: \`${EPAC_ARTIFACT_BUCKET}\`" | |
| echo "- Status: \`${{ job.status }}\`" | |
| } >> "$GITHUB_STEP_SUMMARY" |