[EPAC-2062]: build Hansard search index Lambda

.github/workflows/backend-pr.yml

@@ -47,6 +47,7 @@ jobs:
           - calendar
           - config
           - hansard-subjects-index
+          - hansard-search-index
           - manifest
           - openapi
           - members-publisher

.github/workflows/backend-production.yml

@@ -209,10 +209,14 @@ jobs:
       - name: Sync production artifact environment
         if: ${{ (inputs.service == 'all' || inputs.service == matrix.service) && contains(fromJson(needs.prepare.outputs.artifact_services), matrix.service) }}
         env:
+          SERVICE: ${{ matrix.service }}
           CANONICAL_FUNCTION_NAME: epac-${{ matrix.service }}-production
           LEGACY_FUNCTION_NAME: ${{ matrix.service }}
           ARTIFACT_BUCKET: ${{ vars.EPAC_ARTIFACT_BUCKET_PRODUCTION || vars.EPAC_ARTIFACT_BUCKET || vars.ARTIFACTS_BUCKET }}
           ARTIFACT_PREFIX: ${{ vars.EPAC_ARTIFACT_PREFIX_PRODUCTION || vars.EPAC_ARTIFACT_PREFIX }}
+          HANSARD_SEARCH_PREFIX: ${{ vars.EPAC_HANSARD_SEARCH_PREFIX || 'hansard-search/v1' }}
+          PARLIAMENT_NUMBER: ${{ vars.EPAC_HANSARD_PARLIAMENT_NUMBER || '45' }}
+          SESSION_NUMBER: ${{ vars.EPAC_HANSARD_SESSION_NUMBER || '1' }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN_PRODUCTION }}
         run: |
           set -euo pipefail
@@ -232,10 +236,14 @@ jobs:
             --output json)
           UPDATED_ENV=$(jq -cn \
             --argjson current "${CURRENT_ENV:-null}" \
+            --arg service "$SERVICE" \
             --arg artifact_bucket "$ARTIFACT_BUCKET" \
             --arg artifact_prefix "$ARTIFACT_PREFIX" \
+            --arg hansard_search_prefix "$HANSARD_SEARCH_PREFIX" \
+            --arg parliament_number "$PARLIAMENT_NUMBER" \
+            --arg session_number "$SESSION_NUMBER" \
             --arg sentry_dsn "$SENTRY_DSN" \
-            '{Variables: ((($current // {}) | del(.DATABASE_URL)) + {EPAC_ARTIFACT_BUCKET: $artifact_bucket} + (if $artifact_prefix != "" then {EPAC_ARTIFACT_PREFIX: $artifact_prefix} else {} end) + (if $sentry_dsn != "" then {SENTRY_DSN: $sentry_dsn} else {} end))}')
+            '{Variables: ((($current // {}) | del(.DATABASE_URL)) + {EPAC_ARTIFACT_BUCKET: $artifact_bucket} + (if $artifact_prefix != "" then {EPAC_ARTIFACT_PREFIX: $artifact_prefix} else {} end) + (if $service == "hansard-search-index" then {EPAC_HANSARD_SEARCH_PREFIX: $hansard_search_prefix, PARLIAMENT_NUMBER: $parliament_number, SESSION_NUMBER: $session_number} else {} end) + (if $sentry_dsn != "" then {SENTRY_DSN: $sentry_dsn} else {} end))}')
 
           aws lambda update-function-configuration \
             --function-name "$FUNCTION_NAME" \

.github/workflows/backend-staging.yml

@@ -209,9 +209,13 @@ jobs:
       - name: Sync staging artifact environment
         if: ${{ contains(fromJson(needs.prepare.outputs.artifact_services), matrix.service) }}
         env:
+          SERVICE: ${{ matrix.service }}
           FUNCTION_NAME: epac-${{ matrix.service }}-staging
           ARTIFACT_BUCKET: ${{ vars.EPAC_ARTIFACT_BUCKET_STAGING || vars.EPAC_ARTIFACT_BUCKET || vars.ARTIFACTS_BUCKET }}
           ARTIFACT_PREFIX: ${{ vars.EPAC_ARTIFACT_PREFIX_STAGING || vars.EPAC_ARTIFACT_PREFIX }}
+          HANSARD_SEARCH_PREFIX: ${{ vars.EPAC_HANSARD_SEARCH_PREFIX || 'hansard-search/v1' }}
+          PARLIAMENT_NUMBER: ${{ vars.EPAC_HANSARD_PARLIAMENT_NUMBER || '45' }}
+          SESSION_NUMBER: ${{ vars.EPAC_HANSARD_SESSION_NUMBER || '1' }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN_STAGING }}
         run: |
           set -euo pipefail
@@ -227,10 +231,14 @@ jobs:
             --output json)
           UPDATED_ENV=$(jq -cn \
             --argjson current "${CURRENT_ENV:-null}" \
+            --arg service "$SERVICE" \
             --arg artifact_bucket "$ARTIFACT_BUCKET" \
             --arg artifact_prefix "$ARTIFACT_PREFIX" \
+            --arg hansard_search_prefix "$HANSARD_SEARCH_PREFIX" \
+            --arg parliament_number "$PARLIAMENT_NUMBER" \
+            --arg session_number "$SESSION_NUMBER" \
             --arg sentry_dsn "$SENTRY_DSN" \
-            '{Variables: ((($current // {}) | del(.DATABASE_URL)) + {EPAC_ARTIFACT_BUCKET: $artifact_bucket} + (if $artifact_prefix != "" then {EPAC_ARTIFACT_PREFIX: $artifact_prefix} else {} end) + (if $sentry_dsn != "" then {SENTRY_DSN: $sentry_dsn} else {} end))}')
+            '{Variables: ((($current // {}) | del(.DATABASE_URL)) + {EPAC_ARTIFACT_BUCKET: $artifact_bucket} + (if $artifact_prefix != "" then {EPAC_ARTIFACT_PREFIX: $artifact_prefix} else {} end) + (if $service == "hansard-search-index" then {EPAC_HANSARD_SEARCH_PREFIX: $hansard_search_prefix, PARLIAMENT_NUMBER: $parliament_number, SESSION_NUMBER: $session_number} else {} end) + (if $sentry_dsn != "" then {SENTRY_DSN: $sentry_dsn} else {} end))}')
 
           aws lambda update-function-configuration \
             --function-name "$FUNCTION_NAME" \
@@ -265,4 +273,7 @@ jobs:
         run: ./scripts/ci/ensure-staging-api-routes.sh
 
       - name: Run staging backend smoke tests
+        env:
+          EPAC_ARTIFACT_BUCKET: ${{ vars.EPAC_ARTIFACT_BUCKET_STAGING || vars.EPAC_ARTIFACT_BUCKET || vars.ARTIFACTS_BUCKET }}
+          EPAC_HANSARD_SEARCH_PREFIX: ${{ vars.EPAC_HANSARD_SEARCH_PREFIX || 'hansard-search/v1' }}
         run: python3 scripts/ci/backend_staging_smoke.py --base-url "$STAGING_API_BASE_URL"

.github/workflows/backend-tests.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        lambda: [_shared, daily-fetch, loader, members, sittings, bills, member-content, member-speeches, member-speeches-publisher, member-votes, member-votes-publisher, on-this-day, estimates, riding-boundary, calendar, config, toronto-council, openapi, hansard-subjects-index, members-publisher, sittings-publisher, bills-publisher]
+        lambda: [_shared, daily-fetch, loader, members, sittings, bills, member-content, member-speeches, member-speeches-publisher, member-votes, member-votes-publisher, on-this-day, estimates, riding-boundary, calendar, config, toronto-council, openapi, hansard-subjects-index, hansard-search-index, members-publisher, sittings-publisher, bills-publisher]
     name: Test ${{ matrix.lambda }}
     steps:
       - uses: actions/checkout@v4

.github/workflows/hansard-search-reindex.yml

@@ -0,0 +1,126 @@
+name: Hansard Search Reindex
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to reindex
+        required: true
+        type: choice
+        options:
+          - staging
+          - production
+      parliament_number:
+        description: Optional parliament number override
+        required: false
+        type: string
+      session_number:
+        description: Optional session number override
+        required: false
+        type: string
+
+concurrency:
+  group: hansard-search-reindex-${{ inputs.environment }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  reindex:
+    runs-on: ubuntu-latest
+    env:
+      ENVIRONMENT: ${{ inputs.environment }}
+      FUNCTION_NAME: epac-hansard-search-index-${{ inputs.environment }}
+      PARLIAMENT_NUMBER: ${{ inputs.parliament_number }}
+      SESSION_NUMBER: ${{ inputs.session_number }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ inputs.environment == 'production' && secrets.AWS_BACKEND_PRODUCTION_ROLE_ARN || secrets.AWS_BACKEND_STAGING_ROLE_ARN }}
+          aws-region: us-east-1
+
+      - name: Apply optional session override
+        if: ${{ inputs.parliament_number != '' || inputs.session_number != '' }}
+        run: |
+          set -euo pipefail
+
+          CURRENT_ENV=$(aws lambda get-function-configuration \
+            --function-name "$FUNCTION_NAME" \
+            --query 'Environment.Variables' \
+            --output json)
+          printf '%s' "${CURRENT_ENV:-null}" > /tmp/hansard-search-index.original-env.json
+          UPDATED_ENV=$(jq -cn \
+            --argjson current "${CURRENT_ENV:-null}" \
+            --arg parliament_number "$PARLIAMENT_NUMBER" \
+            --arg session_number "$SESSION_NUMBER" \
+            '{Variables: (($current // {}) + (if $parliament_number != "" then {PARLIAMENT_NUMBER: $parliament_number} else {} end) + (if $session_number != "" then {SESSION_NUMBER: $session_number} else {} end))}')
+
+          aws lambda update-function-configuration \
+            --function-name "$FUNCTION_NAME" \
+            --environment "$UPDATED_ENV" \
+            >/dev/null
+          aws lambda wait function-updated \
+            --function-name "$FUNCTION_NAME"
+
+      - name: Invoke indexer
+        run: |
+          set -euo pipefail
+
+          aws lambda invoke \
+            --function-name "$FUNCTION_NAME" \
+            --payload '{}' \
+            --cli-binary-format raw-in-base64-out \
+            /tmp/hansard-search-index.out.json \
+            > /tmp/hansard-search-index.invoke.json
+
+          {
+            echo "## Hansard search reindex"
+            echo ""
+            echo "- Environment: \`$ENVIRONMENT\`"
+            echo "- Function: \`$FUNCTION_NAME\`"
+            if [ -n "$PARLIAMENT_NUMBER" ]; then
+              echo "- Parliament override: \`$PARLIAMENT_NUMBER\`"
+            fi
+            if [ -n "$SESSION_NUMBER" ]; then
+              echo "- Session override: \`$SESSION_NUMBER\`"
+            fi
+            echo ""
+            echo "### Invoke response"
+            echo '```json'
+            jq . /tmp/hansard-search-index.invoke.json
+            echo '```'
+            echo ""
+            echo "### Function payload"
+            echo '```json'
+            jq . /tmp/hansard-search-index.out.json 2>/dev/null || cat /tmp/hansard-search-index.out.json
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if jq -e '.FunctionError? // empty' /tmp/hansard-search-index.invoke.json >/dev/null; then
+            exit 1
+          fi
+
+      - name: Restore session override
+        if: ${{ always() && (inputs.parliament_number != '' || inputs.session_number != '') }}
+        run: |
+          set -euo pipefail
+
+          if [ ! -f /tmp/hansard-search-index.original-env.json ]; then
+            exit 0
+          fi
+          ORIGINAL_ENV=$(cat /tmp/hansard-search-index.original-env.json)
+          RESTORED_ENV=$(jq -cn \
+            --argjson current "${ORIGINAL_ENV:-null}" \
+            '{Variables: ($current // {})}')
+
+          aws lambda update-function-configuration \
+            --function-name "$FUNCTION_NAME" \
+            --environment "$RESTORED_ENV" \
+            >/dev/null
+          aws lambda wait function-updated \
+            --function-name "$FUNCTION_NAME"

backend/go.work

@@ -10,6 +10,7 @@ use (
 	./daily-fetch
 	./estimates
 	./hansard-backfill
+	./hansard-search-index
 	./hansard-subjects-index
 	./health
 	./loader

backend/hansard-search-index/.gitignore

@@ -0,0 +1,4 @@
+bootstrap
+function.zip
+!testdata/
+!testdata/*.xml

backend/hansard-search-index/README.md

@@ -0,0 +1,141 @@
+# hansard-search-index
+
+`hansard-search-index` is a manually-invoked Go Lambda that builds the v1
+SQLite FTS5 search index for the current House of Commons Hansard session. It
+downloads English Hansard XML from ourcommons.ca, parses interventions and
+paragraphs, writes `/tmp/index.sqlite`, uploads it to S3, then writes a manifest
+after the SQLite object exists.
+
+## Invocation
+
+Staging:
+
+```bash
+aws lambda invoke \
+  --function-name epac-hansard-search-index-staging \
+  --payload '{}' \
+  --cli-binary-format raw-in-base64-out \
+  /tmp/hansard-search-index.out.json
+```
+
+Production:
+
+```bash
+aws lambda invoke \
+  --function-name epac-hansard-search-index-production \
+  --payload '{}' \
+  --cli-binary-format raw-in-base64-out \
+  /tmp/hansard-search-index.out.json
+```
+
+The manual dispatch workflow is:
+
+```text
+https://github.com/RiddimSoftware/epac/actions/workflows/hansard-search-reindex.yml
+```
+
+The workflow accepts `environment`, plus optional `parliament_number` and
+`session_number` overrides. The Lambda reads `PARLIAMENT_NUMBER` and
+`SESSION_NUMBER` from its environment; when workflow overrides are supplied, the
+workflow temporarily updates those environment variables before invoking the
+function, then restores the prior configuration.
+
+## Configuration
+
+Required:
+
+- `EPAC_ARTIFACT_BUCKET` - destination artifact bucket.
+
+Optional:
+
+- `EPAC_HANSARD_SEARCH_PREFIX` - destination prefix, default
+  `hansard-search/v1`.
+- `PARLIAMENT_NUMBER` - parliament to index, default `45`.
+- `SESSION_NUMBER` - session to index, default `1`.
+
+The default session values match the current 45th Parliament, 1st Session at the
+time this Lambda was added. Operators should pass workflow overrides or update
+the Lambda environment when a future session starts.
+
+## Source URL
+
+The downloader uses the existing `backend/download_hansard.py` convention, which
+was verified against ourcommons.ca:
+
+```text
+https://www.ourcommons.ca/Content/House/{parliament}{session}/Debates/{sitting:03d}/HAN{sitting:03d}-E.XML
+```
+
+For example:
+
+```text
+https://www.ourcommons.ca/Content/House/451/Debates/001/HAN001-E.XML
+```
+
+Raw XML is cached for one invocation under `/tmp/raw/45-1-HAN001-E.XML` style
+filenames.
+
+## S3 Layout
+
+With the default prefix, the Lambda writes:
+
+```text
+s3://${EPAC_ARTIFACT_BUCKET}/hansard-search/v1/index.sqlite
+s3://${EPAC_ARTIFACT_BUCKET}/hansard-search/v1/manifest.json
+```
+
+The manifest shape is:
+
+```json
+{
+  "version": "v1",
+  "built_at": "2026-05-25T17:30:00Z",
+  "parliament_number": 45,
+  "session_number": 1,
+  "sitting_count": 47,
+  "intervention_count": 12345,
+  "message_count": 56789,
+  "sqlite_key": "hansard-search/v1/index.sqlite",
+  "sqlite_size_bytes": 31415926,
+  "sqlite_sha256": "<hex>"
+}
+```
+
+## SQLite Schema
+
+The builder recreates the database on every run:
+
+```sql
+CREATE TABLE meta (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+);
+CREATE TABLE interventions (
+    rowid INTEGER PRIMARY KEY,
+    parliament_number INTEGER NOT NULL,
+    session_number INTEGER NOT NULL,
+    sitting_date TEXT NOT NULL,
+    intervention_id TEXT NOT NULL UNIQUE,
+    speaker_name TEXT NOT NULL,
+    party_abbreviation TEXT NOT NULL DEFAULT '',
+    riding_name TEXT NOT NULL DEFAULT '',
+    topic TEXT NOT NULL DEFAULT ''
+);
+CREATE INDEX idx_interventions_sitting_date ON interventions(sitting_date);
+CREATE TABLE messages (
+    rowid INTEGER PRIMARY KEY,
+    intervention_rowid INTEGER NOT NULL REFERENCES interventions(rowid),
+    message_id TEXT NOT NULL UNIQUE,
+    position INTEGER NOT NULL,
+    content TEXT NOT NULL
+);
+CREATE VIRTUAL TABLE messages_fts USING fts5(
+    content,
+    content='messages',
+    content_rowid='rowid',
+    tokenize='porter unicode61 remove_diacritics 1'
+);
+```
+
+Triggers keep `messages_fts` synchronized with `messages`. Dates are stored as
+`YYYY-MM-DD`.