diff --git a/backend/hansard-search/README.md b/backend/hansard-search/README.md index cd822f83..a38c7dfb 100644 --- a/backend/hansard-search/README.md +++ b/backend/hansard-search/README.md @@ -1,10 +1,12 @@ # hansard-search Lambda -**Slice 1 of 3** — scaffold, S3 manifest reader, and SQLite index downloader. +**Slices 1-2 of 3** — scaffold, S3 manifest reader, SQLite index downloader, +and the FTS5 query layer. -This Lambda serves `GET /api/v1/hansard/search`. D1 returns HTTP 503 with -`{"error":"search index not yet available"}` for every request. D2 adds the -FTS5 query adapter and use case; D3 wires the real HTTP handler and OpenAPI spec. +This Lambda serves `GET /api/v1/hansard/search`. It still returns HTTP 503 +until D3 wires the HTTP handler, but D2 adds the `SearchHansard` use case and +the `sqlitefts5` adapter that execute FTS5 `MATCH` queries against the downloaded +SQLite index. ## Environment variables @@ -43,6 +45,12 @@ hansard-search-index Lambda (EPAC-2062): 4. Opens the file read-only (`file:/tmp/index.sqlite?mode=ro&_pragma=query_only(1)`), reads `meta.version`, and returns `ErrSchemaMismatch` if it is not `v1`. +`SearchHansard` use case (query path, added in D2): +1. Validates a non-empty FTS5 query and 1-indexed pagination (`1 <= per_page <= 100`). +2. Sanitizes control characters from the `MATCH` expression while preserving FTS5 operators. +3. Queries the SQLite FTS5 index with optional speaker/topic substring filters. +4. Maps FTS5 parser failures from `modernc.org/sqlite` to `ErrInvalidQuerySyntax`. + ## Architecture ``` @@ -52,10 +60,11 @@ cmd/main.go internal/ domain/ — Manifest value object - usecase/ — OpenSearchIndex (port interfaces ManifestLoader, IndexDownloader) + usecase/ — OpenSearchIndex and SearchHansard application policy adapter/ s3manifest/ — ManifestLoader backed by S3 sqlitefile/ — IndexDownloader: S3 download + SHA-256 + schema-version check + sqlitefts5/ — HansardSearchRepository backed by SQLite FTS5 ``` Dependency rule: `usecase/` has no imports from `aws-sdk-go-v2` or `modernc.org/sqlite`. diff --git a/backend/hansard-search/internal/adapter/sqlitefts5/repository.go b/backend/hansard-search/internal/adapter/sqlitefts5/repository.go new file mode 100644 index 00000000..b8b88407 --- /dev/null +++ b/backend/hansard-search/internal/adapter/sqlitefts5/repository.go @@ -0,0 +1,136 @@ +package sqlitefts5 + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "unicode" + + "epac/hansard-search/internal/usecase" + + sqlite "modernc.org/sqlite" +) + +const searchSQL = ` +SELECT + i.parliament_number, i.session_number, i.sitting_date, + i.intervention_id, m.message_id, + i.speaker_name, i.party_abbreviation, i.riding_name, i.topic, + snippet(messages_fts, 0, '', '', '…', 32) AS snippet, + bm25(messages_fts) AS score +FROM messages_fts +JOIN messages m ON m.rowid = messages_fts.rowid +JOIN interventions i ON i.rowid = m.intervention_rowid +WHERE messages_fts MATCH ? + AND (? = '' OR i.speaker_name LIKE '%' || ? || '%' COLLATE NOCASE) + AND (? = '' OR i.topic LIKE '%' || ? || '%' COLLATE NOCASE) +-- bm25() typically returns negative numbers; lower scores rank better. +ORDER BY score ASC +LIMIT ? OFFSET ? +` + +const countSQL = ` +SELECT COUNT(*) +FROM messages_fts +JOIN messages m ON m.rowid = messages_fts.rowid +JOIN interventions i ON i.rowid = m.intervention_rowid +WHERE messages_fts MATCH ? + AND (? = '' OR i.speaker_name LIKE '%' || ? || '%' COLLATE NOCASE) + AND (? = '' OR i.topic LIKE '%' || ? || '%' COLLATE NOCASE) +` + +type Repository struct { + db *sql.DB +} + +var _ usecase.HansardSearchRepository = (*Repository)(nil) + +func New(db *sql.DB) *Repository { + return &Repository{db: db} +} + +func (r *Repository) Search(ctx context.Context, q usecase.SearchQuery, p usecase.Pagination) (usecase.SearchResults, error) { + if r.db == nil { + return usecase.SearchResults{}, errors.New("sqlite database is required") + } + + query := sanitizeQuery(q.Query) + speaker := strings.TrimSpace(q.Speaker) + topic := strings.TrimSpace(q.Topic) + + var total int + if err := r.db.QueryRowContext(ctx, countSQL, query, speaker, speaker, topic, topic).Scan(&total); err != nil { + return usecase.SearchResults{}, classifySearchError(err) + } + + offset := (p.Page - 1) * p.PerPage + rows, err := r.db.QueryContext(ctx, searchSQL, query, speaker, speaker, topic, topic, p.PerPage, offset) + if err != nil { + return usecase.SearchResults{}, classifySearchError(err) + } + defer rows.Close() + + hits := make([]usecase.SearchHit, 0) + for rows.Next() { + var hit usecase.SearchHit + if err := rows.Scan( + &hit.ParliamentNumber, + &hit.SessionNumber, + &hit.SittingDate, + &hit.InterventionID, + &hit.MessageID, + &hit.SpeakerName, + &hit.PartyAbbreviation, + &hit.RidingName, + &hit.Topic, + &hit.Snippet, + &hit.Score, + ); err != nil { + return usecase.SearchResults{}, classifySearchError(err) + } + hits = append(hits, hit) + } + if err := rows.Err(); err != nil { + return usecase.SearchResults{}, classifySearchError(err) + } + + return usecase.SearchResults{ + Total: total, + Hits: hits, + }, nil +} + +func sanitizeQuery(query string) string { + query = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return ' ' + } + return r + }, query) + + return strings.TrimSpace(query) +} + +func classifySearchError(err error) error { + if err == nil { + return nil + } + if isInvalidQuerySyntax(err) { + return usecase.ErrInvalidQuerySyntax + } + return fmt.Errorf("search hansard sqlite index: %w", err) +} + +func isInvalidQuerySyntax(err error) bool { + var sqliteErr *sqlite.Error + if !errors.As(err, &sqliteErr) { + return false + } + + message := strings.ToLower(sqliteErr.Error()) + return strings.Contains(message, "fts5: syntax error") || + strings.Contains(message, "unterminated string") || + strings.Contains(message, "malformed match expression") +} diff --git a/backend/hansard-search/internal/adapter/sqlitefts5/repository_test.go b/backend/hansard-search/internal/adapter/sqlitefts5/repository_test.go new file mode 100644 index 00000000..013fb302 --- /dev/null +++ b/backend/hansard-search/internal/adapter/sqlitefts5/repository_test.go @@ -0,0 +1,242 @@ +package sqlitefts5_test + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "testing" + + "epac/hansard-search/internal/adapter/sqlitefts5" + "epac/hansard-search/internal/usecase" + + _ "modernc.org/sqlite" +) + +func TestRepositorySearchSimpleWordMatch(t *testing.T) { + db := openFixtureDB(t) + defer db.Close() + + results, err := sqlitefts5.New(db).Search( + context.Background(), + usecase.SearchQuery{Query: "climate"}, + usecase.Pagination{Page: 1, PerPage: 10}, + ) + if err != nil { + t.Fatalf("Search returned error: %v", err) + } + if results.Total != 2 || len(results.Hits) != 2 { + t.Fatalf("results = %#v, want 2 hits", results) + } + if results.Hits[0].Snippet == "" || results.Hits[1].Snippet == "" { + t.Fatalf("expected snippets for all hits: %#v", results.Hits) + } +} + +func TestRepositorySearchPhraseMatch(t *testing.T) { + db := openFixtureDB(t) + defer db.Close() + + results, err := sqlitefts5.New(db).Search( + context.Background(), + usecase.SearchQuery{Query: "\"climate change\""}, + usecase.Pagination{Page: 1, PerPage: 10}, + ) + if err != nil { + t.Fatalf("Search returned error: %v", err) + } + if results.Total != 2 || len(results.Hits) != 2 { + t.Fatalf("results = %#v, want 2 phrase hits", results) + } +} + +func TestRepositorySearchSupportsBooleanOperators(t *testing.T) { + db := openFixtureDB(t) + defer db.Close() + + tests := []struct { + name string + query string + wantTotal int + wantMessage string + }{ + {name: "and", query: "climate AND parliament", wantTotal: 1, wantMessage: "message-1"}, + {name: "or", query: "energy OR housing", wantTotal: 3}, + {name: "not", query: "climate NOT parliament", wantTotal: 1, wantMessage: "message-4"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + results, err := sqlitefts5.New(db).Search( + context.Background(), + usecase.SearchQuery{Query: tt.query}, + usecase.Pagination{Page: 1, PerPage: 10}, + ) + if err != nil { + t.Fatalf("Search returned error: %v", err) + } + if results.Total != tt.wantTotal || len(results.Hits) != tt.wantTotal { + t.Fatalf("results = %#v, want %d hits", results, tt.wantTotal) + } + if tt.wantMessage != "" && results.Hits[0].MessageID != tt.wantMessage { + t.Fatalf("message id = %q, want %q", results.Hits[0].MessageID, tt.wantMessage) + } + }) + } +} + +func TestRepositorySearchSupportsSpeakerFilter(t *testing.T) { + db := openFixtureDB(t) + defer db.Close() + + results, err := sqlitefts5.New(db).Search( + context.Background(), + usecase.SearchQuery{Query: "climate", Speaker: "jAnE"}, + usecase.Pagination{Page: 1, PerPage: 10}, + ) + if err != nil { + t.Fatalf("Search returned error: %v", err) + } + if results.Total != 2 || len(results.Hits) != 2 { + t.Fatalf("results = %#v, want 2 speaker-filtered hits", results) + } +} + +func TestRepositorySearchSupportsTopicFilter(t *testing.T) { + db := openFixtureDB(t) + defer db.Close() + + results, err := sqlitefts5.New(db).Search( + context.Background(), + usecase.SearchQuery{Query: "climate", Topic: "housing"}, + usecase.Pagination{Page: 1, PerPage: 10}, + ) + if err != nil { + t.Fatalf("Search returned error: %v", err) + } + if results.Total != 1 || len(results.Hits) != 1 { + t.Fatalf("results = %#v, want 1 topic-filtered hit", results) + } + if results.Hits[0].MessageID != "message-4" { + t.Fatalf("message id = %q, want message-4", results.Hits[0].MessageID) + } +} + +func TestRepositorySearchSupportsPagination(t *testing.T) { + db := openFixtureDB(t) + defer db.Close() + + pageOne, err := sqlitefts5.New(db).Search( + context.Background(), + usecase.SearchQuery{Query: "climate"}, + usecase.Pagination{Page: 1, PerPage: 1}, + ) + if err != nil { + t.Fatalf("page 1 Search returned error: %v", err) + } + pageTwo, err := sqlitefts5.New(db).Search( + context.Background(), + usecase.SearchQuery{Query: "climate"}, + usecase.Pagination{Page: 2, PerPage: 1}, + ) + if err != nil { + t.Fatalf("page 2 Search returned error: %v", err) + } + + if pageOne.Total != 2 || pageTwo.Total != 2 { + t.Fatalf("totals = %d and %d, want 2", pageOne.Total, pageTwo.Total) + } + if len(pageOne.Hits) != 1 || len(pageTwo.Hits) != 1 { + t.Fatalf("page sizes = %d and %d, want 1 each", len(pageOne.Hits), len(pageTwo.Hits)) + } + if pageOne.Hits[0].MessageID == pageTwo.Hits[0].MessageID { + t.Fatalf("expected distinct paginated hits, got %q twice", pageOne.Hits[0].MessageID) + } +} + +func TestRepositorySearchReturnsEmptyResults(t *testing.T) { + db := openFixtureDB(t) + defer db.Close() + + results, err := sqlitefts5.New(db).Search( + context.Background(), + usecase.SearchQuery{Query: "fisheries"}, + usecase.Pagination{Page: 1, PerPage: 10}, + ) + if err != nil { + t.Fatalf("Search returned error: %v", err) + } + if results.Total != 0 || len(results.Hits) != 0 { + t.Fatalf("results = %#v, want no hits", results) + } +} + +func TestRepositorySearchReturnsInvalidQuerySyntax(t *testing.T) { + db := openFixtureDB(t) + defer db.Close() + + _, err := sqlitefts5.New(db).Search( + context.Background(), + usecase.SearchQuery{Query: "\"unclosed"}, + usecase.Pagination{Page: 1, PerPage: 10}, + ) + if !errors.Is(err, usecase.ErrInvalidQuerySyntax) { + t.Fatalf("error = %v, want ErrInvalidQuerySyntax", err) + } +} + +func openFixtureDB(t *testing.T) *sql.DB { + t.Helper() + + path := filepath.Join(t.TempDir(), "fixture.sqlite") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + + statements := []string{ + `CREATE TABLE interventions ( + rowid INTEGER PRIMARY KEY, + parliament_number INTEGER NOT NULL, + session_number INTEGER NOT NULL, + sitting_date TEXT NOT NULL, + intervention_id TEXT NOT NULL UNIQUE, + speaker_name TEXT NOT NULL, + party_abbreviation TEXT NOT NULL DEFAULT '', + riding_name TEXT NOT NULL DEFAULT '', + topic TEXT NOT NULL DEFAULT '' + )`, + `CREATE TABLE messages ( + rowid INTEGER PRIMARY KEY, + intervention_rowid INTEGER NOT NULL REFERENCES interventions(rowid), + message_id TEXT NOT NULL UNIQUE, + position INTEGER NOT NULL, + content TEXT NOT NULL + )`, + `CREATE VIRTUAL TABLE messages_fts USING fts5(content)`, + `INSERT INTO interventions (rowid, parliament_number, session_number, sitting_date, intervention_id, speaker_name, party_abbreviation, riding_name, topic) VALUES + (1, 45, 1, '2026-05-20', 'intervention-1', 'Jane Smith', 'LIB', 'Ottawa Centre', 'Climate Action'), + (2, 45, 1, '2026-05-21', 'intervention-2', 'Bob Brown', 'CPC', 'Calgary West', 'Energy Policy'), + (3, 45, 1, '2026-05-22', 'intervention-3', 'Alice Green', 'NDP', 'Toronto Centre', 'Housing'), + (4, 45, 1, '2026-05-23', 'intervention-4', 'Jane Smith', 'LIB', 'Ottawa Centre', 'Climate Housing')`, + `INSERT INTO messages (rowid, intervention_rowid, message_id, position, content) VALUES + (1, 1, 'message-1', 1, 'Climate change demands urgent action from Parliament.'), + (2, 2, 'message-2', 1, 'Reliable energy policy should protect workers.'), + (3, 3, 'message-3', 1, 'Housing affordability is central to this debate.'), + (4, 4, 'message-4', 1, 'Climate change affects housing supply.')`, + `INSERT INTO messages_fts (rowid, content) VALUES + (1, 'Climate change demands urgent action from Parliament.'), + (2, 'Reliable energy policy should protect workers.'), + (3, 'Housing affordability is central to this debate.'), + (4, 'Climate change affects housing supply.')`, + } + + for _, statement := range statements { + if _, err := db.Exec(statement); err != nil { + db.Close() + t.Fatalf("exec %q: %v", statement, err) + } + } + + return db +} diff --git a/backend/hansard-search/internal/usecase/open_search_index.go b/backend/hansard-search/internal/usecase/open_search_index.go index 895b80ea..ea177d8d 100644 --- a/backend/hansard-search/internal/usecase/open_search_index.go +++ b/backend/hansard-search/internal/usecase/open_search_index.go @@ -29,8 +29,8 @@ type IndexDownloader interface { Download(ctx context.Context, sqliteKey, expectedSHA256 string) (localPath string, err error) } -// SearchIndex holds a verified, downloaded SQLite index ready for query use cases. -// D2 will extend this with an open *sql.DB connection. +// SearchIndex holds a verified, downloaded SQLite index ready for D3's handler +// wiring to open and pass into the SQLite FTS5 query adapter. type SearchIndex struct { Manifest domain.Manifest LocalPath string diff --git a/backend/hansard-search/internal/usecase/search_hansard.go b/backend/hansard-search/internal/usecase/search_hansard.go new file mode 100644 index 00000000..6ac5da2a --- /dev/null +++ b/backend/hansard-search/internal/usecase/search_hansard.go @@ -0,0 +1,77 @@ +package usecase + +import ( + "context" + "errors" + "strings" + "unicode" +) + +var ( + ErrInvalidQuery = errors.New("invalid hansard search query") + ErrInvalidPagination = errors.New("invalid hansard search pagination") + ErrInvalidQuerySyntax = errors.New("invalid hansard search query syntax") +) + +type SearchQuery struct { + Query string // FTS5 MATCH expression + Speaker string // optional case-insensitive substring filter + Topic string // optional case-insensitive substring filter +} + +type Pagination struct { + Page int // 1-indexed + PerPage int // 1..100 +} + +type SearchHit struct { + ParliamentNumber int + SessionNumber int + SittingDate string + InterventionID string + MessageID string + SpeakerName string + PartyAbbreviation string + RidingName string + Topic string + Snippet string + Score float64 +} + +type SearchResults struct { + Total int + Hits []SearchHit +} + +type HansardSearchRepository interface { + Search(ctx context.Context, q SearchQuery, p Pagination) (SearchResults, error) +} + +type SearchHansard struct { + Repo HansardSearchRepository +} + +func (s SearchHansard) Execute(ctx context.Context, q SearchQuery, p Pagination) (SearchResults, error) { + q.Query = strings.TrimSpace(stripControlCharacters(q.Query)) + if q.Query == "" { + return SearchResults{}, ErrInvalidQuery + } + + if p.Page < 1 || p.PerPage < 1 || p.PerPage > 100 { + return SearchResults{}, ErrInvalidPagination + } + + q.Speaker = strings.TrimSpace(q.Speaker) + q.Topic = strings.TrimSpace(q.Topic) + + return s.Repo.Search(ctx, q, p) +} + +func stripControlCharacters(value string) string { + return strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return ' ' + } + return r + }, value) +} diff --git a/backend/hansard-search/internal/usecase/search_hansard_test.go b/backend/hansard-search/internal/usecase/search_hansard_test.go new file mode 100644 index 00000000..07198a68 --- /dev/null +++ b/backend/hansard-search/internal/usecase/search_hansard_test.go @@ -0,0 +1,105 @@ +package usecase_test + +import ( + "context" + "errors" + "testing" + + "epac/hansard-search/internal/usecase" +) + +type stubHansardSearchRepository struct { + gotQuery usecase.SearchQuery + gotPagination usecase.Pagination + results usecase.SearchResults + err error +} + +func (s *stubHansardSearchRepository) Search(_ context.Context, q usecase.SearchQuery, p usecase.Pagination) (usecase.SearchResults, error) { + s.gotQuery = q + s.gotPagination = p + return s.results, s.err +} + +func TestSearchHansardExecuteReturnsRepositoryResults(t *testing.T) { + repo := &stubHansardSearchRepository{ + results: usecase.SearchResults{ + Total: 1, + Hits: []usecase.SearchHit{ + { + InterventionID: "intervention-1", + MessageID: "message-1", + }, + }, + }, + } + + got, err := usecase.SearchHansard{Repo: repo}.Execute( + context.Background(), + usecase.SearchQuery{Query: "climate"}, + usecase.Pagination{Page: 1, PerPage: 10}, + ) + if err != nil { + t.Fatalf("Execute returned error: %v", err) + } + if got.Total != 1 || len(got.Hits) != 1 || got.Hits[0].InterventionID != "intervention-1" { + t.Fatalf("unexpected results: %#v", got) + } + if repo.gotQuery.Query != "climate" { + t.Fatalf("query = %q, want climate", repo.gotQuery.Query) + } + if repo.gotPagination != (usecase.Pagination{Page: 1, PerPage: 10}) { + t.Fatalf("pagination = %#v", repo.gotPagination) + } +} + +func TestSearchHansardExecuteReturnsInvalidQueryForEmptyInput(t *testing.T) { + repo := &stubHansardSearchRepository{} + + _, err := usecase.SearchHansard{Repo: repo}.Execute( + context.Background(), + usecase.SearchQuery{Query: " \n\t "}, + usecase.Pagination{Page: 1, PerPage: 10}, + ) + if !errors.Is(err, usecase.ErrInvalidQuery) { + t.Fatalf("error = %v, want ErrInvalidQuery", err) + } +} + +func TestSearchHansardExecuteReturnsInvalidPagination(t *testing.T) { + tests := []struct { + name string + pagination usecase.Pagination + }{ + {name: "page zero", pagination: usecase.Pagination{Page: 0, PerPage: 10}}, + {name: "per page zero", pagination: usecase.Pagination{Page: 1, PerPage: 0}}, + {name: "per page too large", pagination: usecase.Pagination{Page: 1, PerPage: 101}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := &stubHansardSearchRepository{} + _, err := usecase.SearchHansard{Repo: repo}.Execute( + context.Background(), + usecase.SearchQuery{Query: "climate"}, + tt.pagination, + ) + if !errors.Is(err, usecase.ErrInvalidPagination) { + t.Fatalf("error = %v, want ErrInvalidPagination", err) + } + }) + } +} + +func TestSearchHansardExecutePreservesInvalidQuerySyntax(t *testing.T) { + repo := &stubHansardSearchRepository{err: usecase.ErrInvalidQuerySyntax} + + _, err := usecase.SearchHansard{Repo: repo}.Execute( + context.Background(), + usecase.SearchQuery{Query: "\"unclosed"}, + usecase.Pagination{Page: 1, PerPage: 10}, + ) + if !errors.Is(err, usecase.ErrInvalidQuerySyntax) { + t.Fatalf("error = %v, want ErrInvalidQuerySyntax", err) + } +} diff --git a/backend/hansard-search/main.go b/backend/hansard-search/main.go index 8e7bdc0e..af72b74a 100644 --- a/backend/hansard-search/main.go +++ b/backend/hansard-search/main.go @@ -1,7 +1,7 @@ // hansard-search Lambda — GET /api/v1/hansard/search // // D1 scaffold: returns HTTP 503 for every request. -// Query logic lands in D2; the HTTP handler is wired in D3. +// D2 adds the query use case + adapter; the HTTP handler wiring lands in D3. package main import ( diff --git a/docs/architecture/use-case-catalog.md b/docs/architecture/use-case-catalog.md index c31d9f5a..f5c1278f 100644 --- a/docs/architecture/use-case-catalog.md +++ b/docs/architecture/use-case-catalog.md @@ -164,15 +164,17 @@ Inputs: Query string, optional entity-type filter. Outputs: Ranked list of matching results with entity type, title, and navigation hint. Entities / values: Hansard, SubjectOfBusiness, ParliamentMember. Ports: HansardRepository, MemberRepository. -Primary adapters (backend): None. Backend search routes and Lambda were retired by EPAC-1921. +Primary adapters (backend): hansard-search Lambda startup/wiring in `backend/hansard-search`, plus the SQLite FTS5 repository adapter in `internal/adapter/sqlitefts5`. Primary adapters (iOS): SearchViewModel, SearchView, NetworkService. Current implementation: + backend/hansard-search/internal/usecase/search_hansard.go + backend/hansard-search/internal/adapter/sqlitefts5/repository.go ios/epac/Views/Search/SearchViewModel.swift ios/epac/Views/Search/SearchView.swift ios/epac/Util/NetworkService.swift ``` -> **Boundary note:** An explicit `SearchHansard` use case type is introduced by EPAC-1742. Until that PR lands, search policy lives in `SearchViewModel`. The catalog entry is documented here so the boundary target is visible. +> **Boundary note:** The backend `SearchHansard` use case now lives in `backend/hansard-search/internal/usecase/` and depends on a `HansardSearchRepository` port implemented by the SQLite FTS5 adapter. The iOS `SearchViewModel` remains a presentation concern for the broader search UI while D3 wires the hansard-search Lambda handler to this backend policy. ---