Skip to content

Commit e8a09df

Browse files
riddim-developer-bot[bot]riddim-developer-bot
riddim-developer-bot[bot]
and
riddim-developer-bot
authored
[EPAC-805]: Add MP biography section from Parliament.ca (#796)
## Scope - Add MP biography domain/use case/repository wiring in iOS and render a Biography card on MP profiles for parliamentary service history, previous roles, education, professional background, and sponsored private members' bills. - Extend the members backend/API and members indexer artifact schema to persist and expose Parliament.ca biography role/service data plus PMB sponsorship references. - Add a sponsored-bill deep link path into BillsView that filters by exact bill numbers while preserving the main segmented bill filter UX. ## Bugfix SPEC * **Spec:** Linear EPAC-805 * **Trace ID:** EPAC-805 ## Testing notes * **Automated tests run:** - `/usr/local/go/bin/go test ./...` in `backend/members` - `/usr/local/go/bin/go test ./...` in `backend/members-indexer` - `jq empty backend/openapi/openapi.json` - `git diff --check` - `xcodebuild test -project ios/epac.xcodeproj -scheme epac -destination 'platform=iOS Simulator,name=iPhone 17 Pro,OS=26.5' -only-testing:epacTests/MPBiographyRepositoryTests -only-testing:epacTests/MemberBiographySnapshotTests -only-testing:epacTests/BillsSelectionTests` - `xcodebuild test -project ios/epac.xcodeproj -scheme epac -destination 'platform=iOS Simulator,name=iPhone 17 Pro,OS=26.5' -only-testing:epacTests/MPBiographyRepositoryTests -only-testing:epacTests/MemberBiographySnapshotTests` * **Manual verification:** Visual snapshot reviewed from `ios/epacTests/__Snapshots__/MemberBiographySnapshotTests/testPopulatedBiographyCard.populated_light.png`. The iOS test host logs an existing calendar TLS fetch failure during startup, but the selected test suites pass. ## Screenshots ![MP biography snapshot](ios/epacTests/__Snapshots__/MemberBiographySnapshotTests/testPopulatedBiographyCard.populated_light.png) ## Release note Adds an MP profile biography section sourced from Parliament.ca, including service history, roles, education/background where available, and sponsored private members' bills. ## Related issue - Closes: EPAC-805 --------- Co-authored-by: riddim-developer-bot <developer-bot@riddimsoftware.com>
1 parent 9e3f506 commit e8a09df

23 files changed

Lines changed: 1518 additions & 51 deletions

‎backend/members-indexer/internal/adapter/ourcommons/fetcher.go‎

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,21 @@ func (f *Fetcher) enrichProfile(ctx context.Context, member *domain.Member) erro
154154
PhotoURL: resolveURL(base, firstMatch(body, photoPattern)),
155155
SourceURL: member.ProfileURL,
156156
}
157+
if rolesURL := firstMatch(body, allRolesPattern); rolesURL != "" {
158+
resolvedRolesURL := resolveURL(base, rolesURL)
159+
rolesBody, err := f.getBytes(ctx, resolvedRolesURL, "text/html")
160+
if err == nil {
161+
member.Biography.YearsServed = parseServicePeriods(rolesBody)
162+
member.Biography.PreviousRoles = parseParliamentaryRoles(rolesBody)
163+
} else if f.logger != nil {
164+
f.logger(map[string]any{
165+
"pipeline": "members-indexer",
166+
"event": "roles_fetch_failed",
167+
"member_id": member.ID,
168+
"error": err.Error(),
169+
})
170+
}
171+
}
157172
member.PMBSponsorships = append(member.PMBSponsorships, parseBillTable(body, base, member.ID, "ce-mip-bill-sponsored-table", "sponsored")...)
158173
member.PMBSponsorships = append(member.PMBSponsorships, parseBillTable(body, base, member.ID, "ce-mip-bill-seconded-table", "jointly_seconded")...)
159174
member.Attendance = parseVoteTable(body, base, member.ID)
@@ -279,6 +294,63 @@ func parseVoteTable(body []byte, base *url.URL, memberID string) []domain.Attend
279294
return out
280295
}
281296

297+
func parseServicePeriods(body []byte) []domain.ServicePeriod {
298+
rows := rows(tableByID(body, "roles-mp"))
299+
out := make([]domain.ServicePeriod, 0, len(rows))
300+
for _, row := range rows {
301+
cells := cells(row)
302+
if len(cells) < 4 || strings.Contains(strings.ToLower(cells[0]), "constituency") {
303+
continue
304+
}
305+
label := joinNonEmpty(cells[0], cells[1])
306+
out = append(out, domain.ServicePeriod{
307+
Label: label,
308+
FromDate: dateOnly(cells[2]),
309+
ToDate: dateOnly(cells[3]),
310+
})
311+
}
312+
return out
313+
}
314+
315+
func parseParliamentaryRoles(body []byte) []domain.ParliamentaryRole {
316+
roles := make([]domain.ParliamentaryRole, 0)
317+
roles = append(roles, parseRoleTable(body, "roles-committees")...)
318+
roles = append(roles, parseRoleTable(body, "roles-iia")...)
319+
return roles
320+
}
321+
322+
func joinNonEmpty(parts ...string) string {
323+
nonEmpty := make([]string, 0, len(parts))
324+
for _, part := range parts {
325+
if cleaned := cleanText(part); cleaned != "" {
326+
nonEmpty = append(nonEmpty, cleaned)
327+
}
328+
}
329+
return strings.Join(nonEmpty, ", ")
330+
}
331+
332+
func parseRoleTable(body []byte, tableID string) []domain.ParliamentaryRole {
333+
tableRows := rows(tableByID(body, tableID))
334+
out := make([]domain.ParliamentaryRole, 0, len(tableRows))
335+
for _, row := range tableRows {
336+
cells := cells(row)
337+
if len(cells) < 5 || strings.Contains(strings.ToLower(cells[1]), "role") {
338+
continue
339+
}
340+
role := domain.ParliamentaryRole{
341+
Title: cleanText(cells[1]),
342+
Organization: cleanText(cells[2]),
343+
StartDate: dateOnly(cells[3]),
344+
EndDate: dateOnly(cells[4]),
345+
}
346+
if role.Title == "" && role.Organization == "" {
347+
continue
348+
}
349+
out = append(out, role)
350+
}
351+
return out
352+
}
353+
282354
func tableByID(body []byte, id string) string {
283355
pattern := regexp.MustCompile(`(?is)<table[^>]*\bid=["']` + regexp.QuoteMeta(id) + `["'][^>]*>(.*?)</table>`)
284356
return firstMatch(body, pattern)
@@ -343,7 +415,13 @@ func dateOnly(raw string) string {
343415
if raw == "" {
344416
return ""
345417
}
346-
for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} {
418+
for _, layout := range []string{
419+
time.RFC3339Nano,
420+
time.RFC3339,
421+
"2006-01-02T15:04:05",
422+
"2006-01-02",
423+
"Monday, January 2, 2006",
424+
} {
347425
if parsed, err := time.Parse(layout, raw); err == nil {
348426
return parsed.Format("2006-01-02")
349427
}
@@ -386,6 +464,7 @@ func stableID(parts ...string) string {
386464
var (
387465
metaDescriptionPattern = regexp.MustCompile(`(?is)<meta[^>]*name=["']description["'][^>]*content=["']([^"']*)["'][^>]*>`)
388466
photoPattern = regexp.MustCompile(`(?is)<img[^>]*class=["'][^"']*ce-mip-mp-picture[^"']*["'][^>]*src=["']([^"']+)["']`)
467+
allRolesPattern = regexp.MustCompile(`(?is)<a[^>]*href=["']([^"']*/roles)["'][^>]*>\s*View all roles`)
389468
allVotesPattern = regexp.MustCompile(`(?is)<a[^>]*href=["']([^"']*/votes)["'][^>]*>\s*All votes by`)
390469
hrefPattern = regexp.MustCompile(`(?is)href=["']([^"']+)["']`)
391470
rowPattern = regexp.MustCompile(`(?is)<tr[^>]*>(.*?)</tr>`)

‎backend/members-indexer/internal/adapter/ourcommons/fetcher_test.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ func TestFetcherBuildsMemberBiographyAttendanceAndPMBSponsorships(t *testing.T)
2727
_, _ = w.Write([]byte(`<html><head><meta name="description" content="Jane Example - Member of Parliament"></head><body>
2828
<img class="ce-mip-mp-picture" src="/photo.jpg">
2929
<dl><dt>Preferred Language:</dt><dd>English / French</dd></dl>
30+
<a href="/Members/en/jane-example(89156)/roles">View all roles Jane Example has held as a member of Parliament.</a>
3031
<table id="ce-mip-bill-sponsored-table"><tbody>
3132
<tr><th>Bill Number</th><th>Title</th></tr>
3233
<tr><td><a href="//parl.ca/LegisInfo/BillDetails.aspx?billId=1">C-234</a></td><td>Living Donor Recognition Medal Act</td></tr>
@@ -39,6 +40,21 @@ func TestFetcherBuildsMemberBiographyAttendanceAndPMBSponsorships(t *testing.T)
3940
<tr><td><a href="/Members/en/votes/45/1/151">No. 151</a></td><td>Report stage</td><td>Yea</td><td>Agreed To</td><td>Wednesday, June 10, 2026</td></tr>
4041
</tbody></table>
4142
</body></html>`))
43+
case "/Members/en/jane-example(89156)/roles":
44+
w.Header().Set("Content-Type", "text/html")
45+
_, _ = w.Write([]byte(`<html><body>
46+
<table id="roles-mp"><tbody>
47+
<tr><th>Constituency</th><th>Province / Territory</th><th>Start Date</th><th>End Date</th></tr>
48+
<tr><td>Ottawa Centre</td><td>Ontario</td><td>Monday, April 28, 2025</td><td></td></tr>
49+
</tbody></table>
50+
<table id="roles-committees"><tbody>
51+
<tr><th>Parliament - Session</th><th>Role</th><th>Committee</th><th>Start Date</th><th>End Date</th></tr>
52+
<tr><td>45-1</td><td>Member</td><td>Health</td><td>Friday, June 13, 2025</td><td></td></tr>
53+
</tbody></table>
54+
<table id="roles-iia"><tbody>
55+
<tr><td>45th</td><td>Member</td><td>Canada-Europe Parliamentary Association</td><td>Wednesday, April 1, 2026</td><td></td></tr>
56+
</tbody></table>
57+
</body></html>`))
4258
default:
4359
t.Fatalf("unexpected path: %s", r.URL.Path)
4460
}
@@ -57,6 +73,12 @@ func TestFetcherBuildsMemberBiographyAttendanceAndPMBSponsorships(t *testing.T)
5773
if member.Name != "Jane Example" || member.Biography.PreferredLanguage != "English / French" {
5874
t.Fatalf("member = %#v", member)
5975
}
76+
if len(member.Biography.YearsServed) != 1 || member.Biography.YearsServed[0].FromDate != "2025-04-28" {
77+
t.Fatalf("years served = %#v", member.Biography.YearsServed)
78+
}
79+
if len(member.Biography.PreviousRoles) != 2 || member.Biography.PreviousRoles[0].Organization != "Health" {
80+
t.Fatalf("previous roles = %#v", member.Biography.PreviousRoles)
81+
}
6082
if len(member.PMBSponsorships) != 2 {
6183
t.Fatalf("sponsorships = %#v", member.PMBSponsorships)
6284
}

‎backend/members-indexer/internal/adapter/sqlite/writer.go‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package sqlite
33
import (
44
"context"
55
"database/sql"
6+
"encoding/json"
67
"fmt"
78
"os"
89
"path/filepath"
@@ -140,7 +141,11 @@ CREATE TABLE member_biographies (
140141
summary TEXT NOT NULL DEFAULT '',
141142
preferred_language TEXT NOT NULL DEFAULT '',
142143
photo_url TEXT NOT NULL DEFAULT '',
143-
source_url TEXT NOT NULL DEFAULT ''
144+
source_url TEXT NOT NULL DEFAULT '',
145+
years_served_json TEXT NOT NULL DEFAULT '[]',
146+
previous_roles_json TEXT NOT NULL DEFAULT '[]',
147+
education_json TEXT NOT NULL DEFAULT '[]',
148+
professional_background_json TEXT NOT NULL DEFAULT '[]'
144149
);
145150
CREATE TABLE attendance_records (
146151
member_id TEXT NOT NULL REFERENCES members(id) ON DELETE CASCADE,
@@ -216,12 +221,21 @@ func insertChildren(ctx context.Context, tx *sql.Tx, member domain.Member, stats
216221
if bio.MemberID == "" {
217222
bio.MemberID = member.ID
218223
}
219-
if bio.SourceURL != "" || bio.Summary != "" || bio.PhotoURL != "" {
224+
if biographyHasContent(bio) {
220225
if _, err := tx.ExecContext(ctx, `
221226
INSERT OR REPLACE INTO member_biographies (
222-
member_id, summary, preferred_language, photo_url, source_url
223-
) VALUES (?, ?, ?, ?, ?)`,
224-
bio.MemberID, bio.Summary, bio.PreferredLanguage, bio.PhotoURL, bio.SourceURL); err != nil {
227+
member_id, summary, preferred_language, photo_url, source_url,
228+
years_served_json, previous_roles_json, education_json, professional_background_json
229+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
230+
bio.MemberID,
231+
bio.Summary,
232+
bio.PreferredLanguage,
233+
bio.PhotoURL,
234+
bio.SourceURL,
235+
mustJSON(bio.YearsServed),
236+
mustJSON(bio.PreviousRoles),
237+
mustJSON(bio.Education),
238+
mustJSON(bio.ProfessionalBackground)); err != nil {
225239
return fmt.Errorf("insert biography %s: %w", member.ID, err)
226240
}
227241
stats.BiographyCount++
@@ -250,6 +264,24 @@ INSERT OR REPLACE INTO pmb_sponsorships (
250264
return nil
251265
}
252266

267+
func biographyHasContent(bio domain.Biography) bool {
268+
return bio.SourceURL != "" ||
269+
bio.Summary != "" ||
270+
bio.PhotoURL != "" ||
271+
len(bio.YearsServed) > 0 ||
272+
len(bio.PreviousRoles) > 0 ||
273+
len(bio.Education) > 0 ||
274+
len(bio.ProfessionalBackground) > 0
275+
}
276+
277+
func mustJSON[T any](value T) string {
278+
data, err := json.Marshal(value)
279+
if err != nil {
280+
return "[]"
281+
}
282+
return string(data)
283+
}
284+
253285
func writeMeta(ctx context.Context, db *sql.DB, stats domain.Stats) error {
254286
values := map[string]string{
255287
"version": domain.ManifestVersion,

‎backend/members-indexer/internal/adapter/sqlite/writer_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,17 @@ func TestWriterCreatesMembersRelationalSchema(t *testing.T) {
2828
Summary: "Jane Example - Member of Parliament",
2929
PreferredLanguage: "English / French",
3030
SourceURL: "https://www.ourcommons.ca/Members/en/89156",
31+
YearsServed: []domain.ServicePeriod{{
32+
Label: "Ottawa Centre, Ontario",
33+
FromDate: "2025-04-28",
34+
}},
35+
PreviousRoles: []domain.ParliamentaryRole{{
36+
Title: "Member",
37+
Organization: "Health",
38+
StartDate: "2025-06-13",
39+
}},
40+
Education: []string{"University of Ottawa, MD"},
41+
ProfessionalBackground: []string{"Family physician"},
3142
},
3243
Attendance: []domain.AttendanceRecord{{
3344
ID: "vote-151", VoteNumber: "151", Subject: "Report stage", Ballot: "Yea", Result: "Agreed To",
@@ -55,6 +66,16 @@ func TestWriterCreatesMembersRelationalSchema(t *testing.T) {
5566
if party != "Liberal" {
5667
t.Fatalf("party = %q", party)
5768
}
69+
var yearsServedJSON, educationJSON string
70+
if err := db.QueryRow("SELECT years_served_json, education_json FROM member_biographies WHERE member_id = ?", "89156").Scan(&yearsServedJSON, &educationJSON); err != nil {
71+
t.Fatalf("query biography details: %v", err)
72+
}
73+
if yearsServedJSON != `[{"label":"Ottawa Centre, Ontario","from_date":"2025-04-28"}]` {
74+
t.Fatalf("years served json = %q", yearsServedJSON)
75+
}
76+
if educationJSON != `["University of Ottawa, MD"]` {
77+
t.Fatalf("education json = %q", educationJSON)
78+
}
5879
}
5980

6081
type fixedClock struct{}

‎backend/members-indexer/internal/domain/domain.go‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,28 @@ type Member struct {
2727
}
2828

2929
type Biography struct {
30-
MemberID string
31-
Summary string
32-
PreferredLanguage string
33-
PhotoURL string
34-
SourceURL string
30+
MemberID string
31+
Summary string
32+
PreferredLanguage string
33+
PhotoURL string
34+
SourceURL string
35+
YearsServed []ServicePeriod
36+
PreviousRoles []ParliamentaryRole
37+
Education []string
38+
ProfessionalBackground []string
39+
}
40+
41+
type ServicePeriod struct {
42+
Label string `json:"label,omitempty"`
43+
FromDate string `json:"from_date,omitempty"`
44+
ToDate string `json:"to_date,omitempty"`
45+
}
46+
47+
type ParliamentaryRole struct {
48+
Title string `json:"title,omitempty"`
49+
Organization string `json:"organization,omitempty"`
50+
StartDate string `json:"start_date,omitempty"`
51+
EndDate string `json:"end_date,omitempty"`
3552
}
3653

3754
type AttendanceRecord struct {

0 commit comments

Comments
 (0)