diff --git a/cmd/fox/aadhaarcmd.go b/cmd/fox/aadhaarcmd.go new file mode 100644 index 0000000..71c3893 --- /dev/null +++ b/cmd/fox/aadhaarcmd.go @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "fmt" + "io" + "os" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/thefoxbyte/foxbyte/internal/auth" + "github.com/thefoxbyte/foxbyte/internal/branch" + "github.com/thefoxbyte/foxbyte/internal/brand" + "github.com/thefoxbyte/foxbyte/internal/edition" +) + +// aadhaarCmd is `fox pii aadhaar` (point 4.5): which person's Aadhaar is +// exposed in a branch, where, and since when. +// +// fox pii aadhaar run [--branch b] [--table-timeout 5m] [--json] read every row now (Enterprise) +// fox pii aadhaar [--branch b] [--json] the latest report +// fox pii aadhaar places [--person n] [--column s.t.c] [--gone] [--limit n] [--json] +// +// A number is shown as UIDAI masks it, XXXX XXXX 1234, and is kept no other +// way. Reading the report is free; a run is Enterprise and is in the security +// log with its counts. +func aadhaarCmd(args []string, b string, ps []string) { + sub := "report" + if len(ps) > 0 { + sub = ps[0] + } + switch sub { + case "run": + requireFeature(edition.PersonalData) + o := branch.AadhaarRunOptions{Actor: cliActor()} + if v := optValue(args, "--table-timeout"); v != "" { + d, err := time.ParseDuration(v) + if err != nil { + must(fmt.Errorf("--table-timeout is a duration (90s, 5m), not %q", v)) + } + o.TableTimeout = d + } + rep, err := branch.RunAadhaarExposure(b, o) + must(err) + if s := historyStore(); s != nil { + s.Audit(auth.EvAadhaarExposure, o.Actor, b, "", rep.Summary()) + s.Close() + } + if hasFlag(args, "--json") { + printJSON(rep) + return + } + printAadhaar(os.Stdout, rep) + case "report": + rep, err := branch.AadhaarExposure(b) + must(err) + if hasFlag(args, "--json") { + printJSON(rep) + return + } + printAadhaar(os.Stdout, rep) + case "places": + f := branch.AadhaarPlaceFilter{Gone: hasFlag(args, "--gone")} + for flag, to := range map[string]*int{"--person": &f.Person, "--limit": &f.Limit} { + if v := optValue(args, flag); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + must(fmt.Errorf("%s is a number, not %q", flag, v)) + } + *to = n + } + } + if c := optValue(args, "--column"); c != "" { + w, err := branch.ParseAadhaarWhere(c) + must(err) + f.Where = w + } + places, err := branch.AadhaarPlaces(b, f) + must(err) + if hasFlag(args, "--json") { + printJSON(places) + return + } + printAadhaarPlaces(os.Stdout, places, f.Gone, b) + default: + usagePII() + } +} + +func shortDay(t time.Time) string { return t.Local().Format("2006-01-02") } + +func printAadhaar(w io.Writer, rep branch.AadhaarReport) { + r := rep.Run + if r == nil { + fmt.Fprintf(w, "No Aadhaar exposure report on %s yet: `%s pii aadhaar run --branch %s` reads it (Enterprise).\n", rep.Branch, brand.CLI, rep.Branch) + return + } + by := "" + if r.Actor != "" { + by = " by " + r.Actor + } + fmt.Fprintf(w, "Aadhaar exposure on %s: run %d, %s%s.\n", rep.Branch, r.ID, r.Finished.Local().Format("2006-01-02 15:04"), by) + fmt.Fprintf(w, "%d table(s), %d column(s) read; %d row(s) held twelve digits in its shape. %d number(s) found, %d distinct; %d place(s) kept.\n", + r.Tables, r.Columns, r.Rows, r.Numbers, r.People, r.Listed) + for _, s := range r.Skipped { + fmt.Fprintf(w, " not read: %s (%s)\n", s.Object, s.Reason) + } + if len(rep.Columns) == 0 { + fmt.Fprintln(w, "\nNo Aadhaar number was found in the tables read.") + } else { + fmt.Fprintln(w, "\nBy column") + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, " COLUMN\tNUMBERS\tROWS\tKEPT\tSINCE") + for _, c := range rep.Columns { + kept := strconv.Itoa(c.Listed) + if c.Capped { + kept += " (first)" + } + since := "–" + if c.Since != nil { + since = shortDay(*c.Since) + } + fmt.Fprintf(tw, " %s\t%d\t%d\t%s\t%s\n", c.Where(), c.Numbers, c.Rows, kept, since) + } + tw.Flush() + } + if len(rep.People) > 0 { + more := "" + if rep.PeopleListed > len(rep.People) { + more = fmt.Sprintf(", the first %d of %d", len(rep.People), rep.PeopleListed) + } + fmt.Fprintf(w, "\nBy person (the most places first%s)\n", more) + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, " PERSON\tNUMBER\tPLACES\tSINCE\tWHERE") + for _, p := range rep.People { + fmt.Fprintf(tw, " %d\t%s\t%d\t%s\t%s\n", p.Person, p.Masked, p.Places, shortDay(p.Since), strings.Join(p.Where, ", ")) + } + tw.Flush() + fmt.Fprintf(w, "One person's rows: %s pii aadhaar places --person --branch %s\n", brand.CLI, rep.Branch) + } + if rep.Gone > 0 { + fmt.Fprintf(w, "\n%d place(s) earlier runs found are no longer there: %s pii aadhaar places --gone --branch %s\n", rep.Gone, brand.CLI, rep.Branch) + } + if len(r.Limits) > 0 { + fmt.Fprintln(w, "\nWhat this report cannot claim:") + for _, l := range r.Limits { + fmt.Fprintf(w, " - %s\n", l) + } + } +} + +func printAadhaarPlaces(w io.Writer, places []branch.AadhaarPlace, gone bool, b string) { + if len(places) == 0 { + if gone { + fmt.Fprintf(w, "No place on %s has gone since an earlier run.\n", b) + } else { + fmt.Fprintf(w, "No places on %s match.\n", b) + } + return + } + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + if gone { + fmt.Fprintln(tw, "NUMBER\tWHERE\tROW\tSINCE\tLAST SEEN") + for _, p := range places { + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\n", p.Masked, aadhaarCol(p), p.Row, shortDay(p.Since), shortDay(p.LastSeen)) + } + } else { + fmt.Fprintln(tw, "PERSON\tNUMBER\tWHERE\tROW\tSINCE") + for _, p := range places { + person := strconv.Itoa(p.Person) + if p.Person == 0 { + person = "–" + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\n", person, p.Masked, aadhaarCol(p), p.Row, shortDay(p.Since)) + } + } + tw.Flush() +} + +func aadhaarCol(p branch.AadhaarPlace) string { + return branch.AadhaarColumn{Schema: p.Schema, Table: p.Table, Column: p.Column, Path: p.Path}.Where() +} diff --git a/cmd/fox/main.go b/cmd/fox/main.go index 12e2755..ebe5f90 100644 --- a/cmd/fox/main.go +++ b/cmd/fox/main.go @@ -126,6 +126,11 @@ Stack: who has gone quiet, and the applications that assert their users (Enterprise) pii unclassified | accept [--note "…"] What appeared since a branch's first scan, not reviewed; accept one as holding nothing + pii aadhaar run [--branch b] [--table-timeout 5m] [--json] + Read every row for Aadhaar numbers: where each is, by row and last four + digits, grouped by person, and since when; never the number (Enterprise) + pii aadhaar [--branch b] [--json] | places [--person n] [--column s.t.c] [--gone] [--limit n] + The latest exposure report, by column and by person; or its places app [list|show ] [--json] The applications that may say which of their users a transaction acts for, and the "aud" their tokens name diff --git a/cmd/fox/piicmd.go b/cmd/fox/piicmd.go index 130dbfd..8db03c5 100644 --- a/cmd/fox/piicmd.go +++ b/cmd/fox/piicmd.go @@ -36,7 +36,7 @@ func piiCmd(args []string) { if b == "" { b = "main" } - ps := positionals(args, "--branch", "--sample", "--state", "--n", "--note") + ps := positionals(args, "--branch", "--sample", "--state", "--n", "--note", "--table-timeout", "--person", "--column", "--limit") sub := "findings" if len(ps) > 0 { sub = ps[0] @@ -175,6 +175,8 @@ func piiCmd(args []string) { fmt.Fprintf(os.Stderr, "note: its table could not be marked for the audit log (%s); `%s audit-log table --personal-data --branch %s` marks it later.\n", d.AuditMarkError, brand.CLI, b) } + case "aadhaar": + aadhaarCmd(args, b, ps[1:]) default: usagePII() } @@ -187,7 +189,10 @@ func usagePII() { " %[1]s pii samples [--branch b] [--n 5] [--json]\n"+ " %[1]s pii mark confirm|reject|reopen [--note \"…\"] [--branch b]\n"+ " %[1]s pii unclassified [--branch b] [--json]\n"+ - " %[1]s pii accept [--note \"…\"] [--branch b]\n", brand.CLI) + " %[1]s pii accept [--note \"…\"] [--branch b]\n"+ + " %[1]s pii aadhaar run [--branch b] [--table-timeout 5m] [--json]\n"+ + " %[1]s pii aadhaar [--branch b] [--json]\n"+ + " %[1]s pii aadhaar places [--branch b] [--person n] [--column schema.table.column] [--gone] [--limit n] [--json]\n", brand.CLI) os.Exit(2) } diff --git a/docs/FOX_Checklist.html b/docs/FOX_Checklist.html index 6f3bad5..0074f89 100644 --- a/docs/FOX_Checklist.html +++ b/docs/FOX_Checklist.html @@ -781,6 +781,13 @@

OPersonal data & complianceStep 4: every access path the design names tested as a masked account and as its agent key (integration §11al, masking.spec.ts). It found eight ways to real values, now closed: a branch it owns, made of main or back in time, reads masks (owning one no longer unmasks); its agent keys log in as a masked agent role, which cannot reach db_client by inheriting or SET ROLE, and MCP’s run_sql and execute_change never fall back to a shared role; agent branches are refused to it (masked branches since step 5); realtime is refused to it where there are masks; the Gateway refuses it rather than log it in as the shared role; the Blackbox’s entries reach it without their statements’ text, and its export is refused; a guard in the database (bb.mask_close_paths, an event trigger) takes masked accounts’ rights on a view or materialized view reading a masked column, on a marked table granted back, and INSERT/UPDATE on a table with a foreign key into a masked column, and gives them back when the rule goes, refusing a GRANT to PUBLIC; fox mask and the Masking page list what it closed and the definer functions that can return real values. Kept, and now tested: COPY, RETURNING, ON CONFLICT, MERGE, errors, pg_stats, EXPLAIN, the console’s query, grid, object browser and exports
  • Step 5: fox branch create <name> --masked [--from b], the Dashboard’s masked option and POST /api/branches with masked (made in the background, 202), fox mask branches, GET /api/masking/branches and the Masking page’s Masked branches table. The copy is recorded before it exists, so the Gateway, the console, the API, MCP and the agents refuse it (423, 57P03) until it is masked and verified, and it never mounts the key; every marked column is overwritten with the mask a masked reader sees, checked row by row; the masks’ machinery is removed; its Blackbox starts afresh, numbering on from the source’s last entry, so a change request from it carries only what was done on it; VACUUM FULL of each masked table and the statistics; the source’s replication slots dropped and the WAL from before removed. Verified: no WAL segment from before, nothing of the source’s Blackbox, and the scan finding no masked category, nor anything confirmed, where no rule masked it (a finding rejected on the source is ignored). A failure, or a run stopped part way, leaves it closed, saying why; running it again starts over. On an open masked branch nobody is masked; a copy of one is one at once; reset makes it again; a masked account’s agent branch is made as one (replacing step 4’s refusal)
  • One-column primary keys only; a JSON path into an array, or a quoted key, is refused for now; masking stays on once set up; a masked account has no DDL rights; a domain’s key is the install’s, shared by every branch, so a branch stopped through a rotation keeps its old masks until set up again; an unmask holder’s reads are logged only while the audit log is on; the lookup is by a column with a rule of its own, not a JSON path; a definer function that can leak is listed, never changed; CREATE TABLE AS copies real values into an ordinary table, which the scanner finds on its next run; the change requests list shows each request’s SQL to anyone on its branches; a masked branch’s blocks shared with its source’s snapshot, and the source’s own WAL and backups, hold real values (they are the source’s); on a masked branch a foreign key from a column without a rule into a hashed column no longer matches (masking skips the check)
    • enterprise/masking unit; internal/branch mask_test.go, maskkey_test.go, maskpaths_test.go, maskedbranch_test.go; VM §11ai, §11aj, §11ak, §11al, §11am; masking.spec.ts + O4Aadhaar exposure report (point 4.5, Enterprise)
      +
    • Every row of every text, number, array and JSON column read for Aadhaar numbers (Verhoeff), in free text and JSON path by path; the database filters for the shape, the engine checks the digit; run-on numbers and cards refused
      • +
    • Places kept as column or path, the row’s primary key (or ctid, said so) and the last four digits; never the number or a hash; grouped by person in memory; since when from the first run or a tracked row’s created_at; gone when a full read no longer finds it
      • +
    • fox pii aadhaar run|places, three API routes, and the console’s Aadhaar exposure page; the branch’s owner or an admin; each run in the security log with counts
      • +
    • A table locked or too slow is skipped and said so, its places not called gone; a branch made from it carries the report; a masked branch starts with none
      • +
    • 4.5b: query history, the audit log’s statement text and the Blackbox (admins only)
      • +
    • No vault yet (point 13); a person is a run’s grouping; places capped at 10,000 a column and 200,000 a run; synchronous
    enterprise/pii unit; VM §11ar; aadhaar.spec.ts
    FOX_Checklist · area O
    diff --git a/docs/FOX_Checklist.pdf b/docs/FOX_Checklist.pdf index 068ad53..7819716 100644 Binary files a/docs/FOX_Checklist.pdf and b/docs/FOX_Checklist.pdf differ diff --git a/docs/FOX_Feature_Implemented.html b/docs/FOX_Feature_Implemented.html index 5a71db0..8c04617 100644 --- a/docs/FOX_Feature_Implemented.html +++ b/docs/FOX_Feature_Implemented.html @@ -119,6 +119,7 @@

    Change log

    Newest first. Every change to FoxByte adds a row here.

    + @@ -416,6 +417,7 @@

    Feature index

  • O1Personal-data discovery: scan, review, keeping it currentWorking
  • O2Access review reportWorking
  • O3Masking personal dataWorking
    • +
  • O4Aadhaar exposure reportWorking
    • FOX_Feature Implemented · index
    @@ -1716,6 +1718,17 @@

    Test matrix

    Added in
    feat/f5-mask-step1; key domains and rotation in feat/f5-mask-step2; unmask, the lookup and the access review’s column in feat/f5-mask-step3; every access path tested, and the paths it found closed, in feat/f5-mask-step4; masked branches in feat/f5-mask-step5.
    +
    O4

    Aadhaar exposure report

    Working
    +
    What it does
    Point 4.5: which person’s Aadhaar is exposed in a branch, where, and since when. A run reads every row of every text, number, array and JSON column, and finds each Aadhaar number (twelve digits, first 2–9, Verhoeff), alone or inside free text (“caller read out 2345 6789 0124”), and inside JSON path by path, a key holding a number included. The report gives each column (or JSON path) with how many numbers and rows it holds, and the people: each number as UIDAI masks it (XXXX XXXX 0124), the places it sits, and since when; and a person’s rows by their primary key (id=42). A number found by an earlier run and no longer there is gone, with when it was last seen.
    +
    How it works
    enterprise/pii/aadhaar.go, as the superuser (a run that obeyed row-level security would call a branch clean that is not), one read-only transaction a table with a time limit (5 minutes unless told, 1 s to 1 h) and a 10 s lock wait; a table it cannot read is skipped and said so. The database filters each table for twelve digits in Aadhaar’s shape, so only rows that may hold one cross the connection, and the check digit is checked in the engine. Twelve digits run on into a longer number (a card’s sixteen, digits after a space) are not an Aadhaar. A place is kept as its column or path, the row’s primary key (a table with none by its ctid, and the report says so) and the number’s last four digits, in bb.aadhaar_places, with the first and last run that found it; bb.aadhaar_columns counts each column in full; bb.aadhaar_runs keeps each run, what it skipped and what it cannot claim. Places of one number are grouped as one person by the run, holding the numbers in memory only. Since when is the first run that found a number in a place, or the row’s created_at on a tracked table (L4, point 6.1) if earlier. In the branch, so a branch made from it carries its report; a masked branch starts with none, since the places name the source’s rows. A place is gone only when the latest run read its table in full; a skipped table’s places were not looked for.
    +
    Never the number
    No full number, and no hash of one, is stored, logged or shown: UIDAI forbids hashes of Aadhaar as reference keys, and twelve digits with a check digit are few enough to find from any hash. A JSON key holding a long run of digits is kept in a path as *. The tables are not granted to db_client. Each run is in the security log (personal-data.aadhaar) with counts only.
    +
    Who
    The branch’s owner or an admin, as the scan (O1); someone with no access is told the branch is not there. Running it is Enterprise (personal-data); reading a report is free.
    +
    How to use
    fox pii aadhaar run [--branch b] [--table-timeout 5m] [--json] · fox pii aadhaar [--branch b] [--json] · fox pii aadhaar places [--person n] [--column schema.table.column] [--gone] [--limit n] [--json] · GET /api/branches/{name}/personal-data/aadhaar, POST …/aadhaar/run, GET …/aadhaar/places · the console’s Aadhaar exposure page (Compliance): by column and by person, each with its rows, and what is gone.
    +
    How to test
    Unit: enterprise/pii/aadhaar_test.go (numbers in text, run-on numbers and cards refused, the database’s filter a superset of the check, JSON paths with a number as a key hidden, row keys, grouping and caps, no number kept), enterprise/masking/sql_test.go (a masked branch empties the report), web/src/pages/AadhaarExposure.test.tsx. Integration §11ar; web/tests/aadhaar.spec.ts.
    +
    Status & limits
    Working, Enterprise. The first part of 4.5: the branch’s tables. FoxByte’s own logs (query history, the audit log’s statement text, the Blackbox’s statements) are the second part (4.5b, admins only); server log files and SIEM platforms sit outside PostgreSQL. There is no vault yet (point 13), so every number found is outside one. A check digit shows a number is well-formed, not issued or whose. A person is a run’s grouping, not an identity kept across runs. At most 10,000 places are kept a column and 200,000 a run, and 2,000,000 distinct numbers grouped (counts are in full, and the report says when it capped). A run is synchronous and reads everything, so on a large branch it takes a while. Views, FoxByte’s masks, binary columns, dates and network addresses are not read.
    +
    Added in
    feat/p4-aadhaar-exposure
    +
    +
    FOX_Feature Implemented · area O
    diff --git a/docs/FOX_Feature_Implemented.pdf b/docs/FOX_Feature_Implemented.pdf index 9da2062..fc67894 100644 Binary files a/docs/FOX_Feature_Implemented.pdf and b/docs/FOX_Feature_Implemented.pdf differ diff --git a/enterprise/masking/maskbranch.go b/enterprise/masking/maskbranch.go index 3b5e906..5b4f0e9 100644 --- a/enterprise/masking/maskbranch.go +++ b/enterprise/masking/maskbranch.go @@ -162,11 +162,16 @@ DROP TABLE IF EXISTS bb.mask_closed, bb.mask_rules, bb.mask_tables; // freshBlackboxSQL empties the Blackbox's tables in one statement, as the // engine, past their append-only triggers. Its sequence is kept, so the // branch's first entry follows the source's last. +// +// The Aadhaar exposure report's places too (point 4.5): each names a row of +// the source by its key and a number's last four digits, which are the +// source's, not the masks'. const freshBlackboxSQL = `SET session_replication_role = replica; DO $bbfresh$ DECLARE t text; ts text[] := '{}'; BEGIN - FOREACH t IN ARRAY ARRAY['bb.ledger_ext', 'bb.ledger_checkpoints', 'bb.agent_sessions', 'bb.schema_ledger'] LOOP + FOREACH t IN ARRAY ARRAY['bb.ledger_ext', 'bb.ledger_checkpoints', 'bb.agent_sessions', 'bb.schema_ledger', + 'bb.aadhaar_places', 'bb.aadhaar_columns', 'bb.aadhaar_runs'] LOOP IF to_regclass(t) IS NOT NULL THEN ts := ts || t; END IF; END LOOP; IF array_length(ts, 1) > 0 THEN diff --git a/enterprise/masking/sql_test.go b/enterprise/masking/sql_test.go index c9e11c3..4b2a21d 100644 --- a/enterprise/masking/sql_test.go +++ b/enterprise/masking/sql_test.go @@ -287,6 +287,7 @@ func TestMaskBranchSQL(t *testing.T) { } } if !strings.Contains(freshBlackboxSQL, "session_replication_role = replica") || !strings.Contains(freshBlackboxSQL, "'bb.schema_ledger'") || + !strings.Contains(freshBlackboxSQL, "'bb.aadhaar_places'") || !strings.Contains(freshBlackboxSQL, "'bb.aadhaar_runs'") || strings.Contains(freshBlackboxSQL, "RESTART IDENTITY") { t.Error("the Blackbox is emptied past its triggers, keeping its numbering") } diff --git a/enterprise/pii/aadhaar.go b/enterprise/pii/aadhaar.go new file mode 100644 index 0000000..16df088 --- /dev/null +++ b/enterprise/pii/aadhaar.go @@ -0,0 +1,639 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package pii + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/thefoxbyte/foxbyte/enterprise/schema" + "github.com/thefoxbyte/foxbyte/internal/branch" +) + +// The Aadhaar exposure report's run (point 4.5): every row of a branch's +// tables read for Aadhaar numbers, and each place one sits kept by its row's +// key and the number's last four digits. +// +// As the superuser, for the scan's reason: a run that obeyed row-level +// security would call a branch clean while rows it could not see held exactly +// what it looks for. Read-only transactions, one a table, each with a time +// limit. The database filters each table for twelve digits in Aadhaar's shape, +// so only rows that may hold one cross the connection; the check digit is +// checked here. The numbers stay in this process's memory for the run, to +// group the places of one number as one person, and are never written. + +func init() { branch.SetAadhaarReporter(runExposure) } + +const ( + // runTimeout bounds a whole run. + runTimeout = 2 * time.Hour + // maxListedPerColumn and maxListed bound the places kept. A column that + // holds every customer's number is one finding, said in its count; the + // report lists the first places, not all of them. + maxListedPerColumn = 10000 + maxListed = 200000 + // maxGrouped bounds the distinct numbers held in memory to group places + // by person. Past it, a new number's places are kept ungrouped (person 0). + maxGrouped = 2000000 + // maxExposureDocument is the largest JSON document walked, as stored. + maxExposureDocument = 16 << 20 +) + +// shapePG is the database's filter: twelve digits as four, four and four, +// with or without a space or hyphen between. A superset of what isAadhaarIn +// accepts, so the filter never drops a row the check would keep. +const shapePG = `[2-9][0-9]{3}[ -]?[0-9]{4}[ -]?[0-9]{4}` + +var shapeRE = regexp.MustCompile(`[2-9][0-9]{3}[ -]?[0-9]{4}[ -]?[0-9]{4}`) + +// findAadhaar returns the Aadhaar numbers in a text, as twelve digits each, +// in order and each once. A match must stand alone: twelve digits inside a +// longer number (a card's sixteen, a phone with its country code run on) are +// that number, not an Aadhaar. +func findAadhaar(s string) []string { + var out []string + for _, m := range shapeRE.FindAllStringIndex(s, -1) { + if joined(s, m[0], -1) || joined(s, m[1]-1, +1) { + continue + } + d := strings.NewReplacer(" ", "", "-", "").Replace(s[m[0]:m[1]]) + if len(d) != 12 || !verhoeff(d) { + continue + } + dup := false + for _, x := range out { + if x == d { + dup = true + break + } + } + if !dup { + out = append(out, d) + } + } + return out +} + +// joined reports whether the digit at i continues into another beyond it, in +// direction dir: a digit next to it, or a space or hyphen and then a digit. +func joined(s string, i, dir int) bool { + j := i + dir + if j < 0 || j >= len(s) { + return false + } + if isDigit(s[j]) { + return true + } + if s[j] == ' ' || s[j] == '-' { + k := j + dir + return k >= 0 && k < len(s) && isDigit(s[k]) + } + return false +} + +func isDigit(b byte) bool { return b >= '0' && b <= '9' } + +// safeKey is a JSON key as a path may keep it. A key holding a long run of +// digits could be the number itself (documents keyed by Aadhaar), and a path +// is kept: it becomes *. +func safeKey(parent, key string) string { + run, longest := 0, 0 + for i := 0; i < len(key); i++ { + if isDigit(key[i]) { + run++ + if run > longest { + longest = run + } + } else if key[i] != ' ' && key[i] != '-' { + run = 0 + } + } + if longest >= 8 { + return parent + ".*" + } + return childPath(parent, key) +} + +// docHit is one number found in a JSON document, and the path it was at. +type docHit struct{ path, number string } + +// walkExposure finds the Aadhaar numbers in one JSON document, path by path: +// values, and keys (a document keyed by number). ok is false for a document +// that is not JSON. +func walkExposure(doc string) (hits []docHit, ok bool) { + dec := json.NewDecoder(bytes.NewReader([]byte(doc))) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, false + } + var walk func(v any, path string, depth int) + walk = func(v any, path string, depth int) { + if depth > maxDepth { + return + } + switch x := v.(type) { + case map[string]any: + keys := make([]string, 0, len(x)) + for k := range x { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + child := safeKey(path, k) + for _, n := range findAadhaar(k) { + hits = append(hits, docHit{child, n}) + } + walk(x[k], child, depth+1) + } + case []any: + for _, c := range x { + walk(c, path+"[*]", depth+1) + } + case string: + for _, n := range findAadhaar(x) { + hits = append(hits, docHit{path, n}) + } + case json.Number: + for _, n := range findAadhaar(x.String()) { + hits = append(hits, docHit{path, n}) + } + } + } + walk(v, "$", 0) + return hits, true +} + +// exposureTable is a table as a run reads it. +type exposureTable struct { + table + read []column + key string // the SQL that names a row + keyless bool // named by ctid, not a primary key + created string // the tracked created_at column, "" if none +} + +type colKey struct{ schema, table, column, path string } + +type placeKey struct { + colKey + row, last4 string +} + +type colStat struct { + numbers, rows int64 + listed int + capped bool +} + +// exposure is a run's working state: what it has found so far. +type exposure struct { + people map[uint64]int32 + grouped bool // every number has a person + places []placeKey + kept map[placeKey]int // index into places and the slices beside it + person []int32 + created []pgtype.Timestamptz + cols map[colKey]*colStat + order []colKey +} + +func newExposure() *exposure { + return &exposure{people: map[uint64]int32{}, grouped: true, kept: map[placeKey]int{}, cols: map[colKey]*colStat{}} +} + +// add records the numbers found in one value of one row. +func (e *exposure) add(ck colKey, row string, created pgtype.Timestamptz, numbers []string) { + if len(numbers) == 0 { + return + } + st := e.cols[ck] + if st == nil { + st = &colStat{} + e.cols[ck] = st + e.order = append(e.order, ck) + } + st.rows++ + for _, n := range numbers { + st.numbers++ + v, _ := strconv.ParseUint(n, 10, 64) + p, ok := e.people[v] + if !ok { + if len(e.people) < maxGrouped { + p = int32(len(e.people) + 1) + e.people[v] = p + } else { + e.grouped = false + } + } + pk := placeKey{ck, row, n[8:]} + if _, dup := e.kept[pk]; dup { + continue + } + if st.listed >= maxListedPerColumn || len(e.places) >= maxListed { + st.capped = true + continue + } + e.kept[pk] = len(e.places) + e.places = append(e.places, pk) + e.person = append(e.person, p) + e.created = append(e.created, created) + st.listed++ + } +} + +// runExposure reads a branch for Aadhaar numbers and records what it found. +func runExposure(name string, o branch.AadhaarRunOptions) (branch.AadhaarRun, error) { + ctx, cancel := context.WithTimeout(context.Background(), runTimeout) + defer cancel() + if err := schema.EnsurePII(name); err != nil { + return branch.AadhaarRun{}, err + } + conn, err := branch.OpenSuper(ctx, name) + if err != nil { + return branch.AadhaarRun{}, err + } + defer conn.Close(context.Background()) + + r := branch.AadhaarRun{Started: time.Now().UTC(), Actor: o.Actor, Skipped: []branch.PIISkip{}} + tables, err := exposureCatalog(ctx, conn) + if err != nil { + return r, fmt.Errorf("reading the catalog: %w", err) + } + e := newExposure() + var st exposureStats + for _, t := range tables { + if t.kind == "f" { + r.Skipped = append(r.Skipped, branch.PIISkip{Object: t.label(), Reason: "a foreign table: its data is held somewhere else"}) + continue + } + if len(t.read) == 0 { + continue + } + n, err := readExposure(ctx, conn, t, o.TableTimeout, e, &st) + if err != nil { + r.Skipped = append(r.Skipped, branch.PIISkip{Object: t.label(), Reason: exposureFailure(err, o.TableTimeout)}) + continue + } + r.Tables++ + r.Columns += len(t.read) + r.Rows += n + if t.keyless { + st.keyless = append(st.keyless, t.label()) + } + } + for _, c := range e.cols { + r.Numbers += c.numbers + } + r.People = len(e.people) + r.Listed = len(e.places) + r.Limits = exposureLimits(e, st) + r.Finished = time.Now().UTC() + return r, recordExposure(ctx, conn, &r, e) +} + +// exposureStats is what a run could not read, or read only in part. +type exposureStats struct { + keyless []string + large int + bad int +} + +// exposureCatalog lists the tables a run reads: the scan's catalog, less +// FoxByte's masks, with each table's key and tracked creation column. +func exposureCatalog(ctx context.Context, conn *pgx.Conn) ([]exposureTable, error) { + tables, err := catalog(ctx, conn) + if err != nil { + return nil, err + } + keys := map[[2]string][]string{} + rows, err := conn.Query(ctx, `SELECT n.nspname, c.relname, array_agg(a.attname ORDER BY k.ord) + FROM pg_index i + JOIN pg_class c ON c.oid = i.indrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.attnum + WHERE i.indisprimary + GROUP BY n.nspname, c.relname`) + if err != nil { + return nil, err + } + for rows.Next() { + var s, t string + var cols []string + if err := rows.Scan(&s, &t, &cols); err != nil { + rows.Close() + return nil, err + } + keys[[2]string{s, t}] = cols + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + created := map[[2]string]string{} + var tracking bool + if err := conn.QueryRow(ctx, `SELECT to_regprocedure('bb.tracked_tables()') IS NOT NULL`).Scan(&tracking); err != nil { + return nil, err + } + if tracking { + rows, err := conn.Query(ctx, `SELECT n.nspname, c.relname, t.created_col + FROM bb.tracked_tables() t JOIN pg_class c ON c.oid = t.rel JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE t.created_col <> ''`) + if err != nil { + return nil, err + } + for rows.Next() { + var s, t, c string + if err := rows.Scan(&s, &t, &c); err != nil { + rows.Close() + return nil, err + } + created[[2]string{s, t}] = c + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + } + + var out []exposureTable + for _, t := range tables { + if t.schema == branch.MaskSchema { + continue // FoxByte's masks: the masks of numbers, not numbers + } + x := exposureTable{table: t} + for _, c := range t.cols { + if exposureReads(c) { + x.read = append(x.read, c) + } + if c.name == created[[2]string{t.schema, t.name}] && c.typname == "timestamptz" { + x.created = c.name + } + } + x.key, x.keyless = rowKeySQL(t, keys[[2]string{t.schema, t.name}]) + out = append(out, x) + } + return out, nil +} + +// exposureReads says whether a run reads a column: anything that can hold the +// digits as text. Strings, numbers, arrays and composites of them, and JSON. +// Not binary (bytea's hex would be read as digits), dates, booleans, network +// addresses or geometry. +func exposureReads(c column) bool { + if isJSON(c) { + return true + } + switch c.typcat { + case "S", "N", "A", "C": + return true + } + return false +} + +// rowKeySQL is the SQL naming a row as a place keeps it: id=42, or +// (a, b)=(1, x) for a key of more than one column. A table with no primary +// key is named by its physical position, with its partition's name for a +// partitioned one. +func rowKeySQL(t table, key []string) (string, bool) { + switch { + case len(key) == 1: + return branch.QuoteLiteral(key[0]+"=") + " || " + branch.QuoteIdent(key[0]) + "::text", false + case len(key) > 1: + vals := make([]string, len(key)) + for i, k := range key { + vals[i] = "coalesce(" + branch.QuoteIdent(k) + "::text, 'NULL')" + } + return branch.QuoteLiteral("("+strings.Join(key, ", ")+")=(") + " || " + + strings.Join(vals, " || ', ' || ") + " || ')'", false + case t.kind == "p": + return "'ctid=' || tableoid::regclass::text || ':' || ctid::text", true + default: + return "'ctid=' || ctid::text", true + } +} + +// readExposure reads one table's rows that may hold a number, and adds what +// they hold. It returns how many rows the filter passed on. +func readExposure(ctx context.Context, conn *pgx.Conn, t exposureTable, timeout time.Duration, e *exposure, st *exposureStats) (int64, error) { + sel := []string{t.key, "NULL::timestamptz"} + if t.created != "" { + sel[1] = branch.QuoteIdent(t.created) + } + where := make([]string, len(t.read)) + for i, c := range t.read { + q := branch.QuoteIdent(c.name) + where[i] = q + "::text ~ " + branch.QuoteLiteral(shapePG) + if isJSON(c) { + sel = append(sel, fmt.Sprintf("CASE WHEN pg_column_size(%s) > %d THEN %s ELSE %s::text END", + q, maxExposureDocument, branch.QuoteLiteral(tooLarge), q)) + } else { + sel = append(sel, q+"::text") + } + } + tx, err := conn.BeginTx(ctx, pgx.TxOptions{AccessMode: pgx.ReadOnly}) + if err != nil { + return 0, err + } + defer tx.Rollback(context.Background()) + if _, err := tx.Exec(ctx, fmt.Sprintf("SET LOCAL statement_timeout = %d", timeout.Milliseconds())); err != nil { + return 0, err + } + if _, err := tx.Exec(ctx, "SET LOCAL lock_timeout = '10s'"); err != nil { + return 0, err + } + rows, err := tx.Query(ctx, fmt.Sprintf("SELECT %s FROM %s WHERE %s", + strings.Join(sel, ", "), t.ref(), strings.Join(where, " OR "))) + if err != nil { + return 0, err + } + defer rows.Close() + var n int64 + // A table's places are added only once its read has finished: a read + // cut off by its time limit is skipped whole, not half-kept. + type hit struct { + ck colKey + row string + created pgtype.Timestamptz + numbers []string + } + var hits []hit + for rows.Next() { + var key string + var created pgtype.Timestamptz + vals := make([]*string, len(t.read)) + dest := []any{&key, &created} + for i := range vals { + dest = append(dest, &vals[i]) + } + if err := rows.Scan(dest...); err != nil { + return 0, err + } + n++ + for i, c := range t.read { + v := vals[i] + if v == nil { + continue + } + base := colKey{t.schema, t.name, c.name, ""} + if !isJSON(c) { + if ns := findAadhaar(*v); len(ns) > 0 { + hits = append(hits, hit{base, key, created, ns}) + } + continue + } + if *v == tooLarge { + st.large++ + continue + } + found, ok := walkExposure(*v) + if !ok { + st.bad++ + continue + } + // One value per path: a document holding a number at two paths + // is two places. + byPath := map[string][]string{} + var paths []string + for _, h := range found { + if _, seen := byPath[h.path]; !seen { + paths = append(paths, h.path) + } + byPath[h.path] = append(byPath[h.path], h.number) + } + for _, p := range paths { + ck := base + ck.path = p + hits = append(hits, hit{ck, key, created, byPath[p]}) + } + } + } + if err := rows.Err(); err != nil { + return 0, err + } + if err := tx.Commit(ctx); err != nil { + return 0, err + } + for _, h := range hits { + e.add(h.ck, h.row, h.created, h.numbers) + } + return n, nil +} + +// exposureFailure says why a table could not be read, as readFailure does, +// with the run's own time limit. +func exposureFailure(err error, timeout time.Duration) string { + if strings.Contains(err.Error(), "statement timeout") { + return "reading it took longer than " + timeout.String() + } + if strings.Contains(err.Error(), "lock timeout") { + return "it was locked by another session for longer than 10s" + } + return readFailure(err) +} + +// exposureLimits are what a run's report cannot claim. Every report carries +// them, found or not. +func exposureLimits(e *exposure, st exposureStats) []string { + l := []string{ + "Every row of every text, number, array and JSON column was read. The database passed on only rows holding twelve digits in Aadhaar's shape (four, four and four, with or without a space or hyphen), and each was then checked by its check digit.", + "A check digit shows a number is well-formed, not that it was issued, or whose it is.", + "There is no Aadhaar vault in FoxByte yet (point 13), so every number found is outside one.", + "Only the branch's tables were read. Query history, the audit log's statement text and the Blackbox are not read by this run.", + "Not read: views (they show what tables hold, and the tables are read), FoxByte's masks, binary columns, and dates, booleans and network addresses.", + "Since when is the first run that found a number in a place, or the row's created_at on a tracked table if that is earlier. A number there before the first run has no earlier date.", + } + capped := false + for _, c := range e.cols { + capped = capped || c.capped + } + if capped { + l = append(l, fmt.Sprintf("At most %d places are kept for a column and %d for a run; a column's counts are in full.", maxListedPerColumn, maxListed)) + } + if !e.grouped { + l = append(l, fmt.Sprintf("More than %d distinct numbers were found; places of numbers past those are kept without a person.", maxGrouped)) + } + if len(st.keyless) > 0 { + l = append(l, "Rows of tables with no primary key are named by their physical position (ctid), which changes when a row is updated: "+strings.Join(st.keyless, ", ")+".") + } + if st.large > 0 { + l = append(l, fmt.Sprintf("%d JSON document(s) larger than %d MB were not read.", st.large, maxExposureDocument>>20)) + } + if st.bad > 0 { + l = append(l, fmt.Sprintf("%d value(s) in JSON columns could not be parsed and were not read.", st.bad)) + } + return l +} + +// recordExposure writes a run, its columns and its places in one transaction. +func recordExposure(ctx context.Context, conn *pgx.Conn, r *branch.AadhaarRun, e *exposure) error { + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + skipped, _ := json.Marshal(r.Skipped) + lims, _ := json.Marshal(r.Limits) + if err := tx.QueryRow(ctx, `INSERT INTO bb.aadhaar_runs + (started_at, finished_at, actor, tables_read, columns_read, rows_read, numbers, people, listed, skipped, limits) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING id`, + r.Started, r.Finished, r.Actor, r.Tables, r.Columns, r.Rows, r.Numbers, r.People, r.Listed, skipped, lims).Scan(&r.ID); err != nil { + return fmt.Errorf("recording the run: %w", err) + } + + if n := len(e.order); n > 0 { + sc, tb, co, pa := make([]string, n), make([]string, n), make([]string, n), make([]string, n) + nums, rws, lst, capd := make([]int64, n), make([]int64, n), make([]int32, n), make([]bool, n) + for i, k := range e.order { + c := e.cols[k] + sc[i], tb[i], co[i], pa[i] = k.schema, k.table, k.column, k.path + nums[i], rws[i], lst[i], capd[i] = c.numbers, c.rows, int32(c.listed), c.capped + } + if _, err := tx.Exec(ctx, `INSERT INTO bb.aadhaar_columns + (run_id, schema_name, table_name, column_name, json_path, numbers, rows_with, listed, capped) + SELECT $1, s, t, c, p, n, r, l, x + FROM unnest($2::text[], $3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::int[], $9::boolean[]) + AS u(s, t, c, p, n, r, l, x)`, + r.ID, sc, tb, co, pa, nums, rws, lst, capd); err != nil { + return fmt.Errorf("recording the columns: %w", err) + } + } + + // Places in batches: one statement over arrays each, a place found + // before keeping when it was first seen. + const batch = 5000 + for from := 0; from < len(e.places); from += batch { + to := min(from+batch, len(e.places)) + n := to - from + sc, tb, co, pa, rk, l4 := make([]string, n), make([]string, n), make([]string, n), make([]string, n), make([]string, n), make([]string, n) + for i, p := range e.places[from:to] { + sc[i], tb[i], co[i], pa[i], rk[i], l4[i] = p.schema, p.table, p.column, p.path, p.row, p.last4 + } + if _, err := tx.Exec(ctx, `INSERT INTO bb.aadhaar_places AS a + (schema_name, table_name, column_name, json_path, row_key, last4, person, + first_run, last_run, first_seen, last_seen, row_created) + SELECT s, t, c, p, k, l, pe, $9, $9, $10, $10, rc + FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::text[], $6::text[], $7::int[], $8::timestamptz[]) + AS u(s, t, c, p, k, l, pe, rc) + ON CONFLICT (schema_name, table_name, column_name, json_path, row_key, last4) DO UPDATE SET + person = EXCLUDED.person, last_run = EXCLUDED.last_run, last_seen = EXCLUDED.last_seen, + row_created = EXCLUDED.row_created`, + sc, tb, co, pa, rk, l4, e.person[from:to], e.created[from:to], r.ID, r.Started); err != nil { + return fmt.Errorf("recording the places: %w", err) + } + } + return tx.Commit(ctx) +} diff --git a/enterprise/pii/aadhaar_test.go b/enterprise/pii/aadhaar_test.go new file mode 100644 index 0000000..705e578 --- /dev/null +++ b/enterprise/pii/aadhaar_test.go @@ -0,0 +1,184 @@ +//go:build enterprise + +// SPDX-License-Identifier: LicenseRef-FoxByte-Enterprise-1.0 + +package pii + +import ( + "reflect" + "strconv" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgtype" +) + +// Synthetic numbers only: eleven digits chosen here, and the twelfth computed +// to pass the check. +var ( + aadA = verhoeffDigit("23456789012") + aadB = verhoeffDigit("98765432109") +) + +func spaced(d string) string { return d[:4] + " " + d[4:8] + " " + d[8:] } + +func TestFindAadhaar(t *testing.T) { + bad := aadA[:11] + string(rune('0'+(int(aadA[11]-'0')+1)%10)) + cases := []struct { + in string + want []string + }{ + {aadA, []string{aadA}}, + {"Aadhaar " + spaced(aadA) + ", verified", []string{aadA}}, + {"id " + aadA[:4] + "-" + aadA[4:8] + "-" + aadA[8:] + ".", []string{aadA}}, + {aadA + " and " + spaced(aadB) + " and " + aadA, []string{aadA, aadB}}, // each once + {"num:" + aadA + ".00", []string{aadA}}, // a numeric's decimals end it + {bad, nil}, // the check digit fails + {"1" + aadA[1:], nil}, // Aadhaar never starts with 0 or 1 + {"9" + aadA, nil}, // inside a longer number + {aadA + "7", nil}, // inside a longer number + // A space between digits joins them (a card's or a phone's groups), + // so a number run on after "+91 " reads as part of a longer one. + {"+91 " + aadA, nil}, + {"4111 " + spaced(aadA), nil}, // a card's groups run on + {spaced(aadA) + " 1234", nil}, // and on + {"", nil}, + } + for _, c := range cases { + if got := findAadhaar(c.in); !reflect.DeepEqual(got, c.want) { + t.Errorf("findAadhaar(%q) = %v, want %v", c.in, findAadhaar(c.in), c.want) + } + } +} + +// The database's filter must pass on every value the check keeps, or a +// number is lost before it is looked at. +func TestShapeFilterCoversTheCheck(t *testing.T) { + for _, v := range []string{aadA, spaced(aadA), aadA[:4] + "-" + aadA[4:8] + "-" + aadA[8:], aadA[:4] + " " + aadA[4:]} { + if len(findAadhaar(v)) == 1 && !shapeRE.MatchString(v) { + t.Errorf("%q is kept by the check but not passed by the filter", v) + } + } + if shapePG != shapeRE.String() { + t.Errorf("the database's filter %q and the check's %q differ", shapePG, shapeRE.String()) + } +} + +func TestWalkExposure(t *testing.T) { + doc := `{"kyc": {"aadhaar": "` + spaced(aadA) + `", "pan": "ABCPE1234F"}, + "notes": ["called", "said ` + aadB + `"], + "by_id": {"` + aadA + `": {"name": "x", "uid": ` + aadB + `}}, + "status": true}` + hits, ok := walkExposure(doc) + if !ok { + t.Fatal("a JSON document did not parse") + } + want := []docHit{ + {"$.by_id.*", aadA}, // a key holding the number: found, and the path does not keep it + {"$.by_id.*.uid", aadB}, // and what is under it + {"$.kyc.aadhaar", aadA}, + {"$.notes[*]", aadB}, + } + if !reflect.DeepEqual(hits, want) { + t.Errorf("walkExposure = %v, want %v", hits, want) + } + for _, h := range hits { + if strings.ContainsAny(h.path, "0123456789") { + t.Errorf("a path keeps digits: %q", h.path) + } + } + if _, ok := walkExposure("{not json"); ok { + t.Error("a broken document parsed") + } +} + +func TestSafeKey(t *testing.T) { + for in, want := range map[string]string{ + "aadhaar": "$.aadhaar", + "line 2": `$."line 2"`, + // Eight digits across hyphens: hidden too. A path that loses a date + // is a smaller loss than one that keeps a number. + "2024-11-12": "$.*", + "order_1234567": `$.order_1234567`, + aadA: "$.*", + spaced(aadA): "$.*", + } { + if got := safeKey("$", in); got != want { + t.Errorf("safeKey(%q) = %q, want %q", in, safeKey("$", in), want) + } + } +} + +func TestRowKeySQL(t *testing.T) { + tb := table{schema: "public", name: "t", kind: "r"} + if got, keyless := rowKeySQL(tb, []string{"id"}); got != `'id=' || "id"::text` || keyless { + t.Errorf("one column: %s %v", got, keyless) + } + got, _ := rowKeySQL(tb, []string{"a", "b"}) + if got != `'(a, b)=(' || coalesce("a"::text, 'NULL') || ', ' || coalesce("b"::text, 'NULL') || ')'` { + t.Errorf("two columns: %s", got) + } + if got, keyless := rowKeySQL(tb, nil); got != "'ctid=' || ctid::text" || !keyless { + t.Errorf("no key: %s %v", got, keyless) + } + tb.kind = "p" + if got, _ := rowKeySQL(tb, nil); !strings.Contains(got, "tableoid::regclass") { + t.Errorf("a partitioned table's position names its partition: %s", got) + } +} + +func TestExposureGroupsAndCaps(t *testing.T) { + e := newExposure() + var none pgtype.Timestamptz + notes := colKey{"public", "tickets", "notes", ""} + cust := colKey{"public", "customers", "aadhaar", ""} + e.add(cust, "id=1", none, []string{aadA}) + e.add(cust, "id=2", none, []string{aadB}) + e.add(notes, "id=9", none, []string{aadA, aadB}) + e.add(notes, "id=9", none, []string{aadA}) // the same place again + if len(e.places) != 4 { + t.Fatalf("places = %d, want 4", len(e.places)) + } + // One person per number, across tables. + if e.person[0] != e.person[2] || e.person[1] != e.person[3] || e.person[0] == e.person[1] { + t.Errorf("persons = %v", e.person) + } + if e.places[2].last4 != aadA[8:] { + t.Errorf("a place keeps the last four digits, got %q", e.places[2].last4) + } + if st := e.cols[notes]; st.numbers != 3 || st.rows != 2 || st.listed != 2 { + t.Errorf("notes: %+v", *st) + } + + // Past a column's cap, numbers are counted and not kept. + e = newExposure() + big := colKey{"public", "big", "uid", ""} + for i := 0; i < maxListedPerColumn+5; i++ { + e.add(big, "id="+strconv.Itoa(i), none, []string{aadA}) + } + st := e.cols[big] + if st.listed != maxListedPerColumn || !st.capped || st.numbers != int64(maxListedPerColumn+5) { + t.Errorf("capped column: %+v", *st) + } + if lim := strings.Join(exposureLimits(e, exposureStats{}), " "); !strings.Contains(lim, "places are kept for a column") { + t.Errorf("a capped run does not say so: %s", lim) + } +} + +// Nothing a run keeps may hold a full number. +func TestExposureKeepsNoNumber(t *testing.T) { + e := newExposure() + var none pgtype.Timestamptz + hits, _ := walkExposure(`{"by": {"` + aadA + `": "` + aadB + `"}}`) + for _, h := range hits { + e.add(colKey{"public", "docs", "body", h.path}, "id=1", none, []string{h.number}) + } + for _, p := range e.places { + all := p.schema + p.table + p.column + p.path + p.row + p.last4 + for _, n := range []string{aadA, aadB} { + if strings.Contains(all, n) || strings.Contains(all, n[:8]) { + t.Errorf("a place keeps a number: %+v", p) + } + } + } +} diff --git a/enterprise/schema/pii.sql b/enterprise/schema/pii.sql index 205d2de..932fc39 100644 --- a/enterprise/schema/pii.sql +++ b/enterprise/schema/pii.sql @@ -130,6 +130,68 @@ CREATE OR REPLACE TRIGGER pii_column_reviews_append_only BEFORE UPDATE OR DELETE CREATE OR REPLACE TRIGGER pii_column_reviews_no_truncate BEFORE TRUNCATE ON bb.pii_column_reviews FOR EACH STATEMENT EXECUTE FUNCTION bb.pii_append_only(); +-- The Aadhaar exposure report (point 4.5): every Aadhaar number in the +-- branch's tables, found by reading every row, and kept as places. A place is +-- where a number sits (a column, or a JSON path in one, and the row's primary +-- key) and the number's last four digits, which UIDAI's own masked form shows. +-- Never the number, and never a hash of it: a hash of Aadhaar is the reference +-- key UIDAI forbids, and a number has few enough values to be found from one. +-- +-- A place keeps when a run first and last saw it, so the report can say since +-- when a number has been exposed. person groups the places of one number, +-- within the run that last saw them, by the number held in memory for the run +-- only: it is a run's numbering, not an identity kept across runs. +CREATE TABLE IF NOT EXISTS bb.aadhaar_runs ( + id bigserial PRIMARY KEY, + started_at timestamptz NOT NULL, + finished_at timestamptz NOT NULL, + actor text NOT NULL DEFAULT '', + tables_read integer NOT NULL, + columns_read integer NOT NULL, + rows_read bigint NOT NULL, -- rows the database's filter passed on + numbers bigint NOT NULL, -- Aadhaar numbers found, place by place + people integer NOT NULL, -- distinct numbers + listed integer NOT NULL, -- places kept + skipped jsonb NOT NULL DEFAULT '[]', -- [{object, reason}] + limits jsonb NOT NULL DEFAULT '[]' +); + +-- What each run found in each column or path, counted in full even where +-- only the first places were kept (capped). +CREATE TABLE IF NOT EXISTS bb.aadhaar_columns ( + run_id bigint NOT NULL REFERENCES bb.aadhaar_runs (id), + schema_name text NOT NULL, + table_name text NOT NULL, + column_name text NOT NULL, + json_path text NOT NULL DEFAULT '', + numbers bigint NOT NULL, + rows_with bigint NOT NULL, + listed integer NOT NULL, + capped boolean NOT NULL DEFAULT false, + PRIMARY KEY (run_id, schema_name, table_name, column_name, json_path) +); + +CREATE TABLE IF NOT EXISTS bb.aadhaar_places ( + id bigserial PRIMARY KEY, + schema_name text NOT NULL, + table_name text NOT NULL, + column_name text NOT NULL, + json_path text NOT NULL DEFAULT '', + row_key text NOT NULL, -- id=42, or (a, b)=(1, x); ctid=(0,7) for a table with no primary key + last4 text NOT NULL CHECK (last4 ~ '^[0-9]{4}$'), + person integer NOT NULL, + first_run bigint NOT NULL REFERENCES bb.aadhaar_runs (id), + last_run bigint NOT NULL REFERENCES bb.aadhaar_runs (id), + first_seen timestamptz NOT NULL, + last_seen timestamptz NOT NULL, + row_created timestamptz, -- the row's created_at, on a tracked table (point 6.1) + UNIQUE (schema_name, table_name, column_name, json_path, row_key, last4) +); +CREATE INDEX IF NOT EXISTS aadhaar_places_run ON bb.aadhaar_places (last_run, person); + +REVOKE ALL ON bb.aadhaar_runs, bb.aadhaar_columns, bb.aadhaar_places FROM PUBLIC; +REVOKE ALL ON SEQUENCE bb.aadhaar_runs_id_seq, bb.aadhaar_places_id_seq FROM PUBLIC; + REVOKE ALL ON bb.pii_scans, bb.pii_findings, bb.pii_decisions, bb.pii_columns, bb.pii_column_reviews FROM PUBLIC; REVOKE ALL ON SEQUENCE bb.pii_scans_id_seq, bb.pii_findings_id_seq, bb.pii_decisions_id_seq, bb.pii_columns_id_seq, bb.pii_column_reviews_id_seq FROM PUBLIC; diff --git a/internal/auth/audit.go b/internal/auth/audit.go index 32f6cb2..a10e80a 100644 --- a/internal/auth/audit.go +++ b/internal/auth/audit.go @@ -119,6 +119,9 @@ const ( // A new column or JSON path accepted as holding nothing to classify // (point 4.7): who, and where. EvPersonalDataAccepted = "personal-data.accepted" + // The Aadhaar exposure report run (point 4.5): which branch, and how much + // it found, as counts. Never a place, a row or a digit. + EvAadhaarExposure = "personal-data.aadhaar" // The access review report produced (point 4.8): who asked, and what it // covered. A report of who can reach what is itself worth knowing about. EvAccessReviewed = "access.reviewed" diff --git a/internal/branch/aadhaar.go b/internal/branch/aadhaar.go new file mode 100644 index 0000000..c7c9ef4 --- /dev/null +++ b/internal/branch/aadhaar.go @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package branch + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/thefoxbyte/foxbyte/internal/edition" +) + +// The Aadhaar exposure report (point 4.5): which person's Aadhaar is exposed +// in a branch, where, and since when. +// +// A run reads every row of every text, number and JSON column, and keeps each +// place a number sits as its column (or JSON path), the row's primary key and +// the number's last four digits. Never the number and never a hash of it +// (UIDAI forbids hashes of Aadhaar as reference keys, and twelve digits with a +// check digit are few enough to be found from any hash). Places of the same +// number are grouped as one person by the run that saw them, holding the +// numbers in memory and nowhere else. +// +// Running it is Enterprise (enterprise/pii supplies it), under the +// personal-data licence feature; reading what runs found is free, as the +// scan's findings are. The tables are the branch's (bb.aadhaar_*), so a +// branch's report travels with it. + +// AadhaarRunOptions is what a run is asked to do. +type AadhaarRunOptions struct { + Actor string // who asked, kept with the run + // TableTimeout bounds one table's read; DefaultAadhaarTableTimeout when 0. + TableTimeout time.Duration +} + +// DefaultAadhaarTableTimeout is how long one table's read may take, unless told. +const ( + DefaultAadhaarTableTimeout = 5 * time.Minute + MaxAadhaarTableTimeout = time.Hour +) + +// AadhaarRun is one run: when, by whom, what it read and what it found. +type AadhaarRun struct { + ID int64 `json:"id"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished"` + Actor string `json:"actor"` + Tables int `json:"tables"` + Columns int `json:"columns"` + Rows int64 `json:"rows"` // rows the database's filter passed on: twelve digits in Aadhaar's shape + Numbers int64 `json:"numbers"` // numbers found, place by place + People int `json:"people"` // distinct numbers + Listed int `json:"listed"` // places kept + Skipped []PIISkip `json:"skipped"` + // Limits are what the run could not read or cannot claim, in sentences. + Limits []string `json:"limits"` +} + +// AadhaarColumn is what a run found in one column, or one JSON path in one. +type AadhaarColumn struct { + Schema string `json:"schema"` + Table string `json:"table"` + Column string `json:"column"` + Path string `json:"path"` + Numbers int64 `json:"numbers"` + Rows int64 `json:"rows"` + Listed int `json:"listed"` + // Capped says only the first Listed places were kept: the column holds + // more numbers than one report lists. + Capped bool `json:"capped"` + // Since is the earliest any of its kept places has been exposed. + Since *time.Time `json:"since"` +} + +// Where is the column's location as a reader writes it. +func (c AadhaarColumn) Where() string { + return PIIFinding{Schema: c.Schema, Table: c.Table, Column: c.Column, Path: c.Path}.Where() +} + +// AadhaarPlace is one number in one place. +type AadhaarPlace struct { + ID int64 `json:"id"` + Schema string `json:"schema"` + Table string `json:"table"` + Column string `json:"column"` + Path string `json:"path"` + Row string `json:"row"` // the row's primary key: id=42 + Last4 string `json:"last4"` // the number's last four digits + Masked string `json:"masked"` // XXXX XXXX 1234 + // Person is the run's number for whose Aadhaar this is: places with the + // same person hold the same number. 0 when the run held too many numbers + // to group them all. + Person int `json:"person"` + FirstSeen time.Time `json:"firstSeen"` + LastSeen time.Time `json:"lastSeen"` + RowCreated *time.Time `json:"rowCreated"` // the row's created_at, on a tracked table + // Since is the earliest the number is known to have been here: the row's + // creation if it is tracked and earlier, else the first run that saw it. + Since time.Time `json:"since"` + // Gone says the latest run read this place's table in full and did not + // find the number there again. + Gone bool `json:"gone"` +} + +// Where is the place as a reader writes it: schema.table.column [path] row. +func (p AadhaarPlace) Where() string { + return PIIFinding{Schema: p.Schema, Table: p.Table, Column: p.Column, Path: p.Path}.Where() + " " + p.Row +} + +// AadhaarPerson is one number, by its masked form, and where it is. +type AadhaarPerson struct { + Person int `json:"person"` + Masked string `json:"masked"` + Places int `json:"places"` // places kept for it + Since time.Time `json:"since"` + // Where are its columns (schema.table.column [path]), each once. + Where []string `json:"where"` +} + +// AadhaarReport is a branch's exposure now: the latest run, what it found +// column by column, and the people whose numbers it found. +type AadhaarReport struct { + Branch string `json:"branch"` + Run *AadhaarRun `json:"run"` // nil before the first run + Columns []AadhaarColumn `json:"columns"` + // People are the numbers found, those in the most places first, up to + // MaxAadhaarPeople; PeopleListed is how many have a kept place. + People []AadhaarPerson `json:"people"` + PeopleListed int `json:"peopleListed"` + // Gone counts places earlier runs found that the latest no longer does. + Gone int `json:"gone"` +} + +// MaxAadhaarPeople is the most people a report returns; the places are read +// with AadhaarPlaces. +const MaxAadhaarPeople = 500 + +// MaskAadhaar is UIDAI's masked form of a number: its last four digits only. +func MaskAadhaar(last4 string) string { return "XXXX XXXX " + last4 } + +var runAadhaar func(name string, o AadhaarRunOptions) (AadhaarRun, error) + +// SetAadhaarReporter installs the run. Called from enterprise/pii's init, and +// from nowhere else. +func SetAadhaarReporter(fn func(name string, o AadhaarRunOptions) (AadhaarRun, error)) { + runAadhaar = fn +} + +// CheckAadhaarRunOptions fills in the defaults and refuses what cannot be right. +func CheckAadhaarRunOptions(o AadhaarRunOptions) (AadhaarRunOptions, error) { + if o.TableTimeout == 0 { + o.TableTimeout = DefaultAadhaarTableTimeout + } + if o.TableTimeout < time.Second || o.TableTimeout > MaxAadhaarTableTimeout { + return o, fmt.Errorf("%w: a table's read may take 1s to %s, not %s", ErrInvalidRequest, MaxAadhaarTableTimeout, o.TableTimeout) + } + return o, nil +} + +// RunAadhaarExposure reads a branch for Aadhaar numbers and returns its +// report afterwards. +func RunAadhaarExposure(name string, o AadhaarRunOptions) (AadhaarReport, error) { + if err := requireFeature(edition.PersonalData); err != nil { + return AadhaarReport{}, err + } + if runAadhaar == nil { + return AadhaarReport{}, ErrPersonalDataNotLicensed + } + o, err := CheckAadhaarRunOptions(o) + if err != nil { + return AadhaarReport{}, err + } + if _, err := runAadhaar(name, o); err != nil { + return AadhaarReport{}, err + } + return AadhaarExposure(name) +} + +// latestAadhaarRun opens a branch and reads its latest run; nil, with no +// error, when it has none or no report tables yet. +func latestAadhaarRun(ctx context.Context, name string) (*pgx.Conn, *AadhaarRun, error) { + conn, err := OpenSuper(ctx, name) + if err != nil { + return nil, nil, err + } + var have bool + if err := conn.QueryRow(ctx, `SELECT to_regclass('bb.aadhaar_runs') IS NOT NULL AND to_regclass('bb.aadhaar_places') IS NOT NULL + AND to_regclass('bb.aadhaar_columns') IS NOT NULL`).Scan(&have); err != nil { + conn.Close(context.Background()) + return nil, nil, err + } + if !have { + return conn, nil, nil + } + var r AadhaarRun + var skipped, limits []byte + err = conn.QueryRow(ctx, `SELECT id, started_at, finished_at, actor, tables_read, columns_read, rows_read, + numbers, people, listed, skipped, limits + FROM bb.aadhaar_runs ORDER BY id DESC LIMIT 1`). + Scan(&r.ID, &r.Started, &r.Finished, &r.Actor, &r.Tables, &r.Columns, &r.Rows, &r.Numbers, &r.People, &r.Listed, &skipped, &limits) + switch { + case errors.Is(err, pgx.ErrNoRows): + return conn, nil, nil + case err != nil: + conn.Close(context.Background()) + return nil, nil, err + } + if err := json.Unmarshal(skipped, &r.Skipped); err != nil { + conn.Close(context.Background()) + return nil, nil, err + } + if err := json.Unmarshal(limits, &r.Limits); err != nil { + conn.Close(context.Background()) + return nil, nil, err + } + return conn, &r, nil +} + +// AadhaarExposure reads a branch's report as its latest run left it. Free in +// every edition, as reading a scan's findings is. +func AadhaarExposure(name string) (AadhaarReport, error) { + if name == "" { + name = "main" + } + rep := AadhaarReport{Branch: name, Columns: []AadhaarColumn{}, People: []AadhaarPerson{}} + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, run, err := latestAadhaarRun(ctx, name) + if err != nil { + return rep, err + } + defer conn.Close(context.Background()) + if run == nil { + return rep, nil + } + rep.Run = run + + rows, err := conn.Query(ctx, `SELECT c.schema_name, c.table_name, c.column_name, c.json_path, c.numbers, c.rows_with, c.listed, c.capped, + (SELECT min(least(p.first_seen, coalesce(p.row_created, p.first_seen))) FROM bb.aadhaar_places p + WHERE p.last_run = c.run_id AND (p.schema_name, p.table_name, p.column_name, p.json_path) + = (c.schema_name, c.table_name, c.column_name, c.json_path)) + FROM bb.aadhaar_columns c WHERE c.run_id = $1 + ORDER BY c.numbers DESC, c.schema_name, c.table_name, c.column_name, c.json_path`, run.ID) + if err != nil { + return rep, err + } + for rows.Next() { + var c AadhaarColumn + if err := rows.Scan(&c.Schema, &c.Table, &c.Column, &c.Path, &c.Numbers, &c.Rows, &c.Listed, &c.Capped, &c.Since); err != nil { + rows.Close() + return rep, err + } + rep.Columns = append(rep.Columns, c) + } + rows.Close() + if err := rows.Err(); err != nil { + return rep, err + } + + // People, from the places the latest run kept: one row each, its columns + // each once. + rows, err = conn.Query(ctx, `SELECT person, min(last4), count(*)::int, + min(least(first_seen, coalesce(row_created, first_seen))), + array_agg(DISTINCT schema_name || '.' || table_name || '.' || column_name || CASE WHEN json_path <> '' THEN ' ' || json_path ELSE '' END), + count(*) OVER ()::int + FROM bb.aadhaar_places WHERE last_run = $1 AND person > 0 + GROUP BY person + ORDER BY count(*) DESC, 4, person + LIMIT $2`, run.ID, MaxAadhaarPeople) + if err != nil { + return rep, err + } + for rows.Next() { + var p AadhaarPerson + var last4 string + if err := rows.Scan(&p.Person, &last4, &p.Places, &p.Since, &p.Where, &rep.PeopleListed); err != nil { + rows.Close() + return rep, err + } + sort.Strings(p.Where) + p.Masked = MaskAadhaar(last4) + rep.People = append(rep.People, p) + } + rows.Close() + if err := rows.Err(); err != nil { + return rep, err + } + err = conn.QueryRow(ctx, `SELECT count(*) FROM bb.aadhaar_places p WHERE `+goneSQL, run.ID).Scan(&rep.Gone) + return rep, err +} + +// goneSQL is the condition, on bb.aadhaar_places p, for a place the latest +// run ($1) no longer found: one an earlier run kept, in a table the latest +// read in full (not skipped), in a column it was not capped in. A place in a +// table the latest run could not read was not looked for, which is not gone. +const goneSQL = `p.last_run < $1 + AND NOT EXISTS (SELECT 1 FROM bb.aadhaar_runs r, jsonb_array_elements(r.skipped) s + WHERE r.id = $1 AND s->>'object' = p.schema_name || '.' || p.table_name) + AND NOT EXISTS (SELECT 1 FROM bb.aadhaar_columns c + WHERE c.run_id = $1 AND c.capped AND (c.schema_name, c.table_name, c.column_name, c.json_path) + = (p.schema_name, p.table_name, p.column_name, p.json_path))` + +// AadhaarPlaceFilter narrows a list of places. +type AadhaarPlaceFilter struct { + Person int // one person's places; 0 for every person + Where string // one column: schema.table.column, or with " $.path" + Gone bool // places no longer found, rather than those found now + Limit int // at most this many; DefaultAadhaarPlaces when 0 +} + +// The most places one list returns, unless told. +const ( + DefaultAadhaarPlaces = 200 + MaxAadhaarPlaces = 10000 +) + +// AadhaarPlaces lists the places of a branch's latest run, or those it no +// longer found. Free, as reading always is. +func AadhaarPlaces(name string, f AadhaarPlaceFilter) ([]AadhaarPlace, error) { + if name == "" { + name = "main" + } + if f.Limit == 0 { + f.Limit = DefaultAadhaarPlaces + } + if f.Limit < 1 || f.Limit > MaxAadhaarPlaces { + return nil, fmt.Errorf("%w: a list is 1 to %d places, not %d", ErrInvalidRequest, MaxAadhaarPlaces, f.Limit) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, run, err := latestAadhaarRun(ctx, name) + if err != nil { + return nil, err + } + defer conn.Close(context.Background()) + out := []AadhaarPlace{} + if run == nil { + return out, nil + } + cond, args := "p.last_run = $1", []any{run.ID} + if f.Gone { + cond = goneSQL + } + if f.Person != 0 { + args = append(args, f.Person) + cond += fmt.Sprintf(" AND p.last_run = $1 AND p.person = $%d", len(args)) + } + if f.Where != "" { + args = append(args, f.Where) + cond += fmt.Sprintf(` AND p.schema_name || '.' || p.table_name || '.' || p.column_name + || CASE WHEN p.json_path <> '' THEN ' ' || p.json_path ELSE '' END = $%d`, len(args)) + } + args = append(args, f.Limit) + rows, err := conn.Query(ctx, `SELECT p.id, p.schema_name, p.table_name, p.column_name, p.json_path, p.row_key, p.last4, + CASE WHEN p.last_run = $1 THEN p.person ELSE 0 END, p.first_seen, p.last_seen, p.row_created + FROM bb.aadhaar_places p WHERE `+cond+fmt.Sprintf(` + ORDER BY p.person, p.schema_name, p.table_name, p.column_name, p.json_path, p.id LIMIT $%d`, len(args)), args...) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var p AadhaarPlace + if err := rows.Scan(&p.ID, &p.Schema, &p.Table, &p.Column, &p.Path, &p.Row, &p.Last4, + &p.Person, &p.FirstSeen, &p.LastSeen, &p.RowCreated); err != nil { + return nil, err + } + p.Masked = MaskAadhaar(p.Last4) + p.Since = p.FirstSeen + if p.RowCreated != nil && p.RowCreated.Before(p.Since) { + p.Since = *p.RowCreated + } + p.Gone = f.Gone + out = append(out, p) + } + return out, rows.Err() +} + +// Summary is what the security log keeps of a run: its coverage and how much +// it found. Counts only: never a place, a row or a digit. +func (r AadhaarReport) Summary() string { + if r.Run == nil { + return "read" + } + x := r.Run + return fmt.Sprintf("run %d: %d table(s), %d column(s), %d skipped; %d number(s) in %d place(s) kept, %d distinct", + x.ID, x.Tables, x.Columns, len(x.Skipped), x.Numbers, x.Listed, x.People) +} + +// ParseAadhaarWhere checks a column as a filter names it: schema.table.column, +// optionally followed by a space and a JSON path. +func ParseAadhaarWhere(s string) (string, error) { + col, _, _ := strings.Cut(s, " ") + if strings.Count(col, ".") < 2 { + return "", fmt.Errorf("%w: a column is schema.table.column, not %q", ErrInvalidRequest, s) + } + return s, nil +} diff --git a/internal/controlplane/aadhaar.go b/internal/controlplane/aadhaar.go new file mode 100644 index 0000000..c075c51 --- /dev/null +++ b/internal/controlplane/aadhaar.go @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controlplane + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/thefoxbyte/foxbyte/internal/auth" + "github.com/thefoxbyte/foxbyte/internal/branch" + "github.com/thefoxbyte/foxbyte/internal/edition" +) + +// The Aadhaar exposure report over HTTP (point 4.5). +// +// GET /api/branches/{name}/personal-data/aadhaar the latest run: by column, and by person +// POST /api/branches/{name}/personal-data/aadhaar/run read every row now, {"tableTimeout": seconds} (Enterprise) +// GET /api/branches/{name}/personal-data/aadhaar/places ?person=n&column=s.t.c&gone=1&limit=200 +// +// Under personal-data, so Manage, as the scan is (needFor): where a person's +// Aadhaar sits is as sensitive as anything FoxByte reports. A place is a row's +// key and the number's last four digits, never the number. Each run is in the +// security log with its counts. +func registerAadhaarExposure(mux *http.ServeMux) { + running := func(w http.ResponseWriter, r *http.Request) (string, bool) { + name := r.PathValue("name") + if _, err := branch.EnsureRunning(name); err != nil { + writeErr(w, 404, err) + return "", false + } + return name, true + } + fail := func(w http.ResponseWriter, err error) { + if errors.Is(err, branch.ErrInvalidRequest) { + writeErr(w, 400, err) + return + } + writeErr(w, 500, err) + } + mux.HandleFunc("GET /api/branches/{name}/personal-data/aadhaar", func(w http.ResponseWriter, r *http.Request) { + name, ok := running(w, r) + if !ok { + return + } + rep, err := branch.AadhaarExposure(name) + if err != nil { + fail(w, err) + return + } + writeJSON(w, 200, rep) + }) + mux.HandleFunc("POST /api/branches/{name}/personal-data/aadhaar/run", func(w http.ResponseWriter, r *http.Request) { + if !requireFeature(w, edition.PersonalData) { + return + } + var body struct { + TableTimeout int `json:"tableTimeout"` // seconds + } + if r.ContentLength != 0 { + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeErr(w, 400, fmt.Errorf("the body must be JSON: {\"tableTimeout\": seconds a table's read may take}")) + return + } + } + o, err := branch.CheckAadhaarRunOptions(branch.AadhaarRunOptions{TableTimeout: time.Duration(body.TableTimeout) * time.Second}) + if err != nil { + writeErr(w, 400, err) + return + } + name, ok := running(w, r) + if !ok { + return + } + u, _ := auth.UserFrom(r.Context()) + o.Actor = u.Email + rep, err := branch.RunAadhaarExposure(name, o) + if err != nil { + fail(w, err) + return + } + if secLog != nil { + secLog.Audit(auth.EvAadhaarExposure, u.Email, name, remoteIP(r), rep.Summary()) + } + writeJSON(w, 200, rep) + }) + mux.HandleFunc("GET /api/branches/{name}/personal-data/aadhaar/places", func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + var f branch.AadhaarPlaceFilter + for _, x := range []struct { + key string + to *int + }{{"person", &f.Person}, {"limit", &f.Limit}} { + if v := q.Get(x.key); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + writeErr(w, 400, fmt.Errorf("%s is a number, not %q", x.key, v)) + return + } + *x.to = n + } + } + if c := q.Get("column"); c != "" { + where, err := branch.ParseAadhaarWhere(c) + if err != nil { + writeErr(w, 400, err) + return + } + f.Where = where + } + f.Gone = q.Get("gone") == "1" || q.Get("gone") == "true" + name, ok := running(w, r) + if !ok { + return + } + ps, err := branch.AadhaarPlaces(name, f) + if err != nil { + fail(w, err) + return + } + writeJSON(w, 200, map[string]any{"branch": name, "places": ps}) + }) +} diff --git a/internal/controlplane/openapi.yaml b/internal/controlplane/openapi.yaml index faaf6ca..c2f9075 100644 --- a/internal/controlplane/openapi.yaml +++ b/internal/controlplane/openapi.yaml @@ -1056,6 +1056,76 @@ paths: "403": { description: Not the branch's owner or an admin. } "401": { $ref: "#/components/responses/Unauthorized" } + /api/branches/{name}/personal-data/aadhaar: + get: + tags: [branches] + summary: The Aadhaar exposure report as the latest run left it (the branch's owner or an admin). + description: >- + What the latest run read and could not read, what it found column by + column (counted in full), the people whose numbers it found (by the + number's last four digits, the most places first, up to 500), and how + many places earlier runs found that it no longer does. Never a full + number, and never a hash of one. Free in every edition; run is null + before the first run. + parameters: [{ $ref: "#/components/parameters/BranchName" }] + responses: + "200": { description: The report., content: { application/json: { schema: { $ref: "#/components/schemas/AadhaarReport" } } } } + "403": { description: Not the branch's owner or an admin. } + "404": { description: No such branch. } + "401": { $ref: "#/components/responses/Unauthorized" } + + /api/branches/{name}/personal-data/aadhaar/run: + post: + tags: [branches] + summary: Read every row of the branch for Aadhaar numbers now (Enterprise; the branch's owner or an admin). + description: >- + Every text, number, array and JSON column, in read-only transactions, + one a table with a time limit (300 seconds unless given). The database + passes on rows holding twelve digits in Aadhaar's shape; each is + checked by its check digit. Each place a number sits is kept as its + column or JSON path, the row's primary key and the number's last four + digits, with when it was first and last seen. The run is in the + security log with its counts. + parameters: [{ $ref: "#/components/parameters/BranchName" }] + requestBody: + content: + application/json: + schema: + type: object + properties: + tableTimeout: { type: integer, minimum: 1, maximum: 3600, description: "Seconds one table's read may take." } + responses: + "200": { description: Read; the report as it now is., content: { application/json: { schema: { $ref: "#/components/schemas/AadhaarReport" } } } } + "400": { description: A table timeout outside 1 to 3600 seconds. } + "403": { description: Not the branch's owner or an admin, or no licence covers personal-data discovery. } + "404": { description: No such branch. } + "401": { $ref: "#/components/responses/Unauthorized" } + + /api/branches/{name}/personal-data/aadhaar/places: + get: + tags: [branches] + summary: The places of the latest Aadhaar exposure run, or those it no longer found (the branch's owner or an admin). + parameters: + - { $ref: "#/components/parameters/BranchName" } + - { name: person, in: query, schema: { type: integer }, description: "One person's places, by the latest run's number for them." } + - { name: column, in: query, schema: { type: string }, description: "One column: schema.table.column, or with a space and a JSON path." } + - { name: gone, in: query, schema: { type: boolean }, description: "Places earlier runs found that the latest no longer does." } + - { name: limit, in: query, schema: { type: integer, minimum: 1, maximum: 10000, default: 200 } } + responses: + "200": + description: The places. + content: + application/json: + schema: + type: object + properties: + branch: { type: string } + places: { type: array, items: { $ref: "#/components/schemas/AadhaarPlace" } } + "400": { description: A person or limit that is not a number, or a column not written schema.table.column. } + "403": { description: Not the branch's owner or an admin. } + "404": { description: No such branch. } + "401": { $ref: "#/components/responses/Unauthorized" } + /api/branches/{name}/personal-data/columns/{id}/accept: post: tags: [branches] @@ -3330,6 +3400,69 @@ components: sessions: { type: object, additionalProperties: { type: integer }, description: By running branch. } lastAsserted: { type: string, format: date-time, nullable: true } limits: { type: array, items: { type: string } } + AadhaarReport: + type: object + properties: + branch: { type: string } + run: + type: object + nullable: true + properties: + id: { type: integer } + started: { type: string, format: date-time } + finished: { type: string, format: date-time } + actor: { type: string } + tables: { type: integer } + columns: { type: integer } + rows: { type: integer, description: "Rows holding something shaped like an Aadhaar number." } + numbers: { type: integer, description: "Numbers found, place by place." } + people: { type: integer, description: "Distinct numbers." } + listed: { type: integer, description: "Places kept." } + skipped: { type: array, items: { type: object, properties: { object: { type: string }, reason: { type: string } } } } + limits: { type: array, items: { type: string } } + columns: + type: array + items: + type: object + properties: + schema: { type: string } + table: { type: string } + column: { type: string } + path: { type: string } + numbers: { type: integer } + rows: { type: integer } + listed: { type: integer } + capped: { type: boolean, description: "Only the first places were kept." } + since: { type: string, format: date-time, nullable: true } + people: + type: array + items: + type: object + properties: + person: { type: integer } + masked: { type: string, example: "XXXX XXXX 0124" } + places: { type: integer } + since: { type: string, format: date-time } + where: { type: array, items: { type: string } } + peopleListed: { type: integer } + gone: { type: integer } + AadhaarPlace: + type: object + properties: + id: { type: integer } + schema: { type: string } + table: { type: string } + column: { type: string } + path: { type: string } + row: { type: string, example: "id=42" } + last4: { type: string } + masked: { type: string, example: "XXXX XXXX 0124" } + person: { type: integer, description: "The latest run's number for whose Aadhaar this is; 0 when ungrouped or gone." } + firstSeen: { type: string, format: date-time } + lastSeen: { type: string, format: date-time } + rowCreated: { type: string, format: date-time, nullable: true } + since: { type: string, format: date-time } + gone: { type: boolean } PersonalDataColumn: type: object properties: diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index fbb78b6..c03806d 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -123,6 +123,7 @@ func Serve(addr string) error { registerPersonalData(api) // /api/branches/{name}/personal-data* (point 4) registerPersonalDataReview(api) // /api/branches/{name}/personal-data/findings/* (point 4.6) registerPersonalDataInventory(api) // /api/branches/{name}/personal-data/{unclassified,columns/*} (point 4.7) + registerAadhaarExposure(api) // /api/branches/{name}/personal-data/aadhaar* (point 4.5) registerEditionSwitch(api, store) // /api/edition/switch (the Upgrade modal's Install button) // The paid edition's routes. Compiled out of the Standard build entirely, // where this call does nothing (internal/controlplane/enterprise_off.go). diff --git a/scripts/integration_test.sh b/scripts/integration_test.sh index f719382..a74e3f0 100644 --- a/scripts/integration_test.sh +++ b/scripts/integration_test.sh @@ -4057,6 +4057,99 @@ assert_eq "…and the table is still there; with the override, as the editor ask "$(pg pg-main "SELECT to_regclass('kt_renamed') IS NOT NULL")|$(curl -sk -H "$AUTH" -H 'Content-Type: application/json' -X POST -d '{"sql":"DROP TABLE \"public\".\"kt_renamed\"","allow_destructive":true}' https://localhost:8080/api/branches/main/query | rderr)|$(pg pg-main "SELECT to_regclass('kt_renamed') IS NULL")" "t||t" pg pg-main "SET bb.allow_destructive = on; DROP TABLE IF EXISTS kt_child, kt_parent, kt_ex, kt_renamed CASCADE" >/dev/null +echo "### 11ar. the Aadhaar exposure report: whose Aadhaar is exposed, where, and since when (point 4.5)" +# Synthetic numbers only, the same as 11r's: 234567890124, 345678901238 and +# 456789012341 pass Verhoeff. 4111 1111 1111 1111 is the usual test card, a +# sixteen-digit number that must not be read as an Aadhaar. +$S branch delete aadt >/dev/null 2>&1 +$S branch create aadt >/dev/null 2>&1 +pg pg-aadt "CREATE TABLE customers (id int PRIMARY KEY, name text, aadhaar text); + INSERT INTO customers VALUES (1, 'A Person', '2345 6789 0124'), (2, 'B Person', '456789012341'), (3, 'C Person', NULL); + -- created_at already there, so tracking it (below) keeps when each row was made. + CREATE TABLE tickets (id int PRIMARY KEY, notes text, created_at timestamptz); + INSERT INTO tickets VALUES + (9, 'customer read out 234567890124 on the call', '2026-01-15 10:00+00'), + (10, 'card 4111 1111 1111 1111, phone +91 98765 43210', '2026-01-16 10:00+00'), + (12, 'again: 4567-8901-2341', '2026-02-01 10:00+00'); + CREATE TABLE orders (id bigint PRIMARY KEY, meta jsonb); + INSERT INTO orders VALUES (1, '{\"kyc\": {\"uid\": 345678901238}, \"status\": \"open\"}'), (2, '{\"by\": {\"456789012341\": \"seen\"}}'); + CREATE TABLE nokey (note text); + INSERT INTO nokey VALUES ('345678901238'), ('nothing here');" >/dev/null +$S track tickets --branch aadt >/dev/null 2>&1 +LEDGER_BEFORE="$(pg pg-aadt "SELECT count(*) FROM bb.schema_ledger")" +RUNS_LOGGED="$($S audit --limit 1000 | grep -c 'personal-data.aadhaar')" +AR="$($S pii aadhaar run --branch aadt 2>&1)"; AR_RC=$? +assert_eq "fox pii aadhaar run reads every table and exits 0" \ + "$AR_RC|$(grep -c '^Aadhaar exposure on aadt: run [0-9]*' <<<"$AR")|$(grep -c '^[0-9]* table(s), [0-9]* column(s) read; [0-9]* row(s) held twelve digits in its shape. 7 number(s) found, 3 distinct; 7 place(s) kept.' <<<"$AR")" "0|1|1" +# The branch is made from main, so main's tables are read too: the counts of +# tables and rows are main's as well, and only what was found is pinned. +grep -q ' 7 number(s) found, 3 distinct; 7 place(s) kept.' <<<"$AR" || { echo "--- what it said ---"; head -12 <<<"$AR"; echo "--- end ---"; } +AJ="$($S pii aadhaar --branch aadt --json 2>/dev/null)" +assert_eq "…finds each number by column, inside JSON path by path, a key holding one hidden as *; not a card or a phone" \ + "$(hjson "print(' '.join(sorted(c['schema']+'.'+c['table']+'.'+c['column']+(' '+c['path'] if c['path'] else '')+':'+str(c['numbers']) for c in d['columns'])))" <<<"$AJ")" \ + "public.customers.aadhaar:2 public.nokey.note:1 public.orders.meta $.by.*:1 public.orders.meta $.kyc.uid:1 public.tickets.notes:2" +assert_eq "…groups the places of one number as one person, shown by its last four digits, the most places first" \ + "$(hjson "print([(p['masked'], p['places']) for p in d['people']])" <<<"$AJ")" \ + "[('XXXX XXXX 2341', 3), ('XXXX XXXX 0124', 2), ('XXXX XXXX 1238', 2)]" +P2341="$(hjson "print(d['people'][0]['person'])" <<<"$AJ")" +PJ2341="$($S pii aadhaar places --person "$P2341" --branch aadt --json 2>/dev/null)" +assert_eq "fox pii aadhaar places names a person's rows by their primary key" \ + "$(hjson "print(sorted(p['table']+(' '+p['path'] if p['path'] else '')+' '+p['row'] for p in d))" <<<"$PJ2341")" \ + "['customers id=2', 'orders $.by.* id=2', 'tickets id=12']" +P0124="$(hjson "print([p['person'] for p in d['people'] if p['masked'].endswith('0124')][0])" <<<"$AJ")" +assert_eq "…since when: a tracked row's created_at, when it is earlier than the first run" \ + "$(hjson "print([p['since'][:10] for p in d['people'] if p['masked'].endswith('0124')][0])" <<<"$AJ")|$($S pii aadhaar places --person "$P0124" --branch aadt --json | hjson "print(sorted((p['table'], p['rowCreated'] is not None) for p in d))")" \ + "2026-01-15|[('customers', False), ('tickets', True)]" +assert_eq "…a row of a table with no primary key named by its position, and the report says so" \ + "$($S pii aadhaar places --column public.nokey.note --branch aadt --json | hjson "print([p['row'][:6] for p in d])")|$(grep -c 'no primary key are named by their physical position (ctid).*public.nokey' <<<"$AR")" "['ctid=(']|1" +assert_eq "…and says what it cannot claim: the check digit, no vault yet, the logs not read" \ + "$(grep -c 'not that it was issued' <<<"$AR")|$(grep -c 'no Aadhaar vault in FoxByte yet' <<<"$AR")|$(grep -c 'Query history, the audit log' <<<"$AR")" "1|1|1" +# The point of the design: the report must not become one more copy of the +# numbers it finds, or a way to find them again. +assert_eq "no full number is stored, nor its first eight digits, in any of the report's tables" \ + "$(pg pg-aadt "SELECT count(*) FROM (SELECT r::text t FROM bb.aadhaar_runs r UNION ALL SELECT c::text FROM bb.aadhaar_columns c UNION ALL SELECT p::text FROM bb.aadhaar_places p) x + WHERE regexp_replace(t, '[ -]', '', 'g') ~ '(23456789|45678901|34567890)'")" "0" +assert_eq "…and a client role cannot read it" \ + "$(sudo docker exec -e PGPASSWORD=foxbyte pg-aadt psql -U dbadmin -d appdb -tAc "SET ROLE db_client; SELECT count(*) FROM bb.aadhaar_places" 2>&1 | grep -c 'permission denied')" "1" +assert_eq "a run reads only: the Blackbox records nothing for it" "$(pg pg-aadt "SELECT count(*) FROM bb.schema_ledger")" "$LEDGER_BEFORE" +assert_eq "…and it is in the security log, with counts and no digit of a number" \ + "$($S audit --limit 1000 | grep -c 'personal-data.aadhaar')|$($S audit --limit 1000 | grep 'personal-data.aadhaar' | grep -cE '0124|2341|1238')" "$((RUNS_LOGGED+1))|0" + +# A second run: a number removed is gone; a table it could not read is not. +# customers is held locked past the run's lock wait, so it is skipped, and its +# places are not looked for, which is not the same as not found. +RUN1="$(hjson "print(d['run']['id'])" <<<"$AJ")" +pg pg-aadt "UPDATE tickets SET notes = 'number removed' WHERE id = 9" >/dev/null +sudo docker exec -e PGPASSWORD=foxbyte pg-aadt psql -U dbadmin -d appdb -qc "BEGIN; LOCK TABLE customers IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(25); COMMIT;" >/dev/null 2>&1 & +LOCKER=$! +sleep 2 +AR2="$($S pii aadhaar run --branch aadt 2>&1)" +wait "$LOCKER" 2>/dev/null +AJ2="$($S pii aadhaar --branch aadt --json 2>/dev/null)" +assert_eq "a table locked past the run's wait is skipped and said so" \ + "$(grep -c 'not read: public.customers (it was locked by another session for longer than 10s)' <<<"$AR2")" "1" +assert_eq "…a number no longer there is gone, and the skipped table's places are not" \ + "$(hjson "print(d['gone'])" <<<"$AJ2")|$($S pii aadhaar places --gone --branch aadt --json | hjson "print([(p['table'], p['row'], p['masked']) for p in d])")" \ + "1|[('tickets', 'id=9', 'XXXX XXXX 0124')]" +assert_eq "…and a place found again keeps when it was first found" \ + "$(pg pg-aadt "SELECT first_run = $RUN1 AND last_run > $RUN1 FROM bb.aadhaar_places WHERE table_name = 'tickets' AND row_key = 'id=12'")" "t" +assert_eq "a table's time limit outside 1s to 1h is refused" \ + "$($S pii aadhaar run --branch aadt --table-timeout 500ms >/dev/null 2>&1; echo $?)|$(code "$KEY" -H 'Content-Type: application/json' -X POST -d '{"tableTimeout":4000}' https://localhost:8080/api/branches/aadt/personal-data/aadhaar/run)" "1|400" +AU=https://localhost:8080/api/branches/aadt/personal-data/aadhaar +assert_eq "the API runs and reads for the branch's owner or an admin, and no one else" \ + "$(code "$KEY" -X POST "$AU/run")|$(code "$KEY" "$AU")|$(code "$KEY" "$AU/places?person=$P2341")|$(code "$KA" "$AU")|$(code "$KA" -X POST "$AU/run")|$(code "$KA" https://localhost:8080/api/branches/main/personal-data/aadhaar)|$(code "$KA" -X POST https://localhost:8080/api/branches/main/personal-data/aadhaar/run)" \ + "200|200|200|404|404|403|403" +assert_eq "…refusing a person that is not a number, or a column not written schema.table.column" \ + "$(code "$KEY" "$AU/places?person=abc")|$(code "$KEY" "$AU/places?column=notes")|$(code "$KEY" "$AU/places?limit=20000")" "400|400|400" +assert_eq "…answering with the report, by person, last four digits only" \ + "$(as "$KEY" "$AU" | hjson "print(d['run']['actor'], d['people'][0]['masked'], len(d['columns']))")" "test@foxbyte.dev XXXX XXXX 2341 5" +$S branch delete aadt2 >/dev/null 2>&1 +$S branch create aadt2 --from aadt >/dev/null 2>&1 +assert_eq "the report travels with the branch: one made from it carries it" \ + "$($S pii aadhaar --branch aadt2 --json | hjson "print(d['run']['id'] == $(as "$KEY" "$AU" | hjson "print(d['run']['id'])"))")" "True" +$S branch delete aadt2 >/dev/null 2>&1 +$S branch delete aadt >/dev/null 2>&1 + echo "### 12. fox uninstall (B1)" # Removal used to be a list of commands to run by hand. This runs last: it takes # the stack apart, so nothing after it has a stack to use. diff --git a/web/src/api.ts b/web/src/api.ts index b5aedf1..923d183 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -567,6 +567,74 @@ export const getUnclassified = (branch: string) => export const acceptColumn = (branch: string, id: number, note = '') => req('POST', `${API}/api/branches/${branch}/personal-data/columns/${id}/accept`, note ? { note } : {}) as Promise +// The Aadhaar exposure report (point 4.5): every row read for Aadhaar +// numbers; each place kept by its row's key and the number's last four digits, +// grouped by person, never the number. +export type AadhaarRun = { + id: number + started: string + finished: string + actor: string + tables: number + columns: number + rows: number + numbers: number + people: number + listed: number + skipped: { object: string; reason: string }[] | null + limits: string[] | null +} +export type AadhaarColumn = { + schema: string + table: string + column: string + path: string + numbers: number + rows: number + listed: number + capped: boolean + since: string | null +} +export type AadhaarPerson = { person: number; masked: string; places: number; since: string; where: string[] | null } +export type AadhaarReport = { + branch: string + run: AadhaarRun | null + columns: AadhaarColumn[] | null + people: AadhaarPerson[] | null + peopleListed: number + gone: number +} +export type AadhaarPlace = { + id: number + schema: string + table: string + column: string + path: string + row: string + last4: string + masked: string + person: number + firstSeen: string + lastSeen: string + rowCreated: string | null + since: string + gone: boolean +} +export type AadhaarPlaceFilter = { person?: number; column?: string; gone?: boolean; limit?: number } +const aadhaarURL = (branch: string) => `${API}/api/branches/${branch}/personal-data/aadhaar` +export const getAadhaar = (branch: string) => req('GET', aadhaarURL(branch)) as Promise +export const runAadhaar = (branch: string, tableTimeout?: number) => + req('POST', `${aadhaarURL(branch)}/run`, tableTimeout ? { tableTimeout } : {}) as Promise +export const getAadhaarPlaces = (branch: string, f: AadhaarPlaceFilter) => { + const q = new URLSearchParams() + if (f.person) q.set('person', String(f.person)) + if (f.column) q.set('column', f.column) + if (f.gone) q.set('gone', '1') + if (f.limit) q.set('limit', String(f.limit)) + const qs = q.toString() + return req('GET', `${aadhaarURL(branch)}/places${qs ? '?' + qs : ''}`) as Promise<{ branch: string; places: AadhaarPlace[] | null }> +} + // The data grid's save (point 12.2): rows added, changed and deleted, applied // together as you, or not at all. export const saveRows = (branch: string, schema: string, table: string, changes: import('./gridquery').RowChange[]) => diff --git a/web/src/components/AppLayout.tsx b/web/src/components/AppLayout.tsx index c67dc9f..21450f8 100644 --- a/web/src/components/AppLayout.tsx +++ b/web/src/components/AppLayout.tsx @@ -46,6 +46,7 @@ export const APP_NAV: NavSection[] = [ label: 'Compliance', items: [ { to: '/personal-data', label: 'Personal data', icon: I.IconPersonalData }, + { to: '/aadhaar-exposure', label: 'Aadhaar exposure', icon: I.IconPersonalData }, { to: '/masking', label: 'Masking', icon: I.IconPersonalData }, { to: '/access-review', label: 'Access review', icon: I.IconKey }, { to: '/applications', label: 'Applications', icon: I.IconKey }, diff --git a/web/src/main.tsx b/web/src/main.tsx index e81c69d..5c19900 100644 --- a/web/src/main.tsx +++ b/web/src/main.tsx @@ -20,6 +20,7 @@ import Import from './pages/Import' import Pipelines from './pages/Pipelines' import PersonalData from './pages/PersonalData' import AccessReview from './pages/AccessReview' +import AadhaarExposure from './pages/AadhaarExposure' import Applications from './pages/Applications' import AuditLog from './pages/AuditLog' import Hardening from './pages/Hardening' @@ -119,6 +120,7 @@ const router = createBrowserRouter([ { path: 'import', element: }, { path: 'pipelines', element: }, { path: 'personal-data', element: }, + { path: 'aadhaar-exposure', element: }, { path: 'access-review', element: }, { path: 'applications', element: }, { path: 'audit-log', element: }, diff --git a/web/src/pages/AadhaarExposure.test.tsx b/web/src/pages/AadhaarExposure.test.tsx new file mode 100644 index 0000000..97f27c4 --- /dev/null +++ b/web/src/pages/AadhaarExposure.test.tsx @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { MemoryRouter } from 'react-router-dom' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('../api', async () => { + const actual = await vi.importActual('../api') + return { + ...actual, + getAadhaar: vi.fn(), + runAadhaar: vi.fn(), + getAadhaarPlaces: vi.fn(), + getBranches: vi.fn(async () => [{ name: 'main' }, { name: 'dev' }]), + getStatus: vi.fn(), + } +}) + +import { getAadhaar, getAadhaarPlaces, getStatus, runAadhaar, type AadhaarReport, type Status } from '../api' +import AadhaarExposure from './AadhaarExposure' + +const status = (features: string[], edition: 'standard' | 'enterprise' = 'enterprise') => + ({ edition, features }) as unknown as Status + +const report: AadhaarReport = { + branch: 'main', + run: { + id: 3, started: '2026-11-12T10:00:00Z', finished: '2026-11-12T10:00:02Z', actor: 'owner@example.com', + tables: 4, columns: 9, rows: 5, numbers: 6, people: 3, listed: 6, + skipped: [{ object: 'public.remote', reason: 'a foreign table: its data is held somewhere else' }], + limits: ['There is no Aadhaar vault in FoxByte yet (point 13), so every number found is outside one.'], + }, + columns: [ + { schema: 'public', table: 'customers', column: 'aadhaar', path: '', numbers: 3, rows: 3, listed: 3, capped: false, since: '2026-10-01T00:00:00Z' }, + { schema: 'public', table: 'orders', column: 'meta', path: '$.kyc.uid', numbers: 2, rows: 2, listed: 2, capped: false, since: '2026-11-12T10:00:00Z' }, + ], + people: [ + { person: 1, masked: 'XXXX XXXX 0124', places: 3, since: '2026-10-01T00:00:00Z', where: ['public.customers.aadhaar', 'public.orders.meta $.kyc.uid', 'public.tickets.notes'] }, + { person: 2, masked: 'XXXX XXXX 5555', places: 1, since: '2026-11-12T10:00:00Z', where: ['public.customers.aadhaar'] }, + ], + peopleListed: 2, + gone: 1, +} + +function page() { + render() +} + +describe('Aadhaar exposure page', () => { + beforeEach(() => { + vi.mocked(getAadhaar).mockReset() + vi.mocked(runAadhaar).mockReset() + vi.mocked(getAadhaarPlaces).mockReset() + vi.mocked(getStatus).mockResolvedValue(status(['personal-data'])) + }) + afterEach(cleanup) + + it('says a branch has no report yet', async () => { + vi.mocked(getAadhaar).mockResolvedValue({ branch: 'main', run: null, columns: [], people: [], peopleListed: 0, gone: 0 }) + page() + expect(await screen.findByText(/has no Aadhaar exposure report yet/)).toBeTruthy() + }) + + it('shows each column and each person by the last four digits, with what it cannot claim', async () => { + vi.mocked(getAadhaar).mockResolvedValue(report) + page() + const cols = await screen.findByRole('table', { name: 'Aadhaar by column' }) + expect(cols.textContent).toContain('public.customers.aadhaar') + expect(cols.textContent).toContain('public.orders.meta $.kyc.uid') + const people = screen.getByRole('table', { name: 'Aadhaar by person' }) + expect(people.textContent).toContain('XXXX XXXX 0124') + expect(people.textContent).toContain('public.tickets.notes') + expect(screen.getByTestId('aadhaar-summary').textContent).toContain('6 number(s) found, 3 distinct') + expect(screen.getByText(/public.remote/)).toBeTruthy() + expect(screen.getByText(/no Aadhaar vault in FoxByte yet/)).toBeTruthy() + expect(screen.getByText(/1 place\(s\) earlier runs found are no longer there/)).toBeTruthy() + }) + + it("opens a person's rows", async () => { + vi.mocked(getAadhaar).mockResolvedValue(report) + vi.mocked(getAadhaarPlaces).mockResolvedValue({ + branch: 'main', + places: [{ id: 7, schema: 'public', table: 'tickets', column: 'notes', path: '', row: 'id=9', last4: '0124', masked: 'XXXX XXXX 0124', + person: 1, firstSeen: '2026-11-12T10:00:00Z', lastSeen: '2026-11-12T10:00:00Z', rowCreated: '2026-10-01T00:00:00Z', + since: '2026-10-01T00:00:00Z', gone: false }], + }) + page() + fireEvent.click(await screen.findByRole('button', { name: 'Rows of person 1' })) + const places = await screen.findByRole('table', { name: 'Aadhaar places' }) + expect(vi.mocked(getAadhaarPlaces)).toHaveBeenCalledWith('main', { person: 1 }) + expect(places.textContent).toContain('id=9') + expect(places.textContent).toContain('row created') + }) + + it('runs the report with the time limit given', async () => { + vi.mocked(getAadhaar).mockResolvedValue({ branch: 'main', run: null, columns: [], people: [], peopleListed: 0, gone: 0 }) + vi.mocked(runAadhaar).mockResolvedValue(report) + page() + const btn = await screen.findByRole('button', { name: 'Run the report' }) + fireEvent.change(screen.getByLabelText('Seconds per table'), { target: { value: '60' } }) + fireEvent.click(btn) + await waitFor(() => expect(vi.mocked(runAadhaar)).toHaveBeenCalledWith('main', 60)) + expect(await screen.findByRole('table', { name: 'Aadhaar by column' })).toBeTruthy() + }) + + it('says a run is Enterprise and still shows an earlier report', async () => { + vi.mocked(getStatus).mockResolvedValue(status([], 'standard')) + vi.mocked(getAadhaar).mockResolvedValue(report) + page() + expect(await screen.findByRole('table', { name: 'Aadhaar by column' })).toBeTruthy() + const btn = screen.getByRole('button', { name: /Run the report \(Enterprise\)/ }) as HTMLButtonElement + expect(btn.disabled).toBe(true) + }) +}) diff --git a/web/src/pages/AadhaarExposure.tsx b/web/src/pages/AadhaarExposure.tsx new file mode 100644 index 0000000..9214287 --- /dev/null +++ b/web/src/pages/AadhaarExposure.tsx @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +import { useCallback, useEffect, useState } from 'react' +import { + getAadhaar, getAadhaarPlaces, runAadhaar, + type AadhaarColumn, type AadhaarPlace, type AadhaarPlaceFilter, type AadhaarReport, type Branch, +} from '../api' +import { useFeatures, why } from '../features' +import { useBranches } from '../useBranches' + +// The Aadhaar exposure report (point 4.5): which person's Aadhaar is exposed +// in a branch, where, and since when. A run reads every row; the engine keeps +// each place as its row's key and the number's last four digits, so nothing +// here can show a whole number. Reading the report is free; a run is +// Enterprise. + +const day = (iso: string | null | undefined) => (iso ? new Date(iso).toLocaleDateString() : '–') +const where = (c: { schema: string; table: string; column: string; path: string }) => + `${c.schema}.${c.table}.${c.column}${c.path ? ' ' + c.path : ''}` + +export default function AadhaarExposure() { + const [branch, setBranch] = useState('main') + const [report, setReport] = useState(null) + const [loading, setLoading] = useState(true) + const [running, setRunning] = useState(false) + const [timeout, setTimeout_] = useState('300') + const [err, setErr] = useState('') + const [showing, setShowing] = useState<{ title: string; filter: AadhaarPlaceFilter } | null>(null) + const { branches } = useBranches() + const { can, edition } = useFeatures() + const licensed = can('personal-data') + + const load = useCallback(async () => { + setLoading(true); setErr(''); setShowing(null) + try { setReport(await getAadhaar(branch)) } + catch (e) { setReport(null); setErr((e as Error).message) } + finally { setLoading(false) } + }, [branch]) + useEffect(() => { load() }, [load]) + + const run = async () => { + const n = Number(timeout) + if (!Number.isInteger(n) || n < 1 || n > 3600) { setErr("A table's read may take 1 to 3,600 seconds."); return } + setRunning(true); setErr(''); setShowing(null) + try { setReport(await runAadhaar(branch, n)) } + catch (e) { setErr((e as Error).message) } + finally { setRunning(false) } + } + + const options = branches.length ? branches : ([{ name: 'main' }] as Branch[]) + const r = report?.run ?? null + const cols = report?.columns ?? [] + const people = report?.people ?? [] + + return ( +
    +

    Aadhaar exposure

    +

    + Every Aadhaar number in a branch's tables: whose (by its last four digits), in which column and row, and since + when. FoxByte reads every row and keeps where a number is, never the number. +

    + + {edition && !licensed && ( +

    + {why(edition, 'The Aadhaar exposure report')} Reports from earlier runs can still be read. +

    + )} + +
    + + + + setTimeout_(e.target.value)} /> + +
    + + {err &&
    {err}
    } + + {loading && !report ?

    Loading…

    : !r ? ( + !err && ( +
    +

    + {branch} has no Aadhaar exposure report yet.{licensed ? ' Run it to read every row for Aadhaar numbers.' : ''} +

    +
    + ) + ) : ( + <> +

    + Run {r.id}, {new Date(r.finished).toLocaleString()}{r.actor ? ` by ${r.actor}` : ''}: {r.tables} table(s), {r.columns} column(s) read. + {' '}{r.numbers} number(s) found, {r.people} distinct; {r.listed} place(s) kept. +

    + + {cols.length === 0 ? ( +
    +

    No Aadhaar number was found in the tables read. The limits below still apply.

    +
    + ) : ( + <> +

    By column

    +
    +
    DateFeature(s)What changedCommit / PR
    12 Nov 2026O4 (new), O3The Aadhaar exposure report, first part: the branch’s tables (point 4.5). fox pii aadhaar run reads every row of every text, number, array and JSON column of a branch for Aadhaar numbers (Verhoeff), in free text and JSON path by path, a key holding a number included; the database filters each table for the shape, so only rows that may hold one leave it, one read-only transaction a table with a time limit and a lock wait. fox pii aadhaar reports by column, counted in full, and by person: each number as UIDAI masks it (XXXX XXXX 0124), its places, and since when (the first run that found it, or a tracked row’s created_at); fox pii aadhaar places gives a person’s or a column’s rows by primary key, and what earlier runs found that is gone. A place is kept as its column or path, the row’s key and the last four digits (bb.aadhaar_*); never the number or a hash of it; people are grouped in memory for the run. Three API routes under personal-data/aadhaar (the branch’s owner or an admin; running it Enterprise, reading free) and a console page, Aadhaar exposure. Each run in the security log with counts. A table locked or too slow is skipped and said so, and its places are not called gone. A masked branch starts with no report (O3: its places name the source’s rows). Query history, the audit log’s text and the Blackbox are the second part (4.5b). Tests: integration 955/955 (§11ar new: numbers by column and inside JSON, a card and a phone not taken, one number in three tables one person, rows by key, a tracked row’s date, a keyless table named by ctid, no number or its first eight digits stored, not readable by a client, nothing in the Blackbox, the security log with no digit, a locked table skipped and not gone, a number removed gone, first-found kept, limits refused, the API’s owner, admin and others, the report carried by a branch), UI 62/62 plus the upgrade test (a run from the page, a person’s rows, no number on the screen, kept on the next visit, a removed number gone), integration-v2 403/403, integration-f5 57/57, web unit 299/299, Go unit tests in both editions, vet.feat/p4-aadhaar-exposure
    11 Nov 2026O3Fixed: a masked account’s read could deadlock with a key rotation. Swapping a table’s new masks in renamed the old shadow table first and replaced the masked view after it, while a masked read locks the view first, then the table and the shadow: a read arriving between the two held the view and waited for the shadow the swap held, the swap waited for the view, and Postgres ended one of them (seen once in the integration suite: four failed reads during fox mask key rotate). The swap now replaces the view first, reading the new shadow by oid, then renames, in a read’s order, so a read arriving mid-swap waits a moment and reads the new masks. Shown on PostgreSQL 18 with the swap paused at the point between: the old order deadlocked, the new one did not, and the view then reads the new shadow under its own name. Tests: integration 936/936, integration-f5 57/57, Go unit tests in both editions (the swap takes the view first), vet for macOS, Linux and Windows.fix/mask-rotate-deadlock
    11 Nov 2026H1Fixed: every branch start wrote 82 entries into the branch’s Blackbox. Since #221, each start of a branch takes dblink away from clients again (a REVOKE of EXECUTE on each of its 41 functions, from PUBLIC and from db_client), and the Blackbox records every REVOKE, held or not: a restarted branch gained 82 entries with no actor, and a branch made from just before an entry held 82 entries made after it. Each REVOKE is now issued only while the grant it removes is there, so a start that changes nothing records nothing, and a grant someone added is still taken away, and recorded, on the next start. Tests: integration-v2 403/403 (it was 401/403 on main), integration 936/936, Go unit tests in both editions (the REVOKEs are guarded by a look at the grant), vet for macOS, Linux and Windows; on the test VM a branch’s restart added 0 entries, and one given two dblink grants lost both on restart and recorded those two REVOKEs.fix/v2-branch-ledger
    11 Nov 2026(tests)Fixed: two unit tests that failed CI now and then. TestSnapshotServesThePreviousAnswerRatherThanHang ended with its deliberately hung read still running; let go as the test ended, the read looked at the test’s clock while the test’s cleanup put the real one back, and go test -race in CI failed on PRs that had not touched it (2 runs in 300). The test now lets the read finish before that cleanup. TestWitnessAnswersOneQuestion freed a port by closing a listener and took it for a primary that is down, but the test servers it then started were sometimes handed that very port, so the “down” primary answered; it now uses port 0, where nothing can listen. Nothing outside the tests changed. Tests: the test 1000 times under the race detector, internal/branch 20 times, the console’s and the Gateway’s snapshot tests 10 times, the witness test 500 times on macOS and Linux, all clean.fix/snapshot-test-race
    + + + + + {cols.map((c: AadhaarColumn) => ( + + + + + + + + + ))} + +
    WhereNumbersRowsKeptSinceRows
    {where(c)}{c.numbers}{c.rows}{c.listed}{c.capped &&
    the first
    }    		{day(c.since)} + +
    + + +

    By person

    +

    + The most places first{(report?.peopleListed ?? 0) > people.length ? `; the first ${people.length} of ${report!.peopleListed}` : ''}. + A person is this run's number for one Aadhaar: places with the same person hold the same number. +

    +
    + + + + + + {people.map(p => ( + + + + + + + + + ))} + +
    PersonNumberPlacesSinceWhereRows
    {p.person}{p.masked}{p.places}{day(p.since)}{(p.where ?? []).map(w =>
    {w}
    )}    		 + +
    +
    + + )} + + {(report?.gone ?? 0) > 0 && ( +

    + +

    + )} + + {showing && setShowing(null)} />} + + {(r.skipped?.length ?? 0) > 0 && ( +
    +

    Not read

    +
      + {r.skipped!.map(k =>
    • {k.object}: {k.reason}
      • )} +
    +
    + )} + +
    +

    What this cannot claim

    +
      + {(r.limits ?? []).map(l =>
    • {l}
      • )} +
    +
    + + )} + + ) +} + +// The rows behind a column, a person, or what is gone. +function Places({ branch, runId, title, filter, onClose }: { + branch: string + runId: number + title: string + filter: AadhaarPlaceFilter + onClose: () => void +}) { + const [places, setPlaces] = useState(null) + const [err, setErr] = useState('') + const key = JSON.stringify(filter) + + useEffect(() => { + let live = true + setPlaces(null); setErr('') + getAadhaarPlaces(branch, JSON.parse(key) as AadhaarPlaceFilter) + .then(r => { if (live) setPlaces(r.places ?? []) }) + .catch(e => { if (live) setErr((e as Error).message) }) + return () => { live = false } + }, [branch, runId, key]) + + const gone = !!filter.gone + return ( +
    +
    +

    Rows: {title}

    + +
    + {err ?
    {err}
    : !places ?

    Loading…

    : places.length === 0 ? ( +

    Nothing here.

    + ) : ( +
    + + + + {!gone && }{gone && } + + + + {places.map(p => ( + + {!gone && } + + + + + {gone && } + + ))} + +
    PersonNumberWhereRowSinceLast seen
    {p.person || '–'}{p.masked}{where(p)}{p.row}{day(p.since)}{p.rowCreated &&
    row created
    }    		{day(p.lastSeen)}
    +
    + )} +
    + ) +} diff --git a/web/tests/aadhaar.spec.ts b/web/tests/aadhaar.spec.ts new file mode 100644 index 0000000..57f5f61 --- /dev/null +++ b/web/tests/aadhaar.spec.ts @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// The Aadhaar exposure page, in a browser, against a real engine (point 4.5). +// +// The component tests prove the page's logic against mocks. This proves a run +// started from the page reads every row of a real branch, groups one number's +// places as one person shown by its last four digits, opens that person's rows, +// keeps the report for the next visit, says when a number has gone, and puts +// no whole number on the screen. public.pii_people (synthetic values) is made +// by scripts/integration_ui.sh; the notes table here, through the API the SQL +// console uses, with this browser's own session. +import { test, expect, type Page } from '@playwright/test' + +const email = process.env.FOX_E2E_EMAIL ?? 'e2e@foxbyte.dev' +const password = process.env.FOX_E2E_PASSWORD ?? 'password123' +const branch = process.env.FOX_E2E_BRANCH ?? 'e2eui' + +async function signIn(page: Page) { + await page.goto('/login', { waitUntil: 'domcontentloaded' }) + await page.getByPlaceholder('you@example.com').fill(email) + await page.getByPlaceholder('password').fill(password) + await page.getByRole('button', { name: /log in/i }).click() + await expect(page.getByRole('heading', { name: 'Dashboard' })).toBeVisible() +} + +test('a run from the page finds whose Aadhaar is where, by its last four digits, and shows no number', async ({ page }) => { + await signIn(page) + // Synthetic: 234567890124 passes Verhoeff; pii_people holds it as row 1. + const notes = `aad_notes_${Date.now().toString().slice(-6)}` + const sql = async (q: string) => { + const r = await page.request.post(`/api/branches/${branch}/query`, { data: { sql: q } }) + expect(r.ok(), await r.text()).toBeTruthy() + } + await sql(`CREATE TABLE public.${notes} (id int PRIMARY KEY, note text)`) + await sql(`INSERT INTO public.${notes} VALUES (7, 'caller read out 2345 6789 0124'), (8, 'card 4111 1111 1111 1111')`) + + await page.getByRole('link', { name: 'Aadhaar exposure' }).click() + await expect(page.getByRole('heading', { name: 'Aadhaar exposure', exact: true })).toBeVisible() + await page.getByLabel('Branch', { exact: true }).selectOption(branch) + await page.getByRole('button', { name: 'Run the report' }).click() + + const cols = page.getByRole('table', { name: 'Aadhaar by column' }) + const col = (where: string) => cols.locator('tr', { has: page.getByText(where, { exact: true }) }) + await expect(col('public.pii_people.aadhaar_no')).toContainText('2', { timeout: 120_000 }) + await expect(col(`public.${notes}.note`)).toBeVisible() + // A card's sixteen digits are not an Aadhaar. + await expect(cols.getByText('public.pii_people.card_no', { exact: true })).toHaveCount(0) + + // One number in two tables is one person, shown as UIDAI masks it. + const people = page.getByRole('table', { name: 'Aadhaar by person' }) + const person = people.locator('tr', { has: page.getByText('XXXX XXXX 0124', { exact: true }) }) + await expect(person).toContainText('public.pii_people.aadhaar_no') + await expect(person).toContainText(`public.${notes}.note`) + await person.getByRole('button', { name: /^Rows of person \d+$/ }).click() + const places = page.getByRole('table', { name: 'Aadhaar places' }) + await expect(places.locator('tr', { has: page.getByText(`public.${notes}.note`, { exact: true }) })).toContainText('id=7') + await expect(places.locator('tr', { has: page.getByText('public.pii_people.aadhaar_no', { exact: true }) })).toContainText('id=1') + await expect(page.getByRole('heading', { name: 'What this cannot claim' })).toBeVisible() + await expect(page.getByTestId('aadhaar-summary')).toContainText('number(s) found') + + // Where, never what: no whole number reaches the screen. + const text = (await page.locator('main').innerText()).replace(/[ -]/g, '') + for (const v of ['234567890124', '456789012341', '4111111111111111']) { + expect(text).not.toContain(v) + } + + // Kept, not recomputed: the report is there on the next visit. + await page.reload({ waitUntil: 'domcontentloaded' }) + await page.getByLabel('Branch', { exact: true }).selectOption(branch) + await expect(col(`public.${notes}.note`)).toBeVisible({ timeout: 30_000 }) + + // A number taken out is gone on the next run, with when it was last seen. + await sql(`UPDATE public.${notes} SET note = 'number removed' WHERE id = 7`) + await page.getByRole('button', { name: 'Run the report' }).click() + await expect(cols.getByText(`public.${notes}.note`, { exact: true })).toHaveCount(0, { timeout: 120_000 }) + await page.getByRole('button', { name: /^\d+ place\(s\) earlier runs found are no longer there$/ }).click() + const gone = page.getByRole('table', { name: 'Aadhaar places' }) + await expect(gone.locator('tr', { has: page.getByText(`public.${notes}.note`, { exact: true }) })).toContainText('XXXX XXXX 0124') + await expect(gone).toContainText('Last seen') + // No clean-up: a DROP here would be one of the branch's changes, which a + // later spec applies to another branch, where the guardrail refuses it. The + // harness remakes the branch, as personaldata.spec.ts leaves its table too. +})