The Aadhaar exposure report, the branch's tables (point 4.5a)

cmd/fox/aadhaarcmd.go

@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/thefoxbyte/foxbyte/internal/auth"
+	"github.com/thefoxbyte/foxbyte/internal/branch"
+	"github.com/thefoxbyte/foxbyte/internal/brand"
+	"github.com/thefoxbyte/foxbyte/internal/edition"
+)
+
+// aadhaarCmd is `fox pii aadhaar` (point 4.5): which person's Aadhaar is
+// exposed in a branch, where, and since when.
+//
+//	fox pii aadhaar run [--branch b] [--table-timeout 5m] [--json]   read every row now (Enterprise)
+//	fox pii aadhaar [--branch b] [--json]                            the latest report
+//	fox pii aadhaar places [--person n] [--column s.t.c] [--gone] [--limit n] [--json]
+//
+// A number is shown as UIDAI masks it, XXXX XXXX 1234, and is kept no other
+// way. Reading the report is free; a run is Enterprise and is in the security
+// log with its counts.
+func aadhaarCmd(args []string, b string, ps []string) {
+	sub := "report"
+	if len(ps) > 0 {
+		sub = ps[0]
+	}
+	switch sub {
+	case "run":
+		requireFeature(edition.PersonalData)
+		o := branch.AadhaarRunOptions{Actor: cliActor()}
+		if v := optValue(args, "--table-timeout"); v != "" {
+			d, err := time.ParseDuration(v)
+			if err != nil {
+				must(fmt.Errorf("--table-timeout is a duration (90s, 5m), not %q", v))
+			}
+			o.TableTimeout = d
+		}
+		rep, err := branch.RunAadhaarExposure(b, o)
+		must(err)
+		if s := historyStore(); s != nil {
+			s.Audit(auth.EvAadhaarExposure, o.Actor, b, "", rep.Summary())
+			s.Close()
+		}
+		if hasFlag(args, "--json") {
+			printJSON(rep)
+			return
+		}
+		printAadhaar(os.Stdout, rep)
+	case "report":
+		rep, err := branch.AadhaarExposure(b)
+		must(err)
+		if hasFlag(args, "--json") {
+			printJSON(rep)
+			return
+		}
+		printAadhaar(os.Stdout, rep)
+	case "places":
+		f := branch.AadhaarPlaceFilter{Gone: hasFlag(args, "--gone")}
+		for flag, to := range map[string]*int{"--person": &f.Person, "--limit": &f.Limit} {
+			if v := optValue(args, flag); v != "" {
+				n, err := strconv.Atoi(v)
+				if err != nil || n < 1 {
+					must(fmt.Errorf("%s is a number, not %q", flag, v))
+				}
+				*to = n
+			}
+		}
+		if c := optValue(args, "--column"); c != "" {
+			w, err := branch.ParseAadhaarWhere(c)
+			must(err)
+			f.Where = w
+		}
+		places, err := branch.AadhaarPlaces(b, f)
+		must(err)
+		if hasFlag(args, "--json") {
+			printJSON(places)
+			return
+		}
+		printAadhaarPlaces(os.Stdout, places, f.Gone, b)
+	default:
+		usagePII()
+	}
+}
+
+func shortDay(t time.Time) string { return t.Local().Format("2006-01-02") }
+
+func printAadhaar(w io.Writer, rep branch.AadhaarReport) {
+	r := rep.Run
+	if r == nil {
+		fmt.Fprintf(w, "No Aadhaar exposure report on %s yet: `%s pii aadhaar run --branch %s` reads it (Enterprise).\n", rep.Branch, brand.CLI, rep.Branch)
+		return
+	}
+	by := ""
+	if r.Actor != "" {
+		by = " by " + r.Actor
+	}
+	fmt.Fprintf(w, "Aadhaar exposure on %s: run %d, %s%s.\n", rep.Branch, r.ID, r.Finished.Local().Format("2006-01-02 15:04"), by)
+	fmt.Fprintf(w, "%d table(s), %d column(s) read; %d row(s) held twelve digits in its shape. %d number(s) found, %d distinct; %d place(s) kept.\n",
+		r.Tables, r.Columns, r.Rows, r.Numbers, r.People, r.Listed)
+	for _, s := range r.Skipped {
+		fmt.Fprintf(w, "  not read: %s (%s)\n", s.Object, s.Reason)
+	}
+	if len(rep.Columns) == 0 {
+		fmt.Fprintln(w, "\nNo Aadhaar number was found in the tables read.")
+	} else {
+		fmt.Fprintln(w, "\nBy column")
+		tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
+		fmt.Fprintln(tw, "  COLUMN\tNUMBERS\tROWS\tKEPT\tSINCE")
+		for _, c := range rep.Columns {
+			kept := strconv.Itoa(c.Listed)
+			if c.Capped {
+				kept += " (first)"
+			}
+			since := "–"
+			if c.Since != nil {
+				since = shortDay(*c.Since)
+			}
+			fmt.Fprintf(tw, "  %s\t%d\t%d\t%s\t%s\n", c.Where(), c.Numbers, c.Rows, kept, since)
+		}
+		tw.Flush()
+	}
+	if len(rep.People) > 0 {
+		more := ""
+		if rep.PeopleListed > len(rep.People) {
+			more = fmt.Sprintf(", the first %d of %d", len(rep.People), rep.PeopleListed)
+		}
+		fmt.Fprintf(w, "\nBy person (the most places first%s)\n", more)
+		tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
+		fmt.Fprintln(tw, "  PERSON\tNUMBER\tPLACES\tSINCE\tWHERE")
+		for _, p := range rep.People {
+			fmt.Fprintf(tw, "  %d\t%s\t%d\t%s\t%s\n", p.Person, p.Masked, p.Places, shortDay(p.Since), strings.Join(p.Where, ", "))
+		}
+		tw.Flush()
+		fmt.Fprintf(w, "One person's rows: %s pii aadhaar places --person <n> --branch %s\n", brand.CLI, rep.Branch)
+	}
+	if rep.Gone > 0 {
+		fmt.Fprintf(w, "\n%d place(s) earlier runs found are no longer there: %s pii aadhaar places --gone --branch %s\n", rep.Gone, brand.CLI, rep.Branch)
+	}
+	if len(r.Limits) > 0 {
+		fmt.Fprintln(w, "\nWhat this report cannot claim:")
+		for _, l := range r.Limits {
+			fmt.Fprintf(w, "  - %s\n", l)
+		}
+	}
+}
+
+func printAadhaarPlaces(w io.Writer, places []branch.AadhaarPlace, gone bool, b string) {
+	if len(places) == 0 {
+		if gone {
+			fmt.Fprintf(w, "No place on %s has gone since an earlier run.\n", b)
+		} else {
+			fmt.Fprintf(w, "No places on %s match.\n", b)
+		}
+		return
+	}
+	tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
+	if gone {
+		fmt.Fprintln(tw, "NUMBER\tWHERE\tROW\tSINCE\tLAST SEEN")
+		for _, p := range places {
+			fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\n", p.Masked, aadhaarCol(p), p.Row, shortDay(p.Since), shortDay(p.LastSeen))
+		}
+	} else {
+		fmt.Fprintln(tw, "PERSON\tNUMBER\tWHERE\tROW\tSINCE")
+		for _, p := range places {
+			person := strconv.Itoa(p.Person)
+			if p.Person == 0 {
+				person = "–"
+			}
+			fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\n", person, p.Masked, aadhaarCol(p), p.Row, shortDay(p.Since))
+		}
+	}
+	tw.Flush()
+}
+
+func aadhaarCol(p branch.AadhaarPlace) string {
+	return branch.AadhaarColumn{Schema: p.Schema, Table: p.Table, Column: p.Column, Path: p.Path}.Where()
+}

cmd/fox/main.go

@@ -126,6 +126,11 @@ Stack:
                        who has gone quiet, and the applications that assert their users (Enterprise)
   pii unclassified | accept <column-id> [--note "…"]
                        What appeared since a branch's first scan, not reviewed; accept one as holding nothing
+  pii aadhaar run [--branch b] [--table-timeout 5m] [--json]
+                       Read every row for Aadhaar numbers: where each is, by row and last four
+                       digits, grouped by person, and since when; never the number (Enterprise)
+  pii aadhaar [--branch b] [--json] | places [--person n] [--column s.t.c] [--gone] [--limit n]
+                       The latest exposure report, by column and by person; or its places
   app [list|show <name>] [--json]
                        The applications that may say which of their users a transaction
                        acts for, and the "aud" their tokens name

cmd/fox/piicmd.go

@@ -36,7 +36,7 @@ func piiCmd(args []string) {
 	if b == "" {
 		b = "main"
 	}
-	ps := positionals(args, "--branch", "--sample", "--state", "--n", "--note")
+	ps := positionals(args, "--branch", "--sample", "--state", "--n", "--note", "--table-timeout", "--person", "--column", "--limit")
 	sub := "findings"
 	if len(ps) > 0 {
 		sub = ps[0]
@@ -175,6 +175,8 @@ func piiCmd(args []string) {
 			fmt.Fprintf(os.Stderr, "note: its table could not be marked for the audit log (%s); `%s audit-log table --personal-data --branch %s` marks it later.\n",
 				d.AuditMarkError, brand.CLI, b)
 		}
+	case "aadhaar":
+		aadhaarCmd(args, b, ps[1:])
 	default:
 		usagePII()
 	}
@@ -187,7 +189,10 @@ func usagePII() {
 		"       %[1]s pii samples <id> [--branch b] [--n 5] [--json]\n"+
 		"       %[1]s pii mark <id> confirm|reject|reopen [--note \"…\"] [--branch b]\n"+
 		"       %[1]s pii unclassified [--branch b] [--json]\n"+
-		"       %[1]s pii accept <column-id> [--note \"…\"] [--branch b]\n", brand.CLI)
+		"       %[1]s pii accept <column-id> [--note \"…\"] [--branch b]\n"+
+		"       %[1]s pii aadhaar run [--branch b] [--table-timeout 5m] [--json]\n"+
+		"       %[1]s pii aadhaar [--branch b] [--json]\n"+
+		"       %[1]s pii aadhaar places [--branch b] [--person n] [--column schema.table.column] [--gone] [--limit n] [--json]\n", brand.CLI)
 	os.Exit(2)
 }
 

docs/FOX_Checklist.html

@@ -781,6 +781,13 @@ <h2><span class="al">O</span>Personal data &amp; compliance<span class="cnt"></s
       <li class="ok">Step 4: every access path the design names tested as a masked account and as its agent key (integration &sect;11al, masking.spec.ts). It found eight ways to real values, now closed: a branch it owns, made of main or back in time, reads masks (owning one no longer unmasks); its agent keys log in as a masked agent role, which cannot reach <code>db_client</code> by inheriting or <code>SET ROLE</code>, and MCP&rsquo;s <code>run_sql</code> and <code>execute_change</code> never fall back to a shared role; agent branches are refused to it (masked branches since step 5); realtime is refused to it where there are masks; the Gateway refuses it rather than log it in as the shared role; the Blackbox&rsquo;s entries reach it without their statements&rsquo; text, and its export is refused; a guard in the database (<code>bb.mask_close_paths</code>, an event trigger) takes masked accounts&rsquo; rights on a view or materialized view reading a masked column, on a marked table granted back, and <code>INSERT</code>/<code>UPDATE</code> on a table with a foreign key into a masked column, and gives them back when the rule goes, refusing a <code>GRANT</code> to <code>PUBLIC</code>; <code>fox mask</code> and the Masking page list what it closed and the definer functions that can return real values. Kept, and now tested: COPY, <code>RETURNING</code>, <code>ON CONFLICT</code>, <code>MERGE</code>, errors, <code>pg_stats</code>, <code>EXPLAIN</code>, the console&rsquo;s query, grid, object browser and exports</li>
       <li class="ok">Step 5: <code>fox branch create &lt;name&gt; --masked [--from b]</code>, the Dashboard&rsquo;s <i>masked</i> option and <code>POST /api/branches</code> with <code>masked</code> (made in the background, 202), <code>fox mask branches</code>, <code>GET /api/masking/branches</code> and the Masking page&rsquo;s Masked branches table. The copy is recorded before it exists, so the Gateway, the console, the API, MCP and the agents refuse it (423, 57P03) until it is masked and verified, and it never mounts the key; every marked column is overwritten with the mask a masked reader sees, checked row by row; the masks&rsquo; machinery is removed; its Blackbox starts afresh, numbering on from the source&rsquo;s last entry, so a change request from it carries only what was done on it; <code>VACUUM FULL</code> of each masked table and the statistics; the source&rsquo;s replication slots dropped and the WAL from before removed. Verified: no WAL segment from before, nothing of the source&rsquo;s Blackbox, and the scan finding no masked category, nor anything confirmed, where no rule masked it (a finding rejected on the source is ignored). A failure, or a run stopped part way, leaves it closed, saying why; running it again starts over. On an open masked branch nobody is masked; a copy of one is one at once; reset makes it again; a masked account&rsquo;s agent branch is made as one (replacing step 4&rsquo;s refusal)</li>
       <li class="lim">One-column primary keys only; a JSON path into an array, or a quoted key, is refused for now; masking stays on once set up; a masked account has no DDL rights; a domain&rsquo;s key is the install&rsquo;s, shared by every branch, so a branch stopped through a rotation keeps its old masks until set up again; an unmask holder&rsquo;s reads are logged only while the audit log is on; the lookup is by a column with a rule of its own, not a JSON path; a definer function that can leak is listed, never changed; <code>CREATE TABLE AS</code> copies real values into an ordinary table, which the scanner finds on its next run; the change requests list shows each request&rsquo;s SQL to anyone on its branches; a masked branch&rsquo;s blocks shared with its source&rsquo;s snapshot, and the source&rsquo;s own WAL and backups, hold real values (they are the source&rsquo;s); on a masked branch a foreign key from a column without a rule into a hashed column no longer matches (masking skips the check)</li></ul></td><td class="tb">enterprise/masking unit; internal/branch mask_test.go, maskkey_test.go, maskpaths_test.go, maskedbranch_test.go; VM &sect;11ai, &sect;11aj, &sect;11ak, &sect;11al, &sect;11am; masking.spec.ts</td></tr>
+    <tr data-id="O4" data-st="part"><td class="c"><i class="ck part"></i></td><td class="id">O4</td><td><b>Aadhaar exposure report</b> (point 4.5, Enterprise)<ul class="subs">
+      <li class="ok">Every row of every text, number, array and JSON column read for Aadhaar numbers (Verhoeff), in free text and JSON path by path; the database filters for the shape, the engine checks the digit; run-on numbers and cards refused</li>
+      <li class="ok">Places kept as column or path, the row&rsquo;s primary key (or ctid, said so) and the last four digits; never the number or a hash; grouped by person in memory; since when from the first run or a tracked row&rsquo;s created_at; gone when a full read no longer finds it</li>
+      <li class="ok"><code>fox pii aadhaar run|places</code>, three API routes, and the console&rsquo;s Aadhaar exposure page; the branch&rsquo;s owner or an admin; each run in the security log with counts</li>
+      <li class="ok">A table locked or too slow is skipped and said so, its places not called gone; a branch made from it carries the report; a masked branch starts with none</li>
+      <li class="todo">4.5b: query history, the audit log&rsquo;s statement text and the Blackbox (admins only)</li>
+      <li class="lim">No vault yet (point 13); a person is a run&rsquo;s grouping; places capped at 10,000 a column and 200,000 a run; synchronous</li></ul></td><td class="tb">enterprise/pii unit; VM &sect;11ar; aadhaar.spec.ts</td></tr>
   </table>
   <div class="foot">FOX_Checklist &middot; area O</div>
 </section>