From eb49f45bcdca3a31971c173dfa55399a39ef4284 Mon Sep 17 00:00:00 2001 From: linkvapeluckyman Date: Wed, 10 Jun 2026 14:56:32 +0530 Subject: [PATCH 1/2] feat(server,client): configurable ingest-time privacy controls for hook payloads Implements the MVP for issue #148: - New lib/privacy.js sanitizer applied at every event-insert site in routes/hooks.js BEFORE persistence and WebSocket broadcast: six built-in detectors (secret-named keys, Bearer tokens, common API-key formats, private-key blocks, opt-in email addresses and home-directory paths) plus up to 100 custom key/value regex rules with mask / hash / drop_field / drop_event_payload actions - Conservative default policy: obvious secrets masked out of the box; hash action uses stable truncated SHA-256 so values stay correlatable - Redacted events carry a _privacy counters block (rules applied, masked / hashed / dropped) without exposing originals; clean payloads are stored byte-identical so analytics, filters, and cost views are unaffected - Fail-safe: sanitizer errors degrade to a metadata-only stub (never raw data) and never fail hook ingestion; invalid rules are rejected at save - REST API: GET/PUT /api/privacy + POST /api/privacy/preview (non-persisting before/after, supports draft policies); documented in OpenAPI/Swagger - Policy persisted in a new additive app_settings key/value table and included in GET /api/settings/export - Settings panel: master toggle, per-detector toggles, rule CRUD, live sample preview, explicit import/reimport warning; localized en/zh/vi - 14 server tests: policy validation, nested payload masking, key formats, summary sanitization on ingest + response, hash stability, drop actions, disabled passthrough, opt-in detectors, large payloads, preview isolation Closes #148 Co-Authored-By: Claude Fable 5 --- ARCHITECTURE.md | 17 +- README.md | 9 + client/src/components/PrivacyControls.tsx | 424 ++++++++++++++++++++++ client/src/i18n/locales/en/settings.json | 57 +++ client/src/i18n/locales/vi/settings.json | 57 +++ client/src/i18n/locales/zh/settings.json | 57 +++ client/src/lib/api.ts | 28 ++ client/src/lib/types.ts | 45 +++ client/src/pages/Settings.tsx | 11 + server/__tests__/api.test.js | 2 + server/__tests__/privacy.test.js | 381 +++++++++++++++++++ server/db.js | 19 + server/index.js | 2 + server/lib/privacy.js | 403 ++++++++++++++++++++ server/openapi.js | 104 ++++++ server/routes/hooks.js | 43 ++- server/routes/privacy.js | 76 ++++ server/routes/settings.js | 4 + 18 files changed, 1723 insertions(+), 16 deletions(-) create mode 100644 client/src/components/PrivacyControls.tsx create mode 100644 server/__tests__/privacy.test.js create mode 100644 server/lib/privacy.js create mode 100644 server/routes/privacy.js diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b5b21d86..69336c11 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -293,12 +293,17 @@ graph TD PRICING[routes/pricing.js
Cost calculation + pricing CRUD] SETTINGS[routes/settings.js
System info + data management] WORKFLOWS[routes/workflows.js
Workflow visualizations] + PRIVACYR[routes/privacy.js
Privacy policy + preview] + PRIVACY[lib/privacy.js
Ingest-time sanitizer] INDEX --> DB INDEX --> WS - INDEX --> HOOKS & SESSIONS & AGENTS & EVENTS & STATS & PRICING & SETTINGS & WORKFLOWS + INDEX --> HOOKS & SESSIONS & AGENTS & EVENTS & STATS & PRICING & SETTINGS & WORKFLOWS & PRIVACYR HOOKS --> DB & WS & TC + HOOKS --> PRIVACY + PRIVACYR --> PRIVACY + PRIVACY --> DB SETTINGS --> DB & TC INDEX --> TC SESSIONS --> DB & WS @@ -329,7 +334,9 @@ graph TD | `routes/stats.js` | Single aggregate query returning total/active counts + status distributions | | `routes/analytics.js` | Extended analytics — token totals, tool usage counts, daily event/session trends, agent type distribution. The client-side analytics heatmap grid is aligned to a Sunday start for correct day-of-week positioning | | `routes/pricing.js` | Model pricing CRUD (list/upsert/delete), per-session and global cost calculation with pattern-based model matching | -| `routes/settings.js` | System info (DB size, hook status, server uptime, transcript cache stats), data export as JSON, session cleanup (abandon stale, purge old), clear all data, reset pricing, reinstall hooks | +| `routes/settings.js` | System info (DB size, hook status, server uptime, transcript cache stats), data export as JSON (includes the active `privacy_policy` so a restore can re-apply the same rules; events export as stored — post-policy), session cleanup (abandon stale, purge old), clear all data, reset pricing, reinstall hooks | +| `routes/privacy.js` | HTTP surface for ingest-time privacy controls: `GET /api/privacy` (active policy + action/match-type vocabulary + defaults), `PUT /api/privacy` (replace policy, validated server-side — invalid regexes, unknown actions, and `drop_field` on value rules are rejected with structured 400s), and `POST /api/privacy/preview` (before/after transformation of a sample payload + optional draft policy, persisting nothing) | +| `lib/privacy.js` | Ingest-time payload sanitizer. The policy (persisted as JSON in `app_settings`, cached in memory, invalidated on PUT) holds six built-in detector toggles — `secret_keys` (key-name match, same regex family as the Config Explorer redaction), `bearer_tokens`, `api_key_formats`, `private_key_blocks`, plus opt-in `email_addresses` and `home_paths` — a `default_action` (`mask` or stable truncated SHA-256 `hash`), and up to 100 custom rules (`match_type: key\|value` regex with `mask` / `hash` / `drop_field` / `drop_event_payload` actions; `drop_event_payload` reduces the payload to a metadata-only stub preserving `session_id`/`tool_name`/`transcript_path`/`cwd`). `sanitizeEventData()` deep-walks payloads (depth cap 32, node cap 20k, value regexes skip strings > 256 KB) and stamps a `_privacy` counters block only when something was redacted — clean payloads are stored byte-identical. `sanitizeText()` covers event summaries and agent task strings. Called from `routes/hooks.js` at every event-insert site **after** routing decisions ran on the raw payload; fail-safe — a sanitizer crash stores a conservative metadata-only stub (never raw data) and never breaks ingestion. Import/re-import intentionally bypasses the policy (records are written as found on disk); the Settings panel and docs call this out explicitly | | `routes/workflows.js` | Aggregate workflow visualization data (agent orchestration graphs, tool transition flows, collaboration networks, workflow pattern detection, model delegation, error propagation, concurrency timelines, session complexity metrics, compaction impact). Accepts `?status=active\|completed` query parameter to filter all data by session status. Per-session drill-in endpoint with agent tree, tool timeline, and event details | | `lib/transcript-cache.js` | Stat-based JSONL transcript cache with incremental byte-offset reads. Shared between `hooks.js` (token extraction on every event) and the periodic compaction scanner (`index.js`). Extracts tokens, compaction entries, API errors (`isApiErrorMessage` + raw error responses), turn durations (`system` subtype `turn_duration`), thinking block counts, and usage extras (service_tier, speed, inference_geo). Uses `(path, mtime, size)` cache key — unchanged files return cached results instantly, grown files only parse new bytes, shrunk files (compaction) trigger full re-read. Each cache entry stores **only** `{mtimeMs, size, bytesRead, result}` — the previous shape that duplicated every growable array at both the top level and inside `result` is gone, halving steady-state memory per entry. Per-entry growable arrays (`turnDurations`, `errors`, `compaction.entries`, `usageExtras.*`) are bounded to `TRANSCRIPT_CACHE_MAX_ARRAY_LEN` (default `1000`, tail-kept) — older items remain in the `events` table thanks to hook dedup, so the cap only affects the in-memory view. Trimming runs both during parse (when an array reaches `2 * MAX_ARRAY_LEN`, amortized O(N)) and at finalize, so even a fresh full-file parse on a multi-day session cannot accumulate an unbounded transient before returning. **Chunked sync byte-stream reader** (`_streamRange`, 4 MiB chunks split on `0x0A` bytes — safe across UTF-8 multibyte sequences — with a growable per-line byte buffer capped at 64 MiB) replaces the previous `readFileSync("utf8")` so transcripts larger than V8's max JS string length (~512 MiB on 64-bit Node 20) parse without aborting Node with `FATAL ERROR: v8::ToLocalChecked Empty MaybeLocal`. Both full and incremental reads share the same line-level state machine (`_initParseState` / `_consumeLine` / `_finalizeState`). LRU eviction caps at 200 entries. Entries evicted on SessionEnd and abandoned session cleanup | | `scripts/import-history.js` | Batch history importer used by (a) server startup auto-import, (b) the `/api/import/*` routes, (c) the `import-history` CLI, and (d) live `SubagentStop` ingestion via the exported `scanAndImportSubagents(dbModule, sessionId, transcriptPath)`. Exposes `importAllSessions(dbModule)` for the default `~/.claude/projects` tree and the generalized `importFromDirectory(dbModule, rootDir, {onProgress})` which walks any directory recursively, classifies each `.jsonl` as session vs subagent (with `findSessionSubagents` probing both `//subagents/*` and `/subagents//*` layouts), and funnels everything through the shared `parseSessionFile` + `importSession` pipeline. `parseSubagentFile` extracts ordered `toolEvents` (tool_use + tool_result paired by `tool_use_id`) so `importSubagentFromJsonl` can emit per-tool `PreToolUse` + `PostToolUse` rows under each subagent's own `agent_id`. The importer dedups against live hook-created subagent rows via `findLiveSubagentForJsonl` (session + subagent_type + start-time within 30 s) so backfill never produces parallel `-jsonl-*` rows. **Re-import is fully incremental**: for each existing session a per-event-type high-water mark (`MAX(created_at) GROUP BY event_type`) is read up-front and only JSONL entries with `ts > cutoff[type]` are inserted for Stop / PostToolUse / TurnDuration / ToolError — so long-running sessions whose transcripts grow across multiple days continue to receive new events on every re-run instead of being blocked by the old "if zero of type X then dump all" check. `sessions.ended_at` is rolled forward to the JSONL's last activity when it surpasses the stored value, and `metadata.user_messages` / `assistant_messages` / `turn_count` are refreshed on every pass. Other idempotency keys are unchanged: `data LIKE '%"tool_use_id":"X"%'` skips any tool event already inserted, compaction agents/events dedup by uuid, API errors dedup by summary, and `baseline_*` columns preserve pre-compaction token totals. Token totals, per-model cost, compactions, subagents, tool events, API errors, and turn durations are identical to live ingestion. Creates `APIError`, `TurnDuration`, and `ToolError` event types during import; subagent tool events carry `imported: true, source: "subagent_jsonl"` in their data payload so analytics can distinguish backfilled rows when needed | @@ -775,6 +782,12 @@ erDiagram TEXT auth "Auth secret" TEXT created_at "ISO 8601" } + + app_settings { + TEXT key PK "Setting name e.g. privacy_policy" + TEXT value "JSON string" + TEXT updated_at "ISO 8601" + } ``` ### Indexes diff --git a/README.md b/README.md index 3b71f558..725b9f1d 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,7 @@ The dashboard offers a comprehensive set of features to monitor and analyze your | **Cost Tracking** | Per-model cost estimation with configurable pricing rules and per-session breakdowns. Compaction-aware token accounting preserves totals across context compressions. Transcript reads are cached with incremental byte-offset updates for efficient token extraction | | **Transcript Cache** | Real-time extraction from JSONL transcripts: tokens, compactions, API errors (`isApiErrorMessage` entries stored as `APIError` events), turn durations (stored as `TurnDuration` events), thinking block counts, and usage extras (service_tier, speed, inference_geo). Per-entry growable arrays are tail-capped at `TRANSCRIPT_CACHE_MAX_ARRAY_LEN` (default `1000`, configurable) — both during parse and at finalize — so even a session that runs for days cannot grow a single cache entry without bound. Each entry stores only `{mtimeMs, size, bytesRead, result}`, so there's no shadow copy of the same data at both the top level and inside `result`. Session metadata is enriched with these fields in real-time | | **Notifications** | Full Web Push (VAPID) pipeline for reliable delivery. Arrive even when the tab is backgrounded or the browser is closed. Explicitly configured for macOS audio support. Configurable per-event toggles with subscription management | +| **Privacy Controls** | Configurable ingest-time privacy policy that redacts, hashes, or drops sensitive data from hook payloads **before** they are written to SQLite or broadcast over WebSocket. Six built-in detectors — secret-named keys (`token`/`secret`/`password`/`api_key`/`auth`/`credential`), `Bearer` tokens, common API-key formats (`sk-ant-…`, `ghp_…`, `AKIA…`, `xox…`, `AIza…`), private-key blocks, and opt-in email-address and home-directory-path detectors — plus custom regex rules matching key names or value text with `mask`, `hash` (stable truncated SHA-256 for correlation), `drop_field`, or `drop_event_payload` (metadata-only stub) actions. Conservative default: obvious secrets are masked out of the box. Redacted events carry a `_privacy` metadata block (rule count, masked/hashed/dropped counters) without exposing original values, and clean payloads are stored byte-identical so existing analytics, filters, and cost views are unaffected. Settings panel with master toggle, per-detector toggles, rule CRUD, and a live **before/after preview** (`POST /api/privacy/preview`) that never persists the sample. Fail-safe: a sanitizer error degrades to dropping the payload — never storing raw data, never failing hook ingestion. Applies to live ingest only; import/re-import writes transcripts as found (explicitly called out in the UI) | | **Update Notifier** | Server periodically runs a non-blocking `git fetch` and compares the local checkout to `origin/master`/`origin/main`/`origin/HEAD`. When upstream is ahead, the UI surfaces a modal with the exact `git pull && npm run setup` command and a one-click **Copy** button; the Sidebar gets a persistent "Check for updates" button with live badge. The dashboard never pulls or restarts itself — the user runs the command in a terminal — so the mechanism cannot break dev sessions, pm2/systemd/Docker supervision, or leave orphaned processes | | **Settings** | System info, hook status, model pricing management, notification preferences, data export, session cleanup. The Model Pricing section exposes an info popover (the `i` icon next to the title) explaining how rule lookup works (first matching pattern wins), the SQL-style `%` wildcard syntax with concrete examples (`claude-opus-4-7%`, `claude-%-haiku`, exact ids), and that prices must be updated manually when Anthropic publishes new rates — already-stored sessions keep the price applied at ingest time. The CLAUDE_HOME box and Import History panel are fully i18n-driven across en/vi/zh | | **MCP Server (Local)** | Enterprise-grade local MCP server in `mcp/` with three transport modes (stdio, HTTP+SSE, interactive REPL), 25 typed tools across 6 domains, strict input schemas, retry/backoff, localhost-only API enforcement, and tiered mutation/destructive safety gates. HTTP mode serves Streamable HTTP (2025-11-25) and legacy SSE (2024-11-05) on configurable port. REPL mode provides tab-completed interactive tool invocation with colored output | @@ -892,6 +893,14 @@ The OpenAPI document is generated from `server/openapi.js`, and Swagger UI is se | `GET` | `/api/workflows` | Aggregate workflow data (orchestration, tools, patterns). Optional `?status=active\|completed` query param filters all 11 data sections by session status | | `GET` | `/api/workflows/session/:id` | Per-session drill-in (agent tree, tool timeline, events) | +### Privacy + +| Method | Path | Description | +| ------ | ---------------------- | ------------------------------------------------------------------------ | +| `GET` | `/api/privacy` | Active ingest-time privacy policy + action/match-type vocabulary | +| `PUT` | `/api/privacy` | Replace the policy (validated server-side; invalid regexes are rejected) | +| `POST` | `/api/privacy/preview` | Before/after preview of a sample payload — nothing is persisted | + ### Settings | Method | Path | Description | diff --git a/client/src/components/PrivacyControls.tsx b/client/src/components/PrivacyControls.tsx new file mode 100644 index 00000000..932213ee --- /dev/null +++ b/client/src/components/PrivacyControls.tsx @@ -0,0 +1,424 @@ +/** + * @file PrivacyControls.tsx + * @description Settings section for ingest-time privacy controls. Lets the + * user toggle the master policy and built-in detectors, manage custom + * redaction rules (key/value regex match with mask / hash / drop_field / + * drop_event_payload actions), and preview how a sample payload would be + * transformed by the current (possibly unsaved) policy — nothing in the + * preview is persisted. Live ingest applies the policy; import/reimport does + * not, which the panel calls out explicitly. + * @author Son Nguyen + */ + +import { useCallback, useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Shield, Plus, Trash2, Eye, AlertTriangle, Check } from "lucide-react"; +import { api } from "../lib/api"; +import { Skeleton } from "./Skeleton"; +import type { + PrivacyAction, + PrivacyMatchType, + PrivacyPolicy, + PrivacyPreviewResult, + PrivacyRule, +} from "../lib/types"; + +const DETECTOR_KEYS = [ + "secret_keys", + "bearer_tokens", + "api_key_formats", + "private_key_blocks", + "email_addresses", + "home_paths", +] as const; + +const ACTIONS: PrivacyAction[] = ["mask", "hash", "drop_field", "drop_event_payload"]; +const MATCH_TYPES: PrivacyMatchType[] = ["key", "value"]; + +const SAMPLE_PAYLOAD = JSON.stringify( + { + session_id: "sample-session", + tool_name: "Bash", + tool_input: { + command: "curl -H 'Authorization: Bearer abc123def456ghi789' https://api.example.com", + api_key: "sk-ant-api03-EXAMPLE-KEY-VALUE-1234567890", + }, + }, + null, + 2 +); + +function MiniToggle({ + checked, + onChange, + label, +}: { + checked: boolean; + onChange: (next: boolean) => void; + label: string; +}) { + return ( + + ); +} + +export function PrivacyControls() { + const { t } = useTranslation("settings"); + const [policy, setPolicy] = useState(null); + const [loading, setLoading] = useState(true); + const [dirty, setDirty] = useState(false); + const [saving, setSaving] = useState(false); + const [saveError, setSaveError] = useState(null); + const [savedFlash, setSavedFlash] = useState(false); + + const [newRule, setNewRule] = useState({ + name: "", + enabled: true, + match_type: "value", + pattern: "", + action: "mask", + }); + + const [sampleText, setSampleText] = useState(SAMPLE_PAYLOAD); + const [preview, setPreview] = useState(null); + const [previewError, setPreviewError] = useState(null); + + useEffect(() => { + api.privacy + .get() + .then((res) => setPolicy(res.policy)) + .catch((err) => console.error("Failed to load privacy policy:", err)) + .finally(() => setLoading(false)); + }, []); + + const edit = useCallback((patch: Partial) => { + setPolicy((prev) => (prev ? { ...prev, ...patch } : prev)); + setDirty(true); + setSavedFlash(false); + }, []); + + const save = async () => { + if (!policy || saving) return; + setSaving(true); + setSaveError(null); + try { + const res = await api.privacy.update(policy); + setPolicy(res.policy); + setDirty(false); + setSavedFlash(true); + } catch (err) { + setSaveError(err instanceof Error ? err.message : String(err)); + } finally { + setSaving(false); + } + }; + + const addRule = () => { + if (!policy || !newRule.name.trim() || !newRule.pattern.trim()) return; + edit({ rules: [...policy.rules, { ...newRule, name: newRule.name.trim() }] }); + setNewRule({ name: "", enabled: true, match_type: "value", pattern: "", action: "mask" }); + }; + + const runPreview = async () => { + if (!policy) return; + setPreviewError(null); + let data: Record; + try { + data = JSON.parse(sampleText); + } catch { + setPreviewError(t("privacy.preview.invalidJson")); + return; + } + try { + const res = await api.privacy.preview({ data, policy }); + setPreview(res); + } catch (err) { + setPreviewError(err instanceof Error ? err.message : String(err)); + } + }; + + if (loading) { + return ( +
+ + +
+ ); + } + if (!policy) { + return
{t("privacy.loadFailed")}
; + } + + return ( +
+ {/* Master toggle */} +
+
+
+ +
+
+

{t("privacy.enable")}

+

{t("privacy.enableDesc")}

+
+
+ edit({ enabled: next })} + label={t("privacy.enable")} + /> +
+ + {/* Import warning */} +
+ +

{t("privacy.importWarning")}

+
+ + {/* Built-in detectors */} +
+

+ {t("privacy.detectors.title")} +

+
+ {DETECTOR_KEYS.map((key) => ( +
+
+

+ {t(`privacy.detectors.${key}`)} +

+

+ {t(`privacy.detectors.${key}Desc`)} +

+
+ edit({ detectors: { ...policy.detectors, [key]: next } })} + label={t(`privacy.detectors.${key}`)} + /> +
+ ))} +
+
+ {t("privacy.defaultAction")} + +
+
+ + {/* Custom rules */} +
+

+ {t("privacy.rules.title")} +

+ {policy.rules.length === 0 ? ( +

{t("privacy.rules.empty")}

+ ) : ( +
    + {policy.rules.map((rule, idx) => ( +
  • +
    + + {rule.name} + + + {rule.match_type}~/{rule.pattern}/ → {t(`privacy.actions.${rule.action}`)} + +
    +
    + + edit({ + rules: policy.rules.map((r, i) => + i === idx ? { ...r, enabled: next } : r + ), + }) + } + label={rule.name} + /> + +
    +
    • + ))} +
+ )} + +
+ setNewRule((r) => ({ ...r, name: e.target.value }))} + placeholder={t("privacy.rules.namePlaceholder")} + className="input text-xs" + aria-label={t("privacy.rules.name")} + /> + + setNewRule((r) => ({ ...r, pattern: e.target.value }))} + placeholder={t("privacy.rules.patternPlaceholder")} + className="input text-xs font-mono" + aria-label={t("privacy.rules.pattern")} + /> + + +
+
+ + {/* Preview */} +
+

+ {t("privacy.preview.title")} +

+