From 3737811502f61a8e06d76c125a9d708ab5825737 Mon Sep 17 00:00:00 2001 From: stefanbaxter Date: Wed, 8 Jul 2026 13:40:33 +0000 Subject: [PATCH 1/4] feat(080): row-type discovery probe + template-seeded generation (cxs2 spec 080) - NEW POST /v1/discover-row-types: exact two-pass row-type discovery (novelty = full snapshot every run; stats via per-table cursors: semantic_events (timestamp,event_gid), data_points (timestamp,signature), entities _version + uniqExact(gid) snapshot counts); monotone map-key/ JSON-path inventories; series head-label enrichment (headSchema). - NEW GET /v1/published-template: the published global template by name (the FR-020 maturation package's seed). - smart-generate: optional template_name (seed via buildCubesFromTemplate, probe-pruned by the request filters) + cube_meta (family-#2 provenance, replaces the 013 default_model/template_checksum stamp); template path forces skip_llm, disables deep-profile sampling (registry pruning must not flap), forces the target table into internalTables (partition scoping), and merges regenerations via mergeTemplateModel so ai_generated/team content survives. - buildCubesFromTemplate: additive filters/cubeName/cubeMeta options. - FraiOS JIT provisioning: the row-type-pipeline provider provisions as team ADMIN (dataschemas select/update perms are owner/admin-only) with an upgrade path for already-provisioned identities. Verified live from cxs2 e2e (somi.is, 13.9M-row semantic_events + 2.4M-row data_points): cxs2 suites US2 6/6, US3 10/10, US5 2/2. cubejs npm test: 620/622 (2 pre-existing partition.test.js failures on clean main). Co-Authored-By: Claude Fable 5 --- docs/tychi-cube-test-setup.md | 499 ++++++++++++++++++ .../cubejs/src/routes/discoverRowTypes.js | 242 +++++++++ services/cubejs/src/routes/index.js | 12 + .../cubejs/src/routes/publishedTemplate.js | 44 ++ services/cubejs/src/routes/smartGenerate.js | 132 ++++- .../cubejs/src/utils/dataSourceHelpers.js | 27 +- .../src/utils/smart-generation/cubeBuilder.js | 65 ++- .../smart-generation/templateResolver.js | 70 +++ 8 files changed, 1046 insertions(+), 45 deletions(-) create mode 100644 docs/tychi-cube-test-setup.md create mode 100644 services/cubejs/src/routes/discoverRowTypes.js create mode 100644 services/cubejs/src/routes/publishedTemplate.js create mode 100644 services/cubejs/src/utils/smart-generation/templateResolver.js diff --git a/docs/tychi-cube-test-setup.md b/docs/tychi-cube-test-setup.md new file mode 100644 index 00000000..7ad10ff4 --- /dev/null +++ b/docs/tychi-cube-test-setup.md @@ -0,0 +1,499 @@ +# Tychi × Cube Test Setup + +Single-page handover so any new session can pick up where this one left off: +making Tychi reliably build a data-exploration dashboard from a natural-language +prompt, against the Synmetrix semantic layer, via the cxs2 MCP. + +--- + +## Goal + +The end-to-end test: + +> User in cxs2 chat (logged in to "Blue Car Rental" / `blue.is` partition): +> **"I would like to explore popular POIs last summer"** + +Expected behaviour: +1. Dex (orchestrator) delegates to Tychi via `communicate(['tychi'])`. +2. Tychi invokes the `data-exploration` skill. +3. Tychi calls `cube_list` (cxs MCP, scoped to active partition). +4. Tychi calls `cube_discover(action="describe", cubeName=)` for the cube it picks (today: `bluecar_stays`). +5. Tychi calls `browser_render_data_visualization` with a config built from the discovered schema. +6. The cxs2 canvas renders a multi-section dashboard (KPIs, time series, map, top-N tables) — verdict `PASS` or `PASS_WITH_WARNINGS`. + +Final-state screenshot saved this session: `dashboard-rendered.png` at synmetrix repo root (untracked). + +--- + +## Repos & roles + +| Repo | Role | Local path | +|---|---|---| +| **synmetrix** | Backend: Hasura + cubejs + actions + cubestore + Postgres. Owns `/api/v1/meta-all`, `/api/v1/discover`, security context, partition-scoped data access. | `/Users/stefanbaxter/Development/synmetrix` | +| **client-v2** | Synmetrix legacy frontend (Vite/React). Talks directly to local synmetrix on port 8000. Used for direct semantic-layer authoring/exploration. | `/Users/stefanbaxter/Development/client-v2` | +| **cxs2** | FraiOS app (Next.js + Convex + WorkOS). Hosts chat, agent canvas, MCP server (`/api/mcp/v1/messages`), generative-UI render tools. The `cube_list` / `cube_discover` / `cube_query` tools that Tychi sees are defined here, in `src/lib/services/mcp/component-tools/cube-analytics/`. | `/Users/stefanbaxter/Development/cxs2` | +| **cxs-agents** | Python orchestrator (Tychi, Dex, Provi …). Connects to cxs2's MCP via `cxs-platform` MCP client. Skill files for Tychi at `agents/Tychi/skills/`. | `/Users/stefanbaxter/Development/cxs-agents` | + +--- + +## Pre-flight (do these once on a fresh session) + +1. **Confirm `agentscope==1.0.18` in the cxs-agents venv.** Stale 1.0.11 was + the cause-of-everything in the previous debugging session + (`Toolkit.register_tool_function() got an unexpected keyword argument + 'func_name'` swallowed by a broad `except`, no CXS MCP attaches, agent + thrashes on `search_tools`): + ```bash + ~/Development/cxs-agents/.venv/bin/pip show agentscope | grep Version + # if < 1.0.18: + ~/Development/cxs-agents/.venv/bin/pip install -r ~/Development/cxs-agents/requirements.txt --upgrade + ``` + +2. **Start cxs2's local infra (Redis + Qdrant + Convex studio) before the dev + server**: + ```bash + cd ~/Development/cxs2 && bun run dev:infra:start && bun run dev:https + ``` + +3. **Point `kubectl` at the dev cluster** if you'll need cluster-admin + queries (only relevant if `meta-all` returns empty and you need to + inspect Hasura state on `dbx.fraios.dev`): + ```bash + kubectl config get-contexts # should list a context for the dev cluster + kubectl config use-context + kubectl get pods -n synmetrix # should list synmetrix-* pods + ``` + +4. **ClickHouse port-forward** (synmetrix CubeJS reaches it via + `host.docker.internal:18123`): + ```bash + kubectl port-forward -n clickhouse svc/clickhouse-fraios-clickhouse 18123:8123 & + ``` + +5. **Log into cxs2 as the right org.** The canonical test assumes you're in + the **Blue Car Rental** organization (synmetrix partition `blue.is`). The + cluster-side data state below is keyed to that team; testing under a + different org will show different cubes. + +6. **Restart the orchestrator after any cxs-agents code change** — Python + doesn't hot-reload: + ```bash + ~/Development/cxs-agents/utils/devctl.sh orchestrator restart + ``` + Cubejs (synmetrix) and cxs2 (Next.js) DO hot-reload, no restart needed. + +--- + +## Local stack + +All four run concurrently: + +| Component | How to run | Port | Notes | +|---|---|---|---| +| Synmetrix backend (hasura, cubejs, actions, cubestore, postgres) | `./cli.sh compose up` (in synmetrix) | 8080 (hasura), 4000 (cubejs), 3001 (actions), …| Cubejs uses `yarn start.dev` w/ nodemon — source-mounted, hot-reloads on local edits. | +| client-v2 (Vite) | `cd ../client-v2 && yarn dev` | 8000 | Independent of cxs2; dev-only. | +| cxs-agents orchestrator | `./utils/devctl.sh orchestrator restart` | 8090 | Restart picks up Python edits. Default agent for `agent restart` is now `orchestrator` (was `baseline`). | +| cxs2 (Next.js HTTPS) | `bun run dev:https` (or already running) | 3000 (HTTPS at `https://local.fraios.dev:3000`) | Next.js + Turbopack hot-reloads. Cert is `local.fraios.dev` → `localhost` won't validate. | +| ClickHouse port-forward | `kubectl port-forward -n clickhouse svc/clickhouse-fraios-clickhouse 18123:8123` | 18123 | Memory note: `dbx.fraios.dev` Tailscale address is deprecated; this kubectl forward is the current method. Synmetrix CubeJS reaches it via `host.docker.internal:18123`. | + +Health checks: + +```bash +curl -sf -o /dev/null -w "cubejs %{http_code}\n" -m 3 http://localhost:4000/readyz +curl -sf -o /dev/null -w "hasura %{http_code}\n" -m 3 http://localhost:8080/healthz +curl -sf -o /dev/null -w "client-v2 %{http_code}\n" -m 3 http://localhost:8000/ +curl -ksf -o /dev/null -w "cxs2 %{http_code}\n" -m 3 https://local.fraios.dev:3000/ +curl -sf -o /dev/null -w "cxs-agents %{http_code}\n" -m 3 http://localhost:8090/health +``` + +Local synmetrix env (`.env`): +- `JWT_KEY` (HS256 secret used by FraiOS-style local tokens) — `LGB6j3RkoVuOuqKzjgnCeq7vwfqBYJDw` in this dev environment. +- Local Hasura admin secret: `devsecret` (from container env, not in repo `.env`). + +--- + +## Driving the test + +1. Open `https://local.fraios.dev:3000/dashboard` in Playwright (or browser). +2. Cache-bust the URL (`?cb=anything`) on first load — older builds may serve cached HTML referencing missing JS bundles. +3. Type the prompt into the "I would like to..." textbox; submit. +4. Wait ~60–120 s. Monitor `cxs-agents/.logs/orchestrator.log` for tool calls. +5. Expect the right pane (canvas) to populate with a `data-visualization` iframe. + +Useful Playwright bits: + +```js +// Submit programmatically (UI controls are flaky) +const tb = document.querySelector('input[type="text"], textarea, [role="textbox"]'); +const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype,'value')?.set + || Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype,'value')?.set; +setter.call(tb, 'I would like to explore popular POIs last summer'); +tb.dispatchEvent(new Event('input', {bubbles:true})); +tb.dispatchEvent(new KeyboardEvent('keydown',{key:'Enter',bubbles:true})); +``` + +--- + +## Fastest A-Z test (one prompt, two grades) + +The whole test boils down to **one prompt + one grader command**. Two +separable grades come out of every run: + +- **A — Ability**: did the chain reach a render? (`cube_list` ≥1, + `cube_discover` ≥1, `browser_render_data_visualization` ≥1) +- **B — Quality**: did Tychi consult the dashboard reference material + before authoring, and what did the cxs2 validator say + (`PASS` / `PASS_WITH_WARNINGS` / `FAIL`, plus counts of `autoFixed`, + `warnings`, `remainingIssues`)? + +### Recipe + +```bash +# 1) Snapshot the log offset BEFORE the prompt +OFF=$(wc -c < ~/Development/cxs-agents/.logs/orchestrator.log) + +# 2) Drive the prompt in the cxs2 chat (Playwright snippet above, or by hand) +# Wait ~60–120s for Tychi to finish. + +# 3) Score the slice — six greps, paste-and-read +LOG=~/Development/cxs-agents/.logs/orchestrator.log +SLICE=$(mktemp); tail -c "+$OFF" "$LOG" > "$SLICE" + +# Infrastructure +echo "cxs MCP attaches: $(grep -ac 'attached.*CXS MCP tools' "$SLICE")" +echo "cxs MCP no-cached misses: $(grep -ac 'no cached CXS MCP clients' "$SLICE")" +echo "cxs MCP TypeError failures: $(grep -ac 'Failed to initialize CXS MCP' "$SLICE")" +echo " (TypeError > 0 → agentscope venv stale; reinstall)" + +# A) ability — tool counts +echo +echo "=== A: ABILITY ===" +grep -aoE '"name":[[:space:]]*"(cube_list|cube_discover|cube_query|browser_render_data_visualization|read_skill_file)"' "$SLICE" \ + | sort | uniq -c | sort -rn + +# B) quality — verdict + reference reads +echo +echo "=== B: QUALITY ===" +echo -n "validator verdict: " +grep -aoE '\\"verdict\\":[[:space:]]*\\"(PASS_WITH_WARNINGS|PASS|FAIL)\\"|"verdict":[[:space:]]*"(PASS_WITH_WARNINGS|PASS|FAIL)"' "$SLICE" \ + | tail -1 | grep -oE '(PASS_WITH_WARNINGS|PASS|FAIL)' | head -1 || echo "(no render reached)" +echo "reference files read:" +grep -aoE '"name":[[:space:]]*"read_skill_file"[^}]*"file_path":[[:space:]]*"[^"]+\.md"' "$SLICE" \ + | grep -oE '[a-z0-9_-]+\.md' | sort | uniq -c | sort -rn || echo " (none)" + +# Side-channel +echo +echo "=== SIDE-CHANNEL ===" +echo "skill invocations: $(grep -ac 'SKILL_INVOKE' "$SLICE")" +echo "destructive plan replacements (should be 0): $(grep -ac 'PLAN_SEED_REPLACING_ACTIVE_PLAN' "$SLICE")" + +rm -f "$SLICE" +``` + +Sample of a healthy run: +``` +cxs MCP attaches: 1 +cxs MCP no-cached misses: 0 +cxs MCP TypeError failures: 0 + +=== A: ABILITY === + 1 "name": "browser_render_data_visualization" + 2 "name": "cube_discover" + 1 "name": "cube_list" + 3 "name": "read_skill_file" + +=== B: QUALITY === +validator verdict: PASS_WITH_WARNINGS +reference files read: + 2 dashboard-principles.md + 1 lazy-classification.md + +=== SIDE-CHANNEL === +skill invocations: 1 +destructive plan replacements (should be 0): 0 +``` + +**A grade** = `cube_list ≥1` AND `cube_discover ≥1` AND `render ≥1`. +**B grade** = `verdict ∈ {PASS, PASS_WITH_WARNINGS}` AND `read_skill_file ≥1`. + +Sample of a healthy run: +``` +A) Ability — did the chain run? + cube_list: 1 + cube_discover: 2 + render: 1 + GRADE: PASS + +B) Quality — references + validator + read_skill_file calls: 3 + Files read (top): + 3 dashboard-principles.md + 1 lazy-classification.md + Validator verdict: PASS_WITH_WARNINGS + autoFixed items: 2 + warnings items: 1 + remainingIssues: 0 + GRADE: WARN +``` + +### What "good quality" looks like + +Tychi's `data-exploration` skill ships these references at +`agents/Tychi/skills/data-exploration/references/`: + +| File | Loaded when … | +|---|---| +| `improve-pipeline.md` | Always (validator pipeline overview) | +| `lazy-classification.md` | Field-cardinality decisions | +| (any other reference files added later) | Per their own when-to-read clauses in the SKILL.md | + +If `read_skill_file calls = 0` in the scorecard, Tychi authored the +dashboard purely from the system prompt — that's the most common cause +of low B-grade runs even when verdict is PASS_WITH_WARNINGS. Two ways to +push reference reads up: + +1. **Skill-side** — strengthen the "you MUST `read_skill_file(...)` + before authoring" prose in `data-exploration/SKILL.md`. The + `semantic-layer` skill's `workflow.steps` frontmatter is the + gold-standard pattern (each step prescribes exactly which file to + read with a "MUST quote outcome" clause). +2. **Tool-boundary** — make `browser_render_data_visualization`'s + nextActions, on FAIL, name a specific reference file to read; today + it already does this for sparse dashboards (Tychi/skills/data-exploration + `dashboard-principles.md` mention). + +### What "good ability" looks like + +`A grade = PASS` requires all three of `cube_list`, `cube_discover`, +`browser_render_data_visualization` to fire. Common failure modes and +where to look: + +| Symptom | Likely cause | Where | +|---|---|---| +| `cxs MCP TypeError failures > 0` | agentscope venv stale | `.venv/bin/pip show agentscope` | +| `no cached CXS MCP` > 0 | cxs MCP attach failed silently (or parent session hadn't connected) | `shared/agents/server/stream.py:206` | +| `cube_list` returns empty | partition mismatch on the active team | synmetrix `metaAll.js` + team `settings.partition` (PR #50 widens to also match `team.name`) | +| `cube_discover` not called | tool descriptions don't pull the agent past `cube_list` | cxs2 `cube-analytics/discover.ts` description | +| `render` not called or hallucinated cube | data-exploration skill prose ignored | strengthen with `workflow.steps` | +| `destructive plan replacements > 0` | wrong-skill misroute (semantic-layer fired for catalog) | cxs-agents PR #207 + further description tightening | + +--- + +## Diagnostics + +### Did Tychi attach the CXS MCP tools? + +```bash +grep -aE "attached.*CXS MCP|no cached CXS MCP|Failed to initialize CXS MCP" \ + ~/Development/cxs-agents/.logs/orchestrator.log | tail +``` + +Healthy looks like: +``` +INFO | client.py:96 | MCP client 'cxs-platform' connected with 11 tools +INFO | stream.py:290 | attach_cached_cxs_mcp_tools_to_agent: attached 11 CXS MCP tools (groups=['cxs']) to agent 'tychi' +``` + +Bad looks like: +``` +WARNING | stream.py:1987 | Failed to initialize CXS MCP 'cxs': ... +INFO | stream.py:244 | attach_cached_cxs_mcp_tools_to_agent: no cached CXS MCP clients +``` + +### Are cube tools visible to the LLM at reasoning time? + +`agents/orchestrator/service_adapter.py` emits a `[TOOL_VISIBILITY]` line per Tychi spawn (added this session, INFO-level): + +``` +INFO | service_adapter.py:1107 | [TOOL_VISIBILITY] agent=tychi + groups={'skills':True,'workers':False,'search':False,'files':False, + 'plan_related':False,'cxs':True,'browser':True,'communication':False} + visible=25 hidden=19 cube_visible=['cube_list','cube_discover','cube_query'] +``` + +If `cxs:False` here, the bootstrap pass didn't activate it. If `cube_visible` is empty, the cxs MCP didn't attach. + +### Tool-call counts for a single conversation + +```bash +# Mark log offset, drive prompt, then: +tail -c + ~/Development/cxs-agents/.logs/orchestrator.log \ + | grep -aoE '"name":\s*"[a-z_]+"' | sort | uniq -c | sort -rn +``` + +Healthy run for the canonical prompt looks like: +- ≥1 `cube_list` +- ≥1 `cube_discover` +- ≥1 `browser_render_data_visualization` +- 1 `communicate` (Dex → Tychi delegation) +- multiple `activate_tool_groups` (skills, cxs, browser, plan_related, communication) + +### What cube_list actually returned + +```bash +TOKEN=$(...mint or extract...) # see below +curl -sk -m 15 -H "Authorization: Bearer $TOKEN" \ + https://dbx.fraios.dev/api/v1/meta-all | jq '{ + count: (.datasources|length), + teams: [.datasources[] | {name:.datasource_name, cubes:(.cubes|length)}] + }' +``` + +### Extracting a fresh user JWT via Playwright + +```js +const r = await fetch('/api/v1/auth/token', { credentials: 'include' }); +const { accessToken } = await r.json(); +console.log(accessToken); // RS256 WorkOS JWT, ~30-min lifetime +``` + +This token is accepted by both **dbx.fraios.dev** synmetrix and **localhost:4000** synmetrix (both verify via WorkOS JWKS — local synmetrix is configured for the same WorkOS app). + +### Minting a local FraiOS HS256 JWT (for development/test) + +```bash +cd services/cubejs && node -e " +const jwt = require('jsonwebtoken'); +console.log(jwt.sign({ + accountId: 'test-account', // REQUIRED — detectTokenType uses this + userId: '', + email: '', + partition: '', + iat: Math.floor(Date.now()/1000), + exp: Math.floor(Date.now()/1000)+3600, +}, process.env.JWT_KEY || 'LGB6j3RkoVuOuqKzjgnCeq7vwfqBYJDw', + { algorithm: 'HS256' }));" +``` + +Note: the `JWT_KEY` in synmetrix `.env` may differ from what the running cubejs container has loaded. Check with `docker exec synmetrix-cubejs-1 env | grep JWT_KEY` before relying on a self-minted token. + +`detectTokenType` requires `payload.accountId` to classify a token as FraiOS (HS256). Without that claim, it falls through to `hasura` and is rejected by `/meta-all`. + +--- + +## What's been fixed this session + +All merged unless noted: + +| # | Repo / PR | Title | What it fixes | +|---|---|---|---| +| 1 | synmetrix #48 | fix(actions): prevent unhandledRejection crash on WorkOS refresh failure | Actions service crashed when WorkOS rejected an expired refresh token — `.finally()` chain produced a dangling rejection. Added `.catch(()=>{})`. | +| 2 | synmetrix #49 | fix(cubejs): guard express error handler against already-sent responses | Fallback error handler called `res.status().send()` unconditionally → `Cannot set headers after they are sent`. Added `res.headersSent` guard. | +| 3 | cxs2 #498 | fix(mcp/cube-analytics): treat non-UUID datasourceId/branchId as null | Solution-link metadata sometimes carried placeholder strings like `"primary"`; the resolver short-circuited partition lookup if any value was set. Added a UUID guard so non-UUIDs become `null` and partition resolution runs. | +| 4 | cxs2 #499 | fix(canvas): guard render handler against unresolved cube metadata | When `validateBeforeRender` threw "Cube metadata unavailable" the raw error bubbled to the agent. Now caught and translated into a structured FAIL with prescriptive `nextActions` (call `cube_list`, then `cube_discover`, then retry render). Also normalised tool-name underscores throughout `CanvasTools.ts`. | +| 5 | cxs2 #500 | fix(mcp/cube-analytics): tighten cube_list/cube_discover/cube_query descriptions | Old descriptions read like reference docs ("what I return"); none mentioned trigger words like *explore* or *what's available*. Rewrote each as a tight input/output spec — the LLM derives ordering from the parameter chain (no "Step 1/2/3" prose). This is the change that finally got Tychi to actually call `cube_list`. | +| 6 | cxs-agents #208 | fix(tychi): make tool-call path reliable end-to-end | Three small bundled changes:
(a) Tychi system prompt — drop skill-naming, add anti-hallucination clause ("never assert presence/absence of data without running the tool");
(b) `_log_tool_visibility` diagnostic + INFO-level bootstrap log;
(c) `devctl.sh DEFAULT_AGENT="orchestrator"`. | +| 7 | cxs-agents #207 | fix(skills/semantic-layer): refuse catalog listing — redirect to cxs MCP tools | The semantic-layer skill's plan-seeder destructively replaces the active plan when invoked. Tychi was reaching for it for catalog listing → killing the dashboard plan. Tightened the skill's description and `call_requirement` with a negative requirement at the top. | +| 8 | synmetrix #50 | fix(cubejs/meta-all): widen partition match to team.name OR team.settings.partition | `meta-all` filtered teams by `team.settings.partition === jwt.partition`. A team's settings.partition can drift from its name; we found a `blue.is` team whose settings.partition was left at `"bluecar.is"`. Match now succeeds if EITHER the team name OR the soft setting matches the JWT partition — single-team scope preserved. Threaded `team.name` through the `findUser` GraphQL projection. | + +Cluster-side state changes on `dbx.fraios.dev` (no PR — direct Hasura admin): +- Copied 2 datasources + 4 dataschemas from `bluecar.is` team to `blue.is` team. +- Promoted `stefan@snjallgogn.is` → `owner` of `blue.is`. +- Corrected `blue.is.team.settings.partition` from `"bluecar.is"` → `"blue.is"`. + +Two upstream dependency changes (no PR but worth recording): +- **`agentscope: 1.0.11 → 1.0.18`** in cxs-agents venv (`requirements.txt` already pinned 1.0.18; venv was stale). 1.0.11's `Toolkit.register_tool_function` lacked the `func_name` kwarg — every CXS MCP attach raised `TypeError`, the broad catch swallowed it, and the cache stayed empty. **One pinned-version mismatch caused: search-tools loops, wrong-skill pivots, hallucinated cube names, stuck plans, duplicated plans.** If the test starts misbehaving again, check this version first (`/Users/stefanbaxter/Development/cxs-agents/.venv/bin/pip show agentscope`). +- Confirmed `bootstrap_groups=["skills","cxs"]` in `agents/tychi/config.py` is honoured by `_post_integration_bootstrap` after the MCP attach completes (it runs AFTER `attach_cached_cxs_mcp_tools_to_agent`, so the `cxs` group exists by then). + +--- + +## Open improvements / known gaps + +These are good candidates for the next session: + +1. **`cube_list` initial empty result.** First `cube_list` per Tychi spawn occasionally returns `{datasources:[]}` because synmetrix's `findUser` 30 s cache served stale data from a prior request before partition was right. Hits the cache miss on retry. Worth surfacing the cache hit/miss in logs, or invalidating on auth state change. + +2. **Tychi's skill machinery still emits a stale `create_plan()` empty-args call once per run** (visible as `Error: PlanNotebook.create_plan() missing 4 required positional arguments`). Doesn't break anything but is noisy. Look at `agents/orchestrator/service_adapter.py` plan seeding around skill invocation. + +3. **`semantic-layer` skill's destructive plan replacement** (`SKILL_PLAN_SEED_REPLACING_ACTIVE_PLAN`) is still possible if Tychi misroutes. PR #207 makes the description discourage it; the harder fix is to make plan-seeding non-destructive when an active plan exists for the conversation. + +4. **`data-exploration` skill prose still says "Activate silently on entry: cxs and browser. If either group is absent, the user is not in cxs2 — say so once and stop."** That's now structurally false (groups are pre-activated by `bootstrap_groups`) and gives the LLM a license to bail. Worth a follow-up edit in `agents/Tychi/skills/data-exploration/SKILL.md`. + +5. **No `workflow.steps` on `data-exploration`.** The semantic-layer skill has a frontmatter `workflow.steps` that the runtime seeds as a plan with examiner-enforced "MUST quote outcome" clauses. data-exploration only has prose. Adding a 3-step contract (`cube_list` → `cube_discover` → `browser_render_data_visualization`) would be a structural belt-and-braces over the description-side fix that already works. + +6. **`Toolkit.register_tool_function` failure-mode** — the broad `except BaseException` in `_connect_single_cxs_mcp` swallowed the TypeError silently for many sessions before we found it. Worth narrowing to known network/timeout errors so future API drifts surface immediately. + +7. **Local meta-all retest of synmetrix #50.** The fix is on `main` and live in the local cubejs (nodemon picked it up), but the local end-to-end test ran into a JWT signature mismatch (self-minted token vs running container's `JWT_KEY`). Easiest path: extract a real session token from cxs2 via `/api/v1/auth/token` and curl `localhost:4000/api/v1/meta-all` with that. + +8. **Tychi's PERSONA.md is stale** — describes a `delegate_to_specialist` architecture with `pipeline`/`cube`/`dev`/`explorer` specialists that have all been disabled (`Tychi specialists disabled (pipeline/explorer/dev). Set TYCHI_ENABLE_SPECIALISTS=1 to re-enable.`). System prompt is correctly loaded from `prompts/system.txt` (PERSONA.md is fallback only) but the file should be deleted or rewritten so it doesn't mislead future readers. + +--- + +## Useful one-liners + +```bash +# Tail orchestrator log filtered for the current investigation +tail -f ~/Development/cxs-agents/.logs/orchestrator.log \ + | grep --line-buffered -aE "TOOL_VISIBILITY|attached.*CXS|SKILL_INVOKE|cube_list|cube_discover|browser_render|verdict|PLAN_SEED" + +# Count tool calls in current run +tail -c + ~/Development/cxs-agents/.logs/orchestrator.log \ + | grep -aoE '"name":\s*"[a-z_]+"' | sort | uniq -c | sort -rn + +# Verify Tychi config has cxs in bootstrap_groups +grep -A2 bootstrap_groups ~/Development/cxs-agents/agents/tychi/config.py + +# Verify agentscope version (must be ≥ 1.0.18) +~/Development/cxs-agents/.venv/bin/pip show agentscope | grep Version + +# Check active synmetrix branch +git -C ~/Development/synmetrix branch --show-current + +# Restart orchestrator (default is now `orchestrator`, was `baseline`) +~/Development/cxs-agents/utils/devctl.sh orchestrator restart +``` + +--- + +## Cluster admin (when needed) + +The `dbx.fraios.dev` cluster's Hasura is reachable via port-forward + admin secret stored in a k8s Secret. The harness asks for explicit user authorization before extracting credentials each session. + +```bash +# Stash admin secret (requires user-confirmed permission scope) +kubectl get secret -n synmetrix synmetrix-secrets -o json \ + | jq -r '.data.HASURA_GRAPHQL_ADMIN_SECRET' | base64 -d > /tmp/hadmin + +# Port-forward Hasura +kubectl port-forward -n synmetrix svc/synmetrix-hasura 18080:8080 & + +# Query +curl -s http://localhost:18080/v1/graphql \ + -H "x-hasura-admin-secret: $(cat /tmp/hadmin)" \ + -H "Content-Type: application/json" \ + -d '{"query":"query{ teams(where:{name:{_eq:\"blue.is\"}}){ id name settings members{ user{ display_name } member_roles{ team_role } } datasources{ id name } } }"}' +``` + +`blue.is` reference IDs (current state on `dbx.fraios.dev`): +- team_id: `c523ec6d-cf6d-4bb1-b5db-cb90d76980d2` +- `dev-clickhouse` datasource_id: `394db2d6-bf14-4517-ac33-f605f73871d6` (1 cube: `bluecar_stays`) +- `prod` datasource_id: `bdf28038-2f63-4203-ae40-4042c44db495` (0 cubes) +- branch_id (dev-clickhouse `main`): `fecbdd1f-68f6-4083-8df2-1c54c55c9911` +- version_id (dev-clickhouse latest): `94f83d27-ca8d-4036-b4eb-1948099e44c5` +- stefan@snjallgogn.is user_id: `3319c92b-9310-43b5-92b9-e6789c497b4f` (`owner` of `blue.is`) + +--- + +## Where everything lives + +Code paths worth knowing: + +| What | Path | +|---|---| +| `/api/v1/meta-all` handler | `synmetrix/services/cubejs/src/routes/metaAll.js` | +| `findUser` + `userQuery` GraphQL | `synmetrix/services/cubejs/src/utils/dataSourceHelpers.js` (line ~244) | +| `provisionUserFromWorkOS` / `provisionUserFromFraiOS` | same file (line ~653 / ~808) | +| `defineUserScope` (request-time auth) | `synmetrix/services/cubejs/src/utils/defineUserScope.js` | +| `resolvePartitionTeamIds` (other routes still use this) | `synmetrix/services/cubejs/src/routes/discover.js` (line ~42) | +| cxs MCP cube tools (descriptions, schemas, handlers) | `cxs2/src/lib/services/mcp/component-tools/cube-analytics/{list,discover,query}.ts` | +| cxs MCP solution-link resolution (with UUID guard) | `cxs2/src/lib/services/mcp/component-tools/cube-analytics/cube-meta.ts` | +| `browser_render_data_visualization` handler (with order-guard nextActions) | `cxs2/src/lib/rpc/CanvasTools.ts` (~line 1516) | +| `validateBeforeRender` | `cxs2/src/lib/dashboard-adjuster/validate-before-render.ts` | +| Tychi system prompt | `cxs-agents/agents/Tychi/prompts/system.txt` | +| Tychi config (incl. `bootstrap_groups`) | `cxs-agents/agents/tychi/config.py` | +| `data-exploration` skill | `cxs-agents/agents/Tychi/skills/data-exploration/SKILL.md` | +| `semantic-layer` skill (with negative-requirement frontmatter) | `cxs-agents/agents/Tychi/skills/semantic-layer/SKILL.md` | +| Orchestrator service adapter (bootstrap + tool-visibility log + cxs MCP attach) | `cxs-agents/agents/orchestrator/service_adapter.py` | +| `attach_cached_cxs_mcp_tools_to_agent` (Tychi inherits parent's MCP) | `cxs-agents/shared/agents/server/stream.py` (~line 206) | +| `_post_integration_bootstrap` (re-applies bootstrap_groups after late integrations) | `cxs-agents/agents/orchestrator/service_adapter.py` (~line 493) | +| devctl (orchestrator default) | `cxs-agents/utils/devctl.sh` | diff --git a/services/cubejs/src/routes/discoverRowTypes.js b/services/cubejs/src/routes/discoverRowTypes.js new file mode 100644 index 00000000..cfc770f1 --- /dev/null +++ b/services/cubejs/src/routes/discoverRowTypes.js @@ -0,0 +1,242 @@ +/** + * Row-type discovery probe (cxs2 spec 080, research D3 — additive). + * + * POST /api/v1/discover-row-types + * Body: { table: 'semantic_events'|'entities'|'data_points', schema, since?, limitPerType? } + * + * Two EXACT passes per run (never sampled): + * 1. Novelty snapshot (cursor-independent, every run): full GROUP BY over the + * discriminator — cheap on LowCardinality/leading-PK columns — so a row + * type present in the data is NEVER missed regardless of cursor semantics. + * 2. Stats pass (per-table strategy): + * - semantic_events: delta via compound cursor (timestamp, event_gid), + * tie-safe strictly-greater tuple compare; counts accumulate caller-side. + * - data_points: delta via compound cursor (timestamp, signature); same. + * - entities: NO row timestamp exists — `_version` (monotone epoch-ms) + * cursor for change detection; counts are a CURRENT-STATE snapshot + * (uniqExact(gid) — ReplacingMergeTree semantics), returned by the + * novelty pass. + * The delta scan also gathers the observed inventory (map keys per Map + * column + JSON paths, both capped) for the changed slice only. + * + * The next cursor derives from the SCAN BOUNDS (max seen tuple), never wall + * clock. Read-only; partition scoping comes from the caller's token scope, + * same as profile-table. + */ + +const IDENT = /^[A-Za-z_][A-Za-z0-9_]*$/; + +const TABLES = { + semantic_events: { + discriminator: 'event', + evidence: 'type', + tsExpr: 'toUnixTimestamp64Milli(timestamp)', + tie: 'event_gid', + countMode: 'accumulate', + }, + entities: { + discriminator: 'type', + // `_version` IS monotone epoch-ms (entities.sql:112) — the seen-timestamp + // axis and the change-detection cursor in one. + tsExpr: '_version', + versionExpr: '_version', + countMode: 'snapshot', + snapshotCountExpr: 'uniqExact(gid)', + }, + data_points: { + discriminator: 'series_gid', + tsExpr: 'toUnixTimestamp(timestamp) * 1000', + tie: 'signature', + countMode: 'accumulate', + }, +}; + +const escapeSql = (v) => String(v).replace(/\\/g, '\\\\').replace(/'/g, "\\'"); + +export default async (req, res, cubejs) => { + const { securityContext } = req; + const { table, schema, since, limitPerType, headSchema } = req.body || {}; + + const config = TABLES[table]; + if (!config) { + return res.status(400).json({ + code: 'discover_row_types_bad_table', + message: `table must be one of: ${Object.keys(TABLES).join(', ')}`, + }); + } + if (!schema || !IDENT.test(schema)) { + return res.status(400).json({ + code: 'discover_row_types_bad_schema', + message: 'A valid schema (database) name is required.', + }); + } + + const mapKeyCap = Math.min(Math.max(Number(limitPerType) || 100, 10), 500); + const jsonPathCap = 200; + + let driver; + try { + const partition = securityContext.userScope?.dataSource?.partition || null; + if (!partition) { + return res.status(403).json({ + code: 'discover_row_types_no_partition', + message: 'The caller token resolves no partition scope.', + }); + } + + driver = await cubejs.options.driverFactory({ securityContext }); + const qualified = `${schema}.${table}`; + const partitionCond = `partition = '${escapeSql(partition)}'`; + const disc = config.discriminator; + + // Discover the open-ended surfaces dynamically from the live DDL. + const described = await driver.query(`DESCRIBE TABLE ${qualified}`); + const mapColumns = described + .filter((c) => IDENT.test(c.name) && /^Map\(/.test(c.type)) + .map((c) => c.name); + const jsonColumns = described + .filter((c) => IDENT.test(c.name) && /^(JSON|Object\('json'\))/.test(c.type)) + .map((c) => c.name); + + // ── Pass 1: novelty snapshot (every run, cursor-independent) ── + const noveltySelect = [ + `${disc} AS key`, + `min(${config.tsExpr}) AS min_ts`, + `max(${config.tsExpr}) AS max_ts`, + ...(config.evidence ? [`groupUniqArray(16)(${config.evidence}) AS observed_types`] : []), + ...(config.countMode === 'snapshot' ? [`${config.snapshotCountExpr} AS snapshot_count`] : []), + ].join(', '); + const noveltyRows = await driver.query( + `SELECT ${noveltySelect} FROM ${qualified} WHERE ${partitionCond} GROUP BY key` + ); + + // ── Pass 2: stats delta (per-table cursor strategy) ── + let deltaCond; + let cursorBoundSelect; + if (config.versionExpr) { + const sinceVersion = Number(since?.version) || 0; + deltaCond = sinceVersion > 0 ? `${config.versionExpr} > ${sinceVersion}` : null; + cursorBoundSelect = `max(${config.versionExpr}) AS bound_ts`; + } else { + const sinceTs = Number(since?.timestamp) || 0; + const sinceTie = typeof since?.tie === 'string' ? since.tie : ''; + deltaCond = + sinceTs > 0 + ? `(${config.tsExpr}, toString(${config.tie})) > (${sinceTs}, '${escapeSql(sinceTie)}')` + : null; + // argMax over the (ts, tie) tuple gives the tie-breaker AT the scan bound + cursorBoundSelect = + `max(${config.tsExpr}) AS bound_ts, ` + + `argMax(toString(${config.tie}), tuple(${config.tsExpr}, toString(${config.tie}))) AS bound_tie`; + } + + const inventorySelects = [ + ...mapColumns.map( + (col) => `arraySort(groupUniqArrayArray(${mapKeyCap})(mapKeys(${col}))) AS mk_${col}` + ), + ...jsonColumns.map( + (col) => `arraySort(groupUniqArrayArray(${jsonPathCap + 1})(JSONAllPaths(${col}))) AS jp_${col}` + ), + ]; + const statsWhere = [partitionCond, ...(deltaCond ? [deltaCond] : [])].join(' AND '); + const statsRows = await driver.query( + `SELECT ${[ + `${disc} AS key`, + 'count() AS delta_count', + `min(${config.versionExpr || config.tsExpr}) AS min_ts`, + `max(${config.versionExpr || config.tsExpr}) AS max_ts`, + ...(config.evidence ? [`groupUniqArray(16)(${config.evidence}) AS observed_types`] : []), + ...inventorySelects, + ].join(', ')} FROM ${qualified} WHERE ${statsWhere} GROUP BY key` + ); + + // Series head-label enrichment (cxs2 080 D12): data_points row types are + // gids — resolve display labels from the series head table when it is + // reachable. The head database is config-driven per environment + // (`cst` vs `ql`); a missing head table is non-fatal (labels stay gids). + const labelsByKey = new Map(); + if (table === 'data_points') { + const headDb = + typeof headSchema === 'string' && IDENT.test(headSchema) ? headSchema : schema; + try { + const rows = await driver.query( + `SELECT toString(gid) AS gid, anyLast(label) AS label ` + + `FROM ${headDb}.timeseries WHERE ${partitionCond} GROUP BY gid` + ); + for (const row of rows || []) { + if (row.label) labelsByKey.set(String(row.gid), String(row.label)); + } + } catch (err) { + console.warn(`[discoverRowTypes] series head lookup failed (non-fatal): ${err.message}`); + } + } + + // Next cursor = scan bound of the whole table slice (not wall clock). + const boundRows = await driver.query( + `SELECT ${cursorBoundSelect} FROM ${qualified} WHERE ${partitionCond}` + ); + const boundTs = Number(boundRows?.[0]?.bound_ts) || 0; + let cursor = since ?? null; + if (boundTs > 0) { + cursor = config.versionExpr + ? { version: boundTs } + : { timestamp: boundTs, tie: String(boundRows?.[0]?.bound_tie ?? '') }; + } + + const statsByKey = new Map(statsRows.map((r) => [String(r.key), r])); + const rowTypes = noveltyRows.map((novelty) => { + const key = String(novelty.key); + const stats = statsByKey.get(key) || null; + const inventory = { mapKeys: {}, jsonPaths: [] }; + let jsonPathsTruncated = false; + if (stats) { + for (const col of mapColumns) { + const keys = stats[`mk_${col}`]; + if (Array.isArray(keys) && keys.length > 0) inventory.mapKeys[col] = keys.map(String); + } + for (const col of jsonColumns) { + const paths = stats[`jp_${col}`]; + if (Array.isArray(paths) && paths.length > 0) { + for (const p of paths) { + if (inventory.jsonPaths.length >= jsonPathCap) { + jsonPathsTruncated = true; + break; + } + inventory.jsonPaths.push(String(p)); + } + } + } + inventory.jsonPaths.sort(); + } + const observedTypes = (stats?.observed_types ?? novelty.observed_types ?? []).map(String); + const label = labelsByKey.get(key); + return { + key, + ...(label ? { label } : {}), + ...(config.evidence ? { observedTypes } : {}), + ...(config.countMode === 'snapshot' + ? { snapshotCount: Number(novelty.snapshot_count) || 0 } + : { deltaCount: Number(stats?.delta_count) || 0 }), + minTs: Number(novelty.min_ts) || 0, + maxTs: Number(novelty.max_ts) || 0, + inventory: { + ...(Object.keys(inventory.mapKeys).length > 0 ? { mapKeys: inventory.mapKeys } : {}), + jsonPaths: inventory.jsonPaths, + ...(jsonPathsTruncated ? { jsonPathsTruncated: true } : {}), + }, + }; + }); + + return res.json({ table, rowTypes, cursor }); + } catch (err) { + console.error('[discoverRowTypes]', err); + return res.status(500).json({ + code: 'discover_row_types_error', + message: err.message || String(err), + }); + } finally { + if (driver && driver.release) { + await driver.release(); + } + } +}; diff --git a/services/cubejs/src/routes/index.js b/services/cubejs/src/routes/index.js index 9bc51961..6edeff77 100644 --- a/services/cubejs/src/routes/index.js +++ b/services/cubejs/src/routes/index.js @@ -36,6 +36,8 @@ import runSql from "./runSql.js"; import smartGenerate from "./smartGenerate.js"; import columnValues from "./columnValues.js"; import discoverNested from "./discoverNested.js"; +import discoverRowTypes from "./discoverRowTypes.js"; +import publishedTemplate from "./publishedTemplate.js"; import discover from "./discover.js"; import metaAll from "./metaAll.js"; import testConnection from "./testConnection.js"; @@ -278,6 +280,16 @@ export default ({ basePath, cubejs }) => { async (req, res) => discoverNested(req, res, cubejs) ); + router.post( + `${basePath}/v1/discover-row-types`, + checkAuthMiddleware, + async (req, res) => discoverRowTypes(req, res, cubejs) + ); + + router.get(`${basePath}/v1/published-template`, async (req, res) => + publishedTemplate(req, res) + ); + router.post( `${basePath}/v1/validate`, checkAuthMiddleware, diff --git a/services/cubejs/src/routes/publishedTemplate.js b/services/cubejs/src/routes/publishedTemplate.js new file mode 100644 index 00000000..e29b1a4b --- /dev/null +++ b/services/cubejs/src/routes/publishedTemplate.js @@ -0,0 +1,44 @@ +import { verifyAndProvision } from "../utils/directVerifyAuth.js"; +import { fetchPublishedTemplate } from "../utils/smart-generation/templateResolver.js"; + +/** + * GET /api/v1/published-template?name= (cxs2 spec 080 — additive). + * + * Returns the published global template (name, fileName, code, checksum) from + * the platform template datasource — the seed the row-type pipeline passes to + * the modeling agent as part of the FR-020 maturation context. Auth: any + * verified FraiOS/WorkOS identity (global templates are platform content that + * is seeded into every tenant's models; no tenant data is exposed). + */ +export default async (req, res) => { + const auth = await verifyAndProvision(req); + if (auth.error) { + return res.status(auth.error.status).json(auth.error); + } + + const name = typeof req.query?.name === "string" ? req.query.name.trim() : ""; + if (!name) { + return res.status(400).json({ + code: "published_template_missing_name", + message: "The name query parameter is required.", + }); + } + + try { + const template = await fetchPublishedTemplate(name); + if (!template) { + return res.status(404).json({ + code: "published_template_not_found", + message: `No published global template named '${name}' was found.`, + }); + } + const { cubes, ...rest } = template; + return res.json(rest); + } catch (err) { + console.error("[publishedTemplate]", err); + return res.status(500).json({ + code: "published_template_error", + message: err.message || String(err), + }); + } +}; diff --git a/services/cubejs/src/routes/smartGenerate.js b/services/cubejs/src/routes/smartGenerate.js index e5e19b53..68756f89 100644 --- a/services/cubejs/src/routes/smartGenerate.js +++ b/services/cubejs/src/routes/smartGenerate.js @@ -5,7 +5,9 @@ import { import createMd5Hex from '../utils/md5Hex.js'; import { profileTable } from '../utils/smart-generation/profiler.js'; import { detectPrimaryKeys } from '../utils/smart-generation/primaryKeyDetector.js'; -import { buildCubes, mergeAIMetrics, deriveCubeNameFromFlatFilters } from '../utils/smart-generation/cubeBuilder.js'; +import { buildCubes, buildCubesFromTemplate, mergeAIMetrics, deriveCubeNameFromFlatFilters, filtersToSqlConditions } from '../utils/smart-generation/cubeBuilder.js'; +import { fetchPublishedTemplate } from '../utils/smart-generation/templateResolver.js'; +import { mergeTemplateModel } from '../utils/smart-generation/templateMerger.js'; import { generateJs, generateYaml, @@ -99,7 +101,19 @@ export default async (req, res, cubejs) => { ? rawProfileData : null; const dryRun = rawDryRun === true; - const skipLlm = req.body.skip_llm === true; + // 080 (research D2): optional template-seeded generation. When present, the + // named global template is the seed (buildCubesFromTemplate: seed + + // probe-prune against `filters`) and LLM stages are skipped — the scaffold + // is deterministic; the modeling agent matures it out of band. + const templateName = typeof req.body.template_name === 'string' && req.body.template_name.trim() + ? req.body.template_name.trim() + : null; + // 080: optional cube-level provenance meta stamped by the caller (marker + // family #2 — managed_by / row_type / registry_ref). + const cubeMeta = (req.body.cube_meta && typeof req.body.cube_meta === 'object' && !Array.isArray(req.body.cube_meta)) + ? req.body.cube_meta + : null; + const skipLlm = req.body.skip_llm === true || templateName !== null; const filters = Array.isArray(rawFilters) ? rawFilters : []; const nestedFilters = Array.isArray(req.body.nestedFilters) ? req.body.nestedFilters : []; const fileNameOverride = typeof rawFileName === 'string' && rawFileName.trim() ? rawFileName.trim() : null; @@ -140,7 +154,15 @@ export default async (req, res, cubejs) => { try { const { userId } = securityContext; const partition = securityContext.userScope?.dataSource?.partition || null; - const internalTables = securityContext.userScope?.dataSource?.internalTables || []; + let internalTables = securityContext.userScope?.dataSource?.internalTables || []; + // 080: template-seeded generation targets the CANONICAL internal tables by + // definition — force partition scoping for the target table even when the + // team's internalTables setting doesn't enumerate it (tenancy is + // non-negotiable for row-type models; profiling AND the cube source WHERE + // both ride this list). + if (templateName && table && !internalTables.includes(table)) { + internalTables = [...internalTables, table]; + } const emitter = createProgressEmitter(res, req.headers.accept); @@ -202,6 +224,11 @@ export default async (req, res, cubejs) => { filters: [...filters, ...ruleFilters], nestedFilters, emitter, + // 080: template-seeded generation prunes registry members against the + // profiled key inventory — sampling would make that pruning flap on + // rare keys (owner directive: full-but-filtered profiling of a single + // row type). Deep profiling stays exact on this path. + ...(templateName ? { sampleThreshold: Number.MAX_SAFE_INTEGER } : {}), }); emitter.emit('primary_keys', 'Detecting primary keys...', 0.5); @@ -240,17 +267,87 @@ export default async (req, res, cubejs) => { } emitter.emit('building', 'Building cube definitions...', 0.6); - const cubeResult = buildCubes(profiledTable, { - partition, - internalTables, - arrayJoinColumns, - maxMapKeys, - primaryKeys, - cubeName: cubeNameOverride, - filters, - nestedFilters, - aliasColumnNames, - }); + let cubeResult; + let templateSeed = null; + if (templateName) { + // 080 (research D2): template-seeded path — resolve the published global + // template and generate via buildCubesFromTemplate (seed + probe-prune + // against `filters`). Everything after (naming, versioning, no-change + // guard, validation) is the shared pipeline. + templateSeed = await fetchPublishedTemplate(templateName); + if (!templateSeed || templateSeed.cubes.length === 0) { + return res.status(400).json({ + code: 'smart_generate_template_not_found', + message: `No published global template named '${templateName}' was found.`, + }); + } + const primaryTemplateCube = templateSeed.cubes[0]; + + // Registry-path pruning needs JSON path presence for the filtered slice + // (reconciler parity — 014 FR-010): probe JSONAllPaths per registry + // column, scoped by partition + the row-type filters. + const registryPathColumns = new Set(); + for (const list of [primaryTemplateCube.dimensions, primaryTemplateCube.measures, primaryTemplateCube.segments]) { + for (const field of list || []) { + const raw = field?.meta?.registry_path; + if (!raw) continue; + const match = /^([A-Za-z_][A-Za-z0-9_]*)\./.exec(raw); + if (match) registryPathColumns.add(match[1]); + } + } + if (registryPathColumns.size > 0 && driver && profiledTable && (profiledTable.row_count || 0) > 0) { + const where = []; + if (partition) where.push(`partition = '${String(partition).replace(/'/g, "''")}'`); + if (filters.length > 0) where.push(filtersToSqlConditions(filters)); + profiledTable.jsonPaths = new Set(); + for (const column of registryPathColumns) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(column)) continue; + try { + const rows = await driver.query( + `SELECT DISTINCT arrayJoin(JSONAllPaths(${column})) AS p ` + + `FROM ${schema ? `${schema}.` : ''}${table}` + + (where.length > 0 ? ` WHERE ${where.join(' AND ')}` : '') + ); + for (const row of rows || []) profiledTable.jsonPaths.add(row.p); + } catch (err) { + console.warn(`[smartGenerate] JSON path probe failed for ${column} (non-fatal): ${err.message}`); + } + } + } + + const { cube } = buildCubesFromTemplate(primaryTemplateCube, profiledTable, { + partition, + internalTables, + templateName, + templateChecksum: templateSeed.checksum, + filters, + cubeName: cubeNameOverride, + cubeMeta, + }); + cubeResult = { + cubes: [cube], + summary: { + cubes_count: 1, + dimensions_count: (cube.dimensions || []).length, + measures_count: (cube.measures || []).length, + columns_profiled: profiledTable?.columns?.size ?? 0, + columns_skipped: 0, + map_keys_discovered: 0, + }, + }; + } else { + cubeResult = buildCubes(profiledTable, { + partition, + internalTables, + arrayJoinColumns, + maxMapKeys, + primaryKeys, + cubeName: cubeNameOverride, + filters, + nestedFilters, + aliasColumnNames, + }); + } // Store filters in cube-level meta for provenance tracking if (filters.length > 0) { @@ -711,7 +808,12 @@ export default async (req, res, cubejs) => { // existing model that has FILTER_PARAMS on array columns will break. // Force replace to start clean. const effectiveMergeStrategy = nestedFilters.length > 0 ? 'replace' : mergeStrategy; - if (existingCode && effectiveMergeStrategy !== 'replace') { + if (templateName) { + // 080 (FR-012): template-seeded regenerations merge with provenance + // classes — template-owned converges to the template, auto_generated + // refreshes from the probe, ai_generated/team-added members SURVIVE. + finalYaml = mergeTemplateModel(existingCode, yamlContent); + } else if (existingCode && effectiveMergeStrategy !== 'replace') { finalYaml = mergeModels(existingCode, yamlContent, effectiveMergeStrategy); } diff --git a/services/cubejs/src/utils/dataSourceHelpers.js b/services/cubejs/src/utils/dataSourceHelpers.js index 4391dfee..cc5b08f6 100644 --- a/services/cubejs/src/utils/dataSourceHelpers.js +++ b/services/cubejs/src/utils/dataSourceHelpers.js @@ -587,7 +587,7 @@ async function findTeamByName(name) { * Matches the Actions flow's ensureOrgTeam() in provision.js:230. * Handles: existing account with missing membership, or membership with missing role. */ -async function ensureTeamMembership(userId, email, partition) { +async function ensureTeamMembership(userId, email, partition, teamRole = "member") { const teamName = deriveTeamName(email, partition); let team = await findTeamByName(teamName); @@ -617,7 +617,9 @@ async function ensureTeamMembership(userId, email, partition) { let memberId = memberResult.data?.insert_members_one?.id; // If on_conflict returned null, membership already exists — look it up - // and ensure member role exists (fixes partial provisioning retry) + // and ensure the required role exists (fixes partial provisioning retry; + // also upgrades an already-provisioned identity when a stronger role is + // required — cxs2 080: the row-type pipeline needs admin for dataschemas). if (!memberId) { const existing = await fetchGraphQL(findMemberByUserAndTeamQuery, { user_id: userId, @@ -626,15 +628,17 @@ async function ensureTeamMembership(userId, email, partition) { const member = existing.data?.members?.[0]; if (member) { memberId = member.id; - // If role already exists, skip - if (member.member_roles?.length > 0) return; + const roles = (member.member_roles || []).map((r) => r.team_role); + // Role already satisfied — nothing to add. + if (roles.includes(teamRole) || roles.includes("owner")) return; + if (teamRole === "member" && roles.length > 0) return; } } if (memberId) { await fetchGraphQL(createMemberRoleMutation, { member_id: memberId, - team_role: "member", + team_role: teamRole, }); } } @@ -809,6 +813,11 @@ export async function provisionUserFromFraiOS(fraiosPayload) { const externalId = fraiosPayload.userId; const email = fraiosPayload.email; const partition = fraiosPayload.partition; + // cxs2 080 (research D7): the row-type pipeline/agent identities write + // dataschemas/versions through the graphql surface — that requires the + // admin team role (dataschemas select/update perms are owner/admin-only). + const teamRole = + fraiosPayload.provider === "row-type-pipeline" ? "admin" : "member"; // Step 1: Check identity cache (reuses workosSubCache) const cached = getWorkosSubCacheEntry(externalId); @@ -818,13 +827,13 @@ export async function provisionUserFromFraiOS(fraiosPayload) { let inflight = inflightProvisions.get(externalId); if (inflight) return inflight; - const provision = _provisionUserFromFraiOS(externalId, email, partition); + const provision = _provisionUserFromFraiOS(externalId, email, partition, teamRole); inflightProvisions.set(externalId, provision); provision.finally(() => inflightProvisions.delete(externalId)); return provision; } -async function _provisionUserFromFraiOS(externalId, email, partition) { +async function _provisionUserFromFraiOS(externalId, email, partition, teamRole = "member") { // Re-check cache after acquiring singleflight const cached = getWorkosSubCacheEntry(externalId); if (cached) return cached.userId; @@ -832,7 +841,7 @@ async function _provisionUserFromFraiOS(externalId, email, partition) { // Step 2: DB lookup by email const account = await findAccountByEmail(email); if (account) { - await ensureTeamMembership(account.user_id, email, partition); + await ensureTeamMembership(account.user_id, email, partition, teamRole); setWorkosSubCacheEntry(externalId, account.user_id, true); return account.user_id; } @@ -907,7 +916,7 @@ async function _provisionUserFromFraiOS(externalId, email, partition) { if (memberId) { await fetchGraphQL(createMemberRoleMutation, { member_id: memberId, - team_role: isTeamCreator ? "owner" : "member", + team_role: isTeamCreator ? "owner" : teamRole, }); } diff --git a/services/cubejs/src/utils/smart-generation/cubeBuilder.js b/services/cubejs/src/utils/smart-generation/cubeBuilder.js index 662f9b25..bae989d0 100644 --- a/services/cubejs/src/utils/smart-generation/cubeBuilder.js +++ b/services/cubejs/src/utils/smart-generation/cubeBuilder.js @@ -206,7 +206,7 @@ function detectNestedLookupKeys(columns) { * @param {Array<{ column: string, operator: string, value: * }>} filters * @returns {string} SQL conditions joined by AND (no WHERE keyword) */ -function filtersToSqlConditions(filters) { +export function filtersToSqlConditions(filters) { const conditions = []; for (const f of filters) { const op = String(f.operator).toUpperCase(); @@ -1556,8 +1556,35 @@ export function buildCubesFromTemplate(templateCube, profiledTable, options = {} internalTables = [], templateName, templateChecksum = null, + // 080 (row-type pipeline, additive): scope the cube source to one row + // type's slice, override the cube name (e.g. `rt_`), and stamp + // caller-supplied provenance meta (marker family #2) INSTEAD of the 013 + // default-model trio. All default to the pre-080 behavior. + filters = [], + cubeName = null, + cubeMeta = null, } = options; + const finalCubeName = cubeName || templateCube.name; + + // Family #2 replaces `default_model`/`template_checksum`; the template's own + // meta (field_policy, slots, …) still carries through underneath, and + // `template` records the seed either way (080 FR-011). + const stampCubeMeta = (templateMeta) => { + if (!cubeMeta) { + return { + ...(templateMeta || {}), + default_model: true, + template: templateName, + template_checksum: templateChecksum, + }; + } + const base = { ...(templateMeta || {}) }; + delete base.default_model; + delete base.template_checksum; + return { ...base, template: templateName, ...cubeMeta }; + }; + const stampTemplateField = (field) => { const meta = { ...(field.meta || {}) }; // template-owned supersedes any stray generation marker @@ -1585,12 +1612,17 @@ export function buildCubesFromTemplate(templateCube, profiledTable, options = {} let source; if (qualifiedTable) { - source = buildCubeSource(schema, table, partition, isInternal, [], []); + source = buildCubeSource(schema, table, partition, isInternal, filters, []); } else if (templateCube.sql) { - // custom-SQL template: wrap so the scope literal still applies - source = partition - ? { sql: `SELECT * FROM (${templateCube.sql}) WHERE partition = '${partition}'` } - : { sql: templateCube.sql }; + // custom-SQL template: wrap so the scope literal (and any row-type + // filters) still applies + const wrapConditions = []; + if (partition) wrapConditions.push(`partition = '${partition}'`); + if (filters.length > 0) wrapConditions.push(filtersToSqlConditions(filters)); + source = + wrapConditions.length > 0 + ? { sql: `SELECT * FROM (${templateCube.sql}) WHERE ${wrapConditions.join(' AND ')}` } + : { sql: templateCube.sql }; } else { source = {}; } @@ -1644,18 +1676,13 @@ export function buildCubesFromTemplate(templateCube, profiledTable, options = {} }; const cube = sanitizeCubeProse({ - name: templateCube.name, + name: finalCubeName, ...(templateCube.title ? { title: templateCube.title } : {}), ...(templateCube.description ? { description: templateCube.description } : {}), ...source, - meta: { - ...(templateCube.meta || {}), - default_model: true, - template: templateName, - template_checksum: templateChecksum, - }, + meta: stampCubeMeta(templateCube.meta), dimensions: templateDims.filter(keepRegistryMember), measures: templateMeasures.filter(keepRegistryMember), ...(templateSegments.length > 0 @@ -1670,7 +1697,8 @@ export function buildCubesFromTemplate(templateCube, profiledTable, options = {} const { cubes: probeCubes } = buildCubes(profiledTable, { partition, internalTables, - cubeName: templateCube.name, + cubeName: finalCubeName, + filters, }); const probeCube = probeCubes[0]; if (probeCube) { @@ -1694,16 +1722,11 @@ export function buildCubesFromTemplate(templateCube, profiledTable, options = {} } const cube = sanitizeCubeProse({ - name: templateCube.name, + name: finalCubeName, ...(templateCube.title ? { title: templateCube.title } : {}), ...(templateCube.description ? { description: templateCube.description } : {}), ...source, - meta: { - ...(templateCube.meta || {}), - default_model: true, - template: templateName, - template_checksum: templateChecksum, - }, + meta: stampCubeMeta(templateCube.meta), dimensions, measures, ...(templateSegments.length > 0 ? { segments: templateSegments } : {}), diff --git a/services/cubejs/src/utils/smart-generation/templateResolver.js b/services/cubejs/src/utils/smart-generation/templateResolver.js new file mode 100644 index 00000000..0f814fd7 --- /dev/null +++ b/services/cubejs/src/utils/smart-generation/templateResolver.js @@ -0,0 +1,70 @@ +import YAML from 'yaml'; + +import { fetchGraphQL } from '../graphql.js'; +import { parseCubesFromJs } from './diffModels.js'; + +/** + * Published-template resolution for template-seeded smart generation (080 D2). + * + * The published template set is the dataschemas on the latest version of the + * active branch of the platform-owned template datasource — the SAME surface + * `fetchPublishedTemplates` (actions, 013 reconciler) consumes; replicated + * here because smart-generate runs in the cubejs service. Keyed by + * `DEFAULT_MODELS_TEMPLATE_DATASOURCE_ID` (already in this service's env). + */ + +const TEMPLATES_QUERY = ` + query ($id: uuid!) { + datasources_by_pk(id: $id) { + id + branches(where: { status: { _eq: active } }, limit: 1) { + id + versions(limit: 1, order_by: { created_at: desc }) { + id + checksum + dataschemas { + id + name + code + } + } + } + } + } +`; + +/** + * Fetch ONE published global template by its base name (file name without + * extension). Returns `{ name, fileName, code, checksum, cubes }` or null + * when no such template is published. + */ +export async function fetchPublishedTemplate(templateName) { + const templateDatasourceId = process.env.DEFAULT_MODELS_TEMPLATE_DATASOURCE_ID; + if (!templateDatasourceId) { + throw new Error('DEFAULT_MODELS_TEMPLATE_DATASOURCE_ID is not configured'); + } + + const res = await fetchGraphQL(TEMPLATES_QUERY, { id: templateDatasourceId }); + const version = res?.data?.datasources_by_pk?.branches?.[0]?.versions?.[0] || null; + const schema = (version?.dataschemas || []).find( + (s) => s.name.replace(/\.(yml|yaml|js)$/i, '') === templateName + ); + if (!schema) return null; + + let cubes = null; + try { + cubes = /\.js$/i.test(schema.name) + ? parseCubesFromJs(schema.code) + : YAML.parse(schema.code)?.cubes; + } catch { + cubes = null; + } + + return { + name: templateName, + fileName: schema.name, + code: schema.code, + checksum: version?.checksum || null, + cubes: cubes || [], + }; +} From 70d88cf8e2587158b4b4be56c863e70742ac156a Mon Sep 17 00:00:00 2001 From: stefanbaxter Date: Wed, 8 Jul 2026 15:00:45 +0000 Subject: [PATCH 2/4] feat(080): type-class discriminator for typeless Segment events + series head labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - discover-row-types (semantic_events): identity = event value, but rows with an empty event register under synthetic 'type:' keys (owner amendment — page/screen/identify/log/custom carry no event name). - discover-row-types (data_points): optional headSchema — series display labels resolved from .timeseries (non-fatal when absent). Co-Authored-By: Claude Fable 5 --- services/cubejs/src/routes/discoverRowTypes.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/cubejs/src/routes/discoverRowTypes.js b/services/cubejs/src/routes/discoverRowTypes.js index cfc770f1..ffc6f396 100644 --- a/services/cubejs/src/routes/discoverRowTypes.js +++ b/services/cubejs/src/routes/discoverRowTypes.js @@ -28,7 +28,12 @@ const IDENT = /^[A-Za-z_][A-Za-z0-9_]*$/; const TABLES = { semantic_events: { - discriminator: 'event', + // Row-type identity = the `event` value when present; Segment-style rows + // without one (page/screen/identify/log/custom) fall back to their TYPE + // CLASS as a synthetic `type:` key (cxs2 080 — owner amendment: + // `type` joins the discriminator; real event names are Title-Case past + // tense and can never collide with the prefix). + discriminator: "if(empty(event), concat('type:', type), event)", evidence: 'type', tsExpr: 'toUnixTimestamp64Milli(timestamp)', tie: 'event_gid', From dc2d65966c3f6a382580163879b91b2ef1ebe318 Mon Sep 17 00:00:00 2001 From: stefanbaxter Date: Wed, 8 Jul 2026 19:19:52 +0000 Subject: [PATCH 3/4] feat(row-types): capture observed source.label set per event type Owner refinement: an event type spans multiple sources across rows (source.label/type are scalar Nullable(String) per row). Add a sourceEvidence config on semantic_events and groupUniqArray(64) over `source.label` in both the novelty + stats passes, surfaced as observedSources alongside observedTypes. groupUniqArray drops NULLs, so sourceless platform events fall out cleanly. Co-Authored-By: Claude Opus 4.8 (1M context) --- services/cubejs/src/routes/discoverRowTypes.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/services/cubejs/src/routes/discoverRowTypes.js b/services/cubejs/src/routes/discoverRowTypes.js index ffc6f396..2fd2efd3 100644 --- a/services/cubejs/src/routes/discoverRowTypes.js +++ b/services/cubejs/src/routes/discoverRowTypes.js @@ -35,6 +35,13 @@ const TABLES = { // tense and can never collide with the prefix). discriminator: "if(empty(event), concat('type:', type), event)", evidence: 'type', + // cxs2 080 (owner refinement): capture the distinct set of SOURCE labels an + // event type is observed from. `source.label` is a scalar Nullable(String) + // per row (the dot is a column-name convention, NOT a Nested/Array column), + // so one event type spans several sources across rows — grouUniqArray drops + // NULLs (system/platform events with no source fall out cleanly). Back-tick + // required because of the dot. + sourceEvidence: '`source.label`', tsExpr: 'toUnixTimestamp64Milli(timestamp)', tie: 'event_gid', countMode: 'accumulate', @@ -109,6 +116,9 @@ export default async (req, res, cubejs) => { `min(${config.tsExpr}) AS min_ts`, `max(${config.tsExpr}) AS max_ts`, ...(config.evidence ? [`groupUniqArray(16)(${config.evidence}) AS observed_types`] : []), + ...(config.sourceEvidence + ? [`groupUniqArray(64)(${config.sourceEvidence}) AS observed_sources`] + : []), ...(config.countMode === 'snapshot' ? [`${config.snapshotCountExpr} AS snapshot_count`] : []), ].join(', '); const noveltyRows = await driver.query( @@ -151,6 +161,9 @@ export default async (req, res, cubejs) => { `min(${config.versionExpr || config.tsExpr}) AS min_ts`, `max(${config.versionExpr || config.tsExpr}) AS max_ts`, ...(config.evidence ? [`groupUniqArray(16)(${config.evidence}) AS observed_types`] : []), + ...(config.sourceEvidence + ? [`groupUniqArray(64)(${config.sourceEvidence}) AS observed_sources`] + : []), ...inventorySelects, ].join(', ')} FROM ${qualified} WHERE ${statsWhere} GROUP BY key` ); @@ -214,11 +227,15 @@ export default async (req, res, cubejs) => { inventory.jsonPaths.sort(); } const observedTypes = (stats?.observed_types ?? novelty.observed_types ?? []).map(String); + const observedSources = (stats?.observed_sources ?? novelty.observed_sources ?? []) + .map((s) => String(s)) + .filter((s) => s.length > 0); const label = labelsByKey.get(key); return { key, ...(label ? { label } : {}), ...(config.evidence ? { observedTypes } : {}), + ...(config.sourceEvidence ? { observedSources } : {}), ...(config.countMode === 'snapshot' ? { snapshotCount: Number(novelty.snapshot_count) || 0 } : { deltaCount: Number(stats?.delta_count) || 0 }), From 7fcdd7b88d85c9d6669e09c97f556adb6617cd6c Mon Sep 17 00:00:00 2001 From: stefanbaxter Date: Wed, 8 Jul 2026 21:07:14 +0000 Subject: [PATCH 4/4] perf(row-types): count + first/last-seen as a full-partition snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner: count() + max(timestamp) are the PRIMARY active-indicator and were missing/zero for most types because they came from the cursor-bounded delta. - Novelty (full-partition) pass now computes total_count + min/max per key — always correct, independent of the delta cursor. count/min/max are cheap columnar aggregates. - The delta pass carries ONLY the expensive union-mergeable evidence (distinct Segment types, source labels, map keys, JSON paths). - Cursor bound is derived from the delta slice (its max IS the global max), removing a second full-partition scan. - Truncation: raise caps (types 100, sources 256) + uniqExact totals + evidenceTruncated flag so capped arrays are no longer silently lossy. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cubejs/src/routes/discoverRowTypes.js | 74 +++++++++++++------ 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/services/cubejs/src/routes/discoverRowTypes.js b/services/cubejs/src/routes/discoverRowTypes.js index 2fd2efd3..925b687f 100644 --- a/services/cubejs/src/routes/discoverRowTypes.js +++ b/services/cubejs/src/routes/discoverRowTypes.js @@ -44,7 +44,13 @@ const TABLES = { sourceEvidence: '`source.label`', tsExpr: 'toUnixTimestamp64Milli(timestamp)', tie: 'event_gid', - countMode: 'accumulate', + // count() + max(ts) are the PRIMARY active-indicator (owner) — computed in + // the FULL novelty pass as a SNAPSHOT, so they are always correct and never + // depend on the delta cursor. count()/min()/max() are cheap columnar + // aggregates; only the dedup (types/sources/JSON inventory) stays on the + // delta for efficiency. + totalCountExpr: 'count()', + cursorMode: 'tie', }, entities: { discriminator: 'type', @@ -52,14 +58,15 @@ const TABLES = { // axis and the change-detection cursor in one. tsExpr: '_version', versionExpr: '_version', - countMode: 'snapshot', - snapshotCountExpr: 'uniqExact(gid)', + totalCountExpr: 'uniqExact(gid)', + cursorMode: 'version', }, data_points: { discriminator: 'series_gid', tsExpr: 'toUnixTimestamp(timestamp) * 1000', tie: 'signature', - countMode: 'accumulate', + totalCountExpr: 'count()', + cursorMode: 'tie', }, }; @@ -111,15 +118,15 @@ export default async (req, res, cubejs) => { .map((c) => c.name); // ── Pass 1: novelty snapshot (every run, cursor-independent) ── + // The SNAPSHOT of the ACTIVE indicators — total count + first/last seen — + // per key over the whole partition. Always correct regardless of the delta + // cursor (owner: count + max(ts) decide whether a type is active). These + // are cheap columnar aggregates; the expensive dedup is on the delta only. const noveltySelect = [ `${disc} AS key`, + `${config.totalCountExpr} AS total_count`, `min(${config.tsExpr}) AS min_ts`, `max(${config.tsExpr}) AS max_ts`, - ...(config.evidence ? [`groupUniqArray(16)(${config.evidence}) AS observed_types`] : []), - ...(config.sourceEvidence - ? [`groupUniqArray(64)(${config.sourceEvidence}) AS observed_sources`] - : []), - ...(config.countMode === 'snapshot' ? [`${config.snapshotCountExpr} AS snapshot_count`] : []), ].join(', '); const noveltyRows = await driver.query( `SELECT ${noveltySelect} FROM ${qualified} WHERE ${partitionCond} GROUP BY key` @@ -154,15 +161,27 @@ export default async (req, res, cubejs) => { ), ]; const statsWhere = [partitionCond, ...(deltaCond ? [deltaCond] : [])].join(' AND '); + // The delta pass carries ONLY the expensive, union-mergeable evidence + // (distinct Segment types, source labels, map keys, JSON paths). Counts and + // first/last-seen come from the full novelty pass above. `uniqExact` gives + // the TRUE cardinality so cxs2 can flag when the (capped) display array was + // truncated instead of silently losing values. + const OBSERVED_TYPES_CAP = 100; + const OBSERVED_SOURCES_CAP = 256; const statsRows = await driver.query( `SELECT ${[ `${disc} AS key`, - 'count() AS delta_count', - `min(${config.versionExpr || config.tsExpr}) AS min_ts`, - `max(${config.versionExpr || config.tsExpr}) AS max_ts`, - ...(config.evidence ? [`groupUniqArray(16)(${config.evidence}) AS observed_types`] : []), + ...(config.evidence + ? [ + `groupUniqArray(${OBSERVED_TYPES_CAP})(${config.evidence}) AS observed_types`, + `uniqExact(${config.evidence}) AS observed_types_total`, + ] + : []), ...(config.sourceEvidence - ? [`groupUniqArray(64)(${config.sourceEvidence}) AS observed_sources`] + ? [ + `groupUniqArray(${OBSERVED_SOURCES_CAP})(${config.sourceEvidence}) AS observed_sources`, + `uniqExact(${config.sourceEvidence}) AS observed_sources_total`, + ] : []), ...inventorySelects, ].join(', ')} FROM ${qualified} WHERE ${statsWhere} GROUP BY key` @@ -189,9 +208,11 @@ export default async (req, res, cubejs) => { } } - // Next cursor = scan bound of the whole table slice (not wall clock). + // Next cursor = the max (ts, tie) of the DELTA slice (not the whole table). + // The delta already contains the newest rows, so its max IS the current + // global bound — scoping to the delta avoids a second full-partition scan. const boundRows = await driver.query( - `SELECT ${cursorBoundSelect} FROM ${qualified} WHERE ${partitionCond}` + `SELECT ${cursorBoundSelect} FROM ${qualified} WHERE ${statsWhere}` ); const boundTs = Number(boundRows?.[0]?.bound_ts) || 0; let cursor = since ?? null; @@ -226,19 +247,28 @@ export default async (req, res, cubejs) => { } inventory.jsonPaths.sort(); } - const observedTypes = (stats?.observed_types ?? novelty.observed_types ?? []).map(String); - const observedSources = (stats?.observed_sources ?? novelty.observed_sources ?? []) + const observedTypes = (stats?.observed_types ?? []).map(String); + const observedSources = (stats?.observed_sources ?? []) .map((s) => String(s)) .filter((s) => s.length > 0); + // Truncation flag — the (capped) display array vs the exact cardinality. + const observedSourcesTotal = Number(stats?.observed_sources_total) || observedSources.length; + const observedTypesTruncated = Number(stats?.observed_types_total) > observedTypes.length; + const observedSourcesTruncated = observedSourcesTotal > observedSources.length; const label = labelsByKey.get(key); return { key, ...(label ? { label } : {}), ...(config.evidence ? { observedTypes } : {}), - ...(config.sourceEvidence ? { observedSources } : {}), - ...(config.countMode === 'snapshot' - ? { snapshotCount: Number(novelty.snapshot_count) || 0 } - : { deltaCount: Number(stats?.delta_count) || 0 }), + ...(config.sourceEvidence + ? { observedSources, observedSourcesTotal } + : {}), + ...(observedTypesTruncated || observedSourcesTruncated + ? { evidenceTruncated: true } + : {}), + // total_count is the SNAPSHOT over the full partition — always correct + // (the active indicator), independent of the delta cursor. + snapshotCount: Number(novelty.total_count) || 0, minTs: Number(novelty.min_ts) || 0, maxTs: Number(novelty.max_ts) || 0, inventory: {