From ce2b247af53dc87d1292079802562739e392e1f9 Mon Sep 17 00:00:00 2001 From: stefanbaxter Date: Wed, 8 Jul 2026 00:13:43 +0000 Subject: [PATCH] feat(smart-gen): bound baked meta, Cube-native descriptions, idempotent reconciles Generated models were ~2/3 per-field meta: volatile value snapshots (ranges, 60-value enum lists, exact cardinalities) that rot as data arrives, leak identifier values (PII) into schema files, and defeat the reconciler's no-change guard so every reconcile minted a version. The model carries shape + semantics; the data plane answers value questions (live queries, /meta/dynamic). Changes: - drop write-only keys: range, raw_type, max_array_length, known_keys (superseded by /meta/dynamic), cube refresh_cadence - unique_values: exact <= 12, else 2 significant figures (consumers only threshold at 7/12/50/500/2000) - lc_values / known_values: baked only as a COMPLETE (<= 12) set that is not identifier-shaped (UUID/email/opaque-token guard) - descriptions move to the Cube-native description property (member + cube); expanded map keys no longer repeat the parent column's text - titles baked only where the acronym-aware form differs from Cube's derived default (Entity GID vs Entity Gid) - merger/templateMerger shed legacy generator keys instead of preserving them forever as presumed user keys Verified live (dev stack, doc-anchor datasource): per-field meta 9.0 -> 3.8 lines; applied model queryable (store_format counts match doc anchors); reconcile-team run 2 = all skipped_no_change; cxs2 079 e2e US1 5/5 + US2 6/6 against this backend. Unit suite 620/622 (2 failures pre-exist on main, partition.test.js). Known follow-up: probe key inventory can flap on rare map keys (profiler sampling) - now the only churn source. Scope + consumer map: docs/plans/2026-07-07-smart-gen-meta-trim-scope.md Co-Authored-By: Claude Fable 5 --- .../2026-07-07-smart-gen-meta-trim-scope.md | 121 +++++++++ .../__tests__/cubeBuilder.test.js | 116 +++++++++ .../__tests__/templateMerger.test.js | 11 + .../src/utils/smart-generation/cubeBuilder.js | 229 ++++++++---------- .../src/utils/smart-generation/merger.js | 8 +- .../utils/smart-generation/templateMerger.js | 11 +- 6 files changed, 370 insertions(+), 126 deletions(-) create mode 100644 docs/plans/2026-07-07-smart-gen-meta-trim-scope.md diff --git a/docs/plans/2026-07-07-smart-gen-meta-trim-scope.md b/docs/plans/2026-07-07-smart-gen-meta-trim-scope.md new file mode 100644 index 00000000..d7e5baf6 --- /dev/null +++ b/docs/plans/2026-07-07-smart-gen-meta-trim-scope.md @@ -0,0 +1,121 @@ +# Scope: Trim generated-model meta (smart-gen + template probe pipeline) + +**Date**: 2026-07-07 · **Status**: IMPLEMENTED 2026-07-08 (branch `chore/smart-gen-meta-trim`) · **Repo**: synmetrix (primary), cxs2 (docs/conventions only) + +> Owner decisions taken at implementation: lc cap = 12; `unique_values` exact ≤ 12 else 2 significant figures; **`meta.description` → Cube-native `description` moved INTO scope** (member + cube level, owner directive — "the current way is just stupid"); trim lands before the fully-managed-models work. +> Live results (local dev stack, doc-anchor datasource): per-field meta 9.0 → 3.8 lines; zero `range`/`raw_type`/`max_array_length`/`known_keys`/`refresh_cadence`; UUID guard verified on `account_id`; `store_format` keeps its complete 8-value enum; 158,604 → `unique_values: 160000`; applied model queryable (Discount Supermarket = 82,180, doc anchor); reconcile run 2 = all `skipped_no_change` (idempotent). Known separate issue: probe KEY INVENTORY can flap on rare map keys (profiler sampling — `holiday_name` appeared in one run, not the next); pre-existing, now the only churn source, follow-up candidate. +**Problem owner**: Stefán · **Origin**: generated models are ~⅔ meta by volume; per-field meta carries redundant type/provenance info and volatile value snapshots (ranges, enum lists, cardinality counts) that go stale immediately and churn managed-model versions on every reconcile. + +## Measurements (local dev stack, 2026-07-07) + +| File | Lines | KB | Field-meta lines | Enumerated-value lines | +| --- | --- | --- | --- | --- | +| wizard cube, 1 event type, 104 dims (`e2e_smoke_mrb8rfo2.yml`) | 1,501 | 60 | 954 (64%) | 263 | +| managed `semantic_events.yml` (template + probe members) | 3,372 | 121 | 2,294 (68%) | 1,027 | + +Concrete rot examples: `received_at` range frozen at a 26-second window (`2026-03-29T12:37:08.168 → 12:37:34.386`); 8 raw user UUIDs baked into `user_id.meta.lc_values` (PII-adjacent); `abstract_event` enum of 32 values that grows weekly; parent map description duplicated verbatim onto every expanded key (38× in one file). + +Churn mechanism: `reconcileTeam.js:255` no-change guard compares exact code strings; volatile stats shift whenever data arrives → near-every reconcile mints a new version per team. + +## Principle + +**The model carries shape + semantics; values live in the data plane.** Anything a consumer can ask the data for at use time (distinct values, ranges, key inventories) is not baked into the model. Existing live surfaces already cover every need: filter values via live queries (FilterBuilder pattern / `/column-values`), map keys via `POST /api/v1/meta/dynamic` (014, TTL-fresh), profiling via `/profile-table`. + +## Goals + +1. Cut generated file size ~50–60%; kill per-field noise. +2. Make generation idempotent w.r.t. data drift → managed-model reconciles stop minting no-op versions (prerequisite for the upcoming "fully managed models" class). +3. Stop baking PII-shaped values (identifier lists) into schema files. +4. Zero capability regression for the verified consumers listed below. + +## Non-goals + +- No change to the PROFILE pipeline or wizard refinement UX — the SSE profile keeps full stats (ranges, up to 60 LC values, array lengths); the wizard's ColumnDetailPanel renders the profile, not baked meta. +- No change to template/param-slot machinery (`param_slot`, `from_template`, `default_model`, `template_checksum`) — 013/014 and cxs2 079 depend on these. +- No new endpoints. No cxs2 behavior changes (docs/conventions only). + +## Key-by-key disposition + +Verified consumers as of 2026-07-07 (grep across synmetrix `services/cubejs/src`, cxs2 `src`, semantic-layer skill). + +### Field-level — DROP (zero readers of the baked copy) + +| Key | Emitted at (cubeBuilder.js) | Evidence | +| --- | --- | --- | +| `range` {min,max,avg} | :709–711, :725–726 | No reader in either repo. Wizard shows ranges from profile. | +| `raw_type` | :679–682 | Only wizard profile UI reads `raw_type` (transient profile field, not baked meta). | +| `max_array_length` | :713–715 | Wizard profile UI only. | +| `known_keys` (native `_map` dims) | :780 | No reader; superseded by `/meta/dynamic` (fresher, TTL-stamped, filter-scoped). | +| per-key `description` copied from parent map column | :694–697 (map-expanded branch) | Pure duplication (38× in managed file). Keep description on the native `_map` dim + non-map fields. | +| `title` when equal to plain humanization | :927–928, :1313–1314 (`titleFromName`) | Cube derives the same title. Emit ONLY when `titleFromName` differs from Cube's default humanization (acronym cases: "Entity GID" vs "Entity Gid" — keep those). | + +### Field-level — BOUND (readers need coarse/bounded info, not snapshots) + +| Key | Disposition | Readers that constrain it | +| --- | --- | --- | +| `unique_values` | Exact when ≤ 12; else round to 2 significant figures (e.g. 8,314 → 8,300). Kills churn; preserves every threshold in use (7 pie, 12 facet, 50 searchable, 500 list, 2000 lazy). | cxs2 `discover.ts:73–103` (lazy gate), `cube-meta-client.ts:261/318/342/372` ×2 copies (chart-type inference, facets), `cubeMetaToFields.ts:154/176` | +| `lc_values` | Cap 60 → **12**, and attach ONLY when the full distinct set fits (uniqueValues ≤ 12) — a truncated list masquerading as an enum misleads facet UIs. Add identifier-shape guard: skip when values match UUID/ULID/hex/email patterns regardless of count (fixes the `user_id` PII case). Keep for genuinely enumerable dims (status/tier/category) — facet discovery (`discoverFacetDimensions`, maxValues=12) and agent filter-hints need real values. | cxs2 `cube-meta-client.ts:376–382` ×2, `discover.ts:113`, `CompetitorPricingFilters.tsx:41` (FILTER_PARAMS dims can't be live-queried), skill `meta-conventions.md:180` | +| `known_values` (nested lookup keys) | Keep (agents can't compose the FILTER_PARAMS lookup pattern without valid values; sets are small/stable — entity types), same identifier guard + 12 cap. | `CompetitorPricingFilters.tsx:41`, lookup SQL pattern itself | + +### Field-level — KEEP AS-IS (load-bearing) + +`auto_generated`, `ai_generated` (merger.js:20/27 — user-vs-generated field distinction on regeneration), `source_column` (smartGenerate.js:611–617 AI-metric survival + used-column tracking), `map_key` + `source_group` (diffModels.js:29–31 change-preview source tags), `nested_lookup_key`, `field_type` (cxs2 chart-type inference `cube-meta-client.ts:259`; keep this one, drop `raw_type`), `filtered_count_for` (cubeBuilder pruning). + +### Cube-level + +| Key | Disposition | +| --- | --- | +| `source_database/source_table/source_partition`, `generation_filters` | Keep — re-generation reads them (`smartGenerate.js:697–698`); merger provenance set (merger.js:283–291). | +| `grain`, `grain_description`, `time_dimension`, `time_zone` | Keep — consumed by cxs2 dashboards + MCP discover (`discover.ts:52/173/258`, `cube-meta-client.ts:418–422`) and the propose gate (`validate-proposal.ts:54`). | +| `generated_at` | Keep — one line per cube, provenance value; not emitted by the template pipeline so no reconcile churn. | +| `refresh_cadence` | **Drop** — zero readers. Must ALSO be added to a legacy-cleanup list in merger (see below) or it survives forever as a "user key". | +| `description` (tableDescription) | Keep. | + +## Code changes (synmetrix) + +1. **`utils/smart-generation/cubeBuilder.js`** — the only emission site: + - Field loop :671–741: drop `raw_type`/`range`/`max_array_length`; per-key description suppression; `unique_values` bucketing helper; `lc_values`/`known_values` cap + identifier-shape guard (new small helper, e.g. `looksLikeIdentifierValues(values)`). + - Native map block :763–785: drop `known_keys` (keep `native_map`, `field_type: 'map'`, description). + - Cube meta builders (~:930–996, :1316–1334): drop `refresh_cadence`. + - Title stamping :927–928/:1313–1314: emit only when differing from Cube's default humanization. + - `buildCubesFromTemplate` probe-member path reuses the same field pipeline — verify no separate emission site remains (:1570ff). +2. **`utils/smart-generation/merger.js`**: + - Cube-level: add dropped legacy keys (`refresh_cadence`) to the provenance/regenerate set (:283–291) so existing files shed them on next merge instead of preserving them as user keys. + - Field-level: no change — auto fields are wholesale-replaced (`mergeFields` :157–210), so trimmed meta self-heals on regeneration/reconcile. +3. **`utils/smart-generation/profiler.js`**: no functional change (LC probe stays at 60 for the wizard profile). Identifier-shape helper may live beside `buildRange` in cubeBuilder instead. +4. **Tests** (repo's own suites): update assertions in `__tests__/lcProbe.test.js` (:164–252), `cubeBuilder.test.js` (:150–152 keeps `generated_at`), `templateMode.test.js` (:188), merger/aiMetricMerge tests. Add cases: identifier-guard, lc cap, unique_values bucketing, title suppression, reconcile no-change on unchanged data (idempotency — the headline behavior). +5. **`routes/reconcileTeam.js`**: no code change; verify `skipped_no_change` now actually fires across two consecutive reconciles with static data (e2e assertion). + +## Coordinated changes (cxs2 — no behavior change required) + +- `src/lib/services/semantic-layer/model-validation/completion/meta-conventions.ts` (:56–57) — update key descriptions (bucketed `unique_values`, lc cap 12, removed keys no longer suggested). +- Semantic-layer skill `references/meta-conventions.md` (:172–185) + `qa_gate.py` / `validate_cube_model.py` — align documented conventions; the skill already says "lc_values only when truly enumerable — never identifiers", which the generator currently violates; after the trim the generator matches the skill. +- `discover.ts`, `cube-meta-client.ts` ×2, `cubeMetaToFields.ts` — **no edits**: all threshold logic keeps working with exact-≤12 / bucketed values, absent keys already have fallbacks (verified: every read is `typeof x === 'number'`-guarded or `?? -1`). + +## Migration / rollout + +- No data migration. Wizard models shed fat meta on next regeneration (auto fields wholesale-replaced); managed models shed it on next reconcile per team. User-edited (non-auto) fields keep whatever meta they have — by design (user content is never touched). +- Old + new files coexist safely: no consumer requires the dropped keys (verified above). +- Deploy: synmetrix cubejs service only; no Hasura/actions/schema change; no cxs2 deploy dependency. + +## Verification (owner rules: live e2e, no mocks) + +1. Regenerate the 079 e2e anchor cube (Sómi, `Sales Report Submitted`) → assert: file < ~40% of previous size; no `range`/`raw_type`/`known_keys`; `user_id` has no `lc_values`; `store_format` keeps its 8 values; facets/chart-inference still work on a dashboard bound to it. +2. Reconcile twice with static data → second run reports `skipped_no_change` for all templates (idempotency). +3. cxs2 079 suites US2–US4 stay green (dynamic discovery path untouched). +4. Tychi `cube_discover fields` on the regenerated cube: context size measurably down, lazy-gate behavior unchanged. + +## Open decisions (owner) + +1. `lc_values` cap = 12 (matches facet maxValues) — or keep a higher bake cap (e.g. 20) for agent hints? +2. `unique_values` bucketing = exact ≤12, 2-sig-fig above — acceptable threshold wobble at bucket edges? +3. Phase 2 candidate (separate): move member `meta.description` → Cube-native `description` (would surface in standard Cube tooling and Explore; touches annotation consumers) — in or out? +4. Should the trim land before or together with the "fully managed models" work? (Trim is a prerequisite for meaningful managed-version history either way.) + +## Forward relevance ("fully managed models", discussion upcoming) + +The trim is foundational for the two-class model (Manual per-datasource vs Managed built-in SL: `semantic_events`, `data_points`, `sales_data`, `entities`, `time_series`): managed reconciles become idempotent (version history = real template changes only), derived files become small enough for cheap fleet-wide re-derivation and diffing, and value-freshness concerns move wholly to the data plane (dynamic discovery), where the managed class needs them anyway. + +## Estimate + +Synmetrix: ~1 day incl. test updates + live verification. cxs2/skill doc sync: ~1 hour. No coordination hazard (either order deploys safely). diff --git a/services/cubejs/src/utils/smart-generation/__tests__/cubeBuilder.test.js b/services/cubejs/src/utils/smart-generation/__tests__/cubeBuilder.test.js index 00391ea4..894a9030 100644 --- a/services/cubejs/src/utils/smart-generation/__tests__/cubeBuilder.test.js +++ b/services/cubejs/src/utils/smart-generation/__tests__/cubeBuilder.test.js @@ -153,6 +153,122 @@ describe('cubeBuilder – buildCubes', () => { }); }); + describe('meta trim — bounded, data-plane-first', () => { + it('bakes no volatile or redundant per-field meta (range / raw_type / max_array_length)', () => { + const t = makeTable({ + columns: new Map([ + col('amount', 'Float64', ColumnType.BASIC, ValueType.NUMBER, { + hasValues: true, uniqueValues: 4200, minValue: 1, maxValue: 950, avgValue: 77.3, + }), + ]), + }); + const { cubes } = buildCubes(t); + const amount = cubes[0].measures.find((m) => m.name === 'amount'); + assert.ok(amount, 'amount measure expected'); + assert.strictEqual(amount.meta.range, undefined); + assert.strictEqual(amount.meta.raw_type, undefined); + assert.strictEqual(amount.meta.max_array_length, undefined); + assert.strictEqual(amount.meta.field_type, 'float', 'field_type stays (chart inference)'); + }); + + it('keeps exact cardinality up to 12 and buckets larger counts to 2 significant figures', () => { + const t = makeTable({ + columns: new Map([ + col('status', 'String', ColumnType.BASIC, ValueType.STRING, { + hasValues: true, uniqueValues: 5, lcValues: ['a', 'b', 'c', 'd', 'e'], + }), + col('city', 'String', ColumnType.BASIC, ValueType.STRING, { + hasValues: true, uniqueValues: 8314, + }), + ]), + }); + const { cubes } = buildCubes(t); + const status = cubes[0].dimensions.find((d) => d.name === 'status'); + const city = cubes[0].dimensions.find((d) => d.name === 'city'); + assert.strictEqual(status.meta.unique_values, 5); + assert.strictEqual(city.meta.unique_values, 8300); + }); + + it('bakes lc_values only as a complete, non-identifier set', () => { + const t = makeTable({ + columns: new Map([ + col('tier', 'String', ColumnType.BASIC, ValueType.STRING, { + hasValues: true, uniqueValues: 3, lcValues: ['bronze', 'gold', 'silver'], + }), + // truncated list: 40 distinct, only 12 observed — misleading as an enum + col('sku_bucket', 'String', ColumnType.BASIC, ValueType.STRING, { + hasValues: true, uniqueValues: 40, + lcValues: Array.from({ length: 12 }, (_, i) => `v${i}`), + }), + // identifier-shaped values are data, not enums (and PII-adjacent) + col('user_ref', 'String', ColumnType.BASIC, ValueType.STRING, { + hasValues: true, uniqueValues: 2, + lcValues: [ + '067289d0-cd97-401e-b0cc-d45630bd9e97', + 'b99d4d95-cdfc-4c8a-8085-a8030c15ad78', + ], + }), + ]), + }); + const { cubes } = buildCubes(t); + const dim = (n) => cubes[0].dimensions.find((d) => d.name === n); + assert.deepStrictEqual(dim('tier').meta.lc_values, ['bronze', 'gold', 'silver']); + assert.strictEqual(dim('sku_bucket').meta.lc_values, undefined); + assert.strictEqual(dim('user_ref').meta.lc_values, undefined); + }); + + it('emits descriptions as the Cube-native property, never meta, and not on expanded map keys', () => { + const t = makeTable({ + columns: new Map([ + col('status', 'String', ColumnType.BASIC, ValueType.STRING, { + hasValues: true, uniqueValues: 2, lcValues: ['on', 'off'], + }), + col('props', 'Map(String, String)', ColumnType.MAP, ValueType.OTHER, { + uniqueKeys: ['color'], + }), + ]), + columnDescriptions: new Map([ + ['status', 'Current status'], + ['props', 'Additional low-cardinality dimensions'], + ]), + }); + const { cubes } = buildCubes(t); + const dim = (n) => cubes[0].dimensions.find((d) => d.name === n); + assert.strictEqual(dim('status').description, 'Current status'); + assert.strictEqual(dim('status').meta.description, undefined); + // expanded key: no parent-description copy + assert.strictEqual(dim('color').description, undefined); + // native map accessor keeps the column description, no baked key inventory + assert.strictEqual(dim('props_map').description, 'Additional low-cardinality dimensions'); + assert.strictEqual(dim('props_map').meta.known_keys, undefined); + }); + + it('bakes titles only where they differ from the derived default (acronyms)', () => { + const t = makeTable({ + columns: new Map([ + col('order_id', 'String', ColumnType.BASIC, ValueType.STRING, { + hasValues: true, uniqueValues: 900, + }), + col('store_format', 'String', ColumnType.BASIC, ValueType.STRING, { + hasValues: true, uniqueValues: 8, + }), + ]), + }); + const { cubes } = buildCubes(t); + const dim = (n) => cubes[0].dimensions.find((d) => d.name === n); + assert.strictEqual(dim('order_id').title, 'Order ID'); + assert.strictEqual(dim('store_format').title, undefined); + }); + + it('cube meta carries no refresh_cadence or description duplicate (native description wins)', () => { + const { cubes } = buildCubes(table); + assert.strictEqual(cubes[0].meta.refresh_cadence, undefined); + assert.strictEqual(cubes[0].meta.description, undefined); + assert.ok(typeof cubes[0].description === 'string' && cubes[0].description.length > 0); + assert.ok(typeof cubes[0].meta.generated_at === 'string'); + }); + }); + describe('empty column exclusion', () => { it('should skip columns where hasValues is false', () => { const t = makeTable({ diff --git a/services/cubejs/src/utils/smart-generation/__tests__/templateMerger.test.js b/services/cubejs/src/utils/smart-generation/__tests__/templateMerger.test.js index 41e9c706..168e73fd 100644 --- a/services/cubejs/src/utils/smart-generation/__tests__/templateMerger.test.js +++ b/services/cubejs/src/utils/smart-generation/__tests__/templateMerger.test.js @@ -129,6 +129,17 @@ describe('templateMerger — three-class merge (D4)', () => { assert.equal(merged.meta.team_note, 'keep me', 'team meta key preserved'); }); + it('sheds legacy generator meta keys the candidate no longer writes', () => { + const legacyExisting = EXISTING.replace( + ' team_note: keep me\n', + ' team_note: keep me\n refresh_cadence: 1 hour\n description: legacy duplicate\n' + ); + const out = YAML.parse(mergeTemplateModel(legacyExisting, CANDIDATE)).cubes[0]; + assert.equal(out.meta.refresh_cadence, undefined, 'legacy refresh_cadence shed'); + assert.equal(out.meta.description, undefined, 'legacy meta description shed (native wins)'); + assert.equal(out.meta.team_note, 'keep me', 'genuine team keys still preserved'); + }); + it('is stable: merging the merged output with the same candidate is a no-op', () => { const once = mergeTemplateModel(EXISTING, CANDIDATE); const twice = mergeTemplateModel(once, CANDIDATE); diff --git a/services/cubejs/src/utils/smart-generation/cubeBuilder.js b/services/cubejs/src/utils/smart-generation/cubeBuilder.js index 5fd3c18c..662f9b25 100644 --- a/services/cubejs/src/utils/smart-generation/cubeBuilder.js +++ b/services/cubejs/src/utils/smart-generation/cubeBuilder.js @@ -27,6 +27,25 @@ function titleFromName(name) { .join(' '); } +/** + * Cube.js derives a member/cube title from the name on its own (words split, + * each capitalized). An explicit title is only worth baking when our + * acronym-aware form differs from that default — everything else is noise. + * + * @param {string} name + * @returns {string|null} Title to bake, or null when Cube's default suffices + */ +function titleWhenNotDefault(name) { + const title = titleFromName(name); + const plain = name + .replace(/_/g, ' ') + .replace(/([a-z])([A-Z])/g, '$1 $2') + .split(' ') + .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) + .join(' '); + return title === plain ? null : title; +} + /** * Sanitize a table name into a valid Cube.js cube identifier. * @@ -296,59 +315,55 @@ function toJsonFieldType(valueType, rawType) { } /** - * Extract the value type from a Map(...) raw type string. - * e.g. "Map(LowCardinality(String), Float32)" → "Float32" - * - * @param {string} rawType - * @returns {string|null} The unwrapped value type, or null if not a Map + * Baked value lists and cardinality counts are snapshots that rot as data + * arrives — the model carries shape + semantics, the data plane answers + * value questions (live queries, /meta/dynamic). What little we do bake is + * bounded by these helpers so reconciles stay idempotent: + * - counts ≤ LC_BAKE_CAP are exact (facet/pie thresholds need them), + * larger counts round to 2 significant figures (consumers only threshold); + * - value lists are baked only when COMPLETE (≤ LC_BAKE_CAP distinct) and + * not identifier-shaped (UUIDs/emails/opaque tokens are data, not enums — + * and baking them leaks PII into schema files). */ -function extractMapValueType(rawType) { - const m = rawType.match(/^Map\s*\((.+)\)\s*$/i); - if (!m) return null; - // Split on top-level comma (respecting parentheses depth) - let depth = 0; - let splitIdx = -1; - for (let i = 0; i < m[1].length; i++) { - if (m[1][i] === '(') depth++; - else if (m[1][i] === ')') depth--; - else if (m[1][i] === ',' && depth === 0) { splitIdx = i; break; } - } - if (splitIdx === -1) return null; - let valPart = m[1].slice(splitIdx + 1).trim(); - // Unwrap Nullable / LowCardinality wrappers - valPart = valPart.replace(/^(Nullable|LowCardinality)\s*\(\s*/gi, '').replace(/\s*\)\s*$/, ''); - return valPart || null; +const LC_BAKE_CAP = 12; + +function bucketCardinality(count) { + const n = Number(count); + if (!Number.isFinite(n) || n < 0) return null; + if (n <= LC_BAKE_CAP) return n; + const magnitude = 10 ** (Math.floor(Math.log10(n)) - 1); + return Math.round(n / magnitude) * magnitude; +} + +const UUID_VALUE_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const EMAIL_VALUE_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; +// Long unbroken token containing a digit — machine ids, not human enum labels +const OPAQUE_TOKEN_RE = /^(?=.*\d)[A-Za-z0-9_-]{20,}$/; + +function looksLikeIdentifierValues(values) { + const sample = (values || []).filter((v) => typeof v === 'string' && v.length > 0); + if (sample.length === 0) return false; + const hits = sample.filter( + (v) => UUID_VALUE_RE.test(v) || EMAIL_VALUE_RE.test(v) || OPAQUE_TOKEN_RE.test(v) + ).length; + return hits / sample.length >= 0.5; } /** - * Build a range object from min/max/avg values. - * For numbers: coerces strings to numbers, includes avg. - * For timestamps: keeps as strings, min/max only (avg is meaningless). + * The list to bake for an enum-ish field, or null when baking would mislead + * (incomplete list, identifier-shaped values) — consumers treat a baked list + * as the complete value set (facet widgets, agent filter hints). * - * @param {*} min - * @param {*} max - * @param {*} avg - * @returns {{ min: number|string, max: number|string, avg?: number }|null} + * @param {*} values - Observed distinct values (profiler, ≤ LC probe limit) + * @param {*} uniqueCount - Distinct count when known (defaults to list length) + * @returns {Array|null} */ -function buildRange(min, max, avg) { - const parts = {}; - // Try numeric first - if (min != null) { - const n = Number(min); - if (!isNaN(n)) parts.min = n; - else if (typeof min === 'string' && min.length > 0) parts.min = min; // timestamp - } - if (max != null) { - const n = Number(max); - if (!isNaN(n)) parts.max = n; - else if (typeof max === 'string' && max.length > 0) parts.max = max; // timestamp - } - // avg only for numerics - if (avg != null) { - const n = Number(avg); - if (!isNaN(n)) parts.avg = n; - } - return Object.keys(parts).length > 0 ? parts : null; +function lcBakeList(values, uniqueCount) { + if (!Array.isArray(values) || values.length === 0) return null; + const count = uniqueCount == null ? values.length : Number(uniqueCount); + if (count > LC_BAKE_CAP || values.length > LC_BAKE_CAP) return null; + if (looksLikeIdentifierValues(values)) return null; + return values; } /** @@ -518,16 +533,15 @@ function processColumns(columns, options) { meta: { auto_generated: true, source_column: columnName, - raw_type: details.rawType, field_type: 'string', nested_lookup_key: true, - known_values: lookup.lcValues, }, }; - if (colDescription) field.meta.description = colDescription; - if (profile && profile.maxArrayLength != null && profile.maxArrayLength > 0) { - field.meta.max_array_length = profile.maxArrayLength; - } + // FILTER_PARAMS lookups are unusable without knowing the valid + // values — bake them, but only as a complete, non-identifier set. + const knownValues = lcBakeList(lookup.lcValues, lookup.lcValues.length); + if (knownValues) field.meta.known_values = knownValues; + if (colDescription) field.description = colDescription; allFields.push(field); } else { // Skip nested sub-columns with no non-empty values — but only for @@ -620,14 +634,10 @@ function processColumns(columns, options) { meta: { auto_generated: true, source_column: columnName, - raw_type: details.rawType, field_type: jsonFieldType, }, }; - if (colDescription) field.meta.description = colDescription; - if (profile && profile.maxArrayLength != null && profile.maxArrayLength > 0) { - field.meta.max_array_length = profile.maxArrayLength; - } + if (colDescription) field.description = colDescription; allFields.push(field); } continue; // skip normal processing for this column @@ -672,15 +682,6 @@ function processColumns(columns, options) { field.meta = { auto_generated: true }; field.meta.source_column = columnName; - // For map-expanded fields, use the map's value type (e.g. "Float32"), - // not the full Map(...) container type - if (field._mapKey) { - const mapValueType = extractMapValueType(details.rawType); - if (mapValueType) field.meta.raw_type = mapValueType; - } else { - field.meta.raw_type = details.rawType; - } - // Add JSON-friendly field_type const isBoolField = isInt8Boolean(details.rawType, profile); if (field._mapKey) { @@ -690,53 +691,40 @@ function processColumns(columns, options) { field.meta.field_type = isBoolField ? 'boolean' : toJsonFieldType(details.valueType, details.rawType); } - // Add column description if available + // Column description — Cube-native property, and only on the column's + // own field: stamping the parent map's description onto every expanded + // key repeats one sentence dozens of times and describes nothing. const colDescription = columnDescriptions.get(columnName) || null; - if (colDescription) { - field.meta.description = colDescription; + if (colDescription && !field._mapKey) { + field.description = colDescription; } - // Add profile stats when available - if (profile) { - // For Map-expanded fields, unique_values from the parent is the key count - // — not meaningful per-field. Only attach for non-map fields. - if (!field._mapKey && profile.uniqueValues > 0) { - field.meta.unique_values = profile.uniqueValues; - } - - // Numeric range: combine min/max/avg into a single "range" field - if (!isBoolField) { - const range = buildRange(profile.minValue, profile.maxValue, profile.avgValue); - if (range) field.meta.range = range; - } - - if (profile.maxArrayLength != null && profile.maxArrayLength > 0) { - field.meta.max_array_length = profile.maxArrayLength; - } - } - - // For Map-expanded fields, add per-key metadata if (field._mapKey) { field.meta.map_key = field._mapKey; - // Per-key stats from profiler (numeric: min/max/avg, string: unique_values) - if (profile && profile.keyStats && profile.keyStats[field._mapKey]) { - const stats = profile.keyStats[field._mapKey]; - const range = buildRange(stats.min, stats.max, stats.avg); - if (range) field.meta.range = range; - if (stats.unique_values != null) field.meta.unique_values = stats.unique_values; + // Per-key stats from profiler (bounded — see lcBakeList/bucketCardinality) + const stats = profile?.keyStats?.[field._mapKey]; + if (stats && stats.unique_values != null) { + const bucketed = bucketCardinality(stats.unique_values); + if (bucketed != null) field.meta.unique_values = bucketed; } - - // String map keys get LC values if (profile && profile.lcValues && typeof profile.lcValues === 'object' && !Array.isArray(profile.lcValues)) { - if (profile.lcValues[field._mapKey]) { - field.meta.lc_values = profile.lcValues[field._mapKey]; - } + const lcList = lcBakeList( + profile.lcValues[field._mapKey], + stats?.unique_values ?? undefined + ); + if (lcList) field.meta.lc_values = lcList; } - } else { - // Non-map fields: attach LC values for categorical data - if (profile && profile.lcValues != null && Array.isArray(profile.lcValues)) { - field.meta.lc_values = profile.lcValues; + } else if (profile) { + // For Map-expanded fields, unique_values from the parent is the key count + // — not meaningful per-field. Only attach for non-map fields. + if (profile.uniqueValues > 0) { + const bucketed = bucketCardinality(profile.uniqueValues); + if (bucketed != null) field.meta.unique_values = bucketed; + } + if (Array.isArray(profile.lcValues)) { + const lcList = lcBakeList(profile.lcValues, profile.uniqueValues); + if (lcList) field.meta.lc_values = lcList; } } @@ -774,13 +762,13 @@ function processColumns(columns, options) { meta: { auto_generated: true, source_column: columnName, - raw_type: details.rawType, field_type: 'map', native_map: true, - known_keys: profile.uniqueKeys, + // Key inventory intentionally NOT baked — /meta/dynamic answers it + // at query time, filter-scoped and freshness-stamped (014). }, }; - if (colDescription) nativeMapField.meta.description = colDescription; + if (colDescription) nativeMapField.description = colDescription; allFields.push(nativeMapField); } } @@ -854,6 +842,8 @@ function processColumns(columns, options) { meta: field.meta, }; + if (field.description) output.description = field.description; + if (field.primary_key) { output.primary_key = true; output.public = true; @@ -923,9 +913,9 @@ function buildRawCube(profiledTable, options) { dimensions.unshift(partitionDim); } - // -- Heuristics: titles on all fields + cube -------------------------------- - for (const dim of dimensions) { if (!dim.title) dim.title = titleFromName(dim.name); } - for (const meas of measures) { if (!meas.title) meas.title = titleFromName(meas.name); } + // -- Heuristics: titles only where Cube's derived default falls short ------- + for (const dim of dimensions) { if (!dim.title) dim.title = titleWhenNotDefault(dim.name) ?? undefined; } + for (const meas of measures) { if (!meas.title) meas.title = titleWhenNotDefault(meas.name) ?? undefined; } // -- Heuristics: meta block ------------------------------------------------- const timeDim = dimensions.find((d) => d.type === 'time' && d.primary_key !== true); @@ -941,15 +931,9 @@ function buildRawCube(profiledTable, options) { grain_description: `Each row represents one ${table} record, keyed by ${grainParts}.`, time_dimension: timeDim ? timeDim.name : null, time_zone: 'UTC', - refresh_cadence: '1 hour', generated_at: new Date().toISOString(), }; - // Include table description if available - if (profiledTable.tableDescription) { - meta.description = profiledTable.tableDescription; - } - if (isInternal && partition) { meta.source_partition = partition; } @@ -990,7 +974,7 @@ function buildRawCube(profiledTable, options) { const cube = { name: cubeName, - title: titleFromName(cubeName), + title: titleWhenNotDefault(cubeName) ?? undefined, description: profiledTable.tableDescription || `Analytical model for ${schema}.${table}, auto-generated from table profiling.`, ...source, meta, @@ -1309,9 +1293,9 @@ function buildArrayJoinCube(profiledTable, arrayJoinGroups, rawCube, options) { existingNames.add(f.name); } - // Titles on all fields - for (const dim of dimensions) { if (!dim.title) dim.title = titleFromName(dim.name); } - for (const meas of measures) { if (!meas.title) meas.title = titleFromName(meas.name); } + // Titles only where Cube's derived default falls short + for (const dim of dimensions) { if (!dim.title) dim.title = titleWhenNotDefault(dim.name) ?? undefined; } + for (const meas of measures) { if (!meas.title) meas.title = titleWhenNotDefault(meas.name) ?? undefined; } const meta = { auto_generated: true, @@ -1328,7 +1312,7 @@ function buildArrayJoinCube(profiledTable, arrayJoinGroups, rawCube, options) { return { name: cubeName, - title: titleFromName(cubeName), + title: titleWhenNotDefault(cubeName) ?? undefined, description: profiledTable.tableDescription || `Analytical model for ${schema}.${table}, auto-generated from table profiling.`, sql, meta, @@ -1405,7 +1389,6 @@ export function buildCubes(profiledTable, options = {}) { ajCube.meta.grain_description = `Each row represents one ${profiledTable.table} record, keyed by ${grainParts}.`; ajCube.meta.time_dimension = timeDim ? timeDim.name : null; ajCube.meta.time_zone = 'UTC'; - ajCube.meta.refresh_cadence = '1 hour'; // - Drill members on count const countMeasure = ajCube.measures.find((m) => m.type === 'count'); diff --git a/services/cubejs/src/utils/smart-generation/merger.js b/services/cubejs/src/utils/smart-generation/merger.js index 06890400..64793041 100644 --- a/services/cubejs/src/utils/smart-generation/merger.js +++ b/services/cubejs/src/utils/smart-generation/merger.js @@ -280,7 +280,11 @@ function mergeCube(existingCube, newCube, keepStale) { const existingMeta = { ...existingCube.meta }; const newMeta = newCube.meta ? { ...newCube.meta } : {}; - // Collect user-added meta keys (anything that isn't a provenance key) + // Collect user-added meta keys (anything that isn't a provenance key). + // Keys the generator used to write but no longer does ('refresh_cadence', + // cube-level 'description' — Cube-native description is authoritative) + // are listed as provenance too, so legacy files shed them on merge + // instead of carrying them forever as presumed user keys. const provenanceKeys = new Set([ 'auto_generated', 'source_database', @@ -290,6 +294,8 @@ function mergeCube(existingCube, newCube, keepStale) { 'generation_filters', 'ai_enrichment_status', 'ai_metrics_count', + 'refresh_cadence', + 'description', ]); for (const [key, value] of Object.entries(existingMeta)) { diff --git a/services/cubejs/src/utils/smart-generation/templateMerger.js b/services/cubejs/src/utils/smart-generation/templateMerger.js index 05884e4d..eba54b89 100644 --- a/services/cubejs/src/utils/smart-generation/templateMerger.js +++ b/services/cubejs/src/utils/smart-generation/templateMerger.js @@ -77,8 +77,15 @@ const mergeCube = (existingCube, candidateCube) => { merged.pre_aggregations = existingCube.pre_aggregations; } - // provenance stamps from the candidate win; team-added meta keys survive - merged.meta = { ...(existingCube.meta || {}), ...(candidateCube.meta || {}) }; + // provenance stamps from the candidate win; team-added meta keys survive. + // Keys the generator used to write but no longer does are shed from the + // existing side, not resurrected through the spread (legacy cleanup — + // cube-level description is Cube-native now). + const existingMeta = { ...(existingCube.meta || {}) }; + for (const legacyKey of ['refresh_cadence', 'description']) { + if (!(legacyKey in (candidateCube.meta || {}))) delete existingMeta[legacyKey]; + } + merged.meta = { ...existingMeta, ...(candidateCube.meta || {}) }; // a re-published template resumes management if ( merged.meta.default_model === true &&