From a705c8d5267045a3521cd5a224e32a806892ef2f Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Wed, 20 May 2026 16:35:51 +0200 Subject: [PATCH 01/22] feat(api-gateway): granularities config and /v1/granularities endpoint --- packages/cubejs-api-gateway/openspec.yml | 44 +++++ packages/cubejs-api-gateway/package.json | 1 + packages/cubejs-api-gateway/src/gateway.ts | 124 +++++++++++++- .../src/helpers/prepare-annotation.ts | 21 ++- .../cubejs-api-gateway/src/types/gateway.ts | 20 +++ .../test/helpers/prepare-annotation.test.ts | 6 +- packages/cubejs-backend-shared/src/env.ts | 12 ++ packages/cubejs-client-core/src/time.ts | 5 + packages/cubejs-client-core/src/types.ts | 4 + .../src/compiler/CubeSymbols.ts | 23 +++ .../src/compiler/CubeToMetaTransformer.ts | 7 + .../src/compiler/CubeValidator.ts | 149 ++++++++++------- .../src/compiler/GlobalGranularitiesConfig.ts | 154 ++++++++++++++++++ .../src/compiler/GranularityResolver.ts | 107 ++++++++++++ .../src/compiler/YamlCompiler.ts | 20 ++- .../src/compiler/index.ts | 19 +++ .../src/compiler/utils.ts | 5 + .../test/unit/cube-validator.test.ts | 87 ++++++++++ .../test/unit/granularities-config.test.ts | 75 +++++++++ .../test/unit/granularities-shape.test.ts | 154 ++++++++++++++++++ .../src/core/optionsValidate.ts | 9 + .../cubejs-server-core/src/core/server.ts | 1 + packages/cubejs-server-core/src/core/types.ts | 3 +- rust/cube/cubeorchestrator/src/transport.rs | 6 + 24 files changed, 989 insertions(+), 67 deletions(-) create mode 100644 packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts create mode 100644 packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts create mode 100644 packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts create mode 100644 packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts diff --git a/packages/cubejs-api-gateway/openspec.yml b/packages/cubejs-api-gateway/openspec.yml index 0a5c71ddddc04..5fe9925a30bac 100644 --- a/packages/cubejs-api-gateway/openspec.yml +++ b/packages/cubejs-api-gateway/openspec.yml @@ -4,6 +4,30 @@ info: version: "1.0.0" title: "Cube.js" paths: + "/v1/granularities": + get: + summary: "List the granularities available for this deployment" + description: "Returns the granularities enabled in this deployment — built-ins plus any custom granularities defined via `CUBEJS_GRANULARITIES` or `config.granularities`. Evaluated per request context." + operationId: "granularitiesV1" + responses: + "200": + description: "successful operation" + content: + application/json: + schema: + $ref: "#/components/schemas/V1GranularitiesResponse" + "4XX": + description: "Request could not be completed" + content: + application/json: + schema: + $ref: "#/components/schemas/V1Error" + "5XX": + description: "Internal Server Error" + content: + application/json: + schema: + $ref: "#/components/schemas/V1Error" "/v1/meta": get: summary: "Load Metadata" @@ -115,8 +139,17 @@ components: properties: name: type: "string" + type: + type: "string" + description: "Built-in (year/quarter/month/...) or user-defined custom granularity." + enum: + - built-in + - custom title: type: "string" + format: + type: "string" + description: "d3-time-format string used by clients to display bucketed timestamps." interval: type: "string" sql: @@ -125,6 +158,17 @@ components: type: "string" origin: type: "string" + V1GranularitiesResponse: + type: "object" + description: "Response shape of GET /v1/granularities." + properties: + data: + type: "object" + properties: + granularities: + type: array + items: + $ref: "#/components/schemas/V1CubeMetaDimensionGranularity" V1CubeMetaDimension: type: "object" required: diff --git a/packages/cubejs-api-gateway/package.json b/packages/cubejs-api-gateway/package.json index aed2702f2524d..67a8444cc3555 100644 --- a/packages/cubejs-api-gateway/package.json +++ b/packages/cubejs-api-gateway/package.json @@ -29,6 +29,7 @@ "dependencies": { "@cubejs-backend/native": "1.7.23", "@cubejs-backend/query-orchestrator": "1.7.23", + "@cubejs-backend/schema-compiler": "1.7.23", "@cubejs-backend/shared": "1.7.23", "@ungap/structured-clone": "^0.3.4", "assert-never": "^1.4.0", diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index e595cc3e61c1b..2db93270ec392 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -32,6 +32,13 @@ import type { import { createProxyMiddleware } from 'http-proxy-middleware'; import { QueryBody } from '@cubejs-backend/query-orchestrator'; +import { + resolveGlobalGranularities, + resolveDimensionGranularities, + normalizeGranularitiesBlock, + buildBuiltInsCatalog, + BUILT_IN_GRANULARITIES, +} from '@cubejs-backend/schema-compiler'; import { QueryType, ApiScopes, @@ -166,6 +173,8 @@ class ApiGateway { protected readonly extendContext?: ExtendContextFn; + protected readonly granularitiesOption?: ApiGatewayOptions['granularities']; + protected readonly dataSourceStorage: any; public readonly checkAuthFn: PreparedCheckAuthFn; @@ -224,6 +233,7 @@ class ApiGateway { this.subscriptionStore = options.subscriptionStore || new LocalSubscriptionStore(); this.enforceSecurityChecks = options.enforceSecurityChecks || (process.env.NODE_ENV === 'production'); this.extendContext = options.extendContext; + this.granularitiesOption = options.granularities; this.checkAuthFn = this.createCheckAuthFn(options); this.checkAuthSystemFn = this.createCheckAuthSystemFn(); @@ -478,6 +488,17 @@ class ApiGateway { }) ); + app.get( + `${this.basePath}/v1/granularities`, + userMiddlewares, + userAsyncHandler(async (req, res) => { + await this.granularities({ + context: req.context, + res: this.resToResultFn(res), + }); + }) + ); + app.post( `${this.basePath}/v1/cubesql`, userMiddlewares, @@ -726,7 +747,9 @@ class ApiGateway { const cubesConfig = onlyViews ? metaConfig.cubes.filter((c: any) => c.config?.type === 'view') : metaConfig.cubes; - const cubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config); + const filteredCubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config); + // Apply after the visibility filter so we only enrich what the client will actually receive. + const cubes = await this.applyGlobalGranularitiesToMetaCubes(context, filteredCubes); const visibleCubeNames = new Set(cubes.map(c => c.name)); const viewGroups = (metaConfig.viewGroups || []) .map(group => this.filterVisibleViewGroup(group, visibleCubeNames)) @@ -750,6 +773,47 @@ class ApiGateway { } } + public async granularities({ context, res }: { + context: RequestContext, + res: ResponseResultFn, + }) { + const requestStarted = new Date(); + try { + await this.assertApiScope('meta', context.securityContext); + const globalConfig = await this.resolveGlobalGranularitiesForRequest(context); + const builtInsCatalog = buildBuiltInsCatalog(globalConfig); + + const granularities: any[] = []; + for (const [name, entry] of Object.entries(builtInsCatalog)) { + granularities.push({ type: 'built-in', name, ...entry }); + } + for (const [name, def] of Object.entries(globalConfig.customGranularities)) { + // Skip names already emitted by `buildBuiltInsCatalog` (their inline overrides are folded in there). + if (!(name in BUILT_IN_GRANULARITIES)) { + const entry: any = { + type: 'custom', + name, + title: def.title || name, + }; + if (def.interval !== undefined) entry.interval = def.interval; + if (def.origin !== undefined) entry.origin = def.origin; + if (def.offset !== undefined) entry.offset = def.offset; + if (def.format !== undefined) entry.format = def.format; + granularities.push(entry); + } + } + res({ data: { granularities } }); + } catch (e: any) { + this.handleError({ + e, + context, + // @ts-ignore + res, + requestStarted, + }); + } + } + public async metaExtended({ context, res, onlyViews }: { context: ExtendedRequestContext, res: ResponseResultFn, @@ -2070,6 +2134,13 @@ class ApiGateway { }); metaConfigResult = this.filterVisibleItemsInMeta(context, metaConfigResult); + // Annotation reads from this meta. Without enrichment, /v1/load and /v1/cubesql would + // omit the type/title/format/interval fields that /v1/meta exposes. + const enrichedCubes = await this.applyGlobalGranularitiesToMetaCubes( + context, + metaConfigResult.map((m: any) => m.config), + ); + metaConfigResult = metaConfigResult.map((m: any, i: number) => ({ ...m, config: enrichedCubes[i] })); const sqlQueries = await this.getSqlQueriesInternal(context, normalizedQueries); @@ -2388,6 +2459,57 @@ class ApiGateway { return this.adapterApi(context); } + protected async resolveGlobalGranularitiesForRequest(context: RequestContext) { + return resolveGlobalGranularities(this.granularitiesOption, context); + } + + // Reconcile each time dimension's `granularitiesBlock` against the request's global config + // and emit the effective granularity array. Built-ins get tagged `built-in`, locals/globals + // become `custom`. Replaces the per-dim `granularities` array on the returned cube. + protected async applyGlobalGranularitiesToMetaCubes(context: RequestContext, cubes: any[]): Promise { + const globalConfig = await this.resolveGlobalGranularitiesForRequest(context); + const builtInsCatalog = buildBuiltInsCatalog(globalConfig); + + return cubes.map(cube => ({ + ...cube, + dimensions: cube.dimensions?.map((dim: any) => { + if (dim.type !== 'time') return dim; + // Re-key the local-custom array CubeToMetaTransformer produced so the resolver can + // merge it back into `granularitiesBlock.custom` cleanly. + const localCustom: Record = {}; + for (const g of dim.granularities || []) { + localCustom[g.name] = { + title: g.title, + interval: g.interval, + offset: g.offset, + origin: g.origin, + ...(g.format !== undefined ? { format: g.format } : {}), + }; + } + const block = dim.granularitiesBlock || normalizeGranularitiesBlock(undefined); + const blockWithLocal = { ...block, custom: { ...block.custom, ...localCustom } }; + const resolved = resolveDimensionGranularities( + blockWithLocal, + globalConfig.enabledBuiltIns, + globalConfig.customGranularities, + builtInsCatalog, + ); + const resolvedArray = Object.entries(resolved).map(([name, def]: [string, any]) => ({ + name, + type: def.type, + title: def.title, + ...(def.interval !== undefined ? { interval: def.interval } : {}), + ...(def.offset !== undefined ? { offset: def.offset } : {}), + ...(def.origin !== undefined ? { origin: def.origin } : {}), + ...(def.format !== undefined ? { format: def.format } : {}), + })); + // Strip the transport-only block; clients only see the resolved `granularities` array. + const { granularitiesBlock, ...rest } = dim; + return { ...rest, granularities: resolvedArray }; + }), + })); + } + public async contextByReq(req: Request, securityContext, requestId: string): Promise { req.securityContext = securityContext; diff --git a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts index 2f53164369ff1..92a7656740d1e 100644 --- a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts +++ b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts @@ -7,6 +7,7 @@ import R from 'ramda'; import { isPredefinedGranularity } from '@cubejs-backend/shared'; +import { BUILT_IN_GRANULARITIES } from '@cubejs-backend/schema-compiler'; import { MetaConfig, MetaConfigMap, toConfigMap } from './to-config-map'; import { MemberType } from '../types/strings'; import { MemberType as MemberTypeEnum } from '../types/enums'; @@ -14,7 +15,10 @@ import { MemberExpression } from '../types/query'; type GranularityMeta = { name: string; + type?: 'built-in' | 'custom'; title: string; + /** d3-time-format string for displaying bucketed timestamps. */ + format?: string; interval: string; offset?: string; origin?: string; @@ -115,14 +119,25 @@ function prepareAnnotation(metaConfig: MetaConfig[], query: any) { if (an) { let granularityMeta: GranularityMeta | undefined; if (isPredefinedGranularity(td.granularity)) { + // Prefer values the meta endpoint already attached (these honor any global + // title/format override). Fall back to BUILT_IN_GRANULARITIES, then to the bare name. + const fromMeta = an[1].granularities?.find(g => g.name === td.granularity); + const builtInDefaults = BUILT_IN_GRANULARITIES[td.granularity] || {}; granularityMeta = { name: td.granularity, - title: td.granularity, - interval: `1 ${td.granularity}`, + type: 'built-in', + title: fromMeta?.title || builtInDefaults.title || td.granularity, + interval: fromMeta?.interval || `1 ${td.granularity}`, + ...(fromMeta?.format || builtInDefaults.format + ? { format: fromMeta?.format || builtInDefaults.format } + : {}), }; } else if (an[1].granularities) { - // No need to send all the granularities defined, only those make sense for this query + // Forward only the granularity in play for this query; siblings stay in /v1/meta. granularityMeta = an[1].granularities.find(g => g.name === td.granularity); + if (granularityMeta && !granularityMeta.type) { + granularityMeta = { ...granularityMeta, type: 'custom' }; + } } const { granularities: _, ...rest } = an[1]; diff --git a/packages/cubejs-api-gateway/src/types/gateway.ts b/packages/cubejs-api-gateway/src/types/gateway.ts index d1578742ebad5..ab5639cadab26 100644 --- a/packages/cubejs-api-gateway/src/types/gateway.ts +++ b/packages/cubejs-api-gateway/src/types/gateway.ts @@ -52,6 +52,19 @@ type ScheduledRefreshContextsFn = type ScheduledRefreshTimeZonesFn = (context: RequestContext) => string[] | Promise; +type GranularityListItem = string | { + name: string; + title?: string; + format?: string; + interval?: string; + origin?: string; + offset?: string; +}; +type GranularityList = GranularityListItem[]; +type GranularitiesOption = + | GranularityList + | ((context: RequestContext) => GranularityList | Promise); + /** * Gateway configuration options interface. */ @@ -64,6 +77,13 @@ interface ApiGatewayOptions { scheduledRefreshTimeZones?: ScheduledRefreshTimeZonesFn; basePath: string; extendContext?: ExtendContextFn; + /** + * Enabled granularities (built-in names and/or custom definitions), or a function called per + * request to produce the same. Drives /v1/granularities and the /v1/meta enrichment. + * Shape mirrors `GranularityList` in @cubejs-backend/schema-compiler; redeclared locally to + * avoid a dependency on schema-compiler from this types module. + */ + granularities?: GranularitiesOption; jwt?: JWTOptions; requestLoggerMiddleware?: RequestLoggerMiddlewareFn; queryRewrite?: QueryRewriteFn; diff --git a/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts b/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts index 2fe52c50fb52d..c87ee9291b411 100644 --- a/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts +++ b/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts @@ -182,6 +182,7 @@ describe('prepareAnnotation helpers', () => { }).timeDimensions ).toEqual({ 'cube_name.member': { + currency: undefined, description: undefined, format: undefined, meta: undefined, @@ -190,6 +191,7 @@ describe('prepareAnnotation helpers', () => { type: undefined, }, 'cube_name.member.day': { + currency: undefined, description: undefined, format: undefined, meta: undefined, @@ -198,8 +200,10 @@ describe('prepareAnnotation helpers', () => { type: undefined, granularity: { name: 'day', - title: 'day', + type: 'built-in', + title: 'Day', interval: '1 day', + format: '%Y-%m-%d', } }, }); diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index ad63f10ba1cac..3a44dd15a8ea8 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -2049,6 +2049,18 @@ const variables: Record any> = { .asString(), accessPolicyMaskNumber: () => get('CUBEJS_ACCESS_POLICY_MASK_NUMBER') .asString(), + // Comma-separated names (built-in or custom). Empty/unset = all 8 built-ins enabled. + granularities: () => get('CUBEJS_GRANULARITIES') + .asArray(','), + // `getEnv` forwards `opts` positionally, so callers pass `{ name }` (matches dbType: { dataSource }). + granularityCustomInterval: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_INTERVAL`) + .asString(), + granularityCustomTitle: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_TITLE`) + .asString(), + granularityCustomOffset: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_OFFSET`) + .asString(), + granularityCustomOrigin: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_ORIGIN`) + .asString(), }; type Vars = typeof variables; diff --git a/packages/cubejs-client-core/src/time.ts b/packages/cubejs-client-core/src/time.ts index 9143dd8bccf04..036860a649432 100644 --- a/packages/cubejs-client-core/src/time.ts +++ b/packages/cubejs-client-core/src/time.ts @@ -22,7 +22,12 @@ export type SqlInterval = string; // TODO: Define a better type as unitOfTime.DurationConstructor in moment.js export type ParsedInterval = Record; +// Runtime shape for custom-granularity time-series math. For built-ins, see +// TimeDimensionPredefinedGranularity (query value) and GranularityAnnotation (response field). export type Granularity = { + type?: 'built-in' | 'custom'; + title?: string; + format?: string; interval: SqlInterval; origin?: string; offset?: SqlInterval; diff --git a/packages/cubejs-client-core/src/types.ts b/packages/cubejs-client-core/src/types.ts index 4a3ee92658c17..4414ade60bc33 100644 --- a/packages/cubejs-client-core/src/types.ts +++ b/packages/cubejs-client-core/src/types.ts @@ -12,7 +12,11 @@ export type TQueryOrderArray = Array<[string, QueryOrder]>; export type GranularityAnnotation = { name: string; + type?: 'built-in' | 'custom'; title: string; + /** d3-time-format string for displaying bucketed timestamps. */ + format?: string; + /** Always present: built-ins use "1 "; customs carry the user-defined interval. */ interval: string; offset?: string; origin?: string; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index e77f848b3fa94..79b4dccc136de 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -5,6 +5,7 @@ import { camelize } from 'inflection'; import { UserError } from './UserError'; import { DynamicReference } from './DynamicReference'; import { camelizeCube } from './utils'; +import { normalizeGranularitiesBlock, NormalizedGranularitiesBlock } from './GranularityResolver'; import type { ErrorReporter } from './ErrorReporter'; import { TranspilerSymbolResolver } from './transpilers'; @@ -16,6 +17,8 @@ export type GranularityDefinition = { sql?: (...args: any[]) => string; name?: string; title?: string; + /** d3-time-format string used by the client to display bucketed timestamps. */ + format?: string; interval?: string; offset?: string; origin?: string; @@ -33,6 +36,7 @@ export type CubeSymbolDefinition = { sql?: (...args: any[]) => string; primaryKey?: boolean; granularities?: Record; + granularitiesBlock?: NormalizedGranularitiesBlock; timeShift?: TimeshiftDefinition[]; format?: string; currency?: string; @@ -581,6 +585,8 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface this.camelCaseTypes(cube.preAggregations); this.camelCaseTypes(cube.accessPolicy); + this.normalizeDimensionGranularities(cube.dimensions); + if (cube.preAggregations) { this.transformPreAggregations(cube.preAggregations); } @@ -603,6 +609,23 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface } as CubeSymbolsDefinition; } + // Stores the canonical `granularitiesBlock` on each time dimension and rewrites + // `granularities` to the dict of locally-defined customs only — preserving the legacy shape + // that BaseQuery, prepare-annotation, and CubeToMetaTransformer already read. + private normalizeDimensionGranularities(dimensions: Record | undefined) { + if (!dimensions) { + return; + } + + for (const dim of Object.values(dimensions)) { + if (dim && dim.type === 'time' && 'granularities' in dim) { + const block: NormalizedGranularitiesBlock = normalizeGranularitiesBlock(dim.granularities); + dim.granularitiesBlock = block; + dim.granularities = block.custom; + } + } + } + private camelCaseTypes(obj: Object | Array | undefined) { if (!obj) { return; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index da4eb611c221e..38b30ff7ee269 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -302,6 +302,10 @@ export class CubeToMetaTransformer implements CompilerInterface { ? this.isVisible(extendedDimDef, !extendedDimDef.primaryKey) : false; const granularitiesObj = extendedDimDef.granularities; + // The gateway reconciles `granularitiesBlock` (includes/excludes/custom) with global + // config per request. Forwarded as-is; the flat `granularities` field alone can't + // represent `includes: '*'` vs `includes: []`. + const { granularitiesBlock } = extendedDimDef as any; const dimType = this.dimensionDataType(extendedDimDef.type || 'string'); const dimFormat = this.transformDimensionFormat(extendedDimDef); const dimCurrency = extendedDimDef.currency?.toUpperCase(); @@ -328,12 +332,15 @@ export class CubeToMetaTransformer implements CompilerInterface { granularitiesObj ? Object.entries(granularitiesObj).map(([gName, gDef]: [string, any]) => ({ name: gName, + type: 'custom', title: this.title(cubeTitle, [gName, gDef], true), + ...(gDef.format !== undefined ? { format: gDef.format } : {}), interval: gDef.interval, offset: gDef.offset, origin: gDef.origin, })) : undefined, + granularitiesBlock, order: extendedDimDef.order, key: extendedDimDef.keyReference, ...(extendedDimDef.links ? { links: extendedDimDef.links.map((link: any) => ({ diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index a7452ea2bff33..ce3d8aa34adaa 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -113,6 +113,74 @@ const GranularityInterval = Joi.string().pattern(/^\d+\s+(second|minute|hour|day // Do not allow negative intervals for granularities, while offsets could be negative const GranularityOffset = Joi.string().pattern(/^-?(\d+\s+)(second|minute|hour|day|week|month|quarter|year)s?(\s-?\d+\s+(second|minute|hour|day|week|month|quarter|year)s?){0,7}$/, 'granularity offset'); +// One custom granularity entry: with-origin, with-offset (interval must be aligned), or sql-defined. +// Reused by the legacy `granularities: { name: {...} }` form and the new `custom: { name: {...} }` form. +const CustomGranularityEntrySchema = Joi.alternatives([ + Joi.object().keys({ + title: Joi.string(), + format: Joi.string(), + interval: GranularityInterval.required(), + origin: Joi.string().required().custom((value, helpers) => { + const date = new Date(value); + + if (Number.isNaN(date.getTime())) { + return helpers.message({ custom: 'Origin should be valid date-only form: YYYY[-MM[-DD]] or date-time form: YYYY-MM-DD[T]HH:mm[:ss[.sss[Z]]]' }); + } + return value; + }), + }), + Joi.object().keys({ + title: Joi.string(), + format: Joi.string(), + interval: GranularityInterval.required().custom((value, helper) => { + const intParsed = value.split(' '); + const msg = { custom: 'Arbitrary intervals cannot be used without origin point specified' }; + + if (intParsed.length !== 2) { + return helper.message(msg); + } + + const v = parseInt(intParsed[0], 10); + const unit = intParsed[1]; + + const validIntervals = { + // Any number of years is valid + year: () => true, + // Only months divisible by a year with no remainder are valid + month: () => 12 % v === 0, + // Only quarters divisible by a year with no remainder are valid + quarter: () => 4 % v === 0, + // Only 1 week is valid + week: () => v === 1, + // Only 1 day is valid + day: () => v === 1, + // Only hours divisible by a day with no remainder are valid + hour: () => 24 % v === 0, + // Only minutes divisible by an hour with no remainder are valid + minute: () => 60 % v === 0, + // Only seconds divisible by a minute with no remainder are valid + second: () => 60 % v === 0, + }; + + const isValid = Object.keys(validIntervals).some(key => unit.includes(key) && validIntervals[key]()); + + return isValid ? value : helper.message(msg); + }), + offset: GranularityOffset.optional(), + }), + Joi.object().keys({ + title: Joi.string(), + format: Joi.string(), + sql: Joi.func().required() + }) +]); + +// `includes` / `excludes` accept a list of granularity names or the wildcard `'*'`. +const GranularityInclusionListSchema = Joi.alternatives([ + Joi.string().valid('*'), + Joi.array().items(Joi.string()), +]); + const formatAlternatives = [ Joi.string().valid('imageUrl', 'link', 'currency', 'percent', 'number', 'id'), Joi.object().keys({ @@ -380,65 +448,32 @@ const BaseDimensionWithoutSubQuery = { }), granularities: Joi.when('type', { is: 'time', - then: Joi.object().pattern(identifierRegex, - Joi.alternatives([ - Joi.object().keys({ - title: Joi.string(), - interval: GranularityInterval.required(), - origin: Joi.string().required().custom((value, helpers) => { - const date = new Date(value); - - if (Number.isNaN(date.getTime())) { - return helpers.message({ custom: 'Origin should be valid date-only form: YYYY[-MM[-DD]] or date-time form: YYYY-MM-DD[T]HH:mm[:ss[.sss[Z]]]' }); - } - return value; - }), - }), - Joi.object().keys({ - title: Joi.string(), - interval: GranularityInterval.required().custom((value, helper) => { - const intParsed = value.split(' '); - const msg = { custom: 'Arbitrary intervals cannot be used without origin point specified' }; - - if (intParsed.length !== 2) { - return helper.message(msg); - } - - const v = parseInt(intParsed[0], 10); - const unit = intParsed[1]; - - const validIntervals = { - // Any number of years is valid - year: () => true, - // Only months divisible by a year with no remainder are valid - month: () => 12 % v === 0, - // Only quarters divisible by a year with no remainder are valid - quarter: () => 4 % v === 0, - // Only 1 week is valid - week: () => v === 1, - // Only 1 day is valid - day: () => v === 1, - // Only hours divisible by a day with no remainder are valid - hour: () => 24 % v === 0, - // Only minutes divisible by an hour with no remainder are valid - minute: () => 60 % v === 0, - // Only seconds divisible by a minute with no remainder are valid - second: () => 60 % v === 0, - }; - - const isValid = Object.keys(validIntervals).some(key => unit.includes(key) && validIntervals[key]()); - - return isValid ? value : helper.message(msg); - }), - offset: GranularityOffset.optional(), + then: Joi.alternatives() + .conditional(Joi.ref('.'), { + // Discriminate by shape: only the new dict form has includes/excludes/custom and no other keys. + is: Joi.object().keys({ + includes: Joi.any(), + excludes: Joi.any(), + custom: Joi.any(), + }).unknown(false), + then: Joi.object().keys({ + includes: GranularityInclusionListSchema, + excludes: GranularityInclusionListSchema, + custom: Joi.object().pattern(identifierRegex, CustomGranularityEntrySchema), + }).custom((value, helper) => { + if (value && value.includes !== undefined && value.excludes !== undefined && value.includes !== '*') { + return helper.message({ custom: '"includes" and "excludes" cannot be used together unless includes is "*"' } as any); + } + return value; }), - Joi.object().keys({ - title: Joi.string(), - sql: Joi.func().required() - }) - ])).optional(), + otherwise: Joi.object().pattern(identifierRegex, CustomGranularityEntrySchema), + }) + .optional(), otherwise: Joi.forbidden() - }) + }), + // Internal field written by CubeSymbols.normalizeDimensionGranularities before the validator runs. + // Not user-facing; declared so unknown-key validation doesn't reject it. + granularitiesBlock: Joi.any() }; const BaseDimension = { diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts new file mode 100644 index 0000000000000..d30aa08ea3bd0 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -0,0 +1,154 @@ +import { getEnv } from '@cubejs-backend/shared'; + +import type { GranularityDefinition } from './CubeSymbols'; + +// Default `title` and `format` for each built-in granularity. Overridable via `config.granularities` +// (file) or `CUBEJS_GRANULARITIES__TITLE` (env, title only). Format syntax: d3-time-format. + +export type BuiltInGranularityDefinition = { + title: string; + format: string; +}; + +export const BUILT_IN_GRANULARITIES: Readonly> = Object.freeze({ + year: { title: 'Year', format: '%Y' }, + quarter: { title: 'Quarter', format: 'Q%q %Y' }, + month: { title: 'Month', format: '%b %Y' }, + week: { title: 'Week', format: '%b %-d, %Y' }, + day: { title: 'Day', format: '%Y-%m-%d' }, + hour: { title: 'Hour', format: '%Y-%m-%d %H:00' }, + minute: { title: 'Minute', format: '%Y-%m-%d %H:%M' }, + second: { title: 'Second', format: '%Y-%m-%d %H:%M:%S' }, +}); + +export const BUILT_IN_GRANULARITY_NAMES = Object.freeze(Object.keys(BUILT_IN_GRANULARITIES)); + +export function isBuiltInGranularity(name: string): boolean { + return name in BUILT_IN_GRANULARITIES; +} + +// Item shape accepted in `config.granularities`: a built-in name, or a custom granularity object. +export type GranularityListItem = string | (GranularityDefinition & { name: string }); +export type GranularityList = GranularityListItem[]; + +export type GlobalGranularitiesConfig = { + enabledBuiltIns: ReadonlyArray; + customGranularities: Readonly>; +}; + +const DEFAULT_CONFIG: GlobalGranularitiesConfig = Object.freeze({ + enabledBuiltIns: BUILT_IN_GRANULARITY_NAMES, + customGranularities: Object.freeze({}), +}); + +// Read `CUBEJS_GRANULARITIES__{INTERVAL,TITLE,OFFSET,ORIGIN}` for a custom granularity name. +// Only consulted for names sourced from `CUBEJS_GRANULARITIES`, not for `config.granularities` entries. +function applyEnvOverrides(name: string, base?: Partial): GranularityDefinition { + // getEnv types `opts` as a Parameters<> tuple but forwards it positionally; cast to bypass that. + const opts = { name } as any; + const interval = getEnv('granularityCustomInterval', opts) ?? base?.interval; + const title = getEnv('granularityCustomTitle', opts) ?? base?.title; + const offset = getEnv('granularityCustomOffset', opts) ?? base?.offset; + const origin = getEnv('granularityCustomOrigin', opts) ?? base?.origin; + + const out: GranularityDefinition = {}; + if (interval !== undefined) out.interval = interval; + if (title !== undefined) out.title = title; + if (offset !== undefined) out.offset = offset; + if (origin !== undefined) out.origin = origin; + return out; +} + +function resolveFromEnv(): GlobalGranularitiesConfig { + const list = getEnv('granularities'); + if (!list || list.length === 0) { + return DEFAULT_CONFIG; + } + + const enabledBuiltIns: string[] = []; + const customGranularities: Record = {}; + for (const name of list) { + const trimmed = name.trim(); + if (trimmed) { + if (isBuiltInGranularity(trimmed)) { + enabledBuiltIns.push(trimmed); + } else { + // Non-built-in name: pull the definition from `CUBEJS_GRANULARITIES__*` env vars. + customGranularities[trimmed] = applyEnvOverrides(trimmed); + } + } + } + return { enabledBuiltIns, customGranularities }; +} + +function resolveFromList(list: GranularityList): GlobalGranularitiesConfig { + const enabledBuiltIns: string[] = []; + const customGranularities: Record = {}; + + for (const item of list) { + if (typeof item === 'string') { + if (isBuiltInGranularity(item)) { + enabledBuiltIns.push(item); + } + // A bare non-built-in string has no definition attached and is silently dropped: + // custom granularities in `config.granularities` must be objects. + } else if (item && typeof item === 'object' && item.name) { + const { name, ...def } = item; + if (isBuiltInGranularity(name)) { + // `{ name: 'year', title: 'Anno' }` both enables 'year' and overrides its title/format. + enabledBuiltIns.push(name); + customGranularities[name] = def; + } else { + customGranularities[name] = def; + } + } + } + return { enabledBuiltIns, customGranularities }; +} + +// `userValue` is the value of `granularities` from the cube.js / cube.py config file. +// undefined -> fall back to `CUBEJS_GRANULARITIES` env vars +// GranularityList -> use this list, replacing env vars entirely (no merge) +// function(ctx) -> called per request; same no-merge replacement as the list form +export async function resolveGlobalGranularities( + userValue: GranularityList | ((ctx: any) => GranularityList | Promise) | undefined, + ctx: any, +): Promise { + if (userValue === undefined) { + return resolveFromEnv(); + } + const resolved = typeof userValue === 'function' ? await userValue(ctx) : userValue; + if (!Array.isArray(resolved)) { + return resolveFromEnv(); + } + return resolveFromList(resolved); +} + +export function getBuiltInGranularityDefaults(name: string) { + return BUILT_IN_GRANULARITIES[name]; +} + +// Resolve title/format/interval for each enabled built-in, applying any override from +// `globalConfig.customGranularities` (e.g. `{ name: 'year', title: 'Jaar' }` localizes the title). +// `interval` is filled in as "1 " so built-ins share the same response shape as customs. +export type BuiltInCatalogEntry = { + title: string; + format: string; + interval: string; +}; + +export function buildBuiltInsCatalog(globalConfig: GlobalGranularitiesConfig): Record { + const catalog: Record = {}; + for (const name of globalConfig.enabledBuiltIns) { + const defaults = BUILT_IN_GRANULARITIES[name]; + if (defaults) { + const override = globalConfig.customGranularities[name]; + catalog[name] = { + title: override?.title || defaults.title, + format: override?.format || defaults.format, + interval: override?.interval || `1 ${name}`, + }; + } + } + return catalog; +} diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts new file mode 100644 index 0000000000000..4f73924a6460d --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -0,0 +1,107 @@ +import type { GranularityDefinition } from './CubeSymbols'; + +// `'*'` is the wildcard form of an includes/excludes list. +export type GranularityInclusionList = '*' | string[]; + +// Canonical form produced by `normalizeGranularitiesBlock`. All downstream readers +// (resolver, meta transformer, pre-agg matcher) only see this shape. +export type NormalizedGranularitiesBlock = { + includes: GranularityInclusionList; + excludes: GranularityInclusionList; + custom: Record; +}; + +// Effective granularity set for one time dimension, ready to serialize into /v1/meta. +export type ResolvedGranularitySet = Record; + +// Used when a time dimension has no `granularities` block at all: take all enabled globals, no local customs. +const EMPTY_BLOCK: NormalizedGranularitiesBlock = { + includes: '*', + excludes: [], + custom: {}, +}; + +// Map any accepted input shape (omitted, legacy array, post-yaml keyed object, new dict) +// onto NormalizedGranularitiesBlock. Validator runs first, so malformed input never reaches here. +export function normalizeGranularitiesBlock(raw: any): NormalizedGranularitiesBlock { + if (raw == null) { + return EMPTY_BLOCK; + } + + if (Array.isArray(raw)) { + // Legacy form. Every array entry is a custom granularity; built-ins are inherited from globals. + return { + includes: '*', + excludes: [], + custom: Object.fromEntries(raw.filter(g => g && g.name).map(g => { + const { name, ...rest } = g; + return [name, rest]; + })), + }; + } + + if (typeof raw === 'object') { + if ('includes' in raw || 'excludes' in raw || 'custom' in raw) { + // New dict form. YamlCompiler has already keyed `custom` by name. + return { + includes: raw.includes ?? '*', + excludes: raw.excludes ?? [], + custom: raw.custom ?? {}, + }; + } + // Already-keyed legacy form (e.g. coming from JS configs, not YAML). Treat as custom-only. + return { + includes: '*', + excludes: [], + custom: raw, + }; + } + + return EMPTY_BLOCK; +} + +// Reconcile a dimension's local block against the global enabled built-ins and global customs, +// producing the effective set. Local customs always survive — even if local excludes is '*'. +export function resolveDimensionGranularities( + localBlock: NormalizedGranularitiesBlock, + globalEnabledBuiltIns: ReadonlyArray, + globalCustom: Readonly>, + allBuiltInsCatalog: Readonly>, +): ResolvedGranularitySet { + const out: ResolvedGranularitySet = {}; + + const localIncludes = localBlock.includes; + const localExcludes = localBlock.excludes; + + const includesAllowsAll = localIncludes === '*'; + const includesSet = includesAllowsAll ? null : new Set(localIncludes); + const excludesBlocksAll = localExcludes === '*'; + const excludesSet = excludesBlocksAll ? null : new Set(localExcludes); + + // Built-ins and global customs are filtered the same way: keep iff included AND not excluded. + if (!excludesBlocksAll) { + for (const builtInName of globalEnabledBuiltIns) { + const passesIncludes = includesAllowsAll || includesSet!.has(builtInName); + const blockedByExcludes = excludesSet!.has(builtInName); + const def = allBuiltInsCatalog[builtInName]; + if (passesIncludes && !blockedByExcludes && def) { + out[builtInName] = { ...def, type: 'built-in' }; + } + } + for (const [name, def] of Object.entries(globalCustom)) { + const passesIncludes = includesAllowsAll || includesSet!.has(name); + const blockedByExcludes = excludesSet!.has(name); + if (passesIncludes && !blockedByExcludes) { + out[name] = { ...def, type: 'custom' }; + } + } + } + + // Local customs are always emitted, even when `excludes: '*'` strips everything else. + // Same-named local entry replaces a global one (last-write-wins). + for (const [name, def] of Object.entries(localBlock.custom)) { + out[name] = { ...def, type: 'custom' }; + } + + return out; +} diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index 42e20a9f4cb63..770cb779668c3 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -450,10 +450,22 @@ export class YamlCompiler { } if (memberType === 'dimension' && granularities) { - granularities = this.yamlArrayToObj(granularities || [], 'dimension.granularity', errorsReport, { - cubeName: ctx.cubeName, - parent: { type: 'time dimension', name } - }); + if (Array.isArray(granularities)) { + // Legacy form: `granularities: [{ name, ... }]` -> `{ name: { ... } }`. + granularities = this.yamlArrayToObj(granularities || [], 'dimension.granularity', errorsReport, { + cubeName: ctx.cubeName, + parent: { type: 'time dimension', name } + }); + } else if (granularities && typeof granularities === 'object' && Array.isArray(granularities.custom)) { + // New dict form: only the inner `custom` array needs keying; includes/excludes pass through. + granularities = { + ...granularities, + custom: this.yamlArrayToObj(granularities.custom, 'dimension.granularity', errorsReport, { + cubeName: ctx.cubeName, + parent: { type: 'time dimension', name } + }), + }; + } res[name] = { granularities, ...res[name] }; } diff --git a/packages/cubejs-schema-compiler/src/compiler/index.ts b/packages/cubejs-schema-compiler/src/compiler/index.ts index ff62cfaa59ddc..86f7b4ff85114 100644 --- a/packages/cubejs-schema-compiler/src/compiler/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/index.ts @@ -11,3 +11,22 @@ export { PreAggregationInfo, EvaluatedCube, } from './CubeEvaluator'; +export { + BUILT_IN_GRANULARITIES, + BUILT_IN_GRANULARITY_NAMES, + isBuiltInGranularity, + BuiltInGranularityDefinition, + GranularityList, + GranularityListItem, + GlobalGranularitiesConfig, + BuiltInCatalogEntry, + resolveGlobalGranularities, + getBuiltInGranularityDefaults, + buildBuiltInsCatalog, +} from './GlobalGranularitiesConfig'; +export { + NormalizedGranularitiesBlock, + ResolvedGranularitySet, + normalizeGranularitiesBlock, + resolveDimensionGranularities, +} from './GranularityResolver'; diff --git a/packages/cubejs-schema-compiler/src/compiler/utils.ts b/packages/cubejs-schema-compiler/src/compiler/utils.ts index f917e6a93be06..9af688a1e92ed 100644 --- a/packages/cubejs-schema-compiler/src/compiler/utils.ts +++ b/packages/cubejs-schema-compiler/src/compiler/utils.ts @@ -4,6 +4,11 @@ import { camelize } from 'inflection'; const IGNORE_CAMELIZE = { 1: { granularities: true, + }, + // Custom granularity names live one level deeper than the legacy flat-array form + // (`granularities.custom.`), so they need their own guard. + 2: { + custom: true, } }; diff --git a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts index 3b6bd2f32bd86..f8ab629974f6e 100644 --- a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts @@ -1716,6 +1716,93 @@ describe('Cube Validation', () => { }); }); + describe('Granularities dict shape (includes/excludes/custom):', () => { + const newCube = (granularities: any) => ({ + name: 'Orders', + fileName: 'fileName', + sql: () => 'select * from tbl', + public: true, + dimensions: { + createdAt: { + public: true, + sql: () => 'created_at', + type: 'time', + granularities, + }, + }, + measures: { + count: { sql: () => 'count', type: 'count' }, + }, + }); + + it('accepts includes-only with built-in names', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: ['year', 'quarter'] }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts excludes-only with built-in names', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ excludes: ['day'] }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts includes "*" wildcard', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: '*' }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts custom-only', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ + custom: { fiscal_year: { interval: '1 year', origin: '2026-04-01' } }, + }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts includes "*" combined with excludes', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: '*', excludes: ['day'] }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts excludes "*" combined with custom', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ + excludes: '*', + custom: { fiscal_year: { interval: '1 year', origin: '2026-04-01' } }, + }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('rejects includes + excludes both as lists (mutually exclusive)', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: ['year'], excludes: ['day'] }); + let captured: string | undefined; + const validationResult = cubeValidator.validate(cube, { + error: (message: any) => { captured = message; }, + } as any); + expect(validationResult.error).toBeTruthy(); + expect(captured).toContain('"includes" and "excludes" cannot be used together'); + }); + + it('legacy flat-array form (post yamlArrayToObj) still validates', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ + fiscal_year: { interval: '1 year', origin: '2026-04-01' }, + }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + }); + describe('Access Policy group/groups support:', () => { const cubeValidator = new CubeValidator(new CubeSymbols()); diff --git a/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts b/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts new file mode 100644 index 0000000000000..be38fae4b89b6 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts @@ -0,0 +1,75 @@ +import { resolveGlobalGranularities, BUILT_IN_GRANULARITY_NAMES } from '../../src/compiler/GlobalGranularitiesConfig'; + +describe('resolveGlobalGranularities', () => { + const originalEnv = { ...process.env }; + + afterEach(() => { + for (const key of Object.keys(process.env)) { + if (key.startsWith('CUBEJS_GRANULARITIES')) delete process.env[key]; + } + Object.assign(process.env, originalEnv); + }); + + it('user value undefined + no env -> defaults to full built-in catalog', async () => { + const cfg = await resolveGlobalGranularities(undefined, {}); + expect([...cfg.enabledBuiltIns].sort()).toEqual([...BUILT_IN_GRANULARITY_NAMES].sort()); + expect(cfg.customGranularities).toEqual({}); + }); + + it('CUBEJS_GRANULARITIES restricts enabled built-ins', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,quarter,month'; + const cfg = await resolveGlobalGranularities(undefined, {}); + expect(cfg.enabledBuiltIns).toEqual(['year', 'quarter', 'month']); + expect(cfg.customGranularities).toEqual({}); + }); + + it('CUBEJS_GRANULARITIES with a custom name + companion env vars produces a custom granularity', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,fiscal_year'; + process.env.CUBEJS_GRANULARITIES_FISCAL_YEAR_INTERVAL = '1 year'; + process.env.CUBEJS_GRANULARITIES_FISCAL_YEAR_ORIGIN = '2026-04-01'; + process.env.CUBEJS_GRANULARITIES_FISCAL_YEAR_TITLE = 'Fiscal Year'; + const cfg = await resolveGlobalGranularities(undefined, {}); + expect(cfg.enabledBuiltIns).toEqual(['year']); + expect(cfg.customGranularities.fiscal_year).toEqual({ + interval: '1 year', + origin: '2026-04-01', + title: 'Fiscal Year', + }); + }); + + it('file-config replaces env vars (full replacement, not merge)', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,quarter,month'; + const cfg = await resolveGlobalGranularities(['day', 'hour'], {}); + expect(cfg.enabledBuiltIns).toEqual(['day', 'hour']); + }); + + it('file-config can mix built-in names + custom objects', async () => { + const cfg = await resolveGlobalGranularities( + ['year', { name: 'fiscal_year', interval: '1 year', origin: '2026-04-01' }], + {}, + ); + expect(cfg.enabledBuiltIns).toEqual(['year']); + expect(cfg.customGranularities.fiscal_year).toEqual({ interval: '1 year', origin: '2026-04-01' }); + }); + + it('file-config function is invoked with the request context', async () => { + let receivedCtx: any; + const cfg = await resolveGlobalGranularities( + (ctx) => { + receivedCtx = ctx; + return ['year']; + }, + { securityContext: { tenant: 't1' } }, + ); + expect(receivedCtx.securityContext.tenant).toBe('t1'); + expect(cfg.enabledBuiltIns).toEqual(['year']); + }); + + it('file-config function may return a Promise', async () => { + const cfg = await resolveGlobalGranularities( + async () => ['quarter'], + {}, + ); + expect(cfg.enabledBuiltIns).toEqual(['quarter']); + }); +}); diff --git a/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts b/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts new file mode 100644 index 0000000000000..39f4924f203d3 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts @@ -0,0 +1,154 @@ +import { + normalizeGranularitiesBlock, + resolveDimensionGranularities, +} from '../../src/compiler/GranularityResolver'; + +const BUILT_INS = { + year: { title: 'Year' }, + quarter: { title: 'Quarter' }, + month: { title: 'Month' }, + week: { title: 'Week' }, + day: { title: 'Day' }, + hour: { title: 'Hour' }, + minute: { title: 'Minute' }, + second: { title: 'Second' }, +}; + +const ALL_ENABLED = Object.keys(BUILT_INS); +const FISCAL_YEAR = { title: 'Fiscal Year', interval: '1 year', origin: '2026-04-01' }; +const GLOBAL_CUSTOM = { fiscal_year: FISCAL_YEAR }; + +describe('normalizeGranularitiesBlock', () => { + it('treats missing input as wide-open: includes * / no excludes / no custom', () => { + expect(normalizeGranularitiesBlock(undefined)).toEqual({ + includes: '*', + excludes: [], + custom: {}, + }); + expect(normalizeGranularitiesBlock(null)).toEqual({ + includes: '*', + excludes: [], + custom: {}, + }); + }); + + it('legacy flat-array form maps each entry into custom; includes stays *', () => { + const out = normalizeGranularitiesBlock([ + { name: 'fiscal_q', interval: '3 months', origin: '2026-04-01' }, + ]); + expect(out.includes).toBe('*'); + expect(out.excludes).toEqual([]); + expect(out.custom).toEqual({ + fiscal_q: { interval: '3 months', origin: '2026-04-01' }, + }); + }); + + it('post-yamlArrayToObj keyed object is preserved as legacy custom-only block', () => { + const out = normalizeGranularitiesBlock({ + fiscal_q: { interval: '3 months', origin: '2026-04-01' }, + }); + expect(out.includes).toBe('*'); + expect(out.custom.fiscal_q).toEqual({ interval: '3 months', origin: '2026-04-01' }); + }); + + it('new dict shape is canonicalized with defaults', () => { + const out = normalizeGranularitiesBlock({ includes: ['year'], custom: { fy: FISCAL_YEAR } }); + expect(out).toEqual({ + includes: ['year'], + excludes: [], + custom: { fy: FISCAL_YEAR }, + }); + }); +}); + +describe('resolveDimensionGranularities — spec resolution table', () => { + it('row 1: granularities omitted -> all enabled global granularities', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(Object.keys(out).sort()).toEqual([...ALL_ENABLED].sort()); + expect(out.year.type).toBe('built-in'); + }); + + it('row 2: legacy flat array -> enabled globals plus local custom', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock([{ name: 'fiscal_q', interval: '3 months', origin: '2026-04-01' }]), + ['year', 'month'], + {}, + BUILT_INS, + ); + expect(out.year.type).toBe('built-in'); + expect(out.month.type).toBe('built-in'); + expect(out.fiscal_q).toMatchObject({ interval: '3 months', origin: '2026-04-01', type: 'custom' }); + }); + + it('row 3: includes [a, b] + custom -> {a, b} plus custom', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ includes: ['year', 'quarter'], custom: { fy: FISCAL_YEAR } }), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(Object.keys(out).sort()).toEqual(['fy', 'quarter', 'year']); + expect(out.year.type).toBe('built-in'); + expect(out.fy.type).toBe('custom'); + }); + + it('row 4: excludes [x] -> all enabled globals minus x', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ excludes: ['day'] }), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(out.day).toBeUndefined(); + expect(out.year.type).toBe('built-in'); + }); + + it('row 5: excludes "*" + custom -> custom only', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ excludes: '*', custom: { fy: FISCAL_YEAR } }), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(Object.keys(out)).toEqual(['fy']); + expect(out.fy.type).toBe('custom'); + }); + + it('row 6: includes "*" + custom -> all enabled globals plus custom', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ includes: '*', custom: { fy: FISCAL_YEAR } }), + ['year', 'month'], + {}, + BUILT_INS, + ); + expect(Object.keys(out).sort()).toEqual(['fy', 'month', 'year']); + expect(out.fy.type).toBe('custom'); + }); + + it('global custom granularities flow through unless excluded', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), + ['year'], + GLOBAL_CUSTOM, + BUILT_INS, + ); + expect(out.fiscal_year.type).toBe('custom'); + expect(out.year.type).toBe('built-in'); + }); + + it('local custom overrides global custom of same name', () => { + const localFy = { interval: '1 year', origin: '2026-01-01' }; + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ custom: { fiscal_year: localFy } }), + ['year'], + GLOBAL_CUSTOM, + BUILT_INS, + ); + expect(out.fiscal_year.origin).toBe('2026-01-01'); + }); +}); diff --git a/packages/cubejs-server-core/src/core/optionsValidate.ts b/packages/cubejs-server-core/src/core/optionsValidate.ts index 0ba331d9f9476..c424dc66c4694 100644 --- a/packages/cubejs-server-core/src/core/optionsValidate.ts +++ b/packages/cubejs-server-core/src/core/optionsValidate.ts @@ -90,6 +90,15 @@ const schemaOptions = Joi.object().keys({ ), schemaVersion: Joi.func(), extendContext: Joi.func(), + granularities: Joi.alternatives().try( + Joi.func(), + Joi.array().items( + Joi.alternatives().try( + Joi.string(), + Joi.object().unknown(true) + ) + ) + ), // Scheduled refresh scheduledRefreshTimer: Joi.alternatives().try( Joi.boolean(), diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index 48b146c8abb70..8c1a61ca6df55 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -481,6 +481,7 @@ export class CubejsServerCore { queryRewrite: this.options.queryRewrite || this.options.queryTransformer, extendContext: this.options.extendContext, + granularities: this.options.granularities, playgroundAuthSecret: getEnv('playgroundAuthSecret'), apiSecrets: this.options.apiSecrets, jwt: this.options.jwt, diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts index 3d1cfee7d79ee..502337dc18c8f 100644 --- a/packages/cubejs-server-core/src/core/types.ts +++ b/packages/cubejs-server-core/src/core/types.ts @@ -10,7 +10,7 @@ import { UserBackgroundContext, } from '@cubejs-backend/api-gateway'; import { BaseDriver, CacheAndQueryDriverType } from '@cubejs-backend/query-orchestrator'; -import { BaseQuery } from '@cubejs-backend/schema-compiler'; +import { BaseQuery, GranularityList } from '@cubejs-backend/schema-compiler'; export interface QueueOptions { concurrency?: number; @@ -217,6 +217,7 @@ export interface CreateOptions { preAggregationsSchema?: string | PreAggregationsSchemaFn; schemaVersion?: (context: RequestContext) => string | Promise; extendContext?: ExtendContextFn; + granularities?: GranularityList | ((context: RequestContext) => GranularityList | Promise); scheduledRefreshTimer?: boolean | number; scheduledRefreshTimeZones?: string[] | ScheduledRefreshTimeZonesFn; scheduledRefreshContexts?: () => Promise; diff --git a/rust/cube/cubeorchestrator/src/transport.rs b/rust/cube/cubeorchestrator/src/transport.rs index a32ff801b7cc4..52d36bd0fddfc 100644 --- a/rust/cube/cubeorchestrator/src/transport.rs +++ b/rust/cube/cubeorchestrator/src/transport.rs @@ -164,7 +164,13 @@ pub type MembersMap = IndexMap; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GranularityMeta { pub name: String, + /// Serialized as `type`: "built-in" or "custom". Field is named `kind` to avoid Rust's keyword. + #[serde(rename = "type", skip_serializing_if = "Option::is_none")] + pub kind: Option, pub title: String, + /// d3-time-format string used by the client to display bucketed timestamps. + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, #[serde(skip_serializing_if = "Option::is_none")] pub interval: Option, #[serde(skip_serializing_if = "Option::is_none")] From d24a272f51f9287b89644ed2f306f82e72024a22 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Fri, 22 May 2026 08:45:46 +0200 Subject: [PATCH 02/22] feat(backend-native): add granularities to Cube.py Configuration --- packages/cubejs-backend-native/python/cube/src/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/cubejs-backend-native/python/cube/src/__init__.py b/packages/cubejs-backend-native/python/cube/src/__init__.py index c793af0869056..d7d4ea822abe8 100644 --- a/packages/cubejs-backend-native/python/cube/src/__init__.py +++ b/packages/cubejs-backend-native/python/cube/src/__init__.py @@ -69,6 +69,9 @@ class Configuration: check_sql_auth: Callable can_switch_sql_user: Callable extend_context: Callable + # Mirrors `granularities` in the JS config: a list of granularity names and/or custom-granularity + # definitions, or a function returning the same. Drives /v1/granularities and /v1/meta enrichment. + granularities: Union[list, Callable[[RequestContext], list]] scheduled_refresh_contexts: Callable context_to_api_scopes: Callable repository_factory: Callable @@ -117,6 +120,7 @@ def __init__(self): self.can_switch_sql_user = None self.query_rewrite = None self.extend_context = None + self.granularities = None self.scheduled_refresh_contexts = None self.scheduled_refresh_time_zones = None self.context_to_api_scopes = None From 4b72d54cfb0e0461b4e2d5fa92419f61c9e392ef Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 16 Jul 2026 18:32:11 +0200 Subject: [PATCH 03/22] feat(granularities): expose effectiveGranularities, keep legacy granularities, resolve on demand --- DEPRECATION.md | 13 ++ packages/cubejs-api-gateway/openspec.yml | 32 +++- packages/cubejs-api-gateway/src/gateway.ts | 165 ++++++++++++------ .../src/helpers/prepare-annotation.ts | 33 +--- .../test/helpers/prepare-annotation.test.ts | 11 +- .../cubejs-api-gateway/test/index.test.ts | 2 + packages/cubejs-client-core/src/types.ts | 23 ++- .../src/compiler/CubeSymbols.ts | 4 + .../src/compiler/CubeToMetaTransformer.ts | 12 +- .../src/compiler/CubeValidator.ts | 58 +++--- .../src/compiler/GlobalGranularitiesConfig.ts | 6 +- .../src/compiler/GranularityResolver.ts | 9 +- 12 files changed, 254 insertions(+), 114 deletions(-) diff --git a/DEPRECATION.md b/DEPRECATION.md index c5e3f98026f37..3282b7c458a3d 100644 --- a/DEPRECATION.md +++ b/DEPRECATION.md @@ -70,6 +70,7 @@ features: | Removed | [Elasticsearch driver](#elasticsearch-driver) | v1.6.0 | v1.7.0 | | Removed | [`context_to_roles`](#context-to-roles) | v1.6.4 | v1.7.0 | | Deprecated | [Node.js 22](#nodejs-22) | v1.7.0 | | +| Deprecated | [`granularities` field on time dimensions in `/v1/meta`](#granularities-field-on-time-dimensions-in-v1meta) | v1.7.4 | | ### Node.js 8 @@ -454,3 +455,15 @@ The `context_to_roles` configuration option has been removed. Please use `contex Node.js 22 is in maintenance mode from [October 21, 2025][link-nodejs-eol]. This means no more new features, only security updates. Please upgrade to Node.js 24 or higher. + +### `granularities` field on time dimensions in `/v1/meta` + +**Deprecated in Release: v1.7.4** + +The `granularities` field returned for each time dimension by the `/v1/meta` endpoint lists only +the custom granularities defined in the data model, without the enabled built-in granularities or a +`type` discriminator. It is deprecated in favor of the `effectiveGranularities` field, which returns +the full reconciled set (enabled built-ins, global custom granularities, and per-dimension custom +granularities), each tagged with `type` (`built-in` or `custom`) and carrying `title`, `format`, and +`interval`. The `granularities` field remains for backward compatibility and will be removed in a +future release; new integrations should read `effectiveGranularities`. diff --git a/packages/cubejs-api-gateway/openspec.yml b/packages/cubejs-api-gateway/openspec.yml index 5fe9925a30bac..b964ad28c1307 100644 --- a/packages/cubejs-api-gateway/openspec.yml +++ b/packages/cubejs-api-gateway/openspec.yml @@ -133,9 +133,31 @@ components: type: "object" V1CubeMetaDimensionGranularity: type: "object" + deprecated: true + description: "Deprecated. Legacy shape listing only model-defined custom granularities. Use V1CubeMetaDimensionEffectiveGranularity via `effectiveGranularities`." required: - name - title + properties: + name: + type: "string" + title: + type: "string" + interval: + type: "string" + sql: + type: "string" + offset: + type: "string" + origin: + type: "string" + V1CubeMetaDimensionEffectiveGranularity: + type: "object" + description: "Reconciled granularity: enabled built-ins plus global and per-dimension custom granularities." + required: + - name + - type + - title properties: name: type: "string" @@ -152,8 +174,6 @@ components: description: "d3-time-format string used by clients to display bucketed timestamps." interval: type: "string" - sql: - type: "string" offset: type: "string" origin: @@ -168,7 +188,7 @@ components: granularities: type: array items: - $ref: "#/components/schemas/V1CubeMetaDimensionGranularity" + $ref: "#/components/schemas/V1CubeMetaDimensionEffectiveGranularity" V1CubeMetaDimension: type: "object" required: @@ -189,9 +209,15 @@ components: description: "When dimension is defined in View, it keeps the original path: Cube.dimension" type: "string" granularities: + deprecated: true + description: "Deprecated. Use `effectiveGranularities`." type: array items: $ref: "#/components/schemas/V1CubeMetaDimensionGranularity" + effectiveGranularities: + type: array + items: + $ref: "#/components/schemas/V1CubeMetaDimensionEffectiveGranularity" meta: type: "object" format: diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 2db93270ec392..c260ba63217a1 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -14,6 +14,7 @@ import { QueryAlias, CacheMode, LoggerFn, + isPredefinedGranularity, } from '@cubejs-backend/shared'; import { ResultArrayWrapper, @@ -39,6 +40,7 @@ import { buildBuiltInsCatalog, BUILT_IN_GRANULARITIES, } from '@cubejs-backend/schema-compiler'; +import type { GlobalGranularitiesConfig } from '@cubejs-backend/schema-compiler'; import { QueryType, ApiScopes, @@ -107,7 +109,7 @@ import { cachedHandler } from './cached-handler'; import { createJWKsFetcher } from './jwk'; import { SQLServer, SQLServerConstructorOptions } from './sql-server'; import { getJsonQueryFromGraphQLQuery, makeSchema } from './graphql'; -import { ConfigItem, prepareAnnotation } from './helpers/prepare-annotation'; +import { ConfigItem, prepareAnnotation, GranularityMeta, GranularityResolverFn } from './helpers/prepare-annotation'; import { transformCube, transformMeasure, @@ -2134,13 +2136,8 @@ class ApiGateway { }); metaConfigResult = this.filterVisibleItemsInMeta(context, metaConfigResult); - // Annotation reads from this meta. Without enrichment, /v1/load and /v1/cubesql would - // omit the type/title/format/interval fields that /v1/meta exposes. - const enrichedCubes = await this.applyGlobalGranularitiesToMetaCubes( - context, - metaConfigResult.map((m: any) => m.config), - ); - metaConfigResult = metaConfigResult.map((m: any, i: number) => ({ ...m, config: enrichedCubes[i] })); + // Resolve the queried granularity's meta on demand instead of rewriting the whole model. + const resolveGranularity = await this.buildGranularityResolver(context, metaConfigResult); const sqlQueries = await this.getSqlQueriesInternal(context, normalizedQueries); @@ -2158,7 +2155,7 @@ class ApiGateway { ); const annotation = prepareAnnotation( - metaConfigResult, normalizedQuery + metaConfigResult, normalizedQuery, resolveGranularity ); return this.prepareResultTransformData( @@ -2244,6 +2241,7 @@ class ApiGateway { }); metaConfigResult = this.filterVisibleItemsInMeta(context, metaConfigResult); + const resolveGranularity = await this.buildGranularityResolver(context, metaConfigResult); const sqlQueries = await this .getSqlQueriesInternal( @@ -2293,7 +2291,7 @@ class ApiGateway { const response = await adapterApi.executeQuery(finalQuery); const annotation = prepareAnnotation( - metaConfigResult, normalizedQueries[0] + metaConfigResult, normalizedQueries[0], resolveGranularity ); // TODO Can we just pass through data? Ensure hidden members can't be queried @@ -2330,7 +2328,7 @@ class ApiGateway { Boolean(sqlQueries[index].slowQuery); const annotation = prepareAnnotation( - metaConfigResult, normalizedQuery + metaConfigResult, normalizedQuery, resolveGranularity ); if (request.streaming) { @@ -2463,51 +2461,114 @@ class ApiGateway { return resolveGlobalGranularities(this.granularitiesOption, context); } - // Reconcile each time dimension's `granularitiesBlock` against the request's global config - // and emit the effective granularity array. Built-ins get tagged `built-in`, locals/globals - // become `custom`. Replaces the per-dim `granularities` array on the returned cube. - protected async applyGlobalGranularitiesToMetaCubes(context: RequestContext, cubes: any[]): Promise { + // Resolve one time dimension's effective granularity set against the request's global config. + // Returns the array serialized for `effectiveGranularities`; leaves the legacy `granularities` + // untouched. + private resolveEffectiveGranularitiesForDim( + dim: any, + globalConfig: GlobalGranularitiesConfig, + builtInsCatalog: Record, + ): any[] { + const localCustom: Record = {}; + for (const g of dim.granularities || []) { + localCustom[g.name] = { + title: g.title, + interval: g.interval, + offset: g.offset, + origin: g.origin, + ...(g.format !== undefined ? { format: g.format } : {}), + }; + } + const block = dim.granularitiesBlock || normalizeGranularitiesBlock(undefined); + const blockWithLocal = { ...block, custom: { ...block.custom, ...localCustom } }; + const resolved = resolveDimensionGranularities( + blockWithLocal, + globalConfig.enabledBuiltIns, + globalConfig.customGranularities, + builtInsCatalog, + ); + return Object.entries(resolved).map(([name, def]: [string, any]) => ({ + name, + type: def.type, + title: def.title, + ...(def.interval !== undefined ? { interval: def.interval } : {}), + ...(def.offset !== undefined ? { offset: def.offset } : {}), + ...(def.origin !== undefined ? { origin: def.origin } : {}), + ...(def.format !== undefined ? { format: def.format } : {}), + })); + } + + // Build a resolver that returns the effective granularity meta for one queried time dimension, + // computing per-dimension sets lazily so query paths don't enrich the whole meta. + protected async buildGranularityResolver(context: RequestContext, metaConfig: any[]): Promise { const globalConfig = await this.resolveGlobalGranularitiesForRequest(context); const builtInsCatalog = buildBuiltInsCatalog(globalConfig); - return cubes.map(cube => ({ - ...cube, - dimensions: cube.dimensions?.map((dim: any) => { - if (dim.type !== 'time') return dim; - // Re-key the local-custom array CubeToMetaTransformer produced so the resolver can - // merge it back into `granularitiesBlock.custom` cleanly. - const localCustom: Record = {}; - for (const g of dim.granularities || []) { - localCustom[g.name] = { - title: g.title, - interval: g.interval, - offset: g.offset, - origin: g.origin, - ...(g.format !== undefined ? { format: g.format } : {}), - }; + // dimension name -> { granularityName -> meta }, filled on first use. + const perDimCache = new Map>(); + + const dimIndex = new Map(); + for (const cube of metaConfig) { + for (const dim of cube.config?.dimensions || []) { + // A queried granularity only ever targets a time dimension; also index dimensions that carry + // granularity definitions even if `type` is absent (defensive against partial meta). + if (dim.type === 'time' || dim.granularities || dim.granularitiesBlock) { + dimIndex.set(dim.name, dim); } - const block = dim.granularitiesBlock || normalizeGranularitiesBlock(undefined); - const blockWithLocal = { ...block, custom: { ...block.custom, ...localCustom } }; - const resolved = resolveDimensionGranularities( - blockWithLocal, - globalConfig.enabledBuiltIns, - globalConfig.customGranularities, - builtInsCatalog, - ); - const resolvedArray = Object.entries(resolved).map(([name, def]: [string, any]) => ({ - name, - type: def.type, - title: def.title, - ...(def.interval !== undefined ? { interval: def.interval } : {}), - ...(def.offset !== undefined ? { offset: def.offset } : {}), - ...(def.origin !== undefined ? { origin: def.origin } : {}), - ...(def.format !== undefined ? { format: def.format } : {}), - })); - // Strip the transport-only block; clients only see the resolved `granularities` array. - const { granularitiesBlock, ...rest } = dim; - return { ...rest, granularities: resolvedArray }; - }), - })); + } + } + + return (dimension: string, granularity: string): GranularityMeta | undefined => { + let byName = perDimCache.get(dimension); + if (!byName) { + const dim = dimIndex.get(dimension); + byName = {}; + if (dim) { + for (const g of this.resolveEffectiveGranularitiesForDim(dim, globalConfig, builtInsCatalog)) { + byName[g.name] = g; + } + } + perDimCache.set(dimension, byName); + } + const resolved = byName[granularity]; + if (resolved) return resolved; + // A built-in that global config disabled still executes; annotate it from defaults so the + // response is never missing the queried granularity. + if (isPredefinedGranularity(granularity)) { + const defaults = BUILT_IN_GRANULARITIES[granularity]; + return { + name: granularity, + type: 'built-in', + title: defaults?.title || granularity, + interval: `1 ${granularity}`, + ...(defaults?.format ? { format: defaults.format } : {}), + }; + } + return undefined; + }; + } + + // Attach `effectiveGranularities` (reconciled built-ins + globals + local customs) to each time + // dimension for /v1/meta. The legacy `granularities` array is preserved as-is (deprecated). + // Non-time dimensions and cubes without time dimensions are returned by reference. + protected async applyGlobalGranularitiesToMetaCubes(context: RequestContext, cubes: any[]): Promise { + const globalConfig = await this.resolveGlobalGranularitiesForRequest(context); + const builtInsCatalog = buildBuiltInsCatalog(globalConfig); + + return cubes.map(cube => { + if (!cube.dimensions?.some((d: any) => d.type === 'time')) { + return cube; + } + return { + ...cube, + dimensions: cube.dimensions.map((dim: any) => { + if (dim.type !== 'time') return dim; + const effectiveGranularities = this.resolveEffectiveGranularitiesForDim(dim, globalConfig, builtInsCatalog); + const { granularitiesBlock, ...rest } = dim; + return { ...rest, effectiveGranularities }; + }), + }; + }); } public async contextByReq(req: Request, securityContext, requestId: string): Promise { diff --git a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts index 92a7656740d1e..a83db80bd7a59 100644 --- a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts +++ b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts @@ -6,8 +6,6 @@ */ import R from 'ramda'; -import { isPredefinedGranularity } from '@cubejs-backend/shared'; -import { BUILT_IN_GRANULARITIES } from '@cubejs-backend/schema-compiler'; import { MetaConfig, MetaConfigMap, toConfigMap } from './to-config-map'; import { MemberType } from '../types/strings'; import { MemberType as MemberTypeEnum } from '../types/enums'; @@ -24,6 +22,10 @@ type GranularityMeta = { origin?: string; }; +// Resolves the effective granularity for a queried time dimension against the request's global +// config, so the load path doesn't have to enrich the whole meta. `dimension` is `cube.member`. +export type GranularityResolverFn = (dimension: string, granularity: string) => GranularityMeta | undefined; + /** * Annotation item for cube's member. */ @@ -81,8 +83,9 @@ const annotation = ( /** * Returns annotations object by MetaConfigs and query. + * `resolveGranularity` computes the effective granularity meta for a queried time dimension. */ -function prepareAnnotation(metaConfig: MetaConfig[], query: any) { +function prepareAnnotation(metaConfig: MetaConfig[], query: any, resolveGranularity?: GranularityResolverFn) { const configMap = toConfigMap(metaConfig); const dimensions = (query.dimensions || []); return { @@ -117,29 +120,7 @@ function prepareAnnotation(metaConfig: MetaConfig[], query: any) { let dimAnnotation: [string, AnnotatedConfigItem] | undefined; if (an) { - let granularityMeta: GranularityMeta | undefined; - if (isPredefinedGranularity(td.granularity)) { - // Prefer values the meta endpoint already attached (these honor any global - // title/format override). Fall back to BUILT_IN_GRANULARITIES, then to the bare name. - const fromMeta = an[1].granularities?.find(g => g.name === td.granularity); - const builtInDefaults = BUILT_IN_GRANULARITIES[td.granularity] || {}; - granularityMeta = { - name: td.granularity, - type: 'built-in', - title: fromMeta?.title || builtInDefaults.title || td.granularity, - interval: fromMeta?.interval || `1 ${td.granularity}`, - ...(fromMeta?.format || builtInDefaults.format - ? { format: fromMeta?.format || builtInDefaults.format } - : {}), - }; - } else if (an[1].granularities) { - // Forward only the granularity in play for this query; siblings stay in /v1/meta. - granularityMeta = an[1].granularities.find(g => g.name === td.granularity); - if (granularityMeta && !granularityMeta.type) { - granularityMeta = { ...granularityMeta, type: 'custom' }; - } - } - + const granularityMeta = resolveGranularity?.(td.dimension, td.granularity); const { granularities: _, ...rest } = an[1]; dimAnnotation = [an[0], { ...rest, granularity: granularityMeta }]; } diff --git a/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts b/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts index c87ee9291b411..ed80425c0c938 100644 --- a/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts +++ b/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts @@ -13,8 +13,17 @@ import prepareAnnotationDef import { annotation, prepareAnnotation, + GranularityResolverFn, } from '../../src/helpers/prepare-annotation'; +// Mimics the gateway's built-in fallback: the resolver the gateway injects resolves `day` from +// BUILT_IN_GRANULARITIES defaults. +const dayResolver: GranularityResolverFn = (_dimension, granularity) => ( + granularity === 'day' + ? { name: 'day', type: 'built-in', title: 'Day', interval: '1 day', format: '%Y-%m-%d' } + : undefined +); + describe('prepareAnnotation helpers', () => { test('export looks as expected', () => { expect(prepareAnnotationDef).toBeDefined(); @@ -179,7 +188,7 @@ describe('prepareAnnotation helpers', () => { dimension: 'cube_name.member', granularity: 'day', }], - }).timeDimensions + }, dayResolver).timeDimensions ).toEqual({ 'cube_name.member': { currency: undefined, diff --git a/packages/cubejs-api-gateway/test/index.test.ts b/packages/cubejs-api-gateway/test/index.test.ts index 87d0825e0bdf7..22701681b6848 100644 --- a/packages/cubejs-api-gateway/test/index.test.ts +++ b/packages/cubejs-api-gateway/test/index.test.ts @@ -334,6 +334,7 @@ describe('API Gateway', () => { .toStrictEqual({ granularity: { name: 'half_year_by_1st_april', + type: 'custom', title: 'Half Year By1 St April', interval: '6 months', offset: '3 months', @@ -356,6 +357,7 @@ describe('API Gateway', () => { .toStrictEqual({ granularity: { name: 'half_year_by_1st_april', + type: 'custom', title: 'Half Year By1 St April', interval: '6 months', offset: '3 months', diff --git a/packages/cubejs-client-core/src/types.ts b/packages/cubejs-client-core/src/types.ts index 4414ade60bc33..055b6bfadd5b4 100644 --- a/packages/cubejs-client-core/src/types.ts +++ b/packages/cubejs-client-core/src/types.ts @@ -437,11 +437,27 @@ export type TCubeMeasure = BaseCubeMember & { currency?: string; }; +/** + * @deprecated Use `EffectiveGranularity` / `CubeTimeDimension.effectiveGranularities`. + * Legacy meta shape: model-defined custom granularities only. + */ export type CubeTimeDimensionGranularity = { name: string; title: string; }; +/** Reconciled granularity for a time dimension: enabled built-ins + global and local customs. */ +export type EffectiveGranularity = { + name: string; + type: 'built-in' | 'custom'; + title: string; + /** d3-time-format string for displaying bucketed timestamps. */ + format?: string; + interval?: string; + offset?: string; + origin?: string; +}; + export type BaseCubeDimension = BaseCubeMember & { primaryKey?: boolean; suggestFilterValues: boolean; @@ -452,7 +468,12 @@ export type BaseCubeDimension = BaseCubeMember & { }; export type CubeTimeDimension = BaseCubeDimension & - { type: 'time'; granularities?: CubeTimeDimensionGranularity[] }; + { + type: 'time'; + /** @deprecated Use `effectiveGranularities`. */ + granularities?: CubeTimeDimensionGranularity[]; + effectiveGranularities?: EffectiveGranularity[]; + }; export type TCubeDimension = (BaseCubeDimension & { type: Exclude }) | diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 79b4dccc136de..506603d37a088 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -619,6 +619,9 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface for (const dim of Object.values(dimensions)) { if (dim && dim.type === 'time' && 'granularities' in dim) { + // Keep the raw user value for the validator (it runs after this and would otherwise only + // see the extracted customs, never the includes/excludes/custom dict). + dim.rawGranularities = dim.granularities; const block: NormalizedGranularitiesBlock = normalizeGranularitiesBlock(dim.granularities); dim.granularitiesBlock = block; dim.granularities = block.custom; @@ -1161,6 +1164,7 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface format: memberRef.override?.format || resolvedMember.format, ...(propagatedCurrency ? { currency: propagatedCurrency } : {}), ...(resolvedMember.granularities ? { granularities: resolvedMember.granularities } : {}), + ...(resolvedMember.granularitiesBlock ? { granularitiesBlock: resolvedMember.granularitiesBlock } : {}), ...(resolvedMember.multiStage && { multiStage: resolvedMember.multiStage }), ...(resolvedMember.keyReference && this.processKeyReferenceForView(resolvedMember.keyReference, targetCube.name, viewAllMembers, memberRef.member)), ...(resolvedMember.mask !== undefined ? { mask: resolvedMember.mask } : {}), diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index 38b30ff7ee269..c7880aa482f97 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -132,6 +132,10 @@ export type DimensionConfig = { public: boolean; primaryKey: boolean; aliasMember?: string; + /** + * @deprecated Use `effectiveGranularities`. Lists only the model's custom granularities; + * omits built-ins, global customs, and the `type` field. See DEPRECATION.md. + */ granularities?: GranularityDefinition[]; order?: 'asc' | 'desc'; key?: string; @@ -302,9 +306,9 @@ export class CubeToMetaTransformer implements CompilerInterface { ? this.isVisible(extendedDimDef, !extendedDimDef.primaryKey) : false; const granularitiesObj = extendedDimDef.granularities; - // The gateway reconciles `granularitiesBlock` (includes/excludes/custom) with global - // config per request. Forwarded as-is; the flat `granularities` field alone can't - // represent `includes: '*'` vs `includes: []`. + // `granularities` keeps its legacy custom-only shape (deprecated); the gateway attaches + // the reconciled set as `effectiveGranularities` per request from `granularitiesBlock` + // and strips the block before responding. const { granularitiesBlock } = extendedDimDef as any; const dimType = this.dimensionDataType(extendedDimDef.type || 'string'); const dimFormat = this.transformDimensionFormat(extendedDimDef); @@ -332,9 +336,7 @@ export class CubeToMetaTransformer implements CompilerInterface { granularitiesObj ? Object.entries(granularitiesObj).map(([gName, gDef]: [string, any]) => ({ name: gName, - type: 'custom', title: this.title(cubeTitle, [gName, gDef], true), - ...(gDef.format !== undefined ? { format: gDef.format } : {}), interval: gDef.interval, offset: gDef.offset, origin: gDef.origin, diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index ce3d8aa34adaa..2a56a79ca9ebd 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -181,6 +181,31 @@ const GranularityInclusionListSchema = Joi.alternatives([ Joi.array().items(Joi.string()), ]); +// Validates a time dimension's granularities in either shape: the new dict form +// (includes/excludes/custom) or the legacy custom map. Applied to both `granularities` (the raw +// user value in unit tests / pre-normalization) and `rawGranularities` (the stashed raw value the +// real pipeline validates, since normalization replaces `granularities` with the custom map first). +const GranularitiesFieldSchema = Joi.alternatives() + .conditional(Joi.ref('.'), { + // Only the new dict form has exclusively includes/excludes/custom keys. + is: Joi.object().keys({ + includes: Joi.any(), + excludes: Joi.any(), + custom: Joi.any(), + }).unknown(false), + then: Joi.object().keys({ + includes: GranularityInclusionListSchema, + excludes: GranularityInclusionListSchema, + custom: Joi.object().pattern(identifierRegex, CustomGranularityEntrySchema), + }).custom((value, helper) => { + if (value && value.includes !== undefined && value.excludes !== undefined && value.includes !== '*') { + return helper.message({ custom: '"includes" and "excludes" cannot be used together unless includes is "*"' } as any); + } + return value; + }), + otherwise: Joi.object().pattern(identifierRegex, CustomGranularityEntrySchema), + }); + const formatAlternatives = [ Joi.string().valid('imageUrl', 'link', 'currency', 'percent', 'number', 'id'), Joi.object().keys({ @@ -446,33 +471,20 @@ const BaseDimensionWithoutSubQuery = { then: Joi.array().items(Joi.string()), otherwise: Joi.forbidden() }), + // Validated in both shapes. In the real pipeline CubeSymbols.normalizeDimensionGranularities runs + // first and replaces `granularities` with the extracted custom map (still valid here) while + // stashing the user's raw value in `rawGranularities`; when validated directly (unit tests) only + // `granularities` is set and carries the raw shape. granularities: Joi.when('type', { is: 'time', - then: Joi.alternatives() - .conditional(Joi.ref('.'), { - // Discriminate by shape: only the new dict form has includes/excludes/custom and no other keys. - is: Joi.object().keys({ - includes: Joi.any(), - excludes: Joi.any(), - custom: Joi.any(), - }).unknown(false), - then: Joi.object().keys({ - includes: GranularityInclusionListSchema, - excludes: GranularityInclusionListSchema, - custom: Joi.object().pattern(identifierRegex, CustomGranularityEntrySchema), - }).custom((value, helper) => { - if (value && value.includes !== undefined && value.excludes !== undefined && value.includes !== '*') { - return helper.message({ custom: '"includes" and "excludes" cannot be used together unless includes is "*"' } as any); - } - return value; - }), - otherwise: Joi.object().pattern(identifierRegex, CustomGranularityEntrySchema), - }) - .optional(), + then: GranularitiesFieldSchema.optional(), + otherwise: Joi.forbidden() + }), + rawGranularities: Joi.when('type', { + is: 'time', + then: GranularitiesFieldSchema.optional(), otherwise: Joi.forbidden() }), - // Internal field written by CubeSymbols.normalizeDimensionGranularities before the validator runs. - // Not user-facing; declared so unknown-key validation doesn't reject it. granularitiesBlock: Joi.any() }; diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts index d30aa08ea3bd0..b0207f43ee87f 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -74,7 +74,11 @@ function resolveFromEnv(): GlobalGranularitiesConfig { enabledBuiltIns.push(trimmed); } else { // Non-built-in name: pull the definition from `CUBEJS_GRANULARITIES__*` env vars. - customGranularities[trimmed] = applyEnvOverrides(trimmed); + // Skip if no interval was provided — a custom granularity without an interval is unusable. + const def = applyEnvOverrides(trimmed); + if (def.interval !== undefined) { + customGranularities[trimmed] = def; + } } } } diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts index 4f73924a6460d..520909aa4a787 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -41,8 +41,13 @@ export function normalizeGranularitiesBlock(raw: any): NormalizedGranularitiesBl } if (typeof raw === 'object') { - if ('includes' in raw || 'excludes' in raw || 'custom' in raw) { - // New dict form. YamlCompiler has already keyed `custom` by name. + // New dict form iff every key is one of includes/excludes/custom. Requiring ALL keys to match + // (not just any) avoids misreading a legacy custom granularity named `includes`/`excludes`/`custom` + // as the dict form. Mirrors the validator's `.unknown(false)` discrimination. + const keys = Object.keys(raw); + const isDictForm = keys.length > 0 && keys.every(k => k === 'includes' || k === 'excludes' || k === 'custom'); + if (isDictForm) { + // YamlCompiler has already keyed `custom` by name. return { includes: raw.includes ?? '*', excludes: raw.excludes ?? [], From 2cad934ed04420ebb95de8dc446d81db3fee1a96 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 16 Jul 2026 19:06:54 +0200 Subject: [PATCH 04/22] fix(granularities): keep overridden built-in type, scope camelize guard to granularities.custom --- .../src/compiler/GranularityResolver.ts | 21 +++++++++++--- .../src/compiler/utils.ts | 28 +++++++++++++------ .../test/unit/granularities-shape.test.ts | 23 +++++++++++++++ 3 files changed, 59 insertions(+), 13 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts index 520909aa4a787..a9faa8b73c35a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -41,11 +41,19 @@ export function normalizeGranularitiesBlock(raw: any): NormalizedGranularitiesBl } if (typeof raw === 'object') { - // New dict form iff every key is one of includes/excludes/custom. Requiring ALL keys to match - // (not just any) avoids misreading a legacy custom granularity named `includes`/`excludes`/`custom` - // as the dict form. Mirrors the validator's `.unknown(false)` discrimination. + // New dict form iff every key is one of includes/excludes/custom AND the values have the dict + // shape (includes/excludes are '*' or arrays, custom is a plain object). The value check keeps a + // legacy custom granularity named `includes`/`excludes`/`custom` — whose value is a granularity + // definition object — from being misread as the dict form. const keys = Object.keys(raw); - const isDictForm = keys.length > 0 && keys.every(k => k === 'includes' || k === 'excludes' || k === 'custom'); + const isInclusionList = (v: any) => v === undefined || v === '*' || Array.isArray(v); + const isCustomMap = (v: any) => v === undefined || (typeof v === 'object' && v !== null && !Array.isArray(v)); + const isDictForm = + keys.length > 0 && + keys.every(k => k === 'includes' || k === 'excludes' || k === 'custom') && + isInclusionList(raw.includes) && + isInclusionList(raw.excludes) && + isCustomMap(raw.custom); if (isDictForm) { // YamlCompiler has already keyed `custom` by name. return { @@ -94,6 +102,11 @@ export function resolveDimensionGranularities( } } for (const [name, def] of Object.entries(globalCustom)) { + // A name shadowing a built-in is an override, already emitted as `type: 'built-in'` above with + // its title/format folded in via `allBuiltInsCatalog`. Skip it here so it isn't relabeled custom. + if (allBuiltInsCatalog[name]) { + continue; + } const passesIncludes = includesAllowsAll || includesSet!.has(name); const blockedByExcludes = excludesSet!.has(name); if (passesIncludes && !blockedByExcludes) { diff --git a/packages/cubejs-schema-compiler/src/compiler/utils.ts b/packages/cubejs-schema-compiler/src/compiler/utils.ts index 9af688a1e92ed..d71892befd12f 100644 --- a/packages/cubejs-schema-compiler/src/compiler/utils.ts +++ b/packages/cubejs-schema-compiler/src/compiler/utils.ts @@ -1,30 +1,40 @@ import { camelize } from 'inflection'; -// It's a map where key - is a level and value - is a map of properties on this level to ignore camelization -const IGNORE_CAMELIZE = { +// Map of level -> keys at that level whose children must not be camelized (they hold user-defined +// identifiers). Each entry can require a specific parent key so the guard is path-scoped rather than +// matching any same-named property elsewhere in the tree. +const IGNORE_CAMELIZE: Record> = { 1: { - granularities: true, + granularities: {}, }, - // Custom granularity names live one level deeper than the legacy flat-array form - // (`granularities.custom.`), so they need their own guard. + // Custom granularity names in the new dict form live at `granularities.custom.`; scope the + // guard to that path so an unrelated `custom` property elsewhere isn't affected. 2: { - custom: true, + custom: { parent: 'granularities' }, } }; -function camelizeObjectPart(obj: unknown, camelizeKeys: boolean, level = 0): unknown { +function shouldIgnoreCamelize(level: number, key: string, parentKey: string | undefined): boolean { + const entry = IGNORE_CAMELIZE[level]?.[key]; + if (!entry) { + return false; + } + return entry.parent === undefined || entry.parent === parentKey; +} + +function camelizeObjectPart(obj: unknown, camelizeKeys: boolean, level = 0, parentKey?: string): unknown { if (!obj) { return obj; } if (Array.isArray(obj)) { for (let i = 0; i < obj.length; i++) { - obj[i] = camelizeObjectPart(obj[i], true, level + 1); + obj[i] = camelizeObjectPart(obj[i], true, level + 1, parentKey); } } else if (typeof obj === 'object') { for (const key of Object.keys(obj)) { if (!(level === 1 && key === 'meta')) { - obj[key] = camelizeObjectPart(obj[key], !IGNORE_CAMELIZE[level]?.[key], level + 1); + obj[key] = camelizeObjectPart(obj[key], !shouldIgnoreCamelize(level, key, parentKey), level + 1, key); } if (camelizeKeys) { diff --git a/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts b/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts index 39f4924f203d3..863627b0ada65 100644 --- a/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts @@ -151,4 +151,27 @@ describe('resolveDimensionGranularities — spec resolution table', () => { ); expect(out.fiscal_year.origin).toBe('2026-01-01'); }); + + it('built-in overridden via global custom stays type built-in (not relabeled custom)', () => { + // `config.granularities: [{ name: 'year', title: 'Anno' }]` both enables 'year' and puts it in + // globalCustom; the resolved entry must remain a built-in. + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), + ['year'], + { year: { title: 'Anno' } }, + { ...BUILT_INS, year: { title: 'Anno' } }, + ); + expect(out.year.type).toBe('built-in'); + expect(out.year.title).toBe('Anno'); + }); +}); + +describe('normalizeGranularitiesBlock — reserved-name disambiguation', () => { + it('custom granularity named "includes" is not misread as the dict form', () => { + const out = normalizeGranularitiesBlock({ + includes: { interval: '1 year', origin: '2026-04-01' }, + }); + expect(out.includes).toBe('*'); + expect(out.custom.includes).toEqual({ interval: '1 year', origin: '2026-04-01' }); + }); }); From 9a192e9ae9a92bdf7f56afce53681db2a71769e8 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 16 Jul 2026 19:12:33 +0200 Subject: [PATCH 05/22] fix(granularities): avoid continue to satisfy no-continue lint rule --- .../src/compiler/GranularityResolver.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts index a9faa8b73c35a..f177f3891f63e 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -103,13 +103,11 @@ export function resolveDimensionGranularities( } for (const [name, def] of Object.entries(globalCustom)) { // A name shadowing a built-in is an override, already emitted as `type: 'built-in'` above with - // its title/format folded in via `allBuiltInsCatalog`. Skip it here so it isn't relabeled custom. - if (allBuiltInsCatalog[name]) { - continue; - } + // its title/format folded in via `allBuiltInsCatalog`; skip it here so it isn't relabeled custom. + const shadowsBuiltIn = !!allBuiltInsCatalog[name]; const passesIncludes = includesAllowsAll || includesSet!.has(name); const blockedByExcludes = excludesSet!.has(name); - if (passesIncludes && !blockedByExcludes) { + if (!shadowsBuiltIn && passesIncludes && !blockedByExcludes) { out[name] = { ...def, type: 'custom' }; } } From 16614e0f9e2ea1d1577ff8af6f4806b64e4a5323 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 16 Jul 2026 19:35:30 +0200 Subject: [PATCH 06/22] fix(granularities): emit granularitiesBlock only when present, update snapshots and stale assertion --- .../src/compiler/CubeToMetaTransformer.ts | 2 +- .../unit/__snapshots__/schema.test.ts.snap | 95 +++++++++++++++++++ .../test/unit/yaml-schema.test.ts | 5 +- 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index c7880aa482f97..cd37537c4b3f4 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -342,7 +342,7 @@ export class CubeToMetaTransformer implements CompilerInterface { origin: gDef.origin, })) : undefined, - granularitiesBlock, + ...(granularitiesBlock ? { granularitiesBlock } : {}), order: extendedDimDef.order, key: extendedDimDef.keyReference, ...(extendedDimDef.links ? { links: extendedDimDef.links.map((link: any) => ({ diff --git a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap index 1e48caa41d7f3..3c3cff455d445 100644 --- a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap +++ b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap @@ -78,7 +78,39 @@ Object { "sql": [Function], }, }, + "granularitiesBlock": Object { + "custom": Object { + "month": Object { + "sql": [Function], + }, + "quarter": Object { + "sql": [Function], + }, + "week": Object { + "sql": [Function], + }, + "year": Object { + "sql": [Function], + }, + }, + "excludes": Array [], + "includes": "*", + }, "ownedByCube": true, + "rawGranularities": Object { + "month": Object { + "sql": [Function], + }, + "quarter": Object { + "sql": [Function], + }, + "week": Object { + "sql": [Function], + }, + "year": Object { + "sql": [Function], + }, + }, "sql": [Function], "timeShift": Array [ Object { @@ -263,7 +295,47 @@ Object { "sql": [Function], }, }, + "granularitiesBlock": Object { + "custom": Object { + "fortnight": Object { + "interval": "2 week", + "origin": "2025-01-01", + }, + "month": Object { + "sql": [Function], + }, + "quarter": Object { + "sql": [Function], + }, + "week": Object { + "sql": [Function], + }, + "year": Object { + "sql": [Function], + }, + }, + "excludes": Array [], + "includes": "*", + }, "ownedByCube": true, + "rawGranularities": Object { + "fortnight": Object { + "interval": "2 week", + "origin": "2025-01-01", + }, + "month": Object { + "sql": [Function], + }, + "quarter": Object { + "sql": [Function], + }, + "week": Object { + "sql": [Function], + }, + "year": Object { + "sql": [Function], + }, + }, "sql": [Function], "timeShift": Array [ Object { @@ -1922,6 +1994,29 @@ Object { "origin": "2020-03-01", }, }, + "granularitiesBlock": Object { + "custom": Object { + "half_year": Object { + "interval": "6 months", + "title": "6 month intervals", + }, + "half_year_by_1st_april": Object { + "interval": "6 months", + "offset": "3 months", + "title": "Half year from Apr to Oct", + }, + "half_year_by_1st_june": Object { + "interval": "6 months", + "origin": "2020-06-01 10:00:00", + }, + "half_year_by_1st_march": Object { + "interval": "6 months", + "origin": "2020-03-01", + }, + }, + "excludes": Array [], + "includes": "*", + }, "meta": undefined, "ownedByCube": false, "sql": [Function], diff --git a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts index 5eb64ab75f8ab..f1e0d81254831 100644 --- a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts @@ -796,7 +796,10 @@ cubes: await compiler.compile(); throw new Error('compile must return an error'); } catch (e: any) { - expect(e.message).toContain('must be defined as array'); + // `granularities: { name: half_year }` is neither the legacy array nor the dict + // (includes/excludes/custom) form; it's read as a custom map whose `name` entry is an + // invalid (non-object) granularity definition. + expect(e.message).toContain('dimensions.created_at.granularities.name'); } }); From 0ee4a7bfd9cb3f30ac3cdd7e2b75407d4d779e15 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 16 Jul 2026 21:24:53 +0200 Subject: [PATCH 07/22] refactor(granularities): resolve effective granularities in the compiler with a bounded per-config variant cache --- packages/cubejs-api-gateway/src/gateway.ts | 146 +------- .../src/helpers/prepare-annotation.ts | 47 ++- .../cubejs-api-gateway/src/types/gateway.ts | 20 -- .../test/helpers/prepare-annotation.test.ts | 77 +++- .../cubejs-api-gateway/test/index.test.ts | 54 +++ packages/cubejs-api-gateway/test/mocks.ts | 23 +- .../src/compiler/CubeToMetaTransformer.ts | 118 ++++++- .../src/compiler/GlobalGranularitiesConfig.ts | 28 +- .../src/compiler/GranularityConfigHash.ts | 29 ++ .../src/compiler/GranularityResolver.ts | 26 ++ .../src/compiler/PrepareCompiler.ts | 13 +- .../src/compiler/index.ts | 5 + .../test/unit/granularity-config-hash.test.ts | 55 +++ .../src/core/CompilerApi.ts | 161 ++++++++- .../cubejs-server-core/src/core/server.ts | 2 +- .../test/unit/granularity-variants.test.ts | 331 ++++++++++++++++++ 16 files changed, 943 insertions(+), 192 deletions(-) create mode 100644 packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts create mode 100644 packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts create mode 100644 packages/cubejs-server-core/test/unit/granularity-variants.test.ts diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index c260ba63217a1..74feec85984d6 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -34,13 +34,9 @@ import { createProxyMiddleware } from 'http-proxy-middleware'; import { QueryBody } from '@cubejs-backend/query-orchestrator'; import { - resolveGlobalGranularities, - resolveDimensionGranularities, - normalizeGranularitiesBlock, buildBuiltInsCatalog, BUILT_IN_GRANULARITIES, } from '@cubejs-backend/schema-compiler'; -import type { GlobalGranularitiesConfig } from '@cubejs-backend/schema-compiler'; import { QueryType, ApiScopes, @@ -109,7 +105,7 @@ import { cachedHandler } from './cached-handler'; import { createJWKsFetcher } from './jwk'; import { SQLServer, SQLServerConstructorOptions } from './sql-server'; import { getJsonQueryFromGraphQLQuery, makeSchema } from './graphql'; -import { ConfigItem, prepareAnnotation, GranularityMeta, GranularityResolverFn } from './helpers/prepare-annotation'; +import { ConfigItem, prepareAnnotation } from './helpers/prepare-annotation'; import { transformCube, transformMeasure, @@ -175,8 +171,6 @@ class ApiGateway { protected readonly extendContext?: ExtendContextFn; - protected readonly granularitiesOption?: ApiGatewayOptions['granularities']; - protected readonly dataSourceStorage: any; public readonly checkAuthFn: PreparedCheckAuthFn; @@ -235,7 +229,6 @@ class ApiGateway { this.subscriptionStore = options.subscriptionStore || new LocalSubscriptionStore(); this.enforceSecurityChecks = options.enforceSecurityChecks || (process.env.NODE_ENV === 'production'); this.extendContext = options.extendContext; - this.granularitiesOption = options.granularities; this.checkAuthFn = this.createCheckAuthFn(options); this.checkAuthSystemFn = this.createCheckAuthSystemFn(); @@ -749,9 +742,9 @@ class ApiGateway { const cubesConfig = onlyViews ? metaConfig.cubes.filter((c: any) => c.config?.type === 'view') : metaConfig.cubes; - const filteredCubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config); - // Apply after the visibility filter so we only enrich what the client will actually receive. - const cubes = await this.applyGlobalGranularitiesToMetaCubes(context, filteredCubes); + // Time dimensions arrive from CompilerApi with `effectiveGranularities` already attached + // (baked at compile for env/static configs, variant-cached for the function form). + const cubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config); const visibleCubeNames = new Set(cubes.map(c => c.name)); const viewGroups = (metaConfig.viewGroups || []) .map(group => this.filterVisibleViewGroup(group, visibleCubeNames)) @@ -782,7 +775,8 @@ class ApiGateway { const requestStarted = new Date(); try { await this.assertApiScope('meta', context.securityContext); - const globalConfig = await this.resolveGlobalGranularitiesForRequest(context); + const compilerApi = await this.getCompilerApi(context); + const globalConfig = await compilerApi.resolveGlobalGranularitiesConfig(context); const builtInsCatalog = buildBuiltInsCatalog(globalConfig); const granularities: any[] = []; @@ -1242,7 +1236,10 @@ class ApiGateway { } else { const metaCacheKey = JSON.stringify(ctx); if (!metaCache.has(metaCacheKey)) { - metaCache.set(metaCacheKey, await compiler.metaConfigExtended(context, ctx)); + // `ctx` (outer context merged with the job's own context) is the request context; + // passing the outer `context` here would select visibility and granularities for + // the wrong tenant. + metaCache.set(metaCacheKey, await compiler.metaConfigExtended(ctx, { requestId: ctx.requestId })); } // checking and fetching result status @@ -2136,8 +2133,6 @@ class ApiGateway { }); metaConfigResult = this.filterVisibleItemsInMeta(context, metaConfigResult); - // Resolve the queried granularity's meta on demand instead of rewriting the whole model. - const resolveGranularity = await this.buildGranularityResolver(context, metaConfigResult); const sqlQueries = await this.getSqlQueriesInternal(context, normalizedQueries); @@ -2155,7 +2150,7 @@ class ApiGateway { ); const annotation = prepareAnnotation( - metaConfigResult, normalizedQuery, resolveGranularity + metaConfigResult, normalizedQuery ); return this.prepareResultTransformData( @@ -2241,7 +2236,6 @@ class ApiGateway { }); metaConfigResult = this.filterVisibleItemsInMeta(context, metaConfigResult); - const resolveGranularity = await this.buildGranularityResolver(context, metaConfigResult); const sqlQueries = await this .getSqlQueriesInternal( @@ -2291,7 +2285,7 @@ class ApiGateway { const response = await adapterApi.executeQuery(finalQuery); const annotation = prepareAnnotation( - metaConfigResult, normalizedQueries[0], resolveGranularity + metaConfigResult, normalizedQueries[0] ); // TODO Can we just pass through data? Ensure hidden members can't be queried @@ -2328,7 +2322,7 @@ class ApiGateway { Boolean(sqlQueries[index].slowQuery); const annotation = prepareAnnotation( - metaConfigResult, normalizedQuery, resolveGranularity + metaConfigResult, normalizedQuery ); if (request.streaming) { @@ -2457,120 +2451,6 @@ class ApiGateway { return this.adapterApi(context); } - protected async resolveGlobalGranularitiesForRequest(context: RequestContext) { - return resolveGlobalGranularities(this.granularitiesOption, context); - } - - // Resolve one time dimension's effective granularity set against the request's global config. - // Returns the array serialized for `effectiveGranularities`; leaves the legacy `granularities` - // untouched. - private resolveEffectiveGranularitiesForDim( - dim: any, - globalConfig: GlobalGranularitiesConfig, - builtInsCatalog: Record, - ): any[] { - const localCustom: Record = {}; - for (const g of dim.granularities || []) { - localCustom[g.name] = { - title: g.title, - interval: g.interval, - offset: g.offset, - origin: g.origin, - ...(g.format !== undefined ? { format: g.format } : {}), - }; - } - const block = dim.granularitiesBlock || normalizeGranularitiesBlock(undefined); - const blockWithLocal = { ...block, custom: { ...block.custom, ...localCustom } }; - const resolved = resolveDimensionGranularities( - blockWithLocal, - globalConfig.enabledBuiltIns, - globalConfig.customGranularities, - builtInsCatalog, - ); - return Object.entries(resolved).map(([name, def]: [string, any]) => ({ - name, - type: def.type, - title: def.title, - ...(def.interval !== undefined ? { interval: def.interval } : {}), - ...(def.offset !== undefined ? { offset: def.offset } : {}), - ...(def.origin !== undefined ? { origin: def.origin } : {}), - ...(def.format !== undefined ? { format: def.format } : {}), - })); - } - - // Build a resolver that returns the effective granularity meta for one queried time dimension, - // computing per-dimension sets lazily so query paths don't enrich the whole meta. - protected async buildGranularityResolver(context: RequestContext, metaConfig: any[]): Promise { - const globalConfig = await this.resolveGlobalGranularitiesForRequest(context); - const builtInsCatalog = buildBuiltInsCatalog(globalConfig); - - // dimension name -> { granularityName -> meta }, filled on first use. - const perDimCache = new Map>(); - - const dimIndex = new Map(); - for (const cube of metaConfig) { - for (const dim of cube.config?.dimensions || []) { - // A queried granularity only ever targets a time dimension; also index dimensions that carry - // granularity definitions even if `type` is absent (defensive against partial meta). - if (dim.type === 'time' || dim.granularities || dim.granularitiesBlock) { - dimIndex.set(dim.name, dim); - } - } - } - - return (dimension: string, granularity: string): GranularityMeta | undefined => { - let byName = perDimCache.get(dimension); - if (!byName) { - const dim = dimIndex.get(dimension); - byName = {}; - if (dim) { - for (const g of this.resolveEffectiveGranularitiesForDim(dim, globalConfig, builtInsCatalog)) { - byName[g.name] = g; - } - } - perDimCache.set(dimension, byName); - } - const resolved = byName[granularity]; - if (resolved) return resolved; - // A built-in that global config disabled still executes; annotate it from defaults so the - // response is never missing the queried granularity. - if (isPredefinedGranularity(granularity)) { - const defaults = BUILT_IN_GRANULARITIES[granularity]; - return { - name: granularity, - type: 'built-in', - title: defaults?.title || granularity, - interval: `1 ${granularity}`, - ...(defaults?.format ? { format: defaults.format } : {}), - }; - } - return undefined; - }; - } - - // Attach `effectiveGranularities` (reconciled built-ins + globals + local customs) to each time - // dimension for /v1/meta. The legacy `granularities` array is preserved as-is (deprecated). - // Non-time dimensions and cubes without time dimensions are returned by reference. - protected async applyGlobalGranularitiesToMetaCubes(context: RequestContext, cubes: any[]): Promise { - const globalConfig = await this.resolveGlobalGranularitiesForRequest(context); - const builtInsCatalog = buildBuiltInsCatalog(globalConfig); - - return cubes.map(cube => { - if (!cube.dimensions?.some((d: any) => d.type === 'time')) { - return cube; - } - return { - ...cube, - dimensions: cube.dimensions.map((dim: any) => { - if (dim.type !== 'time') return dim; - const effectiveGranularities = this.resolveEffectiveGranularitiesForDim(dim, globalConfig, builtInsCatalog); - const { granularitiesBlock, ...rest } = dim; - return { ...rest, effectiveGranularities }; - }), - }; - }); - } - public async contextByReq(req: Request, securityContext, requestId: string): Promise { req.securityContext = securityContext; diff --git a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts index a83db80bd7a59..8f391952b7186 100644 --- a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts +++ b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts @@ -6,6 +6,8 @@ */ import R from 'ramda'; +import { isPredefinedGranularity } from '@cubejs-backend/shared'; +import { BUILT_IN_GRANULARITIES } from '@cubejs-backend/schema-compiler'; import { MetaConfig, MetaConfigMap, toConfigMap } from './to-config-map'; import { MemberType } from '../types/strings'; import { MemberType as MemberTypeEnum } from '../types/enums'; @@ -22,10 +24,6 @@ type GranularityMeta = { origin?: string; }; -// Resolves the effective granularity for a queried time dimension against the request's global -// config, so the load path doesn't have to enrich the whole meta. `dimension` is `cube.member`. -export type GranularityResolverFn = (dimension: string, granularity: string) => GranularityMeta | undefined; - /** * Annotation item for cube's member. */ @@ -43,6 +41,42 @@ type ConfigItem = { granularities?: GranularityMeta[]; }; +/** + * Effective granularity meta for one queried time dimension, read from the (already granularity- + * enriched) meta config. Two fallbacks keep the annotation complete for granularities that + * execute but aren't in the dimension's effective set: + * - a built-in disabled by config is synthesized from `BUILT_IN_GRANULARITIES` defaults; + * - an unknown custom name yields `undefined` (never the deprecated legacy array). + */ +function resolveGranularityMeta( + configMap: MetaConfigMap, + dimension: string, + granularity: string, +): GranularityMeta | undefined { + const cubeName = dimension.split('.')[0]; + const dimConfig: any = configMap[cubeName]?.[MemberTypeEnum.DIMENSIONS] + ?.find((m: any) => m.name === dimension); + + const resolved = dimConfig?.effectiveGranularities + ?.find((g: GranularityMeta) => g.name === granularity); + if (resolved) { + return resolved; + } + + if (isPredefinedGranularity(granularity)) { + const defaults = BUILT_IN_GRANULARITIES[granularity]; + return { + name: granularity, + type: 'built-in', + title: defaults?.title || granularity, + interval: `1 ${granularity}`, + ...(defaults?.format ? { format: defaults.format } : {}), + }; + } + + return undefined; +} + type AnnotatedConfigItem = Omit & { granularity?: GranularityMeta; }; @@ -83,9 +117,8 @@ const annotation = ( /** * Returns annotations object by MetaConfigs and query. - * `resolveGranularity` computes the effective granularity meta for a queried time dimension. */ -function prepareAnnotation(metaConfig: MetaConfig[], query: any, resolveGranularity?: GranularityResolverFn) { +function prepareAnnotation(metaConfig: MetaConfig[], query: any) { const configMap = toConfigMap(metaConfig); const dimensions = (query.dimensions || []); return { @@ -120,7 +153,7 @@ function prepareAnnotation(metaConfig: MetaConfig[], query: any, resolveGranular let dimAnnotation: [string, AnnotatedConfigItem] | undefined; if (an) { - const granularityMeta = resolveGranularity?.(td.dimension, td.granularity); + const granularityMeta = resolveGranularityMeta(configMap, td.dimension, td.granularity); const { granularities: _, ...rest } = an[1]; dimAnnotation = [an[0], { ...rest, granularity: granularityMeta }]; } diff --git a/packages/cubejs-api-gateway/src/types/gateway.ts b/packages/cubejs-api-gateway/src/types/gateway.ts index ab5639cadab26..d1578742ebad5 100644 --- a/packages/cubejs-api-gateway/src/types/gateway.ts +++ b/packages/cubejs-api-gateway/src/types/gateway.ts @@ -52,19 +52,6 @@ type ScheduledRefreshContextsFn = type ScheduledRefreshTimeZonesFn = (context: RequestContext) => string[] | Promise; -type GranularityListItem = string | { - name: string; - title?: string; - format?: string; - interval?: string; - origin?: string; - offset?: string; -}; -type GranularityList = GranularityListItem[]; -type GranularitiesOption = - | GranularityList - | ((context: RequestContext) => GranularityList | Promise); - /** * Gateway configuration options interface. */ @@ -77,13 +64,6 @@ interface ApiGatewayOptions { scheduledRefreshTimeZones?: ScheduledRefreshTimeZonesFn; basePath: string; extendContext?: ExtendContextFn; - /** - * Enabled granularities (built-in names and/or custom definitions), or a function called per - * request to produce the same. Drives /v1/granularities and the /v1/meta enrichment. - * Shape mirrors `GranularityList` in @cubejs-backend/schema-compiler; redeclared locally to - * avoid a dependency on schema-compiler from this types module. - */ - granularities?: GranularitiesOption; jwt?: JWTOptions; requestLoggerMiddleware?: RequestLoggerMiddlewareFn; queryRewrite?: QueryRewriteFn; diff --git a/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts b/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts index ed80425c0c938..6695e4a7c4a3d 100644 --- a/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts +++ b/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts @@ -13,17 +13,8 @@ import prepareAnnotationDef import { annotation, prepareAnnotation, - GranularityResolverFn, } from '../../src/helpers/prepare-annotation'; -// Mimics the gateway's built-in fallback: the resolver the gateway injects resolves `day` from -// BUILT_IN_GRANULARITIES defaults. -const dayResolver: GranularityResolverFn = (_dimension, granularity) => ( - granularity === 'day' - ? { name: 'day', type: 'built-in', title: 'Day', interval: '1 day', format: '%Y-%m-%d' } - : undefined -); - describe('prepareAnnotation helpers', () => { test('export looks as expected', () => { expect(prepareAnnotationDef).toBeDefined(); @@ -188,7 +179,7 @@ describe('prepareAnnotation helpers', () => { dimension: 'cube_name.member', granularity: 'day', }], - }, dayResolver).timeDimensions + }).timeDimensions ).toEqual({ 'cube_name.member': { currency: undefined, @@ -252,4 +243,70 @@ describe('prepareAnnotation helpers', () => { }).timeDimensions ).toEqual({}); }); + + describe('granularity resolution from effectiveGranularities', () => { + const metaConfig = (effectiveGranularities?: any[]) => [{ + config: ({ + name: 'cube_name', + title: 'cube name', + dimensions: [{ + name: 'cube_name.member', + type: 'time', + ...(effectiveGranularities ? { effectiveGranularities } : {}), + }], + }) as { name: string; title: string; }, + }]; + + const tdQuery = (granularity: string) => ({ + dimensions: ['cube_name.member'], + timeDimensions: [{ dimension: 'cube_name.member', granularity }], + }); + + test('reads the queried granularity from the effective set (global override honored)', () => { + const result = prepareAnnotation( + metaConfig([ + { name: 'day', type: 'built-in', title: 'Tag', interval: '1 day', format: '%d.%m.%Y' }, + { name: 'fiscal_year', type: 'custom', title: 'Fiscal Year', interval: '1 year', origin: '2024-02-01' }, + ]), + tdQuery('day'), + ); + expect((result.timeDimensions['cube_name.member.day'] as any).granularity).toEqual({ + name: 'day', type: 'built-in', title: 'Tag', interval: '1 day', format: '%d.%m.%Y', + }); + }); + + test('resolves a custom granularity from the effective set', () => { + const result = prepareAnnotation( + metaConfig([ + { name: 'fiscal_year', type: 'custom', title: 'Fiscal Year', interval: '1 year', origin: '2024-02-01' }, + ]), + tdQuery('fiscal_year'), + ); + expect((result.timeDimensions['cube_name.member.fiscal_year'] as any).granularity).toEqual({ + name: 'fiscal_year', type: 'custom', title: 'Fiscal Year', interval: '1 year', origin: '2024-02-01', + }); + }); + + test('synthesizes a config-disabled built-in from defaults', () => { + const result = prepareAnnotation( + metaConfig([ + { name: 'year', type: 'built-in', title: 'Year', interval: '1 year', format: '%Y' }, + ]), + tdQuery('day'), + ); + expect((result.timeDimensions['cube_name.member.day'] as any).granularity).toEqual({ + name: 'day', type: 'built-in', title: 'Day', interval: '1 day', format: '%Y-%m-%d', + }); + }); + + test('unknown custom granularity yields undefined, never the legacy array', () => { + const result = prepareAnnotation( + metaConfig([ + { name: 'day', type: 'built-in', title: 'Day', interval: '1 day', format: '%Y-%m-%d' }, + ]), + tdQuery('some_custom'), + ); + expect((result.timeDimensions['cube_name.member.some_custom'] as any).granularity).toBeUndefined(); + }); + }); }); diff --git a/packages/cubejs-api-gateway/test/index.test.ts b/packages/cubejs-api-gateway/test/index.test.ts index 22701681b6848..fb1680a2710ef 100644 --- a/packages/cubejs-api-gateway/test/index.test.ts +++ b/packages/cubejs-api-gateway/test/index.test.ts @@ -332,6 +332,7 @@ describe('API Gateway', () => { expect(res.body && res.body.data).toStrictEqual([{ 'Foo.bar': '42' }]); expect(res.body.annotation.timeDimensions['Foo.timeGranularities.half_year_by_1st_april']) .toStrictEqual({ + type: 'time', granularity: { name: 'half_year_by_1st_april', type: 'custom', @@ -355,6 +356,7 @@ describe('API Gateway', () => { expect(res.body && res.body.data).toStrictEqual([{ 'Foo.bar': '42' }]); expect(res.body.annotation.timeDimensions['Foo.timeGranularities.half_year_by_1st_april']) .toStrictEqual({ + type: 'time', granularity: { name: 'half_year_by_1st_april', type: 'custom', @@ -689,6 +691,58 @@ describe('API Gateway', () => { expect(res.body.cubes[0]?.segments.find(segment => segment.name === 'Foo.quux').description).toBe('segment from compilerApi mock'); }); + test('meta endpoint passes through effectiveGranularities from CompilerApi', async () => { + const { app } = await createApiGateway(); + + const res = await request(app) + .get('/cubejs-api/v1/meta') + .set('Authorization', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.t-IDcSemACt8x4iTMCda8Yhe3iZaWbvV5XKSTbuAn0M') + .expect(200); + + const dim = res.body.cubes[0]?.dimensions.find(d => d.name === 'Foo.timeGranularities'); + expect(dim.effectiveGranularities).toEqual([ + { name: 'year', type: 'built-in', title: 'Year', interval: '1 year', format: '%Y' }, + { + name: 'half_year_by_1st_april', + type: 'custom', + title: 'Half Year By1 St April', + interval: '6 months', + offset: '3 months', + }, + ]); + // The deprecated legacy array is preserved unchanged, and the internal block never leaks. + expect(dim.granularities).toEqual([ + { + name: 'half_year_by_1st_april', + title: 'Half Year By1 St April', + interval: '6 months', + offset: '3 months', + }, + ]); + expect(dim.granularitiesBlock).toBeUndefined(); + }); + + test('granularities endpoint returns the context-resolved global config', async () => { + const { app } = await createApiGateway(); + + const res = await request(app) + .get('/cubejs-api/v1/granularities') + .set('Authorization', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.t-IDcSemACt8x4iTMCda8Yhe3iZaWbvV5XKSTbuAn0M') + .expect(200); + + expect(res.body.data.granularities).toEqual([ + { type: 'built-in', name: 'year', title: 'Year', format: '%Y', interval: '1 year' }, + { type: 'built-in', name: 'month', title: 'Month', format: '%b %Y', interval: '1 month' }, + { + type: 'custom', + name: 'fiscal_year', + title: 'Fiscal Year', + interval: '1 year', + origin: '2024-02-01', + }, + ]); + }); + test('meta endpoint returns view groups', async () => { const { app } = await createApiGateway(); diff --git a/packages/cubejs-api-gateway/test/mocks.ts b/packages/cubejs-api-gateway/test/mocks.ts index 5ea9898f8487a..40f6df656a20e 100644 --- a/packages/cubejs-api-gateway/test/mocks.ts +++ b/packages/cubejs-api-gateway/test/mocks.ts @@ -80,6 +80,15 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({ return { query, denied: false }; }, + async resolveGlobalGranularitiesConfig(_ctx: any) { + return { + enabledBuiltIns: ['year', 'month'], + customGranularities: { + fiscal_year: { title: 'Fiscal Year', interval: '1 year', origin: '2024-02-01' }, + }, + }; + }, + async metaConfig(_ctx, options: any = {}) { const cubes = [ { @@ -106,6 +115,7 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({ }, { name: 'Foo.timeGranularities', + type: 'time', isVisible: true, granularities: [ { @@ -114,7 +124,18 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({ interval: '6 months', offset: '3 months' } - ] + ], + // As attached by CompilerApi (baked or variant-selected) before meta reaches the gateway. + effectiveGranularities: [ + { name: 'year', type: 'built-in', title: 'Year', interval: '1 year', format: '%Y' }, + { + name: 'half_year_by_1st_april', + type: 'custom', + title: 'Half Year By1 St April', + interval: '6 months', + offset: '3 months', + }, + ], }, ], segments: [ diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index cd37537c4b3f4..4977cc06fcb40 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -20,6 +20,19 @@ import type { JoinGraph } from './JoinGraph'; import type { ErrorReporter } from './ErrorReporter'; import { CompilerInterface } from './PrepareCompiler'; import { resolveNamedNumericFormat, STANDARD_FORMAT_SPECIFIERS, DEFAULT_FORMAT_SPECIFIER } from './named-numeric-formats'; +import { + EffectiveGranularity, + NormalizedGranularitiesBlock, + normalizeGranularitiesBlock, + resolveDimensionGranularities, + serializeEffectiveGranularities, +} from './GranularityResolver'; +import { + GlobalGranularitiesConfig, + GranularitiesOption, + buildBuiltInsCatalog, + resolveGlobalGranularitiesSync, +} from './GlobalGranularitiesConfig'; export type CustomNumericFormat = { type: 'custom-numeric'; value: string; alias?: string }; export type DimensionCustomTimeFormat = { type: 'custom-time'; value: string }; @@ -137,6 +150,12 @@ export type DimensionConfig = { * omits built-ins, global customs, and the `type` field. See DEPRECATION.md. */ granularities?: GranularityDefinition[]; + /** + * Reconciled granularity set (enabled built-ins + global customs + local customs) for time + * dimensions. Baked in at compile time for env/static global configs; attached per request + * by CompilerApi meta variants when `granularities` is a context function. + */ + effectiveGranularities?: EffectiveGranularity[]; order?: 'asc' | 'desc'; key?: string; links?: LinkConfig[]; @@ -201,12 +220,31 @@ export class CubeToMetaTransformer implements CompilerInterface { */ public queries: TransformedCube[]; + private readonly granularitiesOption?: GranularitiesOption; + + /** + * Per-dimension granularity inputs for time dimensions that customize their granularity set + * (a `granularities` block or local customs). Keyed by `cube.dimension`. Dimensions absent + * from this index use the config-wide default set. Consumed by `CompilerApi` to build + * context-dependent meta variants when `granularities` is a function; never serialized. + */ + public readonly granularityInputs: Map = new Map(); + + // Set during compile() for the context-independent config forms (env / static list); + // null when `granularities` is a function and resolution has to happen per request. + private staticGranularityState: { + config: GlobalGranularitiesConfig; + catalog: Record; + defaultSet: EffectiveGranularity[]; + } | null = null; + public constructor( cubeValidator: CubeValidator, cubeEvaluator: CubeEvaluator, contextEvaluator: ContextEvaluator, viewGroupEvaluator: ViewGroupEvaluator, - joinGraph: JoinGraph + joinGraph: JoinGraph, + granularitiesOption?: GranularitiesOption ) { this.cubeValidator = cubeValidator; this.cubeSymbols = cubeEvaluator; @@ -214,6 +252,7 @@ export class CubeToMetaTransformer implements CompilerInterface { this.contextEvaluator = contextEvaluator; this.viewGroupEvaluator = viewGroupEvaluator; this.joinGraph = joinGraph; + this.granularitiesOption = granularitiesOption; this.cubes = []; this.queries = []; } @@ -223,6 +262,30 @@ export class CubeToMetaTransformer implements CompilerInterface { } public compile(_cubes: any[], errorReporter: ErrorReporter): void { + this.granularityInputs.clear(); + // Env / static-list configs are context-independent, so the effective granularity sets are + // resolved once here and baked into the meta configs. The function form is never called at + // compile time (the compiled model is shared across security contexts); `CompilerApi` + // resolves it per request and enriches cached variants from `granularityInputs`. + if (typeof this.granularitiesOption === 'function') { + this.staticGranularityState = null; + } else { + const config = resolveGlobalGranularitiesSync(this.granularitiesOption); + const catalog = buildBuiltInsCatalog(config); + this.staticGranularityState = { + config, + catalog, + // One shared array for every time dimension without local customization — with large + // models this avoids re-allocating an identical granularity set per dimension. + defaultSet: serializeEffectiveGranularities(resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), + config.enabledBuiltIns, + config.customGranularities, + catalog, + )), + }; + } + this.cubes = this.cubeSymbols.cubeList .filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator)) .map((v) => this.transform(v, errorReporter.inContext(`${v.name} cube`))); @@ -306,14 +369,30 @@ export class CubeToMetaTransformer implements CompilerInterface { ? this.isVisible(extendedDimDef, !extendedDimDef.primaryKey) : false; const granularitiesObj = extendedDimDef.granularities; - // `granularities` keeps its legacy custom-only shape (deprecated); the gateway attaches - // the reconciled set as `effectiveGranularities` per request from `granularitiesBlock` - // and strips the block before responding. + // `granularities` keeps its legacy custom-only shape (deprecated). The reconciled set + // is emitted as `effectiveGranularities`: baked in here for env/static global configs, + // or attached per request by CompilerApi variants when the config is a function. const { granularitiesBlock } = extendedDimDef as any; const dimType = this.dimensionDataType(extendedDimDef.type || 'string'); const dimFormat = this.transformDimensionFormat(extendedDimDef); const dimCurrency = extendedDimDef.currency?.toUpperCase(); + let effectiveGranularities: EffectiveGranularity[] | undefined; + if (dimType === 'time') { + const inputs = this.granularityInputsForDimension(cubeTitle, granularitiesObj, granularitiesBlock); + if (inputs) { + this.granularityInputs.set(`${cubeName}.${dimensionName}`, inputs); + } + if (this.staticGranularityState) { + const s = this.staticGranularityState; + effectiveGranularities = inputs + ? serializeEffectiveGranularities(resolveDimensionGranularities( + inputs, s.config.enabledBuiltIns, s.config.customGranularities, s.catalog, + )) + : s.defaultSet; + } + } + return { name: `${cubeName}.${dimensionName}`, title: this.title(cubeTitle, nameToDimension, false), @@ -342,7 +421,7 @@ export class CubeToMetaTransformer implements CompilerInterface { origin: gDef.origin, })) : undefined, - ...(granularitiesBlock ? { granularitiesBlock } : {}), + ...(effectiveGranularities ? { effectiveGranularities } : {}), order: extendedDimDef.order, key: extendedDimDef.keyReference, ...(extendedDimDef.links ? { links: extendedDimDef.links.map((link: any) => ({ @@ -384,6 +463,35 @@ export class CubeToMetaTransformer implements CompilerInterface { }; } + /** + * Granularity-resolution inputs for one time dimension, or null when the dimension has no + * local customization and can use the config-wide default set. Local custom titles resolve + * here (short-title semantics, matching the legacy `granularities` array); every other field + * is projected from the raw definition so that e.g. `sql` never leaks into meta output. + */ + private granularityInputsForDimension( + cubeTitle: string, + granularitiesObj: Record | undefined, + granularitiesBlock: NormalizedGranularitiesBlock | undefined, + ): NormalizedGranularitiesBlock | null { + const hasLocalCustoms = granularitiesObj && Object.keys(granularitiesObj).length > 0; + if (!granularitiesBlock && !hasLocalCustoms) { + return null; + } + const block = granularitiesBlock || normalizeGranularitiesBlock(undefined); + const custom: Record = {}; + for (const [gName, gDef] of Object.entries({ ...block.custom, ...(granularitiesObj || {}) })) { + custom[gName] = { + title: this.title(cubeTitle, [gName, gDef], true), + interval: gDef.interval, + offset: gDef.offset, + origin: gDef.origin, + ...(gDef.format !== undefined ? { format: gDef.format } : {}), + }; + } + return { includes: block.includes, excludes: block.excludes, custom }; + } + public queriesForContext(contextId: string | null | undefined): TransformedCube[] { // return All queries if no context pass if (contextId == null || contextId.length === 0) { diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts index b0207f43ee87f..2e39afe7b3123 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -110,22 +110,34 @@ function resolveFromList(list: GranularityList): GlobalGranularitiesConfig { return { enabledBuiltIns, customGranularities }; } +// Config forms accepted for `granularities` in cube.js / cube.py. +export type GranularitiesOption = + GranularityList | ((ctx: any) => GranularityList | Promise) | undefined; + +// Sync resolution for the context-independent forms (undefined -> env, static list). +// The function form is per-context and must go through `resolveGlobalGranularities`. +export function resolveGlobalGranularitiesSync( + userValue: Exclude, +): GlobalGranularitiesConfig { + if (!Array.isArray(userValue)) { + return resolveFromEnv(); + } + return resolveFromList(userValue); +} + // `userValue` is the value of `granularities` from the cube.js / cube.py config file. // undefined -> fall back to `CUBEJS_GRANULARITIES` env vars // GranularityList -> use this list, replacing env vars entirely (no merge) // function(ctx) -> called per request; same no-merge replacement as the list form export async function resolveGlobalGranularities( - userValue: GranularityList | ((ctx: any) => GranularityList | Promise) | undefined, + userValue: GranularitiesOption, ctx: any, ): Promise { - if (userValue === undefined) { - return resolveFromEnv(); - } - const resolved = typeof userValue === 'function' ? await userValue(ctx) : userValue; - if (!Array.isArray(resolved)) { - return resolveFromEnv(); + if (typeof userValue === 'function') { + const resolved = await userValue(ctx); + return resolveGlobalGranularitiesSync(Array.isArray(resolved) ? resolved : undefined); } - return resolveFromList(resolved); + return resolveGlobalGranularitiesSync(userValue); } export function getBuiltInGranularityDefaults(name: string) { diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts new file mode 100644 index 0000000000000..944a93676dcbd --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts @@ -0,0 +1,29 @@ +import crypto from 'crypto'; + +import type { GlobalGranularitiesConfig } from './GlobalGranularitiesConfig'; + +// Emission-order-sensitive canonical form: `enabledBuiltIns` order and custom-granularity +// insertion order both affect the resolved set that clients receive, so neither is sorted. +// Only the known serializable fields participate; anything else a config spreads onto a +// definition (including functions) is ignored so the hash stays deterministic. +const asHashableString = (value: unknown): string | undefined => ( + typeof value === 'string' ? value : undefined +); + +// Canonical sha256 of a resolved global granularities config. Two configs share a hash iff +// they produce identical effective granularity sets, so the hash is safe to use as a cache +// key for enriched meta variants and as a compilerId discriminator. +export function granularityConfigHash(config: GlobalGranularitiesConfig): string { + const canonical = { + builtIns: [...config.enabledBuiltIns], + custom: Object.entries(config.customGranularities).map(([name, def]) => ({ + name, + title: asHashableString(def.title), + format: asHashableString(def.format), + interval: asHashableString(def.interval), + offset: asHashableString(def.offset), + origin: asHashableString(def.origin), + })), + }; + return crypto.createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); +} diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts index f177f3891f63e..cbd197173edd6 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -73,6 +73,32 @@ export function normalizeGranularitiesBlock(raw: any): NormalizedGranularitiesBl return EMPTY_BLOCK; } +// Wire shape of one entry in a time dimension's `effectiveGranularities`. +export type EffectiveGranularity = { + name: string; + type: 'built-in' | 'custom'; + title: string; + interval?: string; + offset?: string; + origin?: string; + format?: string; +}; + +// Serialize a resolved set for /v1/meta. Field order and the conditional inclusion of +// interval/offset/origin/format are part of the wire contract — keep byte-compatible. +// `title` falls back to the granularity name (same as /v1/granularities) so it is always present. +export function serializeEffectiveGranularities(resolved: ResolvedGranularitySet): EffectiveGranularity[] { + return Object.entries(resolved).map(([name, def]) => ({ + name, + type: def.type, + title: def.title || name, + ...(def.interval !== undefined ? { interval: def.interval } : {}), + ...(def.offset !== undefined ? { offset: def.offset } : {}), + ...(def.origin !== undefined ? { origin: def.origin } : {}), + ...(def.format !== undefined ? { format: def.format } : {}), + })); +} + // Reconcile a dimension's local block against the global enabled built-ins and global customs, // producing the effective set. Local customs always survive — even if local excludes is '*'. export function resolveDimensionGranularities( diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index e36e6cb02589b..615ba49face69 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -26,6 +26,7 @@ import { CompilerCache } from './CompilerCache'; import { YamlCompiler } from './YamlCompiler'; import { ViewCompilationGate } from './ViewCompilationGate'; import type { ErrorReporter } from './ErrorReporter'; +import type { GranularitiesOption } from './GlobalGranularitiesConfig'; export type PrepareCompilerOptions = { nativeInstance?: NativeInstance, @@ -40,6 +41,10 @@ export type PrepareCompilerOptions = { compiledScriptCache?: LRUCache; compiledYamlCache?: LRUCache; compiledJinjaCache?: LRUCache; + // Global `granularities` config (env fallback / static list / context function). Env and + // static forms are resolved at compile time by CubeToMetaTransformer; the function form is + // resolved per request by CompilerApi. + granularities?: GranularitiesOption; }; export interface CompilerInterface { @@ -56,6 +61,12 @@ export type Compiler = { compilerCache: CompilerCache; headCommitId?: string; compilerId: string; + /** + * Granularity-enriched meta variants, keyed by canonical global-config hash. Owned by the + * compiled model (not CompilerApi) so a recompile discards it implicitly. Populated lazily + * by CompilerApi when `granularities` is a context function; bounded LRU. + */ + granularityVariants?: Map>; }; export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}): Compiler => { @@ -69,7 +80,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const contextEvaluator = new ContextEvaluator(cubeEvaluator); const viewGroupEvaluator = new ViewGroupEvaluator(cubeEvaluator, cubeValidator); const joinGraph = new JoinGraph(cubeValidator, cubeEvaluator); - const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, viewGroupEvaluator, joinGraph); + const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, viewGroupEvaluator, joinGraph, options.granularities); const { maxQueryCacheSize, maxQueryCacheAge } = options; const compilerCache = new CompilerCache({ maxQueryCacheSize, maxQueryCacheAge }); const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler); diff --git a/packages/cubejs-schema-compiler/src/compiler/index.ts b/packages/cubejs-schema-compiler/src/compiler/index.ts index 86f7b4ff85114..12067767c8460 100644 --- a/packages/cubejs-schema-compiler/src/compiler/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/index.ts @@ -18,15 +18,20 @@ export { BuiltInGranularityDefinition, GranularityList, GranularityListItem, + GranularitiesOption, GlobalGranularitiesConfig, BuiltInCatalogEntry, resolveGlobalGranularities, + resolveGlobalGranularitiesSync, getBuiltInGranularityDefaults, buildBuiltInsCatalog, } from './GlobalGranularitiesConfig'; export { NormalizedGranularitiesBlock, ResolvedGranularitySet, + EffectiveGranularity, normalizeGranularitiesBlock, resolveDimensionGranularities, + serializeEffectiveGranularities, } from './GranularityResolver'; +export { granularityConfigHash } from './GranularityConfigHash'; diff --git a/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts b/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts new file mode 100644 index 0000000000000..136ffd97a42e3 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts @@ -0,0 +1,55 @@ +import { granularityConfigHash } from '../../src/compiler/GranularityConfigHash'; +import type { GlobalGranularitiesConfig } from '../../src/compiler/GlobalGranularitiesConfig'; + +const config = ( + enabledBuiltIns: string[], + customGranularities: Record = {}, +): GlobalGranularitiesConfig => ({ enabledBuiltIns, customGranularities }); + +describe('granularityConfigHash', () => { + it('is stable for identical configs', () => { + const a = config(['year', 'month'], { fy: { interval: '1 year', origin: '2024-02-01' } }); + const b = config(['year', 'month'], { fy: { interval: '1 year', origin: '2024-02-01' } }); + expect(granularityConfigHash(a)).toBe(granularityConfigHash(b)); + }); + + it('is sensitive to built-in order (order is part of the wire contract)', () => { + expect(granularityConfigHash(config(['year', 'month']))) + .not.toBe(granularityConfigHash(config(['month', 'year']))); + }); + + it('is sensitive to custom emission order', () => { + const a = config(['year'], { a: { interval: '1 week' }, b: { interval: '2 week' } }); + const b = config(['year'], { b: { interval: '2 week' }, a: { interval: '1 week' } }); + expect(granularityConfigHash(a)).not.toBe(granularityConfigHash(b)); + }); + + it('is sensitive to every known definition field', () => { + const base = { interval: '1 year', title: 'FY', format: '%Y', offset: undefined, origin: '2024-02-01' }; + const baseHash = granularityConfigHash(config(['year'], { fy: base })); + for (const [field, value] of [ + ['interval', '2 year'], ['title', 'Fiscal'], ['format', '%y'], ['origin', '2025-02-01'], + ] as const) { + expect(granularityConfigHash(config(['year'], { fy: { ...base, [field]: value } }))) + .not.toBe(baseHash); + } + }); + + it('ignores unknown and non-serializable definition props', () => { + const plain = config(['year'], { fy: { interval: '1 year' } }); + const dirty = config(['year'], { + fy: { + interval: '1 year', + sql: () => 'now()', + somethingElse: { nested: true }, + title: (() => 'not a string') as any, + }, + }); + expect(granularityConfigHash(dirty)).toBe(granularityConfigHash(plain)); + }); + + it('distinguishes an absent field from a present one', () => { + expect(granularityConfigHash(config(['year'], { fy: { interval: '1 year' } }))) + .not.toBe(granularityConfigHash(config(['year'], { fy: { interval: '1 year', title: 'FY' } }))); + }); +}); diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index c6e1481713747..3fdaf171bc7b1 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -4,11 +4,16 @@ import { AccessPolicyDefinition, BaseQuery, CanUsePreAggregationFn, + buildBuiltInsCatalog, compile, Compiler, createQuery, CubeDefinition, EvaluatedCube, + GlobalGranularitiesConfig, + GranularitiesOption, + granularityConfigHash, + normalizeGranularitiesBlock, PreAggregationFilters, PreAggregationInfo, PreAggregationReferences, @@ -16,6 +21,10 @@ import { prepareCompiler, queryClass, QueryFactory, + resolveDimensionGranularities, + resolveGlobalGranularities, + resolveGlobalGranularitiesSync, + serializeEffectiveGranularities, TransformedQuery, ViewIncludedMember, } from '@cubejs-backend/schema-compiler'; @@ -51,6 +60,7 @@ export interface CompilerApiOptions { devServer?: boolean; fastReload?: boolean; allowNodeRequire?: boolean; + granularities?: GranularitiesOption; } export interface GetSqlOptions { @@ -87,6 +97,13 @@ export interface DataSourceInfo { } export class CompilerApi { + /** + * Bound on cached granularity-enriched meta variants per compiled model. Each entry costs + * O(model) memory; expected distinct-config cardinality for a `granularities` context + * function is 1–4, so this is headroom, not a target. Exceeding it evicts LRU and logs. + */ + protected static readonly MAX_GRANULARITY_VARIANTS = 16; + protected readonly repository: SchemaFileRepository; protected readonly dbType: DbTypeInternalFn; @@ -133,6 +150,8 @@ export class CompilerApi { protected compilerVersion?: string; + protected readonly granularities?: GranularitiesOption; + protected queryFactory?: QueryFactory; public constructor(repository: SchemaFileRepository, dbType: DbTypeInternalFn, options: CompilerApiOptions) { @@ -152,6 +171,7 @@ export class CompilerApi { this.allowJsDuplicatePropsInSchema = options.allowJsDuplicatePropsInSchema; this.sqlCache = options.sqlCache; this.standalone = options.standalone; + this.granularities = options.granularities; this.nativeInstance = this.createNativeInstance(); // Caching stuff @@ -224,6 +244,13 @@ export class CompilerApi { compilerVersion += `_${crypto.createHash('md5').update(JSON.stringify(files)).digest('hex')}`; } + // Env/static granularity configs are baked into the compiled meta, so a config change must + // force a recompile. The function form is resolved per request instead (variant cache) and + // must never churn the compiler version. + if (typeof this.granularities !== 'function') { + compilerVersion += `_gran_${granularityConfigHash(resolveGlobalGranularitiesSync(this.granularities))}`; + } + if (!this.compilers || this.compilerVersion !== compilerVersion) { this.compilers = this.compileSchema(compilerVersion, options.requestId).catch(e => { this.compilers = undefined; @@ -267,6 +294,7 @@ export class CompilerApi { compiledScriptCache: this.compiledScriptCache, compiledJinjaCache: this.compiledJinjaCache, compiledYamlCache: this.compiledYamlCache, + granularities: this.granularities, }); this.queryFactory = await this.createQueryFactory(compilers); @@ -1064,24 +1092,144 @@ export class CompilerApi { }; } - protected mixInVisibilityMaskHash(compilerId: string, visibilityMaskHash: string): string { + protected mixInMaskHash(compilerId: string, maskHash: string): string { const uuidBytes = Buffer.from(uuidParse(compilerId)); - const hashBytes = Buffer.from(visibilityMaskHash, 'hex'); + const hashBytes = Buffer.from(maskHash, 'hex'); return uuidv4({ random: crypto.createHash('sha256').update(uuidBytes).update(hashBytes).digest() .subarray(0, 16) as any }); } + protected mixInVisibilityMaskHash(compilerId: string, visibilityMaskHash: string): string { + return this.mixInMaskHash(compilerId, visibilityMaskHash); + } + + /** + * Global granularity config for a request context. O(config): env/static forms ignore the + * context; the function form is invoked with it. Used by the /v1/granularities endpoint. + */ + public async resolveGlobalGranularitiesConfig(context: Context): Promise { + return resolveGlobalGranularities(this.granularities, context); + } + + /** + * Meta cubes with `effectiveGranularities` attached, plus the hash to mix into compilerId. + * + * Env/static configs are context-independent: the transformer bakes the effective sets into + * the base meta at compile time (their hash is folded into compilerVersion), so this returns + * the base cubes untouched with a null hash. + * + * A function config resolves per request. Enriched variants are cached on the compiled model + * keyed by the canonical config hash — a bounded promise-valued LRU, so concurrent misses of + * one hash dedup to a single O(model) build, a failed build self-evicts, and the whole cache + * is discarded with the compilers object on recompile. + */ + protected async selectGranularityVariant( + compilers: Compiler, + requestContext: Context, + ): Promise<{ cubes: any[]; granularityHash: string | null }> { + if (typeof this.granularities !== 'function') { + return { cubes: compilers.metaTransformer.cubes, granularityHash: null }; + } + + const config = await resolveGlobalGranularities(this.granularities, requestContext); + const granularityHash = granularityConfigHash(config); + + if (!compilers.granularityVariants) { + compilers.granularityVariants = new Map(); + } + const cache = compilers.granularityVariants; + + let variant = cache.get(granularityHash); + if (variant) { + // Refresh LRU recency. + cache.delete(granularityHash); + cache.set(granularityHash, variant); + } else { + if (cache.size >= CompilerApi.MAX_GRANULARITY_VARIANTS) { + const oldest = cache.keys().next().value; + cache.delete(oldest); + this.logger('Granularity variant cache is full', { + warning: `More than ${CompilerApi.MAX_GRANULARITY_VARIANTS} distinct granularity configs seen for one compiled model; ` + + 'evicting the least recently used variant. A `granularities` function returning unstable values ' + + 'causes per-request meta rebuilds and churns compilerId-based caches (e.g. in CubeSQL).', + }); + } + variant = Promise.resolve().then(() => this.buildGranularityVariant(compilers, config)); + cache.set(granularityHash, variant); + variant.catch(() => { + if (cache.get(granularityHash) === variant) { + cache.delete(granularityHash); + } + }); + } + + return { cubes: await variant, granularityHash }; + } + + /** + * One O(model) pass attaching `effectiveGranularities` to every time dimension. Never mutates + * the base meta: enriched cubes get new cube/config/dimensions containers, while untouched + * members stay shared by reference (all downstream consumers — visibility patch, gateway + * filters — copy-on-write rather than mutate). Time dimensions without local customization + * share one default set per variant instead of allocating identical arrays per dimension. + */ + protected buildGranularityVariant(compilers: Compiler, config: GlobalGranularitiesConfig): any[] { + const catalog = buildBuiltInsCatalog(config); + const inputs = compilers.metaTransformer.granularityInputs; + const defaultSet = serializeEffectiveGranularities(resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), + config.enabledBuiltIns, + config.customGranularities, + catalog, + )); + + return compilers.metaTransformer.cubes.map((cube: any) => { + if (!cube.config.dimensions?.some((d: any) => d.type === 'time')) { + return cube; + } + return { + config: { + ...cube.config, + dimensions: cube.config.dimensions.map((dim: any) => { + if (dim.type !== 'time') { + return dim; + } + const block = inputs.get(dim.name); + const effectiveGranularities = block + ? serializeEffectiveGranularities(resolveDimensionGranularities( + block, config.enabledBuiltIns, config.customGranularities, catalog, + )) + : defaultSet; + return { ...dim, effectiveGranularities }; + }), + }, + }; + }); + } + public async metaConfig( requestContext: Context, options: { includeCompilerId?: boolean; includeViewGroups?: boolean; skipVisibilityPatch?: boolean; requestId?: string } = {} ): Promise { const { includeCompilerId, includeViewGroups, skipVisibilityPatch, ...restOptions } = options; const compilers = await this.getCompilers(restOptions); - const { cubes } = compilers.metaTransformer; + const { cubes, granularityHash } = await this.selectGranularityVariant(compilers, requestContext); + + // Fixed composition order: base compilerId, then visibility mask, then granularity hash. + const composeCompilerId = (visibilityMaskHash: string | null) => { + let id = compilers.compilerId; + if (visibilityMaskHash) { + id = this.mixInMaskHash(id, visibilityMaskHash); + } + if (granularityHash) { + id = this.mixInMaskHash(id, granularityHash); + } + return id; + }; if (skipVisibilityPatch) { if (includeCompilerId || includeViewGroups) { - const result: any = { cubes, compilerId: compilers.compilerId }; + const result: any = { cubes, compilerId: composeCompilerId(null) }; if (includeViewGroups) { result.viewGroups = compilers.metaTransformer.viewGroups; } @@ -1098,7 +1246,7 @@ export class CompilerApi { if (includeCompilerId || includeViewGroups) { const result: any = { cubes: patchedCubes, - compilerId: visibilityMaskHash ? this.mixInVisibilityMaskHash(compilers.compilerId, visibilityMaskHash) : compilers.compilerId, + compilerId: composeCompilerId(visibilityMaskHash), }; if (includeViewGroups) { result.viewGroups = compilers.metaTransformer.viewGroups; @@ -1113,10 +1261,11 @@ export class CompilerApi { options?: { requestId?: string } ): Promise<{ metaConfig: any; cubeDefinitions: Record }> { const compilers = await this.getCompilers(options); + const { cubes } = await this.selectGranularityVariant(compilers, requestContext); const { cubes: patchedCubes } = await this.patchVisibilityByAccessPolicy( compilers, requestContext, - compilers.metaTransformer?.cubes + cubes ); return { metaConfig: patchedCubes, diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index 8c1a61ca6df55..8d047e4340bab 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -481,7 +481,6 @@ export class CubejsServerCore { queryRewrite: this.options.queryRewrite || this.options.queryTransformer, extendContext: this.options.extendContext, - granularities: this.options.granularities, playgroundAuthSecret: getEnv('playgroundAuthSecret'), apiSecrets: this.options.apiSecrets, jwt: this.options.jwt, @@ -742,6 +741,7 @@ export class CubejsServerCore { compileContext: options.context, dialectClass: options.dialectClass, externalDialectClass: options.externalDialectClass, + granularities: this.options.granularities, allowJsDuplicatePropsInSchema: options.allowJsDuplicatePropsInSchema, sqlCache: this.options.sqlCache, standalone: this.standalone, diff --git a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts new file mode 100644 index 0000000000000..9cd16ebf7fed8 --- /dev/null +++ b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts @@ -0,0 +1,331 @@ +import { SchemaFileRepository } from '@cubejs-backend/shared'; +import type { Compiler, GlobalGranularitiesConfig } from '@cubejs-backend/schema-compiler'; +import { CompilerApi } from '../../src/core/CompilerApi'; +import { DbTypeInternalFn } from '../../src/core/types'; + +class TestableCompilerApi extends CompilerApi { + public buildCount = 0; + + public failNextBuild = false; + + protected buildGranularityVariant(compilers: Compiler, config: GlobalGranularitiesConfig): any[] { + if (this.failNextBuild) { + this.failNextBuild = false; + throw new Error('injected variant build failure'); + } + this.buildCount++; + return super.buildGranularityVariant(compilers, config); + } + + public version(): string | undefined { + return this.compilerVersion; + } + + public async variantCache(): Promise> | undefined> { + return (await this.getCompilers()).granularityVariants; + } +} + +const repository: SchemaFileRepository = { + localPath: () => '/mock/path', + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'orders.js', + content: ` + cube('Orders', { + sql: 'SELECT * FROM orders', + measures: { count: { type: 'count' } }, + dimensions: { + id: { sql: 'id', type: 'number', primaryKey: true }, + created_at: { sql: 'created_at', type: 'time' }, + updated_at: { sql: 'updated_at', type: 'time' }, + }, + }); + cube('Events', { + sql: 'SELECT * FROM events', + measures: { count: { type: 'count' } }, + dimensions: { + ts: { + sql: 'ts', + type: 'time', + granularities: { + fiscal_year: { interval: '1 year', origin: '2024-02-01' }, + }, + }, + }, + }); + cube('Products', { + sql: 'SELECT * FROM products', + measures: { count: { type: 'count' } }, + dimensions: { + name: { sql: 'name', type: 'string' }, + }, + }); + `, + }, + ]), +}; + +const mockDbType: DbTypeInternalFn = async () => 'postgres'; + +const noopLogger = () => { /* silent */ }; + +const createApi = (options: any = {}) => new TestableCompilerApi(repository, mockDbType, { + logger: options.capturedLogs + ? (msg: string, params: any) => options.capturedLogs.push({ msg, params }) + : noopLogger, + ...options, +}); + +const ctxFor = (tenant: string) => ({ securityContext: { tenant }, requestId: `req-${tenant}` }); + +const dimByName = (cubes: any[], name: string) => cubes + .flatMap((c: any) => c.config.dimensions) + .find((d: any) => d.name === name); + +const granularityNames = (dim: any) => dim.effectiveGranularities.map((g: any) => g.name); + +const ALL_BUILT_INS = ['year', 'quarter', 'month', 'week', 'day', 'hour', 'minute', 'second']; + +describe('granularity variants in CompilerApi', () => { + describe('env/static configs (baked at compile time)', () => { + afterEach(() => { + delete process.env.CUBEJS_GRANULARITIES; + }); + + test('no config: every time dimension gets all built-ins, no variant builds happen', async () => { + const api = createApi(); + const cubes = await api.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toEqual(ALL_BUILT_INS); + // Local custom granularity survives on top of the enabled built-ins. + expect(granularityNames(dimByName(cubes, 'Events.ts'))).toEqual([...ALL_BUILT_INS, 'fiscal_year']); + // Non-time dimensions are untouched. + expect(dimByName(cubes, 'Products.name').effectiveGranularities).toBeUndefined(); + + await api.metaConfig(ctxFor('b'), {}); + expect(api.buildCount).toBe(0); + expect(await api.variantCache()).toBeUndefined(); + api.dispose(); + }); + + test('static list is baked in and internal fields stay off the wire', async () => { + const api = createApi({ granularities: ['year', { name: 'half', interval: '6 months' }] }); + const cubes = await api.metaConfig(ctxFor('a'), {}); + const dim = dimByName(cubes, 'Orders.created_at'); + expect(dim.effectiveGranularities).toEqual([ + { name: 'year', type: 'built-in', title: 'Year', interval: '1 year', format: '%Y' }, + { name: 'half', type: 'custom', title: 'half', interval: '6 months' }, + ]); + expect(dim.granularitiesBlock).toBeUndefined(); + // Legacy shape for the customized dimension is preserved (deprecated but not broken). + const eventsTs = dimByName(cubes, 'Events.ts'); + expect(eventsTs.granularities).toEqual([ + { name: 'fiscal_year', title: 'Fiscal Year', interval: '1 year', offset: undefined, origin: '2024-02-01' }, + ]); + expect(api.buildCount).toBe(0); + api.dispose(); + }); + + test('time dimensions without customization share one default set instance', async () => { + const api = createApi(); + const cubes = await api.metaConfig(ctxFor('a'), {}); + const created = dimByName(cubes, 'Orders.created_at'); + const updated = dimByName(cubes, 'Orders.updated_at'); + expect(created.effectiveGranularities).toBe(updated.effectiveGranularities); + expect(dimByName(cubes, 'Events.ts').effectiveGranularities) + .not.toBe(created.effectiveGranularities); + api.dispose(); + }); + + test('the resolved config hash is folded into compilerVersion, so an env change recompiles', async () => { + const api = createApi(); + let cubes = await api.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toEqual(ALL_BUILT_INS); + const versionBefore = api.version(); + expect(versionBefore).toContain('_gran_'); + + process.env.CUBEJS_GRANULARITIES = 'year,month'; + cubes = await api.metaConfig(ctxFor('a'), {}); + expect(api.version()).not.toBe(versionBefore); + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toEqual(['year', 'month']); + api.dispose(); + }); + }); + + describe('function config (per-request variants)', () => { + const perTenant = (ctx: any) => (ctx.securityContext.tenant === 'a' + ? ['year', 'month'] + : ['week', { name: 'sprint', interval: '2 weeks' }]); + + test('tenants get their own sets; repeats and alternation hit the cache; no leaks', async () => { + const api = createApi({ granularities: perTenant }); + + for (let i = 0; i < 3; i++) { + const cubesA = await api.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubesA, 'Orders.created_at'))).toEqual(['year', 'month']); + expect(granularityNames(dimByName(cubesA, 'Events.ts'))).toEqual(['year', 'month', 'fiscal_year']); + + const cubesB = await api.metaConfig(ctxFor('b'), {}); + expect(granularityNames(dimByName(cubesB, 'Orders.created_at'))).toEqual(['week', 'sprint']); + expect(granularityNames(dimByName(cubesB, 'Events.ts'))).toEqual(['week', 'sprint', 'fiscal_year']); + } + + expect(api.buildCount).toBe(2); + expect((await api.variantCache())!.size).toBe(2); + api.dispose(); + }); + + test('base meta cubes are never mutated by variant enrichment', async () => { + const api = createApi({ granularities: perTenant }); + await api.metaConfig(ctxFor('a'), {}); + const compilers = await (api as any).getCompilers(); + const baseDim = dimByName(compilers.metaTransformer.cubes, 'Orders.created_at'); + expect(baseDim.effectiveGranularities).toBeUndefined(); + api.dispose(); + }); + + test('distinct compilerIds per tenant, both distinct from the base', async () => { + const api = createApi({ granularities: perTenant }); + const a = await api.metaConfig(ctxFor('a'), { includeCompilerId: true }); + const b = await api.metaConfig(ctxFor('b'), { includeCompilerId: true }); + const base = (await (api as any).getCompilers()).compilerId; + expect(a.compilerId).not.toBe(b.compilerId); + expect(a.compilerId).not.toBe(base); + expect(b.compilerId).not.toBe(base); + // Same tenant, same id — stable across calls. + const a2 = await api.metaConfig(ctxFor('a'), { includeCompilerId: true }); + expect(a2.compilerId).toBe(a.compilerId); + api.dispose(); + }); + + test('static and function forms producing the same list emit identical meta', async () => { + const list = ['year', 'month', { name: 'half', interval: '6 months', title: 'Half' }]; + const staticApi = createApi({ granularities: list }); + const fnApi = createApi({ granularities: () => list }); + const staticCubes = await staticApi.metaConfig(ctxFor('a'), {}); + const fnCubes = await fnApi.metaConfig(ctxFor('a'), {}); + expect(JSON.parse(JSON.stringify(fnCubes))).toEqual(JSON.parse(JSON.stringify(staticCubes))); + staticApi.dispose(); + fnApi.dispose(); + }); + + test('concurrent identical-config misses dedup to a single build', async () => { + const api = createApi({ granularities: perTenant }); + await Promise.all( + Array.from({ length: 10 }, () => api.metaConfig(ctxFor('a'), {})) + ); + expect(api.buildCount).toBe(1); + api.dispose(); + }); + + test('uncustomized time dimensions share the default set within a variant', async () => { + const api = createApi({ granularities: perTenant }); + const cubes = await api.metaConfig(ctxFor('a'), {}); + expect(dimByName(cubes, 'Orders.created_at').effectiveGranularities) + .toBe(dimByName(cubes, 'Orders.updated_at').effectiveGranularities); + // A cube without time dimensions is passed through by reference, not copied. + const compilers = await (api as any).getCompilers(); + const baseProducts = compilers.metaTransformer.cubes.find((c: any) => c.config.name === 'Products'); + const variantProducts = (await api.metaConfig(ctxFor('a'), {})) + .find((c: any) => c.config.name === 'Products'); + expect(variantProducts).toBe(baseProducts); + api.dispose(); + }); + + test('LRU eviction beyond the bound, with a logged warning and rebuild on re-request', async () => { + const capturedLogs: any[] = []; + const api = createApi({ + capturedLogs, + granularities: (ctx: any) => [{ name: `g_${ctx.securityContext.tenant}`, interval: '1 week' }], + }); + + for (let i = 0; i < 17; i++) { + await api.metaConfig(ctxFor(`t${i}`), {}); + } + expect(api.buildCount).toBe(17); + expect((await api.variantCache())!.size).toBe(16); + expect(capturedLogs.some(l => l.msg === 'Granularity variant cache is full')).toBe(true); + + // t0 was evicted (least recently used) — asking again rebuilds. + await api.metaConfig(ctxFor('t0'), {}); + expect(api.buildCount).toBe(18); + // t16 is still cached — no rebuild. + await api.metaConfig(ctxFor('t16'), {}); + expect(api.buildCount).toBe(18); + api.dispose(); + }); + + test('a failed variant build self-evicts and the next request retries', async () => { + const api = createApi({ granularities: perTenant }); + api.failNextBuild = true; + await expect(api.metaConfig(ctxFor('a'), {})).rejects.toThrow('injected variant build failure'); + const cubes = await api.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toEqual(['year', 'month']); + expect(api.buildCount).toBe(1); + api.dispose(); + }); + + test('a recompile discards the variant cache with the compilers object', async () => { + let version = 'v1'; + const api = createApi({ granularities: perTenant, schemaVersion: () => version }); + await api.metaConfig(ctxFor('a'), {}); + expect(api.buildCount).toBe(1); + + version = 'v2'; + await api.metaConfig(ctxFor('a'), {}); + expect(api.buildCount).toBe(2); + expect((await api.variantCache())!.size).toBe(1); + api.dispose(); + }); + }); + + describe('composition with RBAC visibility', () => { + const rbacRepository: SchemaFileRepository = { + localPath: () => '/mock/path', + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'orders.js', + content: ` + cube('Orders', { + sql: 'SELECT * FROM orders', + measures: { count: { type: 'count' } }, + dimensions: { + created_at: { sql: 'created_at', type: 'time' }, + secret: { sql: 'secret', type: 'string' }, + }, + accessPolicy: [ + { + group: '*', + rowLevel: { allowAll: true }, + memberLevel: { includes: ['count', 'created_at'] }, + }, + ], + }); + `, + }, + ]), + }; + + test('granularity variant selects first, visibility patches on top, compilerId mixes both', async () => { + const api = new TestableCompilerApi(rbacRepository, mockDbType, { + logger: noopLogger, + granularities: (ctx: any) => (ctx.securityContext.tenant === 'a' ? ['year'] : ['month']), + }); + + const result = await api.metaConfig(ctxFor('a'), { includeCompilerId: true }); + const createdAt = dimByName(result.cubes, 'Orders.created_at'); + expect(granularityNames(createdAt)).toEqual(['year']); + // RBAC hid `secret` but kept the enriched time dimension intact. + const secret = dimByName(result.cubes, 'Orders.secret'); + expect(secret.isVisible).toBe(false); + expect(createdAt.isVisible).toBe(true); + + // compilerId differs across tenants (granularity), and from the base (visibility + granularity). + const base = (await (api as any).getCompilers()).compilerId; + const resultB = await api.metaConfig(ctxFor('b'), { includeCompilerId: true }); + expect(result.compilerId).not.toBe(resultB.compilerId); + expect(result.compilerId).not.toBe(base); + api.dispose(); + }); + }); +}); From c57346e040c280fc89b8ee43da6da25c69e925a0 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 16 Jul 2026 22:28:02 +0200 Subject: [PATCH 08/22] refactor(granularities): raise variant cache bound to 64 to cover combinatorial calendar configs --- .../cubejs-server-core/src/core/CompilerApi.ts | 13 +++++++++---- .../test/unit/granularity-variants.test.ts | 15 ++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index 3fdaf171bc7b1..1cdb88102c601 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -98,11 +98,16 @@ export interface DataSourceInfo { export class CompilerApi { /** - * Bound on cached granularity-enriched meta variants per compiled model. Each entry costs - * O(model) memory; expected distinct-config cardinality for a `granularities` context - * function is 1–4, so this is headroom, not a target. Exceeding it evicts LRU and logs. + * Bound on cached granularity-enriched meta variants per compiled model. Legitimate + * distinct-config cardinality is the product of low-cardinality context facts (fiscal-year + * origins × locales × week conventions), so real working sets stay well under this; the bound + * exists to contain a `granularities` function accidentally keyed on something high-cardinality + * (a user id, a timestamp), which would otherwise pin one enriched meta copy (~2 MB on a + * 3k-cube model) per unique config forever. Exceeding it evicts LRU and logs. Kept below + * CubeSQL's LRU-100 compiler cache — distinct configs mint distinct compilerIds, and + * cardinality past that ceiling hurts downstream regardless of this cache. */ - protected static readonly MAX_GRANULARITY_VARIANTS = 16; + protected static readonly MAX_GRANULARITY_VARIANTS = 64; protected readonly repository: SchemaFileRepository; diff --git a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts index 9cd16ebf7fed8..1e0cd4890da41 100644 --- a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts +++ b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts @@ -233,25 +233,26 @@ describe('granularity variants in CompilerApi', () => { }); test('LRU eviction beyond the bound, with a logged warning and rebuild on re-request', async () => { + const bound = (CompilerApi as any).MAX_GRANULARITY_VARIANTS; const capturedLogs: any[] = []; const api = createApi({ capturedLogs, granularities: (ctx: any) => [{ name: `g_${ctx.securityContext.tenant}`, interval: '1 week' }], }); - for (let i = 0; i < 17; i++) { + for (let i = 0; i < bound + 1; i++) { await api.metaConfig(ctxFor(`t${i}`), {}); } - expect(api.buildCount).toBe(17); - expect((await api.variantCache())!.size).toBe(16); + expect(api.buildCount).toBe(bound + 1); + expect((await api.variantCache())!.size).toBe(bound); expect(capturedLogs.some(l => l.msg === 'Granularity variant cache is full')).toBe(true); // t0 was evicted (least recently used) — asking again rebuilds. await api.metaConfig(ctxFor('t0'), {}); - expect(api.buildCount).toBe(18); - // t16 is still cached — no rebuild. - await api.metaConfig(ctxFor('t16'), {}); - expect(api.buildCount).toBe(18); + expect(api.buildCount).toBe(bound + 2); + // The newest entry is still cached — no rebuild. + await api.metaConfig(ctxFor(`t${bound}`), {}); + expect(api.buildCount).toBe(bound + 2); api.dispose(); }); From 89c6dbdcf255993791d22acdc5ac4b97d7f4a807 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 16 Jul 2026 22:34:13 +0200 Subject: [PATCH 09/22] feat(server-core): make granularity variant cache bound configurable via CUBEJS_MAX_GRANULARITY_VARIANTS --- packages/cubejs-backend-shared/src/env.ts | 8 +++++ .../src/core/CompilerApi.ts | 31 ++++++++-------- .../test/unit/granularity-variants.test.ts | 35 +++++++++++-------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 3a44dd15a8ea8..761905ed545a9 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -2052,6 +2052,14 @@ const variables: Record any> = { // Comma-separated names (built-in or custom). Empty/unset = all 8 built-ins enabled. granularities: () => get('CUBEJS_GRANULARITIES') .asArray(','), + /** + * Max distinct `granularities` context-function configs cached as enriched meta variants + * per compiled model. A bound, not a target: contains a config function accidentally keyed + * on a high-cardinality context fact. Exceeding it evicts LRU and logs a warning. + */ + maxGranularityVariants: () => get('CUBEJS_MAX_GRANULARITY_VARIANTS') + .default(64) + .asInt(), // `getEnv` forwards `opts` positionally, so callers pass `{ name }` (matches dbType: { dataSource }). granularityCustomInterval: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_INTERVAL`) .asString(), diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index 1cdb88102c601..f6cbd0b93d04c 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -32,7 +32,7 @@ import { GraphQLSchema } from 'graphql'; import { parse as uuidParse, v4 as uuidv4 } from 'uuid'; import { LRUCache } from 'lru-cache'; import { NativeInstance } from '@cubejs-backend/native'; -import { disposedProxy } from '@cubejs-backend/shared'; +import { disposedProxy, getEnv } from '@cubejs-backend/shared'; import type { SchemaFileRepository } from '@cubejs-backend/shared'; import { NormalizedQuery, MemberExpression } from '@cubejs-backend/api-gateway'; import { DriverCapabilities } from '@cubejs-backend/base-driver'; @@ -98,16 +98,17 @@ export interface DataSourceInfo { export class CompilerApi { /** - * Bound on cached granularity-enriched meta variants per compiled model. Legitimate - * distinct-config cardinality is the product of low-cardinality context facts (fiscal-year - * origins × locales × week conventions), so real working sets stay well under this; the bound - * exists to contain a `granularities` function accidentally keyed on something high-cardinality - * (a user id, a timestamp), which would otherwise pin one enriched meta copy (~2 MB on a - * 3k-cube model) per unique config forever. Exceeding it evicts LRU and logs. Kept below - * CubeSQL's LRU-100 compiler cache — distinct configs mint distinct compilerIds, and - * cardinality past that ceiling hurts downstream regardless of this cache. + * Bound on cached granularity-enriched meta variants per compiled model + * (`CUBEJS_MAX_GRANULARITY_VARIANTS`, default 64). Legitimate distinct-config cardinality is + * the product of low-cardinality context facts (fiscal-year origins × locales × week + * conventions), so real working sets stay well under the default; the bound exists to contain + * a `granularities` function accidentally keyed on something high-cardinality (a user id, a + * timestamp), which would otherwise pin one enriched meta copy (~2 MB on a 3k-cube model) per + * unique config forever. Exceeding it evicts LRU and logs. The default sits below CubeSQL's + * LRU-100 compiler cache — distinct configs mint distinct compilerIds, and cardinality past + * that ceiling hurts downstream regardless of this cache. */ - protected static readonly MAX_GRANULARITY_VARIANTS = 64; + protected readonly maxGranularityVariants: number; protected readonly repository: SchemaFileRepository; @@ -177,6 +178,7 @@ export class CompilerApi { this.sqlCache = options.sqlCache; this.standalone = options.standalone; this.granularities = options.granularities; + this.maxGranularityVariants = getEnv('maxGranularityVariants'); this.nativeInstance = this.createNativeInstance(); // Caching stuff @@ -1150,13 +1152,14 @@ export class CompilerApi { cache.delete(granularityHash); cache.set(granularityHash, variant); } else { - if (cache.size >= CompilerApi.MAX_GRANULARITY_VARIANTS) { + if (cache.size >= this.maxGranularityVariants) { const oldest = cache.keys().next().value; cache.delete(oldest); this.logger('Granularity variant cache is full', { - warning: `More than ${CompilerApi.MAX_GRANULARITY_VARIANTS} distinct granularity configs seen for one compiled model; ` + - 'evicting the least recently used variant. A `granularities` function returning unstable values ' + - 'causes per-request meta rebuilds and churns compilerId-based caches (e.g. in CubeSQL).', + warning: `More than ${this.maxGranularityVariants} distinct granularity configs seen for one compiled model; ` + + 'evicting the least recently used variant (raise CUBEJS_MAX_GRANULARITY_VARIANTS if this cardinality is intended). ' + + 'A `granularities` function returning unstable values causes per-request meta rebuilds ' + + 'and churns compilerId-based caches (e.g. in CubeSQL).', }); } variant = Promise.resolve().then(() => this.buildGranularityVariant(compilers, config)); diff --git a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts index 1e0cd4890da41..3d1ac4a5b8bfa 100644 --- a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts +++ b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts @@ -232,28 +232,33 @@ describe('granularity variants in CompilerApi', () => { api.dispose(); }); - test('LRU eviction beyond the bound, with a logged warning and rebuild on re-request', async () => { - const bound = (CompilerApi as any).MAX_GRANULARITY_VARIANTS; + test('LRU eviction beyond the CUBEJS_MAX_GRANULARITY_VARIANTS bound, with a logged warning and rebuild on re-request', async () => { + const bound = 4; + process.env.CUBEJS_MAX_GRANULARITY_VARIANTS = String(bound); const capturedLogs: any[] = []; const api = createApi({ capturedLogs, granularities: (ctx: any) => [{ name: `g_${ctx.securityContext.tenant}`, interval: '1 week' }], }); - for (let i = 0; i < bound + 1; i++) { - await api.metaConfig(ctxFor(`t${i}`), {}); + try { + for (let i = 0; i < bound + 1; i++) { + await api.metaConfig(ctxFor(`t${i}`), {}); + } + expect(api.buildCount).toBe(bound + 1); + expect((await api.variantCache())!.size).toBe(bound); + expect(capturedLogs.some(l => l.msg === 'Granularity variant cache is full')).toBe(true); + + // t0 was evicted (least recently used) — asking again rebuilds. + await api.metaConfig(ctxFor('t0'), {}); + expect(api.buildCount).toBe(bound + 2); + // The newest entry is still cached — no rebuild. + await api.metaConfig(ctxFor(`t${bound}`), {}); + expect(api.buildCount).toBe(bound + 2); + } finally { + delete process.env.CUBEJS_MAX_GRANULARITY_VARIANTS; + api.dispose(); } - expect(api.buildCount).toBe(bound + 1); - expect((await api.variantCache())!.size).toBe(bound); - expect(capturedLogs.some(l => l.msg === 'Granularity variant cache is full')).toBe(true); - - // t0 was evicted (least recently used) — asking again rebuilds. - await api.metaConfig(ctxFor('t0'), {}); - expect(api.buildCount).toBe(bound + 2); - // The newest entry is still cached — no rebuild. - await api.metaConfig(ctxFor(`t${bound}`), {}); - expect(api.buildCount).toBe(bound + 2); - api.dispose(); }); test('a failed variant build self-evicts and the next request retries', async () => { From 46e8dc6c3ee70941867b24fae3bdd7cf7a5a5516 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 16 Jul 2026 22:41:57 +0200 Subject: [PATCH 10/22] refactor(granularities): drop CUBEJS_MAX_GRANULARITY_VARIANTS env var, keep fixed bound of 64, tighten comments --- .../src/helpers/prepare-annotation.ts | 10 ++-- packages/cubejs-backend-shared/src/env.ts | 8 --- .../src/compiler/CubeToMetaTransformer.ts | 29 +++-------- .../src/compiler/GranularityConfigHash.ts | 11 ++--- .../src/compiler/PrepareCompiler.ts | 12 ++--- .../src/core/CompilerApi.ts | 49 ++++++------------- .../test/unit/granularity-variants.test.ts | 35 ++++++------- 7 files changed, 50 insertions(+), 104 deletions(-) diff --git a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts index 8f391952b7186..0ef992c0d9a25 100644 --- a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts +++ b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts @@ -41,13 +41,9 @@ type ConfigItem = { granularities?: GranularityMeta[]; }; -/** - * Effective granularity meta for one queried time dimension, read from the (already granularity- - * enriched) meta config. Two fallbacks keep the annotation complete for granularities that - * execute but aren't in the dimension's effective set: - * - a built-in disabled by config is synthesized from `BUILT_IN_GRANULARITIES` defaults; - * - an unknown custom name yields `undefined` (never the deprecated legacy array). - */ +// Effective granularity meta for one queried time dimension, read from the enriched meta config. +// A queried granularity outside the effective set still annotates: disabled built-ins are +// synthesized from defaults; unknown customs yield undefined (never the deprecated legacy array). function resolveGranularityMeta( configMap: MetaConfigMap, dimension: string, diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 761905ed545a9..3a44dd15a8ea8 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -2052,14 +2052,6 @@ const variables: Record any> = { // Comma-separated names (built-in or custom). Empty/unset = all 8 built-ins enabled. granularities: () => get('CUBEJS_GRANULARITIES') .asArray(','), - /** - * Max distinct `granularities` context-function configs cached as enriched meta variants - * per compiled model. A bound, not a target: contains a config function accidentally keyed - * on a high-cardinality context fact. Exceeding it evicts LRU and logs a warning. - */ - maxGranularityVariants: () => get('CUBEJS_MAX_GRANULARITY_VARIANTS') - .default(64) - .asInt(), // `getEnv` forwards `opts` positionally, so callers pass `{ name }` (matches dbType: { dataSource }). granularityCustomInterval: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_INTERVAL`) .asString(), diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index 4977cc06fcb40..a161fa8ab520f 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -150,11 +150,7 @@ export type DimensionConfig = { * omits built-ins, global customs, and the `type` field. See DEPRECATION.md. */ granularities?: GranularityDefinition[]; - /** - * Reconciled granularity set (enabled built-ins + global customs + local customs) for time - * dimensions. Baked in at compile time for env/static global configs; attached per request - * by CompilerApi meta variants when `granularities` is a context function. - */ + /** Reconciled set for time dimensions: enabled built-ins + global customs + local customs. */ effectiveGranularities?: EffectiveGranularity[]; order?: 'asc' | 'desc'; key?: string; @@ -222,12 +218,8 @@ export class CubeToMetaTransformer implements CompilerInterface { private readonly granularitiesOption?: GranularitiesOption; - /** - * Per-dimension granularity inputs for time dimensions that customize their granularity set - * (a `granularities` block or local customs). Keyed by `cube.dimension`. Dimensions absent - * from this index use the config-wide default set. Consumed by `CompilerApi` to build - * context-dependent meta variants when `granularities` is a function; never serialized. - */ + // Inputs for time dimensions that customize their granularity set, keyed by `cube.dimension`; + // absent dims use the config-wide default. Read by CompilerApi variant builds; never serialized. public readonly granularityInputs: Map = new Map(); // Set during compile() for the context-independent config forms (env / static list); @@ -263,10 +255,9 @@ export class CubeToMetaTransformer implements CompilerInterface { public compile(_cubes: any[], errorReporter: ErrorReporter): void { this.granularityInputs.clear(); - // Env / static-list configs are context-independent, so the effective granularity sets are - // resolved once here and baked into the meta configs. The function form is never called at - // compile time (the compiled model is shared across security contexts); `CompilerApi` - // resolves it per request and enriches cached variants from `granularityInputs`. + // Env/static configs are resolved once here and baked in. The function form must never run + // at compile time (the compiled model is shared across security contexts) — CompilerApi + // resolves it per request from `granularityInputs`. if (typeof this.granularitiesOption === 'function') { this.staticGranularityState = null; } else { @@ -463,12 +454,8 @@ export class CubeToMetaTransformer implements CompilerInterface { }; } - /** - * Granularity-resolution inputs for one time dimension, or null when the dimension has no - * local customization and can use the config-wide default set. Local custom titles resolve - * here (short-title semantics, matching the legacy `granularities` array); every other field - * is projected from the raw definition so that e.g. `sql` never leaks into meta output. - */ + // Resolution inputs for one time dimension; null = no local customization, use the default set. + // Fields are projected from raw definitions so e.g. `sql` never leaks into meta output. private granularityInputsForDimension( cubeTitle: string, granularitiesObj: Record | undefined, diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts index 944a93676dcbd..8c0cc2c04203e 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts @@ -2,17 +2,14 @@ import crypto from 'crypto'; import type { GlobalGranularitiesConfig } from './GlobalGranularitiesConfig'; -// Emission-order-sensitive canonical form: `enabledBuiltIns` order and custom-granularity -// insertion order both affect the resolved set that clients receive, so neither is sorted. -// Only the known serializable fields participate; anything else a config spreads onto a -// definition (including functions) is ignored so the hash stays deterministic. +// Only known serializable fields participate, so the hash stays deterministic even when a +// config spreads extra props (e.g. functions) onto a definition. const asHashableString = (value: unknown): string | undefined => ( typeof value === 'string' ? value : undefined ); -// Canonical sha256 of a resolved global granularities config. Two configs share a hash iff -// they produce identical effective granularity sets, so the hash is safe to use as a cache -// key for enriched meta variants and as a compilerId discriminator. +// Canonical sha256 of a resolved global granularities config: equal hash iff identical effective +// sets. Order-sensitive on purpose — built-in and custom order affect the emitted meta. export function granularityConfigHash(config: GlobalGranularitiesConfig): string { const canonical = { builtIns: [...config.enabledBuiltIns], diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 615ba49face69..2814a5e984930 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -41,9 +41,8 @@ export type PrepareCompilerOptions = { compiledScriptCache?: LRUCache; compiledYamlCache?: LRUCache; compiledJinjaCache?: LRUCache; - // Global `granularities` config (env fallback / static list / context function). Env and - // static forms are resolved at compile time by CubeToMetaTransformer; the function form is - // resolved per request by CompilerApi. + // Global `granularities` config: env/static forms are resolved at compile time by + // CubeToMetaTransformer; the function form per request by CompilerApi. granularities?: GranularitiesOption; }; @@ -61,11 +60,8 @@ export type Compiler = { compilerCache: CompilerCache; headCommitId?: string; compilerId: string; - /** - * Granularity-enriched meta variants, keyed by canonical global-config hash. Owned by the - * compiled model (not CompilerApi) so a recompile discards it implicitly. Populated lazily - * by CompilerApi when `granularities` is a context function; bounded LRU. - */ + // Granularity-enriched meta variants keyed by config hash; owned by the compiled model so a + // recompile discards it. Populated lazily by CompilerApi (bounded LRU, function form only). granularityVariants?: Map>; }; diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index f6cbd0b93d04c..b0dd3f0a641c9 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -32,7 +32,7 @@ import { GraphQLSchema } from 'graphql'; import { parse as uuidParse, v4 as uuidv4 } from 'uuid'; import { LRUCache } from 'lru-cache'; import { NativeInstance } from '@cubejs-backend/native'; -import { disposedProxy, getEnv } from '@cubejs-backend/shared'; +import { disposedProxy } from '@cubejs-backend/shared'; import type { SchemaFileRepository } from '@cubejs-backend/shared'; import { NormalizedQuery, MemberExpression } from '@cubejs-backend/api-gateway'; import { DriverCapabilities } from '@cubejs-backend/base-driver'; @@ -97,18 +97,11 @@ export interface DataSourceInfo { } export class CompilerApi { - /** - * Bound on cached granularity-enriched meta variants per compiled model - * (`CUBEJS_MAX_GRANULARITY_VARIANTS`, default 64). Legitimate distinct-config cardinality is - * the product of low-cardinality context facts (fiscal-year origins × locales × week - * conventions), so real working sets stay well under the default; the bound exists to contain - * a `granularities` function accidentally keyed on something high-cardinality (a user id, a - * timestamp), which would otherwise pin one enriched meta copy (~2 MB on a 3k-cube model) per - * unique config forever. Exceeding it evicts LRU and logs. The default sits below CubeSQL's - * LRU-100 compiler cache — distinct configs mint distinct compilerIds, and cardinality past - * that ceiling hurts downstream regardless of this cache. - */ - protected readonly maxGranularityVariants: number; + // Bound on cached granularity-enriched meta variants per compiled model (~2 MB each on a + // 3k-cube model). Contains a `granularities` function keyed on a high-cardinality context + // fact; legitimate configs (calendars × locales) stay well under it. Exceeding evicts LRU + // and logs. Kept below CubeSQL's LRU-100 compilerId-keyed cache. + protected static readonly MAX_GRANULARITY_VARIANTS = 64; protected readonly repository: SchemaFileRepository; @@ -178,7 +171,6 @@ export class CompilerApi { this.sqlCache = options.sqlCache; this.standalone = options.standalone; this.granularities = options.granularities; - this.maxGranularityVariants = getEnv('maxGranularityVariants'); this.nativeInstance = this.createNativeInstance(); // Caching stuff @@ -1120,15 +1112,9 @@ export class CompilerApi { /** * Meta cubes with `effectiveGranularities` attached, plus the hash to mix into compilerId. - * - * Env/static configs are context-independent: the transformer bakes the effective sets into - * the base meta at compile time (their hash is folded into compilerVersion), so this returns - * the base cubes untouched with a null hash. - * - * A function config resolves per request. Enriched variants are cached on the compiled model - * keyed by the canonical config hash — a bounded promise-valued LRU, so concurrent misses of - * one hash dedup to a single O(model) build, a failed build self-evicts, and the whole cache - * is discarded with the compilers object on recompile. + * Env/static configs are baked into the base meta at compile time (null hash); a function + * config resolves per request against a bounded promise-valued LRU of enriched variants keyed + * by config hash and owned by the compiled model, so a recompile discards it. */ protected async selectGranularityVariant( compilers: Compiler, @@ -1152,14 +1138,13 @@ export class CompilerApi { cache.delete(granularityHash); cache.set(granularityHash, variant); } else { - if (cache.size >= this.maxGranularityVariants) { + if (cache.size >= CompilerApi.MAX_GRANULARITY_VARIANTS) { const oldest = cache.keys().next().value; cache.delete(oldest); this.logger('Granularity variant cache is full', { - warning: `More than ${this.maxGranularityVariants} distinct granularity configs seen for one compiled model; ` + - 'evicting the least recently used variant (raise CUBEJS_MAX_GRANULARITY_VARIANTS if this cardinality is intended). ' + - 'A `granularities` function returning unstable values causes per-request meta rebuilds ' + - 'and churns compilerId-based caches (e.g. in CubeSQL).', + warning: `More than ${CompilerApi.MAX_GRANULARITY_VARIANTS} distinct granularity configs seen for one compiled model; ` + + 'evicting the least recently used variant. A `granularities` function returning unstable values ' + + 'causes per-request meta rebuilds and churns compilerId-based caches (e.g. in CubeSQL).', }); } variant = Promise.resolve().then(() => this.buildGranularityVariant(compilers, config)); @@ -1175,11 +1160,9 @@ export class CompilerApi { } /** - * One O(model) pass attaching `effectiveGranularities` to every time dimension. Never mutates - * the base meta: enriched cubes get new cube/config/dimensions containers, while untouched - * members stay shared by reference (all downstream consumers — visibility patch, gateway - * filters — copy-on-write rather than mutate). Time dimensions without local customization - * share one default set per variant instead of allocating identical arrays per dimension. + * One O(model) pass attaching `effectiveGranularities` to every time dimension. Copies only + * what it changes — the base meta is never mutated, untouched members stay shared by + * reference, and uncustomized time dimensions share one default set per variant. */ protected buildGranularityVariant(compilers: Compiler, config: GlobalGranularitiesConfig): any[] { const catalog = buildBuiltInsCatalog(config); diff --git a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts index 3d1ac4a5b8bfa..1e0cd4890da41 100644 --- a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts +++ b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts @@ -232,33 +232,28 @@ describe('granularity variants in CompilerApi', () => { api.dispose(); }); - test('LRU eviction beyond the CUBEJS_MAX_GRANULARITY_VARIANTS bound, with a logged warning and rebuild on re-request', async () => { - const bound = 4; - process.env.CUBEJS_MAX_GRANULARITY_VARIANTS = String(bound); + test('LRU eviction beyond the bound, with a logged warning and rebuild on re-request', async () => { + const bound = (CompilerApi as any).MAX_GRANULARITY_VARIANTS; const capturedLogs: any[] = []; const api = createApi({ capturedLogs, granularities: (ctx: any) => [{ name: `g_${ctx.securityContext.tenant}`, interval: '1 week' }], }); - try { - for (let i = 0; i < bound + 1; i++) { - await api.metaConfig(ctxFor(`t${i}`), {}); - } - expect(api.buildCount).toBe(bound + 1); - expect((await api.variantCache())!.size).toBe(bound); - expect(capturedLogs.some(l => l.msg === 'Granularity variant cache is full')).toBe(true); - - // t0 was evicted (least recently used) — asking again rebuilds. - await api.metaConfig(ctxFor('t0'), {}); - expect(api.buildCount).toBe(bound + 2); - // The newest entry is still cached — no rebuild. - await api.metaConfig(ctxFor(`t${bound}`), {}); - expect(api.buildCount).toBe(bound + 2); - } finally { - delete process.env.CUBEJS_MAX_GRANULARITY_VARIANTS; - api.dispose(); + for (let i = 0; i < bound + 1; i++) { + await api.metaConfig(ctxFor(`t${i}`), {}); } + expect(api.buildCount).toBe(bound + 1); + expect((await api.variantCache())!.size).toBe(bound); + expect(capturedLogs.some(l => l.msg === 'Granularity variant cache is full')).toBe(true); + + // t0 was evicted (least recently used) — asking again rebuilds. + await api.metaConfig(ctxFor('t0'), {}); + expect(api.buildCount).toBe(bound + 2); + // The newest entry is still cached — no rebuild. + await api.metaConfig(ctxFor(`t${bound}`), {}); + expect(api.buildCount).toBe(bound + 2); + api.dispose(); }); test('a failed variant build self-evicts and the next request retries', async () => { From ceb4ae27dfa8832384b38d914471ebf7492e001e Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 16 Jul 2026 22:48:19 +0200 Subject: [PATCH 11/22] refactor(granularities): fold granularityConfigHash into GlobalGranularitiesConfig --- .../src/compiler/GlobalGranularitiesConfig.ts | 25 ++++++++++++++++++ .../src/compiler/GranularityConfigHash.ts | 26 ------------------- .../src/compiler/index.ts | 2 +- .../test/unit/granularity-config-hash.test.ts | 2 +- 4 files changed, 27 insertions(+), 28 deletions(-) delete mode 100644 packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts index 2e39afe7b3123..b73b8355bb110 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -1,3 +1,4 @@ +import crypto from 'crypto'; import { getEnv } from '@cubejs-backend/shared'; import type { GranularityDefinition } from './CubeSymbols'; @@ -168,3 +169,27 @@ export function buildBuiltInsCatalog(globalConfig: GlobalGranularitiesConfig): R } return catalog; } + +// Only known serializable fields participate, so the hash stays deterministic even when a +// config spreads extra props (e.g. functions) onto a definition. +const asHashableString = (value: unknown): string | undefined => ( + typeof value === 'string' ? value : undefined +); + +// Canonical sha256 of a resolved config: equal hash iff identical effective sets. Order-sensitive +// on purpose — built-in and custom order affect the emitted meta. Used as the meta-variant cache +// key and compilerId discriminator. +export function granularityConfigHash(config: GlobalGranularitiesConfig): string { + const canonical = { + builtIns: [...config.enabledBuiltIns], + custom: Object.entries(config.customGranularities).map(([name, def]) => ({ + name, + title: asHashableString(def.title), + format: asHashableString(def.format), + interval: asHashableString(def.interval), + offset: asHashableString(def.offset), + origin: asHashableString(def.origin), + })), + }; + return crypto.createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); +} diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts deleted file mode 100644 index 8c0cc2c04203e..0000000000000 --- a/packages/cubejs-schema-compiler/src/compiler/GranularityConfigHash.ts +++ /dev/null @@ -1,26 +0,0 @@ -import crypto from 'crypto'; - -import type { GlobalGranularitiesConfig } from './GlobalGranularitiesConfig'; - -// Only known serializable fields participate, so the hash stays deterministic even when a -// config spreads extra props (e.g. functions) onto a definition. -const asHashableString = (value: unknown): string | undefined => ( - typeof value === 'string' ? value : undefined -); - -// Canonical sha256 of a resolved global granularities config: equal hash iff identical effective -// sets. Order-sensitive on purpose — built-in and custom order affect the emitted meta. -export function granularityConfigHash(config: GlobalGranularitiesConfig): string { - const canonical = { - builtIns: [...config.enabledBuiltIns], - custom: Object.entries(config.customGranularities).map(([name, def]) => ({ - name, - title: asHashableString(def.title), - format: asHashableString(def.format), - interval: asHashableString(def.interval), - offset: asHashableString(def.offset), - origin: asHashableString(def.origin), - })), - }; - return crypto.createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); -} diff --git a/packages/cubejs-schema-compiler/src/compiler/index.ts b/packages/cubejs-schema-compiler/src/compiler/index.ts index 12067767c8460..691eaab9f7d32 100644 --- a/packages/cubejs-schema-compiler/src/compiler/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/index.ts @@ -25,6 +25,7 @@ export { resolveGlobalGranularitiesSync, getBuiltInGranularityDefaults, buildBuiltInsCatalog, + granularityConfigHash, } from './GlobalGranularitiesConfig'; export { NormalizedGranularitiesBlock, @@ -34,4 +35,3 @@ export { resolveDimensionGranularities, serializeEffectiveGranularities, } from './GranularityResolver'; -export { granularityConfigHash } from './GranularityConfigHash'; diff --git a/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts b/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts index 136ffd97a42e3..c17a6991cfa7c 100644 --- a/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts @@ -1,4 +1,4 @@ -import { granularityConfigHash } from '../../src/compiler/GranularityConfigHash'; +import { granularityConfigHash } from '../../src/compiler/GlobalGranularitiesConfig'; import type { GlobalGranularitiesConfig } from '../../src/compiler/GlobalGranularitiesConfig'; const config = ( From 2a3e2c6da8ff8b6c78df2dd8420c9d4d2e1a92cb Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 16 Jul 2026 23:41:45 +0200 Subject: [PATCH 12/22] fix(granularities): make global custom granularities queryable in SQL; harden resolution (proto keys, built-in interval, view block, validator, config validation, hash) --- .../src/adapter/BaseQuery.js | 12 +++- .../src/adapter/Granularity.ts | 13 ++++- .../src/compiler/CubeSymbols.ts | 50 ++++++++++++++-- .../src/compiler/CubeValidator.ts | 12 ++-- .../src/compiler/GlobalGranularitiesConfig.ts | 58 +++++++++++++------ .../test/unit/cube-validator.test.ts | 10 ++++ .../test/unit/granularities-config.test.ts | 53 ++++++++++++++++- .../test/unit/views.test.ts | 46 +++++++++++++++ .../src/core/CompilerApi.ts | 55 +++++++++++++++++- .../test/unit/granularity-variants.test.ts | 45 ++++++++++++++ 10 files changed, 320 insertions(+), 34 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 162fb2290b202..73e9918735247 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -139,11 +139,11 @@ export class BaseQuery { */ constructor(compilers, options) { this.compilers = compilers; + this.options = options || {}; /** @type {import('../compiler/CubeEvaluator').CubeEvaluator} */ - this.cubeEvaluator = compilers.cubeEvaluator; + this.cubeEvaluator = compilers.cubeEvaluator.withGranularityDefinitions(this.options.granularityDefinitions); /** @type {import('../compiler/JoinGraph').JoinGraph} */ this.joinGraph = compilers.joinGraph; - this.options = options || {}; this.orderHashToString = this.orderHashToString.bind(this); this.defaultOrder = this.defaultOrder.bind(this); @@ -270,6 +270,7 @@ export class BaseQuery { segments: this.options.segments, order: this.options.order, contextSymbols: this.options.contextSymbols, + granularityDefinitions: this.options.granularityDefinitions, timezone: this.options.timezone, limit: this.options.limit, offset: this.options.offset, @@ -4208,6 +4209,7 @@ export class BaseQuery { preAggregationQuery: this.options.preAggregationQuery, useOriginalSqlPreAggregationsInPreAggregation: this.options.useOriginalSqlPreAggregationsInPreAggregation, contextSymbols: this.contextSymbols, + granularityDefinitions: this.options.granularityDefinitions, preAggregationsSchema: this.preAggregationsSchemaOption, cubeLatticeCache: this.options.cubeLatticeCache, historyQueries: this.options.historyQueries, @@ -4364,7 +4366,11 @@ export class BaseQuery { if (path.length === 3 && this.cubeEvaluator.isDimension(path.slice(0, 2))) { const dimensionDef = this.cubeEvaluator.dimensionByPath(path.slice(0, 2)); if (dimensionDef.type === 'time' && - this.cubeEvaluator.resolveGranularity([path[0], path[1], 'granularities', path[2]])) { + this.cubeEvaluator.resolveGranularity( + [path[0], path[1], 'granularities', path[2]], + undefined, + this.options.granularityDefinitions + )) { const td = this.newTimeDimension({ dimension: `${path[0]}.${path[1]}`, granularity: path[2], diff --git a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts index 4e7d452fd6d99..080efdb856631 100644 --- a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts +++ b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts @@ -39,9 +39,18 @@ export class Granularity { this.granularityInterval = `1 ${this.granularity}`; } else { const customGranularity = this.query.cacheValue( - ['customGranularity', timeDimension.dimension, this.granularity], + [ + 'customGranularity', + timeDimension.dimension, + this.granularity, + JSON.stringify(query.options.granularityDefinitions?.[timeDimension.dimension]?.[this.granularity] || null), + ], () => query.cubeEvaluator - .resolveGranularity([...query.cubeEvaluator.parsePath('dimensions', timeDimension.dimension), 'granularities', this.granularity]) + .resolveGranularity( + [...query.cubeEvaluator.parsePath('dimensions', timeDimension.dimension), 'granularities', this.granularity], + undefined, + query.options.granularityDefinitions, + ) ); if (!customGranularity) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 506603d37a088..9aed28930babc 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -618,7 +618,11 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface } for (const dim of Object.values(dimensions)) { - if (dim && dim.type === 'time' && 'granularities' in dim) { + // A view's included dimension already carries a propagated granularitiesBlock (with the + // source dimension's includes/excludes) alongside the custom-only `granularities` map. + // Re-normalizing the custom-only map here would reset includes to '*' and drop the source's + // includes/excludes, so only normalize dimensions that haven't been normalized yet. + if (dim && dim.type === 'time' && 'granularities' in dim && !dim.granularitiesBlock) { // Keep the raw user value for the validator (it runs after this and would otherwise only // see the extracted customs, never the includes/excludes/custom dict). dim.rawGranularities = dim.granularities; @@ -1523,7 +1527,11 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface } if (refProperty && cube[refProperty].type === 'time' && - self.resolveGranularity([cubeName, refProperty, 'granularities', propertyName], cube) + self.resolveGranularity( + [cubeName, refProperty, 'granularities', propertyName], + cube, + query?.options?.granularityDefinitions + ) ) { return { toString: () => this.withSymbolsCallContext( @@ -1554,7 +1562,11 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface * @param {string|string[]} path * @param [refCube] Optional cube object to operate on */ - public resolveGranularity(path: string | string[], refCube?: any) { + public resolveGranularity( + path: string | string[], + refCube?: any, + granularityDefinitions?: Record>, + ) { const [cubeName, dimName, gr, granName] = Array.isArray(path) ? path : path.split('.'); const cube = refCube || this.symbols[cubeName]; @@ -1572,7 +1584,37 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface return { interval: `1 ${granName}` }; } - return cube?.[dimName]?.[gr]?.[granName]; + return cube?.[dimName]?.[gr]?.[granName] || + granularityDefinitions?.[`${cubeName}.${dimName}`]?.[granName]; + } + + /** + * Returns a request-owned evaluator facade that keeps all state and method execution on the + * shared evaluator, while binding custom-granularity fallback to one request's effective set. + * The native SQL planner calls resolveGranularity on the evaluator bridge directly, so the + * request context has to live at this seam rather than only at JS adapter call sites. + */ + public withGranularityDefinitions( + granularityDefinitions?: Record>, + ): this { + if (!granularityDefinitions || Object.keys(granularityDefinitions).length === 0) { + return this; + } + + const evaluator = this; + return new Proxy(this, { + get(target, property) { + if (property === 'resolveGranularity') { + return (path: string | string[], refCube?: any) => evaluator.resolveGranularity( + path, + refCube, + granularityDefinitions, + ); + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); } protected cubeDependenciesProxy(parentIndex, cubeName) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index 2a56a79ca9ebd..93573fe396690 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -187,11 +187,15 @@ const GranularityInclusionListSchema = Joi.alternatives([ // real pipeline validates, since normalization replaces `granularities` with the custom map first). const GranularitiesFieldSchema = Joi.alternatives() .conditional(Joi.ref('.'), { - // Only the new dict form has exclusively includes/excludes/custom keys. + // Dict form iff keys are a subset of includes/excludes/custom AND their VALUES have the dict + // shape (includes/excludes are '*' or arrays, custom is a plain object). The value check + // mirrors normalizeGranularitiesBlock so a legacy custom granularity literally named + // `includes`/`excludes`/`custom` — whose value is a granularity DEFINITION object — is not + // misread as the dict form and rejected. is: Joi.object().keys({ - includes: Joi.any(), - excludes: Joi.any(), - custom: Joi.any(), + includes: Joi.alternatives(Joi.string().valid('*'), Joi.array()), + excludes: Joi.alternatives(Joi.string().valid('*'), Joi.array()), + custom: Joi.object().pattern(Joi.string(), Joi.object()), }).unknown(false), then: Joi.object().keys({ includes: GranularityInclusionListSchema, diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts index b73b8355bb110..077602af074da 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -25,7 +25,9 @@ export const BUILT_IN_GRANULARITIES: Readonly 0) { + customGranularities[trimmed] = override; + } } else { // Non-built-in name: pull the definition from `CUBEJS_GRANULARITIES__*` env vars. // Skip if no interval was provided — a custom granularity without an interval is unusable. @@ -86,6 +94,20 @@ function resolveFromEnv(): GlobalGranularitiesConfig { return { enabledBuiltIns, customGranularities }; } +// Project a definition onto exactly the known string fields, dropping anything else (functions, +// nested objects, numbers). Guarantees every value the serializer emits and the hash reads is a +// plain string, so the two can never diverge (no cross-tenant cache-key collision), and stray +// props a config spreads in can't leak onto the wire. +function sanitizeDefinition(def: Partial): GranularityDefinition { + const out: GranularityDefinition = {}; + for (const key of ['title', 'format', 'interval', 'offset', 'origin'] as const) { + if (typeof def[key] === 'string') { + out[key] = def[key]; + } + } + return out; +} + function resolveFromList(list: GranularityList): GlobalGranularitiesConfig { const enabledBuiltIns: string[] = []; const customGranularities: Record = {}; @@ -99,12 +121,15 @@ function resolveFromList(list: GranularityList): GlobalGranularitiesConfig { // custom granularities in `config.granularities` must be objects. } else if (item && typeof item === 'object' && item.name) { const { name, ...def } = item; + const clean = sanitizeDefinition(def); if (isBuiltInGranularity(name)) { // `{ name: 'year', title: 'Anno' }` both enables 'year' and overrides its title/format. enabledBuiltIns.push(name); - customGranularities[name] = def; - } else { - customGranularities[name] = def; + customGranularities[name] = clean; + } else if (clean.interval !== undefined) { + // A custom granularity without an interval is unusable (SQL can't bucket it) and must not + // be advertised — drop it rather than exposing a granularity that fails at query time. + customGranularities[name] = clean; } } } @@ -163,32 +188,31 @@ export function buildBuiltInsCatalog(globalConfig: GlobalGranularitiesConfig): R catalog[name] = { title: override?.title || defaults.title, format: override?.format || defaults.format, - interval: override?.interval || `1 ${name}`, + // Built-in interval is fixed at `1 ` — the SQL layer always buckets predefined + // granularities that way, so an override interval must NOT be advertised (it would lie + // about how the data is bucketed). Title/format overrides are display-only and safe. + interval: `1 ${name}`, }; } } return catalog; } -// Only known serializable fields participate, so the hash stays deterministic even when a -// config spreads extra props (e.g. functions) onto a definition. -const asHashableString = (value: unknown): string | undefined => ( - typeof value === 'string' ? value : undefined -); - // Canonical sha256 of a resolved config: equal hash iff identical effective sets. Order-sensitive // on purpose — built-in and custom order affect the emitted meta. Used as the meta-variant cache -// key and compilerId discriminator. +// key and compilerId discriminator. Values are already sanitized to strings at resolution time, +// so the projected fields here are exactly what the serializer emits — the hash can never +// disagree with the wire output (which would let two different tenant configs share one variant). export function granularityConfigHash(config: GlobalGranularitiesConfig): string { const canonical = { builtIns: [...config.enabledBuiltIns], custom: Object.entries(config.customGranularities).map(([name, def]) => ({ name, - title: asHashableString(def.title), - format: asHashableString(def.format), - interval: asHashableString(def.interval), - offset: asHashableString(def.offset), - origin: asHashableString(def.origin), + title: def.title, + format: def.format, + interval: def.interval, + offset: def.offset, + origin: def.origin, })), }; return crypto.createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); diff --git a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts index f8ab629974f6e..222f82ac24f52 100644 --- a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts @@ -1801,6 +1801,16 @@ describe('Cube Validation', () => { const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); expect(validationResult.error).toBeFalsy(); }); + + // Regression: a legacy custom granularity literally named includes/excludes/custom (value is a + // definition object) must be accepted as the legacy map, not misread as the dict form. Mirrors + // the value-shape disambiguation in normalizeGranularitiesBlock. + it.each(['includes', 'excludes', 'custom'])('accepts a legacy custom granularity named "%s"', (name) => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ [name]: { interval: '1 year', origin: '2026-04-01' } }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); }); describe('Access Policy group/groups support:', () => { diff --git a/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts b/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts index be38fae4b89b6..bcb816ddeab17 100644 --- a/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts @@ -1,4 +1,10 @@ -import { resolveGlobalGranularities, BUILT_IN_GRANULARITY_NAMES } from '../../src/compiler/GlobalGranularitiesConfig'; +import { + resolveGlobalGranularities, + buildBuiltInsCatalog, + granularityConfigHash, + isBuiltInGranularity, + BUILT_IN_GRANULARITY_NAMES, +} from '../../src/compiler/GlobalGranularitiesConfig'; describe('resolveGlobalGranularities', () => { const originalEnv = { ...process.env }; @@ -72,4 +78,49 @@ describe('resolveGlobalGranularities', () => { ); expect(cfg.enabledBuiltIns).toEqual(['quarter']); }); + + // Regression: CUBEJS_GRANULARITIES__TITLE was dropped for built-in names. + it('env title/format override applies to a built-in (folded into catalog, stays built-in)', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,month'; + process.env.CUBEJS_GRANULARITIES_YEAR_TITLE = 'Jaar'; + const cfg = await resolveGlobalGranularities(undefined, {}); + expect(cfg.enabledBuiltIns).toEqual(['year', 'month']); + const catalog = buildBuiltInsCatalog(cfg); + expect(catalog.year.title).toBe('Jaar'); + expect(catalog.year.interval).toBe('1 year'); + }); + + // Regression: a config-provided interval must not override a built-in's fixed 1-unit interval. + it('interval override on a built-in is ignored (SQL always buckets 1 unit)', async () => { + const cfg = await resolveGlobalGranularities([{ name: 'month', interval: '2 months', title: 'Bi' }], {}); + const catalog = buildBuiltInsCatalog(cfg); + expect(catalog.month.interval).toBe('1 month'); + expect(catalog.month.title).toBe('Bi'); + }); + + // Regression: a config custom without an interval is unusable and must be dropped, not advertised. + it('drops a config custom granularity that has no interval', async () => { + const cfg = await resolveGlobalGranularities([{ name: 'fiscal_year', title: 'Fiscal Year' }], {}); + expect(cfg.customGranularities.fiscal_year).toBeUndefined(); + }); + + // Regression: non-string definition values must not survive (hash/serialize must agree). + it('sanitizes non-string custom fields so the config hash matches the wire output', async () => { + const cfgClean = await resolveGlobalGranularities([{ name: 'fy', interval: '1 year', origin: '2024-02-01' }], {}); + const cfgDirty = await resolveGlobalGranularities( + [{ name: 'fy', interval: '1 year', origin: '2024-02-01', title: 123 as any, junk: () => 'x' } as any], + {}, + ); + expect(cfgDirty.customGranularities.fy).toEqual({ interval: '1 year', origin: '2024-02-01' }); + // Two configs that serialize identically must hash identically. + expect(granularityConfigHash(cfgDirty)).toBe(granularityConfigHash(cfgClean)); + }); + + // Regression (adversarial): prototype-chain names must not be classified as built-ins. + it('does not classify prototype-chain names as built-in granularities', () => { + expect(isBuiltInGranularity('__proto__')).toBe(false); + expect(isBuiltInGranularity('constructor')).toBe(false); + expect(isBuiltInGranularity('hasOwnProperty')).toBe(false); + expect(isBuiltInGranularity('year')).toBe(true); + }); }); diff --git a/packages/cubejs-schema-compiler/test/unit/views.test.ts b/packages/cubejs-schema-compiler/test/unit/views.test.ts index 41e9b08e92025..f54299de78126 100644 --- a/packages/cubejs-schema-compiler/test/unit/views.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/views.test.ts @@ -589,4 +589,50 @@ describe('Views YAML', () => { await expect(compiler.compile()).rejects.toThrow('test_view cube: Member \'unknown\''); }); + + // Regression: a time dimension restricted via the dict form (includes/excludes) must keep that + // restriction when included in a view. The view dimension carries a propagated granularitiesBlock; + // re-normalizing the custom-only map used to reset includes to '*' and expose every built-in. + it('preserves a source time dimension includes/excludes when included in a view', async () => { + const { compiler, cubeEvaluator, metaTransformer } = prepareYamlCompiler(` + cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: created_at + sql: created_at + type: time + granularities: + includes: + - year + - month + + views: + - name: orders_view + cubes: + - join_path: orders + includes: + - count + - created_at + `); + + await compiler.compile(); + + // The compiled view dimension keeps the source's includes, not the reset '*'. + const viewDim = cubeEvaluator.getCubeDefinition('orders_view').dimensions!.created_at; + expect(viewDim.granularitiesBlock).toBeDefined(); + expect(viewDim.granularitiesBlock!.includes).toEqual(['year', 'month']); + + // And the meta reflects only year + month (no other built-ins leak through the view). + const viewMeta = metaTransformer.cubes.map((d) => d.config).find((d) => d.name === 'orders_view'); + const metaDim: any = viewMeta!.dimensions.find((d: any) => d.name === 'orders_view.created_at'); + expect(metaDim.effectiveGranularities.map((g: any) => g.name)).toEqual(['year', 'month']); + }); }); diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index b0dd3f0a641c9..916fdb7112b7f 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -342,7 +342,9 @@ export class CompilerApi { public async getSqlGenerator(query: NormalizedQuery, dataSource?: string): Promise<{ sqlGenerator: any; compilers: Compiler }> { const dbType = await this.getDbType(dataSource); const compilers = await this.getCompilers({ requestId: query.requestId }); - let sqlGenerator = await this.createQueryByDataSource(compilers, query, dataSource, dbType); + const granularityDefinitions = await this.granularityDefinitionsForQuery(compilers, query); + const queryWithGranularities = { ...query, granularityDefinitions }; + let sqlGenerator = await this.createQueryByDataSource(compilers, queryWithGranularities, dataSource, dbType); if (!sqlGenerator) { throw new Error(`Unknown dbType: ${dbType}`); @@ -358,7 +360,7 @@ export class CompilerApi { // TODO consider more efficient way than instantiating query sqlGenerator = await this.createQueryByDataSource( compilers, - query, + queryWithGranularities, dataSource, _dbType ); @@ -374,6 +376,49 @@ export class CompilerApi { return { sqlGenerator, compilers }; } + /** + * Request-scoped custom granularities keyed by time dimension. The shared cube evaluator cannot + * contain function-config results because it is reused across security contexts, so query-time + * resolution receives this immutable lookup instead. Building it through the same resolver and + * per-dimension inputs as meta keeps includes/excludes behavior identical on both paths. + */ + protected async granularityDefinitionsForQuery( + compilers: Compiler, + query: NormalizedQuery, + ): Promise>> { + const { contextSymbols } = query as any; + const requestContext = { + securityContext: contextSymbols?.securityContext || {}, + requestId: query.requestId, + }; + const config = await resolveGlobalGranularities(this.granularities, requestContext); + const catalog = buildBuiltInsCatalog(config); + const inputs = compilers.metaTransformer.granularityInputs; + const definitions: Record> = {}; + + for (const cube of compilers.metaTransformer.cubes) { + for (const dimension of (cube.config.dimensions || []).filter((d: any) => d.type === 'time')) { + const resolved = resolveDimensionGranularities( + inputs.get(dimension.name) || normalizeGranularitiesBlock(undefined), + config.enabledBuiltIns, + config.customGranularities, + catalog, + ); + const customs = Object.fromEntries( + Object.entries(resolved) + .filter(([name, definition]) => definition.type === 'custom' && + Object.prototype.hasOwnProperty.call(config.customGranularities, name)) + .map(([name, { type: _type, ...definition }]) => [name, definition]) + ); + if (Object.keys(customs).length > 0) { + definitions[dimension.name] = customs; + } + } + } + + return definitions; + } + public async getSql(query: NormalizedQuery, options: GetSqlOptions = {}): Promise { const { includeDebugInfo, exportAnnotatedSql, preAggregationsOnly } = options; const { sqlGenerator, compilers } = await this.getSqlGenerator(query); @@ -398,7 +443,11 @@ export class CompilerApi { if (this.sqlCache) { // eslint-disable-next-line @typescript-eslint/no-unused-vars const { requestId, ...keyOptions } = query; - const key = { query: keyOptions, options }; + const key = { + query: keyOptions, + options, + granularityDefinitions: sqlGenerator.options.granularityDefinitions, + }; return compilers.compilerCache.getQueryCache(key).cache(['sql'], getSqlFn); } else { return getSqlFn(); diff --git a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts index 1e0cd4890da41..3fb3bf1c3c49d 100644 --- a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts +++ b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts @@ -39,6 +39,11 @@ const repository: SchemaFileRepository = { id: { sql: 'id', type: 'number', primaryKey: true }, created_at: { sql: 'created_at', type: 'time' }, updated_at: { sql: 'updated_at', type: 'time' }, + excluded_at: { + sql: 'updated_at', + type: 'time', + granularities: { excludes: ['fiscal_year', 'sprint'] }, + }, }, }); cube('Events', { @@ -87,6 +92,14 @@ const granularityNames = (dim: any) => dim.effectiveGranularities.map((g: any) = const ALL_BUILT_INS = ['year', 'quarter', 'month', 'week', 'day', 'hour', 'minute', 'second']; +const queryFor = (tenant: string, dimension: string, granularity: string): any => ({ + measures: [dimension.startsWith('Events.') ? 'Events.count' : 'Orders.count'], + timeDimensions: [{ dimension, granularity }], + timezone: 'UTC', + contextSymbols: { securityContext: { tenant } }, + requestId: `sql-${tenant}-${dimension}-${granularity}`, +}); + describe('granularity variants in CompilerApi', () => { describe('env/static configs (baked at compile time)', () => { afterEach(() => { @@ -126,6 +139,24 @@ describe('granularity variants in CompilerApi', () => { api.dispose(); }); + test('static global customs resolve in SQL, while dimension excludes and legacy behavior remain intact', async () => { + const api = createApi({ + granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-02-01' }], + }); + + const globalSql = await api.getSql(queryFor('a', 'Orders.created_at', 'fiscal_year')); + expect(globalSql.sql[0]).toContain('created_at'); + await expect(api.getSql(queryFor('a', 'Orders.excluded_at', 'fiscal_year'))) + .rejects.toThrow('Granularity "fiscal_year" does not exist in dimension Orders.excluded_at'); + + // Existing local customs and predefined granularities still use their original paths. + const localSql = await api.getSql(queryFor('a', 'Events.ts', 'fiscal_year')); + expect(localSql.sql[0]).toContain('ts'); + const builtInSql = await api.getSql(queryFor('a', 'Orders.updated_at', 'day')); + expect(builtInSql.sql[0]).toContain('updated_at'); + api.dispose(); + }); + test('time dimensions without customization share one default set instance', async () => { const api = createApi(); const cubes = await api.metaConfig(ctxFor('a'), {}); @@ -175,6 +206,20 @@ describe('granularity variants in CompilerApi', () => { api.dispose(); }); + test('context-function global customs resolve in SQL without crossing dimension exclusions', async () => { + const api = createApi({ granularities: perTenant }); + + await expect(api.getSql(queryFor('a', 'Orders.created_at', 'sprint'))) + .rejects.toThrow('Granularity "sprint" does not exist in dimension Orders.created_at'); + const sql = await api.getSql(queryFor('b', 'Orders.created_at', 'sprint')); + expect(sql.sql[0]).toContain('created_at'); + await expect(api.getSql(queryFor('b', 'Orders.excluded_at', 'sprint'))) + .rejects.toThrow('Granularity "sprint" does not exist in dimension Orders.excluded_at'); + await expect(api.getSql(queryFor('a', 'Orders.created_at', 'sprint'))) + .rejects.toThrow('Granularity "sprint" does not exist in dimension Orders.created_at'); + api.dispose(); + }); + test('base meta cubes are never mutated by variant enrichment', async () => { const api = createApi({ granularities: perTenant }); await api.metaConfig(ctxFor('a'), {}); From 7adba3279c30d19c592e026dd6c0df9b86b64159 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Fri, 17 Jul 2026 10:06:34 +0200 Subject: [PATCH 13/22] fix(granularities): resolve config from securityContext on both meta and SQL paths; drop empty-interval env customs; non-array function return uses default catalog --- .../src/compiler/GlobalGranularitiesConfig.ts | 16 ++++++--- .../test/unit/granularities-config.test.ts | 17 ++++++++++ .../src/core/CompilerApi.ts | 34 ++++++++++++++----- .../test/unit/granularity-variants.test.ts | 13 +++++++ 4 files changed, 67 insertions(+), 13 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts index 077602af074da..54c5252fa3de0 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -49,10 +49,13 @@ const DEFAULT_CONFIG: GlobalGranularitiesConfig = Object.freeze({ function applyEnvOverrides(name: string, base?: Partial): GranularityDefinition { // getEnv types `opts` as a Parameters<> tuple but forwards it positionally; cast to bypass that. const opts = { name } as any; - const interval = getEnv('granularityCustomInterval', opts) ?? base?.interval; - const title = getEnv('granularityCustomTitle', opts) ?? base?.title; - const offset = getEnv('granularityCustomOffset', opts) ?? base?.offset; - const origin = getEnv('granularityCustomOrigin', opts) ?? base?.origin; + // A set-but-empty env var (e.g. `CUBEJS_GRANULARITIES_FOO_INTERVAL=`) reads as '' — treat that + // as absent so an unusable empty interval isn't advertised and later fails at query time. + const nonEmpty = (v: string | undefined) => (v === undefined || v === '' ? undefined : v); + const interval = nonEmpty(getEnv('granularityCustomInterval', opts)) ?? base?.interval; + const title = nonEmpty(getEnv('granularityCustomTitle', opts)) ?? base?.title; + const offset = nonEmpty(getEnv('granularityCustomOffset', opts)) ?? base?.offset; + const origin = nonEmpty(getEnv('granularityCustomOrigin', opts)) ?? base?.origin; const out: GranularityDefinition = {}; if (interval !== undefined) out.interval = interval; @@ -161,7 +164,10 @@ export async function resolveGlobalGranularities( ): Promise { if (typeof userValue === 'function') { const resolved = await userValue(ctx); - return resolveGlobalGranularitiesSync(Array.isArray(resolved) ? resolved : undefined); + // A function opts out of env vars entirely, so a non-array return (null / undefined / a stray + // object) means "no explicit config" → the default built-in catalog, NOT an env fallback that + // would leak CUBEJS_GRANULARITIES into a context the function meant to leave unconfigured. + return Array.isArray(resolved) ? resolveFromList(resolved) : DEFAULT_CONFIG; } return resolveGlobalGranularitiesSync(userValue); } diff --git a/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts b/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts index bcb816ddeab17..08a08b2ff4206 100644 --- a/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts @@ -116,6 +116,23 @@ describe('resolveGlobalGranularities', () => { expect(granularityConfigHash(cfgDirty)).toBe(granularityConfigHash(cfgClean)); }); + // Regression: a set-but-empty interval env var must be treated as absent (unusable), not advertised. + it('drops a custom granularity whose env interval is set-but-empty', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,foo'; + process.env.CUBEJS_GRANULARITIES_FOO_INTERVAL = ''; + const cfg = await resolveGlobalGranularities(undefined, {}); + expect(cfg.enabledBuiltIns).toEqual(['year']); + expect(cfg.customGranularities.foo).toBeUndefined(); + }); + + // Regression: a function returning a non-array opts out of env (no leak), yielding the default catalog. + it('function returning a non-array yields the default catalog, not an env fallback', async () => { + process.env.CUBEJS_GRANULARITIES = 'week'; + const cfg = await resolveGlobalGranularities(() => null as any, {}); + expect([...cfg.enabledBuiltIns].sort()).toEqual([...BUILT_IN_GRANULARITY_NAMES].sort()); + expect(cfg.customGranularities).toEqual({}); + }); + // Regression (adversarial): prototype-chain names must not be classified as built-ins. it('does not classify prototype-chain names as built-in granularities', () => { expect(isBuiltInGranularity('__proto__')).toBe(false); diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index 916fdb7112b7f..b46972afeead7 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -386,15 +386,21 @@ export class CompilerApi { compilers: Compiler, query: NormalizedQuery, ): Promise>> { + // Resolve through the same securityContext-only seam as the meta path so both paths agree on + // the effective set — a `granularities` function must key only on securityContext. const { contextSymbols } = query as any; - const requestContext = { - securityContext: contextSymbols?.securityContext || {}, - requestId: query.requestId, - }; - const config = await resolveGlobalGranularities(this.granularities, requestContext); + const config = await this.resolveGranularities({ securityContext: contextSymbols?.securityContext }); + const definitions: Record> = {}; + + // Only GLOBAL customs need threading (locals already resolve via the compiled symbols map). With + // none configured there's nothing to add, so skip the O(model) scan — the common case allocates + // nothing and query-time resolution falls straight through to the symbols map. + if (Object.keys(config.customGranularities).length === 0) { + return definitions; + } + const catalog = buildBuiltInsCatalog(config); const inputs = compilers.metaTransformer.granularityInputs; - const definitions: Record> = {}; for (const cube of compilers.metaTransformer.cubes) { for (const dimension of (cube.config.dimensions || []).filter((d: any) => d.type === 'time')) { @@ -1151,12 +1157,24 @@ export class CompilerApi { return this.mixInMaskHash(compilerId, visibilityMaskHash); } + /** + * Resolve the global granularity config for a request. A `granularities` function may depend + * only on `securityContext` (like queryRewrite and access policies): the meta path and the SQL + * path receive different-shaped request objects, but both carry the security context, so keying + * on it — and nothing else — is what guarantees the two paths resolve the SAME config and never + * advertise a granularity that then fails at query time. + */ + private async resolveGranularities(context: Context): Promise { + const securityContext = context?.securityContext ?? {}; + return resolveGlobalGranularities(this.granularities, { securityContext }); + } + /** * Global granularity config for a request context. O(config): env/static forms ignore the * context; the function form is invoked with it. Used by the /v1/granularities endpoint. */ public async resolveGlobalGranularitiesConfig(context: Context): Promise { - return resolveGlobalGranularities(this.granularities, context); + return this.resolveGranularities(context); } /** @@ -1173,7 +1191,7 @@ export class CompilerApi { return { cubes: compilers.metaTransformer.cubes, granularityHash: null }; } - const config = await resolveGlobalGranularities(this.granularities, requestContext); + const config = await this.resolveGranularities(requestContext); const granularityHash = granularityConfigHash(config); if (!compilers.granularityVariants) { diff --git a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts index 3fb3bf1c3c49d..2fb322962432d 100644 --- a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts +++ b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts @@ -229,6 +229,19 @@ describe('granularity variants in CompilerApi', () => { api.dispose(); }); + // Regression: meta and SQL paths must resolve the config from the SAME securityContext-only + // view. A function keyed on securityContext advertises `sprint` for tenant b in meta AND + // resolves it in SQL — no path can advertise a custom the other can't execute. + test('meta and SQL paths agree on the effective set (securityContext-keyed function)', async () => { + const api = createApi({ granularities: perTenant }); + const cubes = await api.metaConfig(ctxFor('b'), {}); + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toContain('sprint'); + // Same tenant, SQL path: the advertised custom actually resolves. + const sql = await api.getSql(queryFor('b', 'Orders.created_at', 'sprint')); + expect(sql.sql[0]).toContain('created_at'); + api.dispose(); + }); + test('distinct compilerIds per tenant, both distinct from the base', async () => { const api = createApi({ granularities: perTenant }); const a = await api.metaConfig(ctxFor('a'), { includeCompilerId: true }); From d7a77d2ac2b047f669098821b55c725929127e60 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Fri, 17 Jul 2026 10:16:23 +0200 Subject: [PATCH 14/22] =?UTF-8?q?refactor(granularities):=20dedupe=20resol?= =?UTF-8?q?ve=E2=86=92serialize=20via=20effectiveGranularitiesFor=20+=20GR?= =?UTF-8?q?ANULARITY=5FSTRING=5FFIELDS=20constant?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/compiler/CubeToMetaTransformer.ts | 14 ++---- .../src/compiler/GlobalGranularitiesConfig.ts | 10 ++-- .../src/compiler/GranularityResolver.ts | 47 ++++++++++++++----- .../src/compiler/index.ts | 2 + .../src/core/CompilerApi.ts | 30 ++++++------ 5 files changed, 58 insertions(+), 45 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index a161fa8ab520f..8a12edfaaea4c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -24,8 +24,7 @@ import { EffectiveGranularity, NormalizedGranularitiesBlock, normalizeGranularitiesBlock, - resolveDimensionGranularities, - serializeEffectiveGranularities, + effectiveGranularitiesFor, } from './GranularityResolver'; import { GlobalGranularitiesConfig, @@ -268,12 +267,7 @@ export class CubeToMetaTransformer implements CompilerInterface { catalog, // One shared array for every time dimension without local customization — with large // models this avoids re-allocating an identical granularity set per dimension. - defaultSet: serializeEffectiveGranularities(resolveDimensionGranularities( - normalizeGranularitiesBlock(undefined), - config.enabledBuiltIns, - config.customGranularities, - catalog, - )), + defaultSet: effectiveGranularitiesFor(undefined, config.enabledBuiltIns, config.customGranularities, catalog), }; } @@ -377,9 +371,7 @@ export class CubeToMetaTransformer implements CompilerInterface { if (this.staticGranularityState) { const s = this.staticGranularityState; effectiveGranularities = inputs - ? serializeEffectiveGranularities(resolveDimensionGranularities( - inputs, s.config.enabledBuiltIns, s.config.customGranularities, s.catalog, - )) + ? effectiveGranularitiesFor(inputs, s.config.enabledBuiltIns, s.config.customGranularities, s.catalog) : s.defaultSet; } } diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts index 54c5252fa3de0..4b48d196214b8 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -2,6 +2,7 @@ import crypto from 'crypto'; import { getEnv } from '@cubejs-backend/shared'; import type { GranularityDefinition } from './CubeSymbols'; +import { GRANULARITY_STRING_FIELDS } from './GranularityResolver'; // Default `title` and `format` for each built-in granularity. Overridable via `config.granularities` // (file) or `CUBEJS_GRANULARITIES__TITLE` (env, title only). Format syntax: d3-time-format. @@ -103,7 +104,7 @@ function resolveFromEnv(): GlobalGranularitiesConfig { // props a config spreads in can't leak onto the wire. function sanitizeDefinition(def: Partial): GranularityDefinition { const out: GranularityDefinition = {}; - for (const key of ['title', 'format', 'interval', 'offset', 'origin'] as const) { + for (const key of GRANULARITY_STRING_FIELDS) { if (typeof def[key] === 'string') { out[key] = def[key]; } @@ -214,11 +215,8 @@ export function granularityConfigHash(config: GlobalGranularitiesConfig): string builtIns: [...config.enabledBuiltIns], custom: Object.entries(config.customGranularities).map(([name, def]) => ({ name, - title: def.title, - format: def.format, - interval: def.interval, - offset: def.offset, - origin: def.origin, + // Same fields, same order the serializer emits — driven by one constant so they can't drift. + ...Object.fromEntries(GRANULARITY_STRING_FIELDS.map((f) => [f, def[f]])), })), }; return crypto.createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts index cbd197173edd6..01ad2e9b48c28 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -73,6 +73,12 @@ export function normalizeGranularitiesBlock(raw: any): NormalizedGranularitiesBl return EMPTY_BLOCK; } +// The string-valued fields of a granularity definition, in wire-emission order. Single source of +// truth for every place that projects a definition (serialize, hash, sanitize, meta) so those +// projections can't drift — notably the hash and the serializer, which MUST agree or two tenant +// configs could collide on one variant-cache key. +export const GRANULARITY_STRING_FIELDS = ['title', 'interval', 'offset', 'origin', 'format'] as const; + // Wire shape of one entry in a time dimension's `effectiveGranularities`. export type EffectiveGranularity = { name: string; @@ -84,19 +90,19 @@ export type EffectiveGranularity = { format?: string; }; -// Serialize a resolved set for /v1/meta. Field order and the conditional inclusion of -// interval/offset/origin/format are part of the wire contract — keep byte-compatible. -// `title` falls back to the granularity name (same as /v1/granularities) so it is always present. +// Serialize a resolved set for /v1/meta. `title` always present (falls back to the name); the +// other string fields are included only when defined. Order follows GRANULARITY_STRING_FIELDS, +// which also drives the config hash — the two share the field list so they can't drift. export function serializeEffectiveGranularities(resolved: ResolvedGranularitySet): EffectiveGranularity[] { - return Object.entries(resolved).map(([name, def]) => ({ - name, - type: def.type, - title: def.title || name, - ...(def.interval !== undefined ? { interval: def.interval } : {}), - ...(def.offset !== undefined ? { offset: def.offset } : {}), - ...(def.origin !== undefined ? { origin: def.origin } : {}), - ...(def.format !== undefined ? { format: def.format } : {}), - })); + return Object.entries(resolved).map(([name, def]) => { + const out: EffectiveGranularity = { name, type: def.type, title: def.title || name }; + for (const field of GRANULARITY_STRING_FIELDS) { + if (field !== 'title' && def[field] !== undefined) { + out[field] = def[field]; + } + } + return out; + }); } // Reconcile a dimension's local block against the global enabled built-ins and global customs, @@ -147,3 +153,20 @@ export function resolveDimensionGranularities( return out; } + +// One-shot: reconcile a dimension's block against the global config and serialize to the wire set. +// The single seam every path uses to turn a (block, config) into `effectiveGranularities`, so the +// resolve→serialize contract lives in exactly one place. A missing block means "no local block". +export function effectiveGranularitiesFor( + block: NormalizedGranularitiesBlock | undefined, + globalEnabledBuiltIns: ReadonlyArray, + globalCustom: Readonly>, + allBuiltInsCatalog: Readonly>, +): EffectiveGranularity[] { + return serializeEffectiveGranularities(resolveDimensionGranularities( + block ?? normalizeGranularitiesBlock(undefined), + globalEnabledBuiltIns, + globalCustom, + allBuiltInsCatalog, + )); +} diff --git a/packages/cubejs-schema-compiler/src/compiler/index.ts b/packages/cubejs-schema-compiler/src/compiler/index.ts index 691eaab9f7d32..26b60fcb0f059 100644 --- a/packages/cubejs-schema-compiler/src/compiler/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/index.ts @@ -31,7 +31,9 @@ export { NormalizedGranularitiesBlock, ResolvedGranularitySet, EffectiveGranularity, + GRANULARITY_STRING_FIELDS, normalizeGranularitiesBlock, resolveDimensionGranularities, serializeEffectiveGranularities, + effectiveGranularitiesFor, } from './GranularityResolver'; diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index b46972afeead7..58beb31e1c9dd 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -21,10 +21,10 @@ import { prepareCompiler, queryClass, QueryFactory, + effectiveGranularitiesFor, resolveDimensionGranularities, resolveGlobalGranularities, resolveGlobalGranularitiesSync, - serializeEffectiveGranularities, TransformedQuery, ViewIncludedMember, } from '@cubejs-backend/schema-compiler'; @@ -400,21 +400,25 @@ export class CompilerApi { } const catalog = buildBuiltInsCatalog(config); + const { enabledBuiltIns, customGranularities } = config; const inputs = compilers.metaTransformer.granularityInputs; + const emptyBlock = normalizeGranularitiesBlock(undefined); for (const cube of compilers.metaTransformer.cubes) { for (const dimension of (cube.config.dimensions || []).filter((d: any) => d.type === 'time')) { const resolved = resolveDimensionGranularities( - inputs.get(dimension.name) || normalizeGranularitiesBlock(undefined), - config.enabledBuiltIns, - config.customGranularities, + inputs.get(dimension.name) || emptyBlock, + enabledBuiltIns, + customGranularities, catalog, ); + // Only GLOBAL customs need threading to SQL — locals already resolve via the symbols map, + // built-ins via the predefined path. Strip the meta-only `type` tag the resolver added. const customs = Object.fromEntries( Object.entries(resolved) - .filter(([name, definition]) => definition.type === 'custom' && - Object.prototype.hasOwnProperty.call(config.customGranularities, name)) - .map(([name, { type: _type, ...definition }]) => [name, definition]) + .filter(([name, def]) => def.type === 'custom' && + Object.prototype.hasOwnProperty.call(customGranularities, name)) + .map(([name, { type: _type, ...def }]) => [name, def]) ); if (Object.keys(customs).length > 0) { definitions[dimension.name] = customs; @@ -1234,12 +1238,8 @@ export class CompilerApi { protected buildGranularityVariant(compilers: Compiler, config: GlobalGranularitiesConfig): any[] { const catalog = buildBuiltInsCatalog(config); const inputs = compilers.metaTransformer.granularityInputs; - const defaultSet = serializeEffectiveGranularities(resolveDimensionGranularities( - normalizeGranularitiesBlock(undefined), - config.enabledBuiltIns, - config.customGranularities, - catalog, - )); + const { enabledBuiltIns, customGranularities } = config; + const defaultSet = effectiveGranularitiesFor(undefined, enabledBuiltIns, customGranularities, catalog); return compilers.metaTransformer.cubes.map((cube: any) => { if (!cube.config.dimensions?.some((d: any) => d.type === 'time')) { @@ -1254,9 +1254,7 @@ export class CompilerApi { } const block = inputs.get(dim.name); const effectiveGranularities = block - ? serializeEffectiveGranularities(resolveDimensionGranularities( - block, config.enabledBuiltIns, config.customGranularities, catalog, - )) + ? effectiveGranularitiesFor(block, enabledBuiltIns, customGranularities, catalog) : defaultSet; return { ...dim, effectiveGranularities }; }), From 773d9ee243723a22c9f41571e4211bfffdfe1972 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Fri, 17 Jul 2026 15:22:21 +0200 Subject: [PATCH 15/22] perf(granularities): cache the SQL-path global-custom lookup per config-hash on the compiled model --- .../src/compiler/PrepareCompiler.ts | 4 + .../src/core/CompilerApi.ts | 49 ++++++++++-- .../test/unit/granularity-variants.test.ts | 79 +++++++++++++++++++ 3 files changed, 124 insertions(+), 8 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 2814a5e984930..55a4b14145632 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -63,6 +63,10 @@ export type Compiler = { // Granularity-enriched meta variants keyed by config hash; owned by the compiled model so a // recompile discards it. Populated lazily by CompilerApi (bounded LRU, function form only). granularityVariants?: Map>; + // Per-request SQL-path global-custom lookups (dim -> name -> def) keyed by config hash. Same + // ownership/lifecycle as granularityVariants: the map is a pure function of (model, config), + // so it's cached here and discarded on recompile. Bounded to match the variant cache. + granularityDefinitions?: Map>>; }; export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}): Compiler => { diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index 58beb31e1c9dd..004314ba3af25 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -381,6 +381,11 @@ export class CompilerApi { * contain function-config results because it is reused across security contexts, so query-time * resolution receives this immutable lookup instead. Building it through the same resolver and * per-dimension inputs as meta keeps includes/excludes behavior identical on both paths. + * + * The result is a pure function of (compiled model, resolved config), so it is cached on the + * compiled model keyed by the canonical config hash — the same discriminator the meta variant + * cache uses. Steady state is O(config) (resolve + hash + map lookup); the O(model) scan runs + * once per distinct config, not once per query. Recompile discards the cache with `compilers`. */ protected async granularityDefinitionsForQuery( compilers: Compiler, @@ -390,19 +395,49 @@ export class CompilerApi { // the effective set — a `granularities` function must key only on securityContext. const { contextSymbols } = query as any; const config = await this.resolveGranularities({ securityContext: contextSymbols?.securityContext }); - const definitions: Record> = {}; - // Only GLOBAL customs need threading (locals already resolve via the compiled symbols map). With - // none configured there's nothing to add, so skip the O(model) scan — the common case allocates - // nothing and query-time resolution falls straight through to the symbols map. + // Only GLOBAL customs need threading (locals already resolve via the compiled symbols map, and + // built-ins via the predefined path). With none configured there's nothing to add, so skip both + // the cache and the scan — the common case falls straight through to the symbols map. if (Object.keys(config.customGranularities).length === 0) { - return definitions; + return {}; } - const catalog = buildBuiltInsCatalog(config); + if (!compilers.granularityDefinitions) { + compilers.granularityDefinitions = new Map(); + } + const cache = compilers.granularityDefinitions; + const hash = granularityConfigHash(config); + + let definitions = cache.get(hash); + if (definitions) { + // Refresh LRU recency. + cache.delete(hash); + cache.set(hash, definitions); + } else { + if (cache.size >= CompilerApi.MAX_GRANULARITY_VARIANTS) { + cache.delete(cache.keys().next().value); + } + definitions = this.buildGranularityDefinitions(compilers, config); + cache.set(hash, definitions); + } + return definitions; + } + + /** + * One O(model) pass building the SQL-path global-custom lookup: `dim -> { customName -> def }`, + * respecting each dimension's includes/excludes exactly as the meta path does. Emits only global + * customs (locals/built-ins resolve elsewhere) with the meta-only `type` tag stripped. + */ + protected buildGranularityDefinitions( + compilers: Compiler, + config: GlobalGranularitiesConfig, + ): Record> { const { enabledBuiltIns, customGranularities } = config; + const catalog = buildBuiltInsCatalog(config); const inputs = compilers.metaTransformer.granularityInputs; const emptyBlock = normalizeGranularitiesBlock(undefined); + const definitions: Record> = {}; for (const cube of compilers.metaTransformer.cubes) { for (const dimension of (cube.config.dimensions || []).filter((d: any) => d.type === 'time')) { @@ -412,8 +447,6 @@ export class CompilerApi { customGranularities, catalog, ); - // Only GLOBAL customs need threading to SQL — locals already resolve via the symbols map, - // built-ins via the predefined path. Strip the meta-only `type` tag the resolver added. const customs = Object.fromEntries( Object.entries(resolved) .filter(([name, def]) => def.type === 'custom' && diff --git a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts index 2fb322962432d..bf3acdd293d0a 100644 --- a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts +++ b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts @@ -8,6 +8,8 @@ class TestableCompilerApi extends CompilerApi { public failNextBuild = false; + public defsBuildCount = 0; + protected buildGranularityVariant(compilers: Compiler, config: GlobalGranularitiesConfig): any[] { if (this.failNextBuild) { this.failNextBuild = false; @@ -17,6 +19,11 @@ class TestableCompilerApi extends CompilerApi { return super.buildGranularityVariant(compilers, config); } + protected buildGranularityDefinitions(compilers: Compiler, config: GlobalGranularitiesConfig): any { + this.defsBuildCount++; + return super.buildGranularityDefinitions(compilers, config); + } + public version(): string | undefined { return this.compilerVersion; } @@ -24,6 +31,10 @@ class TestableCompilerApi extends CompilerApi { public async variantCache(): Promise> | undefined> { return (await this.getCompilers()).granularityVariants; } + + public async definitionsCache(): Promise | undefined> { + return (await this.getCompilers()).granularityDefinitions; + } } const repository: SchemaFileRepository = { @@ -338,6 +349,74 @@ describe('granularity variants in CompilerApi', () => { }); }); + describe('SQL-path global-custom definitions cache', () => { + const perTenant = (ctx: any) => (ctx.securityContext.tenant === 'a' + ? [{ name: 'sprint', interval: '2 weeks', origin: '2024-01-01' }] + : [{ name: 'fortnight', interval: '2 weeks', origin: '2024-01-08' }]); + + test('scan runs once per distinct config, then serves from cache; result is byte-identical', async () => { + const api = createApi({ granularities: perTenant }); + + const a1 = await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); + const a2 = await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); + // Two identical-config queries → one scan. + expect(api.defsBuildCount).toBe(1); + expect(a1.sql[0]).toBe(a2.sql[0]); + + // A different tenant is a distinct config → one more scan, and a second cache entry. + await api.getSql(queryFor('b', 'Orders.created_at', 'fortnight')); + expect(api.defsBuildCount).toBe(2); + expect((await api.definitionsCache())!.size).toBe(2); + api.dispose(); + }); + + test('no scan and no cache entry when no global customs are configured', async () => { + const api = createApi({ granularities: () => ['year', 'month'] }); + await api.getSql(queryFor('a', 'Orders.created_at', 'month')); + await api.getSql(queryFor('a', 'Orders.created_at', 'year')); + expect(api.defsBuildCount).toBe(0); + expect(await api.definitionsCache()).toBeUndefined(); + api.dispose(); + }); + + test('distinct tenants never share a definitions entry (no cross-tenant bleed)', async () => { + const api = createApi({ granularities: perTenant }); + // Tenant a's custom is `sprint`; tenant b's is `fortnight`. Each resolves only its own. + const aSql = await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); + expect(aSql.sql[0]).toContain('created_at'); + await expect(api.getSql(queryFor('a', 'Orders.created_at', 'fortnight'))) + .rejects.toThrow('Granularity "fortnight" does not exist in dimension Orders.created_at'); + const bSql = await api.getSql(queryFor('b', 'Orders.created_at', 'fortnight')); + expect(bSql.sql[0]).toContain('created_at'); + await expect(api.getSql(queryFor('b', 'Orders.created_at', 'sprint'))) + .rejects.toThrow('Granularity "sprint" does not exist in dimension Orders.created_at'); + api.dispose(); + }); + + test('a recompile discards the definitions cache with the compilers object', async () => { + let version = 'v1'; + const api = createApi({ granularities: perTenant, schemaVersion: () => version }); + await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); + expect(api.defsBuildCount).toBe(1); + + version = 'v2'; + await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); + expect(api.defsBuildCount).toBe(2); + expect((await api.definitionsCache())!.size).toBe(1); + api.dispose(); + }); + + test('static config caches a single entry reused across queries', async () => { + const api = createApi({ granularities: [{ name: 'sprint', interval: '2 weeks', origin: '2024-01-01' }] }); + await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); + await api.getSql(queryFor('b', 'Orders.created_at', 'sprint')); + // Context-independent config → one scan, one entry, regardless of tenant. + expect(api.defsBuildCount).toBe(1); + expect((await api.definitionsCache())!.size).toBe(1); + api.dispose(); + }); + }); + describe('composition with RBAC visibility', () => { const rbacRepository: SchemaFileRepository = { localPath: () => '/mock/path', From 9aac80e04f0aa8ca987af69c19be40cb4439c49e Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Fri, 17 Jul 2026 15:50:40 +0200 Subject: [PATCH 16/22] perf(granularities): skip per-dimension reconciliation for dims without a local block (share one global-custom map by reference) --- .../src/core/CompilerApi.ts | 54 ++++++++++++------- .../test/unit/granularity-variants.test.ts | 26 ++++++++- 2 files changed, 59 insertions(+), 21 deletions(-) diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index 004314ba3af25..850ca93f01ab9 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -13,6 +13,7 @@ import { GlobalGranularitiesConfig, GranularitiesOption, granularityConfigHash, + NormalizedGranularitiesBlock, normalizeGranularitiesBlock, PreAggregationFilters, PreAggregationInfo, @@ -436,29 +437,41 @@ export class CompilerApi { const { enabledBuiltIns, customGranularities } = config; const catalog = buildBuiltInsCatalog(config); const inputs = compilers.metaTransformer.granularityInputs; - const emptyBlock = normalizeGranularitiesBlock(undefined); const definitions: Record> = {}; - for (const cube of compilers.metaTransformer.cubes) { - for (const dimension of (cube.config.dimensions || []).filter((d: any) => d.type === 'time')) { - const resolved = resolveDimensionGranularities( - inputs.get(dimension.name) || emptyBlock, - enabledBuiltIns, - customGranularities, - catalog, - ); - const customs = Object.fromEntries( - Object.entries(resolved) - .filter(([name, def]) => def.type === 'custom' && - Object.prototype.hasOwnProperty.call(customGranularities, name)) - .map(([name, { type: _type, ...def }]) => [name, def]) - ); - if (Object.keys(customs).length > 0) { - definitions[dimension.name] = customs; + // Keep only the global customs a dimension's resolved set actually exposes, `type` tag stripped. + const globalCustomsOf = (block: NormalizedGranularitiesBlock) => Object.fromEntries( + Object.entries(resolveDimensionGranularities(block, enabledBuiltIns, customGranularities, catalog)) + .filter(([name, def]) => def.type === 'custom' && + Object.prototype.hasOwnProperty.call(customGranularities, name)) + .map(([name, { type: _type, ...def }]) => [name, def]) + ); + + // A time dimension WITHOUT a local block resolves against the empty block, so it exposes every + // global custom with no filtering — the same map for all of them. Local blocks (rare) are the + // only dimensions needing individual reconciliation, so only those are walked. No full model + // scan: cost is O(global customs) + O(local-block dims), not O(all time dimensions). + const shared = globalCustomsOf(normalizeGranularitiesBlock(undefined)); + + if (Object.keys(shared).length > 0) { + // A time dimension is "plain" iff it isn't in `granularityInputs` (which holds only dims that + // declared a local granularities block). Assign the shared map by reference to each plain dim. + for (const cube of compilers.metaTransformer.cubes) { + for (const dim of cube.config.dimensions || []) { + if (dim.type === 'time' && !inputs.has(dim.name)) { + definitions[dim.name] = shared; + } } } } + for (const [dimName, block] of inputs) { + const customs = globalCustomsOf(block); + if (Object.keys(customs).length > 0) { + definitions[dimName] = customs; + } + } + return definitions; } @@ -1264,9 +1277,10 @@ export class CompilerApi { } /** - * One O(model) pass attaching `effectiveGranularities` to every time dimension. Copies only - * what it changes — the base meta is never mutated, untouched members stay shared by - * reference, and uncustomized time dimensions share one default set per variant. + * Attach `effectiveGranularities` to every time dimension for one config. Must copy each cube + * (the base meta is never mutated), but the expensive reconciliation runs ONLY for the rare + * dimensions with a local block; every plain dimension shares one precomputed `defaultSet` by + * reference — so the per-dimension cost is O(local-block dims), not O(all time dimensions). */ protected buildGranularityVariant(compilers: Compiler, config: GlobalGranularitiesConfig): any[] { const catalog = buildBuiltInsCatalog(config); diff --git a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts index bf3acdd293d0a..44f180f8c86b2 100644 --- a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts +++ b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts @@ -19,9 +19,12 @@ class TestableCompilerApi extends CompilerApi { return super.buildGranularityVariant(compilers, config); } + public lastDefinitions: any; + protected buildGranularityDefinitions(compilers: Compiler, config: GlobalGranularitiesConfig): any { this.defsBuildCount++; - return super.buildGranularityDefinitions(compilers, config); + this.lastDefinitions = super.buildGranularityDefinitions(compilers, config); + return this.lastDefinitions; } public version(): string | undefined { @@ -415,6 +418,27 @@ describe('granularity variants in CompilerApi', () => { expect((await api.definitionsCache())!.size).toBe(1); api.dispose(); }); + + // Plain time dimensions (no local block) share ONE global-custom map by reference; a dimension + // with a local block (Orders.excluded_at excludes sprint) is reconciled individually. + test('plain dimensions share the global-custom map by reference; local-block dims are individual', async () => { + const api = createApi({ granularities: [{ name: 'sprint', interval: '2 weeks', origin: '2024-01-01' }] }); + await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); + const defs = api.lastDefinitions; + + // created_at and updated_at are both plain (no local block) → the very same object. + expect(defs['Orders.created_at']).toBeDefined(); + expect(defs['Orders.created_at']).toBe(defs['Orders.updated_at']); + expect(defs['Orders.created_at'].sprint).toEqual({ interval: '2 weeks', origin: '2024-01-01' }); + + // excluded_at excludes sprint → not in the shared map; sprint absent there. + expect(defs['Orders.excluded_at']?.sprint).toBeUndefined(); + expect(defs['Orders.excluded_at']).not.toBe(defs['Orders.created_at']); + // Querying the excluded custom on it fails, matching meta. + await expect(api.getSql(queryFor('a', 'Orders.excluded_at', 'sprint'))) + .rejects.toThrow('Granularity "sprint" does not exist in dimension Orders.excluded_at'); + api.dispose(); + }); }); describe('composition with RBAC visibility', () => { From db42ad07c04b0b7d6b1b9693688436ed3cfda467 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Fri, 17 Jul 2026 16:29:40 +0200 Subject: [PATCH 17/22] perf(granularities): store sparse effective sets in the variant cache, attach onto base cubes at read time --- .../src/compiler/GranularityResolver.ts | 9 +++ .../src/compiler/PrepareCompiler.ts | 10 ++- .../src/compiler/index.ts | 1 + .../src/core/CompilerApi.ts | 68 +++++++++++-------- .../test/unit/granularity-variants.test.ts | 32 +++++++-- 5 files changed, 85 insertions(+), 35 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts index 01ad2e9b48c28..99ea608dee5f5 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -90,6 +90,15 @@ export type EffectiveGranularity = { format?: string; }; +// The per-config effective granularity data, sparse: one `defaultSet` shared by every time +// dimension without a local block, plus per-dimension `overrides` for the rare dims that declared +// one. Attached onto the base meta cubes at read time (see CompilerApi.attachEffectiveGranularities) +// instead of storing a full enriched cube copy. +export type GranularitySets = { + defaultSet: EffectiveGranularity[]; + overrides: Map; +}; + // Serialize a resolved set for /v1/meta. `title` always present (falls back to the name); the // other string fields are included only when defined. Order follows GRANULARITY_STRING_FIELDS, // which also drives the config hash — the two share the field list so they can't drift. diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 55a4b14145632..e5f219a38de32 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -27,6 +27,7 @@ import { YamlCompiler } from './YamlCompiler'; import { ViewCompilationGate } from './ViewCompilationGate'; import type { ErrorReporter } from './ErrorReporter'; import type { GranularitiesOption } from './GlobalGranularitiesConfig'; +import type { GranularitySets } from './GranularityResolver'; export type PrepareCompilerOptions = { nativeInstance?: NativeInstance, @@ -60,9 +61,12 @@ export type Compiler = { compilerCache: CompilerCache; headCommitId?: string; compilerId: string; - // Granularity-enriched meta variants keyed by config hash; owned by the compiled model so a - // recompile discards it. Populated lazily by CompilerApi (bounded LRU, function form only). - granularityVariants?: Map>; + // Per-config effective granularity SETS keyed by config hash — NOT enriched cube copies. The + // sparse `{ defaultSet, overrides }` shape is a few KB (one shared array + the rare local-block + // dims) rather than a ~MB clone of the whole meta; CompilerApi attaches these onto the base + // cubes cheaply at read time. Owned by the compiled model so a recompile discards it. Bounded + // LRU, function form only. + granularityVariants?: Map>; // Per-request SQL-path global-custom lookups (dim -> name -> def) keyed by config hash. Same // ownership/lifecycle as granularityVariants: the map is a pure function of (model, config), // so it's cached here and discarded on recompile. Bounded to match the variant cache. diff --git a/packages/cubejs-schema-compiler/src/compiler/index.ts b/packages/cubejs-schema-compiler/src/compiler/index.ts index 26b60fcb0f059..ccba0c6674e27 100644 --- a/packages/cubejs-schema-compiler/src/compiler/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/index.ts @@ -31,6 +31,7 @@ export { NormalizedGranularitiesBlock, ResolvedGranularitySet, EffectiveGranularity, + GranularitySets, GRANULARITY_STRING_FIELDS, normalizeGranularitiesBlock, resolveDimensionGranularities, diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index 850ca93f01ab9..0d80d60a5514f 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -9,9 +9,11 @@ import { Compiler, createQuery, CubeDefinition, + EffectiveGranularity, EvaluatedCube, GlobalGranularitiesConfig, GranularitiesOption, + GranularitySets, granularityConfigHash, NormalizedGranularitiesBlock, normalizeGranularitiesBlock, @@ -1230,8 +1232,9 @@ export class CompilerApi { /** * Meta cubes with `effectiveGranularities` attached, plus the hash to mix into compilerId. * Env/static configs are baked into the base meta at compile time (null hash); a function - * config resolves per request against a bounded promise-valued LRU of enriched variants keyed - * by config hash and owned by the compiled model, so a recompile discards it. + * config resolves per request. The cache holds the sparse per-config granularity SETS (a few KB) + * — not enriched cube copies — keyed by config hash and owned by the compiled model, so a + * recompile discards it; the (cheap) attach onto base cubes happens here at read time. */ protected async selectGranularityVariant( compilers: Compiler, @@ -1249,11 +1252,11 @@ export class CompilerApi { } const cache = compilers.granularityVariants; - let variant = cache.get(granularityHash); - if (variant) { + let sets = cache.get(granularityHash); + if (sets) { // Refresh LRU recency. cache.delete(granularityHash); - cache.set(granularityHash, variant); + cache.set(granularityHash, sets); } else { if (cache.size >= CompilerApi.MAX_GRANULARITY_VARIANTS) { const oldest = cache.keys().next().value; @@ -1264,47 +1267,56 @@ export class CompilerApi { 'causes per-request meta rebuilds and churns compilerId-based caches (e.g. in CubeSQL).', }); } - variant = Promise.resolve().then(() => this.buildGranularityVariant(compilers, config)); - cache.set(granularityHash, variant); - variant.catch(() => { - if (cache.get(granularityHash) === variant) { + sets = Promise.resolve().then(() => this.buildGranularitySets(compilers, config)); + cache.set(granularityHash, sets); + sets.catch(() => { + if (cache.get(granularityHash) === sets) { cache.delete(granularityHash); } }); } - return { cubes: await variant, granularityHash }; + return { cubes: this.attachEffectiveGranularities(compilers.metaTransformer.cubes, await sets), granularityHash }; } /** - * Attach `effectiveGranularities` to every time dimension for one config. Must copy each cube - * (the base meta is never mutated), but the expensive reconciliation runs ONLY for the rare - * dimensions with a local block; every plain dimension shares one precomputed `defaultSet` by - * reference — so the per-dimension cost is O(local-block dims), not O(all time dimensions). + * The sparse effective granularity data for one config. The expensive reconciliation runs ONLY + * for the rare dimensions with a local block; every plain dimension uses one shared `defaultSet`. + * No cube copying here — that happens cheaply in `attachEffectiveGranularities` at read time. */ - protected buildGranularityVariant(compilers: Compiler, config: GlobalGranularitiesConfig): any[] { + protected buildGranularitySets(compilers: Compiler, config: GlobalGranularitiesConfig): GranularitySets { const catalog = buildBuiltInsCatalog(config); - const inputs = compilers.metaTransformer.granularityInputs; const { enabledBuiltIns, customGranularities } = config; - const defaultSet = effectiveGranularitiesFor(undefined, enabledBuiltIns, customGranularities, catalog); + const overrides = new Map(); + for (const [dimName, block] of compilers.metaTransformer.granularityInputs) { + overrides.set(dimName, effectiveGranularitiesFor(block, enabledBuiltIns, customGranularities, catalog)); + } + return { + defaultSet: effectiveGranularitiesFor(undefined, enabledBuiltIns, customGranularities, catalog), + overrides, + }; + } - return compilers.metaTransformer.cubes.map((cube: any) => { + /** + * Attach the per-config effective sets onto the base meta cubes without mutating them: copy only + * the cubes/dimensions touched, and reference the shared `defaultSet` for plain time dimensions + * (per-dim `overrides` for the rare local-block ones). O(time dimensions) shallow work — no + * reconciliation, no deep clone of measures/segments/etc. + */ + protected attachEffectiveGranularities(baseCubes: any[], sets: GranularitySets): any[] { + return baseCubes.map((cube: any) => { if (!cube.config.dimensions?.some((d: any) => d.type === 'time')) { return cube; } return { + ...cube, config: { ...cube.config, - dimensions: cube.config.dimensions.map((dim: any) => { - if (dim.type !== 'time') { - return dim; - } - const block = inputs.get(dim.name); - const effectiveGranularities = block - ? effectiveGranularitiesFor(block, enabledBuiltIns, customGranularities, catalog) - : defaultSet; - return { ...dim, effectiveGranularities }; - }), + dimensions: cube.config.dimensions.map((dim: any) => ( + dim.type === 'time' + ? { ...dim, effectiveGranularities: sets.overrides.get(dim.name) ?? sets.defaultSet } + : dim + )), }, }; }); diff --git a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts index 44f180f8c86b2..1307f881e972b 100644 --- a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts +++ b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts @@ -1,5 +1,5 @@ import { SchemaFileRepository } from '@cubejs-backend/shared'; -import type { Compiler, GlobalGranularitiesConfig } from '@cubejs-backend/schema-compiler'; +import type { Compiler, GlobalGranularitiesConfig, GranularitySets } from '@cubejs-backend/schema-compiler'; import { CompilerApi } from '../../src/core/CompilerApi'; import { DbTypeInternalFn } from '../../src/core/types'; @@ -10,13 +10,13 @@ class TestableCompilerApi extends CompilerApi { public defsBuildCount = 0; - protected buildGranularityVariant(compilers: Compiler, config: GlobalGranularitiesConfig): any[] { + protected buildGranularitySets(compilers: Compiler, config: GlobalGranularitiesConfig): GranularitySets { if (this.failNextBuild) { this.failNextBuild = false; throw new Error('injected variant build failure'); } this.buildCount++; - return super.buildGranularityVariant(compilers, config); + return super.buildGranularitySets(compilers, config); } public lastDefinitions: any; @@ -31,7 +31,7 @@ class TestableCompilerApi extends CompilerApi { return this.compilerVersion; } - public async variantCache(): Promise> | undefined> { + public async variantCache(): Promise> | undefined> { return (await this.getCompilers()).granularityVariants; } @@ -243,6 +243,30 @@ describe('granularity variants in CompilerApi', () => { api.dispose(); }); + // The cache stores the sparse per-config sets ({ defaultSet, overrides }), NOT an enriched cube + // array — plain dims reference one shared defaultSet; only local-block dims are in overrides. + test('variant cache stores sparse sets, not cube copies', async () => { + const api = createApi({ granularities: perTenant }); + const cubes = await api.metaConfig(ctxFor('a'), {}); + + const entry = await (await api.variantCache())!.get( + [...(await api.variantCache())!.keys()][0] + )!; + expect(Array.isArray(entry)).toBe(false); + expect(Array.isArray(entry.defaultSet)).toBe(true); + expect(entry.overrides instanceof Map).toBe(true); + // Events.ts has a local block → in overrides; plain dims are not. + expect(entry.overrides.has('Events.ts')).toBe(true); + expect(entry.overrides.has('Orders.created_at')).toBe(false); + + // And the attached read-time result shares the one defaultSet across plain dims by reference. + const created = dimByName(cubes, 'Orders.created_at'); + const updated = dimByName(cubes, 'Orders.updated_at'); + expect(created.effectiveGranularities).toBe(updated.effectiveGranularities); + expect(created.effectiveGranularities).toBe(entry.defaultSet); + api.dispose(); + }); + // Regression: meta and SQL paths must resolve the config from the SAME securityContext-only // view. A function keyed on securityContext advertises `sprint` for tenant b in meta AND // resolves it in SQL — no path can advertise a custom the other can't execute. From 33dec12d8e9050d0f85b67ed3502bd9108b2e7a6 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Fri, 17 Jul 2026 17:04:10 +0200 Subject: [PATCH 18/22] perf(granularities): key query cache by config hash not the shared defs map; memoize proxy bound methods; dedup definitions cache; hash only output-affecting built-in fields; fix custom-named/proto-name bugs; skip compilerId when unneeded --- packages/cubejs-api-gateway/src/gateway.ts | 8 +- .../src/adapter/BaseQuery.js | 6 +- .../src/compiler/CubeSymbols.ts | 28 +++++-- .../src/compiler/GlobalGranularitiesConfig.ts | 16 ++-- .../src/compiler/GranularityResolver.ts | 15 +++- .../src/compiler/PrepareCompiler.ts | 7 +- .../test/unit/granularities-shape.test.ts | 27 +++++++ .../test/unit/granularity-config-hash.test.ts | 29 +++++++ .../src/core/CompilerApi.ts | 76 ++++++++++++++----- 9 files changed, 170 insertions(+), 42 deletions(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 74feec85984d6..d71720a836367 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -35,7 +35,7 @@ import { createProxyMiddleware } from 'http-proxy-middleware'; import { QueryBody } from '@cubejs-backend/query-orchestrator'; import { buildBuiltInsCatalog, - BUILT_IN_GRANULARITIES, + isBuiltInGranularity, } from '@cubejs-backend/schema-compiler'; import { QueryType, @@ -784,8 +784,10 @@ class ApiGateway { granularities.push({ type: 'built-in', name, ...entry }); } for (const [name, def] of Object.entries(globalConfig.customGranularities)) { - // Skip names already emitted by `buildBuiltInsCatalog` (their inline overrides are folded in there). - if (!(name in BUILT_IN_GRANULARITIES)) { + // Skip names already emitted by `buildBuiltInsCatalog` (their inline overrides are folded in + // there). Use isBuiltInGranularity (hasOwnProperty), not `in`, so a custom named e.g. + // `constructor`/`toString` isn't misclassified as a built-in via the prototype chain and dropped. + if (!isBuiltInGranularity(name)) { const entry: any = { type: 'custom', name, diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 73e9918735247..f5e628c1de549 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -270,7 +270,10 @@ export class BaseQuery { segments: this.options.segments, order: this.options.order, contextSymbols: this.options.contextSymbols, - granularityDefinitions: this.options.granularityDefinitions, + // Key on the O(1) config hash, NOT the granularityDefinitions map: the map shares one object + // across all plain dimensions by reference, and JSON.stringify would expand it once per time + // dimension, bloating the (serialized, LRU-retained) cache key by O(dimensions × customs). + granularityHash: this.options.granularityHash, timezone: this.options.timezone, limit: this.options.limit, offset: this.options.offset, @@ -4210,6 +4213,7 @@ export class BaseQuery { useOriginalSqlPreAggregationsInPreAggregation: this.options.useOriginalSqlPreAggregationsInPreAggregation, contextSymbols: this.contextSymbols, granularityDefinitions: this.options.granularityDefinitions, + granularityHash: this.options.granularityHash, preAggregationsSchema: this.preAggregationsSchemaOption, cubeLatticeCache: this.options.cubeLatticeCache, historyQueries: this.options.historyQueries, diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 9aed28930babc..6a389d8c12ea7 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -1593,6 +1593,11 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface * shared evaluator, while binding custom-granularity fallback to one request's effective set. * The native SQL planner calls resolveGranularity on the evaluator bridge directly, so the * request context has to live at this seam rather than only at JS adapter call sites. + * + * Must be a Proxy (not Object.create): the native bridge serializes this object's OWN enumerable + * fields (e.g. `primaryKeys`), which a prototype-delegating object would hide. The Proxy forwards + * every property to the real evaluator; to avoid allocating a bound function on every method + * access on the hot query path, bound methods are memoized in `boundCache` (bind once, reuse). */ public withGranularityDefinitions( granularityDefinitions?: Record>, @@ -1602,17 +1607,28 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface } const evaluator = this; + const boundResolveGranularity = (path: string | string[], refCube?: any) => evaluator.resolveGranularity( + path, + refCube, + granularityDefinitions, + ); + const boundCache = new Map(); return new Proxy(this, { get(target, property) { if (property === 'resolveGranularity') { - return (path: string | string[], refCube?: any) => evaluator.resolveGranularity( - path, - refCube, - granularityDefinitions, - ); + return boundResolveGranularity; } const value = Reflect.get(target, property, target); - return typeof value === 'function' ? value.bind(target) : value; + if (typeof value !== 'function') { + return value; + } + const cached = boundCache.get(property); + if (cached) { + return cached; + } + const bound = value.bind(target); + boundCache.set(property, bound); + return bound; }, }); } diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts index 4b48d196214b8..307eee97d81a6 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -205,18 +205,24 @@ export function buildBuiltInsCatalog(globalConfig: GlobalGranularitiesConfig): R return catalog; } +// Fields that a config override actually changes in the emitted output, per granularity name. For +// a name shadowing a built-in, buildBuiltInsCatalog honors only title/format (interval/offset/origin +// are fixed at `1 ` for predefined granularities), so hashing the ignored fields would churn +// the cache/compilerId on changes that produce identical output. A real custom uses every field. +const hashableFields = (name: string): ReadonlyArray => ( + isBuiltInGranularity(name) ? ['title', 'format'] : GRANULARITY_STRING_FIELDS +); + // Canonical sha256 of a resolved config: equal hash iff identical effective sets. Order-sensitive // on purpose — built-in and custom order affect the emitted meta. Used as the meta-variant cache -// key and compilerId discriminator. Values are already sanitized to strings at resolution time, -// so the projected fields here are exactly what the serializer emits — the hash can never -// disagree with the wire output (which would let two different tenant configs share one variant). +// key and compilerId discriminator. Only the fields that actually affect output participate (see +// hashableFields), so the hash never disagrees with the wire output nor churns on ignored fields. export function granularityConfigHash(config: GlobalGranularitiesConfig): string { const canonical = { builtIns: [...config.enabledBuiltIns], custom: Object.entries(config.customGranularities).map(([name, def]) => ({ name, - // Same fields, same order the serializer emits — driven by one constant so they can't drift. - ...Object.fromEntries(GRANULARITY_STRING_FIELDS.map((f) => [f, def[f]])), + ...Object.fromEntries(hashableFields(name).map((f) => [f, def[f]])), })), }; return crypto.createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts index 99ea608dee5f5..bd5a52f9059b5 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -42,12 +42,19 @@ export function normalizeGranularitiesBlock(raw: any): NormalizedGranularitiesBl if (typeof raw === 'object') { // New dict form iff every key is one of includes/excludes/custom AND the values have the dict - // shape (includes/excludes are '*' or arrays, custom is a plain object). The value check keeps a - // legacy custom granularity named `includes`/`excludes`/`custom` — whose value is a granularity - // definition object — from being misread as the dict form. + // shape. The value checks keep a legacy custom granularity NAMED `includes`/`excludes`/`custom` + // from being misread as the dict form: + // - includes/excludes must be '*' or an array (a legacy custom named `includes` has an object + // definition value, which fails this); + // - `custom` must be a map of name -> definition OBJECT. A legacy custom named `custom` has a + // definition whose own values are strings/functions (interval:'1 year', sql:()=>...), never + // nested objects — so requiring every `custom` value to be an object distinguishes the dict + // form ({custom:{fy:{...}}}) from the legacy custom-named-`custom` ({custom:{interval:...}}). const keys = Object.keys(raw); const isInclusionList = (v: any) => v === undefined || v === '*' || Array.isArray(v); - const isCustomMap = (v: any) => v === undefined || (typeof v === 'object' && v !== null && !Array.isArray(v)); + const isDefinitionObject = (v: any) => typeof v === 'object' && v !== null && !Array.isArray(v); + const isCustomMap = (v: any) => v === undefined || + (isDefinitionObject(v) && Object.values(v).every(isDefinitionObject)); const isDictForm = keys.length > 0 && keys.every(k => k === 'includes' || k === 'excludes' || k === 'custom') && diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index e5f219a38de32..2cfd5c2650703 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -68,9 +68,10 @@ export type Compiler = { // LRU, function form only. granularityVariants?: Map>; // Per-request SQL-path global-custom lookups (dim -> name -> def) keyed by config hash. Same - // ownership/lifecycle as granularityVariants: the map is a pure function of (model, config), - // so it's cached here and discarded on recompile. Bounded to match the variant cache. - granularityDefinitions?: Map>>; + // ownership/lifecycle as granularityVariants: a pure function of (model, config), cached here + // and discarded on recompile, bounded to match the variant cache. Promise-valued so concurrent + // misses of one hash coalesce into a single build. + granularityDefinitions?: Map>>>; }; export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}): Compiler => { diff --git a/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts b/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts index 863627b0ada65..8a5b0dbd60bd9 100644 --- a/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts @@ -174,4 +174,31 @@ describe('normalizeGranularitiesBlock — reserved-name disambiguation', () => { expect(out.includes).toBe('*'); expect(out.custom.includes).toEqual({ interval: '1 year', origin: '2026-04-01' }); }); + + // Regression: a legacy custom granularity literally named `custom` whose value is a DEFINITION + // (string-valued fields) must NOT be misread as the dict form's custom map (name -> definition). + it('custom granularity named "custom" (definition value) is not misread as the dict form', () => { + const out = normalizeGranularitiesBlock({ + custom: { interval: '1 year', origin: '2026-04-01' }, + }); + expect(out.includes).toBe('*'); + expect(out.custom.custom).toEqual({ interval: '1 year', origin: '2026-04-01' }); + }); + + it('custom granularity named "custom" with only an sql function is not misread as the dict form', () => { + const sql = () => 'date_trunc(\'year\', x)'; + const out = normalizeGranularitiesBlock({ custom: { sql } }); + expect(out.custom.custom).toEqual({ sql }); + }); + + // The genuine dict form (custom is a map of name -> definition OBJECT) is still recognized. + it('genuine dict form with a custom map is recognized', () => { + const out = normalizeGranularitiesBlock({ + includes: ['year'], + custom: { fiscal_year: { interval: '1 year', origin: '2026-04-01' } }, + }); + expect(out.includes).toEqual(['year']); + expect(out.custom.fiscal_year).toEqual({ interval: '1 year', origin: '2026-04-01' }); + expect(out.custom.custom).toBeUndefined(); + }); }); diff --git a/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts b/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts index c17a6991cfa7c..75e4c0c51b6a0 100644 --- a/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts @@ -52,4 +52,33 @@ describe('granularityConfigHash', () => { expect(granularityConfigHash(config(['year'], { fy: { interval: '1 year' } }))) .not.toBe(granularityConfigHash(config(['year'], { fy: { interval: '1 year', title: 'FY' } }))); }); + + // Regression (F6): for a name shadowing a built-in, only title/format affect output — the SQL + // layer fixes a built-in's interval at `1 `. So changing interval/offset/origin on a + // built-in override must NOT change the hash (else it churns the cache/compilerId for identical + // output), while changing title/format must. + describe('built-in override hashes only the fields that affect output', () => { + const withYear = (def: any) => config(['year'], { year: def }); + + it('ignores interval/offset/origin overrides on a built-in name', () => { + const baseHash = granularityConfigHash(withYear({ title: 'Year' })); + expect(granularityConfigHash(withYear({ title: 'Year', interval: '2 years' }))).toBe(baseHash); + expect(granularityConfigHash(withYear({ title: 'Year', offset: '1 day' }))).toBe(baseHash); + expect(granularityConfigHash(withYear({ title: 'Year', origin: '2024-02-01' }))).toBe(baseHash); + }); + + it('still reflects title/format overrides on a built-in name', () => { + const baseHash = granularityConfigHash(withYear({ title: 'Year' })); + expect(granularityConfigHash(withYear({ title: 'Jaar' }))).not.toBe(baseHash); + expect(granularityConfigHash(withYear({ title: 'Year', format: '%y' }))).not.toBe(baseHash); + }); + + it('still hashes every field for a real (non-built-in) custom', () => { + const baseHash = granularityConfigHash(config([], { fy: { interval: '1 year', origin: '2024-02-01' } })); + expect(granularityConfigHash(config([], { fy: { interval: '2 years', origin: '2024-02-01' } }))) + .not.toBe(baseHash); + expect(granularityConfigHash(config([], { fy: { interval: '1 year', origin: '2025-02-01' } }))) + .not.toBe(baseHash); + }); + }); }); diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index 0d80d60a5514f..bd374c89ea143 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -154,6 +154,10 @@ export class CompilerApi { protected readonly granularities?: GranularitiesOption; + // Memoized hash of a STATIC-list granularities config (immutable for the instance lifetime). + // Undefined until first computed; the env form is never memoized here (env can change at runtime). + private staticGranularityHashMemo?: string; + protected queryFactory?: QueryFactory; public constructor(repository: SchemaFileRepository, dbType: DbTypeInternalFn, options: CompilerApiOptions) { @@ -248,9 +252,10 @@ export class CompilerApi { // Env/static granularity configs are baked into the compiled meta, so a config change must // force a recompile. The function form is resolved per request instead (variant cache) and - // must never churn the compiler version. + // must never churn the compiler version. A static LIST is immutable for the instance lifetime, + // so its hash is memoized; the env form stays per-call since env vars can change at runtime. if (typeof this.granularities !== 'function') { - compilerVersion += `_gran_${granularityConfigHash(resolveGlobalGranularitiesSync(this.granularities))}`; + compilerVersion += `_gran_${this.staticGranularityHash()}`; } if (!this.compilers || this.compilerVersion !== compilerVersion) { @@ -345,8 +350,11 @@ export class CompilerApi { public async getSqlGenerator(query: NormalizedQuery, dataSource?: string): Promise<{ sqlGenerator: any; compilers: Compiler }> { const dbType = await this.getDbType(dataSource); const compilers = await this.getCompilers({ requestId: query.requestId }); - const granularityDefinitions = await this.granularityDefinitionsForQuery(compilers, query); - const queryWithGranularities = { ...query, granularityDefinitions }; + // `granularityDefinitions` (a reference-shared map) flows to resolveGranularity for SQL, but the + // query cache must key on the O(1) `granularityHash` instead — JSON.stringify'ing the map would + // expand the shared references once per time dimension and bloat the cache key. + const { definitions: granularityDefinitions, hash: granularityHash } = await this.granularityDefinitionsForQuery(compilers, query); + const queryWithGranularities = { ...query, granularityDefinitions, granularityHash }; let sqlGenerator = await this.createQueryByDataSource(compilers, queryWithGranularities, dataSource, dbType); if (!sqlGenerator) { @@ -393,7 +401,7 @@ export class CompilerApi { protected async granularityDefinitionsForQuery( compilers: Compiler, query: NormalizedQuery, - ): Promise>> { + ): Promise<{ definitions: Record>; hash: string | null }> { // Resolve through the same securityContext-only seam as the meta path so both paths agree on // the effective set — a `granularities` function must key only on securityContext. const { contextSymbols } = query as any; @@ -401,9 +409,10 @@ export class CompilerApi { // Only GLOBAL customs need threading (locals already resolve via the compiled symbols map, and // built-ins via the predefined path). With none configured there's nothing to add, so skip both - // the cache and the scan — the common case falls straight through to the symbols map. + // the cache and the scan — the common case falls straight through to the symbols map. A null + // hash then keeps the query-cache key free of any granularity discriminator. if (Object.keys(config.customGranularities).length === 0) { - return {}; + return { definitions: {}, hash: null }; } if (!compilers.granularityDefinitions) { @@ -412,19 +421,26 @@ export class CompilerApi { const cache = compilers.granularityDefinitions; const hash = granularityConfigHash(config); - let definitions = cache.get(hash); - if (definitions) { + // Promise-valued (like granularityVariants) so concurrent misses of one hash coalesce into a + // single build rather than each re-running the O(model) scan. + let built = cache.get(hash); + if (built) { // Refresh LRU recency. cache.delete(hash); - cache.set(hash, definitions); + cache.set(hash, built); } else { if (cache.size >= CompilerApi.MAX_GRANULARITY_VARIANTS) { cache.delete(cache.keys().next().value); } - definitions = this.buildGranularityDefinitions(compilers, config); - cache.set(hash, definitions); + built = Promise.resolve().then(() => this.buildGranularityDefinitions(compilers, config)); + cache.set(hash, built); + built.catch(() => { + if (cache.get(hash) === built) { + cache.delete(hash); + } + }); } - return definitions; + return { definitions: await built, hash }; } /** @@ -504,7 +520,8 @@ export class CompilerApi { const key = { query: keyOptions, options, - granularityDefinitions: sqlGenerator.options.granularityDefinitions, + // Key on the O(1) hash, not the reference-shared definitions map (see getQueryCache). + granularityHash: sqlGenerator.options.granularityHash, }; return compilers.compilerCache.getQueryCache(key).cache(['sql'], getSqlFn); } else { @@ -1216,6 +1233,20 @@ export class CompilerApi { * on it — and nothing else — is what guarantees the two paths resolve the SAME config and never * advertise a granularity that then fails at query time. */ + // Hash of the static/env granularities config for the compilerVersion suffix. Memoized for the + // array (static-list) form since it can't change; recomputed for the env form (undefined) since + // CUBEJS_GRANULARITIES* can change between calls. Never called for the function form. + private staticGranularityHash(): string { + if (Array.isArray(this.granularities)) { + if (this.staticGranularityHashMemo === undefined) { + this.staticGranularityHashMemo = granularityConfigHash(resolveGlobalGranularitiesSync(this.granularities)); + } + return this.staticGranularityHashMemo; + } + // Only reached for the env form (undefined); the function form never calls this. + return granularityConfigHash(resolveGlobalGranularitiesSync(undefined)); + } + private async resolveGranularities(context: Context): Promise { const securityContext = context?.securityContext ?? {}; return resolveGlobalGranularities(this.granularities, { securityContext }); @@ -1330,7 +1361,9 @@ export class CompilerApi { const compilers = await this.getCompilers(restOptions); const { cubes, granularityHash } = await this.selectGranularityVariant(compilers, requestContext); - // Fixed composition order: base compilerId, then visibility mask, then granularity hash. + // Fixed composition order: base compilerId, then visibility mask, then granularity hash. Only + // computed when the caller actually wants the id — the hashing is skipped when a caller asks + // for view groups alone (the gateway always requests view groups). const composeCompilerId = (visibilityMaskHash: string | null) => { let id = compilers.compilerId; if (visibilityMaskHash) { @@ -1344,7 +1377,10 @@ export class CompilerApi { if (skipVisibilityPatch) { if (includeCompilerId || includeViewGroups) { - const result: any = { cubes, compilerId: composeCompilerId(null) }; + const result: any = { cubes }; + if (includeCompilerId) { + result.compilerId = composeCompilerId(null); + } if (includeViewGroups) { result.viewGroups = compilers.metaTransformer.viewGroups; } @@ -1359,10 +1395,10 @@ export class CompilerApi { cubes ); if (includeCompilerId || includeViewGroups) { - const result: any = { - cubes: patchedCubes, - compilerId: composeCompilerId(visibilityMaskHash), - }; + const result: any = { cubes: patchedCubes }; + if (includeCompilerId) { + result.compilerId = composeCompilerId(visibilityMaskHash); + } if (includeViewGroups) { result.viewGroups = compilers.metaTransformer.viewGroups; } From 459aa2ea8b1fb5e484d96772791c2bfb1b22f611 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Wed, 22 Jul 2026 23:46:46 +0200 Subject: [PATCH 19/22] refactor(granularities): resolve global config once per appId at compile time, bake into compiled model (CUB-2567) --- packages/cubejs-api-gateway/openspec.yml | 2 +- packages/cubejs-api-gateway/src/gateway.ts | 7 +- packages/cubejs-api-gateway/test/mocks.ts | 2 +- .../src/adapter/BaseQuery.js | 12 +- .../src/adapter/Granularity.ts | 3 - .../src/compiler/CubeSymbols.ts | 53 +- .../src/compiler/CubeToMetaTransformer.ts | 177 ++++-- .../src/compiler/GlobalGranularitiesConfig.ts | 6 +- .../src/compiler/GranularityResolver.ts | 9 - .../src/compiler/PrepareCompiler.ts | 23 +- .../src/compiler/index.ts | 1 - .../src/core/CompilerApi.ts | 334 ++-------- .../test/unit/granularities-bake.test.ts | 568 ++++++++++++++++++ .../test/unit/granularity-variants.test.ts | 517 ---------------- 14 files changed, 779 insertions(+), 935 deletions(-) create mode 100644 packages/cubejs-server-core/test/unit/granularities-bake.test.ts delete mode 100644 packages/cubejs-server-core/test/unit/granularity-variants.test.ts diff --git a/packages/cubejs-api-gateway/openspec.yml b/packages/cubejs-api-gateway/openspec.yml index b964ad28c1307..6e187211ae0ec 100644 --- a/packages/cubejs-api-gateway/openspec.yml +++ b/packages/cubejs-api-gateway/openspec.yml @@ -7,7 +7,7 @@ paths: "/v1/granularities": get: summary: "List the granularities available for this deployment" - description: "Returns the granularities enabled in this deployment — built-ins plus any custom granularities defined via `CUBEJS_GRANULARITIES` or `config.granularities`. Evaluated per request context." + description: "Returns the granularities enabled in this deployment — built-ins plus any custom granularities defined via `CUBEJS_GRANULARITIES` or `config.granularities`. Resolved once per application ID at data-model compile time and served from the compiled model." operationId: "granularitiesV1" responses: "200": diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index d71720a836367..62000e7bc7d80 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -742,8 +742,8 @@ class ApiGateway { const cubesConfig = onlyViews ? metaConfig.cubes.filter((c: any) => c.config?.type === 'view') : metaConfig.cubes; - // Time dimensions arrive from CompilerApi with `effectiveGranularities` already attached - // (baked at compile for env/static configs, variant-cached for the function form). + // Time dimensions arrive from CompilerApi with `effectiveGranularities` already attached — + // baked into the compiled model at compile time (resolved once per appId, all config forms). const cubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config); const visibleCubeNames = new Set(cubes.map(c => c.name)); const viewGroups = (metaConfig.viewGroups || []) @@ -776,7 +776,8 @@ class ApiGateway { try { await this.assertApiScope('meta', context.securityContext); const compilerApi = await this.getCompilerApi(context); - const globalConfig = await compilerApi.resolveGlobalGranularitiesConfig(context); + // Serve the per-appId catalog baked into the compiled model — no per-request resolution. + const globalConfig = await compilerApi.getGlobalGranularitiesConfig({ requestId: context.requestId }); const builtInsCatalog = buildBuiltInsCatalog(globalConfig); const granularities: any[] = []; diff --git a/packages/cubejs-api-gateway/test/mocks.ts b/packages/cubejs-api-gateway/test/mocks.ts index 40f6df656a20e..c55c3d692f313 100644 --- a/packages/cubejs-api-gateway/test/mocks.ts +++ b/packages/cubejs-api-gateway/test/mocks.ts @@ -80,7 +80,7 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({ return { query, denied: false }; }, - async resolveGlobalGranularitiesConfig(_ctx: any) { + async getGlobalGranularitiesConfig(_options: any = {}) { return { enabledBuiltIns: ['year', 'month'], customGranularities: { diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index f5e628c1de549..c75544c8b2095 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -141,7 +141,7 @@ export class BaseQuery { this.compilers = compilers; this.options = options || {}; /** @type {import('../compiler/CubeEvaluator').CubeEvaluator} */ - this.cubeEvaluator = compilers.cubeEvaluator.withGranularityDefinitions(this.options.granularityDefinitions); + this.cubeEvaluator = compilers.cubeEvaluator; /** @type {import('../compiler/JoinGraph').JoinGraph} */ this.joinGraph = compilers.joinGraph; @@ -270,10 +270,6 @@ export class BaseQuery { segments: this.options.segments, order: this.options.order, contextSymbols: this.options.contextSymbols, - // Key on the O(1) config hash, NOT the granularityDefinitions map: the map shares one object - // across all plain dimensions by reference, and JSON.stringify would expand it once per time - // dimension, bloating the (serialized, LRU-retained) cache key by O(dimensions × customs). - granularityHash: this.options.granularityHash, timezone: this.options.timezone, limit: this.options.limit, offset: this.options.offset, @@ -4212,8 +4208,6 @@ export class BaseQuery { preAggregationQuery: this.options.preAggregationQuery, useOriginalSqlPreAggregationsInPreAggregation: this.options.useOriginalSqlPreAggregationsInPreAggregation, contextSymbols: this.contextSymbols, - granularityDefinitions: this.options.granularityDefinitions, - granularityHash: this.options.granularityHash, preAggregationsSchema: this.preAggregationsSchemaOption, cubeLatticeCache: this.options.cubeLatticeCache, historyQueries: this.options.historyQueries, @@ -4371,9 +4365,7 @@ export class BaseQuery { const dimensionDef = this.cubeEvaluator.dimensionByPath(path.slice(0, 2)); if (dimensionDef.type === 'time' && this.cubeEvaluator.resolveGranularity( - [path[0], path[1], 'granularities', path[2]], - undefined, - this.options.granularityDefinitions + [path[0], path[1], 'granularities', path[2]] )) { const td = this.newTimeDimension({ dimension: `${path[0]}.${path[1]}`, diff --git a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts index 080efdb856631..514fed014d400 100644 --- a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts +++ b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts @@ -43,13 +43,10 @@ export class Granularity { 'customGranularity', timeDimension.dimension, this.granularity, - JSON.stringify(query.options.granularityDefinitions?.[timeDimension.dimension]?.[this.granularity] || null), ], () => query.cubeEvaluator .resolveGranularity( [...query.cubeEvaluator.parsePath('dimensions', timeDimension.dimension), 'granularities', this.granularity], - undefined, - query.options.granularityDefinitions, ) ); diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 6a389d8c12ea7..516b70ba2fcc2 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -1529,8 +1529,7 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface cube[refProperty].type === 'time' && self.resolveGranularity( [cubeName, refProperty, 'granularities', propertyName], - cube, - query?.options?.granularityDefinitions + cube ) ) { return { @@ -1565,7 +1564,6 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface public resolveGranularity( path: string | string[], refCube?: any, - granularityDefinitions?: Record>, ) { const [cubeName, dimName, gr, granName] = Array.isArray(path) ? path : path.split('.'); const cube = refCube || this.symbols[cubeName]; @@ -1584,53 +1582,8 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface return { interval: `1 ${granName}` }; } - return cube?.[dimName]?.[gr]?.[granName] || - granularityDefinitions?.[`${cubeName}.${dimName}`]?.[granName]; - } - - /** - * Returns a request-owned evaluator facade that keeps all state and method execution on the - * shared evaluator, while binding custom-granularity fallback to one request's effective set. - * The native SQL planner calls resolveGranularity on the evaluator bridge directly, so the - * request context has to live at this seam rather than only at JS adapter call sites. - * - * Must be a Proxy (not Object.create): the native bridge serializes this object's OWN enumerable - * fields (e.g. `primaryKeys`), which a prototype-delegating object would hide. The Proxy forwards - * every property to the real evaluator; to avoid allocating a bound function on every method - * access on the hot query path, bound methods are memoized in `boundCache` (bind once, reuse). - */ - public withGranularityDefinitions( - granularityDefinitions?: Record>, - ): this { - if (!granularityDefinitions || Object.keys(granularityDefinitions).length === 0) { - return this; - } - - const evaluator = this; - const boundResolveGranularity = (path: string | string[], refCube?: any) => evaluator.resolveGranularity( - path, - refCube, - granularityDefinitions, - ); - const boundCache = new Map(); - return new Proxy(this, { - get(target, property) { - if (property === 'resolveGranularity') { - return boundResolveGranularity; - } - const value = Reflect.get(target, property, target); - if (typeof value !== 'function') { - return value; - } - const cached = boundCache.get(property); - if (cached) { - return cached; - } - const bound = value.bind(target); - boundCache.set(property, bound); - return bound; - }, - }); + // Custom granularities (local + baked-in globals) resolve from the compiled cube symbols. + return cube?.[dimName]?.[gr]?.[granName]; } protected cubeDependenciesProxy(parentIndex, cubeName) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index 8a12edfaaea4c..6c89ae6445fab 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -23,14 +23,16 @@ import { resolveNamedNumericFormat, STANDARD_FORMAT_SPECIFIERS, DEFAULT_FORMAT_S import { EffectiveGranularity, NormalizedGranularitiesBlock, + ResolvedGranularitySet, + GRANULARITY_STRING_FIELDS, normalizeGranularitiesBlock, - effectiveGranularitiesFor, + resolveDimensionGranularities, + serializeEffectiveGranularities, } from './GranularityResolver'; import { GlobalGranularitiesConfig, - GranularitiesOption, + DEFAULT_GRANULARITIES_CONFIG, buildBuiltInsCatalog, - resolveGlobalGranularitiesSync, } from './GlobalGranularitiesConfig'; export type CustomNumericFormat = { type: 'custom-numeric'; value: string; alias?: string }; @@ -215,19 +217,23 @@ export class CubeToMetaTransformer implements CompilerInterface { */ public queries: TransformedCube[]; - private readonly granularitiesOption?: GranularitiesOption; - - // Inputs for time dimensions that customize their granularity set, keyed by `cube.dimension`; - // absent dims use the config-wide default. Read by CompilerApi variant builds; never serialized. - public readonly granularityInputs: Map = new Map(); - - // Set during compile() for the context-independent config forms (env / static list); - // null when `granularities` is a function and resolution has to happen per request. - private staticGranularityState: { + // Resolved-once global granularities config for this appId, baked into the compiled model. + // CompilerApi resolves all config forms (env / static / function) before compile and passes the + // result here — the transformer never sees a function. + private readonly granularitiesConfig?: GlobalGranularitiesConfig; + + // Set during compile() from the resolved config. Everything a time dimension needs is precomputed + // once here so the per-dimension loop does no repeated resolution for the common (plain) case: + // - `defaultSet`: the serialized effective set shared by every dimension without a local block; + // - `defaultGlobalCustoms`: the global-custom map (name -> def, `type` stripped) baked into those + // same plain dimensions — invariant across them, so computed once and shared by reference. + // Both derive from a single `resolveDimensionGranularities(EMPTY_BLOCK, config)` pass. + private granularityState!: { config: GlobalGranularitiesConfig; catalog: Record; defaultSet: EffectiveGranularity[]; - } | null = null; + defaultGlobalCustoms: Record; + }; public constructor( cubeValidator: CubeValidator, @@ -235,7 +241,7 @@ export class CubeToMetaTransformer implements CompilerInterface { contextEvaluator: ContextEvaluator, viewGroupEvaluator: ViewGroupEvaluator, joinGraph: JoinGraph, - granularitiesOption?: GranularitiesOption + granularitiesConfig?: GlobalGranularitiesConfig ) { this.cubeValidator = cubeValidator; this.cubeSymbols = cubeEvaluator; @@ -243,33 +249,37 @@ export class CubeToMetaTransformer implements CompilerInterface { this.contextEvaluator = contextEvaluator; this.viewGroupEvaluator = viewGroupEvaluator; this.joinGraph = joinGraph; - this.granularitiesOption = granularitiesOption; + this.granularitiesConfig = granularitiesConfig; this.cubes = []; this.queries = []; } + // The resolved global config baked into this compiled model. Exposed for the /v1/granularities + // endpoint, which serves the per-appId catalog from the compiled model rather than re-resolving. + public get globalGranularitiesConfig(): GlobalGranularitiesConfig | undefined { + return this.granularityState?.config; + } + public get viewGroups(): CompiledViewGroup[] { return this.viewGroupEvaluator.compiledViewGroups; } public compile(_cubes: any[], errorReporter: ErrorReporter): void { - this.granularityInputs.clear(); - // Env/static configs are resolved once here and baked in. The function form must never run - // at compile time (the compiled model is shared across security contexts) — CompilerApi - // resolves it per request from `granularityInputs`. - if (typeof this.granularitiesOption === 'function') { - this.staticGranularityState = null; - } else { - const config = resolveGlobalGranularitiesSync(this.granularitiesOption); - const catalog = buildBuiltInsCatalog(config); - this.staticGranularityState = { - config, - catalog, - // One shared array for every time dimension without local customization — with large - // models this avoids re-allocating an identical granularity set per dimension. - defaultSet: effectiveGranularitiesFor(undefined, config.enabledBuiltIns, config.customGranularities, catalog), - }; - } + // The config is already resolved (env / static / function) by CompilerApi at compile time and + // baked in here — a missing config means the default catalog. + const config = this.granularitiesConfig ?? DEFAULT_GRANULARITIES_CONFIG; + const catalog = buildBuiltInsCatalog(config); + // Resolve the no-local-block ("default") set once; every plain time dimension shares both its + // serialized wire form and its global-custom map by reference (no per-dimension resolution). + const defaultResolved = resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), config.enabledBuiltIns, config.customGranularities, catalog, + ); + this.granularityState = { + config, + catalog, + defaultSet: serializeEffectiveGranularities(defaultResolved), + defaultGlobalCustoms: this.globalCustomsOf(defaultResolved, config, {}), + }; this.cubes = this.cubeSymbols.cubeList .filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator)) @@ -353,10 +363,10 @@ export class CubeToMetaTransformer implements CompilerInterface { const dimensionVisibility = isCubeVisible ? this.isVisible(extendedDimDef, !extendedDimDef.primaryKey) : false; - const granularitiesObj = extendedDimDef.granularities; - // `granularities` keeps its legacy custom-only shape (deprecated). The reconciled set - // is emitted as `effectiveGranularities`: baked in here for env/static global configs, - // or attached per request by CompilerApi variants when the config is a function. + // Snapshot the dimension's LOCAL customs before any merge below: the deprecated + // `granularities` meta field must keep listing only the model's own custom granularities. + const localCustoms = extendedDimDef.granularities; + const localCustomEntries = localCustoms ? Object.entries(localCustoms) : []; const { granularitiesBlock } = extendedDimDef as any; const dimType = this.dimensionDataType(extendedDimDef.type || 'string'); const dimFormat = this.transformDimensionFormat(extendedDimDef); @@ -364,16 +374,25 @@ export class CubeToMetaTransformer implements CompilerInterface { let effectiveGranularities: EffectiveGranularity[] | undefined; if (dimType === 'time') { - const inputs = this.granularityInputsForDimension(cubeTitle, granularitiesObj, granularitiesBlock); + const s = this.granularityState; + const inputs = this.granularityInputsForDimension(cubeTitle, localCustoms, granularitiesBlock); + // Dimensions with a local block resolve individually; plain ones reuse the shared default + // (both the serialized set and the global-custom map) computed once in compile(). + let globalCustoms: Record; if (inputs) { - this.granularityInputs.set(`${cubeName}.${dimensionName}`, inputs); - } - if (this.staticGranularityState) { - const s = this.staticGranularityState; - effectiveGranularities = inputs - ? effectiveGranularitiesFor(inputs, s.config.enabledBuiltIns, s.config.customGranularities, s.catalog) - : s.defaultSet; + const resolved = resolveDimensionGranularities( + inputs, s.config.enabledBuiltIns, s.config.customGranularities, s.catalog, + ); + effectiveGranularities = serializeEffectiveGranularities(resolved); + globalCustoms = this.globalCustomsOf(resolved, s.config, localCustoms ?? {}); + } else { + effectiveGranularities = s.defaultSet; + globalCustoms = s.defaultGlobalCustoms; } + + // Bake the effective GLOBAL customs into the dimension's `granularities` map (SQL resolves + // customs by name from this map, and pre-agg matching reads it). Locals win over globals. + this.mergeGlobalCustomsIntoDimension(cubeName, dimensionName, extendedDimDef, localCustoms, globalCustoms); } return { @@ -395,8 +414,8 @@ export class CubeToMetaTransformer implements CompilerInterface { primaryKey: !!extendedDimDef.primaryKey, aliasMember: extendedDimDef.aliasMember, granularities: - granularitiesObj - ? Object.entries(granularitiesObj).map(([gName, gDef]: [string, any]) => ({ + localCustomEntries.length > 0 + ? localCustomEntries.map(([gName, gDef]: [string, any]) => ({ name: gName, title: this.title(cubeTitle, [gName, gDef], true), interval: gDef.interval, @@ -471,6 +490,72 @@ export class CubeToMetaTransformer implements CompilerInterface { return { includes: block.includes, excludes: block.excludes, custom }; } + // From an already-resolved set, extract the GLOBAL customs a dimension exposes: entries that are + // custom, defined in the global config, and not shadowed by a local of the same name. Projected + // through GRANULARITY_STRING_FIELDS (the shared field list, so it can't drift from serialize/hash). + private globalCustomsOf( + resolved: ResolvedGranularitySet, + config: GlobalGranularitiesConfig, + localCustoms: Record, + ): Record { + const out: Record = {}; + for (const [name, def] of Object.entries(resolved)) { + if (def.type === 'custom' && + Object.prototype.hasOwnProperty.call(config.customGranularities, name) && + !Object.prototype.hasOwnProperty.call(localCustoms, name) + ) { + const projected: GranularityDefinition = {} as GranularityDefinition; + for (const field of GRANULARITY_STRING_FIELDS) { + if (def[field] !== undefined) { + (projected as any)[field] = def[field]; + } + } + out[name] = projected; + } + } + return out; + } + + // Bake the precomputed effective GLOBAL customs (`globalCustoms`) into a time dimension's + // `granularities` map — decision 3 of the per-appId compile-time bake — so the SQL symbol path + // resolves them by name and they participate in pre-agg matching. Locals always win. + // + // The bake must reach BOTH object graphs the downstream paths read: the `cubeList`/`evaluatedCubes` + // object (`dimDef`, read by timeDimensionsForCube → pre-agg matching) AND the separate + // `symbols[cube][dim]` object that CubeSymbols.resolveGranularity reads for SQL. These are distinct + // objects (built by different transforms) even for a plain cube, so the second write is + // load-bearing, not a no-op. If only `dimDef` were written, SQL couldn't resolve a baked global and + // pre-agg matching would throw when it builds a Granularity for it. We REASSIGN `granularities` + // (never mutate in place) so a view dim sharing its source's map by reference isn't contaminated + // before it is itself processed. + private mergeGlobalCustomsIntoDimension( + cubeName: string, + dimensionName: string, + dimDef: ExtendedCubeSymbolDefinition, + localCustoms: Record | undefined, + globalCustoms: Record, + ): void { + if (Object.keys(globalCustoms).length === 0) { + return; + } + const hasLocals = !!localCustoms && Object.keys(localCustoms).length > 0; + + // With no locals the baked map IS the shared globalCustoms — assign it by reference (every plain + // dimension then shares one object). Copy-on-write only when locals must be layered on top. + const write = (existing: Record | undefined) => ( + existing && Object.keys(existing).length > 0 + ? { ...globalCustoms, ...existing } // globals first, locals last so locals win on collisions + : globalCustoms + ); + + dimDef.granularities = write(hasLocals ? localCustoms : undefined); + + const symbolDim = (this.cubeEvaluator as any).symbols?.[cubeName]?.[dimensionName]; + if (symbolDim && symbolDim !== dimDef) { + symbolDim.granularities = write(symbolDim.granularities as Record | undefined); + } + } + public queriesForContext(contextId: string | null | undefined): TransformedCube[] { // return All queries if no context pass if (contextId == null || contextId.length === 0) { diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts index 307eee97d81a6..01ee3d3116563 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -40,7 +40,7 @@ export type GlobalGranularitiesConfig = { customGranularities: Readonly>; }; -const DEFAULT_CONFIG: GlobalGranularitiesConfig = Object.freeze({ +export const DEFAULT_GRANULARITIES_CONFIG: GlobalGranularitiesConfig = Object.freeze({ enabledBuiltIns: BUILT_IN_GRANULARITY_NAMES, customGranularities: Object.freeze({}), }); @@ -69,7 +69,7 @@ function applyEnvOverrides(name: string, base?: Partial): function resolveFromEnv(): GlobalGranularitiesConfig { const list = getEnv('granularities'); if (!list || list.length === 0) { - return DEFAULT_CONFIG; + return DEFAULT_GRANULARITIES_CONFIG; } const enabledBuiltIns: string[] = []; @@ -168,7 +168,7 @@ export async function resolveGlobalGranularities( // A function opts out of env vars entirely, so a non-array return (null / undefined / a stray // object) means "no explicit config" → the default built-in catalog, NOT an env fallback that // would leak CUBEJS_GRANULARITIES into a context the function meant to leave unconfigured. - return Array.isArray(resolved) ? resolveFromList(resolved) : DEFAULT_CONFIG; + return Array.isArray(resolved) ? resolveFromList(resolved) : DEFAULT_GRANULARITIES_CONFIG; } return resolveGlobalGranularitiesSync(userValue); } diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts index bd5a52f9059b5..f81a0aa939397 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -97,15 +97,6 @@ export type EffectiveGranularity = { format?: string; }; -// The per-config effective granularity data, sparse: one `defaultSet` shared by every time -// dimension without a local block, plus per-dimension `overrides` for the rare dims that declared -// one. Attached onto the base meta cubes at read time (see CompilerApi.attachEffectiveGranularities) -// instead of storing a full enriched cube copy. -export type GranularitySets = { - defaultSet: EffectiveGranularity[]; - overrides: Map; -}; - // Serialize a resolved set for /v1/meta. `title` always present (falls back to the name); the // other string fields are included only when defined. Order follows GRANULARITY_STRING_FIELDS, // which also drives the config hash — the two share the field list so they can't drift. diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 2cfd5c2650703..6083aa5d552bc 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -26,8 +26,7 @@ import { CompilerCache } from './CompilerCache'; import { YamlCompiler } from './YamlCompiler'; import { ViewCompilationGate } from './ViewCompilationGate'; import type { ErrorReporter } from './ErrorReporter'; -import type { GranularitiesOption } from './GlobalGranularitiesConfig'; -import type { GranularitySets } from './GranularityResolver'; +import type { GlobalGranularitiesConfig } from './GlobalGranularitiesConfig'; export type PrepareCompilerOptions = { nativeInstance?: NativeInstance, @@ -42,9 +41,10 @@ export type PrepareCompilerOptions = { compiledScriptCache?: LRUCache; compiledYamlCache?: LRUCache; compiledJinjaCache?: LRUCache; - // Global `granularities` config: env/static forms are resolved at compile time by - // CubeToMetaTransformer; the function form per request by CompilerApi. - granularities?: GranularitiesOption; + // Resolved-once global granularities config for this appId, baked into the compiled model by + // CubeToMetaTransformer. CompilerApi resolves all config forms (env / static / function) before + // compile and passes the result here. + granularitiesConfig?: GlobalGranularitiesConfig; }; export interface CompilerInterface { @@ -61,17 +61,6 @@ export type Compiler = { compilerCache: CompilerCache; headCommitId?: string; compilerId: string; - // Per-config effective granularity SETS keyed by config hash — NOT enriched cube copies. The - // sparse `{ defaultSet, overrides }` shape is a few KB (one shared array + the rare local-block - // dims) rather than a ~MB clone of the whole meta; CompilerApi attaches these onto the base - // cubes cheaply at read time. Owned by the compiled model so a recompile discards it. Bounded - // LRU, function form only. - granularityVariants?: Map>; - // Per-request SQL-path global-custom lookups (dim -> name -> def) keyed by config hash. Same - // ownership/lifecycle as granularityVariants: a pure function of (model, config), cached here - // and discarded on recompile, bounded to match the variant cache. Promise-valued so concurrent - // misses of one hash coalesce into a single build. - granularityDefinitions?: Map>>>; }; export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}): Compiler => { @@ -85,7 +74,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const contextEvaluator = new ContextEvaluator(cubeEvaluator); const viewGroupEvaluator = new ViewGroupEvaluator(cubeEvaluator, cubeValidator); const joinGraph = new JoinGraph(cubeValidator, cubeEvaluator); - const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, viewGroupEvaluator, joinGraph, options.granularities); + const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, viewGroupEvaluator, joinGraph, options.granularitiesConfig); const { maxQueryCacheSize, maxQueryCacheAge } = options; const compilerCache = new CompilerCache({ maxQueryCacheSize, maxQueryCacheAge }); const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler); diff --git a/packages/cubejs-schema-compiler/src/compiler/index.ts b/packages/cubejs-schema-compiler/src/compiler/index.ts index ccba0c6674e27..26b60fcb0f059 100644 --- a/packages/cubejs-schema-compiler/src/compiler/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/index.ts @@ -31,7 +31,6 @@ export { NormalizedGranularitiesBlock, ResolvedGranularitySet, EffectiveGranularity, - GranularitySets, GRANULARITY_STRING_FIELDS, normalizeGranularitiesBlock, resolveDimensionGranularities, diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index bd374c89ea143..74d63f4740cc0 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -4,19 +4,14 @@ import { AccessPolicyDefinition, BaseQuery, CanUsePreAggregationFn, - buildBuiltInsCatalog, compile, Compiler, createQuery, CubeDefinition, - EffectiveGranularity, EvaluatedCube, GlobalGranularitiesConfig, GranularitiesOption, - GranularitySets, granularityConfigHash, - NormalizedGranularitiesBlock, - normalizeGranularitiesBlock, PreAggregationFilters, PreAggregationInfo, PreAggregationReferences, @@ -24,10 +19,7 @@ import { prepareCompiler, queryClass, QueryFactory, - effectiveGranularitiesFor, - resolveDimensionGranularities, resolveGlobalGranularities, - resolveGlobalGranularitiesSync, TransformedQuery, ViewIncludedMember, } from '@cubejs-backend/schema-compiler'; @@ -100,12 +92,6 @@ export interface DataSourceInfo { } export class CompilerApi { - // Bound on cached granularity-enriched meta variants per compiled model (~2 MB each on a - // 3k-cube model). Contains a `granularities` function keyed on a high-cardinality context - // fact; legitimate configs (calendars × locales) stay well under it. Exceeding evicts LRU - // and logs. Kept below CubeSQL's LRU-100 compilerId-keyed cache. - protected static readonly MAX_GRANULARITY_VARIANTS = 64; - protected readonly repository: SchemaFileRepository; protected readonly dbType: DbTypeInternalFn; @@ -154,9 +140,12 @@ export class CompilerApi { protected readonly granularities?: GranularitiesOption; - // Memoized hash of a STATIC-list granularities config (immutable for the instance lifetime). - // Undefined until first computed; the env form is never memoized here (env can change at runtime). - private staticGranularityHashMemo?: string; + // Resolved-once-per-appId global granularities config, baked into the compiled model. Resolved in + // getCompilers with the appId-level compile context (all three forms — env / static / function). + // The in-flight PROMISE is memoized (not just the value) so concurrent initial getCompilers calls + // share a single resolve/await — the function form is invoked exactly once per appId. Cleared on + // rejection so a failed resolve can be retried. + private resolvedGranularitiesPromise?: Promise<{ config: GlobalGranularitiesConfig; hash: string }>; protected queryFactory?: QueryFactory; @@ -250,16 +239,15 @@ export class CompilerApi { compilerVersion += `_${crypto.createHash('md5').update(JSON.stringify(files)).digest('hex')}`; } - // Env/static granularity configs are baked into the compiled meta, so a config change must - // force a recompile. The function form is resolved per request instead (variant cache) and - // must never churn the compiler version. A static LIST is immutable for the instance lifetime, - // so its hash is memoized; the env form stays per-call since env vars can change at runtime. - if (typeof this.granularities !== 'function') { - compilerVersion += `_gran_${this.staticGranularityHash()}`; - } + // Resolve the global granularities config ONCE per appId with the compile context (handles all + // forms — env / static / function — awaiting the function). The resolved config is baked into + // the compiled model, so its hash goes into compilerVersion for every form: a config change + // then forces a recompile. Memoized on the instance so repeated calls don't re-resolve/re-await. + const { config: resolvedGranularities, hash: granularitiesHash } = await this.getResolvedGranularities(); + compilerVersion += `_gran_${granularitiesHash}`; if (!this.compilers || this.compilerVersion !== compilerVersion) { - this.compilers = this.compileSchema(compilerVersion, options.requestId).catch(e => { + this.compilers = this.compileSchema(compilerVersion, resolvedGranularities, options.requestId).catch(e => { this.compilers = undefined; throw e; }); @@ -269,6 +257,33 @@ export class CompilerApi { return this.compilers; } + // Resolve `config.granularities` with the appId-level compile context. The static-list and function + // forms are IMMUTABLE for the instance (a static list can't change; the function is frozen per + // appId by design), so they are memoized — including the in-flight promise, so concurrent initial + // compiles share one resolve/await and the function is invoked exactly once. The ENV form + // (`granularities === undefined`, reading CUBEJS_GRANULARITIES*) is NOT memoized: env can change at + // runtime, and re-resolving each call lets the folded hash trigger a recompile — matching rev-1. + private getResolvedGranularities(): Promise<{ config: GlobalGranularitiesConfig; hash: string }> { + const resolve = () => resolveGlobalGranularities( + this.granularities, + { securityContext: this.compileContext?.securityContext }, + ).then(config => ({ config, hash: granularityConfigHash(config) })); + + // Env form: always re-resolve (cheap env parse) so a runtime change is picked up. + if (this.granularities === undefined) { + return resolve(); + } + + if (!this.resolvedGranularitiesPromise) { + this.resolvedGranularitiesPromise = resolve(); + // Allow a retry if the resolve (e.g. a throwing function form) fails. + this.resolvedGranularitiesPromise.catch(() => { + this.resolvedGranularitiesPromise = undefined; + }); + } + return this.resolvedGranularitiesPromise; + } + /** * Creates the compilers instances without model compilation, * because it could fail and no compilers will be returned. @@ -284,7 +299,7 @@ export class CompilerApi { }); } - public async compileSchema(compilerVersion: string, requestId?: string): Promise { + public async compileSchema(compilerVersion: string, granularitiesConfig: GlobalGranularitiesConfig, requestId?: string): Promise { const startCompilingTime = new Date().getTime(); try { this.logger(this.compilers ? 'Recompiling schema' : 'Compiling schema', { @@ -301,7 +316,7 @@ export class CompilerApi { compiledScriptCache: this.compiledScriptCache, compiledJinjaCache: this.compiledJinjaCache, compiledYamlCache: this.compiledYamlCache, - granularities: this.granularities, + granularitiesConfig, }); this.queryFactory = await this.createQueryFactory(compilers); @@ -350,12 +365,9 @@ export class CompilerApi { public async getSqlGenerator(query: NormalizedQuery, dataSource?: string): Promise<{ sqlGenerator: any; compilers: Compiler }> { const dbType = await this.getDbType(dataSource); const compilers = await this.getCompilers({ requestId: query.requestId }); - // `granularityDefinitions` (a reference-shared map) flows to resolveGranularity for SQL, but the - // query cache must key on the O(1) `granularityHash` instead — JSON.stringify'ing the map would - // expand the shared references once per time dimension and bloat the cache key. - const { definitions: granularityDefinitions, hash: granularityHash } = await this.granularityDefinitionsForQuery(compilers, query); - const queryWithGranularities = { ...query, granularityDefinitions, granularityHash }; - let sqlGenerator = await this.createQueryByDataSource(compilers, queryWithGranularities, dataSource, dbType); + // Custom granularities (local + baked-in globals) resolve from the compiled symbols, so no + // per-request granularity threading is needed here. + let sqlGenerator = await this.createQueryByDataSource(compilers, query, dataSource, dbType); if (!sqlGenerator) { throw new Error(`Unknown dbType: ${dbType}`); @@ -371,7 +383,7 @@ export class CompilerApi { // TODO consider more efficient way than instantiating query sqlGenerator = await this.createQueryByDataSource( compilers, - queryWithGranularities, + query, dataSource, _dbType ); @@ -387,112 +399,6 @@ export class CompilerApi { return { sqlGenerator, compilers }; } - /** - * Request-scoped custom granularities keyed by time dimension. The shared cube evaluator cannot - * contain function-config results because it is reused across security contexts, so query-time - * resolution receives this immutable lookup instead. Building it through the same resolver and - * per-dimension inputs as meta keeps includes/excludes behavior identical on both paths. - * - * The result is a pure function of (compiled model, resolved config), so it is cached on the - * compiled model keyed by the canonical config hash — the same discriminator the meta variant - * cache uses. Steady state is O(config) (resolve + hash + map lookup); the O(model) scan runs - * once per distinct config, not once per query. Recompile discards the cache with `compilers`. - */ - protected async granularityDefinitionsForQuery( - compilers: Compiler, - query: NormalizedQuery, - ): Promise<{ definitions: Record>; hash: string | null }> { - // Resolve through the same securityContext-only seam as the meta path so both paths agree on - // the effective set — a `granularities` function must key only on securityContext. - const { contextSymbols } = query as any; - const config = await this.resolveGranularities({ securityContext: contextSymbols?.securityContext }); - - // Only GLOBAL customs need threading (locals already resolve via the compiled symbols map, and - // built-ins via the predefined path). With none configured there's nothing to add, so skip both - // the cache and the scan — the common case falls straight through to the symbols map. A null - // hash then keeps the query-cache key free of any granularity discriminator. - if (Object.keys(config.customGranularities).length === 0) { - return { definitions: {}, hash: null }; - } - - if (!compilers.granularityDefinitions) { - compilers.granularityDefinitions = new Map(); - } - const cache = compilers.granularityDefinitions; - const hash = granularityConfigHash(config); - - // Promise-valued (like granularityVariants) so concurrent misses of one hash coalesce into a - // single build rather than each re-running the O(model) scan. - let built = cache.get(hash); - if (built) { - // Refresh LRU recency. - cache.delete(hash); - cache.set(hash, built); - } else { - if (cache.size >= CompilerApi.MAX_GRANULARITY_VARIANTS) { - cache.delete(cache.keys().next().value); - } - built = Promise.resolve().then(() => this.buildGranularityDefinitions(compilers, config)); - cache.set(hash, built); - built.catch(() => { - if (cache.get(hash) === built) { - cache.delete(hash); - } - }); - } - return { definitions: await built, hash }; - } - - /** - * One O(model) pass building the SQL-path global-custom lookup: `dim -> { customName -> def }`, - * respecting each dimension's includes/excludes exactly as the meta path does. Emits only global - * customs (locals/built-ins resolve elsewhere) with the meta-only `type` tag stripped. - */ - protected buildGranularityDefinitions( - compilers: Compiler, - config: GlobalGranularitiesConfig, - ): Record> { - const { enabledBuiltIns, customGranularities } = config; - const catalog = buildBuiltInsCatalog(config); - const inputs = compilers.metaTransformer.granularityInputs; - const definitions: Record> = {}; - - // Keep only the global customs a dimension's resolved set actually exposes, `type` tag stripped. - const globalCustomsOf = (block: NormalizedGranularitiesBlock) => Object.fromEntries( - Object.entries(resolveDimensionGranularities(block, enabledBuiltIns, customGranularities, catalog)) - .filter(([name, def]) => def.type === 'custom' && - Object.prototype.hasOwnProperty.call(customGranularities, name)) - .map(([name, { type: _type, ...def }]) => [name, def]) - ); - - // A time dimension WITHOUT a local block resolves against the empty block, so it exposes every - // global custom with no filtering — the same map for all of them. Local blocks (rare) are the - // only dimensions needing individual reconciliation, so only those are walked. No full model - // scan: cost is O(global customs) + O(local-block dims), not O(all time dimensions). - const shared = globalCustomsOf(normalizeGranularitiesBlock(undefined)); - - if (Object.keys(shared).length > 0) { - // A time dimension is "plain" iff it isn't in `granularityInputs` (which holds only dims that - // declared a local granularities block). Assign the shared map by reference to each plain dim. - for (const cube of compilers.metaTransformer.cubes) { - for (const dim of cube.config.dimensions || []) { - if (dim.type === 'time' && !inputs.has(dim.name)) { - definitions[dim.name] = shared; - } - } - } - } - - for (const [dimName, block] of inputs) { - const customs = globalCustomsOf(block); - if (Object.keys(customs).length > 0) { - definitions[dimName] = customs; - } - } - - return definitions; - } - public async getSql(query: NormalizedQuery, options: GetSqlOptions = {}): Promise { const { includeDebugInfo, exportAnnotatedSql, preAggregationsOnly } = options; const { sqlGenerator, compilers } = await this.getSqlGenerator(query); @@ -520,8 +426,6 @@ export class CompilerApi { const key = { query: keyOptions, options, - // Key on the O(1) hash, not the reference-shared definitions map (see getQueryCache). - granularityHash: sqlGenerator.options.granularityHash, }; return compilers.compilerCache.getQueryCache(key).cache(['sql'], getSqlFn); } else { @@ -1227,130 +1131,14 @@ export class CompilerApi { } /** - * Resolve the global granularity config for a request. A `granularities` function may depend - * only on `securityContext` (like queryRewrite and access policies): the meta path and the SQL - * path receive different-shaped request objects, but both carry the security context, so keying - * on it — and nothing else — is what guarantees the two paths resolve the SAME config and never - * advertise a granularity that then fails at query time. + * The resolved-once global granularities config baked into the compiled model. Serves the + * /v1/granularities endpoint (per-appId catalog) — no per-request resolution. */ - // Hash of the static/env granularities config for the compilerVersion suffix. Memoized for the - // array (static-list) form since it can't change; recomputed for the env form (undefined) since - // CUBEJS_GRANULARITIES* can change between calls. Never called for the function form. - private staticGranularityHash(): string { - if (Array.isArray(this.granularities)) { - if (this.staticGranularityHashMemo === undefined) { - this.staticGranularityHashMemo = granularityConfigHash(resolveGlobalGranularitiesSync(this.granularities)); - } - return this.staticGranularityHashMemo; - } - // Only reached for the env form (undefined); the function form never calls this. - return granularityConfigHash(resolveGlobalGranularitiesSync(undefined)); - } - - private async resolveGranularities(context: Context): Promise { - const securityContext = context?.securityContext ?? {}; - return resolveGlobalGranularities(this.granularities, { securityContext }); - } - - /** - * Global granularity config for a request context. O(config): env/static forms ignore the - * context; the function form is invoked with it. Used by the /v1/granularities endpoint. - */ - public async resolveGlobalGranularitiesConfig(context: Context): Promise { - return this.resolveGranularities(context); - } - - /** - * Meta cubes with `effectiveGranularities` attached, plus the hash to mix into compilerId. - * Env/static configs are baked into the base meta at compile time (null hash); a function - * config resolves per request. The cache holds the sparse per-config granularity SETS (a few KB) - * — not enriched cube copies — keyed by config hash and owned by the compiled model, so a - * recompile discards it; the (cheap) attach onto base cubes happens here at read time. - */ - protected async selectGranularityVariant( - compilers: Compiler, - requestContext: Context, - ): Promise<{ cubes: any[]; granularityHash: string | null }> { - if (typeof this.granularities !== 'function') { - return { cubes: compilers.metaTransformer.cubes, granularityHash: null }; - } - - const config = await this.resolveGranularities(requestContext); - const granularityHash = granularityConfigHash(config); - - if (!compilers.granularityVariants) { - compilers.granularityVariants = new Map(); - } - const cache = compilers.granularityVariants; - - let sets = cache.get(granularityHash); - if (sets) { - // Refresh LRU recency. - cache.delete(granularityHash); - cache.set(granularityHash, sets); - } else { - if (cache.size >= CompilerApi.MAX_GRANULARITY_VARIANTS) { - const oldest = cache.keys().next().value; - cache.delete(oldest); - this.logger('Granularity variant cache is full', { - warning: `More than ${CompilerApi.MAX_GRANULARITY_VARIANTS} distinct granularity configs seen for one compiled model; ` + - 'evicting the least recently used variant. A `granularities` function returning unstable values ' + - 'causes per-request meta rebuilds and churns compilerId-based caches (e.g. in CubeSQL).', - }); - } - sets = Promise.resolve().then(() => this.buildGranularitySets(compilers, config)); - cache.set(granularityHash, sets); - sets.catch(() => { - if (cache.get(granularityHash) === sets) { - cache.delete(granularityHash); - } - }); - } - - return { cubes: this.attachEffectiveGranularities(compilers.metaTransformer.cubes, await sets), granularityHash }; - } - - /** - * The sparse effective granularity data for one config. The expensive reconciliation runs ONLY - * for the rare dimensions with a local block; every plain dimension uses one shared `defaultSet`. - * No cube copying here — that happens cheaply in `attachEffectiveGranularities` at read time. - */ - protected buildGranularitySets(compilers: Compiler, config: GlobalGranularitiesConfig): GranularitySets { - const catalog = buildBuiltInsCatalog(config); - const { enabledBuiltIns, customGranularities } = config; - const overrides = new Map(); - for (const [dimName, block] of compilers.metaTransformer.granularityInputs) { - overrides.set(dimName, effectiveGranularitiesFor(block, enabledBuiltIns, customGranularities, catalog)); - } - return { - defaultSet: effectiveGranularitiesFor(undefined, enabledBuiltIns, customGranularities, catalog), - overrides, - }; - } - - /** - * Attach the per-config effective sets onto the base meta cubes without mutating them: copy only - * the cubes/dimensions touched, and reference the shared `defaultSet` for plain time dimensions - * (per-dim `overrides` for the rare local-block ones). O(time dimensions) shallow work — no - * reconciliation, no deep clone of measures/segments/etc. - */ - protected attachEffectiveGranularities(baseCubes: any[], sets: GranularitySets): any[] { - return baseCubes.map((cube: any) => { - if (!cube.config.dimensions?.some((d: any) => d.type === 'time')) { - return cube; - } - return { - ...cube, - config: { - ...cube.config, - dimensions: cube.config.dimensions.map((dim: any) => ( - dim.type === 'time' - ? { ...dim, effectiveGranularities: sets.overrides.get(dim.name) ?? sets.defaultSet } - : dim - )), - }, - }; - }); + public async getGlobalGranularitiesConfig(options: { requestId?: string } = {}): Promise { + const compilers = await this.getCompilers(options); + // `getCompilers` has compiled the model, so the baked config is always present; fall back to the + // memoized resolve only defensively (should not happen on a successfully compiled model). + return compilers.metaTransformer.globalGranularitiesConfig ?? (await this.getResolvedGranularities()).config; } public async metaConfig( @@ -1359,19 +1147,17 @@ export class CompilerApi { ): Promise { const { includeCompilerId, includeViewGroups, skipVisibilityPatch, ...restOptions } = options; const compilers = await this.getCompilers(restOptions); - const { cubes, granularityHash } = await this.selectGranularityVariant(compilers, requestContext); + // Granularities are baked into the compiled model, so the base meta cubes are served directly. + const cubes = compilers.metaTransformer.cubes; - // Fixed composition order: base compilerId, then visibility mask, then granularity hash. Only - // computed when the caller actually wants the id — the hashing is skipped when a caller asks - // for view groups alone (the gateway always requests view groups). + // Fixed composition order: base compilerId, then visibility mask. Only computed when the caller + // actually wants the id — the hashing is skipped when a caller asks for view groups alone (the + // gateway always requests view groups). const composeCompilerId = (visibilityMaskHash: string | null) => { let id = compilers.compilerId; if (visibilityMaskHash) { id = this.mixInMaskHash(id, visibilityMaskHash); } - if (granularityHash) { - id = this.mixInMaskHash(id, granularityHash); - } return id; }; @@ -1412,11 +1198,11 @@ export class CompilerApi { options?: { requestId?: string } ): Promise<{ metaConfig: any; cubeDefinitions: Record }> { const compilers = await this.getCompilers(options); - const { cubes } = await this.selectGranularityVariant(compilers, requestContext); + // Granularities are baked into the compiled model, so the base meta cubes are used directly. const { cubes: patchedCubes } = await this.patchVisibilityByAccessPolicy( compilers, requestContext, - cubes + compilers.metaTransformer.cubes ); return { metaConfig: patchedCubes, diff --git a/packages/cubejs-server-core/test/unit/granularities-bake.test.ts b/packages/cubejs-server-core/test/unit/granularities-bake.test.ts new file mode 100644 index 0000000000000..fa5ac864e9da3 --- /dev/null +++ b/packages/cubejs-server-core/test/unit/granularities-bake.test.ts @@ -0,0 +1,568 @@ +import { SchemaFileRepository } from '@cubejs-backend/shared'; +import { CompilerApi } from '../../src/core/CompilerApi'; +import { DbTypeInternalFn } from '../../src/core/types'; + +// CUB-2567 rev 2: global granularity config is resolved ONCE per appId at compile time +// (env | static list | function(ctx)), its hash folded into compilerVersion for ALL forms, and the +// effective per-time-dimension set baked into the compiled model. Global CUSTOM granularities are +// merged into each time dimension's `granularities` symbol map at compile (locals win). Both +// /v1/meta and the SQL path read the baked values — no per-request resolution. This suite verifies +// the bake, the fold-into-compilerVersion, and the resolve-once guarantee. + +// A CompilerApi that counts how many times the granularities FUNCTION form is invoked, so we can +// assert it runs once per compile (per appId), not once per metaConfig/getSql call. +class TestableCompilerApi extends CompilerApi { + public version(): string | undefined { + return this.compilerVersion; + } +} + +const repository: SchemaFileRepository = { + localPath: () => '/mock/path', + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'orders.js', + content: ` + cube('Orders', { + sql: 'SELECT * FROM orders', + measures: { count: { type: 'count' } }, + dimensions: { + id: { sql: 'id', type: 'number', primaryKey: true }, + created_at: { sql: 'created_at', type: 'time' }, + updated_at: { sql: 'updated_at', type: 'time' }, + excluded_at: { + sql: 'updated_at', + type: 'time', + granularities: { excludes: ['fiscal_year', 'sprint'] }, + }, + collide_at: { + sql: 'created_at', + type: 'time', + // Local custom named 'fiscal_year' collides with a same-named global custom. + granularities: { fiscal_year: { interval: '1 year', origin: '2020-01-01' } }, + }, + }, + }); + cube('Events', { + sql: 'SELECT * FROM events', + measures: { count: { type: 'count' } }, + dimensions: { + ts: { + sql: 'ts', + type: 'time', + granularities: { + fiscal_year: { interval: '1 year', origin: '2024-02-01' }, + }, + }, + }, + }); + cube('Products', { + sql: 'SELECT * FROM products', + measures: { count: { type: 'count' } }, + dimensions: { + name: { sql: 'name', type: 'string' }, + }, + }); + `, + }, + ]), +}; + +const mockDbType: DbTypeInternalFn = async () => 'postgres'; + +const noopLogger = () => { /* silent */ }; + +const createApi = (options: any = {}) => new TestableCompilerApi(repository, mockDbType, { + logger: noopLogger, + ...options, +}); + +const ctxFor = (tenant: string) => ({ securityContext: { tenant }, requestId: `req-${tenant}` }); + +const dimByName = (cubes: any[], name: string) => cubes + .flatMap((c: any) => c.config.dimensions) + .find((d: any) => d.name === name); + +const granularityNames = (dim: any) => dim.effectiveGranularities.map((g: any) => g.name); + +const ALL_BUILT_INS = ['year', 'quarter', 'month', 'week', 'day', 'hour', 'minute', 'second']; + +const queryFor = (tenant: string, dimension: string, granularity: string): any => ({ + measures: [dimension.startsWith('Events.') ? 'Events.count' : 'Orders.count'], + timeDimensions: [{ dimension, granularity }], + timezone: 'UTC', + contextSymbols: { securityContext: { tenant } }, + requestId: `sql-${tenant}-${dimension}-${granularity}`, +}); + +// The compiled dimension symbol's `granularities` map — the map the SQL layer and the pre-agg +// matcher resolve customs by name from. Global customs are baked into it at compile time. +const bakedDimGranularities = async (api: CompilerApi, dimPath: string): Promise> => { + const compilers = await (api as any).getCompilers(); + return compilers.cubeEvaluator.dimensionByPath(dimPath).granularities || {}; +}; + +// The SEPARATE `symbols[cube][dim]` map that CubeSymbols.resolveGranularity reads for SQL. The +// compile-time bake dual-writes here too; without it SQL can't resolve a baked global custom. +const symbolsDimGranularities = async (api: CompilerApi, cube: string, dim: string): Promise> => { + const compilers = await (api as any).getCompilers(); + return (compilers.cubeEvaluator as any).symbols?.[cube]?.[dim]?.granularities || {}; +}; + +describe('granularities baked at compile time (CUB-2567 rev 2)', () => { + describe('env / static config', () => { + afterEach(() => { + delete process.env.CUBEJS_GRANULARITIES; + }); + + // 1. No config -> every time dimension gets all built-ins baked as effectiveGranularities; + // internal fields stay off the wire. + test('no config: every time dimension gets all built-ins baked; internal fields stay off the wire', async () => { + const api = createApi(); + const cubes = await api.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toEqual(ALL_BUILT_INS); + // Local custom granularity survives on top of the enabled built-ins. + expect(granularityNames(dimByName(cubes, 'Events.ts'))).toEqual([...ALL_BUILT_INS, 'fiscal_year']); + // Non-time dimensions are untouched. + expect(dimByName(cubes, 'Products.name').effectiveGranularities).toBeUndefined(); + // The raw normalized block never reaches the wire. + expect(dimByName(cubes, 'Orders.created_at').granularitiesBlock).toBeUndefined(); + api.dispose(); + }); + + // 2. Static global customs are baked into BOTH dimension object graphs the downstream paths + // read: the evaluatedCubes/cubeList copy (dimensionByPath, timeDimensionsForCube, pre-agg + // matching) AND the separate symbols copy (CubeSymbols.resolveGranularity for SQL). The merge + // must reach both — writing only the first makes SQL unable to resolve a global custom, and + // pre-agg matching (granularityHierarchies builds a Granularity for every baked custom) then + // throws. This test drives a real getSql() on a model with a configured global custom to lock + // that in, and confirms per-dimension excludes are honored. + test('static global customs are baked into the consumed map; SQL resolves them; excludes honored', async () => { + const api = createApi({ + granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-02-01' }], + }); + + // Baked into a plain dimension's granularities map (the map SQL / pre-agg matching read). + const baked = await bakedDimGranularities(api, 'Orders.created_at'); + expect(baked.fiscal_year).toMatchObject({ interval: '1 year', origin: '2024-02-01' }); + + // The dimension that excludes the global custom does NOT get it baked in, matching meta. + const bakedExcluded = await bakedDimGranularities(api, 'Orders.excluded_at'); + expect(bakedExcluded.fiscal_year).toBeUndefined(); + + // Meta agrees: excluded dimension omits it; plain dimensions advertise it. + const cubes = await api.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toContain('fiscal_year'); + expect(granularityNames(dimByName(cubes, 'Orders.excluded_at'))).not.toContain('fiscal_year'); + + // Real SQL path: a query at the global custom grain must resolve (both object graphs carry the + // merge) and not throw during pre-agg matching. Previously this threw "Granularity does not + // exist" because the symbols copy lacked the merge. + const { sql } = await api.getSql({ + measures: [], + dimensions: [], + segments: [], + filters: [], + timeDimensions: [{ + dimension: 'Orders.created_at', + // Custom granularity name — allowed at runtime; the wire type only enumerates built-ins. + granularity: 'fiscal_year' as any, + dateRange: ['2024-01-01', '2025-01-01'], + }], + order: [], + } as any); + const sqlText = Array.isArray(sql) ? sql[0] : sql; + expect(typeof sqlText).toBe('string'); + expect(sqlText.length).toBeGreaterThan(0); + api.dispose(); + }); + + // 3. Time dimensions without local customization share ONE default set instance (memory saver). + test('time dimensions without local customization share one default set instance', async () => { + const api = createApi(); + const cubes = await api.metaConfig(ctxFor('a'), {}); + const created = dimByName(cubes, 'Orders.created_at'); + const updated = dimByName(cubes, 'Orders.updated_at'); + expect(created.effectiveGranularities).toBe(updated.effectiveGranularities); + // A dimension with a local block gets its own set, not the shared default. + expect(dimByName(cubes, 'Events.ts').effectiveGranularities) + .not.toBe(created.effectiveGranularities); + api.dispose(); + }); + + // 4. Resolved config hash is folded into compilerVersion, so a different env config yields a + // different compilerVersion (and therefore a recompile / different compiled model). + // + // Note: the resolved config is cached per CompilerApi instance for its lifetime (resolved once + // per appId), so changing the env var on the SAME instance does NOT re-resolve. The fold-into- + // compilerVersion contract is verified across two instances reading different env values — a + // process restart / new appId is exactly when env is re-read. + test('resolved env config hash folds into compilerVersion (different env -> different version)', async () => { + const apiDefault = createApi(); + const cubesDefault = await apiDefault.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubesDefault, 'Orders.created_at'))).toEqual(ALL_BUILT_INS); + const versionDefault = apiDefault.version(); + expect(versionDefault).toContain('_gran_'); + + process.env.CUBEJS_GRANULARITIES = 'year,month'; + const apiEnv = createApi(); + const cubesEnv = await apiEnv.metaConfig(ctxFor('a'), {}); + // Different resolved config -> different folded hash -> different compilerVersion. + expect(apiEnv.version()).not.toBe(versionDefault); + expect(granularityNames(dimByName(cubesEnv, 'Orders.created_at'))).toEqual(['year', 'month']); + + apiDefault.dispose(); + apiEnv.dispose(); + }); + }); + + describe('function config resolved once per compile', () => { + // 5. THE key new-architecture guarantee: the function form is invoked ONCE per compile (per + // appId), not once per metaConfig/getSql call. Multiple reads on the same instance add zero + // additional invocations. + test('function form is invoked once per compile, not per metaConfig/getSql call', async () => { + let calls = 0; + const api = createApi({ + granularities: () => { + calls += 1; + return ['year', 'month']; + }, + }); + + // First read triggers the single compile-time resolution. + await api.metaConfig(ctxFor('a'), {}); + expect(calls).toBe(1); + + // Further reads (meta or SQL, any context) reuse the baked compile — no re-invocation. + await api.metaConfig(ctxFor('b'), {}); + await api.metaConfig(ctxFor('a'), {}); + await api.getSqlGenerator(queryFor('a', 'Orders.created_at', 'month')); + await api.getSqlGenerator(queryFor('b', 'Orders.created_at', 'year')); + expect(calls).toBe(1); + api.dispose(); + }); + + // 5b. INTENDED per-appId contract (CUB-2567 rev 2 narrowing decision): within a single appId + // (one CompilerApi instance), granularities do NOT vary by request securityContext. The + // function form is resolved ONCE with the instance's compile context, so two requests with + // DIFFERENT request securityContexts see the SAME baked granularities. + test('two request contexts on the same appId get identical baked granularities', async () => { + let seenCtx: any; + const api = createApi({ + // Would branch per-tenant IF it were called per request — but it is resolved once with the + // instance's compile context, so both requests below get the compile-context result ('c'). + granularities: (ctx: any) => { + seenCtx = ctx; + return ctx?.securityContext?.tenant === 'c' ? ['year', 'quarter'] : ['month']; + }, + compileContext: { securityContext: { tenant: 'c' } }, + }); + + const cubesA = await api.metaConfig(ctxFor('a'), {}); + const cubesB = await api.metaConfig(ctxFor('b'), {}); + + // Both requests see the compile-context resolution, not their own securityContext. + expect(seenCtx?.securityContext?.tenant).toBe('c'); + expect(granularityNames(dimByName(cubesA, 'Orders.created_at'))).toEqual(['year', 'quarter']); + expect(granularityNames(dimByName(cubesB, 'Orders.created_at'))) + .toEqual(granularityNames(dimByName(cubesA, 'Orders.created_at'))); + api.dispose(); + }); + + // 6. Function form baked + folded into compilerVersion: two CompilerApi instances with different + // appId-level compile contexts (function returns differ) produce a different compilerVersion and + // different baked meta. + test('different compile contexts (different function results) -> different version and meta', async () => { + const granularities = (ctx: any) => (ctx.securityContext.tenant === 'a' + ? ['year'] + : ['month']); + + const apiA = createApi({ granularities, compileContext: { securityContext: { tenant: 'a' } } }); + const apiB = createApi({ granularities, compileContext: { securityContext: { tenant: 'b' } } }); + + const cubesA = await apiA.metaConfig(ctxFor('x'), {}); + const cubesB = await apiB.metaConfig(ctxFor('x'), {}); + + // Different baked effective sets. + expect(granularityNames(dimByName(cubesA, 'Orders.created_at'))).toEqual(['year']); + expect(granularityNames(dimByName(cubesB, 'Orders.created_at'))).toEqual(['month']); + + // Different compilerVersion (the resolved-config hash is folded in). compilerId is NOT asserted: + // two independent CompilerApi instances get random UUIDs regardless of granularities, so that + // check would be vacuous. + expect(apiA.version()).not.toBe(apiB.version()); + apiA.dispose(); + apiB.dispose(); + }); + }); + + describe('meta and SQL read the same baked values', () => { + // 7. Meta and SQL read the SAME baked values, so a custom the meta advertises on a dimension is + // exactly the one baked into that dimension's granularities map (the map the SQL path / pre-agg + // matcher consume) — no meta/SQL contract gap. (Real SQL execution of a baked global custom is + // exercised in test 2 and the view test; here we assert the per-dimension meta/baked-map parity.) + test('meta and the baked SQL/matcher map agree on the effective set', async () => { + const api = createApi({ + granularities: [{ name: 'sprint', interval: '2 weeks', origin: '2024-01-01' }], + }); + const cubes = await api.metaConfig(ctxFor('a'), {}); + + // Plain dimension: meta advertises the global custom, and the baked map carries the same one. + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toContain('sprint'); + const baked = await bakedDimGranularities(api, 'Orders.created_at'); + expect(baked.sprint).toMatchObject({ interval: '2 weeks', origin: '2024-01-01' }); + + // Excluded dimension: absent from meta AND absent from the baked map — they agree on the gap. + expect(granularityNames(dimByName(cubes, 'Orders.excluded_at'))).not.toContain('sprint'); + const bakedExcluded = await bakedDimGranularities(api, 'Orders.excluded_at'); + expect(bakedExcluded.sprint).toBeUndefined(); + api.dispose(); + }); + }); + + describe('global customs participate in pre-agg / hierarchy matching', () => { + // 8a. NEW deliberate behavior: a global custom merged into td.granularities is visible to the + // granularity-hierarchy / pre-agg matching path. The matcher resolves custom granularities off + // the compiled dimension symbol's `granularities` map, so this first test locks in that the + // global custom is present in that map (and the by-reference / copy-on-write sharing). Test 8b + // then drives the REAL matcher end-to-end. + test('global custom is baked into the dimension granularities map the matcher consumes', async () => { + const api = createApi({ + granularities: [{ name: 'sprint', interval: '2 weeks', origin: '2024-01-01' }], + }); + // Force a compile. + await api.metaConfig(ctxFor('a'), {}); + + const baked = await bakedDimGranularities(api, 'Orders.created_at'); + expect(baked.sprint).toMatchObject({ interval: '2 weeks', origin: '2024-01-01' }); + + // Plain dimensions (no local block) share the SAME globalCustoms object by reference — the + // by-reference merge assigns one shared map to every plain dim (copy-on-write only for locals). + const bakedUpdated = await bakedDimGranularities(api, 'Orders.updated_at'); + expect(bakedUpdated).toBe(baked); + + // A dimension with a local block gets a fresh copy-on-write object, not the shared map. + const bakedCollide = await bakedDimGranularities(api, 'Orders.collide_at'); + expect(bakedCollide).not.toBe(baked); + + // A dimension that excludes it does NOT get it baked in. + const bakedExcluded = await bakedDimGranularities(api, 'Orders.excluded_at'); + expect(bakedExcluded.sprint).toBeUndefined(); + api.dispose(); + }); + + // 8b. REAL pre-agg matching: a rollup defined on the GLOBAL-custom grain must be matchable by a + // query at that grain. This drives the actual matcher end-to-end via getSql (which runs + // PreAggregations.transformQueryToCanUseForm + canUsePreAggregationForTransformedQueryFn and + // resolves the custom granularity off the baked map). A matching query uses the pre-agg; a query + // at an incompatible grain does not. + const preAggRepository: SchemaFileRepository = { + localPath: () => '/mock/path', + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'orders.js', + content: ` + cube('Orders', { + sql: 'SELECT * FROM orders', + measures: { count: { type: 'count' } }, + dimensions: { created_at: { sql: 'created_at', type: 'time' } }, + preAggregations: { + byFiscal: { + type: 'rollup', + measures: [Orders.count], + timeDimension: Orders.created_at, + // Rollup granularity is the GLOBAL custom — only resolvable if the bake reached + // the map the matcher consumes. + granularity: 'fiscal_year', + partitionGranularity: 'year', + }, + }, + }); + `, + }, + ]), + }; + + const preAggQuery = (grain: string): any => ({ + measures: ['Orders.count'], + dimensions: [], + segments: [], + filters: [], + timeDimensions: [{ + dimension: 'Orders.created_at', + // Custom grain name — allowed at runtime; the wire type only enumerates built-ins. + granularity: grain as any, + dateRange: ['2024-01-01', '2025-01-01'], + }], + order: [], + timezone: 'UTC', + }); + + test('a rollup on the global-custom grain is matched by a query at that grain', async () => { + const api = new TestableCompilerApi(preAggRepository, mockDbType, { + logger: noopLogger, + granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-01-01' }], + }); + + // Query at the global-custom grain -> the matcher resolves the custom and USES the pre-agg. + const matchSql = await api.getSql(preAggQuery('fiscal_year'), { includeDebugInfo: true }); + expect(matchSql.preAggregations.map((p: any) => p.preAggregationId)).toContain('Orders.byFiscal'); + + // Query at an incompatible grain -> no pre-agg matches. + const missSql = await api.getSql(preAggQuery('month'), { includeDebugInfo: true }); + expect(missSql.preAggregations).toHaveLength(0); + api.dispose(); + }); + }); + + describe('global custom on a view member (dual-write to symbols)', () => { + // Orders with a plain time dimension + a view that includes it. A global custom must be baked + // into the VIEW member's dimension too, in BOTH object graphs (evaluatedCubes AND symbols), so a + // query against the view member at the global grain resolves in SQL. + const viewRepository: SchemaFileRepository = { + localPath: () => '/mock/path', + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'orders.js', + content: ` + cube('Orders', { + sql: 'SELECT * FROM orders', + measures: { count: { type: 'count' } }, + dimensions: { + id: { sql: 'id', type: 'number', primaryKey: true }, + created_at: { sql: 'created_at', type: 'time' }, + }, + }); + view('OrdersView', { + cubes: [ + { join_path: 'Orders', includes: ['count', 'created_at'] }, + ], + }); + `, + }, + ]), + }; + + const createViewApi = () => new TestableCompilerApi(viewRepository, mockDbType, { + logger: noopLogger, + granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-02-01' }], + }); + + test('global custom is baked into the view member and resolves in SQL', async () => { + const api = createViewApi(); + + // Baked into the view member in BOTH graphs: evaluatedCubes (dimensionByPath / matcher) ... + const bakedEval = await bakedDimGranularities(api, 'OrdersView.created_at'); + expect(bakedEval.fiscal_year).toMatchObject({ interval: '1 year', origin: '2024-02-01' }); + // ... AND the symbols copy (CubeSymbols.resolveGranularity for SQL) — the load-bearing dual write. + const bakedSym = await symbolsDimGranularities(api, 'OrdersView', 'created_at'); + expect(bakedSym.fiscal_year).toMatchObject({ interval: '1 year', origin: '2024-02-01' }); + + // Meta advertises it on the view member. + const cubes = await api.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubes, 'OrdersView.created_at'))).toContain('fiscal_year'); + + // Real SQL against the VIEW member at the global custom grain resolves and does not throw. + const { sql } = await api.getSql({ + measures: ['OrdersView.count'], + dimensions: [], + segments: [], + filters: [], + timeDimensions: [{ + dimension: 'OrdersView.created_at', + // Custom granularity name — allowed at runtime; the wire type only enumerates built-ins. + granularity: 'fiscal_year' as any, + dateRange: ['2024-01-01', '2025-01-01'], + }], + order: [], + } as any); + const sqlText = Array.isArray(sql) ? sql[0] : sql; + expect(typeof sqlText).toBe('string'); + expect(sqlText.length).toBeGreaterThan(0); + api.dispose(); + }); + }); + + describe('locals win over globals on name collision', () => { + // 9. A dimension-local custom with the same name as a global custom keeps the LOCAL definition + // after the merge (locals always win). + test('a local custom shadows a same-named global custom after bake', async () => { + const api = createApi({ + // Global fiscal_year has origin 2024-02-01; the local on Orders.collide_at / Events.ts uses + // a different origin and must win. + granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-02-01' }], + }); + await api.metaConfig(ctxFor('a'), {}); + + // Orders.collide_at declares local fiscal_year { origin: 2020-01-01 } -> local wins. + const collide = await bakedDimGranularities(api, 'Orders.collide_at'); + expect(collide.fiscal_year).toMatchObject({ origin: '2020-01-01' }); + + // Events.ts declares local fiscal_year { origin: 2024-02-01 } — same name, local definition + // preserved (not replaced by the global). + const eventsTs = await bakedDimGranularities(api, 'Events.ts'); + expect(eventsTs.fiscal_year).toMatchObject({ origin: '2024-02-01' }); + + // A plain dimension with no local fiscal_year takes the GLOBAL one. + const created = await bakedDimGranularities(api, 'Orders.created_at'); + expect(created.fiscal_year).toMatchObject({ origin: '2024-02-01' }); + api.dispose(); + }); + }); + + describe('composition with RBAC visibility', () => { + const rbacRepository: SchemaFileRepository = { + localPath: () => '/mock/path', + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'orders.js', + content: ` + cube('Orders', { + sql: 'SELECT * FROM orders', + measures: { count: { type: 'count' } }, + dimensions: { + created_at: { sql: 'created_at', type: 'time' }, + secret: { sql: 'secret', type: 'string' }, + }, + accessPolicy: [ + { + group: '*', + rowLevel: { allowAll: true }, + memberLevel: { includes: ['count', 'created_at'] }, + }, + ], + }); + `, + }, + ]), + }; + + // 10. compilerId still mixes the visibility mask; granularities no longer contribute a separate + // per-request hash (they live in compilerVersion / the base compilerId). So the composed + // compilerId is stable across requests with the same visibility and differs with the mask. + test('compilerId mixes the visibility mask; stable per visibility, differs across masks', async () => { + const api = new TestableCompilerApi(rbacRepository, mockDbType, { + logger: noopLogger, + granularities: ['year', 'month'], + contextToGroups: async (ctx: any) => (ctx.securityContext.tenant === 'admin' ? ['admin'] : []), + }); + + const base = (await (api as any).getCompilers()).compilerId; + + // RBAC hides `secret` for a non-admin; the composed compilerId differs from the base. + const nonAdmin = await api.metaConfig(ctxFor('user'), { includeCompilerId: true }); + const secret = dimByName(nonAdmin.cubes, 'Orders.secret'); + expect(secret.isVisible).toBe(false); + const createdAt = dimByName(nonAdmin.cubes, 'Orders.created_at'); + expect(createdAt.isVisible).toBe(true); + // Granularities are baked, not per-request — the effective set is still present. + expect(granularityNames(createdAt)).toEqual(['year', 'month']); + expect(nonAdmin.compilerId).not.toBe(base); + + // Same visibility mask across two requests -> identical composed compilerId. + const nonAdmin2 = await api.metaConfig(ctxFor('user2'), { includeCompilerId: true }); + expect(nonAdmin2.compilerId).toBe(nonAdmin.compilerId); + api.dispose(); + }); + }); +}); diff --git a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts b/packages/cubejs-server-core/test/unit/granularity-variants.test.ts deleted file mode 100644 index 1307f881e972b..0000000000000 --- a/packages/cubejs-server-core/test/unit/granularity-variants.test.ts +++ /dev/null @@ -1,517 +0,0 @@ -import { SchemaFileRepository } from '@cubejs-backend/shared'; -import type { Compiler, GlobalGranularitiesConfig, GranularitySets } from '@cubejs-backend/schema-compiler'; -import { CompilerApi } from '../../src/core/CompilerApi'; -import { DbTypeInternalFn } from '../../src/core/types'; - -class TestableCompilerApi extends CompilerApi { - public buildCount = 0; - - public failNextBuild = false; - - public defsBuildCount = 0; - - protected buildGranularitySets(compilers: Compiler, config: GlobalGranularitiesConfig): GranularitySets { - if (this.failNextBuild) { - this.failNextBuild = false; - throw new Error('injected variant build failure'); - } - this.buildCount++; - return super.buildGranularitySets(compilers, config); - } - - public lastDefinitions: any; - - protected buildGranularityDefinitions(compilers: Compiler, config: GlobalGranularitiesConfig): any { - this.defsBuildCount++; - this.lastDefinitions = super.buildGranularityDefinitions(compilers, config); - return this.lastDefinitions; - } - - public version(): string | undefined { - return this.compilerVersion; - } - - public async variantCache(): Promise> | undefined> { - return (await this.getCompilers()).granularityVariants; - } - - public async definitionsCache(): Promise | undefined> { - return (await this.getCompilers()).granularityDefinitions; - } -} - -const repository: SchemaFileRepository = { - localPath: () => '/mock/path', - dataSchemaFiles: () => Promise.resolve([ - { - fileName: 'orders.js', - content: ` - cube('Orders', { - sql: 'SELECT * FROM orders', - measures: { count: { type: 'count' } }, - dimensions: { - id: { sql: 'id', type: 'number', primaryKey: true }, - created_at: { sql: 'created_at', type: 'time' }, - updated_at: { sql: 'updated_at', type: 'time' }, - excluded_at: { - sql: 'updated_at', - type: 'time', - granularities: { excludes: ['fiscal_year', 'sprint'] }, - }, - }, - }); - cube('Events', { - sql: 'SELECT * FROM events', - measures: { count: { type: 'count' } }, - dimensions: { - ts: { - sql: 'ts', - type: 'time', - granularities: { - fiscal_year: { interval: '1 year', origin: '2024-02-01' }, - }, - }, - }, - }); - cube('Products', { - sql: 'SELECT * FROM products', - measures: { count: { type: 'count' } }, - dimensions: { - name: { sql: 'name', type: 'string' }, - }, - }); - `, - }, - ]), -}; - -const mockDbType: DbTypeInternalFn = async () => 'postgres'; - -const noopLogger = () => { /* silent */ }; - -const createApi = (options: any = {}) => new TestableCompilerApi(repository, mockDbType, { - logger: options.capturedLogs - ? (msg: string, params: any) => options.capturedLogs.push({ msg, params }) - : noopLogger, - ...options, -}); - -const ctxFor = (tenant: string) => ({ securityContext: { tenant }, requestId: `req-${tenant}` }); - -const dimByName = (cubes: any[], name: string) => cubes - .flatMap((c: any) => c.config.dimensions) - .find((d: any) => d.name === name); - -const granularityNames = (dim: any) => dim.effectiveGranularities.map((g: any) => g.name); - -const ALL_BUILT_INS = ['year', 'quarter', 'month', 'week', 'day', 'hour', 'minute', 'second']; - -const queryFor = (tenant: string, dimension: string, granularity: string): any => ({ - measures: [dimension.startsWith('Events.') ? 'Events.count' : 'Orders.count'], - timeDimensions: [{ dimension, granularity }], - timezone: 'UTC', - contextSymbols: { securityContext: { tenant } }, - requestId: `sql-${tenant}-${dimension}-${granularity}`, -}); - -describe('granularity variants in CompilerApi', () => { - describe('env/static configs (baked at compile time)', () => { - afterEach(() => { - delete process.env.CUBEJS_GRANULARITIES; - }); - - test('no config: every time dimension gets all built-ins, no variant builds happen', async () => { - const api = createApi(); - const cubes = await api.metaConfig(ctxFor('a'), {}); - expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toEqual(ALL_BUILT_INS); - // Local custom granularity survives on top of the enabled built-ins. - expect(granularityNames(dimByName(cubes, 'Events.ts'))).toEqual([...ALL_BUILT_INS, 'fiscal_year']); - // Non-time dimensions are untouched. - expect(dimByName(cubes, 'Products.name').effectiveGranularities).toBeUndefined(); - - await api.metaConfig(ctxFor('b'), {}); - expect(api.buildCount).toBe(0); - expect(await api.variantCache()).toBeUndefined(); - api.dispose(); - }); - - test('static list is baked in and internal fields stay off the wire', async () => { - const api = createApi({ granularities: ['year', { name: 'half', interval: '6 months' }] }); - const cubes = await api.metaConfig(ctxFor('a'), {}); - const dim = dimByName(cubes, 'Orders.created_at'); - expect(dim.effectiveGranularities).toEqual([ - { name: 'year', type: 'built-in', title: 'Year', interval: '1 year', format: '%Y' }, - { name: 'half', type: 'custom', title: 'half', interval: '6 months' }, - ]); - expect(dim.granularitiesBlock).toBeUndefined(); - // Legacy shape for the customized dimension is preserved (deprecated but not broken). - const eventsTs = dimByName(cubes, 'Events.ts'); - expect(eventsTs.granularities).toEqual([ - { name: 'fiscal_year', title: 'Fiscal Year', interval: '1 year', offset: undefined, origin: '2024-02-01' }, - ]); - expect(api.buildCount).toBe(0); - api.dispose(); - }); - - test('static global customs resolve in SQL, while dimension excludes and legacy behavior remain intact', async () => { - const api = createApi({ - granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-02-01' }], - }); - - const globalSql = await api.getSql(queryFor('a', 'Orders.created_at', 'fiscal_year')); - expect(globalSql.sql[0]).toContain('created_at'); - await expect(api.getSql(queryFor('a', 'Orders.excluded_at', 'fiscal_year'))) - .rejects.toThrow('Granularity "fiscal_year" does not exist in dimension Orders.excluded_at'); - - // Existing local customs and predefined granularities still use their original paths. - const localSql = await api.getSql(queryFor('a', 'Events.ts', 'fiscal_year')); - expect(localSql.sql[0]).toContain('ts'); - const builtInSql = await api.getSql(queryFor('a', 'Orders.updated_at', 'day')); - expect(builtInSql.sql[0]).toContain('updated_at'); - api.dispose(); - }); - - test('time dimensions without customization share one default set instance', async () => { - const api = createApi(); - const cubes = await api.metaConfig(ctxFor('a'), {}); - const created = dimByName(cubes, 'Orders.created_at'); - const updated = dimByName(cubes, 'Orders.updated_at'); - expect(created.effectiveGranularities).toBe(updated.effectiveGranularities); - expect(dimByName(cubes, 'Events.ts').effectiveGranularities) - .not.toBe(created.effectiveGranularities); - api.dispose(); - }); - - test('the resolved config hash is folded into compilerVersion, so an env change recompiles', async () => { - const api = createApi(); - let cubes = await api.metaConfig(ctxFor('a'), {}); - expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toEqual(ALL_BUILT_INS); - const versionBefore = api.version(); - expect(versionBefore).toContain('_gran_'); - - process.env.CUBEJS_GRANULARITIES = 'year,month'; - cubes = await api.metaConfig(ctxFor('a'), {}); - expect(api.version()).not.toBe(versionBefore); - expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toEqual(['year', 'month']); - api.dispose(); - }); - }); - - describe('function config (per-request variants)', () => { - const perTenant = (ctx: any) => (ctx.securityContext.tenant === 'a' - ? ['year', 'month'] - : ['week', { name: 'sprint', interval: '2 weeks' }]); - - test('tenants get their own sets; repeats and alternation hit the cache; no leaks', async () => { - const api = createApi({ granularities: perTenant }); - - for (let i = 0; i < 3; i++) { - const cubesA = await api.metaConfig(ctxFor('a'), {}); - expect(granularityNames(dimByName(cubesA, 'Orders.created_at'))).toEqual(['year', 'month']); - expect(granularityNames(dimByName(cubesA, 'Events.ts'))).toEqual(['year', 'month', 'fiscal_year']); - - const cubesB = await api.metaConfig(ctxFor('b'), {}); - expect(granularityNames(dimByName(cubesB, 'Orders.created_at'))).toEqual(['week', 'sprint']); - expect(granularityNames(dimByName(cubesB, 'Events.ts'))).toEqual(['week', 'sprint', 'fiscal_year']); - } - - expect(api.buildCount).toBe(2); - expect((await api.variantCache())!.size).toBe(2); - api.dispose(); - }); - - test('context-function global customs resolve in SQL without crossing dimension exclusions', async () => { - const api = createApi({ granularities: perTenant }); - - await expect(api.getSql(queryFor('a', 'Orders.created_at', 'sprint'))) - .rejects.toThrow('Granularity "sprint" does not exist in dimension Orders.created_at'); - const sql = await api.getSql(queryFor('b', 'Orders.created_at', 'sprint')); - expect(sql.sql[0]).toContain('created_at'); - await expect(api.getSql(queryFor('b', 'Orders.excluded_at', 'sprint'))) - .rejects.toThrow('Granularity "sprint" does not exist in dimension Orders.excluded_at'); - await expect(api.getSql(queryFor('a', 'Orders.created_at', 'sprint'))) - .rejects.toThrow('Granularity "sprint" does not exist in dimension Orders.created_at'); - api.dispose(); - }); - - test('base meta cubes are never mutated by variant enrichment', async () => { - const api = createApi({ granularities: perTenant }); - await api.metaConfig(ctxFor('a'), {}); - const compilers = await (api as any).getCompilers(); - const baseDim = dimByName(compilers.metaTransformer.cubes, 'Orders.created_at'); - expect(baseDim.effectiveGranularities).toBeUndefined(); - api.dispose(); - }); - - // The cache stores the sparse per-config sets ({ defaultSet, overrides }), NOT an enriched cube - // array — plain dims reference one shared defaultSet; only local-block dims are in overrides. - test('variant cache stores sparse sets, not cube copies', async () => { - const api = createApi({ granularities: perTenant }); - const cubes = await api.metaConfig(ctxFor('a'), {}); - - const entry = await (await api.variantCache())!.get( - [...(await api.variantCache())!.keys()][0] - )!; - expect(Array.isArray(entry)).toBe(false); - expect(Array.isArray(entry.defaultSet)).toBe(true); - expect(entry.overrides instanceof Map).toBe(true); - // Events.ts has a local block → in overrides; plain dims are not. - expect(entry.overrides.has('Events.ts')).toBe(true); - expect(entry.overrides.has('Orders.created_at')).toBe(false); - - // And the attached read-time result shares the one defaultSet across plain dims by reference. - const created = dimByName(cubes, 'Orders.created_at'); - const updated = dimByName(cubes, 'Orders.updated_at'); - expect(created.effectiveGranularities).toBe(updated.effectiveGranularities); - expect(created.effectiveGranularities).toBe(entry.defaultSet); - api.dispose(); - }); - - // Regression: meta and SQL paths must resolve the config from the SAME securityContext-only - // view. A function keyed on securityContext advertises `sprint` for tenant b in meta AND - // resolves it in SQL — no path can advertise a custom the other can't execute. - test('meta and SQL paths agree on the effective set (securityContext-keyed function)', async () => { - const api = createApi({ granularities: perTenant }); - const cubes = await api.metaConfig(ctxFor('b'), {}); - expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toContain('sprint'); - // Same tenant, SQL path: the advertised custom actually resolves. - const sql = await api.getSql(queryFor('b', 'Orders.created_at', 'sprint')); - expect(sql.sql[0]).toContain('created_at'); - api.dispose(); - }); - - test('distinct compilerIds per tenant, both distinct from the base', async () => { - const api = createApi({ granularities: perTenant }); - const a = await api.metaConfig(ctxFor('a'), { includeCompilerId: true }); - const b = await api.metaConfig(ctxFor('b'), { includeCompilerId: true }); - const base = (await (api as any).getCompilers()).compilerId; - expect(a.compilerId).not.toBe(b.compilerId); - expect(a.compilerId).not.toBe(base); - expect(b.compilerId).not.toBe(base); - // Same tenant, same id — stable across calls. - const a2 = await api.metaConfig(ctxFor('a'), { includeCompilerId: true }); - expect(a2.compilerId).toBe(a.compilerId); - api.dispose(); - }); - - test('static and function forms producing the same list emit identical meta', async () => { - const list = ['year', 'month', { name: 'half', interval: '6 months', title: 'Half' }]; - const staticApi = createApi({ granularities: list }); - const fnApi = createApi({ granularities: () => list }); - const staticCubes = await staticApi.metaConfig(ctxFor('a'), {}); - const fnCubes = await fnApi.metaConfig(ctxFor('a'), {}); - expect(JSON.parse(JSON.stringify(fnCubes))).toEqual(JSON.parse(JSON.stringify(staticCubes))); - staticApi.dispose(); - fnApi.dispose(); - }); - - test('concurrent identical-config misses dedup to a single build', async () => { - const api = createApi({ granularities: perTenant }); - await Promise.all( - Array.from({ length: 10 }, () => api.metaConfig(ctxFor('a'), {})) - ); - expect(api.buildCount).toBe(1); - api.dispose(); - }); - - test('uncustomized time dimensions share the default set within a variant', async () => { - const api = createApi({ granularities: perTenant }); - const cubes = await api.metaConfig(ctxFor('a'), {}); - expect(dimByName(cubes, 'Orders.created_at').effectiveGranularities) - .toBe(dimByName(cubes, 'Orders.updated_at').effectiveGranularities); - // A cube without time dimensions is passed through by reference, not copied. - const compilers = await (api as any).getCompilers(); - const baseProducts = compilers.metaTransformer.cubes.find((c: any) => c.config.name === 'Products'); - const variantProducts = (await api.metaConfig(ctxFor('a'), {})) - .find((c: any) => c.config.name === 'Products'); - expect(variantProducts).toBe(baseProducts); - api.dispose(); - }); - - test('LRU eviction beyond the bound, with a logged warning and rebuild on re-request', async () => { - const bound = (CompilerApi as any).MAX_GRANULARITY_VARIANTS; - const capturedLogs: any[] = []; - const api = createApi({ - capturedLogs, - granularities: (ctx: any) => [{ name: `g_${ctx.securityContext.tenant}`, interval: '1 week' }], - }); - - for (let i = 0; i < bound + 1; i++) { - await api.metaConfig(ctxFor(`t${i}`), {}); - } - expect(api.buildCount).toBe(bound + 1); - expect((await api.variantCache())!.size).toBe(bound); - expect(capturedLogs.some(l => l.msg === 'Granularity variant cache is full')).toBe(true); - - // t0 was evicted (least recently used) — asking again rebuilds. - await api.metaConfig(ctxFor('t0'), {}); - expect(api.buildCount).toBe(bound + 2); - // The newest entry is still cached — no rebuild. - await api.metaConfig(ctxFor(`t${bound}`), {}); - expect(api.buildCount).toBe(bound + 2); - api.dispose(); - }); - - test('a failed variant build self-evicts and the next request retries', async () => { - const api = createApi({ granularities: perTenant }); - api.failNextBuild = true; - await expect(api.metaConfig(ctxFor('a'), {})).rejects.toThrow('injected variant build failure'); - const cubes = await api.metaConfig(ctxFor('a'), {}); - expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toEqual(['year', 'month']); - expect(api.buildCount).toBe(1); - api.dispose(); - }); - - test('a recompile discards the variant cache with the compilers object', async () => { - let version = 'v1'; - const api = createApi({ granularities: perTenant, schemaVersion: () => version }); - await api.metaConfig(ctxFor('a'), {}); - expect(api.buildCount).toBe(1); - - version = 'v2'; - await api.metaConfig(ctxFor('a'), {}); - expect(api.buildCount).toBe(2); - expect((await api.variantCache())!.size).toBe(1); - api.dispose(); - }); - }); - - describe('SQL-path global-custom definitions cache', () => { - const perTenant = (ctx: any) => (ctx.securityContext.tenant === 'a' - ? [{ name: 'sprint', interval: '2 weeks', origin: '2024-01-01' }] - : [{ name: 'fortnight', interval: '2 weeks', origin: '2024-01-08' }]); - - test('scan runs once per distinct config, then serves from cache; result is byte-identical', async () => { - const api = createApi({ granularities: perTenant }); - - const a1 = await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); - const a2 = await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); - // Two identical-config queries → one scan. - expect(api.defsBuildCount).toBe(1); - expect(a1.sql[0]).toBe(a2.sql[0]); - - // A different tenant is a distinct config → one more scan, and a second cache entry. - await api.getSql(queryFor('b', 'Orders.created_at', 'fortnight')); - expect(api.defsBuildCount).toBe(2); - expect((await api.definitionsCache())!.size).toBe(2); - api.dispose(); - }); - - test('no scan and no cache entry when no global customs are configured', async () => { - const api = createApi({ granularities: () => ['year', 'month'] }); - await api.getSql(queryFor('a', 'Orders.created_at', 'month')); - await api.getSql(queryFor('a', 'Orders.created_at', 'year')); - expect(api.defsBuildCount).toBe(0); - expect(await api.definitionsCache()).toBeUndefined(); - api.dispose(); - }); - - test('distinct tenants never share a definitions entry (no cross-tenant bleed)', async () => { - const api = createApi({ granularities: perTenant }); - // Tenant a's custom is `sprint`; tenant b's is `fortnight`. Each resolves only its own. - const aSql = await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); - expect(aSql.sql[0]).toContain('created_at'); - await expect(api.getSql(queryFor('a', 'Orders.created_at', 'fortnight'))) - .rejects.toThrow('Granularity "fortnight" does not exist in dimension Orders.created_at'); - const bSql = await api.getSql(queryFor('b', 'Orders.created_at', 'fortnight')); - expect(bSql.sql[0]).toContain('created_at'); - await expect(api.getSql(queryFor('b', 'Orders.created_at', 'sprint'))) - .rejects.toThrow('Granularity "sprint" does not exist in dimension Orders.created_at'); - api.dispose(); - }); - - test('a recompile discards the definitions cache with the compilers object', async () => { - let version = 'v1'; - const api = createApi({ granularities: perTenant, schemaVersion: () => version }); - await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); - expect(api.defsBuildCount).toBe(1); - - version = 'v2'; - await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); - expect(api.defsBuildCount).toBe(2); - expect((await api.definitionsCache())!.size).toBe(1); - api.dispose(); - }); - - test('static config caches a single entry reused across queries', async () => { - const api = createApi({ granularities: [{ name: 'sprint', interval: '2 weeks', origin: '2024-01-01' }] }); - await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); - await api.getSql(queryFor('b', 'Orders.created_at', 'sprint')); - // Context-independent config → one scan, one entry, regardless of tenant. - expect(api.defsBuildCount).toBe(1); - expect((await api.definitionsCache())!.size).toBe(1); - api.dispose(); - }); - - // Plain time dimensions (no local block) share ONE global-custom map by reference; a dimension - // with a local block (Orders.excluded_at excludes sprint) is reconciled individually. - test('plain dimensions share the global-custom map by reference; local-block dims are individual', async () => { - const api = createApi({ granularities: [{ name: 'sprint', interval: '2 weeks', origin: '2024-01-01' }] }); - await api.getSql(queryFor('a', 'Orders.created_at', 'sprint')); - const defs = api.lastDefinitions; - - // created_at and updated_at are both plain (no local block) → the very same object. - expect(defs['Orders.created_at']).toBeDefined(); - expect(defs['Orders.created_at']).toBe(defs['Orders.updated_at']); - expect(defs['Orders.created_at'].sprint).toEqual({ interval: '2 weeks', origin: '2024-01-01' }); - - // excluded_at excludes sprint → not in the shared map; sprint absent there. - expect(defs['Orders.excluded_at']?.sprint).toBeUndefined(); - expect(defs['Orders.excluded_at']).not.toBe(defs['Orders.created_at']); - // Querying the excluded custom on it fails, matching meta. - await expect(api.getSql(queryFor('a', 'Orders.excluded_at', 'sprint'))) - .rejects.toThrow('Granularity "sprint" does not exist in dimension Orders.excluded_at'); - api.dispose(); - }); - }); - - describe('composition with RBAC visibility', () => { - const rbacRepository: SchemaFileRepository = { - localPath: () => '/mock/path', - dataSchemaFiles: () => Promise.resolve([ - { - fileName: 'orders.js', - content: ` - cube('Orders', { - sql: 'SELECT * FROM orders', - measures: { count: { type: 'count' } }, - dimensions: { - created_at: { sql: 'created_at', type: 'time' }, - secret: { sql: 'secret', type: 'string' }, - }, - accessPolicy: [ - { - group: '*', - rowLevel: { allowAll: true }, - memberLevel: { includes: ['count', 'created_at'] }, - }, - ], - }); - `, - }, - ]), - }; - - test('granularity variant selects first, visibility patches on top, compilerId mixes both', async () => { - const api = new TestableCompilerApi(rbacRepository, mockDbType, { - logger: noopLogger, - granularities: (ctx: any) => (ctx.securityContext.tenant === 'a' ? ['year'] : ['month']), - }); - - const result = await api.metaConfig(ctxFor('a'), { includeCompilerId: true }); - const createdAt = dimByName(result.cubes, 'Orders.created_at'); - expect(granularityNames(createdAt)).toEqual(['year']); - // RBAC hid `secret` but kept the enriched time dimension intact. - const secret = dimByName(result.cubes, 'Orders.secret'); - expect(secret.isVisible).toBe(false); - expect(createdAt.isVisible).toBe(true); - - // compilerId differs across tenants (granularity), and from the base (visibility + granularity). - const base = (await (api as any).getCompilers()).compilerId; - const resultB = await api.metaConfig(ctxFor('b'), { includeCompilerId: true }); - expect(result.compilerId).not.toBe(resultB.compilerId); - expect(result.compilerId).not.toBe(base); - api.dispose(); - }); - }); -}); From 675da83ede085397cc6bb906783e90c06082b51f Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 23 Jul 2026 00:09:24 +0200 Subject: [PATCH 20/22] fix(granularities): use object destructuring in metaConfig (lint) --- packages/cubejs-server-core/src/core/CompilerApi.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index 74d63f4740cc0..540b22a9db38e 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -1148,7 +1148,7 @@ export class CompilerApi { const { includeCompilerId, includeViewGroups, skipVisibilityPatch, ...restOptions } = options; const compilers = await this.getCompilers(restOptions); // Granularities are baked into the compiled model, so the base meta cubes are served directly. - const cubes = compilers.metaTransformer.cubes; + const { cubes } = compilers.metaTransformer; // Fixed composition order: base compilerId, then visibility mask. Only computed when the caller // actually wants the id — the hashing is skipped when a caller asks for view groups alone (the From aebce83ad25b3b5baefd85a696a523df79fa5100 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Thu, 23 Jul 2026 00:35:35 +0200 Subject: [PATCH 21/22] refactor(granularities): move /v1/granularities catalog assembly into CompilerApi, tighten comments (CUB-2567 review) --- packages/cubejs-api-gateway/src/gateway.ts | 32 +---------- .../cubejs-api-gateway/test/index.test.ts | 2 +- packages/cubejs-api-gateway/test/mocks.ts | 13 +++-- .../src/compiler/CubeToMetaTransformer.ts | 24 +++------ .../src/compiler/PrepareCompiler.ts | 5 +- .../src/core/CompilerApi.ts | 53 ++++++++++++------- 6 files changed, 49 insertions(+), 80 deletions(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 62000e7bc7d80..1cb9a293051aa 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -33,10 +33,6 @@ import type { import { createProxyMiddleware } from 'http-proxy-middleware'; import { QueryBody } from '@cubejs-backend/query-orchestrator'; -import { - buildBuiltInsCatalog, - isBuiltInGranularity, -} from '@cubejs-backend/schema-compiler'; import { QueryType, ApiScopes, @@ -742,8 +738,6 @@ class ApiGateway { const cubesConfig = onlyViews ? metaConfig.cubes.filter((c: any) => c.config?.type === 'view') : metaConfig.cubes; - // Time dimensions arrive from CompilerApi with `effectiveGranularities` already attached — - // baked into the compiled model at compile time (resolved once per appId, all config forms). const cubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config); const visibleCubeNames = new Set(cubes.map(c => c.name)); const viewGroups = (metaConfig.viewGroups || []) @@ -776,31 +770,7 @@ class ApiGateway { try { await this.assertApiScope('meta', context.securityContext); const compilerApi = await this.getCompilerApi(context); - // Serve the per-appId catalog baked into the compiled model — no per-request resolution. - const globalConfig = await compilerApi.getGlobalGranularitiesConfig({ requestId: context.requestId }); - const builtInsCatalog = buildBuiltInsCatalog(globalConfig); - - const granularities: any[] = []; - for (const [name, entry] of Object.entries(builtInsCatalog)) { - granularities.push({ type: 'built-in', name, ...entry }); - } - for (const [name, def] of Object.entries(globalConfig.customGranularities)) { - // Skip names already emitted by `buildBuiltInsCatalog` (their inline overrides are folded in - // there). Use isBuiltInGranularity (hasOwnProperty), not `in`, so a custom named e.g. - // `constructor`/`toString` isn't misclassified as a built-in via the prototype chain and dropped. - if (!isBuiltInGranularity(name)) { - const entry: any = { - type: 'custom', - name, - title: def.title || name, - }; - if (def.interval !== undefined) entry.interval = def.interval; - if (def.origin !== undefined) entry.origin = def.origin; - if (def.offset !== undefined) entry.offset = def.offset; - if (def.format !== undefined) entry.format = def.format; - granularities.push(entry); - } - } + const granularities = await compilerApi.getGranularities({ requestId: context.requestId }); res({ data: { granularities } }); } catch (e: any) { this.handleError({ diff --git a/packages/cubejs-api-gateway/test/index.test.ts b/packages/cubejs-api-gateway/test/index.test.ts index fb1680a2710ef..d3e5201fd4483 100644 --- a/packages/cubejs-api-gateway/test/index.test.ts +++ b/packages/cubejs-api-gateway/test/index.test.ts @@ -722,7 +722,7 @@ describe('API Gateway', () => { expect(dim.granularitiesBlock).toBeUndefined(); }); - test('granularities endpoint returns the context-resolved global config', async () => { + test('granularities endpoint returns the per-appId catalog baked into the compiled model', async () => { const { app } = await createApiGateway(); const res = await request(app) diff --git a/packages/cubejs-api-gateway/test/mocks.ts b/packages/cubejs-api-gateway/test/mocks.ts index c55c3d692f313..47c56f3a4890a 100644 --- a/packages/cubejs-api-gateway/test/mocks.ts +++ b/packages/cubejs-api-gateway/test/mocks.ts @@ -80,13 +80,12 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({ return { query, denied: false }; }, - async getGlobalGranularitiesConfig(_options: any = {}) { - return { - enabledBuiltIns: ['year', 'month'], - customGranularities: { - fiscal_year: { title: 'Fiscal Year', interval: '1 year', origin: '2024-02-01' }, - }, - }; + async getGranularities(_options: any = {}) { + return [ + { type: 'built-in', name: 'year', title: 'Year', format: '%Y', interval: '1 year' }, + { type: 'built-in', name: 'month', title: 'Month', format: '%b %Y', interval: '1 month' }, + { type: 'custom', name: 'fiscal_year', title: 'Fiscal Year', interval: '1 year', origin: '2024-02-01' }, + ]; }, async metaConfig(_ctx, options: any = {}) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index 6c89ae6445fab..aba31f97363ba 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -222,12 +222,8 @@ export class CubeToMetaTransformer implements CompilerInterface { // result here — the transformer never sees a function. private readonly granularitiesConfig?: GlobalGranularitiesConfig; - // Set during compile() from the resolved config. Everything a time dimension needs is precomputed - // once here so the per-dimension loop does no repeated resolution for the common (plain) case: - // - `defaultSet`: the serialized effective set shared by every dimension without a local block; - // - `defaultGlobalCustoms`: the global-custom map (name -> def, `type` stripped) baked into those - // same plain dimensions — invariant across them, so computed once and shared by reference. - // Both derive from a single `resolveDimensionGranularities(EMPTY_BLOCK, config)` pass. + // Precomputed once in compile() so plain dimensions need no per-dimension resolution: `defaultSet` + // and `defaultGlobalCustoms` are shared by reference across every dimension without a local block. private granularityState!: { config: GlobalGranularitiesConfig; catalog: Record; @@ -516,18 +512,10 @@ export class CubeToMetaTransformer implements CompilerInterface { return out; } - // Bake the precomputed effective GLOBAL customs (`globalCustoms`) into a time dimension's - // `granularities` map — decision 3 of the per-appId compile-time bake — so the SQL symbol path - // resolves them by name and they participate in pre-agg matching. Locals always win. - // - // The bake must reach BOTH object graphs the downstream paths read: the `cubeList`/`evaluatedCubes` - // object (`dimDef`, read by timeDimensionsForCube → pre-agg matching) AND the separate - // `symbols[cube][dim]` object that CubeSymbols.resolveGranularity reads for SQL. These are distinct - // objects (built by different transforms) even for a plain cube, so the second write is - // load-bearing, not a no-op. If only `dimDef` were written, SQL couldn't resolve a baked global and - // pre-agg matching would throw when it builds a Granularity for it. We REASSIGN `granularities` - // (never mutate in place) so a view dim sharing its source's map by reference isn't contaminated - // before it is itself processed. + // Bake global customs into a dimension's `granularities` map (locals win). Must write BOTH the + // `dimDef` object (pre-agg matching) and the distinct `symbols[cube][dim]` object (SQL + // resolveGranularity) — writing one leaves the other unable to resolve the custom. Reassign, never + // mutate in place, so a view dim sharing its source's map by reference isn't contaminated. private mergeGlobalCustomsIntoDimension( cubeName: string, dimensionName: string, diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 6083aa5d552bc..516a213453c8c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -41,9 +41,8 @@ export type PrepareCompilerOptions = { compiledScriptCache?: LRUCache; compiledYamlCache?: LRUCache; compiledJinjaCache?: LRUCache; - // Resolved-once global granularities config for this appId, baked into the compiled model by - // CubeToMetaTransformer. CompilerApi resolves all config forms (env / static / function) before - // compile and passes the result here. + // Resolved global granularities config (all forms resolved by CompilerApi before compile), baked + // into the compiled model by CubeToMetaTransformer. granularitiesConfig?: GlobalGranularitiesConfig; }; diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index 540b22a9db38e..e70d32d7c7ba5 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -7,11 +7,13 @@ import { compile, Compiler, createQuery, + buildBuiltInsCatalog, CubeDefinition, EvaluatedCube, GlobalGranularitiesConfig, GranularitiesOption, granularityConfigHash, + isBuiltInGranularity, PreAggregationFilters, PreAggregationInfo, PreAggregationReferences, @@ -141,10 +143,8 @@ export class CompilerApi { protected readonly granularities?: GranularitiesOption; // Resolved-once-per-appId global granularities config, baked into the compiled model. Resolved in - // getCompilers with the appId-level compile context (all three forms — env / static / function). - // The in-flight PROMISE is memoized (not just the value) so concurrent initial getCompilers calls - // share a single resolve/await — the function form is invoked exactly once per appId. Cleared on - // rejection so a failed resolve can be retried. + // Memoized resolved global granularities config (+ hash) for the static/function forms; see + // getResolvedGranularities. private resolvedGranularitiesPromise?: Promise<{ config: GlobalGranularitiesConfig; hash: string }>; protected queryFactory?: QueryFactory; @@ -239,10 +239,7 @@ export class CompilerApi { compilerVersion += `_${crypto.createHash('md5').update(JSON.stringify(files)).digest('hex')}`; } - // Resolve the global granularities config ONCE per appId with the compile context (handles all - // forms — env / static / function — awaiting the function). The resolved config is baked into - // the compiled model, so its hash goes into compilerVersion for every form: a config change - // then forces a recompile. Memoized on the instance so repeated calls don't re-resolve/re-await. + // Fold the resolved granularities hash into compilerVersion so a config change forces a recompile. const { config: resolvedGranularities, hash: granularitiesHash } = await this.getResolvedGranularities(); compilerVersion += `_gran_${granularitiesHash}`; @@ -257,12 +254,10 @@ export class CompilerApi { return this.compilers; } - // Resolve `config.granularities` with the appId-level compile context. The static-list and function - // forms are IMMUTABLE for the instance (a static list can't change; the function is frozen per - // appId by design), so they are memoized — including the in-flight promise, so concurrent initial - // compiles share one resolve/await and the function is invoked exactly once. The ENV form - // (`granularities === undefined`, reading CUBEJS_GRANULARITIES*) is NOT memoized: env can change at - // runtime, and re-resolving each call lets the folded hash trigger a recompile — matching rev-1. + // Resolve `config.granularities` with the appId-level compile context. Static-list/function forms + // are immutable for the instance, so they memoize the in-flight promise (concurrent compiles share + // one await; the function runs once per appId). The env form re-resolves each call so a runtime + // CUBEJS_GRANULARITIES* change still flows through the folded hash to trigger a recompile. private getResolvedGranularities(): Promise<{ config: GlobalGranularitiesConfig; hash: string }> { const resolve = () => resolveGlobalGranularities( this.granularities, @@ -1131,14 +1126,32 @@ export class CompilerApi { } /** - * The resolved-once global granularities config baked into the compiled model. Serves the - * /v1/granularities endpoint (per-appId catalog) — no per-request resolution. + * The `/v1/granularities` catalog for this appId — built-ins plus global customs — assembled from + * the config baked into the compiled model. No per-request resolution. */ - public async getGlobalGranularitiesConfig(options: { requestId?: string } = {}): Promise { + public async getGranularities(options: { requestId?: string } = {}): Promise { const compilers = await this.getCompilers(options); - // `getCompilers` has compiled the model, so the baked config is always present; fall back to the - // memoized resolve only defensively (should not happen on a successfully compiled model). - return compilers.metaTransformer.globalGranularitiesConfig ?? (await this.getResolvedGranularities()).config; + const config = compilers.metaTransformer.globalGranularitiesConfig ?? (await this.getResolvedGranularities()).config; + + const granularities: any[] = Object.entries(buildBuiltInsCatalog(config)) + .map(([name, entry]) => ({ type: 'built-in', name, ...entry })); + + for (const [name, def] of Object.entries(config.customGranularities)) { + // Skip customs that override a built-in — those are already folded into the catalog above. + // hasOwnProperty (not `in`) so a custom named e.g. `toString` isn't dropped via the prototype. + if (!isBuiltInGranularity(name)) { + granularities.push({ + type: 'custom', + name, + title: def.title || name, + ...(def.interval !== undefined ? { interval: def.interval } : {}), + ...(def.origin !== undefined ? { origin: def.origin } : {}), + ...(def.offset !== undefined ? { offset: def.offset } : {}), + ...(def.format !== undefined ? { format: def.format } : {}), + }); + } + } + return granularities; } public async metaConfig( From dba5209699114a1783d1ec1c20b7b4d5f8febb86 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Wed, 29 Jul 2026 20:30:29 +0200 Subject: [PATCH 22/22] refactor(granularities): resolve and apply global config in CubeSymbols instead of a bespoke CompilerApi compile flow --- .../src/compiler/CubeEvaluator.ts | 10 +- .../src/compiler/CubeSymbols.ts | 119 ++++++++++++++--- .../src/compiler/CubeToMetaTransformer.ts | 123 +++--------------- .../src/compiler/DataSchemaCompiler.ts | 2 +- .../src/compiler/GlobalGranularitiesConfig.ts | 23 ++++ .../src/compiler/PrepareCompiler.ts | 40 ++++-- .../src/compiler/index.ts | 1 + .../src/core/CompilerApi.ts | 73 ++++------- .../test/unit/granularities-bake.test.ts | 51 ++++---- 9 files changed, 229 insertions(+), 213 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 4b2fa35409886..e8db87580316a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -20,6 +20,7 @@ import { UserError } from './UserError'; import { BaseQuery, PreAggregationDefinitionExtended } from '../adapter'; import type { CubeValidator } from './CubeValidator'; import type { ErrorReporter } from './ErrorReporter'; +import type { GlobalGranularitiesConfig } from './GlobalGranularitiesConfig'; import { FinishedJoinTree } from './JoinGraph'; export type SegmentDefinition = { @@ -210,13 +211,14 @@ export class CubeEvaluator extends CubeSymbols { private isRbacEnabledCache: boolean | null = null; public constructor( - protected readonly cubeValidator: CubeValidator + protected readonly cubeValidator: CubeValidator, + granularitiesResolver?: () => Promise, ) { - super(true); + super(true, granularitiesResolver); } - public compile(cubes: any[], errorReporter: ErrorReporter) { - super.compile(cubes, errorReporter); + public async compile(cubes: any[], errorReporter: ErrorReporter) { + await super.compile(cubes, errorReporter); const validCubes = this.cubeList.filter(cube => this.cubeValidator.isCubeValid(cube)).sort((a, b) => { if (a.isView) { return 1; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 516b70ba2fcc2..db525f3017b2b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -5,7 +5,16 @@ import { camelize } from 'inflection'; import { UserError } from './UserError'; import { DynamicReference } from './DynamicReference'; import { camelizeCube } from './utils'; -import { normalizeGranularitiesBlock, NormalizedGranularitiesBlock } from './GranularityResolver'; +import { + normalizeGranularitiesBlock, + NormalizedGranularitiesBlock, + resolveDimensionGranularities, +} from './GranularityResolver'; +import { + buildBuiltInsCatalog, + DEFAULT_GRANULARITIES_CONFIG, + GlobalGranularitiesConfig, +} from './GlobalGranularitiesConfig'; import type { ErrorReporter } from './ErrorReporter'; import { TranspilerSymbolResolver } from './transpilers'; @@ -297,13 +306,30 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface private resolveSymbolsCallContext: any; - public constructor(evaluateViews = false) { + // Resolves the global granularities config; shared across the compilers of one prepareCompiler so + // a `config.granularities` function runs once per compile. + private readonly granularitiesResolver?: () => Promise; + + // Global granularities resolved at the start of compile(); the default catalog until then. + private globalGranularities: GlobalGranularitiesConfig = DEFAULT_GRANULARITIES_CONFIG; + + public constructor( + evaluateViews = false, + granularitiesResolver?: () => Promise, + ) { this.symbols = {}; this.builtCubes = {}; this.cubeDefinitions = {}; this.funcArgumentsValues = {}; this.cubeList = []; this.evaluateViews = evaluateViews; + this.granularitiesResolver = granularitiesResolver; + } + + // The resolved global config for this compile. Read by CubeToMetaTransformer (for + // `effectiveGranularities`) and by the /v1/granularities catalog. + public get globalGranularitiesConfig(): GlobalGranularitiesConfig { + return this.globalGranularities; } public free() { @@ -315,7 +341,14 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface this.resolveSymbolsCallContext = undefined; } - public compile(cubes: CubeDefinition[], errorReporter: ErrorReporter) { + public async compile(cubes: CubeDefinition[], errorReporter: ErrorReporter) { + // Resolve the global granularities config before any cube is transformed: the merge below writes + // global customs onto time dimensions, and the SQL layer resolves them from that same map. + // The phase driver chains each compile() through `.then()`, so awaiting here is safe. + if (this.granularitiesResolver) { + this.globalGranularities = await this.granularitiesResolver(); + } + this.cubeDefinitions = Object.fromEntries( cubes.map((c): [string, CubeDefinition] => [c.name, c]) ); @@ -609,28 +642,73 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface } as CubeSymbolsDefinition; } - // Stores the canonical `granularitiesBlock` on each time dimension and rewrites - // `granularities` to the dict of locally-defined customs only — preserving the legacy shape - // that BaseQuery, prepare-annotation, and CubeToMetaTransformer already read. + // Stores the canonical `granularitiesBlock` on each time dimension, rewrites `granularities` to + // the locally-defined customs — preserving the legacy shape that BaseQuery, prepare-annotation + // and CubeToMetaTransformer read — then layers the effective global customs on top so they are + // resolvable in SQL and visible to pre-aggregation matching. private normalizeDimensionGranularities(dimensions: Record | undefined) { if (!dimensions) { return; } + // Every time dimension participates: global customs apply even to one that declares no + // `granularities` of its own. for (const dim of Object.values(dimensions)) { - // A view's included dimension already carries a propagated granularitiesBlock (with the - // source dimension's includes/excludes) alongside the custom-only `granularities` map. - // Re-normalizing the custom-only map here would reset includes to '*' and drop the source's - // includes/excludes, so only normalize dimensions that haven't been normalized yet. - if (dim && dim.type === 'time' && 'granularities' in dim && !dim.granularitiesBlock) { - // Keep the raw user value for the validator (it runs after this and would otherwise only - // see the extracted customs, never the includes/excludes/custom dict). - dim.rawGranularities = dim.granularities; - const block: NormalizedGranularitiesBlock = normalizeGranularitiesBlock(dim.granularities); - dim.granularitiesBlock = block; - dim.granularities = block.custom; + if (dim && dim.type === 'time') { + // A view's included dimension already carries a propagated granularitiesBlock (with the + // source dimension's includes/excludes) alongside the custom-only `granularities` map. + // Re-normalizing the custom-only map here would reset includes to '*' and drop the source's + // includes/excludes, so only normalize dimensions that haven't been normalized yet. A + // dimension declaring no `granularities` needs no block — it takes the global config as-is, + // and attaching one would leak an empty block into the compiled model. + if (!dim.granularitiesBlock && 'granularities' in dim) { + // Keep the raw user value for the validator (it runs after this and would otherwise only + // see the extracted customs, never the includes/excludes/custom dict). + dim.rawGranularities = dim.granularities; + const block: NormalizedGranularitiesBlock = normalizeGranularitiesBlock(dim.granularities); + dim.granularitiesBlock = block; + dim.granularities = block.custom; + } + + this.mergeGlobalCustomsIntoDimension(dim); + } + } + } + + // Layer the dimension's effective global customs onto its `granularities` map (locals win on a + // name collision). Reassigns rather than mutating, so a view dimension sharing its source's map + // by reference isn't contaminated. + private mergeGlobalCustomsIntoDimension(dim: any) { + const globalCustom = this.globalGranularities.customGranularities; + if (Object.keys(globalCustom).length === 0) { + return; + } + + // `granularitiesBlock.custom` is the model's own customs, unaffected by the merge below. + const locals: Record = dim.granularitiesBlock?.custom ?? {}; + const resolved = resolveDimensionGranularities( + dim.granularitiesBlock ?? normalizeGranularitiesBlock(undefined), + this.globalGranularities.enabledBuiltIns, + globalCustom, + buildBuiltInsCatalog(this.globalGranularities), + ); + + // Only genuine global customs are baked in: built-ins resolve by name without a definition, and + // a local of the same name already wins. + const merged: Record = {}; + for (const [name, def] of Object.entries(resolved)) { + if (def.type === 'custom' && + Object.prototype.hasOwnProperty.call(globalCustom, name) && + !Object.prototype.hasOwnProperty.call(locals, name) + ) { + merged[name] = { ...def }; + delete (merged[name] as any).type; } } + + if (Object.keys(merged).length > 0) { + dim.granularities = { ...merged, ...locals }; + } } private camelCaseTypes(obj: Object | Array | undefined) { @@ -1582,8 +1660,11 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface return { interval: `1 ${granName}` }; } - // Custom granularities (local + baked-in globals) resolve from the compiled cube symbols. - return cube?.[dimName]?.[gr]?.[granName]; + // A local custom wins; otherwise fall back to a global custom from the resolved config. + return cube?.[dimName]?.[gr]?.[granName] ?? + (Object.prototype.hasOwnProperty.call(this.globalGranularities.customGranularities, granName) + ? this.globalGranularities.customGranularities[granName] + : undefined); } protected cubeDependenciesProxy(parentIndex, cubeName) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index aba31f97363ba..68becca595909 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -23,8 +23,6 @@ import { resolveNamedNumericFormat, STANDARD_FORMAT_SPECIFIERS, DEFAULT_FORMAT_S import { EffectiveGranularity, NormalizedGranularitiesBlock, - ResolvedGranularitySet, - GRANULARITY_STRING_FIELDS, normalizeGranularitiesBlock, resolveDimensionGranularities, serializeEffectiveGranularities, @@ -217,18 +215,12 @@ export class CubeToMetaTransformer implements CompilerInterface { */ public queries: TransformedCube[]; - // Resolved-once global granularities config for this appId, baked into the compiled model. - // CompilerApi resolves all config forms (env / static / function) before compile and passes the - // result here — the transformer never sees a function. - private readonly granularitiesConfig?: GlobalGranularitiesConfig; - // Precomputed once in compile() so plain dimensions need no per-dimension resolution: `defaultSet` - // and `defaultGlobalCustoms` are shared by reference across every dimension without a local block. + // is shared by reference across every dimension without a local block. private granularityState!: { config: GlobalGranularitiesConfig; catalog: Record; defaultSet: EffectiveGranularity[]; - defaultGlobalCustoms: Record; }; public constructor( @@ -237,7 +229,6 @@ export class CubeToMetaTransformer implements CompilerInterface { contextEvaluator: ContextEvaluator, viewGroupEvaluator: ViewGroupEvaluator, joinGraph: JoinGraph, - granularitiesConfig?: GlobalGranularitiesConfig ) { this.cubeValidator = cubeValidator; this.cubeSymbols = cubeEvaluator; @@ -245,36 +236,26 @@ export class CubeToMetaTransformer implements CompilerInterface { this.contextEvaluator = contextEvaluator; this.viewGroupEvaluator = viewGroupEvaluator; this.joinGraph = joinGraph; - this.granularitiesConfig = granularitiesConfig; this.cubes = []; this.queries = []; } - // The resolved global config baked into this compiled model. Exposed for the /v1/granularities - // endpoint, which serves the per-appId catalog from the compiled model rather than re-resolving. - public get globalGranularitiesConfig(): GlobalGranularitiesConfig | undefined { - return this.granularityState?.config; - } - public get viewGroups(): CompiledViewGroup[] { return this.viewGroupEvaluator.compiledViewGroups; } public compile(_cubes: any[], errorReporter: ErrorReporter): void { - // The config is already resolved (env / static / function) by CompilerApi at compile time and - // baked in here — a missing config means the default catalog. - const config = this.granularitiesConfig ?? DEFAULT_GRANULARITIES_CONFIG; + // CubeSymbols resolved the global config at the start of its own compile phase. + const config = this.cubeEvaluator.globalGranularitiesConfig ?? DEFAULT_GRANULARITIES_CONFIG; const catalog = buildBuiltInsCatalog(config); - // Resolve the no-local-block ("default") set once; every plain time dimension shares both its - // serialized wire form and its global-custom map by reference (no per-dimension resolution). - const defaultResolved = resolveDimensionGranularities( - normalizeGranularitiesBlock(undefined), config.enabledBuiltIns, config.customGranularities, catalog, - ); + // Resolve the no-local-block ("default") set once; every plain time dimension shares the + // serialized wire form by reference (no per-dimension resolution). this.granularityState = { config, catalog, - defaultSet: serializeEffectiveGranularities(defaultResolved), - defaultGlobalCustoms: this.globalCustomsOf(defaultResolved, config, {}), + defaultSet: serializeEffectiveGranularities(resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), config.enabledBuiltIns, config.customGranularities, catalog, + )), }; this.cubes = this.cubeSymbols.cubeList @@ -359,11 +340,11 @@ export class CubeToMetaTransformer implements CompilerInterface { const dimensionVisibility = isCubeVisible ? this.isVisible(extendedDimDef, !extendedDimDef.primaryKey) : false; - // Snapshot the dimension's LOCAL customs before any merge below: the deprecated - // `granularities` meta field must keep listing only the model's own custom granularities. - const localCustoms = extendedDimDef.granularities; - const localCustomEntries = localCustoms ? Object.entries(localCustoms) : []; const { granularitiesBlock } = extendedDimDef as any; + // The deprecated `granularities` meta field lists only the model's own custom + // granularities — `granularitiesBlock.custom`, which the global merge leaves untouched. + const localCustoms = granularitiesBlock?.custom ?? extendedDimDef.granularities; + const localCustomEntries = localCustoms ? Object.entries(localCustoms) : []; const dimType = this.dimensionDataType(extendedDimDef.type || 'string'); const dimFormat = this.transformDimensionFormat(extendedDimDef); const dimCurrency = extendedDimDef.currency?.toUpperCase(); @@ -372,23 +353,13 @@ export class CubeToMetaTransformer implements CompilerInterface { if (dimType === 'time') { const s = this.granularityState; const inputs = this.granularityInputsForDimension(cubeTitle, localCustoms, granularitiesBlock); - // Dimensions with a local block resolve individually; plain ones reuse the shared default - // (both the serialized set and the global-custom map) computed once in compile(). - let globalCustoms: Record; - if (inputs) { - const resolved = resolveDimensionGranularities( + // Dimensions with a local block resolve individually; plain ones reuse the shared + // default set computed once in compile(). + effectiveGranularities = inputs + ? serializeEffectiveGranularities(resolveDimensionGranularities( inputs, s.config.enabledBuiltIns, s.config.customGranularities, s.catalog, - ); - effectiveGranularities = serializeEffectiveGranularities(resolved); - globalCustoms = this.globalCustomsOf(resolved, s.config, localCustoms ?? {}); - } else { - effectiveGranularities = s.defaultSet; - globalCustoms = s.defaultGlobalCustoms; - } - - // Bake the effective GLOBAL customs into the dimension's `granularities` map (SQL resolves - // customs by name from this map, and pre-agg matching reads it). Locals win over globals. - this.mergeGlobalCustomsIntoDimension(cubeName, dimensionName, extendedDimDef, localCustoms, globalCustoms); + )) + : s.defaultSet; } return { @@ -486,64 +457,6 @@ export class CubeToMetaTransformer implements CompilerInterface { return { includes: block.includes, excludes: block.excludes, custom }; } - // From an already-resolved set, extract the GLOBAL customs a dimension exposes: entries that are - // custom, defined in the global config, and not shadowed by a local of the same name. Projected - // through GRANULARITY_STRING_FIELDS (the shared field list, so it can't drift from serialize/hash). - private globalCustomsOf( - resolved: ResolvedGranularitySet, - config: GlobalGranularitiesConfig, - localCustoms: Record, - ): Record { - const out: Record = {}; - for (const [name, def] of Object.entries(resolved)) { - if (def.type === 'custom' && - Object.prototype.hasOwnProperty.call(config.customGranularities, name) && - !Object.prototype.hasOwnProperty.call(localCustoms, name) - ) { - const projected: GranularityDefinition = {} as GranularityDefinition; - for (const field of GRANULARITY_STRING_FIELDS) { - if (def[field] !== undefined) { - (projected as any)[field] = def[field]; - } - } - out[name] = projected; - } - } - return out; - } - - // Bake global customs into a dimension's `granularities` map (locals win). Must write BOTH the - // `dimDef` object (pre-agg matching) and the distinct `symbols[cube][dim]` object (SQL - // resolveGranularity) — writing one leaves the other unable to resolve the custom. Reassign, never - // mutate in place, so a view dim sharing its source's map by reference isn't contaminated. - private mergeGlobalCustomsIntoDimension( - cubeName: string, - dimensionName: string, - dimDef: ExtendedCubeSymbolDefinition, - localCustoms: Record | undefined, - globalCustoms: Record, - ): void { - if (Object.keys(globalCustoms).length === 0) { - return; - } - const hasLocals = !!localCustoms && Object.keys(localCustoms).length > 0; - - // With no locals the baked map IS the shared globalCustoms — assign it by reference (every plain - // dimension then shares one object). Copy-on-write only when locals must be layered on top. - const write = (existing: Record | undefined) => ( - existing && Object.keys(existing).length > 0 - ? { ...globalCustoms, ...existing } // globals first, locals last so locals win on collisions - : globalCustoms - ); - - dimDef.granularities = write(hasLocals ? localCustoms : undefined); - - const symbolDim = (this.cubeEvaluator as any).symbols?.[cubeName]?.[dimensionName]; - if (symbolDim && symbolDim !== dimDef) { - symbolDim.granularities = write(symbolDim.granularities as Record | undefined); - } - } - public queriesForContext(contextId: string | null | undefined): TransformedCube[] { // return All queries if no context pass if (contextId == null || contextId.length === 0) { diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index 3d95c5b1bcf56..47f18811b4054 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -224,7 +224,7 @@ export class DataSchemaCompiler { try { return compileServices .map((compileService) => (() => compileService.compile(objects, errorsReport))) - .reduce((p, fn) => p.then(fn), Promise.resolve()) + .reduce>((p, fn) => p.then(() => fn()).then(() => undefined), Promise.resolve()) .catch((error) => { errorsReport.error(error); }); diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts index 01ee3d3116563..c4dd896933215 100644 --- a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -205,6 +205,29 @@ export function buildBuiltInsCatalog(globalConfig: GlobalGranularitiesConfig): R return catalog; } +// Wire shape of the `/v1/granularities` catalog: every built-in enabled for this model, plus the +// global customs. A custom that shadows a built-in is already folded into the built-in entry. +export function buildGranularitiesCatalog(config: GlobalGranularitiesConfig): Array> { + const catalog: Array> = Object.entries(buildBuiltInsCatalog(config)) + .map(([name, entry]) => ({ type: 'built-in', name, ...entry })); + + for (const [name, def] of Object.entries(config.customGranularities)) { + if (!isBuiltInGranularity(name)) { + catalog.push({ + type: 'custom', + name, + title: def.title || name, + ...Object.fromEntries( + GRANULARITY_STRING_FIELDS + .filter((f) => f !== 'title' && def[f] !== undefined) + .map((f) => [f, def[f] as string]), + ), + }); + } + } + return catalog; +} + // Fields that a config override actually changes in the emitted output, per granularity name. For // a name shadowing a built-in, buildBuiltInsCatalog honors only title/format (interval/offset/origin // are fixed at `1 ` for predefined granularities), so hashing the ignored fields would churn diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 516a213453c8c..22db4453c7ed5 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -26,7 +26,11 @@ import { CompilerCache } from './CompilerCache'; import { YamlCompiler } from './YamlCompiler'; import { ViewCompilationGate } from './ViewCompilationGate'; import type { ErrorReporter } from './ErrorReporter'; -import type { GlobalGranularitiesConfig } from './GlobalGranularitiesConfig'; +import { + type GlobalGranularitiesConfig, + type GranularitiesOption, + resolveGlobalGranularities, +} from './GlobalGranularitiesConfig'; export type PrepareCompilerOptions = { nativeInstance?: NativeInstance, @@ -41,13 +45,19 @@ export type PrepareCompilerOptions = { compiledScriptCache?: LRUCache; compiledYamlCache?: LRUCache; compiledJinjaCache?: LRUCache; - // Resolved global granularities config (all forms resolved by CompilerApi before compile), baked - // into the compiled model by CubeToMetaTransformer. - granularitiesConfig?: GlobalGranularitiesConfig; + // `config.granularities` as authored (env / static list / function). Resolved once at the start + // of compile and applied by CubeSymbols. + granularities?: GranularitiesOption; + // An already-resolved config, when the caller had to resolve it anyway (CompilerApi folds its + // hash into compilerVersion). Takes precedence over `granularities`, so a function form is + // invoked once per compile rather than once here and once there. + resolvedGranularities?: GlobalGranularitiesConfig; }; export interface CompilerInterface { - compile: (cubes: any[], errorReporter: ErrorReporter) => void; + // The phase driver chains each compile() through `.then()`, so a compiler may return a promise; + // any other return value is ignored. + compile: (cubes: any[], errorReporter: ErrorReporter) => unknown; } export type Compiler = { @@ -65,15 +75,27 @@ export type Compiler = { export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}): Compiler => { const nativeInstance = options.nativeInstance || new NativeInstance(); const cubeDictionary = new CubeDictionary(); - const cubeSymbols = new CubeSymbols(); - const viewCompiler = new CubeSymbols(true); + // One resolve shared by every compiler below, so a `config.granularities` function is invoked + // once per compile rather than once per compiler instance. + let granularitiesPromise: Promise | undefined; + const resolveGranularities = () => { + if (!granularitiesPromise) { + granularitiesPromise = options.resolvedGranularities + ? Promise.resolve(options.resolvedGranularities) + : resolveGlobalGranularities(options.granularities, options.compileContext); + } + return granularitiesPromise; + }; + + const cubeSymbols = new CubeSymbols(false, resolveGranularities); + const viewCompiler = new CubeSymbols(true, resolveGranularities); const viewCompilationGate = new ViewCompilationGate(); const cubeValidator = new CubeValidator(cubeSymbols); - const cubeEvaluator = new CubeEvaluator(cubeValidator); + const cubeEvaluator = new CubeEvaluator(cubeValidator, resolveGranularities); const contextEvaluator = new ContextEvaluator(cubeEvaluator); const viewGroupEvaluator = new ViewGroupEvaluator(cubeEvaluator, cubeValidator); const joinGraph = new JoinGraph(cubeValidator, cubeEvaluator); - const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, viewGroupEvaluator, joinGraph, options.granularitiesConfig); + const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, viewGroupEvaluator, joinGraph); const { maxQueryCacheSize, maxQueryCacheAge } = options; const compilerCache = new CompilerCache({ maxQueryCacheSize, maxQueryCacheAge }); const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler); diff --git a/packages/cubejs-schema-compiler/src/compiler/index.ts b/packages/cubejs-schema-compiler/src/compiler/index.ts index 26b60fcb0f059..4261468acfa47 100644 --- a/packages/cubejs-schema-compiler/src/compiler/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/index.ts @@ -25,6 +25,7 @@ export { resolveGlobalGranularitiesSync, getBuiltInGranularityDefaults, buildBuiltInsCatalog, + buildGranularitiesCatalog, granularityConfigHash, } from './GlobalGranularitiesConfig'; export { diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index e70d32d7c7ba5..477457d11a9d0 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -7,13 +7,12 @@ import { compile, Compiler, createQuery, - buildBuiltInsCatalog, + buildGranularitiesCatalog, CubeDefinition, EvaluatedCube, GlobalGranularitiesConfig, GranularitiesOption, granularityConfigHash, - isBuiltInGranularity, PreAggregationFilters, PreAggregationInfo, PreAggregationReferences, @@ -142,10 +141,7 @@ export class CompilerApi { protected readonly granularities?: GranularitiesOption; - // Resolved-once-per-appId global granularities config, baked into the compiled model. Resolved in - // Memoized resolved global granularities config (+ hash) for the static/function forms; see - // getResolvedGranularities. - private resolvedGranularitiesPromise?: Promise<{ config: GlobalGranularitiesConfig; hash: string }>; + private resolvedGranularitiesPromise?: Promise; protected queryFactory?: QueryFactory; @@ -239,9 +235,11 @@ export class CompilerApi { compilerVersion += `_${crypto.createHash('md5').update(JSON.stringify(files)).digest('hex')}`; } - // Fold the resolved granularities hash into compilerVersion so a config change forces a recompile. - const { config: resolvedGranularities, hash: granularitiesHash } = await this.getResolvedGranularities(); - compilerVersion += `_gran_${granularitiesHash}`; + // Global custom granularities are merged into the compiled model (they take part in + // pre-aggregation matching), so a config change must force a recompile. The resolved config is + // handed to the compiler so a `config.granularities` function runs once per compile. + const resolvedGranularities = await this.resolveGranularities(); + compilerVersion += `_gran_${granularityConfigHash(resolvedGranularities)}`; if (!this.compilers || this.compilerVersion !== compilerVersion) { this.compilers = this.compileSchema(compilerVersion, resolvedGranularities, options.requestId).catch(e => { @@ -254,24 +252,18 @@ export class CompilerApi { return this.compilers; } - // Resolve `config.granularities` with the appId-level compile context. Static-list/function forms - // are immutable for the instance, so they memoize the in-flight promise (concurrent compiles share - // one await; the function runs once per appId). The env form re-resolves each call so a runtime - // CUBEJS_GRANULARITIES* change still flows through the folded hash to trigger a recompile. - private getResolvedGranularities(): Promise<{ config: GlobalGranularitiesConfig; hash: string }> { - const resolve = () => resolveGlobalGranularities( - this.granularities, - { securityContext: this.compileContext?.securityContext }, - ).then(config => ({ config, hash: granularityConfigHash(config) })); - - // Env form: always re-resolve (cheap env parse) so a runtime change is picked up. + /** + * The resolved global granularities config. Memoized for the static/function forms — both are + * fixed for the instance, and a `config.granularities` function must not run per request. The env + * form re-resolves (a cheap parse) so a runtime CUBEJS_GRANULARITIES change still forces a recompile. + */ + private resolveGranularities(): Promise { if (this.granularities === undefined) { - return resolve(); + return resolveGlobalGranularities(undefined, this.compileContext); } - if (!this.resolvedGranularitiesPromise) { - this.resolvedGranularitiesPromise = resolve(); - // Allow a retry if the resolve (e.g. a throwing function form) fails. + this.resolvedGranularitiesPromise = resolveGlobalGranularities(this.granularities, this.compileContext); + // Let a failed resolve (e.g. a throwing function form) be retried. this.resolvedGranularitiesPromise.catch(() => { this.resolvedGranularitiesPromise = undefined; }); @@ -294,7 +286,11 @@ export class CompilerApi { }); } - public async compileSchema(compilerVersion: string, granularitiesConfig: GlobalGranularitiesConfig, requestId?: string): Promise { + public async compileSchema( + compilerVersion: string, + resolvedGranularities: GlobalGranularitiesConfig, + requestId?: string, + ): Promise { const startCompilingTime = new Date().getTime(); try { this.logger(this.compilers ? 'Recompiling schema' : 'Compiling schema', { @@ -311,7 +307,7 @@ export class CompilerApi { compiledScriptCache: this.compiledScriptCache, compiledJinjaCache: this.compiledJinjaCache, compiledYamlCache: this.compiledYamlCache, - granularitiesConfig, + resolvedGranularities, }); this.queryFactory = await this.createQueryFactory(compilers); @@ -1126,32 +1122,11 @@ export class CompilerApi { } /** - * The `/v1/granularities` catalog for this appId — built-ins plus global customs — assembled from - * the config baked into the compiled model. No per-request resolution. + * The `/v1/granularities` catalog for this model — built-ins plus global customs. */ public async getGranularities(options: { requestId?: string } = {}): Promise { const compilers = await this.getCompilers(options); - const config = compilers.metaTransformer.globalGranularitiesConfig ?? (await this.getResolvedGranularities()).config; - - const granularities: any[] = Object.entries(buildBuiltInsCatalog(config)) - .map(([name, entry]) => ({ type: 'built-in', name, ...entry })); - - for (const [name, def] of Object.entries(config.customGranularities)) { - // Skip customs that override a built-in — those are already folded into the catalog above. - // hasOwnProperty (not `in`) so a custom named e.g. `toString` isn't dropped via the prototype. - if (!isBuiltInGranularity(name)) { - granularities.push({ - type: 'custom', - name, - title: def.title || name, - ...(def.interval !== undefined ? { interval: def.interval } : {}), - ...(def.origin !== undefined ? { origin: def.origin } : {}), - ...(def.offset !== undefined ? { offset: def.offset } : {}), - ...(def.format !== undefined ? { format: def.format } : {}), - }); - } - } - return granularities; + return buildGranularitiesCatalog(compilers.cubeEvaluator.globalGranularitiesConfig); } public async metaConfig( diff --git a/packages/cubejs-server-core/test/unit/granularities-bake.test.ts b/packages/cubejs-server-core/test/unit/granularities-bake.test.ts index fa5ac864e9da3..8cfcb279f81c1 100644 --- a/packages/cubejs-server-core/test/unit/granularities-bake.test.ts +++ b/packages/cubejs-server-core/test/unit/granularities-bake.test.ts @@ -2,12 +2,12 @@ import { SchemaFileRepository } from '@cubejs-backend/shared'; import { CompilerApi } from '../../src/core/CompilerApi'; import { DbTypeInternalFn } from '../../src/core/types'; -// CUB-2567 rev 2: global granularity config is resolved ONCE per appId at compile time -// (env | static list | function(ctx)), its hash folded into compilerVersion for ALL forms, and the -// effective per-time-dimension set baked into the compiled model. Global CUSTOM granularities are -// merged into each time dimension's `granularities` symbol map at compile (locals win). Both -// /v1/meta and the SQL path read the baked values — no per-request resolution. This suite verifies -// the bake, the fold-into-compilerVersion, and the resolve-once guarantee. +// Global granularity config (env | static list | function(ctx)) is resolved once per compile and +// applied by CubeSymbols: global CUSTOM granularities are merged into each time dimension's +// `granularities` map (locals win), so both /v1/meta and the SQL path — including pre-aggregation +// matching — read the same values with no per-request resolution. The resolved hash is folded into +// compilerVersion, since the config affects the compiled model. This suite verifies the merge, the +// fold-into-compilerVersion, and the resolve-once guarantee. // A CompilerApi that counts how many times the granularities FUNCTION form is invoked, so we can // assert it runs once per compile (per appId), not once per metaConfig/getSql call. @@ -102,14 +102,14 @@ const bakedDimGranularities = async (api: CompilerApi, dimPath: string): Promise return compilers.cubeEvaluator.dimensionByPath(dimPath).granularities || {}; }; -// The SEPARATE `symbols[cube][dim]` map that CubeSymbols.resolveGranularity reads for SQL. The -// compile-time bake dual-writes here too; without it SQL can't resolve a baked global custom. +// The `symbols[cube][dim]` map CubeSymbols.resolveGranularity reads for SQL. It derives from the +// same dimension definition the merge writes, so a global custom is resolvable here too. const symbolsDimGranularities = async (api: CompilerApi, cube: string, dim: string): Promise> => { const compilers = await (api as any).getCompilers(); return (compilers.cubeEvaluator as any).symbols?.[cube]?.[dim]?.granularities || {}; }; -describe('granularities baked at compile time (CUB-2567 rev 2)', () => { +describe('global granularities resolved and applied at compile time', () => { describe('env / static config', () => { afterEach(() => { delete process.env.CUBEJS_GRANULARITIES; @@ -130,13 +130,12 @@ describe('granularities baked at compile time (CUB-2567 rev 2)', () => { api.dispose(); }); - // 2. Static global customs are baked into BOTH dimension object graphs the downstream paths - // read: the evaluatedCubes/cubeList copy (dimensionByPath, timeDimensionsForCube, pre-agg - // matching) AND the separate symbols copy (CubeSymbols.resolveGranularity for SQL). The merge - // must reach both — writing only the first makes SQL unable to resolve a global custom, and - // pre-agg matching (granularityHierarchies builds a Granularity for every baked custom) then - // throws. This test drives a real getSql() on a model with a configured global custom to lock - // that in, and confirms per-dimension excludes are honored. + // 2. Static global customs reach every dimension map the downstream paths read: the + // evaluatedCubes/cubeList copy (dimensionByPath, timeDimensionsForCube, pre-agg matching) and + // the symbols copy (CubeSymbols.resolveGranularity for SQL). Both derive from the definition + // the merge writes, so a global custom resolves in SQL and pre-agg matching doesn't throw + // "Granularity does not exist". Drives a real getSql() to lock that in, and confirms + // per-dimension excludes are honored. test('static global customs are baked into the consumed map; SQL resolves them; excludes honored', async () => { const api = createApi({ granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-02-01' }], @@ -177,16 +176,16 @@ describe('granularities baked at compile time (CUB-2567 rev 2)', () => { api.dispose(); }); - // 3. Time dimensions without local customization share ONE default set instance (memory saver). - test('time dimensions without local customization share one default set instance', async () => { + // 3. Time dimensions without local customization all resolve to the same effective set. + test('time dimensions without local customization resolve to the same set', async () => { const api = createApi(); const cubes = await api.metaConfig(ctxFor('a'), {}); const created = dimByName(cubes, 'Orders.created_at'); const updated = dimByName(cubes, 'Orders.updated_at'); - expect(created.effectiveGranularities).toBe(updated.effectiveGranularities); - // A dimension with a local block gets its own set, not the shared default. + expect(created.effectiveGranularities).toEqual(updated.effectiveGranularities); + // A dimension with a local block resolves to a different set. expect(dimByName(cubes, 'Events.ts').effectiveGranularities) - .not.toBe(created.effectiveGranularities); + .not.toEqual(created.effectiveGranularities); api.dispose(); }); @@ -336,14 +335,14 @@ describe('granularities baked at compile time (CUB-2567 rev 2)', () => { const baked = await bakedDimGranularities(api, 'Orders.created_at'); expect(baked.sprint).toMatchObject({ interval: '2 weeks', origin: '2024-01-01' }); - // Plain dimensions (no local block) share the SAME globalCustoms object by reference — the - // by-reference merge assigns one shared map to every plain dim (copy-on-write only for locals). + // Every plain dimension (no local block) gets the same global customs merged in. const bakedUpdated = await bakedDimGranularities(api, 'Orders.updated_at'); - expect(bakedUpdated).toBe(baked); + expect(bakedUpdated).toEqual(baked); - // A dimension with a local block gets a fresh copy-on-write object, not the shared map. + // A dimension with local customs keeps them alongside the global one. const bakedCollide = await bakedDimGranularities(api, 'Orders.collide_at'); - expect(bakedCollide).not.toBe(baked); + expect(bakedCollide).not.toEqual(baked); + expect(bakedCollide.sprint).toBeDefined(); // A dimension that excludes it does NOT get it baked in. const bakedExcluded = await bakedDimGranularities(api, 'Orders.excluded_at');