diff --git a/DEPRECATION.md b/DEPRECATION.md index c5e3f98026f37..3282b7c458a3d 100644 --- a/DEPRECATION.md +++ b/DEPRECATION.md @@ -70,6 +70,7 @@ features: | Removed | [Elasticsearch driver](#elasticsearch-driver) | v1.6.0 | v1.7.0 | | Removed | [`context_to_roles`](#context-to-roles) | v1.6.4 | v1.7.0 | | Deprecated | [Node.js 22](#nodejs-22) | v1.7.0 | | +| Deprecated | [`granularities` field on time dimensions in `/v1/meta`](#granularities-field-on-time-dimensions-in-v1meta) | v1.7.4 | | ### Node.js 8 @@ -454,3 +455,15 @@ The `context_to_roles` configuration option has been removed. Please use `contex Node.js 22 is in maintenance mode from [October 21, 2025][link-nodejs-eol]. This means no more new features, only security updates. Please upgrade to Node.js 24 or higher. + +### `granularities` field on time dimensions in `/v1/meta` + +**Deprecated in Release: v1.7.4** + +The `granularities` field returned for each time dimension by the `/v1/meta` endpoint lists only +the custom granularities defined in the data model, without the enabled built-in granularities or a +`type` discriminator. It is deprecated in favor of the `effectiveGranularities` field, which returns +the full reconciled set (enabled built-ins, global custom granularities, and per-dimension custom +granularities), each tagged with `type` (`built-in` or `custom`) and carrying `title`, `format`, and +`interval`. The `granularities` field remains for backward compatibility and will be removed in a +future release; new integrations should read `effectiveGranularities`. diff --git a/packages/cubejs-api-gateway/openspec.yml b/packages/cubejs-api-gateway/openspec.yml index 0a5c71ddddc04..6e187211ae0ec 100644 --- a/packages/cubejs-api-gateway/openspec.yml +++ b/packages/cubejs-api-gateway/openspec.yml @@ -4,6 +4,30 @@ info: version: "1.0.0" title: "Cube.js" paths: + "/v1/granularities": + get: + summary: "List the granularities available for this deployment" + description: "Returns the granularities enabled in this deployment — built-ins plus any custom granularities defined via `CUBEJS_GRANULARITIES` or `config.granularities`. Resolved once per application ID at data-model compile time and served from the compiled model." + operationId: "granularitiesV1" + responses: + "200": + description: "successful operation" + content: + application/json: + schema: + $ref: "#/components/schemas/V1GranularitiesResponse" + "4XX": + description: "Request could not be completed" + content: + application/json: + schema: + $ref: "#/components/schemas/V1Error" + "5XX": + description: "Internal Server Error" + content: + application/json: + schema: + $ref: "#/components/schemas/V1Error" "/v1/meta": get: summary: "Load Metadata" @@ -109,6 +133,8 @@ components: type: "object" V1CubeMetaDimensionGranularity: type: "object" + deprecated: true + description: "Deprecated. Legacy shape listing only model-defined custom granularities. Use V1CubeMetaDimensionEffectiveGranularity via `effectiveGranularities`." required: - name - title @@ -125,6 +151,44 @@ components: type: "string" origin: type: "string" + V1CubeMetaDimensionEffectiveGranularity: + type: "object" + description: "Reconciled granularity: enabled built-ins plus global and per-dimension custom granularities." + required: + - name + - type + - title + properties: + name: + type: "string" + type: + type: "string" + description: "Built-in (year/quarter/month/...) or user-defined custom granularity." + enum: + - built-in + - custom + title: + type: "string" + format: + type: "string" + description: "d3-time-format string used by clients to display bucketed timestamps." + interval: + type: "string" + offset: + type: "string" + origin: + type: "string" + V1GranularitiesResponse: + type: "object" + description: "Response shape of GET /v1/granularities." + properties: + data: + type: "object" + properties: + granularities: + type: array + items: + $ref: "#/components/schemas/V1CubeMetaDimensionEffectiveGranularity" V1CubeMetaDimension: type: "object" required: @@ -145,9 +209,15 @@ components: description: "When dimension is defined in View, it keeps the original path: Cube.dimension" type: "string" granularities: + deprecated: true + description: "Deprecated. Use `effectiveGranularities`." type: array items: $ref: "#/components/schemas/V1CubeMetaDimensionGranularity" + effectiveGranularities: + type: array + items: + $ref: "#/components/schemas/V1CubeMetaDimensionEffectiveGranularity" meta: type: "object" format: diff --git a/packages/cubejs-api-gateway/package.json b/packages/cubejs-api-gateway/package.json index aed2702f2524d..67a8444cc3555 100644 --- a/packages/cubejs-api-gateway/package.json +++ b/packages/cubejs-api-gateway/package.json @@ -29,6 +29,7 @@ "dependencies": { "@cubejs-backend/native": "1.7.23", "@cubejs-backend/query-orchestrator": "1.7.23", + "@cubejs-backend/schema-compiler": "1.7.23", "@cubejs-backend/shared": "1.7.23", "@ungap/structured-clone": "^0.3.4", "assert-never": "^1.4.0", diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index e595cc3e61c1b..1cb9a293051aa 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -14,6 +14,7 @@ import { QueryAlias, CacheMode, LoggerFn, + isPredefinedGranularity, } from '@cubejs-backend/shared'; import { ResultArrayWrapper, @@ -478,6 +479,17 @@ class ApiGateway { }) ); + app.get( + `${this.basePath}/v1/granularities`, + userMiddlewares, + userAsyncHandler(async (req, res) => { + await this.granularities({ + context: req.context, + res: this.resToResultFn(res), + }); + }) + ); + app.post( `${this.basePath}/v1/cubesql`, userMiddlewares, @@ -750,6 +762,27 @@ class ApiGateway { } } + public async granularities({ context, res }: { + context: RequestContext, + res: ResponseResultFn, + }) { + const requestStarted = new Date(); + try { + await this.assertApiScope('meta', context.securityContext); + const compilerApi = await this.getCompilerApi(context); + const granularities = await compilerApi.getGranularities({ requestId: context.requestId }); + res({ data: { granularities } }); + } catch (e: any) { + this.handleError({ + e, + context, + // @ts-ignore + res, + requestStarted, + }); + } + } + public async metaExtended({ context, res, onlyViews }: { context: ExtendedRequestContext, res: ResponseResultFn, @@ -1176,7 +1209,10 @@ class ApiGateway { } else { const metaCacheKey = JSON.stringify(ctx); if (!metaCache.has(metaCacheKey)) { - metaCache.set(metaCacheKey, await compiler.metaConfigExtended(context, ctx)); + // `ctx` (outer context merged with the job's own context) is the request context; + // passing the outer `context` here would select visibility and granularities for + // the wrong tenant. + metaCache.set(metaCacheKey, await compiler.metaConfigExtended(ctx, { requestId: ctx.requestId })); } // checking and fetching result status diff --git a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts index 2f53164369ff1..0ef992c0d9a25 100644 --- a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts +++ b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts @@ -7,6 +7,7 @@ import R from 'ramda'; import { isPredefinedGranularity } from '@cubejs-backend/shared'; +import { BUILT_IN_GRANULARITIES } from '@cubejs-backend/schema-compiler'; import { MetaConfig, MetaConfigMap, toConfigMap } from './to-config-map'; import { MemberType } from '../types/strings'; import { MemberType as MemberTypeEnum } from '../types/enums'; @@ -14,7 +15,10 @@ import { MemberExpression } from '../types/query'; type GranularityMeta = { name: string; + type?: 'built-in' | 'custom'; title: string; + /** d3-time-format string for displaying bucketed timestamps. */ + format?: string; interval: string; offset?: string; origin?: string; @@ -37,6 +41,38 @@ type ConfigItem = { granularities?: GranularityMeta[]; }; +// Effective granularity meta for one queried time dimension, read from the enriched meta config. +// A queried granularity outside the effective set still annotates: disabled built-ins are +// synthesized from defaults; unknown customs yield undefined (never the deprecated legacy array). +function resolveGranularityMeta( + configMap: MetaConfigMap, + dimension: string, + granularity: string, +): GranularityMeta | undefined { + const cubeName = dimension.split('.')[0]; + const dimConfig: any = configMap[cubeName]?.[MemberTypeEnum.DIMENSIONS] + ?.find((m: any) => m.name === dimension); + + const resolved = dimConfig?.effectiveGranularities + ?.find((g: GranularityMeta) => g.name === granularity); + if (resolved) { + return resolved; + } + + if (isPredefinedGranularity(granularity)) { + const defaults = BUILT_IN_GRANULARITIES[granularity]; + return { + name: granularity, + type: 'built-in', + title: defaults?.title || granularity, + interval: `1 ${granularity}`, + ...(defaults?.format ? { format: defaults.format } : {}), + }; + } + + return undefined; +} + type AnnotatedConfigItem = Omit & { granularity?: GranularityMeta; }; @@ -113,18 +149,7 @@ function prepareAnnotation(metaConfig: MetaConfig[], query: any) { let dimAnnotation: [string, AnnotatedConfigItem] | undefined; if (an) { - let granularityMeta: GranularityMeta | undefined; - if (isPredefinedGranularity(td.granularity)) { - granularityMeta = { - name: td.granularity, - title: td.granularity, - interval: `1 ${td.granularity}`, - }; - } else if (an[1].granularities) { - // No need to send all the granularities defined, only those make sense for this query - granularityMeta = an[1].granularities.find(g => g.name === td.granularity); - } - + const granularityMeta = resolveGranularityMeta(configMap, td.dimension, td.granularity); const { granularities: _, ...rest } = an[1]; dimAnnotation = [an[0], { ...rest, granularity: granularityMeta }]; } diff --git a/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts b/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts index 2fe52c50fb52d..6695e4a7c4a3d 100644 --- a/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts +++ b/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts @@ -182,6 +182,7 @@ describe('prepareAnnotation helpers', () => { }).timeDimensions ).toEqual({ 'cube_name.member': { + currency: undefined, description: undefined, format: undefined, meta: undefined, @@ -190,6 +191,7 @@ describe('prepareAnnotation helpers', () => { type: undefined, }, 'cube_name.member.day': { + currency: undefined, description: undefined, format: undefined, meta: undefined, @@ -198,8 +200,10 @@ describe('prepareAnnotation helpers', () => { type: undefined, granularity: { name: 'day', - title: 'day', + type: 'built-in', + title: 'Day', interval: '1 day', + format: '%Y-%m-%d', } }, }); @@ -239,4 +243,70 @@ describe('prepareAnnotation helpers', () => { }).timeDimensions ).toEqual({}); }); + + describe('granularity resolution from effectiveGranularities', () => { + const metaConfig = (effectiveGranularities?: any[]) => [{ + config: ({ + name: 'cube_name', + title: 'cube name', + dimensions: [{ + name: 'cube_name.member', + type: 'time', + ...(effectiveGranularities ? { effectiveGranularities } : {}), + }], + }) as { name: string; title: string; }, + }]; + + const tdQuery = (granularity: string) => ({ + dimensions: ['cube_name.member'], + timeDimensions: [{ dimension: 'cube_name.member', granularity }], + }); + + test('reads the queried granularity from the effective set (global override honored)', () => { + const result = prepareAnnotation( + metaConfig([ + { name: 'day', type: 'built-in', title: 'Tag', interval: '1 day', format: '%d.%m.%Y' }, + { name: 'fiscal_year', type: 'custom', title: 'Fiscal Year', interval: '1 year', origin: '2024-02-01' }, + ]), + tdQuery('day'), + ); + expect((result.timeDimensions['cube_name.member.day'] as any).granularity).toEqual({ + name: 'day', type: 'built-in', title: 'Tag', interval: '1 day', format: '%d.%m.%Y', + }); + }); + + test('resolves a custom granularity from the effective set', () => { + const result = prepareAnnotation( + metaConfig([ + { name: 'fiscal_year', type: 'custom', title: 'Fiscal Year', interval: '1 year', origin: '2024-02-01' }, + ]), + tdQuery('fiscal_year'), + ); + expect((result.timeDimensions['cube_name.member.fiscal_year'] as any).granularity).toEqual({ + name: 'fiscal_year', type: 'custom', title: 'Fiscal Year', interval: '1 year', origin: '2024-02-01', + }); + }); + + test('synthesizes a config-disabled built-in from defaults', () => { + const result = prepareAnnotation( + metaConfig([ + { name: 'year', type: 'built-in', title: 'Year', interval: '1 year', format: '%Y' }, + ]), + tdQuery('day'), + ); + expect((result.timeDimensions['cube_name.member.day'] as any).granularity).toEqual({ + name: 'day', type: 'built-in', title: 'Day', interval: '1 day', format: '%Y-%m-%d', + }); + }); + + test('unknown custom granularity yields undefined, never the legacy array', () => { + const result = prepareAnnotation( + metaConfig([ + { name: 'day', type: 'built-in', title: 'Day', interval: '1 day', format: '%Y-%m-%d' }, + ]), + tdQuery('some_custom'), + ); + expect((result.timeDimensions['cube_name.member.some_custom'] as any).granularity).toBeUndefined(); + }); + }); }); diff --git a/packages/cubejs-api-gateway/test/index.test.ts b/packages/cubejs-api-gateway/test/index.test.ts index 87d0825e0bdf7..d3e5201fd4483 100644 --- a/packages/cubejs-api-gateway/test/index.test.ts +++ b/packages/cubejs-api-gateway/test/index.test.ts @@ -332,8 +332,10 @@ describe('API Gateway', () => { expect(res.body && res.body.data).toStrictEqual([{ 'Foo.bar': '42' }]); expect(res.body.annotation.timeDimensions['Foo.timeGranularities.half_year_by_1st_april']) .toStrictEqual({ + type: 'time', granularity: { name: 'half_year_by_1st_april', + type: 'custom', title: 'Half Year By1 St April', interval: '6 months', offset: '3 months', @@ -354,8 +356,10 @@ describe('API Gateway', () => { expect(res.body && res.body.data).toStrictEqual([{ 'Foo.bar': '42' }]); expect(res.body.annotation.timeDimensions['Foo.timeGranularities.half_year_by_1st_april']) .toStrictEqual({ + type: 'time', granularity: { name: 'half_year_by_1st_april', + type: 'custom', title: 'Half Year By1 St April', interval: '6 months', offset: '3 months', @@ -687,6 +691,58 @@ describe('API Gateway', () => { expect(res.body.cubes[0]?.segments.find(segment => segment.name === 'Foo.quux').description).toBe('segment from compilerApi mock'); }); + test('meta endpoint passes through effectiveGranularities from CompilerApi', async () => { + const { app } = await createApiGateway(); + + const res = await request(app) + .get('/cubejs-api/v1/meta') + .set('Authorization', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.t-IDcSemACt8x4iTMCda8Yhe3iZaWbvV5XKSTbuAn0M') + .expect(200); + + const dim = res.body.cubes[0]?.dimensions.find(d => d.name === 'Foo.timeGranularities'); + expect(dim.effectiveGranularities).toEqual([ + { name: 'year', type: 'built-in', title: 'Year', interval: '1 year', format: '%Y' }, + { + name: 'half_year_by_1st_april', + type: 'custom', + title: 'Half Year By1 St April', + interval: '6 months', + offset: '3 months', + }, + ]); + // The deprecated legacy array is preserved unchanged, and the internal block never leaks. + expect(dim.granularities).toEqual([ + { + name: 'half_year_by_1st_april', + title: 'Half Year By1 St April', + interval: '6 months', + offset: '3 months', + }, + ]); + expect(dim.granularitiesBlock).toBeUndefined(); + }); + + test('granularities endpoint returns the per-appId catalog baked into the compiled model', async () => { + const { app } = await createApiGateway(); + + const res = await request(app) + .get('/cubejs-api/v1/granularities') + .set('Authorization', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.t-IDcSemACt8x4iTMCda8Yhe3iZaWbvV5XKSTbuAn0M') + .expect(200); + + expect(res.body.data.granularities).toEqual([ + { type: 'built-in', name: 'year', title: 'Year', format: '%Y', interval: '1 year' }, + { type: 'built-in', name: 'month', title: 'Month', format: '%b %Y', interval: '1 month' }, + { + type: 'custom', + name: 'fiscal_year', + title: 'Fiscal Year', + interval: '1 year', + origin: '2024-02-01', + }, + ]); + }); + test('meta endpoint returns view groups', async () => { const { app } = await createApiGateway(); diff --git a/packages/cubejs-api-gateway/test/mocks.ts b/packages/cubejs-api-gateway/test/mocks.ts index 5ea9898f8487a..47c56f3a4890a 100644 --- a/packages/cubejs-api-gateway/test/mocks.ts +++ b/packages/cubejs-api-gateway/test/mocks.ts @@ -80,6 +80,14 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({ return { query, denied: false }; }, + async getGranularities(_options: any = {}) { + return [ + { type: 'built-in', name: 'year', title: 'Year', format: '%Y', interval: '1 year' }, + { type: 'built-in', name: 'month', title: 'Month', format: '%b %Y', interval: '1 month' }, + { type: 'custom', name: 'fiscal_year', title: 'Fiscal Year', interval: '1 year', origin: '2024-02-01' }, + ]; + }, + async metaConfig(_ctx, options: any = {}) { const cubes = [ { @@ -106,6 +114,7 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({ }, { name: 'Foo.timeGranularities', + type: 'time', isVisible: true, granularities: [ { @@ -114,7 +123,18 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({ interval: '6 months', offset: '3 months' } - ] + ], + // As attached by CompilerApi (baked or variant-selected) before meta reaches the gateway. + effectiveGranularities: [ + { name: 'year', type: 'built-in', title: 'Year', interval: '1 year', format: '%Y' }, + { + name: 'half_year_by_1st_april', + type: 'custom', + title: 'Half Year By1 St April', + interval: '6 months', + offset: '3 months', + }, + ], }, ], segments: [ diff --git a/packages/cubejs-backend-native/python/cube/src/__init__.py b/packages/cubejs-backend-native/python/cube/src/__init__.py index c793af0869056..d7d4ea822abe8 100644 --- a/packages/cubejs-backend-native/python/cube/src/__init__.py +++ b/packages/cubejs-backend-native/python/cube/src/__init__.py @@ -69,6 +69,9 @@ class Configuration: check_sql_auth: Callable can_switch_sql_user: Callable extend_context: Callable + # Mirrors `granularities` in the JS config: a list of granularity names and/or custom-granularity + # definitions, or a function returning the same. Drives /v1/granularities and /v1/meta enrichment. + granularities: Union[list, Callable[[RequestContext], list]] scheduled_refresh_contexts: Callable context_to_api_scopes: Callable repository_factory: Callable @@ -117,6 +120,7 @@ def __init__(self): self.can_switch_sql_user = None self.query_rewrite = None self.extend_context = None + self.granularities = None self.scheduled_refresh_contexts = None self.scheduled_refresh_time_zones = None self.context_to_api_scopes = None diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index ad63f10ba1cac..3a44dd15a8ea8 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -2049,6 +2049,18 @@ const variables: Record any> = { .asString(), accessPolicyMaskNumber: () => get('CUBEJS_ACCESS_POLICY_MASK_NUMBER') .asString(), + // Comma-separated names (built-in or custom). Empty/unset = all 8 built-ins enabled. + granularities: () => get('CUBEJS_GRANULARITIES') + .asArray(','), + // `getEnv` forwards `opts` positionally, so callers pass `{ name }` (matches dbType: { dataSource }). + granularityCustomInterval: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_INTERVAL`) + .asString(), + granularityCustomTitle: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_TITLE`) + .asString(), + granularityCustomOffset: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_OFFSET`) + .asString(), + granularityCustomOrigin: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_ORIGIN`) + .asString(), }; type Vars = typeof variables; diff --git a/packages/cubejs-client-core/src/time.ts b/packages/cubejs-client-core/src/time.ts index 9143dd8bccf04..036860a649432 100644 --- a/packages/cubejs-client-core/src/time.ts +++ b/packages/cubejs-client-core/src/time.ts @@ -22,7 +22,12 @@ export type SqlInterval = string; // TODO: Define a better type as unitOfTime.DurationConstructor in moment.js export type ParsedInterval = Record; +// Runtime shape for custom-granularity time-series math. For built-ins, see +// TimeDimensionPredefinedGranularity (query value) and GranularityAnnotation (response field). export type Granularity = { + type?: 'built-in' | 'custom'; + title?: string; + format?: string; interval: SqlInterval; origin?: string; offset?: SqlInterval; diff --git a/packages/cubejs-client-core/src/types.ts b/packages/cubejs-client-core/src/types.ts index 4a3ee92658c17..055b6bfadd5b4 100644 --- a/packages/cubejs-client-core/src/types.ts +++ b/packages/cubejs-client-core/src/types.ts @@ -12,7 +12,11 @@ export type TQueryOrderArray = Array<[string, QueryOrder]>; export type GranularityAnnotation = { name: string; + type?: 'built-in' | 'custom'; title: string; + /** d3-time-format string for displaying bucketed timestamps. */ + format?: string; + /** Always present: built-ins use "1 "; customs carry the user-defined interval. */ interval: string; offset?: string; origin?: string; @@ -433,11 +437,27 @@ export type TCubeMeasure = BaseCubeMember & { currency?: string; }; +/** + * @deprecated Use `EffectiveGranularity` / `CubeTimeDimension.effectiveGranularities`. + * Legacy meta shape: model-defined custom granularities only. + */ export type CubeTimeDimensionGranularity = { name: string; title: string; }; +/** Reconciled granularity for a time dimension: enabled built-ins + global and local customs. */ +export type EffectiveGranularity = { + name: string; + type: 'built-in' | 'custom'; + title: string; + /** d3-time-format string for displaying bucketed timestamps. */ + format?: string; + interval?: string; + offset?: string; + origin?: string; +}; + export type BaseCubeDimension = BaseCubeMember & { primaryKey?: boolean; suggestFilterValues: boolean; @@ -448,7 +468,12 @@ export type BaseCubeDimension = BaseCubeMember & { }; export type CubeTimeDimension = BaseCubeDimension & - { type: 'time'; granularities?: CubeTimeDimensionGranularity[] }; + { + type: 'time'; + /** @deprecated Use `effectiveGranularities`. */ + granularities?: CubeTimeDimensionGranularity[]; + effectiveGranularities?: EffectiveGranularity[]; + }; export type TCubeDimension = (BaseCubeDimension & { type: Exclude }) | diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 162fb2290b202..c75544c8b2095 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -139,11 +139,11 @@ export class BaseQuery { */ constructor(compilers, options) { this.compilers = compilers; + this.options = options || {}; /** @type {import('../compiler/CubeEvaluator').CubeEvaluator} */ this.cubeEvaluator = compilers.cubeEvaluator; /** @type {import('../compiler/JoinGraph').JoinGraph} */ this.joinGraph = compilers.joinGraph; - this.options = options || {}; this.orderHashToString = this.orderHashToString.bind(this); this.defaultOrder = this.defaultOrder.bind(this); @@ -4364,7 +4364,9 @@ export class BaseQuery { if (path.length === 3 && this.cubeEvaluator.isDimension(path.slice(0, 2))) { const dimensionDef = this.cubeEvaluator.dimensionByPath(path.slice(0, 2)); if (dimensionDef.type === 'time' && - this.cubeEvaluator.resolveGranularity([path[0], path[1], 'granularities', path[2]])) { + this.cubeEvaluator.resolveGranularity( + [path[0], path[1], 'granularities', path[2]] + )) { const td = this.newTimeDimension({ dimension: `${path[0]}.${path[1]}`, granularity: path[2], diff --git a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts index 4e7d452fd6d99..514fed014d400 100644 --- a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts +++ b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts @@ -39,9 +39,15 @@ export class Granularity { this.granularityInterval = `1 ${this.granularity}`; } else { const customGranularity = this.query.cacheValue( - ['customGranularity', timeDimension.dimension, this.granularity], + [ + 'customGranularity', + timeDimension.dimension, + this.granularity, + ], () => query.cubeEvaluator - .resolveGranularity([...query.cubeEvaluator.parsePath('dimensions', timeDimension.dimension), 'granularities', this.granularity]) + .resolveGranularity( + [...query.cubeEvaluator.parsePath('dimensions', timeDimension.dimension), 'granularities', this.granularity], + ) ); if (!customGranularity) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 4b2fa35409886..e8db87580316a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -20,6 +20,7 @@ import { UserError } from './UserError'; import { BaseQuery, PreAggregationDefinitionExtended } from '../adapter'; import type { CubeValidator } from './CubeValidator'; import type { ErrorReporter } from './ErrorReporter'; +import type { GlobalGranularitiesConfig } from './GlobalGranularitiesConfig'; import { FinishedJoinTree } from './JoinGraph'; export type SegmentDefinition = { @@ -210,13 +211,14 @@ export class CubeEvaluator extends CubeSymbols { private isRbacEnabledCache: boolean | null = null; public constructor( - protected readonly cubeValidator: CubeValidator + protected readonly cubeValidator: CubeValidator, + granularitiesResolver?: () => Promise, ) { - super(true); + super(true, granularitiesResolver); } - public compile(cubes: any[], errorReporter: ErrorReporter) { - super.compile(cubes, errorReporter); + public async compile(cubes: any[], errorReporter: ErrorReporter) { + await super.compile(cubes, errorReporter); const validCubes = this.cubeList.filter(cube => this.cubeValidator.isCubeValid(cube)).sort((a, b) => { if (a.isView) { return 1; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index e77f848b3fa94..db525f3017b2b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -5,6 +5,16 @@ import { camelize } from 'inflection'; import { UserError } from './UserError'; import { DynamicReference } from './DynamicReference'; import { camelizeCube } from './utils'; +import { + normalizeGranularitiesBlock, + NormalizedGranularitiesBlock, + resolveDimensionGranularities, +} from './GranularityResolver'; +import { + buildBuiltInsCatalog, + DEFAULT_GRANULARITIES_CONFIG, + GlobalGranularitiesConfig, +} from './GlobalGranularitiesConfig'; import type { ErrorReporter } from './ErrorReporter'; import { TranspilerSymbolResolver } from './transpilers'; @@ -16,6 +26,8 @@ export type GranularityDefinition = { sql?: (...args: any[]) => string; name?: string; title?: string; + /** d3-time-format string used by the client to display bucketed timestamps. */ + format?: string; interval?: string; offset?: string; origin?: string; @@ -33,6 +45,7 @@ export type CubeSymbolDefinition = { sql?: (...args: any[]) => string; primaryKey?: boolean; granularities?: Record; + granularitiesBlock?: NormalizedGranularitiesBlock; timeShift?: TimeshiftDefinition[]; format?: string; currency?: string; @@ -293,13 +306,30 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface private resolveSymbolsCallContext: any; - public constructor(evaluateViews = false) { + // Resolves the global granularities config; shared across the compilers of one prepareCompiler so + // a `config.granularities` function runs once per compile. + private readonly granularitiesResolver?: () => Promise; + + // Global granularities resolved at the start of compile(); the default catalog until then. + private globalGranularities: GlobalGranularitiesConfig = DEFAULT_GRANULARITIES_CONFIG; + + public constructor( + evaluateViews = false, + granularitiesResolver?: () => Promise, + ) { this.symbols = {}; this.builtCubes = {}; this.cubeDefinitions = {}; this.funcArgumentsValues = {}; this.cubeList = []; this.evaluateViews = evaluateViews; + this.granularitiesResolver = granularitiesResolver; + } + + // The resolved global config for this compile. Read by CubeToMetaTransformer (for + // `effectiveGranularities`) and by the /v1/granularities catalog. + public get globalGranularitiesConfig(): GlobalGranularitiesConfig { + return this.globalGranularities; } public free() { @@ -311,7 +341,14 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface this.resolveSymbolsCallContext = undefined; } - public compile(cubes: CubeDefinition[], errorReporter: ErrorReporter) { + public async compile(cubes: CubeDefinition[], errorReporter: ErrorReporter) { + // Resolve the global granularities config before any cube is transformed: the merge below writes + // global customs onto time dimensions, and the SQL layer resolves them from that same map. + // The phase driver chains each compile() through `.then()`, so awaiting here is safe. + if (this.granularitiesResolver) { + this.globalGranularities = await this.granularitiesResolver(); + } + this.cubeDefinitions = Object.fromEntries( cubes.map((c): [string, CubeDefinition] => [c.name, c]) ); @@ -581,6 +618,8 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface this.camelCaseTypes(cube.preAggregations); this.camelCaseTypes(cube.accessPolicy); + this.normalizeDimensionGranularities(cube.dimensions); + if (cube.preAggregations) { this.transformPreAggregations(cube.preAggregations); } @@ -603,6 +642,75 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface } as CubeSymbolsDefinition; } + // Stores the canonical `granularitiesBlock` on each time dimension, rewrites `granularities` to + // the locally-defined customs — preserving the legacy shape that BaseQuery, prepare-annotation + // and CubeToMetaTransformer read — then layers the effective global customs on top so they are + // resolvable in SQL and visible to pre-aggregation matching. + private normalizeDimensionGranularities(dimensions: Record | undefined) { + if (!dimensions) { + return; + } + + // Every time dimension participates: global customs apply even to one that declares no + // `granularities` of its own. + for (const dim of Object.values(dimensions)) { + if (dim && dim.type === 'time') { + // A view's included dimension already carries a propagated granularitiesBlock (with the + // source dimension's includes/excludes) alongside the custom-only `granularities` map. + // Re-normalizing the custom-only map here would reset includes to '*' and drop the source's + // includes/excludes, so only normalize dimensions that haven't been normalized yet. A + // dimension declaring no `granularities` needs no block — it takes the global config as-is, + // and attaching one would leak an empty block into the compiled model. + if (!dim.granularitiesBlock && 'granularities' in dim) { + // Keep the raw user value for the validator (it runs after this and would otherwise only + // see the extracted customs, never the includes/excludes/custom dict). + dim.rawGranularities = dim.granularities; + const block: NormalizedGranularitiesBlock = normalizeGranularitiesBlock(dim.granularities); + dim.granularitiesBlock = block; + dim.granularities = block.custom; + } + + this.mergeGlobalCustomsIntoDimension(dim); + } + } + } + + // Layer the dimension's effective global customs onto its `granularities` map (locals win on a + // name collision). Reassigns rather than mutating, so a view dimension sharing its source's map + // by reference isn't contaminated. + private mergeGlobalCustomsIntoDimension(dim: any) { + const globalCustom = this.globalGranularities.customGranularities; + if (Object.keys(globalCustom).length === 0) { + return; + } + + // `granularitiesBlock.custom` is the model's own customs, unaffected by the merge below. + const locals: Record = dim.granularitiesBlock?.custom ?? {}; + const resolved = resolveDimensionGranularities( + dim.granularitiesBlock ?? normalizeGranularitiesBlock(undefined), + this.globalGranularities.enabledBuiltIns, + globalCustom, + buildBuiltInsCatalog(this.globalGranularities), + ); + + // Only genuine global customs are baked in: built-ins resolve by name without a definition, and + // a local of the same name already wins. + const merged: Record = {}; + for (const [name, def] of Object.entries(resolved)) { + if (def.type === 'custom' && + Object.prototype.hasOwnProperty.call(globalCustom, name) && + !Object.prototype.hasOwnProperty.call(locals, name) + ) { + merged[name] = { ...def }; + delete (merged[name] as any).type; + } + } + + if (Object.keys(merged).length > 0) { + dim.granularities = { ...merged, ...locals }; + } + } + private camelCaseTypes(obj: Object | Array | undefined) { if (!obj) { return; @@ -1138,6 +1246,7 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface format: memberRef.override?.format || resolvedMember.format, ...(propagatedCurrency ? { currency: propagatedCurrency } : {}), ...(resolvedMember.granularities ? { granularities: resolvedMember.granularities } : {}), + ...(resolvedMember.granularitiesBlock ? { granularitiesBlock: resolvedMember.granularitiesBlock } : {}), ...(resolvedMember.multiStage && { multiStage: resolvedMember.multiStage }), ...(resolvedMember.keyReference && this.processKeyReferenceForView(resolvedMember.keyReference, targetCube.name, viewAllMembers, memberRef.member)), ...(resolvedMember.mask !== undefined ? { mask: resolvedMember.mask } : {}), @@ -1496,7 +1605,10 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface } if (refProperty && cube[refProperty].type === 'time' && - self.resolveGranularity([cubeName, refProperty, 'granularities', propertyName], cube) + self.resolveGranularity( + [cubeName, refProperty, 'granularities', propertyName], + cube + ) ) { return { toString: () => this.withSymbolsCallContext( @@ -1527,7 +1639,10 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface * @param {string|string[]} path * @param [refCube] Optional cube object to operate on */ - public resolveGranularity(path: string | string[], refCube?: any) { + public resolveGranularity( + path: string | string[], + refCube?: any, + ) { const [cubeName, dimName, gr, granName] = Array.isArray(path) ? path : path.split('.'); const cube = refCube || this.symbols[cubeName]; @@ -1545,7 +1660,11 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface return { interval: `1 ${granName}` }; } - return cube?.[dimName]?.[gr]?.[granName]; + // A local custom wins; otherwise fall back to a global custom from the resolved config. + return cube?.[dimName]?.[gr]?.[granName] ?? + (Object.prototype.hasOwnProperty.call(this.globalGranularities.customGranularities, granName) + ? this.globalGranularities.customGranularities[granName] + : undefined); } protected cubeDependenciesProxy(parentIndex, cubeName) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index da4eb611c221e..68becca595909 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -20,6 +20,18 @@ import type { JoinGraph } from './JoinGraph'; import type { ErrorReporter } from './ErrorReporter'; import { CompilerInterface } from './PrepareCompiler'; import { resolveNamedNumericFormat, STANDARD_FORMAT_SPECIFIERS, DEFAULT_FORMAT_SPECIFIER } from './named-numeric-formats'; +import { + EffectiveGranularity, + NormalizedGranularitiesBlock, + normalizeGranularitiesBlock, + resolveDimensionGranularities, + serializeEffectiveGranularities, +} from './GranularityResolver'; +import { + GlobalGranularitiesConfig, + DEFAULT_GRANULARITIES_CONFIG, + buildBuiltInsCatalog, +} from './GlobalGranularitiesConfig'; export type CustomNumericFormat = { type: 'custom-numeric'; value: string; alias?: string }; export type DimensionCustomTimeFormat = { type: 'custom-time'; value: string }; @@ -132,7 +144,13 @@ export type DimensionConfig = { public: boolean; primaryKey: boolean; aliasMember?: string; + /** + * @deprecated Use `effectiveGranularities`. Lists only the model's custom granularities; + * omits built-ins, global customs, and the `type` field. See DEPRECATION.md. + */ granularities?: GranularityDefinition[]; + /** Reconciled set for time dimensions: enabled built-ins + global customs + local customs. */ + effectiveGranularities?: EffectiveGranularity[]; order?: 'asc' | 'desc'; key?: string; links?: LinkConfig[]; @@ -197,12 +215,20 @@ export class CubeToMetaTransformer implements CompilerInterface { */ public queries: TransformedCube[]; + // Precomputed once in compile() so plain dimensions need no per-dimension resolution: `defaultSet` + // is shared by reference across every dimension without a local block. + private granularityState!: { + config: GlobalGranularitiesConfig; + catalog: Record; + defaultSet: EffectiveGranularity[]; + }; + public constructor( cubeValidator: CubeValidator, cubeEvaluator: CubeEvaluator, contextEvaluator: ContextEvaluator, viewGroupEvaluator: ViewGroupEvaluator, - joinGraph: JoinGraph + joinGraph: JoinGraph, ) { this.cubeValidator = cubeValidator; this.cubeSymbols = cubeEvaluator; @@ -219,6 +245,19 @@ export class CubeToMetaTransformer implements CompilerInterface { } public compile(_cubes: any[], errorReporter: ErrorReporter): void { + // CubeSymbols resolved the global config at the start of its own compile phase. + const config = this.cubeEvaluator.globalGranularitiesConfig ?? DEFAULT_GRANULARITIES_CONFIG; + const catalog = buildBuiltInsCatalog(config); + // Resolve the no-local-block ("default") set once; every plain time dimension shares the + // serialized wire form by reference (no per-dimension resolution). + this.granularityState = { + config, + catalog, + defaultSet: serializeEffectiveGranularities(resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), config.enabledBuiltIns, config.customGranularities, catalog, + )), + }; + this.cubes = this.cubeSymbols.cubeList .filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator)) .map((v) => this.transform(v, errorReporter.inContext(`${v.name} cube`))); @@ -301,11 +340,28 @@ export class CubeToMetaTransformer implements CompilerInterface { const dimensionVisibility = isCubeVisible ? this.isVisible(extendedDimDef, !extendedDimDef.primaryKey) : false; - const granularitiesObj = extendedDimDef.granularities; + const { granularitiesBlock } = extendedDimDef as any; + // The deprecated `granularities` meta field lists only the model's own custom + // granularities — `granularitiesBlock.custom`, which the global merge leaves untouched. + const localCustoms = granularitiesBlock?.custom ?? extendedDimDef.granularities; + const localCustomEntries = localCustoms ? Object.entries(localCustoms) : []; const dimType = this.dimensionDataType(extendedDimDef.type || 'string'); const dimFormat = this.transformDimensionFormat(extendedDimDef); const dimCurrency = extendedDimDef.currency?.toUpperCase(); + let effectiveGranularities: EffectiveGranularity[] | undefined; + if (dimType === 'time') { + const s = this.granularityState; + const inputs = this.granularityInputsForDimension(cubeTitle, localCustoms, granularitiesBlock); + // Dimensions with a local block resolve individually; plain ones reuse the shared + // default set computed once in compile(). + effectiveGranularities = inputs + ? serializeEffectiveGranularities(resolveDimensionGranularities( + inputs, s.config.enabledBuiltIns, s.config.customGranularities, s.catalog, + )) + : s.defaultSet; + } + return { name: `${cubeName}.${dimensionName}`, title: this.title(cubeTitle, nameToDimension, false), @@ -325,8 +381,8 @@ export class CubeToMetaTransformer implements CompilerInterface { primaryKey: !!extendedDimDef.primaryKey, aliasMember: extendedDimDef.aliasMember, granularities: - granularitiesObj - ? Object.entries(granularitiesObj).map(([gName, gDef]: [string, any]) => ({ + localCustomEntries.length > 0 + ? localCustomEntries.map(([gName, gDef]: [string, any]) => ({ name: gName, title: this.title(cubeTitle, [gName, gDef], true), interval: gDef.interval, @@ -334,6 +390,7 @@ export class CubeToMetaTransformer implements CompilerInterface { origin: gDef.origin, })) : undefined, + ...(effectiveGranularities ? { effectiveGranularities } : {}), order: extendedDimDef.order, key: extendedDimDef.keyReference, ...(extendedDimDef.links ? { links: extendedDimDef.links.map((link: any) => ({ @@ -375,6 +432,31 @@ export class CubeToMetaTransformer implements CompilerInterface { }; } + // Resolution inputs for one time dimension; null = no local customization, use the default set. + // Fields are projected from raw definitions so e.g. `sql` never leaks into meta output. + private granularityInputsForDimension( + cubeTitle: string, + granularitiesObj: Record | undefined, + granularitiesBlock: NormalizedGranularitiesBlock | undefined, + ): NormalizedGranularitiesBlock | null { + const hasLocalCustoms = granularitiesObj && Object.keys(granularitiesObj).length > 0; + if (!granularitiesBlock && !hasLocalCustoms) { + return null; + } + const block = granularitiesBlock || normalizeGranularitiesBlock(undefined); + const custom: Record = {}; + for (const [gName, gDef] of Object.entries({ ...block.custom, ...(granularitiesObj || {}) })) { + custom[gName] = { + title: this.title(cubeTitle, [gName, gDef], true), + interval: gDef.interval, + offset: gDef.offset, + origin: gDef.origin, + ...(gDef.format !== undefined ? { format: gDef.format } : {}), + }; + } + return { includes: block.includes, excludes: block.excludes, custom }; + } + public queriesForContext(contextId: string | null | undefined): TransformedCube[] { // return All queries if no context pass if (contextId == null || contextId.length === 0) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index a7452ea2bff33..93573fe396690 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -113,6 +113,103 @@ const GranularityInterval = Joi.string().pattern(/^\d+\s+(second|minute|hour|day // Do not allow negative intervals for granularities, while offsets could be negative const GranularityOffset = Joi.string().pattern(/^-?(\d+\s+)(second|minute|hour|day|week|month|quarter|year)s?(\s-?\d+\s+(second|minute|hour|day|week|month|quarter|year)s?){0,7}$/, 'granularity offset'); +// One custom granularity entry: with-origin, with-offset (interval must be aligned), or sql-defined. +// Reused by the legacy `granularities: { name: {...} }` form and the new `custom: { name: {...} }` form. +const CustomGranularityEntrySchema = Joi.alternatives([ + Joi.object().keys({ + title: Joi.string(), + format: Joi.string(), + interval: GranularityInterval.required(), + origin: Joi.string().required().custom((value, helpers) => { + const date = new Date(value); + + if (Number.isNaN(date.getTime())) { + return helpers.message({ custom: 'Origin should be valid date-only form: YYYY[-MM[-DD]] or date-time form: YYYY-MM-DD[T]HH:mm[:ss[.sss[Z]]]' }); + } + return value; + }), + }), + Joi.object().keys({ + title: Joi.string(), + format: Joi.string(), + interval: GranularityInterval.required().custom((value, helper) => { + const intParsed = value.split(' '); + const msg = { custom: 'Arbitrary intervals cannot be used without origin point specified' }; + + if (intParsed.length !== 2) { + return helper.message(msg); + } + + const v = parseInt(intParsed[0], 10); + const unit = intParsed[1]; + + const validIntervals = { + // Any number of years is valid + year: () => true, + // Only months divisible by a year with no remainder are valid + month: () => 12 % v === 0, + // Only quarters divisible by a year with no remainder are valid + quarter: () => 4 % v === 0, + // Only 1 week is valid + week: () => v === 1, + // Only 1 day is valid + day: () => v === 1, + // Only hours divisible by a day with no remainder are valid + hour: () => 24 % v === 0, + // Only minutes divisible by an hour with no remainder are valid + minute: () => 60 % v === 0, + // Only seconds divisible by a minute with no remainder are valid + second: () => 60 % v === 0, + }; + + const isValid = Object.keys(validIntervals).some(key => unit.includes(key) && validIntervals[key]()); + + return isValid ? value : helper.message(msg); + }), + offset: GranularityOffset.optional(), + }), + Joi.object().keys({ + title: Joi.string(), + format: Joi.string(), + sql: Joi.func().required() + }) +]); + +// `includes` / `excludes` accept a list of granularity names or the wildcard `'*'`. +const GranularityInclusionListSchema = Joi.alternatives([ + Joi.string().valid('*'), + Joi.array().items(Joi.string()), +]); + +// Validates a time dimension's granularities in either shape: the new dict form +// (includes/excludes/custom) or the legacy custom map. Applied to both `granularities` (the raw +// user value in unit tests / pre-normalization) and `rawGranularities` (the stashed raw value the +// real pipeline validates, since normalization replaces `granularities` with the custom map first). +const GranularitiesFieldSchema = Joi.alternatives() + .conditional(Joi.ref('.'), { + // Dict form iff keys are a subset of includes/excludes/custom AND their VALUES have the dict + // shape (includes/excludes are '*' or arrays, custom is a plain object). The value check + // mirrors normalizeGranularitiesBlock so a legacy custom granularity literally named + // `includes`/`excludes`/`custom` — whose value is a granularity DEFINITION object — is not + // misread as the dict form and rejected. + is: Joi.object().keys({ + includes: Joi.alternatives(Joi.string().valid('*'), Joi.array()), + excludes: Joi.alternatives(Joi.string().valid('*'), Joi.array()), + custom: Joi.object().pattern(Joi.string(), Joi.object()), + }).unknown(false), + then: Joi.object().keys({ + includes: GranularityInclusionListSchema, + excludes: GranularityInclusionListSchema, + custom: Joi.object().pattern(identifierRegex, CustomGranularityEntrySchema), + }).custom((value, helper) => { + if (value && value.includes !== undefined && value.excludes !== undefined && value.includes !== '*') { + return helper.message({ custom: '"includes" and "excludes" cannot be used together unless includes is "*"' } as any); + } + return value; + }), + otherwise: Joi.object().pattern(identifierRegex, CustomGranularityEntrySchema), + }); + const formatAlternatives = [ Joi.string().valid('imageUrl', 'link', 'currency', 'percent', 'number', 'id'), Joi.object().keys({ @@ -378,67 +475,21 @@ const BaseDimensionWithoutSubQuery = { then: Joi.array().items(Joi.string()), otherwise: Joi.forbidden() }), + // Validated in both shapes. In the real pipeline CubeSymbols.normalizeDimensionGranularities runs + // first and replaces `granularities` with the extracted custom map (still valid here) while + // stashing the user's raw value in `rawGranularities`; when validated directly (unit tests) only + // `granularities` is set and carries the raw shape. granularities: Joi.when('type', { is: 'time', - then: Joi.object().pattern(identifierRegex, - Joi.alternatives([ - Joi.object().keys({ - title: Joi.string(), - interval: GranularityInterval.required(), - origin: Joi.string().required().custom((value, helpers) => { - const date = new Date(value); - - if (Number.isNaN(date.getTime())) { - return helpers.message({ custom: 'Origin should be valid date-only form: YYYY[-MM[-DD]] or date-time form: YYYY-MM-DD[T]HH:mm[:ss[.sss[Z]]]' }); - } - return value; - }), - }), - Joi.object().keys({ - title: Joi.string(), - interval: GranularityInterval.required().custom((value, helper) => { - const intParsed = value.split(' '); - const msg = { custom: 'Arbitrary intervals cannot be used without origin point specified' }; - - if (intParsed.length !== 2) { - return helper.message(msg); - } - - const v = parseInt(intParsed[0], 10); - const unit = intParsed[1]; - - const validIntervals = { - // Any number of years is valid - year: () => true, - // Only months divisible by a year with no remainder are valid - month: () => 12 % v === 0, - // Only quarters divisible by a year with no remainder are valid - quarter: () => 4 % v === 0, - // Only 1 week is valid - week: () => v === 1, - // Only 1 day is valid - day: () => v === 1, - // Only hours divisible by a day with no remainder are valid - hour: () => 24 % v === 0, - // Only minutes divisible by an hour with no remainder are valid - minute: () => 60 % v === 0, - // Only seconds divisible by a minute with no remainder are valid - second: () => 60 % v === 0, - }; - - const isValid = Object.keys(validIntervals).some(key => unit.includes(key) && validIntervals[key]()); - - return isValid ? value : helper.message(msg); - }), - offset: GranularityOffset.optional(), - }), - Joi.object().keys({ - title: Joi.string(), - sql: Joi.func().required() - }) - ])).optional(), + then: GranularitiesFieldSchema.optional(), otherwise: Joi.forbidden() - }) + }), + rawGranularities: Joi.when('type', { + is: 'time', + then: GranularitiesFieldSchema.optional(), + otherwise: Joi.forbidden() + }), + granularitiesBlock: Joi.any() }; const BaseDimension = { diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index 3d95c5b1bcf56..47f18811b4054 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -224,7 +224,7 @@ export class DataSchemaCompiler { try { return compileServices .map((compileService) => (() => compileService.compile(objects, errorsReport))) - .reduce((p, fn) => p.then(fn), Promise.resolve()) + .reduce>((p, fn) => p.then(() => fn()).then(() => undefined), Promise.resolve()) .catch((error) => { errorsReport.error(error); }); diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts new file mode 100644 index 0000000000000..c4dd896933215 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -0,0 +1,252 @@ +import crypto from 'crypto'; +import { getEnv } from '@cubejs-backend/shared'; + +import type { GranularityDefinition } from './CubeSymbols'; +import { GRANULARITY_STRING_FIELDS } from './GranularityResolver'; + +// Default `title` and `format` for each built-in granularity. Overridable via `config.granularities` +// (file) or `CUBEJS_GRANULARITIES__TITLE` (env, title only). Format syntax: d3-time-format. + +export type BuiltInGranularityDefinition = { + title: string; + format: string; +}; + +export const BUILT_IN_GRANULARITIES: Readonly> = Object.freeze({ + year: { title: 'Year', format: '%Y' }, + quarter: { title: 'Quarter', format: 'Q%q %Y' }, + month: { title: 'Month', format: '%b %Y' }, + week: { title: 'Week', format: '%b %-d, %Y' }, + day: { title: 'Day', format: '%Y-%m-%d' }, + hour: { title: 'Hour', format: '%Y-%m-%d %H:00' }, + minute: { title: 'Minute', format: '%Y-%m-%d %H:%M' }, + second: { title: 'Second', format: '%Y-%m-%d %H:%M:%S' }, +}); + +export const BUILT_IN_GRANULARITY_NAMES = Object.freeze(Object.keys(BUILT_IN_GRANULARITIES)); + +export function isBuiltInGranularity(name: string): boolean { + // hasOwnProperty, not `in`: `'__proto__' in {}` / `'constructor' in {}` are true via the + // prototype chain, which would misclassify those names as built-in granularities. + return Object.prototype.hasOwnProperty.call(BUILT_IN_GRANULARITIES, name); +} + +// Item shape accepted in `config.granularities`: a built-in name, or a custom granularity object. +export type GranularityListItem = string | (GranularityDefinition & { name: string }); +export type GranularityList = GranularityListItem[]; + +export type GlobalGranularitiesConfig = { + enabledBuiltIns: ReadonlyArray; + customGranularities: Readonly>; +}; + +export const DEFAULT_GRANULARITIES_CONFIG: GlobalGranularitiesConfig = Object.freeze({ + enabledBuiltIns: BUILT_IN_GRANULARITY_NAMES, + customGranularities: Object.freeze({}), +}); + +// Read `CUBEJS_GRANULARITIES__{INTERVAL,TITLE,OFFSET,ORIGIN}` for a custom granularity name. +// Only consulted for names sourced from `CUBEJS_GRANULARITIES`, not for `config.granularities` entries. +function applyEnvOverrides(name: string, base?: Partial): GranularityDefinition { + // getEnv types `opts` as a Parameters<> tuple but forwards it positionally; cast to bypass that. + const opts = { name } as any; + // A set-but-empty env var (e.g. `CUBEJS_GRANULARITIES_FOO_INTERVAL=`) reads as '' — treat that + // as absent so an unusable empty interval isn't advertised and later fails at query time. + const nonEmpty = (v: string | undefined) => (v === undefined || v === '' ? undefined : v); + const interval = nonEmpty(getEnv('granularityCustomInterval', opts)) ?? base?.interval; + const title = nonEmpty(getEnv('granularityCustomTitle', opts)) ?? base?.title; + const offset = nonEmpty(getEnv('granularityCustomOffset', opts)) ?? base?.offset; + const origin = nonEmpty(getEnv('granularityCustomOrigin', opts)) ?? base?.origin; + + const out: GranularityDefinition = {}; + if (interval !== undefined) out.interval = interval; + if (title !== undefined) out.title = title; + if (offset !== undefined) out.offset = offset; + if (origin !== undefined) out.origin = origin; + return out; +} + +function resolveFromEnv(): GlobalGranularitiesConfig { + const list = getEnv('granularities'); + if (!list || list.length === 0) { + return DEFAULT_GRANULARITIES_CONFIG; + } + + const enabledBuiltIns: string[] = []; + const customGranularities: Record = {}; + for (const name of list) { + const trimmed = name.trim(); + if (trimmed) { + if (isBuiltInGranularity(trimmed)) { + enabledBuiltIns.push(trimmed); + // A built-in may still carry per-name env overrides (title/format via _TITLE etc.); fold + // them into customGranularities so buildBuiltInsCatalog picks them up (built-in stays built-in). + const override = applyEnvOverrides(trimmed); + if (Object.keys(override).length > 0) { + customGranularities[trimmed] = override; + } + } else { + // Non-built-in name: pull the definition from `CUBEJS_GRANULARITIES__*` env vars. + // Skip if no interval was provided — a custom granularity without an interval is unusable. + const def = applyEnvOverrides(trimmed); + if (def.interval !== undefined) { + customGranularities[trimmed] = def; + } + } + } + } + return { enabledBuiltIns, customGranularities }; +} + +// Project a definition onto exactly the known string fields, dropping anything else (functions, +// nested objects, numbers). Guarantees every value the serializer emits and the hash reads is a +// plain string, so the two can never diverge (no cross-tenant cache-key collision), and stray +// props a config spreads in can't leak onto the wire. +function sanitizeDefinition(def: Partial): GranularityDefinition { + const out: GranularityDefinition = {}; + for (const key of GRANULARITY_STRING_FIELDS) { + if (typeof def[key] === 'string') { + out[key] = def[key]; + } + } + return out; +} + +function resolveFromList(list: GranularityList): GlobalGranularitiesConfig { + const enabledBuiltIns: string[] = []; + const customGranularities: Record = {}; + + for (const item of list) { + if (typeof item === 'string') { + if (isBuiltInGranularity(item)) { + enabledBuiltIns.push(item); + } + // A bare non-built-in string has no definition attached and is silently dropped: + // custom granularities in `config.granularities` must be objects. + } else if (item && typeof item === 'object' && item.name) { + const { name, ...def } = item; + const clean = sanitizeDefinition(def); + if (isBuiltInGranularity(name)) { + // `{ name: 'year', title: 'Anno' }` both enables 'year' and overrides its title/format. + enabledBuiltIns.push(name); + customGranularities[name] = clean; + } else if (clean.interval !== undefined) { + // A custom granularity without an interval is unusable (SQL can't bucket it) and must not + // be advertised — drop it rather than exposing a granularity that fails at query time. + customGranularities[name] = clean; + } + } + } + return { enabledBuiltIns, customGranularities }; +} + +// Config forms accepted for `granularities` in cube.js / cube.py. +export type GranularitiesOption = + GranularityList | ((ctx: any) => GranularityList | Promise) | undefined; + +// Sync resolution for the context-independent forms (undefined -> env, static list). +// The function form is per-context and must go through `resolveGlobalGranularities`. +export function resolveGlobalGranularitiesSync( + userValue: Exclude, +): GlobalGranularitiesConfig { + if (!Array.isArray(userValue)) { + return resolveFromEnv(); + } + return resolveFromList(userValue); +} + +// `userValue` is the value of `granularities` from the cube.js / cube.py config file. +// undefined -> fall back to `CUBEJS_GRANULARITIES` env vars +// GranularityList -> use this list, replacing env vars entirely (no merge) +// function(ctx) -> called per request; same no-merge replacement as the list form +export async function resolveGlobalGranularities( + userValue: GranularitiesOption, + ctx: any, +): Promise { + if (typeof userValue === 'function') { + const resolved = await userValue(ctx); + // A function opts out of env vars entirely, so a non-array return (null / undefined / a stray + // object) means "no explicit config" → the default built-in catalog, NOT an env fallback that + // would leak CUBEJS_GRANULARITIES into a context the function meant to leave unconfigured. + return Array.isArray(resolved) ? resolveFromList(resolved) : DEFAULT_GRANULARITIES_CONFIG; + } + return resolveGlobalGranularitiesSync(userValue); +} + +export function getBuiltInGranularityDefaults(name: string) { + return BUILT_IN_GRANULARITIES[name]; +} + +// Resolve title/format/interval for each enabled built-in, applying any override from +// `globalConfig.customGranularities` (e.g. `{ name: 'year', title: 'Jaar' }` localizes the title). +// `interval` is filled in as "1 " so built-ins share the same response shape as customs. +export type BuiltInCatalogEntry = { + title: string; + format: string; + interval: string; +}; + +export function buildBuiltInsCatalog(globalConfig: GlobalGranularitiesConfig): Record { + const catalog: Record = {}; + for (const name of globalConfig.enabledBuiltIns) { + const defaults = BUILT_IN_GRANULARITIES[name]; + if (defaults) { + const override = globalConfig.customGranularities[name]; + catalog[name] = { + title: override?.title || defaults.title, + format: override?.format || defaults.format, + // Built-in interval is fixed at `1 ` — the SQL layer always buckets predefined + // granularities that way, so an override interval must NOT be advertised (it would lie + // about how the data is bucketed). Title/format overrides are display-only and safe. + interval: `1 ${name}`, + }; + } + } + return catalog; +} + +// Wire shape of the `/v1/granularities` catalog: every built-in enabled for this model, plus the +// global customs. A custom that shadows a built-in is already folded into the built-in entry. +export function buildGranularitiesCatalog(config: GlobalGranularitiesConfig): Array> { + const catalog: Array> = Object.entries(buildBuiltInsCatalog(config)) + .map(([name, entry]) => ({ type: 'built-in', name, ...entry })); + + for (const [name, def] of Object.entries(config.customGranularities)) { + if (!isBuiltInGranularity(name)) { + catalog.push({ + type: 'custom', + name, + title: def.title || name, + ...Object.fromEntries( + GRANULARITY_STRING_FIELDS + .filter((f) => f !== 'title' && def[f] !== undefined) + .map((f) => [f, def[f] as string]), + ), + }); + } + } + return catalog; +} + +// Fields that a config override actually changes in the emitted output, per granularity name. For +// a name shadowing a built-in, buildBuiltInsCatalog honors only title/format (interval/offset/origin +// are fixed at `1 ` for predefined granularities), so hashing the ignored fields would churn +// the cache/compilerId on changes that produce identical output. A real custom uses every field. +const hashableFields = (name: string): ReadonlyArray => ( + isBuiltInGranularity(name) ? ['title', 'format'] : GRANULARITY_STRING_FIELDS +); + +// Canonical sha256 of a resolved config: equal hash iff identical effective sets. Order-sensitive +// on purpose — built-in and custom order affect the emitted meta. Used as the meta-variant cache +// key and compilerId discriminator. Only the fields that actually affect output participate (see +// hashableFields), so the hash never disagrees with the wire output nor churns on ignored fields. +export function granularityConfigHash(config: GlobalGranularitiesConfig): string { + const canonical = { + builtIns: [...config.enabledBuiltIns], + custom: Object.entries(config.customGranularities).map(([name, def]) => ({ + name, + ...Object.fromEntries(hashableFields(name).map((f) => [f, def[f]])), + })), + }; + return crypto.createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); +} diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts new file mode 100644 index 0000000000000..f81a0aa939397 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -0,0 +1,179 @@ +import type { GranularityDefinition } from './CubeSymbols'; + +// `'*'` is the wildcard form of an includes/excludes list. +export type GranularityInclusionList = '*' | string[]; + +// Canonical form produced by `normalizeGranularitiesBlock`. All downstream readers +// (resolver, meta transformer, pre-agg matcher) only see this shape. +export type NormalizedGranularitiesBlock = { + includes: GranularityInclusionList; + excludes: GranularityInclusionList; + custom: Record; +}; + +// Effective granularity set for one time dimension, ready to serialize into /v1/meta. +export type ResolvedGranularitySet = Record; + +// Used when a time dimension has no `granularities` block at all: take all enabled globals, no local customs. +const EMPTY_BLOCK: NormalizedGranularitiesBlock = { + includes: '*', + excludes: [], + custom: {}, +}; + +// Map any accepted input shape (omitted, legacy array, post-yaml keyed object, new dict) +// onto NormalizedGranularitiesBlock. Validator runs first, so malformed input never reaches here. +export function normalizeGranularitiesBlock(raw: any): NormalizedGranularitiesBlock { + if (raw == null) { + return EMPTY_BLOCK; + } + + if (Array.isArray(raw)) { + // Legacy form. Every array entry is a custom granularity; built-ins are inherited from globals. + return { + includes: '*', + excludes: [], + custom: Object.fromEntries(raw.filter(g => g && g.name).map(g => { + const { name, ...rest } = g; + return [name, rest]; + })), + }; + } + + if (typeof raw === 'object') { + // New dict form iff every key is one of includes/excludes/custom AND the values have the dict + // shape. The value checks keep a legacy custom granularity NAMED `includes`/`excludes`/`custom` + // from being misread as the dict form: + // - includes/excludes must be '*' or an array (a legacy custom named `includes` has an object + // definition value, which fails this); + // - `custom` must be a map of name -> definition OBJECT. A legacy custom named `custom` has a + // definition whose own values are strings/functions (interval:'1 year', sql:()=>...), never + // nested objects — so requiring every `custom` value to be an object distinguishes the dict + // form ({custom:{fy:{...}}}) from the legacy custom-named-`custom` ({custom:{interval:...}}). + const keys = Object.keys(raw); + const isInclusionList = (v: any) => v === undefined || v === '*' || Array.isArray(v); + const isDefinitionObject = (v: any) => typeof v === 'object' && v !== null && !Array.isArray(v); + const isCustomMap = (v: any) => v === undefined || + (isDefinitionObject(v) && Object.values(v).every(isDefinitionObject)); + const isDictForm = + keys.length > 0 && + keys.every(k => k === 'includes' || k === 'excludes' || k === 'custom') && + isInclusionList(raw.includes) && + isInclusionList(raw.excludes) && + isCustomMap(raw.custom); + if (isDictForm) { + // YamlCompiler has already keyed `custom` by name. + return { + includes: raw.includes ?? '*', + excludes: raw.excludes ?? [], + custom: raw.custom ?? {}, + }; + } + // Already-keyed legacy form (e.g. coming from JS configs, not YAML). Treat as custom-only. + return { + includes: '*', + excludes: [], + custom: raw, + }; + } + + return EMPTY_BLOCK; +} + +// The string-valued fields of a granularity definition, in wire-emission order. Single source of +// truth for every place that projects a definition (serialize, hash, sanitize, meta) so those +// projections can't drift — notably the hash and the serializer, which MUST agree or two tenant +// configs could collide on one variant-cache key. +export const GRANULARITY_STRING_FIELDS = ['title', 'interval', 'offset', 'origin', 'format'] as const; + +// Wire shape of one entry in a time dimension's `effectiveGranularities`. +export type EffectiveGranularity = { + name: string; + type: 'built-in' | 'custom'; + title: string; + interval?: string; + offset?: string; + origin?: string; + format?: string; +}; + +// Serialize a resolved set for /v1/meta. `title` always present (falls back to the name); the +// other string fields are included only when defined. Order follows GRANULARITY_STRING_FIELDS, +// which also drives the config hash — the two share the field list so they can't drift. +export function serializeEffectiveGranularities(resolved: ResolvedGranularitySet): EffectiveGranularity[] { + return Object.entries(resolved).map(([name, def]) => { + const out: EffectiveGranularity = { name, type: def.type, title: def.title || name }; + for (const field of GRANULARITY_STRING_FIELDS) { + if (field !== 'title' && def[field] !== undefined) { + out[field] = def[field]; + } + } + return out; + }); +} + +// Reconcile a dimension's local block against the global enabled built-ins and global customs, +// producing the effective set. Local customs always survive — even if local excludes is '*'. +export function resolveDimensionGranularities( + localBlock: NormalizedGranularitiesBlock, + globalEnabledBuiltIns: ReadonlyArray, + globalCustom: Readonly>, + allBuiltInsCatalog: Readonly>, +): ResolvedGranularitySet { + const out: ResolvedGranularitySet = {}; + + const localIncludes = localBlock.includes; + const localExcludes = localBlock.excludes; + + const includesAllowsAll = localIncludes === '*'; + const includesSet = includesAllowsAll ? null : new Set(localIncludes); + const excludesBlocksAll = localExcludes === '*'; + const excludesSet = excludesBlocksAll ? null : new Set(localExcludes); + + // Built-ins and global customs are filtered the same way: keep iff included AND not excluded. + if (!excludesBlocksAll) { + for (const builtInName of globalEnabledBuiltIns) { + const passesIncludes = includesAllowsAll || includesSet!.has(builtInName); + const blockedByExcludes = excludesSet!.has(builtInName); + const def = allBuiltInsCatalog[builtInName]; + if (passesIncludes && !blockedByExcludes && def) { + out[builtInName] = { ...def, type: 'built-in' }; + } + } + for (const [name, def] of Object.entries(globalCustom)) { + // A name shadowing a built-in is an override, already emitted as `type: 'built-in'` above with + // its title/format folded in via `allBuiltInsCatalog`; skip it here so it isn't relabeled custom. + const shadowsBuiltIn = !!allBuiltInsCatalog[name]; + const passesIncludes = includesAllowsAll || includesSet!.has(name); + const blockedByExcludes = excludesSet!.has(name); + if (!shadowsBuiltIn && passesIncludes && !blockedByExcludes) { + out[name] = { ...def, type: 'custom' }; + } + } + } + + // Local customs are always emitted, even when `excludes: '*'` strips everything else. + // Same-named local entry replaces a global one (last-write-wins). + for (const [name, def] of Object.entries(localBlock.custom)) { + out[name] = { ...def, type: 'custom' }; + } + + return out; +} + +// One-shot: reconcile a dimension's block against the global config and serialize to the wire set. +// The single seam every path uses to turn a (block, config) into `effectiveGranularities`, so the +// resolve→serialize contract lives in exactly one place. A missing block means "no local block". +export function effectiveGranularitiesFor( + block: NormalizedGranularitiesBlock | undefined, + globalEnabledBuiltIns: ReadonlyArray, + globalCustom: Readonly>, + allBuiltInsCatalog: Readonly>, +): EffectiveGranularity[] { + return serializeEffectiveGranularities(resolveDimensionGranularities( + block ?? normalizeGranularitiesBlock(undefined), + globalEnabledBuiltIns, + globalCustom, + allBuiltInsCatalog, + )); +} diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index e36e6cb02589b..22db4453c7ed5 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -26,6 +26,11 @@ import { CompilerCache } from './CompilerCache'; import { YamlCompiler } from './YamlCompiler'; import { ViewCompilationGate } from './ViewCompilationGate'; import type { ErrorReporter } from './ErrorReporter'; +import { + type GlobalGranularitiesConfig, + type GranularitiesOption, + resolveGlobalGranularities, +} from './GlobalGranularitiesConfig'; export type PrepareCompilerOptions = { nativeInstance?: NativeInstance, @@ -40,10 +45,19 @@ export type PrepareCompilerOptions = { compiledScriptCache?: LRUCache; compiledYamlCache?: LRUCache; compiledJinjaCache?: LRUCache; + // `config.granularities` as authored (env / static list / function). Resolved once at the start + // of compile and applied by CubeSymbols. + granularities?: GranularitiesOption; + // An already-resolved config, when the caller had to resolve it anyway (CompilerApi folds its + // hash into compilerVersion). Takes precedence over `granularities`, so a function form is + // invoked once per compile rather than once here and once there. + resolvedGranularities?: GlobalGranularitiesConfig; }; export interface CompilerInterface { - compile: (cubes: any[], errorReporter: ErrorReporter) => void; + // The phase driver chains each compile() through `.then()`, so a compiler may return a promise; + // any other return value is ignored. + compile: (cubes: any[], errorReporter: ErrorReporter) => unknown; } export type Compiler = { @@ -61,11 +75,23 @@ export type Compiler = { export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}): Compiler => { const nativeInstance = options.nativeInstance || new NativeInstance(); const cubeDictionary = new CubeDictionary(); - const cubeSymbols = new CubeSymbols(); - const viewCompiler = new CubeSymbols(true); + // One resolve shared by every compiler below, so a `config.granularities` function is invoked + // once per compile rather than once per compiler instance. + let granularitiesPromise: Promise | undefined; + const resolveGranularities = () => { + if (!granularitiesPromise) { + granularitiesPromise = options.resolvedGranularities + ? Promise.resolve(options.resolvedGranularities) + : resolveGlobalGranularities(options.granularities, options.compileContext); + } + return granularitiesPromise; + }; + + const cubeSymbols = new CubeSymbols(false, resolveGranularities); + const viewCompiler = new CubeSymbols(true, resolveGranularities); const viewCompilationGate = new ViewCompilationGate(); const cubeValidator = new CubeValidator(cubeSymbols); - const cubeEvaluator = new CubeEvaluator(cubeValidator); + const cubeEvaluator = new CubeEvaluator(cubeValidator, resolveGranularities); const contextEvaluator = new ContextEvaluator(cubeEvaluator); const viewGroupEvaluator = new ViewGroupEvaluator(cubeEvaluator, cubeValidator); const joinGraph = new JoinGraph(cubeValidator, cubeEvaluator); diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index 42e20a9f4cb63..770cb779668c3 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -450,10 +450,22 @@ export class YamlCompiler { } if (memberType === 'dimension' && granularities) { - granularities = this.yamlArrayToObj(granularities || [], 'dimension.granularity', errorsReport, { - cubeName: ctx.cubeName, - parent: { type: 'time dimension', name } - }); + if (Array.isArray(granularities)) { + // Legacy form: `granularities: [{ name, ... }]` -> `{ name: { ... } }`. + granularities = this.yamlArrayToObj(granularities || [], 'dimension.granularity', errorsReport, { + cubeName: ctx.cubeName, + parent: { type: 'time dimension', name } + }); + } else if (granularities && typeof granularities === 'object' && Array.isArray(granularities.custom)) { + // New dict form: only the inner `custom` array needs keying; includes/excludes pass through. + granularities = { + ...granularities, + custom: this.yamlArrayToObj(granularities.custom, 'dimension.granularity', errorsReport, { + cubeName: ctx.cubeName, + parent: { type: 'time dimension', name } + }), + }; + } res[name] = { granularities, ...res[name] }; } diff --git a/packages/cubejs-schema-compiler/src/compiler/index.ts b/packages/cubejs-schema-compiler/src/compiler/index.ts index ff62cfaa59ddc..4261468acfa47 100644 --- a/packages/cubejs-schema-compiler/src/compiler/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/index.ts @@ -11,3 +11,30 @@ export { PreAggregationInfo, EvaluatedCube, } from './CubeEvaluator'; +export { + BUILT_IN_GRANULARITIES, + BUILT_IN_GRANULARITY_NAMES, + isBuiltInGranularity, + BuiltInGranularityDefinition, + GranularityList, + GranularityListItem, + GranularitiesOption, + GlobalGranularitiesConfig, + BuiltInCatalogEntry, + resolveGlobalGranularities, + resolveGlobalGranularitiesSync, + getBuiltInGranularityDefaults, + buildBuiltInsCatalog, + buildGranularitiesCatalog, + granularityConfigHash, +} from './GlobalGranularitiesConfig'; +export { + NormalizedGranularitiesBlock, + ResolvedGranularitySet, + EffectiveGranularity, + GRANULARITY_STRING_FIELDS, + normalizeGranularitiesBlock, + resolveDimensionGranularities, + serializeEffectiveGranularities, + effectiveGranularitiesFor, +} from './GranularityResolver'; diff --git a/packages/cubejs-schema-compiler/src/compiler/utils.ts b/packages/cubejs-schema-compiler/src/compiler/utils.ts index f917e6a93be06..d71892befd12f 100644 --- a/packages/cubejs-schema-compiler/src/compiler/utils.ts +++ b/packages/cubejs-schema-compiler/src/compiler/utils.ts @@ -1,25 +1,40 @@ import { camelize } from 'inflection'; -// It's a map where key - is a level and value - is a map of properties on this level to ignore camelization -const IGNORE_CAMELIZE = { +// Map of level -> keys at that level whose children must not be camelized (they hold user-defined +// identifiers). Each entry can require a specific parent key so the guard is path-scoped rather than +// matching any same-named property elsewhere in the tree. +const IGNORE_CAMELIZE: Record> = { 1: { - granularities: true, + granularities: {}, + }, + // Custom granularity names in the new dict form live at `granularities.custom.`; scope the + // guard to that path so an unrelated `custom` property elsewhere isn't affected. + 2: { + custom: { parent: 'granularities' }, } }; -function camelizeObjectPart(obj: unknown, camelizeKeys: boolean, level = 0): unknown { +function shouldIgnoreCamelize(level: number, key: string, parentKey: string | undefined): boolean { + const entry = IGNORE_CAMELIZE[level]?.[key]; + if (!entry) { + return false; + } + return entry.parent === undefined || entry.parent === parentKey; +} + +function camelizeObjectPart(obj: unknown, camelizeKeys: boolean, level = 0, parentKey?: string): unknown { if (!obj) { return obj; } if (Array.isArray(obj)) { for (let i = 0; i < obj.length; i++) { - obj[i] = camelizeObjectPart(obj[i], true, level + 1); + obj[i] = camelizeObjectPart(obj[i], true, level + 1, parentKey); } } else if (typeof obj === 'object') { for (const key of Object.keys(obj)) { if (!(level === 1 && key === 'meta')) { - obj[key] = camelizeObjectPart(obj[key], !IGNORE_CAMELIZE[level]?.[key], level + 1); + obj[key] = camelizeObjectPart(obj[key], !shouldIgnoreCamelize(level, key, parentKey), level + 1, key); } if (camelizeKeys) { diff --git a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap index 1e48caa41d7f3..3c3cff455d445 100644 --- a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap +++ b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap @@ -78,7 +78,39 @@ Object { "sql": [Function], }, }, + "granularitiesBlock": Object { + "custom": Object { + "month": Object { + "sql": [Function], + }, + "quarter": Object { + "sql": [Function], + }, + "week": Object { + "sql": [Function], + }, + "year": Object { + "sql": [Function], + }, + }, + "excludes": Array [], + "includes": "*", + }, "ownedByCube": true, + "rawGranularities": Object { + "month": Object { + "sql": [Function], + }, + "quarter": Object { + "sql": [Function], + }, + "week": Object { + "sql": [Function], + }, + "year": Object { + "sql": [Function], + }, + }, "sql": [Function], "timeShift": Array [ Object { @@ -263,7 +295,47 @@ Object { "sql": [Function], }, }, + "granularitiesBlock": Object { + "custom": Object { + "fortnight": Object { + "interval": "2 week", + "origin": "2025-01-01", + }, + "month": Object { + "sql": [Function], + }, + "quarter": Object { + "sql": [Function], + }, + "week": Object { + "sql": [Function], + }, + "year": Object { + "sql": [Function], + }, + }, + "excludes": Array [], + "includes": "*", + }, "ownedByCube": true, + "rawGranularities": Object { + "fortnight": Object { + "interval": "2 week", + "origin": "2025-01-01", + }, + "month": Object { + "sql": [Function], + }, + "quarter": Object { + "sql": [Function], + }, + "week": Object { + "sql": [Function], + }, + "year": Object { + "sql": [Function], + }, + }, "sql": [Function], "timeShift": Array [ Object { @@ -1922,6 +1994,29 @@ Object { "origin": "2020-03-01", }, }, + "granularitiesBlock": Object { + "custom": Object { + "half_year": Object { + "interval": "6 months", + "title": "6 month intervals", + }, + "half_year_by_1st_april": Object { + "interval": "6 months", + "offset": "3 months", + "title": "Half year from Apr to Oct", + }, + "half_year_by_1st_june": Object { + "interval": "6 months", + "origin": "2020-06-01 10:00:00", + }, + "half_year_by_1st_march": Object { + "interval": "6 months", + "origin": "2020-03-01", + }, + }, + "excludes": Array [], + "includes": "*", + }, "meta": undefined, "ownedByCube": false, "sql": [Function], diff --git a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts index 3b6bd2f32bd86..222f82ac24f52 100644 --- a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts @@ -1716,6 +1716,103 @@ describe('Cube Validation', () => { }); }); + describe('Granularities dict shape (includes/excludes/custom):', () => { + const newCube = (granularities: any) => ({ + name: 'Orders', + fileName: 'fileName', + sql: () => 'select * from tbl', + public: true, + dimensions: { + createdAt: { + public: true, + sql: () => 'created_at', + type: 'time', + granularities, + }, + }, + measures: { + count: { sql: () => 'count', type: 'count' }, + }, + }); + + it('accepts includes-only with built-in names', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: ['year', 'quarter'] }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts excludes-only with built-in names', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ excludes: ['day'] }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts includes "*" wildcard', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: '*' }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts custom-only', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ + custom: { fiscal_year: { interval: '1 year', origin: '2026-04-01' } }, + }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts includes "*" combined with excludes', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: '*', excludes: ['day'] }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts excludes "*" combined with custom', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ + excludes: '*', + custom: { fiscal_year: { interval: '1 year', origin: '2026-04-01' } }, + }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('rejects includes + excludes both as lists (mutually exclusive)', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: ['year'], excludes: ['day'] }); + let captured: string | undefined; + const validationResult = cubeValidator.validate(cube, { + error: (message: any) => { captured = message; }, + } as any); + expect(validationResult.error).toBeTruthy(); + expect(captured).toContain('"includes" and "excludes" cannot be used together'); + }); + + it('legacy flat-array form (post yamlArrayToObj) still validates', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ + fiscal_year: { interval: '1 year', origin: '2026-04-01' }, + }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + // Regression: a legacy custom granularity literally named includes/excludes/custom (value is a + // definition object) must be accepted as the legacy map, not misread as the dict form. Mirrors + // the value-shape disambiguation in normalizeGranularitiesBlock. + it.each(['includes', 'excludes', 'custom'])('accepts a legacy custom granularity named "%s"', (name) => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ [name]: { interval: '1 year', origin: '2026-04-01' } }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + }); + describe('Access Policy group/groups support:', () => { const cubeValidator = new CubeValidator(new CubeSymbols()); diff --git a/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts b/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts new file mode 100644 index 0000000000000..08a08b2ff4206 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts @@ -0,0 +1,143 @@ +import { + resolveGlobalGranularities, + buildBuiltInsCatalog, + granularityConfigHash, + isBuiltInGranularity, + BUILT_IN_GRANULARITY_NAMES, +} from '../../src/compiler/GlobalGranularitiesConfig'; + +describe('resolveGlobalGranularities', () => { + const originalEnv = { ...process.env }; + + afterEach(() => { + for (const key of Object.keys(process.env)) { + if (key.startsWith('CUBEJS_GRANULARITIES')) delete process.env[key]; + } + Object.assign(process.env, originalEnv); + }); + + it('user value undefined + no env -> defaults to full built-in catalog', async () => { + const cfg = await resolveGlobalGranularities(undefined, {}); + expect([...cfg.enabledBuiltIns].sort()).toEqual([...BUILT_IN_GRANULARITY_NAMES].sort()); + expect(cfg.customGranularities).toEqual({}); + }); + + it('CUBEJS_GRANULARITIES restricts enabled built-ins', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,quarter,month'; + const cfg = await resolveGlobalGranularities(undefined, {}); + expect(cfg.enabledBuiltIns).toEqual(['year', 'quarter', 'month']); + expect(cfg.customGranularities).toEqual({}); + }); + + it('CUBEJS_GRANULARITIES with a custom name + companion env vars produces a custom granularity', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,fiscal_year'; + process.env.CUBEJS_GRANULARITIES_FISCAL_YEAR_INTERVAL = '1 year'; + process.env.CUBEJS_GRANULARITIES_FISCAL_YEAR_ORIGIN = '2026-04-01'; + process.env.CUBEJS_GRANULARITIES_FISCAL_YEAR_TITLE = 'Fiscal Year'; + const cfg = await resolveGlobalGranularities(undefined, {}); + expect(cfg.enabledBuiltIns).toEqual(['year']); + expect(cfg.customGranularities.fiscal_year).toEqual({ + interval: '1 year', + origin: '2026-04-01', + title: 'Fiscal Year', + }); + }); + + it('file-config replaces env vars (full replacement, not merge)', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,quarter,month'; + const cfg = await resolveGlobalGranularities(['day', 'hour'], {}); + expect(cfg.enabledBuiltIns).toEqual(['day', 'hour']); + }); + + it('file-config can mix built-in names + custom objects', async () => { + const cfg = await resolveGlobalGranularities( + ['year', { name: 'fiscal_year', interval: '1 year', origin: '2026-04-01' }], + {}, + ); + expect(cfg.enabledBuiltIns).toEqual(['year']); + expect(cfg.customGranularities.fiscal_year).toEqual({ interval: '1 year', origin: '2026-04-01' }); + }); + + it('file-config function is invoked with the request context', async () => { + let receivedCtx: any; + const cfg = await resolveGlobalGranularities( + (ctx) => { + receivedCtx = ctx; + return ['year']; + }, + { securityContext: { tenant: 't1' } }, + ); + expect(receivedCtx.securityContext.tenant).toBe('t1'); + expect(cfg.enabledBuiltIns).toEqual(['year']); + }); + + it('file-config function may return a Promise', async () => { + const cfg = await resolveGlobalGranularities( + async () => ['quarter'], + {}, + ); + expect(cfg.enabledBuiltIns).toEqual(['quarter']); + }); + + // Regression: CUBEJS_GRANULARITIES__TITLE was dropped for built-in names. + it('env title/format override applies to a built-in (folded into catalog, stays built-in)', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,month'; + process.env.CUBEJS_GRANULARITIES_YEAR_TITLE = 'Jaar'; + const cfg = await resolveGlobalGranularities(undefined, {}); + expect(cfg.enabledBuiltIns).toEqual(['year', 'month']); + const catalog = buildBuiltInsCatalog(cfg); + expect(catalog.year.title).toBe('Jaar'); + expect(catalog.year.interval).toBe('1 year'); + }); + + // Regression: a config-provided interval must not override a built-in's fixed 1-unit interval. + it('interval override on a built-in is ignored (SQL always buckets 1 unit)', async () => { + const cfg = await resolveGlobalGranularities([{ name: 'month', interval: '2 months', title: 'Bi' }], {}); + const catalog = buildBuiltInsCatalog(cfg); + expect(catalog.month.interval).toBe('1 month'); + expect(catalog.month.title).toBe('Bi'); + }); + + // Regression: a config custom without an interval is unusable and must be dropped, not advertised. + it('drops a config custom granularity that has no interval', async () => { + const cfg = await resolveGlobalGranularities([{ name: 'fiscal_year', title: 'Fiscal Year' }], {}); + expect(cfg.customGranularities.fiscal_year).toBeUndefined(); + }); + + // Regression: non-string definition values must not survive (hash/serialize must agree). + it('sanitizes non-string custom fields so the config hash matches the wire output', async () => { + const cfgClean = await resolveGlobalGranularities([{ name: 'fy', interval: '1 year', origin: '2024-02-01' }], {}); + const cfgDirty = await resolveGlobalGranularities( + [{ name: 'fy', interval: '1 year', origin: '2024-02-01', title: 123 as any, junk: () => 'x' } as any], + {}, + ); + expect(cfgDirty.customGranularities.fy).toEqual({ interval: '1 year', origin: '2024-02-01' }); + // Two configs that serialize identically must hash identically. + expect(granularityConfigHash(cfgDirty)).toBe(granularityConfigHash(cfgClean)); + }); + + // Regression: a set-but-empty interval env var must be treated as absent (unusable), not advertised. + it('drops a custom granularity whose env interval is set-but-empty', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,foo'; + process.env.CUBEJS_GRANULARITIES_FOO_INTERVAL = ''; + const cfg = await resolveGlobalGranularities(undefined, {}); + expect(cfg.enabledBuiltIns).toEqual(['year']); + expect(cfg.customGranularities.foo).toBeUndefined(); + }); + + // Regression: a function returning a non-array opts out of env (no leak), yielding the default catalog. + it('function returning a non-array yields the default catalog, not an env fallback', async () => { + process.env.CUBEJS_GRANULARITIES = 'week'; + const cfg = await resolveGlobalGranularities(() => null as any, {}); + expect([...cfg.enabledBuiltIns].sort()).toEqual([...BUILT_IN_GRANULARITY_NAMES].sort()); + expect(cfg.customGranularities).toEqual({}); + }); + + // Regression (adversarial): prototype-chain names must not be classified as built-ins. + it('does not classify prototype-chain names as built-in granularities', () => { + expect(isBuiltInGranularity('__proto__')).toBe(false); + expect(isBuiltInGranularity('constructor')).toBe(false); + expect(isBuiltInGranularity('hasOwnProperty')).toBe(false); + expect(isBuiltInGranularity('year')).toBe(true); + }); +}); diff --git a/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts b/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts new file mode 100644 index 0000000000000..8a5b0dbd60bd9 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts @@ -0,0 +1,204 @@ +import { + normalizeGranularitiesBlock, + resolveDimensionGranularities, +} from '../../src/compiler/GranularityResolver'; + +const BUILT_INS = { + year: { title: 'Year' }, + quarter: { title: 'Quarter' }, + month: { title: 'Month' }, + week: { title: 'Week' }, + day: { title: 'Day' }, + hour: { title: 'Hour' }, + minute: { title: 'Minute' }, + second: { title: 'Second' }, +}; + +const ALL_ENABLED = Object.keys(BUILT_INS); +const FISCAL_YEAR = { title: 'Fiscal Year', interval: '1 year', origin: '2026-04-01' }; +const GLOBAL_CUSTOM = { fiscal_year: FISCAL_YEAR }; + +describe('normalizeGranularitiesBlock', () => { + it('treats missing input as wide-open: includes * / no excludes / no custom', () => { + expect(normalizeGranularitiesBlock(undefined)).toEqual({ + includes: '*', + excludes: [], + custom: {}, + }); + expect(normalizeGranularitiesBlock(null)).toEqual({ + includes: '*', + excludes: [], + custom: {}, + }); + }); + + it('legacy flat-array form maps each entry into custom; includes stays *', () => { + const out = normalizeGranularitiesBlock([ + { name: 'fiscal_q', interval: '3 months', origin: '2026-04-01' }, + ]); + expect(out.includes).toBe('*'); + expect(out.excludes).toEqual([]); + expect(out.custom).toEqual({ + fiscal_q: { interval: '3 months', origin: '2026-04-01' }, + }); + }); + + it('post-yamlArrayToObj keyed object is preserved as legacy custom-only block', () => { + const out = normalizeGranularitiesBlock({ + fiscal_q: { interval: '3 months', origin: '2026-04-01' }, + }); + expect(out.includes).toBe('*'); + expect(out.custom.fiscal_q).toEqual({ interval: '3 months', origin: '2026-04-01' }); + }); + + it('new dict shape is canonicalized with defaults', () => { + const out = normalizeGranularitiesBlock({ includes: ['year'], custom: { fy: FISCAL_YEAR } }); + expect(out).toEqual({ + includes: ['year'], + excludes: [], + custom: { fy: FISCAL_YEAR }, + }); + }); +}); + +describe('resolveDimensionGranularities — spec resolution table', () => { + it('row 1: granularities omitted -> all enabled global granularities', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(Object.keys(out).sort()).toEqual([...ALL_ENABLED].sort()); + expect(out.year.type).toBe('built-in'); + }); + + it('row 2: legacy flat array -> enabled globals plus local custom', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock([{ name: 'fiscal_q', interval: '3 months', origin: '2026-04-01' }]), + ['year', 'month'], + {}, + BUILT_INS, + ); + expect(out.year.type).toBe('built-in'); + expect(out.month.type).toBe('built-in'); + expect(out.fiscal_q).toMatchObject({ interval: '3 months', origin: '2026-04-01', type: 'custom' }); + }); + + it('row 3: includes [a, b] + custom -> {a, b} plus custom', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ includes: ['year', 'quarter'], custom: { fy: FISCAL_YEAR } }), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(Object.keys(out).sort()).toEqual(['fy', 'quarter', 'year']); + expect(out.year.type).toBe('built-in'); + expect(out.fy.type).toBe('custom'); + }); + + it('row 4: excludes [x] -> all enabled globals minus x', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ excludes: ['day'] }), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(out.day).toBeUndefined(); + expect(out.year.type).toBe('built-in'); + }); + + it('row 5: excludes "*" + custom -> custom only', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ excludes: '*', custom: { fy: FISCAL_YEAR } }), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(Object.keys(out)).toEqual(['fy']); + expect(out.fy.type).toBe('custom'); + }); + + it('row 6: includes "*" + custom -> all enabled globals plus custom', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ includes: '*', custom: { fy: FISCAL_YEAR } }), + ['year', 'month'], + {}, + BUILT_INS, + ); + expect(Object.keys(out).sort()).toEqual(['fy', 'month', 'year']); + expect(out.fy.type).toBe('custom'); + }); + + it('global custom granularities flow through unless excluded', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), + ['year'], + GLOBAL_CUSTOM, + BUILT_INS, + ); + expect(out.fiscal_year.type).toBe('custom'); + expect(out.year.type).toBe('built-in'); + }); + + it('local custom overrides global custom of same name', () => { + const localFy = { interval: '1 year', origin: '2026-01-01' }; + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ custom: { fiscal_year: localFy } }), + ['year'], + GLOBAL_CUSTOM, + BUILT_INS, + ); + expect(out.fiscal_year.origin).toBe('2026-01-01'); + }); + + it('built-in overridden via global custom stays type built-in (not relabeled custom)', () => { + // `config.granularities: [{ name: 'year', title: 'Anno' }]` both enables 'year' and puts it in + // globalCustom; the resolved entry must remain a built-in. + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), + ['year'], + { year: { title: 'Anno' } }, + { ...BUILT_INS, year: { title: 'Anno' } }, + ); + expect(out.year.type).toBe('built-in'); + expect(out.year.title).toBe('Anno'); + }); +}); + +describe('normalizeGranularitiesBlock — reserved-name disambiguation', () => { + it('custom granularity named "includes" is not misread as the dict form', () => { + const out = normalizeGranularitiesBlock({ + includes: { interval: '1 year', origin: '2026-04-01' }, + }); + expect(out.includes).toBe('*'); + expect(out.custom.includes).toEqual({ interval: '1 year', origin: '2026-04-01' }); + }); + + // Regression: a legacy custom granularity literally named `custom` whose value is a DEFINITION + // (string-valued fields) must NOT be misread as the dict form's custom map (name -> definition). + it('custom granularity named "custom" (definition value) is not misread as the dict form', () => { + const out = normalizeGranularitiesBlock({ + custom: { interval: '1 year', origin: '2026-04-01' }, + }); + expect(out.includes).toBe('*'); + expect(out.custom.custom).toEqual({ interval: '1 year', origin: '2026-04-01' }); + }); + + it('custom granularity named "custom" with only an sql function is not misread as the dict form', () => { + const sql = () => 'date_trunc(\'year\', x)'; + const out = normalizeGranularitiesBlock({ custom: { sql } }); + expect(out.custom.custom).toEqual({ sql }); + }); + + // The genuine dict form (custom is a map of name -> definition OBJECT) is still recognized. + it('genuine dict form with a custom map is recognized', () => { + const out = normalizeGranularitiesBlock({ + includes: ['year'], + custom: { fiscal_year: { interval: '1 year', origin: '2026-04-01' } }, + }); + expect(out.includes).toEqual(['year']); + expect(out.custom.fiscal_year).toEqual({ interval: '1 year', origin: '2026-04-01' }); + expect(out.custom.custom).toBeUndefined(); + }); +}); diff --git a/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts b/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts new file mode 100644 index 0000000000000..75e4c0c51b6a0 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/granularity-config-hash.test.ts @@ -0,0 +1,84 @@ +import { granularityConfigHash } from '../../src/compiler/GlobalGranularitiesConfig'; +import type { GlobalGranularitiesConfig } from '../../src/compiler/GlobalGranularitiesConfig'; + +const config = ( + enabledBuiltIns: string[], + customGranularities: Record = {}, +): GlobalGranularitiesConfig => ({ enabledBuiltIns, customGranularities }); + +describe('granularityConfigHash', () => { + it('is stable for identical configs', () => { + const a = config(['year', 'month'], { fy: { interval: '1 year', origin: '2024-02-01' } }); + const b = config(['year', 'month'], { fy: { interval: '1 year', origin: '2024-02-01' } }); + expect(granularityConfigHash(a)).toBe(granularityConfigHash(b)); + }); + + it('is sensitive to built-in order (order is part of the wire contract)', () => { + expect(granularityConfigHash(config(['year', 'month']))) + .not.toBe(granularityConfigHash(config(['month', 'year']))); + }); + + it('is sensitive to custom emission order', () => { + const a = config(['year'], { a: { interval: '1 week' }, b: { interval: '2 week' } }); + const b = config(['year'], { b: { interval: '2 week' }, a: { interval: '1 week' } }); + expect(granularityConfigHash(a)).not.toBe(granularityConfigHash(b)); + }); + + it('is sensitive to every known definition field', () => { + const base = { interval: '1 year', title: 'FY', format: '%Y', offset: undefined, origin: '2024-02-01' }; + const baseHash = granularityConfigHash(config(['year'], { fy: base })); + for (const [field, value] of [ + ['interval', '2 year'], ['title', 'Fiscal'], ['format', '%y'], ['origin', '2025-02-01'], + ] as const) { + expect(granularityConfigHash(config(['year'], { fy: { ...base, [field]: value } }))) + .not.toBe(baseHash); + } + }); + + it('ignores unknown and non-serializable definition props', () => { + const plain = config(['year'], { fy: { interval: '1 year' } }); + const dirty = config(['year'], { + fy: { + interval: '1 year', + sql: () => 'now()', + somethingElse: { nested: true }, + title: (() => 'not a string') as any, + }, + }); + expect(granularityConfigHash(dirty)).toBe(granularityConfigHash(plain)); + }); + + it('distinguishes an absent field from a present one', () => { + expect(granularityConfigHash(config(['year'], { fy: { interval: '1 year' } }))) + .not.toBe(granularityConfigHash(config(['year'], { fy: { interval: '1 year', title: 'FY' } }))); + }); + + // Regression (F6): for a name shadowing a built-in, only title/format affect output — the SQL + // layer fixes a built-in's interval at `1 `. So changing interval/offset/origin on a + // built-in override must NOT change the hash (else it churns the cache/compilerId for identical + // output), while changing title/format must. + describe('built-in override hashes only the fields that affect output', () => { + const withYear = (def: any) => config(['year'], { year: def }); + + it('ignores interval/offset/origin overrides on a built-in name', () => { + const baseHash = granularityConfigHash(withYear({ title: 'Year' })); + expect(granularityConfigHash(withYear({ title: 'Year', interval: '2 years' }))).toBe(baseHash); + expect(granularityConfigHash(withYear({ title: 'Year', offset: '1 day' }))).toBe(baseHash); + expect(granularityConfigHash(withYear({ title: 'Year', origin: '2024-02-01' }))).toBe(baseHash); + }); + + it('still reflects title/format overrides on a built-in name', () => { + const baseHash = granularityConfigHash(withYear({ title: 'Year' })); + expect(granularityConfigHash(withYear({ title: 'Jaar' }))).not.toBe(baseHash); + expect(granularityConfigHash(withYear({ title: 'Year', format: '%y' }))).not.toBe(baseHash); + }); + + it('still hashes every field for a real (non-built-in) custom', () => { + const baseHash = granularityConfigHash(config([], { fy: { interval: '1 year', origin: '2024-02-01' } })); + expect(granularityConfigHash(config([], { fy: { interval: '2 years', origin: '2024-02-01' } }))) + .not.toBe(baseHash); + expect(granularityConfigHash(config([], { fy: { interval: '1 year', origin: '2025-02-01' } }))) + .not.toBe(baseHash); + }); + }); +}); diff --git a/packages/cubejs-schema-compiler/test/unit/views.test.ts b/packages/cubejs-schema-compiler/test/unit/views.test.ts index 41e9b08e92025..f54299de78126 100644 --- a/packages/cubejs-schema-compiler/test/unit/views.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/views.test.ts @@ -589,4 +589,50 @@ describe('Views YAML', () => { await expect(compiler.compile()).rejects.toThrow('test_view cube: Member \'unknown\''); }); + + // Regression: a time dimension restricted via the dict form (includes/excludes) must keep that + // restriction when included in a view. The view dimension carries a propagated granularitiesBlock; + // re-normalizing the custom-only map used to reset includes to '*' and expose every built-in. + it('preserves a source time dimension includes/excludes when included in a view', async () => { + const { compiler, cubeEvaluator, metaTransformer } = prepareYamlCompiler(` + cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: created_at + sql: created_at + type: time + granularities: + includes: + - year + - month + + views: + - name: orders_view + cubes: + - join_path: orders + includes: + - count + - created_at + `); + + await compiler.compile(); + + // The compiled view dimension keeps the source's includes, not the reset '*'. + const viewDim = cubeEvaluator.getCubeDefinition('orders_view').dimensions!.created_at; + expect(viewDim.granularitiesBlock).toBeDefined(); + expect(viewDim.granularitiesBlock!.includes).toEqual(['year', 'month']); + + // And the meta reflects only year + month (no other built-ins leak through the view). + const viewMeta = metaTransformer.cubes.map((d) => d.config).find((d) => d.name === 'orders_view'); + const metaDim: any = viewMeta!.dimensions.find((d: any) => d.name === 'orders_view.created_at'); + expect(metaDim.effectiveGranularities.map((g: any) => g.name)).toEqual(['year', 'month']); + }); }); diff --git a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts index 5eb64ab75f8ab..f1e0d81254831 100644 --- a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts @@ -796,7 +796,10 @@ cubes: await compiler.compile(); throw new Error('compile must return an error'); } catch (e: any) { - expect(e.message).toContain('must be defined as array'); + // `granularities: { name: half_year }` is neither the legacy array nor the dict + // (includes/excludes/custom) form; it's read as a custom map whose `name` entry is an + // invalid (non-object) granularity definition. + expect(e.message).toContain('dimensions.created_at.granularities.name'); } }); diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index c6e1481713747..477457d11a9d0 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -7,8 +7,12 @@ import { compile, Compiler, createQuery, + buildGranularitiesCatalog, CubeDefinition, EvaluatedCube, + GlobalGranularitiesConfig, + GranularitiesOption, + granularityConfigHash, PreAggregationFilters, PreAggregationInfo, PreAggregationReferences, @@ -16,6 +20,7 @@ import { prepareCompiler, queryClass, QueryFactory, + resolveGlobalGranularities, TransformedQuery, ViewIncludedMember, } from '@cubejs-backend/schema-compiler'; @@ -51,6 +56,7 @@ export interface CompilerApiOptions { devServer?: boolean; fastReload?: boolean; allowNodeRequire?: boolean; + granularities?: GranularitiesOption; } export interface GetSqlOptions { @@ -133,6 +139,10 @@ export class CompilerApi { protected compilerVersion?: string; + protected readonly granularities?: GranularitiesOption; + + private resolvedGranularitiesPromise?: Promise; + protected queryFactory?: QueryFactory; public constructor(repository: SchemaFileRepository, dbType: DbTypeInternalFn, options: CompilerApiOptions) { @@ -152,6 +162,7 @@ export class CompilerApi { this.allowJsDuplicatePropsInSchema = options.allowJsDuplicatePropsInSchema; this.sqlCache = options.sqlCache; this.standalone = options.standalone; + this.granularities = options.granularities; this.nativeInstance = this.createNativeInstance(); // Caching stuff @@ -224,8 +235,14 @@ export class CompilerApi { compilerVersion += `_${crypto.createHash('md5').update(JSON.stringify(files)).digest('hex')}`; } + // Global custom granularities are merged into the compiled model (they take part in + // pre-aggregation matching), so a config change must force a recompile. The resolved config is + // handed to the compiler so a `config.granularities` function runs once per compile. + const resolvedGranularities = await this.resolveGranularities(); + compilerVersion += `_gran_${granularityConfigHash(resolvedGranularities)}`; + if (!this.compilers || this.compilerVersion !== compilerVersion) { - this.compilers = this.compileSchema(compilerVersion, options.requestId).catch(e => { + this.compilers = this.compileSchema(compilerVersion, resolvedGranularities, options.requestId).catch(e => { this.compilers = undefined; throw e; }); @@ -235,6 +252,25 @@ export class CompilerApi { return this.compilers; } + /** + * The resolved global granularities config. Memoized for the static/function forms — both are + * fixed for the instance, and a `config.granularities` function must not run per request. The env + * form re-resolves (a cheap parse) so a runtime CUBEJS_GRANULARITIES change still forces a recompile. + */ + private resolveGranularities(): Promise { + if (this.granularities === undefined) { + return resolveGlobalGranularities(undefined, this.compileContext); + } + if (!this.resolvedGranularitiesPromise) { + this.resolvedGranularitiesPromise = resolveGlobalGranularities(this.granularities, this.compileContext); + // Let a failed resolve (e.g. a throwing function form) be retried. + this.resolvedGranularitiesPromise.catch(() => { + this.resolvedGranularitiesPromise = undefined; + }); + } + return this.resolvedGranularitiesPromise; + } + /** * Creates the compilers instances without model compilation, * because it could fail and no compilers will be returned. @@ -250,7 +286,11 @@ export class CompilerApi { }); } - public async compileSchema(compilerVersion: string, requestId?: string): Promise { + public async compileSchema( + compilerVersion: string, + resolvedGranularities: GlobalGranularitiesConfig, + requestId?: string, + ): Promise { const startCompilingTime = new Date().getTime(); try { this.logger(this.compilers ? 'Recompiling schema' : 'Compiling schema', { @@ -267,6 +307,7 @@ export class CompilerApi { compiledScriptCache: this.compiledScriptCache, compiledJinjaCache: this.compiledJinjaCache, compiledYamlCache: this.compiledYamlCache, + resolvedGranularities, }); this.queryFactory = await this.createQueryFactory(compilers); @@ -315,6 +356,8 @@ export class CompilerApi { public async getSqlGenerator(query: NormalizedQuery, dataSource?: string): Promise<{ sqlGenerator: any; compilers: Compiler }> { const dbType = await this.getDbType(dataSource); const compilers = await this.getCompilers({ requestId: query.requestId }); + // Custom granularities (local + baked-in globals) resolve from the compiled symbols, so no + // per-request granularity threading is needed here. let sqlGenerator = await this.createQueryByDataSource(compilers, query, dataSource, dbType); if (!sqlGenerator) { @@ -371,7 +414,10 @@ export class CompilerApi { if (this.sqlCache) { // eslint-disable-next-line @typescript-eslint/no-unused-vars const { requestId, ...keyOptions } = query; - const key = { query: keyOptions, options }; + const key = { + query: keyOptions, + options, + }; return compilers.compilerCache.getQueryCache(key).cache(['sql'], getSqlFn); } else { return getSqlFn(); @@ -1064,24 +1110,51 @@ export class CompilerApi { }; } - protected mixInVisibilityMaskHash(compilerId: string, visibilityMaskHash: string): string { + protected mixInMaskHash(compilerId: string, maskHash: string): string { const uuidBytes = Buffer.from(uuidParse(compilerId)); - const hashBytes = Buffer.from(visibilityMaskHash, 'hex'); + const hashBytes = Buffer.from(maskHash, 'hex'); return uuidv4({ random: crypto.createHash('sha256').update(uuidBytes).update(hashBytes).digest() .subarray(0, 16) as any }); } + protected mixInVisibilityMaskHash(compilerId: string, visibilityMaskHash: string): string { + return this.mixInMaskHash(compilerId, visibilityMaskHash); + } + + /** + * The `/v1/granularities` catalog for this model — built-ins plus global customs. + */ + public async getGranularities(options: { requestId?: string } = {}): Promise { + const compilers = await this.getCompilers(options); + return buildGranularitiesCatalog(compilers.cubeEvaluator.globalGranularitiesConfig); + } + public async metaConfig( requestContext: Context, options: { includeCompilerId?: boolean; includeViewGroups?: boolean; skipVisibilityPatch?: boolean; requestId?: string } = {} ): Promise { const { includeCompilerId, includeViewGroups, skipVisibilityPatch, ...restOptions } = options; const compilers = await this.getCompilers(restOptions); + // Granularities are baked into the compiled model, so the base meta cubes are served directly. const { cubes } = compilers.metaTransformer; + // Fixed composition order: base compilerId, then visibility mask. Only computed when the caller + // actually wants the id — the hashing is skipped when a caller asks for view groups alone (the + // gateway always requests view groups). + const composeCompilerId = (visibilityMaskHash: string | null) => { + let id = compilers.compilerId; + if (visibilityMaskHash) { + id = this.mixInMaskHash(id, visibilityMaskHash); + } + return id; + }; + if (skipVisibilityPatch) { if (includeCompilerId || includeViewGroups) { - const result: any = { cubes, compilerId: compilers.compilerId }; + const result: any = { cubes }; + if (includeCompilerId) { + result.compilerId = composeCompilerId(null); + } if (includeViewGroups) { result.viewGroups = compilers.metaTransformer.viewGroups; } @@ -1096,10 +1169,10 @@ export class CompilerApi { cubes ); if (includeCompilerId || includeViewGroups) { - const result: any = { - cubes: patchedCubes, - compilerId: visibilityMaskHash ? this.mixInVisibilityMaskHash(compilers.compilerId, visibilityMaskHash) : compilers.compilerId, - }; + const result: any = { cubes: patchedCubes }; + if (includeCompilerId) { + result.compilerId = composeCompilerId(visibilityMaskHash); + } if (includeViewGroups) { result.viewGroups = compilers.metaTransformer.viewGroups; } @@ -1113,10 +1186,11 @@ export class CompilerApi { options?: { requestId?: string } ): Promise<{ metaConfig: any; cubeDefinitions: Record }> { const compilers = await this.getCompilers(options); + // Granularities are baked into the compiled model, so the base meta cubes are used directly. const { cubes: patchedCubes } = await this.patchVisibilityByAccessPolicy( compilers, requestContext, - compilers.metaTransformer?.cubes + compilers.metaTransformer.cubes ); return { metaConfig: patchedCubes, diff --git a/packages/cubejs-server-core/src/core/optionsValidate.ts b/packages/cubejs-server-core/src/core/optionsValidate.ts index 0ba331d9f9476..c424dc66c4694 100644 --- a/packages/cubejs-server-core/src/core/optionsValidate.ts +++ b/packages/cubejs-server-core/src/core/optionsValidate.ts @@ -90,6 +90,15 @@ const schemaOptions = Joi.object().keys({ ), schemaVersion: Joi.func(), extendContext: Joi.func(), + granularities: Joi.alternatives().try( + Joi.func(), + Joi.array().items( + Joi.alternatives().try( + Joi.string(), + Joi.object().unknown(true) + ) + ) + ), // Scheduled refresh scheduledRefreshTimer: Joi.alternatives().try( Joi.boolean(), diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index 48b146c8abb70..8d047e4340bab 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -741,6 +741,7 @@ export class CubejsServerCore { compileContext: options.context, dialectClass: options.dialectClass, externalDialectClass: options.externalDialectClass, + granularities: this.options.granularities, allowJsDuplicatePropsInSchema: options.allowJsDuplicatePropsInSchema, sqlCache: this.options.sqlCache, standalone: this.standalone, diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts index 3d1cfee7d79ee..502337dc18c8f 100644 --- a/packages/cubejs-server-core/src/core/types.ts +++ b/packages/cubejs-server-core/src/core/types.ts @@ -10,7 +10,7 @@ import { UserBackgroundContext, } from '@cubejs-backend/api-gateway'; import { BaseDriver, CacheAndQueryDriverType } from '@cubejs-backend/query-orchestrator'; -import { BaseQuery } from '@cubejs-backend/schema-compiler'; +import { BaseQuery, GranularityList } from '@cubejs-backend/schema-compiler'; export interface QueueOptions { concurrency?: number; @@ -217,6 +217,7 @@ export interface CreateOptions { preAggregationsSchema?: string | PreAggregationsSchemaFn; schemaVersion?: (context: RequestContext) => string | Promise; extendContext?: ExtendContextFn; + granularities?: GranularityList | ((context: RequestContext) => GranularityList | Promise); scheduledRefreshTimer?: boolean | number; scheduledRefreshTimeZones?: string[] | ScheduledRefreshTimeZonesFn; scheduledRefreshContexts?: () => Promise; diff --git a/packages/cubejs-server-core/test/unit/granularities-bake.test.ts b/packages/cubejs-server-core/test/unit/granularities-bake.test.ts new file mode 100644 index 0000000000000..8cfcb279f81c1 --- /dev/null +++ b/packages/cubejs-server-core/test/unit/granularities-bake.test.ts @@ -0,0 +1,567 @@ +import { SchemaFileRepository } from '@cubejs-backend/shared'; +import { CompilerApi } from '../../src/core/CompilerApi'; +import { DbTypeInternalFn } from '../../src/core/types'; + +// Global granularity config (env | static list | function(ctx)) is resolved once per compile and +// applied by CubeSymbols: global CUSTOM granularities are merged into each time dimension's +// `granularities` map (locals win), so both /v1/meta and the SQL path — including pre-aggregation +// matching — read the same values with no per-request resolution. The resolved hash is folded into +// compilerVersion, since the config affects the compiled model. This suite verifies the merge, the +// fold-into-compilerVersion, and the resolve-once guarantee. + +// A CompilerApi that counts how many times the granularities FUNCTION form is invoked, so we can +// assert it runs once per compile (per appId), not once per metaConfig/getSql call. +class TestableCompilerApi extends CompilerApi { + public version(): string | undefined { + return this.compilerVersion; + } +} + +const repository: SchemaFileRepository = { + localPath: () => '/mock/path', + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'orders.js', + content: ` + cube('Orders', { + sql: 'SELECT * FROM orders', + measures: { count: { type: 'count' } }, + dimensions: { + id: { sql: 'id', type: 'number', primaryKey: true }, + created_at: { sql: 'created_at', type: 'time' }, + updated_at: { sql: 'updated_at', type: 'time' }, + excluded_at: { + sql: 'updated_at', + type: 'time', + granularities: { excludes: ['fiscal_year', 'sprint'] }, + }, + collide_at: { + sql: 'created_at', + type: 'time', + // Local custom named 'fiscal_year' collides with a same-named global custom. + granularities: { fiscal_year: { interval: '1 year', origin: '2020-01-01' } }, + }, + }, + }); + cube('Events', { + sql: 'SELECT * FROM events', + measures: { count: { type: 'count' } }, + dimensions: { + ts: { + sql: 'ts', + type: 'time', + granularities: { + fiscal_year: { interval: '1 year', origin: '2024-02-01' }, + }, + }, + }, + }); + cube('Products', { + sql: 'SELECT * FROM products', + measures: { count: { type: 'count' } }, + dimensions: { + name: { sql: 'name', type: 'string' }, + }, + }); + `, + }, + ]), +}; + +const mockDbType: DbTypeInternalFn = async () => 'postgres'; + +const noopLogger = () => { /* silent */ }; + +const createApi = (options: any = {}) => new TestableCompilerApi(repository, mockDbType, { + logger: noopLogger, + ...options, +}); + +const ctxFor = (tenant: string) => ({ securityContext: { tenant }, requestId: `req-${tenant}` }); + +const dimByName = (cubes: any[], name: string) => cubes + .flatMap((c: any) => c.config.dimensions) + .find((d: any) => d.name === name); + +const granularityNames = (dim: any) => dim.effectiveGranularities.map((g: any) => g.name); + +const ALL_BUILT_INS = ['year', 'quarter', 'month', 'week', 'day', 'hour', 'minute', 'second']; + +const queryFor = (tenant: string, dimension: string, granularity: string): any => ({ + measures: [dimension.startsWith('Events.') ? 'Events.count' : 'Orders.count'], + timeDimensions: [{ dimension, granularity }], + timezone: 'UTC', + contextSymbols: { securityContext: { tenant } }, + requestId: `sql-${tenant}-${dimension}-${granularity}`, +}); + +// The compiled dimension symbol's `granularities` map — the map the SQL layer and the pre-agg +// matcher resolve customs by name from. Global customs are baked into it at compile time. +const bakedDimGranularities = async (api: CompilerApi, dimPath: string): Promise> => { + const compilers = await (api as any).getCompilers(); + return compilers.cubeEvaluator.dimensionByPath(dimPath).granularities || {}; +}; + +// The `symbols[cube][dim]` map CubeSymbols.resolveGranularity reads for SQL. It derives from the +// same dimension definition the merge writes, so a global custom is resolvable here too. +const symbolsDimGranularities = async (api: CompilerApi, cube: string, dim: string): Promise> => { + const compilers = await (api as any).getCompilers(); + return (compilers.cubeEvaluator as any).symbols?.[cube]?.[dim]?.granularities || {}; +}; + +describe('global granularities resolved and applied at compile time', () => { + describe('env / static config', () => { + afterEach(() => { + delete process.env.CUBEJS_GRANULARITIES; + }); + + // 1. No config -> every time dimension gets all built-ins baked as effectiveGranularities; + // internal fields stay off the wire. + test('no config: every time dimension gets all built-ins baked; internal fields stay off the wire', async () => { + const api = createApi(); + const cubes = await api.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toEqual(ALL_BUILT_INS); + // Local custom granularity survives on top of the enabled built-ins. + expect(granularityNames(dimByName(cubes, 'Events.ts'))).toEqual([...ALL_BUILT_INS, 'fiscal_year']); + // Non-time dimensions are untouched. + expect(dimByName(cubes, 'Products.name').effectiveGranularities).toBeUndefined(); + // The raw normalized block never reaches the wire. + expect(dimByName(cubes, 'Orders.created_at').granularitiesBlock).toBeUndefined(); + api.dispose(); + }); + + // 2. Static global customs reach every dimension map the downstream paths read: the + // evaluatedCubes/cubeList copy (dimensionByPath, timeDimensionsForCube, pre-agg matching) and + // the symbols copy (CubeSymbols.resolveGranularity for SQL). Both derive from the definition + // the merge writes, so a global custom resolves in SQL and pre-agg matching doesn't throw + // "Granularity does not exist". Drives a real getSql() to lock that in, and confirms + // per-dimension excludes are honored. + test('static global customs are baked into the consumed map; SQL resolves them; excludes honored', async () => { + const api = createApi({ + granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-02-01' }], + }); + + // Baked into a plain dimension's granularities map (the map SQL / pre-agg matching read). + const baked = await bakedDimGranularities(api, 'Orders.created_at'); + expect(baked.fiscal_year).toMatchObject({ interval: '1 year', origin: '2024-02-01' }); + + // The dimension that excludes the global custom does NOT get it baked in, matching meta. + const bakedExcluded = await bakedDimGranularities(api, 'Orders.excluded_at'); + expect(bakedExcluded.fiscal_year).toBeUndefined(); + + // Meta agrees: excluded dimension omits it; plain dimensions advertise it. + const cubes = await api.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toContain('fiscal_year'); + expect(granularityNames(dimByName(cubes, 'Orders.excluded_at'))).not.toContain('fiscal_year'); + + // Real SQL path: a query at the global custom grain must resolve (both object graphs carry the + // merge) and not throw during pre-agg matching. Previously this threw "Granularity does not + // exist" because the symbols copy lacked the merge. + const { sql } = await api.getSql({ + measures: [], + dimensions: [], + segments: [], + filters: [], + timeDimensions: [{ + dimension: 'Orders.created_at', + // Custom granularity name — allowed at runtime; the wire type only enumerates built-ins. + granularity: 'fiscal_year' as any, + dateRange: ['2024-01-01', '2025-01-01'], + }], + order: [], + } as any); + const sqlText = Array.isArray(sql) ? sql[0] : sql; + expect(typeof sqlText).toBe('string'); + expect(sqlText.length).toBeGreaterThan(0); + api.dispose(); + }); + + // 3. Time dimensions without local customization all resolve to the same effective set. + test('time dimensions without local customization resolve to the same set', async () => { + const api = createApi(); + const cubes = await api.metaConfig(ctxFor('a'), {}); + const created = dimByName(cubes, 'Orders.created_at'); + const updated = dimByName(cubes, 'Orders.updated_at'); + expect(created.effectiveGranularities).toEqual(updated.effectiveGranularities); + // A dimension with a local block resolves to a different set. + expect(dimByName(cubes, 'Events.ts').effectiveGranularities) + .not.toEqual(created.effectiveGranularities); + api.dispose(); + }); + + // 4. Resolved config hash is folded into compilerVersion, so a different env config yields a + // different compilerVersion (and therefore a recompile / different compiled model). + // + // Note: the resolved config is cached per CompilerApi instance for its lifetime (resolved once + // per appId), so changing the env var on the SAME instance does NOT re-resolve. The fold-into- + // compilerVersion contract is verified across two instances reading different env values — a + // process restart / new appId is exactly when env is re-read. + test('resolved env config hash folds into compilerVersion (different env -> different version)', async () => { + const apiDefault = createApi(); + const cubesDefault = await apiDefault.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubesDefault, 'Orders.created_at'))).toEqual(ALL_BUILT_INS); + const versionDefault = apiDefault.version(); + expect(versionDefault).toContain('_gran_'); + + process.env.CUBEJS_GRANULARITIES = 'year,month'; + const apiEnv = createApi(); + const cubesEnv = await apiEnv.metaConfig(ctxFor('a'), {}); + // Different resolved config -> different folded hash -> different compilerVersion. + expect(apiEnv.version()).not.toBe(versionDefault); + expect(granularityNames(dimByName(cubesEnv, 'Orders.created_at'))).toEqual(['year', 'month']); + + apiDefault.dispose(); + apiEnv.dispose(); + }); + }); + + describe('function config resolved once per compile', () => { + // 5. THE key new-architecture guarantee: the function form is invoked ONCE per compile (per + // appId), not once per metaConfig/getSql call. Multiple reads on the same instance add zero + // additional invocations. + test('function form is invoked once per compile, not per metaConfig/getSql call', async () => { + let calls = 0; + const api = createApi({ + granularities: () => { + calls += 1; + return ['year', 'month']; + }, + }); + + // First read triggers the single compile-time resolution. + await api.metaConfig(ctxFor('a'), {}); + expect(calls).toBe(1); + + // Further reads (meta or SQL, any context) reuse the baked compile — no re-invocation. + await api.metaConfig(ctxFor('b'), {}); + await api.metaConfig(ctxFor('a'), {}); + await api.getSqlGenerator(queryFor('a', 'Orders.created_at', 'month')); + await api.getSqlGenerator(queryFor('b', 'Orders.created_at', 'year')); + expect(calls).toBe(1); + api.dispose(); + }); + + // 5b. INTENDED per-appId contract (CUB-2567 rev 2 narrowing decision): within a single appId + // (one CompilerApi instance), granularities do NOT vary by request securityContext. The + // function form is resolved ONCE with the instance's compile context, so two requests with + // DIFFERENT request securityContexts see the SAME baked granularities. + test('two request contexts on the same appId get identical baked granularities', async () => { + let seenCtx: any; + const api = createApi({ + // Would branch per-tenant IF it were called per request — but it is resolved once with the + // instance's compile context, so both requests below get the compile-context result ('c'). + granularities: (ctx: any) => { + seenCtx = ctx; + return ctx?.securityContext?.tenant === 'c' ? ['year', 'quarter'] : ['month']; + }, + compileContext: { securityContext: { tenant: 'c' } }, + }); + + const cubesA = await api.metaConfig(ctxFor('a'), {}); + const cubesB = await api.metaConfig(ctxFor('b'), {}); + + // Both requests see the compile-context resolution, not their own securityContext. + expect(seenCtx?.securityContext?.tenant).toBe('c'); + expect(granularityNames(dimByName(cubesA, 'Orders.created_at'))).toEqual(['year', 'quarter']); + expect(granularityNames(dimByName(cubesB, 'Orders.created_at'))) + .toEqual(granularityNames(dimByName(cubesA, 'Orders.created_at'))); + api.dispose(); + }); + + // 6. Function form baked + folded into compilerVersion: two CompilerApi instances with different + // appId-level compile contexts (function returns differ) produce a different compilerVersion and + // different baked meta. + test('different compile contexts (different function results) -> different version and meta', async () => { + const granularities = (ctx: any) => (ctx.securityContext.tenant === 'a' + ? ['year'] + : ['month']); + + const apiA = createApi({ granularities, compileContext: { securityContext: { tenant: 'a' } } }); + const apiB = createApi({ granularities, compileContext: { securityContext: { tenant: 'b' } } }); + + const cubesA = await apiA.metaConfig(ctxFor('x'), {}); + const cubesB = await apiB.metaConfig(ctxFor('x'), {}); + + // Different baked effective sets. + expect(granularityNames(dimByName(cubesA, 'Orders.created_at'))).toEqual(['year']); + expect(granularityNames(dimByName(cubesB, 'Orders.created_at'))).toEqual(['month']); + + // Different compilerVersion (the resolved-config hash is folded in). compilerId is NOT asserted: + // two independent CompilerApi instances get random UUIDs regardless of granularities, so that + // check would be vacuous. + expect(apiA.version()).not.toBe(apiB.version()); + apiA.dispose(); + apiB.dispose(); + }); + }); + + describe('meta and SQL read the same baked values', () => { + // 7. Meta and SQL read the SAME baked values, so a custom the meta advertises on a dimension is + // exactly the one baked into that dimension's granularities map (the map the SQL path / pre-agg + // matcher consume) — no meta/SQL contract gap. (Real SQL execution of a baked global custom is + // exercised in test 2 and the view test; here we assert the per-dimension meta/baked-map parity.) + test('meta and the baked SQL/matcher map agree on the effective set', async () => { + const api = createApi({ + granularities: [{ name: 'sprint', interval: '2 weeks', origin: '2024-01-01' }], + }); + const cubes = await api.metaConfig(ctxFor('a'), {}); + + // Plain dimension: meta advertises the global custom, and the baked map carries the same one. + expect(granularityNames(dimByName(cubes, 'Orders.created_at'))).toContain('sprint'); + const baked = await bakedDimGranularities(api, 'Orders.created_at'); + expect(baked.sprint).toMatchObject({ interval: '2 weeks', origin: '2024-01-01' }); + + // Excluded dimension: absent from meta AND absent from the baked map — they agree on the gap. + expect(granularityNames(dimByName(cubes, 'Orders.excluded_at'))).not.toContain('sprint'); + const bakedExcluded = await bakedDimGranularities(api, 'Orders.excluded_at'); + expect(bakedExcluded.sprint).toBeUndefined(); + api.dispose(); + }); + }); + + describe('global customs participate in pre-agg / hierarchy matching', () => { + // 8a. NEW deliberate behavior: a global custom merged into td.granularities is visible to the + // granularity-hierarchy / pre-agg matching path. The matcher resolves custom granularities off + // the compiled dimension symbol's `granularities` map, so this first test locks in that the + // global custom is present in that map (and the by-reference / copy-on-write sharing). Test 8b + // then drives the REAL matcher end-to-end. + test('global custom is baked into the dimension granularities map the matcher consumes', async () => { + const api = createApi({ + granularities: [{ name: 'sprint', interval: '2 weeks', origin: '2024-01-01' }], + }); + // Force a compile. + await api.metaConfig(ctxFor('a'), {}); + + const baked = await bakedDimGranularities(api, 'Orders.created_at'); + expect(baked.sprint).toMatchObject({ interval: '2 weeks', origin: '2024-01-01' }); + + // Every plain dimension (no local block) gets the same global customs merged in. + const bakedUpdated = await bakedDimGranularities(api, 'Orders.updated_at'); + expect(bakedUpdated).toEqual(baked); + + // A dimension with local customs keeps them alongside the global one. + const bakedCollide = await bakedDimGranularities(api, 'Orders.collide_at'); + expect(bakedCollide).not.toEqual(baked); + expect(bakedCollide.sprint).toBeDefined(); + + // A dimension that excludes it does NOT get it baked in. + const bakedExcluded = await bakedDimGranularities(api, 'Orders.excluded_at'); + expect(bakedExcluded.sprint).toBeUndefined(); + api.dispose(); + }); + + // 8b. REAL pre-agg matching: a rollup defined on the GLOBAL-custom grain must be matchable by a + // query at that grain. This drives the actual matcher end-to-end via getSql (which runs + // PreAggregations.transformQueryToCanUseForm + canUsePreAggregationForTransformedQueryFn and + // resolves the custom granularity off the baked map). A matching query uses the pre-agg; a query + // at an incompatible grain does not. + const preAggRepository: SchemaFileRepository = { + localPath: () => '/mock/path', + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'orders.js', + content: ` + cube('Orders', { + sql: 'SELECT * FROM orders', + measures: { count: { type: 'count' } }, + dimensions: { created_at: { sql: 'created_at', type: 'time' } }, + preAggregations: { + byFiscal: { + type: 'rollup', + measures: [Orders.count], + timeDimension: Orders.created_at, + // Rollup granularity is the GLOBAL custom — only resolvable if the bake reached + // the map the matcher consumes. + granularity: 'fiscal_year', + partitionGranularity: 'year', + }, + }, + }); + `, + }, + ]), + }; + + const preAggQuery = (grain: string): any => ({ + measures: ['Orders.count'], + dimensions: [], + segments: [], + filters: [], + timeDimensions: [{ + dimension: 'Orders.created_at', + // Custom grain name — allowed at runtime; the wire type only enumerates built-ins. + granularity: grain as any, + dateRange: ['2024-01-01', '2025-01-01'], + }], + order: [], + timezone: 'UTC', + }); + + test('a rollup on the global-custom grain is matched by a query at that grain', async () => { + const api = new TestableCompilerApi(preAggRepository, mockDbType, { + logger: noopLogger, + granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-01-01' }], + }); + + // Query at the global-custom grain -> the matcher resolves the custom and USES the pre-agg. + const matchSql = await api.getSql(preAggQuery('fiscal_year'), { includeDebugInfo: true }); + expect(matchSql.preAggregations.map((p: any) => p.preAggregationId)).toContain('Orders.byFiscal'); + + // Query at an incompatible grain -> no pre-agg matches. + const missSql = await api.getSql(preAggQuery('month'), { includeDebugInfo: true }); + expect(missSql.preAggregations).toHaveLength(0); + api.dispose(); + }); + }); + + describe('global custom on a view member (dual-write to symbols)', () => { + // Orders with a plain time dimension + a view that includes it. A global custom must be baked + // into the VIEW member's dimension too, in BOTH object graphs (evaluatedCubes AND symbols), so a + // query against the view member at the global grain resolves in SQL. + const viewRepository: SchemaFileRepository = { + localPath: () => '/mock/path', + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'orders.js', + content: ` + cube('Orders', { + sql: 'SELECT * FROM orders', + measures: { count: { type: 'count' } }, + dimensions: { + id: { sql: 'id', type: 'number', primaryKey: true }, + created_at: { sql: 'created_at', type: 'time' }, + }, + }); + view('OrdersView', { + cubes: [ + { join_path: 'Orders', includes: ['count', 'created_at'] }, + ], + }); + `, + }, + ]), + }; + + const createViewApi = () => new TestableCompilerApi(viewRepository, mockDbType, { + logger: noopLogger, + granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-02-01' }], + }); + + test('global custom is baked into the view member and resolves in SQL', async () => { + const api = createViewApi(); + + // Baked into the view member in BOTH graphs: evaluatedCubes (dimensionByPath / matcher) ... + const bakedEval = await bakedDimGranularities(api, 'OrdersView.created_at'); + expect(bakedEval.fiscal_year).toMatchObject({ interval: '1 year', origin: '2024-02-01' }); + // ... AND the symbols copy (CubeSymbols.resolveGranularity for SQL) — the load-bearing dual write. + const bakedSym = await symbolsDimGranularities(api, 'OrdersView', 'created_at'); + expect(bakedSym.fiscal_year).toMatchObject({ interval: '1 year', origin: '2024-02-01' }); + + // Meta advertises it on the view member. + const cubes = await api.metaConfig(ctxFor('a'), {}); + expect(granularityNames(dimByName(cubes, 'OrdersView.created_at'))).toContain('fiscal_year'); + + // Real SQL against the VIEW member at the global custom grain resolves and does not throw. + const { sql } = await api.getSql({ + measures: ['OrdersView.count'], + dimensions: [], + segments: [], + filters: [], + timeDimensions: [{ + dimension: 'OrdersView.created_at', + // Custom granularity name — allowed at runtime; the wire type only enumerates built-ins. + granularity: 'fiscal_year' as any, + dateRange: ['2024-01-01', '2025-01-01'], + }], + order: [], + } as any); + const sqlText = Array.isArray(sql) ? sql[0] : sql; + expect(typeof sqlText).toBe('string'); + expect(sqlText.length).toBeGreaterThan(0); + api.dispose(); + }); + }); + + describe('locals win over globals on name collision', () => { + // 9. A dimension-local custom with the same name as a global custom keeps the LOCAL definition + // after the merge (locals always win). + test('a local custom shadows a same-named global custom after bake', async () => { + const api = createApi({ + // Global fiscal_year has origin 2024-02-01; the local on Orders.collide_at / Events.ts uses + // a different origin and must win. + granularities: [{ name: 'fiscal_year', interval: '1 year', origin: '2024-02-01' }], + }); + await api.metaConfig(ctxFor('a'), {}); + + // Orders.collide_at declares local fiscal_year { origin: 2020-01-01 } -> local wins. + const collide = await bakedDimGranularities(api, 'Orders.collide_at'); + expect(collide.fiscal_year).toMatchObject({ origin: '2020-01-01' }); + + // Events.ts declares local fiscal_year { origin: 2024-02-01 } — same name, local definition + // preserved (not replaced by the global). + const eventsTs = await bakedDimGranularities(api, 'Events.ts'); + expect(eventsTs.fiscal_year).toMatchObject({ origin: '2024-02-01' }); + + // A plain dimension with no local fiscal_year takes the GLOBAL one. + const created = await bakedDimGranularities(api, 'Orders.created_at'); + expect(created.fiscal_year).toMatchObject({ origin: '2024-02-01' }); + api.dispose(); + }); + }); + + describe('composition with RBAC visibility', () => { + const rbacRepository: SchemaFileRepository = { + localPath: () => '/mock/path', + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'orders.js', + content: ` + cube('Orders', { + sql: 'SELECT * FROM orders', + measures: { count: { type: 'count' } }, + dimensions: { + created_at: { sql: 'created_at', type: 'time' }, + secret: { sql: 'secret', type: 'string' }, + }, + accessPolicy: [ + { + group: '*', + rowLevel: { allowAll: true }, + memberLevel: { includes: ['count', 'created_at'] }, + }, + ], + }); + `, + }, + ]), + }; + + // 10. compilerId still mixes the visibility mask; granularities no longer contribute a separate + // per-request hash (they live in compilerVersion / the base compilerId). So the composed + // compilerId is stable across requests with the same visibility and differs with the mask. + test('compilerId mixes the visibility mask; stable per visibility, differs across masks', async () => { + const api = new TestableCompilerApi(rbacRepository, mockDbType, { + logger: noopLogger, + granularities: ['year', 'month'], + contextToGroups: async (ctx: any) => (ctx.securityContext.tenant === 'admin' ? ['admin'] : []), + }); + + const base = (await (api as any).getCompilers()).compilerId; + + // RBAC hides `secret` for a non-admin; the composed compilerId differs from the base. + const nonAdmin = await api.metaConfig(ctxFor('user'), { includeCompilerId: true }); + const secret = dimByName(nonAdmin.cubes, 'Orders.secret'); + expect(secret.isVisible).toBe(false); + const createdAt = dimByName(nonAdmin.cubes, 'Orders.created_at'); + expect(createdAt.isVisible).toBe(true); + // Granularities are baked, not per-request — the effective set is still present. + expect(granularityNames(createdAt)).toEqual(['year', 'month']); + expect(nonAdmin.compilerId).not.toBe(base); + + // Same visibility mask across two requests -> identical composed compilerId. + const nonAdmin2 = await api.metaConfig(ctxFor('user2'), { includeCompilerId: true }); + expect(nonAdmin2.compilerId).toBe(nonAdmin.compilerId); + api.dispose(); + }); + }); +}); diff --git a/rust/cube/cubeorchestrator/src/transport.rs b/rust/cube/cubeorchestrator/src/transport.rs index a32ff801b7cc4..52d36bd0fddfc 100644 --- a/rust/cube/cubeorchestrator/src/transport.rs +++ b/rust/cube/cubeorchestrator/src/transport.rs @@ -164,7 +164,13 @@ pub type MembersMap = IndexMap; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GranularityMeta { pub name: String, + /// Serialized as `type`: "built-in" or "custom". Field is named `kind` to avoid Rust's keyword. + #[serde(rename = "type", skip_serializing_if = "Option::is_none")] + pub kind: Option, pub title: String, + /// d3-time-format string used by the client to display bucketed timestamps. + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, #[serde(skip_serializing_if = "Option::is_none")] pub interval: Option, #[serde(skip_serializing_if = "Option::is_none")]