refactor(granularities): resolve and apply global config in CubeSymbols instead of a bespoke CompilerApi compile flow

Author: igorlukanin

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -20,6 +20,7 @@ import { UserError } from './UserError';
 import { BaseQuery, PreAggregationDefinitionExtended } from '../adapter';
 import type { CubeValidator } from './CubeValidator';
 import type { ErrorReporter } from './ErrorReporter';
+import type { GlobalGranularitiesConfig } from './GlobalGranularitiesConfig';
 import { FinishedJoinTree } from './JoinGraph';
 
 export type SegmentDefinition = {
@@ -210,13 +211,14 @@ export class CubeEvaluator extends CubeSymbols {
   private isRbacEnabledCache: boolean | null = null;
 
   public constructor(
-    protected readonly cubeValidator: CubeValidator
+    protected readonly cubeValidator: CubeValidator,
+    granularitiesResolver?: () => Promise<GlobalGranularitiesConfig>,
   ) {
-    super(true);
+    super(true, granularitiesResolver);
   }
 
-  public compile(cubes: any[], errorReporter: ErrorReporter) {
-    super.compile(cubes, errorReporter);
+  public async compile(cubes: any[], errorReporter: ErrorReporter) {
+    await super.compile(cubes, errorReporter);
     const validCubes = this.cubeList.filter(cube => this.cubeValidator.isCubeValid(cube)).sort((a, b) => {
       if (a.isView) {
         return 1;

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts

@@ -5,7 +5,16 @@ import { camelize } from 'inflection';
 import { UserError } from './UserError';
 import { DynamicReference } from './DynamicReference';
 import { camelizeCube } from './utils';
-import { normalizeGranularitiesBlock, NormalizedGranularitiesBlock } from './GranularityResolver';
+import {
+  normalizeGranularitiesBlock,
+  NormalizedGranularitiesBlock,
+  resolveDimensionGranularities,
+} from './GranularityResolver';
+import {
+  buildBuiltInsCatalog,
+  DEFAULT_GRANULARITIES_CONFIG,
+  GlobalGranularitiesConfig,
+} from './GlobalGranularitiesConfig';
 
 import type { ErrorReporter } from './ErrorReporter';
 import { TranspilerSymbolResolver } from './transpilers';
@@ -297,13 +306,30 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface
 
   private resolveSymbolsCallContext: any;
 
-  public constructor(evaluateViews = false) {
+  // Resolves the global granularities config; shared across the compilers of one prepareCompiler so
+  // a `config.granularities` function runs once per compile.
+  private readonly granularitiesResolver?: () => Promise<GlobalGranularitiesConfig>;
+
+  // Global granularities resolved at the start of compile(); the default catalog until then.
+  private globalGranularities: GlobalGranularitiesConfig = DEFAULT_GRANULARITIES_CONFIG;
+
+  public constructor(
+    evaluateViews = false,
+    granularitiesResolver?: () => Promise<GlobalGranularitiesConfig>,
+  ) {
     this.symbols = {};
     this.builtCubes = {};
     this.cubeDefinitions = {};
     this.funcArgumentsValues = {};
     this.cubeList = [];
     this.evaluateViews = evaluateViews;
+    this.granularitiesResolver = granularitiesResolver;
+  }
+
+  // The resolved global config for this compile. Read by CubeToMetaTransformer (for
+  // `effectiveGranularities`) and by the /v1/granularities catalog.
+  public get globalGranularitiesConfig(): GlobalGranularitiesConfig {
+    return this.globalGranularities;
   }
 
   public free() {
@@ -315,7 +341,14 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface
     this.resolveSymbolsCallContext = undefined;
   }
 
-  public compile(cubes: CubeDefinition[], errorReporter: ErrorReporter) {
+  public async compile(cubes: CubeDefinition[], errorReporter: ErrorReporter) {
+    // Resolve the global granularities config before any cube is transformed: the merge below writes
+    // global customs onto time dimensions, and the SQL layer resolves them from that same map.
+    // The phase driver chains each compile() through `.then()`, so awaiting here is safe.
+    if (this.granularitiesResolver) {
+      this.globalGranularities = await this.granularitiesResolver();
+    }
+
     this.cubeDefinitions = Object.fromEntries(
       cubes.map((c): [string, CubeDefinition] => [c.name, c])
     );
@@ -609,28 +642,73 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface
     } as CubeSymbolsDefinition;
   }
 
-  // Stores the canonical `granularitiesBlock` on each time dimension and rewrites
-  // `granularities` to the dict of locally-defined customs only — preserving the legacy shape
-  // that BaseQuery, prepare-annotation, and CubeToMetaTransformer already read.
+  // Stores the canonical `granularitiesBlock` on each time dimension, rewrites `granularities` to
+  // the locally-defined customs — preserving the legacy shape that BaseQuery, prepare-annotation
+  // and CubeToMetaTransformer read — then layers the effective global customs on top so they are
+  // resolvable in SQL and visible to pre-aggregation matching.
   private normalizeDimensionGranularities(dimensions: Record<string, any> | undefined) {
     if (!dimensions) {
       return;
     }
 
+    // Every time dimension participates: global customs apply even to one that declares no
+    // `granularities` of its own.
     for (const dim of Object.values(dimensions)) {
-      // A view's included dimension already carries a propagated granularitiesBlock (with the
-      // source dimension's includes/excludes) alongside the custom-only `granularities` map.
-      // Re-normalizing the custom-only map here would reset includes to '*' and drop the source's
-      // includes/excludes, so only normalize dimensions that haven't been normalized yet.
-      if (dim && dim.type === 'time' && 'granularities' in dim && !dim.granularitiesBlock) {
-        // Keep the raw user value for the validator (it runs after this and would otherwise only
-        // see the extracted customs, never the includes/excludes/custom dict).
-        dim.rawGranularities = dim.granularities;
-        const block: NormalizedGranularitiesBlock = normalizeGranularitiesBlock(dim.granularities);
-        dim.granularitiesBlock = block;
-        dim.granularities = block.custom;
+      if (dim && dim.type === 'time') {
+        // A view's included dimension already carries a propagated granularitiesBlock (with the
+        // source dimension's includes/excludes) alongside the custom-only `granularities` map.
+        // Re-normalizing the custom-only map here would reset includes to '*' and drop the source's
+        // includes/excludes, so only normalize dimensions that haven't been normalized yet. A
+        // dimension declaring no `granularities` needs no block — it takes the global config as-is,
+        // and attaching one would leak an empty block into the compiled model.
+        if (!dim.granularitiesBlock && 'granularities' in dim) {
+          // Keep the raw user value for the validator (it runs after this and would otherwise only
+          // see the extracted customs, never the includes/excludes/custom dict).
+          dim.rawGranularities = dim.granularities;
+          const block: NormalizedGranularitiesBlock = normalizeGranularitiesBlock(dim.granularities);
+          dim.granularitiesBlock = block;
+          dim.granularities = block.custom;
+        }
+
+        this.mergeGlobalCustomsIntoDimension(dim);
+      }
+    }
+  }
+
+  // Layer the dimension's effective global customs onto its `granularities` map (locals win on a
+  // name collision). Reassigns rather than mutating, so a view dimension sharing its source's map
+  // by reference isn't contaminated.
+  private mergeGlobalCustomsIntoDimension(dim: any) {
+    const globalCustom = this.globalGranularities.customGranularities;
+    if (Object.keys(globalCustom).length === 0) {
+      return;
+    }
+
+    // `granularitiesBlock.custom` is the model's own customs, unaffected by the merge below.
+    const locals: Record<string, GranularityDefinition> = dim.granularitiesBlock?.custom ?? {};
+    const resolved = resolveDimensionGranularities(
+      dim.granularitiesBlock ?? normalizeGranularitiesBlock(undefined),
+      this.globalGranularities.enabledBuiltIns,
+      globalCustom,
+      buildBuiltInsCatalog(this.globalGranularities),
+    );
+
+    // Only genuine global customs are baked in: built-ins resolve by name without a definition, and
+    // a local of the same name already wins.
+    const merged: Record<string, GranularityDefinition> = {};
+    for (const [name, def] of Object.entries(resolved)) {
+      if (def.type === 'custom' &&
+        Object.prototype.hasOwnProperty.call(globalCustom, name) &&
+        !Object.prototype.hasOwnProperty.call(locals, name)
+      ) {
+        merged[name] = { ...def };
+        delete (merged[name] as any).type;
       }
     }
+
+    if (Object.keys(merged).length > 0) {
+      dim.granularities = { ...merged, ...locals };
+    }
   }
 
   private camelCaseTypes(obj: Object | Array<any> | undefined) {
@@ -1582,8 +1660,11 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface
       return { interval: `1 ${granName}` };
     }
 
-    // Custom granularities (local + baked-in globals) resolve from the compiled cube symbols.
-    return cube?.[dimName]?.[gr]?.[granName];
+    // A local custom wins; otherwise fall back to a global custom from the resolved config.
+    return cube?.[dimName]?.[gr]?.[granName] ??
+      (Object.prototype.hasOwnProperty.call(this.globalGranularities.customGranularities, granName)
+        ? this.globalGranularities.customGranularities[granName]
+        : undefined);
   }
 
   protected cubeDependenciesProxy(parentIndex, cubeName) {

packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts

@@ -23,8 +23,6 @@ import { resolveNamedNumericFormat, STANDARD_FORMAT_SPECIFIERS, DEFAULT_FORMAT_S
 import {
   EffectiveGranularity,
   NormalizedGranularitiesBlock,
-  ResolvedGranularitySet,
-  GRANULARITY_STRING_FIELDS,
   normalizeGranularitiesBlock,
   resolveDimensionGranularities,
   serializeEffectiveGranularities,
@@ -217,18 +215,12 @@ export class CubeToMetaTransformer implements CompilerInterface {
    */
   public queries: TransformedCube[];
 
-  // Resolved-once global granularities config for this appId, baked into the compiled model.
-  // CompilerApi resolves all config forms (env / static / function) before compile and passes the
-  // result here — the transformer never sees a function.
-  private readonly granularitiesConfig?: GlobalGranularitiesConfig;
-
   // Precomputed once in compile() so plain dimensions need no per-dimension resolution: `defaultSet`
-  // and `defaultGlobalCustoms` are shared by reference across every dimension without a local block.
+  // is shared by reference across every dimension without a local block.
   private granularityState!: {
     config: GlobalGranularitiesConfig;
     catalog: Record<string, GranularityDefinition>;
     defaultSet: EffectiveGranularity[];
-    defaultGlobalCustoms: Record<string, GranularityDefinition>;
   };
 
   public constructor(
@@ -237,44 +229,33 @@ export class CubeToMetaTransformer implements CompilerInterface {
     contextEvaluator: ContextEvaluator,
     viewGroupEvaluator: ViewGroupEvaluator,
     joinGraph: JoinGraph,
-    granularitiesConfig?: GlobalGranularitiesConfig
   ) {
     this.cubeValidator = cubeValidator;
     this.cubeSymbols = cubeEvaluator;
     this.cubeEvaluator = cubeEvaluator;
     this.contextEvaluator = contextEvaluator;
     this.viewGroupEvaluator = viewGroupEvaluator;
     this.joinGraph = joinGraph;
-    this.granularitiesConfig = granularitiesConfig;
     this.cubes = [];
     this.queries = [];
   }
 
-  // The resolved global config baked into this compiled model. Exposed for the /v1/granularities
-  // endpoint, which serves the per-appId catalog from the compiled model rather than re-resolving.
-  public get globalGranularitiesConfig(): GlobalGranularitiesConfig | undefined {
-    return this.granularityState?.config;
-  }
-
   public get viewGroups(): CompiledViewGroup[] {
     return this.viewGroupEvaluator.compiledViewGroups;
   }
 
   public compile(_cubes: any[], errorReporter: ErrorReporter): void {
-    // The config is already resolved (env / static / function) by CompilerApi at compile time and
-    // baked in here — a missing config means the default catalog.
-    const config = this.granularitiesConfig ?? DEFAULT_GRANULARITIES_CONFIG;
+    // CubeSymbols resolved the global config at the start of its own compile phase.
+    const config = this.cubeEvaluator.globalGranularitiesConfig ?? DEFAULT_GRANULARITIES_CONFIG;
     const catalog = buildBuiltInsCatalog(config);
-    // Resolve the no-local-block ("default") set once; every plain time dimension shares both its
-    // serialized wire form and its global-custom map by reference (no per-dimension resolution).
-    const defaultResolved = resolveDimensionGranularities(
-      normalizeGranularitiesBlock(undefined), config.enabledBuiltIns, config.customGranularities, catalog,
-    );
+    // Resolve the no-local-block ("default") set once; every plain time dimension shares the
+    // serialized wire form by reference (no per-dimension resolution).
     this.granularityState = {
       config,
       catalog,
-      defaultSet: serializeEffectiveGranularities(defaultResolved),
-      defaultGlobalCustoms: this.globalCustomsOf(defaultResolved, config, {}),
+      defaultSet: serializeEffectiveGranularities(resolveDimensionGranularities(
+        normalizeGranularitiesBlock(undefined), config.enabledBuiltIns, config.customGranularities, catalog,
+      )),
     };
 
     this.cubes = this.cubeSymbols.cubeList
@@ -359,11 +340,11 @@ export class CubeToMetaTransformer implements CompilerInterface {
           const dimensionVisibility = isCubeVisible
             ? this.isVisible(extendedDimDef, !extendedDimDef.primaryKey)
             : false;
-          // Snapshot the dimension's LOCAL customs before any merge below: the deprecated
-          // `granularities` meta field must keep listing only the model's own custom granularities.
-          const localCustoms = extendedDimDef.granularities;
-          const localCustomEntries = localCustoms ? Object.entries(localCustoms) : [];
           const { granularitiesBlock } = extendedDimDef as any;
+          // The deprecated `granularities` meta field lists only the model's own custom
+          // granularities — `granularitiesBlock.custom`, which the global merge leaves untouched.
+          const localCustoms = granularitiesBlock?.custom ?? extendedDimDef.granularities;
+          const localCustomEntries = localCustoms ? Object.entries(localCustoms) : [];
           const dimType = this.dimensionDataType(extendedDimDef.type || 'string');
           const dimFormat = this.transformDimensionFormat(extendedDimDef);
           const dimCurrency = extendedDimDef.currency?.toUpperCase();
@@ -372,23 +353,13 @@ export class CubeToMetaTransformer implements CompilerInterface {
           if (dimType === 'time') {
             const s = this.granularityState;
             const inputs = this.granularityInputsForDimension(cubeTitle, localCustoms, granularitiesBlock);
-            // Dimensions with a local block resolve individually; plain ones reuse the shared default
-            // (both the serialized set and the global-custom map) computed once in compile().
-            let globalCustoms: Record<string, GranularityDefinition>;
-            if (inputs) {
-              const resolved = resolveDimensionGranularities(
+            // Dimensions with a local block resolve individually; plain ones reuse the shared
+            // default set computed once in compile().
+            effectiveGranularities = inputs
+              ? serializeEffectiveGranularities(resolveDimensionGranularities(
                 inputs, s.config.enabledBuiltIns, s.config.customGranularities, s.catalog,
-              );
-              effectiveGranularities = serializeEffectiveGranularities(resolved);
-              globalCustoms = this.globalCustomsOf(resolved, s.config, localCustoms ?? {});
-            } else {
-              effectiveGranularities = s.defaultSet;
-              globalCustoms = s.defaultGlobalCustoms;
-            }
-
-            // Bake the effective GLOBAL customs into the dimension's `granularities` map (SQL resolves
-            // customs by name from this map, and pre-agg matching reads it). Locals win over globals.
-            this.mergeGlobalCustomsIntoDimension(cubeName, dimensionName, extendedDimDef, localCustoms, globalCustoms);
+              ))
+              : s.defaultSet;
           }
 
           return {
@@ -486,64 +457,6 @@ export class CubeToMetaTransformer implements CompilerInterface {
     return { includes: block.includes, excludes: block.excludes, custom };
   }
 
-  // From an already-resolved set, extract the GLOBAL customs a dimension exposes: entries that are
-  // custom, defined in the global config, and not shadowed by a local of the same name. Projected
-  // through GRANULARITY_STRING_FIELDS (the shared field list, so it can't drift from serialize/hash).
-  private globalCustomsOf(
-    resolved: ResolvedGranularitySet,
-    config: GlobalGranularitiesConfig,
-    localCustoms: Record<string, GranularityDefinition>,
-  ): Record<string, GranularityDefinition> {
-    const out: Record<string, GranularityDefinition> = {};
-    for (const [name, def] of Object.entries(resolved)) {
-      if (def.type === 'custom' &&
-        Object.prototype.hasOwnProperty.call(config.customGranularities, name) &&
-        !Object.prototype.hasOwnProperty.call(localCustoms, name)
-      ) {
-        const projected: GranularityDefinition = {} as GranularityDefinition;
-        for (const field of GRANULARITY_STRING_FIELDS) {
-          if (def[field] !== undefined) {
-            (projected as any)[field] = def[field];
-          }
-        }
-        out[name] = projected;
-      }
-    }
-    return out;
-  }
-
-  // Bake global customs into a dimension's `granularities` map (locals win). Must write BOTH the
-  // `dimDef` object (pre-agg matching) and the distinct `symbols[cube][dim]` object (SQL
-  // resolveGranularity) — writing one leaves the other unable to resolve the custom. Reassign, never
-  // mutate in place, so a view dim sharing its source's map by reference isn't contaminated.
-  private mergeGlobalCustomsIntoDimension(
-    cubeName: string,
-    dimensionName: string,
-    dimDef: ExtendedCubeSymbolDefinition,
-    localCustoms: Record<string, GranularityDefinition> | undefined,
-    globalCustoms: Record<string, GranularityDefinition>,
-  ): void {
-    if (Object.keys(globalCustoms).length === 0) {
-      return;
-    }
-    const hasLocals = !!localCustoms && Object.keys(localCustoms).length > 0;
-
-    // With no locals the baked map IS the shared globalCustoms — assign it by reference (every plain
-    // dimension then shares one object). Copy-on-write only when locals must be layered on top.
-    const write = (existing: Record<string, GranularityDefinition> | undefined) => (
-      existing && Object.keys(existing).length > 0
-        ? { ...globalCustoms, ...existing } // globals first, locals last so locals win on collisions
-        : globalCustoms
-    );
-
-    dimDef.granularities = write(hasLocals ? localCustoms : undefined);
-
-    const symbolDim = (this.cubeEvaluator as any).symbols?.[cubeName]?.[dimensionName];
-    if (symbolDim && symbolDim !== dimDef) {
-      symbolDim.granularities = write(symbolDim.granularities as Record<string, GranularityDefinition> | undefined);
-    }
-  }
-
   public queriesForContext(contextId: string | null | undefined): TransformedCube[] {
     // return All queries if no context pass
     if (contextId == null || contextId.length === 0) {

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -224,7 +224,7 @@ export class DataSchemaCompiler {
     try {
       return compileServices
         .map((compileService) => (() => compileService.compile(objects, errorsReport)))
-        .reduce((p, fn) => p.then(fn), Promise.resolve())
+        .reduce<Promise<void>>((p, fn) => p.then(() => fn()).then(() => undefined), Promise.resolve())
         .catch((error) => {
           errorsReport.error(error);
         });