diff --git a/docs-mintlify/admin/connect-to-data/oauth-authentication.mdx b/docs-mintlify/admin/connect-to-data/oauth-authentication.mdx
index c18f0e2ef4a23..290a9527aa805 100644
--- a/docs-mintlify/admin/connect-to-data/oauth-authentication.mdx
+++ b/docs-mintlify/admin/connect-to-data/oauth-authentication.mdx
@@ -26,6 +26,48 @@ Because every user connects with different credentials, you also need
per-user query orchestrator state. Without this, one user's cached
connection could leak to another.
+
+
+Cube caches one database connection per
+[`context_to_orchestrator_id`][ref-context-to-orchestrator-id]. **The
+orchestrator ID must therefore distinguish every user your
+`driver_factory` can return a different connection for** — otherwise two
+users share one pool and one user's credential is reused for another's
+queries.
+
+Do not add the token itself to the ID. When the token rotates, Cube
+notices that `driver_factory` now resolves a different configuration and
+rebuilds the connection in place, so a username is enough. Keying on the
+token instead creates a new orchestrator — with its own pool, queues and
+pre-aggregation cache — on every rotation.
+
+
+
+
+
+Rebuilding the connection needs a Cube version that supports it. If a rotated
+token is only picked up when you redeploy, yours predates it: the connection
+resolved by the first request is kept until the deployment restarts.
+
+On those versions, putting the credential's **expiry** in the orchestrator ID
+is the only way to pick up a rotation without restarting — at the cost of a
+fresh orchestrator per rotation. Add a coarse clock to it once that expiry has
+passed, for example `int(time.time() // 300)`, because a credential whose
+refresh is failing upstream stops rotating altogether: an ID that changes only
+with the credential would pin the dead connection for as long as the process
+runs.
+
+
+
+
+
+Return the resolved credential from `driver_factory`, not a function that
+fetches one. Cube compares the configuration values the factory returns,
+and a function compares as unchanged however the credential behind it
+rotates — the connection would never be rebuilt.
+
+
+
## Prerequisites
- A [Cube Cloud][ref-cube-cloud] deployment connected to an
@@ -152,35 +194,89 @@ value with the correct `type` and driver-specific options. See the
```python cube.py
from cube import config
+from datetime import datetime, timezone
import os
+import time
+
+# A token is handed to the driver once, but the pool keeps opening new sessions
+# with it afterwards. Reject one that is too close to expiry to survive that
+# gap, rather than one that is merely still valid at this instant.
+EXPIRY_SKEW_SECONDS = 120
+
+
+def _parse_expiry(value):
+ """Seconds since the epoch, or None if the value is absent or unparseable."""
+ if not value:
+ return None
+ if isinstance(value, (int, float)):
+ # Epoch milliseconds if the value is far too large to be seconds.
+ return value / 1000 if value > 1e11 else float(value)
+ try:
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ except ValueError:
+ return None
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=timezone.utc)
+ return parsed.timestamp()
+
+
+def _credentials(ctx: dict) -> dict:
+ """The user's credentials for this data source, or an empty dict."""
+ # For other data sources, swap "databricks" for "snowflake", etc.
+ cube_cloud = (ctx.get("securityContext") or {}).get("cubeCloud") or {}
+
+ return (cube_cloud.get("userCredentials") or {}).get("databricks") or {}
+
+
+def _access_token(ctx: dict):
+ """The user's OAuth token, or None to fall back to the service account."""
+ creds = _credentials(ctx)
+
+ access_token = creds.get("accessToken")
+ if not access_token:
+ return None
+
+ expires_at = _parse_expiry(creds.get("accessTokenExpiresAt"))
+
+ # Gate on the expiry rather than on `status`: a failed background refresh
+ # can flag the record while the token already in hand is still valid, and
+ # treating that as fatal drops the user onto the service account for no
+ # reason.
+ if expires_at is not None:
+ return access_token if expires_at > time.time() + EXPIRY_SKEW_SECONDS else None
+
+ # No expiry advertised, or one that cannot be read, so there is nothing to
+ # compare against and `status` is all there is to go on. Requiring an expiry
+ # here would send every user of a provider that omits it to the service
+ # account.
+ return access_token if creds.get("status") == "active" else None
@config("driver_factory")
def driver_factory(ctx: dict) -> dict:
- # Extract the Cube Cloud security context, which contains
- # per-user OAuth credentials when available.
- # For other data sources, swap "databricks" for "snowflake", etc.
- databricks_creds = (
- ctx
- .get("securityContext", {})
- .get("cubeCloud", {})
- .get("userCredentials", {})
- .get("databricks", {})
- )
-
- # Only use the OAuth token when the credential status is "active".
- # An expired or revoked token falls back to the service account.
- oauth_token = (
- databricks_creds.get("accessToken")
- if databricks_creds.get("status") == "active"
- else None
- )
-
+ access_token = _access_token(ctx)
+ creds = _credentials(ctx)
+
+ # Cube rebuilds this connection whenever the returned configuration
+ # changes, so returning a rotated token here is enough to replace it.
+ #
+ # `expiresAt` covers what a comparison cannot see. A credential whose
+ # refresh is failing upstream stops rotating: it resolves to an identical
+ # configuration indefinitely while the connection built from it is already
+ # dead. Stating the deadline replaces that connection anyway.
+ #
+ # Only for the user's own token — the service-account credential below has
+ # no deadline of its own, and giving it the user's would rebuild the
+ # fallback connection forever.
return {
"type": "databricks-jdbc",
"url": os.environ["CUBEJS_DB_DATABRICKS_URL"],
- # Prefer the user's OAuth token; fall back to the service account token
- "token": oauth_token or os.environ["CUBEJS_DB_DATABRICKS_TOKEN"],
+ "token": access_token or os.environ["CUBEJS_DB_DATABRICKS_TOKEN"],
+ **(
+ {"expiresAt": creds.get("accessTokenExpiresAt")}
+ if access_token
+ else {}
+ ),
"acceptPolicy": True,
"catalog": os.environ.get("CUBEJS_DB_DATABRICKS_CATALOG"),
}
@@ -188,14 +284,12 @@ def driver_factory(ctx: dict) -> dict:
@config("context_to_orchestrator_id")
def context_to_orchestrator_id(ctx: dict) -> str:
- # Give each user a separate orchestrator instance (DB connections,
- # execution queues, pre-aggregation caches)
- username = (
- ctx
- .get("securityContext", {})
- .get("cubeCloud", {})
- .get("username", "default")
- )
+ # One orchestrator per user: separate DB connections, execution queues and
+ # pre-aggregation caches. Deliberately not keyed on the token — see the
+ # warning above.
+ cube_cloud = (ctx.get("securityContext") or {}).get("cubeCloud") or {}
+ username = cube_cloud.get("username") or "default"
+
return f"CUBE_APP_{username}"
```
@@ -204,37 +298,74 @@ def context_to_orchestrator_id(ctx: dict) -> str:
```javascript cube.js
+// A token is handed to the driver once, but the pool keeps opening new sessions
+// with it afterwards. Reject one that is too close to expiry to survive that
+// gap, rather than one that is merely still valid at this instant.
+const EXPIRY_SKEW_MS = 120 * 1000;
+
+/** The user's OAuth token, or undefined to fall back to the service account. */
+function accessToken(securityContext) {
+ // For other data sources, swap `databricks` for `snowflake`, etc.
+ const creds = securityContext?.cubeCloud?.userCredentials?.databricks ?? {};
+
+ if (!creds.accessToken) {
+ return undefined;
+ }
+
+ const raw = creds.accessTokenExpiresAt;
+ // Epoch milliseconds if the value is far too large to be seconds. Reading
+ // seconds as milliseconds would land in 1970 and reject every token.
+ const expiresAt =
+ typeof raw === "number"
+ ? (raw > 1e11 ? raw : raw * 1000)
+ : Date.parse(raw ?? "");
+
+ // Gate on the expiry rather than on `status`: a failed background refresh can
+ // flag the record while the token already in hand is still valid, and
+ // treating that as fatal drops the user onto the service account for no
+ // reason.
+ if (!Number.isNaN(expiresAt)) {
+ return expiresAt > Date.now() + EXPIRY_SKEW_MS ? creds.accessToken : undefined;
+ }
+
+ // No expiry advertised, or one that cannot be read, so there is nothing to
+ // compare against and `status` is all there is to go on. Requiring an expiry
+ // here would send every user of a provider that omits it to the service
+ // account.
+ return creds.status === "active" ? creds.accessToken : undefined;
+}
+
module.exports = {
+ // Cube rebuilds this connection whenever the returned configuration changes,
+ // so returning a rotated token here is enough to replace it.
+ //
+ // `expiresAt` covers what a comparison cannot see. A credential whose refresh
+ // is failing upstream stops rotating: it resolves to an identical
+ // configuration indefinitely while the connection built from it is already
+ // dead. Stating the deadline replaces that connection anyway.
+ //
+ // Only for the user's own token — the service-account credential below has no
+ // deadline of its own, and giving it the user's would rebuild the fallback
+ // connection forever.
driverFactory: ({ securityContext }) => {
- // Extract the Cube Cloud security context, which contains
- // per-user OAuth credentials when available.
- // For other data sources, swap `databricks` for `snowflake`, etc.
- const databricksCreds =
- securityContext?.cubeCloud?.userCredentials?.databricks ?? {};
-
- // Only use the OAuth token when the credential status is "active".
- // An expired or revoked token falls back to the service account.
- const oauthToken =
- databricksCreds.status === "active"
- ? databricksCreds.accessToken
- : null;
+ const token = accessToken(securityContext);
+ const creds = securityContext?.cubeCloud?.userCredentials?.databricks;
return {
type: "databricks-jdbc",
url: process.env.CUBEJS_DB_DATABRICKS_URL,
- // Prefer the user's OAuth token; fall back to the service account token
- token: oauthToken || process.env.CUBEJS_DB_DATABRICKS_TOKEN,
+ token: token ?? process.env.CUBEJS_DB_DATABRICKS_TOKEN,
+ ...(token ? { expiresAt: creds?.accessTokenExpiresAt } : {}),
acceptPolicy: true,
catalog: process.env.CUBEJS_DB_DATABRICKS_CATALOG,
};
},
- // Give each user a separate orchestrator instance (DB connections,
- // execution queues, pre-aggregation caches)
- contextToOrchestratorId: ({ securityContext }) => {
- const username = securityContext?.cubeCloud?.username ?? "default";
- return `CUBE_APP_${username}`;
- },
+ // One orchestrator per user: separate DB connections, execution queues and
+ // pre-aggregation caches. Deliberately not keyed on the token — see the
+ // warning above.
+ contextToOrchestratorId: ({ securityContext }) =>
+ `CUBE_APP_${securityContext?.cubeCloud?.username ?? "default"}`,
};
```
@@ -248,21 +379,77 @@ module.exports = {
credentials to `securityContext.cubeCloud.userCredentials.`
(for example, `.databricks` or `.snowflake`).
-2. **`driver_factory` resolves the credential** — If the credential status
- is `active`, the user's OAuth token is used. Otherwise, Cube falls back
- to the service account credential stored in environment variables.
+2. **`driver_factory` resolves the credential** — If the user has a token
+ that has not expired, it is used. Otherwise, Cube falls back to the
+ service account credential stored in environment variables.
3. **Per-user orchestrator** —
- [`context_to_orchestrator_id`][ref-context-to-orchestrator-id] returns
- a unique key per username, so each user gets their own database
+ [`context_to_orchestrator_id`][ref-context-to-orchestrator-id] returns a
+ key derived from the username, so each user gets their own database
connection pool, execution queues, and pre-aggregation table cache.
- Without this, Cube would share a single cached connection across all
- users, causing one user's credentials to be reused for another user's
- queries.
+ Without it, every user shares one cached connection and the first
+ user's credential is reused for everyone else's queries.
+
+4. **Rotation replaces the connection** — on the next request after a
+ rotation, Cube compares what `driver_factory` now resolves against what
+ the cached connection was built from. When they differ it builds a
+ replacement and drains the old pool, so in-flight queries finish on the
+ connection they started on.
+
+5. **A stated lifetime replaces it even when nothing changed** — `expiresAt`
+ is enforced on its own, so a credential that stops rotating rather than
+ rotating to something new does not pin a dead connection. It is excluded
+ from the comparison in step 4, so a factory may recompute the deadline on
+ every call without that reading as a new connection.
+
+## Operational notes
+
+- **Replacements are rate-limited.** A data source is replaced at most once
+ every 30 seconds, and inside that window the current connection is reused
+ without asking `driver_factory` again. A credential that rotates hourly is
+ never held up by this; one that resolves to a different value on every call
+ degrades to reuse instead of reconnecting on every query.
+
+- **One orchestrator per user, not per token.** The orchestrator survives
+ rotations, so pre-aggregation caches and queues stay warm and the
+ orchestrator count tracks your concurrent user count rather than growing
+ with every rotation.
+- **Don't make [`context_to_app_id`][ref-context-to-appid] per-user.** The
+ data model is identical for every user — only the connection differs —
+ so a per-user app ID forces a full data-model recompile per user on
+ every replica for no benefit. Leave it unset, or return a constant if
+ your deployment already sets one.
+- **Give the service account the minimum it needs to pass a connection
+ check.** If it has no access at all, liveness checks and any query that
+ falls back to it fail with an opaque authorization error from the driver
+ rather than something diagnosable.
+- **A credential that stops refreshing is the case to plan for.** Rotation is
+ the easy half: the configuration changes and the connection is replaced. A
+ refresh that starts failing upstream leaves the same credential in place
+ until it expires and beyond, which no comparison can distinguish from a
+ credential nobody has touched. Return `expiresAt` and the connection is
+ replaced on its own deadline; a factory that raises instead of falling back
+ is given up after it has refused for long enough, so the error reaches the
+ caller rather than 401s from a pool nobody rebuilds.
+- **A `driver_factory` that reaches an external dependency should catch its
+ own failures.** Sustained refusal is read as the factory declining to serve
+ the connection: after several minutes of raising on every check, the pooled
+ connection is released and the error surfaces to the caller. That is the
+ intended behaviour for a credential that has genuinely stopped working, but
+ nothing here can tell it apart from a secret store that is briefly
+ unreachable. If yours fetches from Vault, a token endpoint or anything else
+ that can blink, catch the error and return the last known-good credential
+ (or the service account) rather than letting it propagate.
+- **Falling back is silent.** A missing or near-expired token sends the
+ query to the service account instead of failing, so results reflect the
+ service account's permissions rather than the user's. If that is not
+ acceptable for your deployment, raise an error in `driver_factory`
+ instead of returning the fallback credential.
[ref-config]: /reference/configuration/config
[ref-driver-factory]: /reference/configuration/config#driver_factory
[ref-context-to-orchestrator-id]: /reference/configuration/config#context_to_orchestrator_id
+[ref-context-to-appid]: /reference/configuration/config#context_to_app_id
[ref-databricks-jdbc]: /admin/connect-to-data/data-sources/databricks-jdbc
[ref-snowflake]: /admin/connect-to-data/data-sources/snowflake
[ref-data-sources]: /admin/connect-to-data/data-sources
diff --git a/docs-mintlify/reference/configuration/config.mdx b/docs-mintlify/reference/configuration/config.mdx
index db729738c9849..68be24f52284f 100644
--- a/docs-mintlify/reference/configuration/config.mdx
+++ b/docs-mintlify/reference/configuration/config.mdx
@@ -442,6 +442,35 @@ in the drivers' [source code][link-github-cube-drivers].
+The optional `expiresAt` element states when the connection this configuration
+describes stops being usable, as epoch milliseconds (or seconds) or an ISO 8601
+string — and in JavaScript, a `Date`. In Python, pass `dt.timestamp()` or
+`dt.isoformat()`: a `datetime` cannot cross the config bridge and raises
+`Unable to represent PyDateTime type as CLR from Python`. It is not passed to
+the driver.
+
+A deadline that has already passed is ignored, with a warning: replacing a
+driver cannot move a deadline the factory keeps re-asserting. So is one that
+falls inside the interval replacements are rate-limited to, since a lifetime
+that short could only ever be honoured by replacing the connection on every
+window. A credential rotating that fast is picked up by its configuration
+changing instead. Both are judged when `driver_factory` states the deadline, so
+a lifetime that was accepted is held to whatever it said.
+
+A driver is resolved once and then cached, and replaced when the configuration
+`driver_factory` returns changes. A connection built from a credential that
+*stops* rotating — an expired token whose refresh is failing — is the case that
+leaves behind: the factory keeps resolving an identical configuration while the
+connection is already dead. `expiresAt` states the lifetime outright, so the
+connection is replaced when it elapses whether or not anything else changed.
+
+It is deliberately excluded from the comparison, so a factory may return a
+deadline recomputed on every call without that reading as a new connection.
+
+
+
+
+
A custom `driver_factory` takes precedence over the [`CUBEJS_PRE_AGGREGATIONS_*`
environment variables][ref-preagg-data-source], for every data source rather than only the
ones it handles. If both are configured, the `driver_factory` connection is used for
diff --git a/packages/cubejs-server-core/src/core/driver-config-expiry.ts b/packages/cubejs-server-core/src/core/driver-config-expiry.ts
new file mode 100644
index 0000000000000..c6769171a7e35
--- /dev/null
+++ b/packages/cubejs-server-core/src/core/driver-config-expiry.ts
@@ -0,0 +1,83 @@
+/**
+ * @copyright Cube Dev, Inc.
+ * @license Apache-2.0
+ * @fileoverview The optional lifetime a `driverFactory` can put on the
+ * configuration it returns.
+ */
+
+import type { DriverConfig } from './types';
+
+/**
+ * Above this a number is read as epoch milliseconds, below it as epoch seconds.
+ * 1e11 milliseconds is 1973 and 1e11 seconds is the year 5138, so nothing
+ * anyone can mean today is ambiguous. Both spellings are accepted because the
+ * value is usually copied straight off a credential, and the ecosystems
+ * disagree: JavaScript counts milliseconds, Python's `time.time()` seconds.
+ */
+const MILLISECONDS_THRESHOLD = 1e11;
+
+/**
+ * A driver lifetime as a POSIX timestamp in milliseconds, or undefined when
+ * none was given or the value cannot be read as a moment in time.
+ *
+ * Unreadable input is dropped rather than rejected: an expiry is an
+ * optimisation over comparing configurations, and failing a deployment's
+ * queries over a malformed one would be a worse outcome than the behaviour it
+ * had before the field existed.
+ */
+export function parseDriverExpiry(value: unknown): number | undefined {
+ if (value === undefined || value === null) {
+ return undefined;
+ }
+
+ if (value instanceof Date) {
+ const time = value.getTime();
+
+ return Number.isNaN(time) ? undefined : time;
+ }
+
+ if (typeof value === 'number') {
+ if (!Number.isFinite(value) || value <= 0) {
+ return undefined;
+ }
+
+ return value > MILLISECONDS_THRESHOLD ? value : value * 1000;
+ }
+
+ if (typeof value === 'string') {
+ const text = value.trim();
+
+ if (!text) {
+ return undefined;
+ }
+
+ // A stringified timestamp, which `Date.parse` would read as a year.
+ if (/^\d+(\.\d+)?$/.test(text)) {
+ return parseDriverExpiry(Number(text));
+ }
+
+ const parsed = Date.parse(text);
+
+ return Number.isNaN(parsed) ? undefined : parsed;
+ }
+
+ return undefined;
+}
+
+/**
+ * The same configuration without its lifetime, for the two places that must not
+ * see it: the fingerprint that decides whether the connection changed, and the
+ * options handed to the driver's own constructor.
+ *
+ * Returns the input untouched when there is nothing to strip, so the common
+ * case allocates nothing.
+ */
+export function withoutDriverExpiry(config: DriverConfig): DriverConfig {
+ if (!config || typeof config !== 'object' || !('expiresAt' in config)) {
+ return config;
+ }
+
+ const { expiresAt: _lifetime, ...rest } = config;
+
+ return rest;
+}
diff --git a/packages/cubejs-server-core/src/core/driver-config-fingerprint.ts b/packages/cubejs-server-core/src/core/driver-config-fingerprint.ts
new file mode 100644
index 0000000000000..f1c4e963a0149
--- /dev/null
+++ b/packages/cubejs-server-core/src/core/driver-config-fingerprint.ts
@@ -0,0 +1,105 @@
+/**
+ * @copyright Cube Dev, Inc.
+ * @license Apache-2.0
+ * @fileoverview Fingerprinting for driver configurations and security contexts.
+ */
+
+import crypto from 'crypto';
+
+/**
+ * Deterministic JSON used for fingerprinting. Object keys are emitted in sorted
+ * order so two structures that differ only in property order hash the same, and
+ * values JSON cannot represent are reduced to stable placeholders rather than
+ * silently disappearing. Throws on a circular structure, which callers treat as
+ * "not fingerprintable".
+ */
+function stableStringify(value: unknown, seen: Set): string {
+ if (value === undefined || value === null) {
+ return 'null';
+ }
+
+ const type = typeof value;
+
+ if (type === 'string' || type === 'number' || type === 'boolean') {
+ return JSON.stringify(value);
+ }
+
+ if (type === 'bigint') {
+ return JSON.stringify((value as bigint).toString());
+ }
+
+ // A closure's identity cannot be compared meaningfully across calls, so it
+ // contributes a constant. Two configs differing only in a function body are
+ // therefore treated as equal — deliberately conservative: it can only lead to
+ // reusing a connection, never to swapping one out unnecessarily.
+ //
+ // The practical consequence is that a config carrying its credential as a
+ // provider callback rather than a resolved value fingerprints identically
+ // however the credential rotates, so such a driver is never rebuilt. A
+ // `driverFactory` that needs rotation to be noticed has to return the
+ // resolved value.
+ if (type === 'function' || type === 'symbol') {
+ return JSON.stringify(`[${type}]`);
+ }
+
+ if (value instanceof Date) {
+ return JSON.stringify(value.toISOString());
+ }
+
+ if (seen.has(value)) {
+ throw new Error('Circular structure cannot be fingerprinted');
+ }
+
+ seen.add(value);
+
+ try {
+ if (Array.isArray(value)) {
+ return `[${value.map((item) => stableStringify(item, seen)).join(',')}]`;
+ }
+
+ // Own enumerable keys only, so a class instance holding its values behind
+ // prototype accessors fingerprints as `{}` — constant, and therefore another
+ // shape whose rotation goes unnoticed. Plain configs are unaffected.
+ const entries = Object.keys(value as Record)
+ .sort()
+ .reduce((acc, key) => {
+ const entry = (value as Record)[key];
+
+ // Match JSON.stringify: undefined-valued properties are absent, so
+ // `{ a: undefined }` and `{}` fingerprint the same.
+ if (entry !== undefined) {
+ acc.push(`${JSON.stringify(key)}:${stableStringify(entry, seen)}`);
+ }
+
+ return acc;
+ }, []);
+
+ return `{${entries.join(',')}}`;
+ } finally {
+ seen.delete(value);
+ }
+}
+
+/**
+ * A short, stable digest of `value`, or `null` when it cannot be fingerprinted.
+ *
+ * Hashed rather than kept verbatim because the values being compared include
+ * database passwords and OAuth access tokens: a raw copy would live for the
+ * lifetime of the process and surface in any heap dump. `null` means "cannot
+ * tell whether this changed", and every caller must treat that as "assume it
+ * did not" so behaviour falls back to the previous resolve-once semantics.
+ */
+export function fingerprint(value: unknown): string | null {
+ try {
+ return crypto
+ .createHash('sha256')
+ .update(stableStringify(value, new Set()))
+ // 32 hex chars = 128 bits, which is far more than an equality check over
+ // the handful of configurations one process resolves needs, and keeps the
+ // digest short enough to sit in a log line.
+ .digest('hex')
+ .slice(0, 32);
+ } catch (e) {
+ return null;
+ }
+}
diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts
index 48b146c8abb70..57b7b68938056 100644
--- a/packages/cubejs-server-core/src/core/server.ts
+++ b/packages/cubejs-server-core/src/core/server.ts
@@ -37,6 +37,8 @@ import { agentCollect } from './agentCollect';
import { OrchestratorStorage } from './OrchestratorStorage';
import { createLogger } from './logger';
import { OptsHandler } from './OptsHandler';
+import { fingerprint } from './driver-config-fingerprint';
+import { parseDriverExpiry, withoutDriverExpiry } from './driver-config-expiry';
import {
driverDependencies,
lookupDriverClass,
@@ -74,6 +76,219 @@ import {
const { version } = require('../../../package.json');
+/**
+ * Rebuilds of one data source's driver before the log escalates to naming a
+ * likely misconfiguration. A rotating credential rebuilds a few times a day, so
+ * reaching this within a process means contexts are displacing each other.
+ */
+const DRIVER_REBUILD_WARN_THRESHOLD = 50;
+
+/**
+ * How many times one request will retry after losing the race to rebuild a
+ * driver before settling for whatever is cached. Bounds the work a single
+ * request can be made to do when contexts keep displacing each other's driver.
+ */
+const MAX_DRIVER_REBUILD_ATTEMPTS = 3;
+
+/**
+ * How long a data source keeps a freshly rebuilt driver before another
+ * configuration change may replace it. Inside the window the cached driver is
+ * reused without even asking the factory, exactly as before this file learned
+ * to rebuild at all.
+ *
+ * A rebuild tears down a connection pool, so the rate has to be bounded by
+ * something other than how often contexts happen to differ. Without this, a
+ * deployment whose `driverFactory` returns a configuration that is not stable
+ * across calls — a credential minted per call, say, or one carrying a nonce —
+ * would rebuild on request after request. That deployment works today, because
+ * the driver is resolved once and the difference is never noticed; it must not
+ * be turned into pool churn.
+ *
+ * The cost is that a credential which rotates twice inside one window is picked
+ * up a window late rather than immediately. At half a minute against
+ * credentials that live for an hour, that is not a tradeoff worth agonising
+ * over — and the alternative, before this change, was "not until redeploy".
+ */
+const DRIVER_REBUILD_MIN_INTERVAL_MS = 30 * 1000;
+
+/**
+ * Separate refusal incidents, spanning the grace window, before the cached
+ * driver is given up rather than reused. One of the two routes to a give-up —
+ * see `PROBE_FAILURE_GRACE_MS` for the other, and `PROBE_FAILURE_COALESCE_MS`
+ * for why the unit is incidents rather than failed checks.
+ *
+ * A probe failure is the factory declining to produce a connection for this
+ * context. One is transient — a secret store blinking, a timeout — and reusing
+ * what is cached is right. Sustained refusal is not: a `driverFactory` written
+ * to fail closed on an unusable credential is stating that this connection must
+ * not serve queries, and honouring that only when the factory happens to return
+ * is how an expired credential goes on serving errors from a pool nobody
+ * rebuilds.
+ */
+const MAX_PROBE_FAILURE_INCIDENTS = 3;
+
+/**
+ * How long refusal has to go on before the driver is given up — both how long
+ * repeated incidents must span, and how long a single unbroken one must run.
+ *
+ * Those are the two routes, and each covers a traffic profile the other cannot
+ * reach. Where probes are sparse every refusal stands alone, so the count is
+ * what accumulates. Where they are dense they coalesce into one incident that
+ * never ends, and only its duration distinguishes it from a blink.
+ *
+ * Note what this means, because it is the thing to check before assuming
+ * otherwise: one continuous dependency outage *is* enough, if it lasts. That is
+ * deliberate — it is the same call this bound made when the window was widened
+ * to minutes, and the recipe tells a `driver_factory` reaching an external
+ * dependency to catch its own failures rather than propagate them.
+ *
+ * Minutes rather than seconds because a probe failure is not evidence about the
+ * cached connection — it is evidence about whatever the factory had to reach to
+ * answer. A secret store restarting, a token endpoint returning 503, a DNS blip
+ * inside the factory: in every one of those the cached credential is untouched
+ * and still valid, and giving the pool up makes a dependency's outage into a
+ * query outage. A credential that has genuinely stopped working is not urgent
+ * to the second, so the bar is set where a dependency can restart under it.
+ */
+const PROBE_FAILURE_GRACE_MS = 5 * 60 * 1000;
+
+/**
+ * How long one refusal stays on the record before it is forgotten.
+ *
+ * Deliberately separate from the grace window, because the two pull opposite
+ * ways. The grace window wants to be long, so a dependency can restart under
+ * it. Retention wants to be long enough that a *sparse* deployment can still
+ * reach the bound: probes are only issued when the security context fingerprint
+ * changes, so a few-user deployment may probe once every several minutes, and
+ * if a refusal expired at the grace window such a deployment would reset to one
+ * every time and never give up a credential however permanently dead it was.
+ *
+ * Longer than the grace window, then, but far short of the days-apart flakes
+ * that made a never-expiring record wrong: a refusal half an hour stale is not
+ * evidence about the one happening now.
+ *
+ * Retention this long would, on its own, let two unrelated blinks half an hour
+ * apart reach the bound between them. What keeps that from happening is that
+ * the count is of incidents rather than of refusals — see
+ * `PROBE_FAILURE_COALESCE_MS`.
+ */
+const PROBE_FAILURE_RETENTION_MS = 30 * 60 * 1000;
+
+/**
+ * How close together two refusals have to be to count as one.
+ *
+ * Retention outliving the grace window is what makes the bound reachable in a
+ * sparse deployment, but on its own it also makes it reachable across unrelated
+ * incidents: concurrent probes all fail on one blink of a dependency, and a
+ * burst of three plus a single refusal six minutes later would otherwise
+ * satisfy both conditions and drain a working pool for what was two brief
+ * outages.
+ *
+ * Counting incidents rather than refusals removes that without giving the
+ * sparse case back: a burst is one, and the bound still wants three. An
+ * incident is bounded by its own duration as well as by the count, so a
+ * refusal stream arriving faster than this window is caught by having lasted
+ * rather than by being counted.
+ */
+const PROBE_FAILURE_COALESCE_MS = 2 * 1000;
+
+/**
+ * What a cached driver was built from. `null` on either fingerprint means
+ * "cannot tell whether it changed", which is always read as "assume it did
+ * not"; `expiresAt` is undefined when the configuration named no lifetime.
+ */
+type DriverOrigin = {
+ securityContextFingerprint: string | null;
+ configFingerprint: string | null;
+ expiresAt: number | undefined;
+ /**
+ * Whether this driver's unusable lifetime has been reported. The carry-over
+ * that resolves it runs on every security context change, with no rate limit
+ * of its own, so without this the warning is emitted per request — trading
+ * the pool churn this guard removes for log volume on the hot path.
+ */
+ lifetimeIgnoredReported: boolean;
+};
+
+/**
+ * Probe failures for one alias set inside one rolling window: how many, when
+ * the window opened, and when it was last extended.
+ *
+ * `lastFailureAt` is what makes the window rolling, against
+ * `PROBE_FAILURE_RETENTION_MS`. Probes are only issued when the security
+ * context fingerprint changes, so in a quiet deployment two of them can be
+ * hours apart with nothing in between to clear the count — and three unrelated
+ * flakes on three different days are not a sustained refusal, however they look
+ * to a counter that only ever goes up.
+ */
+type DriverProbeFailures = {
+ count: number;
+ firstFailureAt: number;
+ lastFailureAt: number;
+ /**
+ * When the incident being counted began — the first refusal of the current
+ * unbroken run, rather than of the window. An incident that has itself lasted
+ * the grace window is no longer a blink, which is what keeps a continuous
+ * stream of refusals from coalescing into one uncountable incident forever.
+ */
+ incidentStartedAt: number;
+};
+
+/** A `driverFactory` result together with the context that produced it. */
+type DriverFactoryResult = {
+ value: DriverConfig | BaseDriver;
+ securityContextFingerprint: string | null;
+};
+
+/** Why a cached driver was found stale, for the operator reading the log. */
+type DriverStalenessReason = 'configuration change' | 'lifetime elapsed';
+
+/**
+ * Every reason a cached driver is replaced. A refusal is not a staleness
+ * verdict — the factory never produced a configuration to compare — but it
+ * tears down the same connection pool, so it is counted and rate-limited
+ * alongside the verdicts rather than slipping past both brakes.
+ */
+type DriverReplacementReason =
+ | DriverStalenessReason
+ | 'repeated staleness check failures';
+
+/**
+ * The verdict on a cached driver. `factoryResult` is present only when the
+ * probe already resolved one, so the rebuild does not call the factory twice;
+ * `probeFailed` marks the reuse that happened because the factory threw, which
+ * the caller counts.
+ */
+type DriverStaleness =
+ | { stale: false, probeFailed?: boolean }
+ | { stale: true, reason: DriverStalenessReason, factoryResult?: DriverFactoryResult };
+
+/** Rebuild history of the one driver an alias set resolves to. */
+type DriverRebuildState = {
+ count: number;
+ lastRebuildAt: number;
+ /**
+ * Whether the suppression window opened by that rebuild has already been
+ * logged. Reset by each rebuild, so a thrashing deployment reports once per
+ * window rather than once per query.
+ */
+ suppressionReported: boolean;
+};
+
+/**
+ * Fingerprint of everything in a driver configuration that identifies the
+ * connection — which is all of it except the lifetime.
+ *
+ * The lifetime is excluded deliberately. It is enforced on its own, and it is
+ * the one field a factory is expected to return a different value for on every
+ * call, being a deadline recomputed from whatever credential it just read.
+ * Including it would read each of those calls as a changed connection and
+ * rebuild the pool on a timer.
+ */
+function driverConfigFingerprint(value: DriverConfig): string | null {
+ return fingerprint(withoutDriverExpiry(value));
+}
+
function wrapToFnIfNeeded(possibleFn: T | ((a: R) => T)): (a: R) => T {
if (typeof possibleFn === 'function') {
return possibleFn;
@@ -130,6 +345,19 @@ export class CubejsServerCore {
protected readonly orchestratorStorage: OrchestratorStorage = new OrchestratorStorage();
+ /**
+ * The request context each cached orchestrator most recently served.
+ *
+ * An orchestrator's driver factory closes over the context of the request
+ * that created it, and the driver it resolves is then cached for the life of
+ * the process. When that driver's configuration is derived from the context —
+ * a per-user OAuth token, say — it goes stale the moment the credential
+ * rotates. Tracking the latest context lets the factory notice. Keyed by the
+ * api instance so an entry disappears with the orchestrator it belongs to.
+ */
+ protected readonly orchestratorRequestContexts =
+ new WeakMap();
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
protected repositoryFactory: ((context: RequestContext) => SchemaFileRepository) | (() => FileRepository);
@@ -573,9 +801,22 @@ export class CubejsServerCore {
const orchestratorId = await this.contextToOrchestratorId(context);
if (this.orchestratorStorage.has(orchestratorId)) {
- return this.orchestratorStorage.get(orchestratorId);
+ const cachedOrchestratorApi = this.orchestratorStorage.get(orchestratorId);
+ const cachedContextRef = this.orchestratorRequestContexts.get(cachedOrchestratorApi);
+
+ // Keep the driver factory's view of the request context current. Without
+ // this it stays pinned to whichever request happened to create the
+ // orchestrator, and a driver built from context-derived credentials can
+ // never be rebuilt when they rotate.
+ if (cachedContextRef) {
+ cachedContextRef.current = context;
+ }
+
+ return cachedOrchestratorApi;
}
+ const requestContextRef: { current: RequestContext } = { current: context };
+
/**
* Hash table to store promises which will be resolved with the
* datasource drivers. DriverFactoryByDataSource function is closure
@@ -583,6 +824,28 @@ export class CubejsServerCore {
*/
const driverPromise: Record> = {};
+ /**
+ * What each cached driver in `driverPromise` was built from, so a changed
+ * configuration can be detected. Keyed identically to `driverPromise`.
+ */
+ const driverOrigin: Record = {};
+
+ /**
+ * Rebuild history per alias set, which both rate-limits rebuilds and makes
+ * a deployment whose `contextToOrchestratorId` does not partition by
+ * whatever `driverFactory` reads — every user sharing one orchestrator, say
+ * — diagnosable: it keeps resolving a changed configuration rather than
+ * doing so once per credential rotation.
+ */
+ const driverRebuilds: Record = {};
+
+ /**
+ * Consecutive staleness probes that threw, per alias set. Reset by any
+ * probe or build that resolves a configuration, so only *sustained* refusal
+ * reaches the bound.
+ */
+ const driverProbeFailures: Record = {};
+
let externalPreAggregationsDriverPromise: Promise | null = null;
const contextToDbType: DbTypeInternalFn = this.contextToDbType.bind(this);
@@ -596,75 +859,400 @@ export class CubejsServerCore {
(await this.orchestratorOptions(context)) || {},
);
- const orchestratorApi = this.createOrchestratorApi(
+ /**
+ * Driver factory function `DriverFactoryByDataSource`. Named so the rebuild
+ * path can re-enter it when another caller wins the race to replace a key.
+ */
+ const resolveDataSourceDriver = async (
+ dataSource = 'default',
+ preAggregations = false,
+ attempt = 0,
+ ): Promise => {
+ const factoryKey = preAggregations ? `${dataSource}@pre_agg` : dataSource;
+
+ const hasSeparatePreAggEnv = hasPreAggregationsEnvVars(dataSource);
+ const usePreAgg = preAggregations && hasSeparatePreAggEnv && !this.optsHandler.isCustomDriverFactory();
+
+ const driverContext = (): DriverContext => ({
+ ...requestContextRef.current,
+ dataSource,
+ preAggregations: usePreAgg || false,
+ });
+
/**
- * Driver factory function `DriverFactoryByDataSource`.
+ * Every key that resolves to the one driver built here. Without separate
+ * pre-aggregation credentials `usePreAgg` is false whichever key was
+ * asked for, so both describe an identically configured driver and share
+ * a single instance — they must therefore be written, and invalidated,
+ * together. Doing it per requested key instead lets the two diverge into
+ * two pools where the deployment expects one.
*/
- async (dataSource = 'default', preAggregations = false) => {
- const factoryKey = preAggregations ? `${dataSource}@pre_agg` : dataSource;
- if (driverPromise[factoryKey]) {
- return driverPromise[factoryKey];
- }
+ const aliasedKeys = hasSeparatePreAggEnv
+ ? [factoryKey]
+ : [dataSource, `${dataSource}@pre_agg`];
- const hasSeparatePreAggEnv = hasPreAggregationsEnvVars(dataSource);
- const usePreAgg = preAggregations && hasSeparatePreAggEnv && !this.optsHandler.isCustomDriverFactory();
+ const invalidate = () => aliasedKeys.forEach((key) => {
+ driverPromise[key] = null;
+ delete driverOrigin[key];
+ });
- if (preAggregations && hasSeparatePreAggEnv && this.optsHandler.isCustomDriverFactory()) {
- this.logger('Pre-aggregation driver conflict', {
- error: 'Both driverFactory and PRE_AGGREGATIONS env vars are defined. driverFactory will take precedence.',
+ /**
+ * Drop every key pointing at `driver` and release it off the request path.
+ *
+ * Every key, not just the one asked for: a surviving alias would keep
+ * handing out a driver whose pool is being drained, and would release it a
+ * second time when it was itself found stale.
+ *
+ * `release` drains the pool, so queries already running on the replaced
+ * driver finish before its connections close. It is deliberately not
+ * awaited — this request should not wait on the previous driver's
+ * in-flight work — and its failure must not fail this request.
+ */
+ const replaceCachedDriver = (driver: Promise) => {
+ Object.keys(driverPromise)
+ .filter((key) => driverPromise[key] === driver)
+ .forEach((key) => {
+ driverPromise[key] = null;
+ delete driverOrigin[key];
+ });
+
+ driver
+ .then((resolved) => resolved.release())
+ .catch((error) => this.logger('Driver release error', {
dataSource,
+ error: (error as Error).stack || (error as Error).toString(),
+ }));
+ };
+
+ /**
+ * Rebuilds are counted and rate-limited per alias set, not per key: a
+ * rotation seen first through `default@pre_agg` and then through `default`
+ * is one rebuild of one shared driver, and must not read as two counters
+ * at 1 — nor rebuild twice.
+ */
+ const rebuildKey = aliasedKeys[0];
+
+ /**
+ * Count a replacement against the alias set's rebuild history, open a
+ * suppression window on it, and report it.
+ *
+ * Both paths that tear a pool down come through here — a configuration
+ * the factory changed, and a factory that will no longer produce one.
+ * They cost the same thing, so they are bounded by the same state: a
+ * replacement that skipped this would rebuild straight past the interval
+ * that exists to stop pool churn, and never reach the diagnostic that
+ * names it.
+ */
+ const recordDriverRebuild = (reason: DriverReplacementReason, warning: string) => {
+ // Re-read rather than reusing what was captured before the staleness
+ // probe awaited: reaching here means no concurrent rebuild landed, but
+ // the count is the one piece of state that would silently lose an
+ // increment if that ever stopped being true.
+ const state = driverRebuilds[rebuildKey]
+ || { count: 0, lastRebuildAt: 0, suppressionReported: false };
+
+ state.count += 1;
+ state.lastRebuildAt = Date.now();
+ state.suppressionReported = false;
+ driverRebuilds[rebuildKey] = state;
+
+ // Carries `warning` so it survives the default log level: a plain-params
+ // message matches no allowlist in `prodLogger`/`devLogger` and is
+ // dropped below `trace`. Tearing down a connection pool is an event an
+ // operator needs to be able to correlate against, and the threshold
+ // message below arrives too late to reconstruct the first rebuilds.
+ this.logger('Rebuilding driver', {
+ dataSource,
+ preAggregations,
+ rebuildCount: state.count,
+ reason,
+ warning,
+ });
+
+ // A credential rotation rebuilds a handful of times a day. Rebuilding
+ // this often means the orchestrator id does not partition by whatever
+ // the factory reads, so contexts that need different connections keep
+ // displacing each other's driver — or that the factory is not resolving
+ // reliably enough to keep any connection.
+ if (state.count === DRIVER_REBUILD_WARN_THRESHOLD) {
+ this.logger('Driver rebuilt repeatedly', {
+ dataSource,
+ rebuildCount: state.count,
+ warning: 'Driver keeps being replaced for one orchestrator. '
+ + 'contextToOrchestratorId likely does not distinguish the contexts '
+ + 'driverFactory returns different connections for, or driverFactory '
+ + 'is not resolving a configuration reliably.',
});
}
+ };
- driverPromise[factoryKey] = (async () => {
- let driver: BaseDriver | null = null;
-
- try {
- driver = await this.resolveDriver(
- {
- ...context,
- dataSource,
- preAggregations: usePreAgg || false,
- },
- orchestratorOptions,
- );
-
- if (typeof driver === 'object' && driver != null) {
- if (driver.setLogger) {
- driver.setLogger(this.logger);
- }
+ // Already resolved by the staleness check below, so the factory is not
+ // asked twice for the same rebuild.
+ let resolvedFactoryResult: DriverFactoryResult | undefined;
+
+ const cached = driverPromise[factoryKey];
+ const rebuildState = driverRebuilds[rebuildKey];
+
+ if (
+ cached &&
+ rebuildState &&
+ Date.now() - rebuildState.lastRebuildAt < DRIVER_REBUILD_MIN_INTERVAL_MS
+ ) {
+ // Inside the window this is a plain cache hit: the factory is not asked
+ // whether anything changed, because acting on the answer is what has to
+ // be rate-limited and asking a user-supplied function on every query is
+ // not free either. A configuration that really did change is picked up
+ // by the first resolution after the window closes.
+ if (!rebuildState.suppressionReported) {
+ rebuildState.suppressionReported = true;
+
+ // Carries `warning` so it survives the default log level, as the
+ // rebuild it follows does.
+ this.logger('Driver rebuild suppressed', {
+ dataSource,
+ preAggregations,
+ rebuildCount: rebuildState.count,
+ warning: 'Driver was rebuilt less than '
+ + `${DRIVER_REBUILD_MIN_INTERVAL_MS / 1000}s ago; reusing it without `
+ + 'rechecking its configuration. Sustained suppression means the '
+ + 'configuration is not stable across driverFactory calls, or that '
+ + 'contextToOrchestratorId does not distinguish the contexts '
+ + 'driverFactory returns different connections for.',
+ });
+ }
- await driver.testConnection();
+ return cached;
+ }
- return driver;
- }
+ if (cached) {
+ const staleness = await this.resolveDriverStaleness(
+ driverOrigin[factoryKey],
+ driverContext(),
+ );
- throw new Error(
- `Unexpected return type, driverFactory must return driver (dataSource: "${dataSource}"), actual: ${getRealType(driver)}`
- );
- } catch (e) {
- driverPromise[factoryKey] = null;
+ // `resolveDriverStaleness` awaits the user's factory, so another caller
+ // may have replaced or invalidated this key in the meantime. Its work
+ // supersedes ours, and `cached` is no longer ours to reuse or release:
+ // it has either been handed to that caller's requests or already
+ // released by it.
+ const superseding = driverPromise[factoryKey];
+
+ if (superseding !== cached) {
+ // Retry, so this request ends up on a driver matching its own
+ // context — but bounded. Where contexts keep displacing each other
+ // this request could otherwise lose every round and pay for a
+ // user-supplied factory call each time. Past the bound, take what is
+ // cached: degrading to a reused driver is this design's fallback
+ // everywhere else, and it is strictly better than starving.
+ if (attempt < MAX_DRIVER_REBUILD_ATTEMPTS) {
+ return resolveDataSourceDriver(dataSource, preAggregations, attempt + 1);
+ }
- if (!preAggregations && !hasSeparatePreAggEnv) {
- driverPromise[`${dataSource}@pre_agg`] = null;
- }
+ if (superseding) {
+ return superseding;
+ }
+
+ // Invalidated rather than replaced — the winning caller's own build
+ // failed, so it released `cached` and left nothing to reuse. Build
+ // below, which cannot recurse again, carrying the probe's result when
+ // it already resolved one so the factory is not asked twice.
+ resolvedFactoryResult = staleness.stale ? staleness.factoryResult : undefined;
+ // `=== false` rather than `!`: this package compiles with
+ // `strictNullChecks` off, where the negation does not narrow the union
+ // and `probeFailed` below would not typecheck.
+ } else if (staleness.stale === false) {
+ if (!staleness.probeFailed) {
+ delete driverProbeFailures[rebuildKey];
+
+ return cached;
+ }
+
+ const now = Date.now();
+ const previousFailures = driverProbeFailures[rebuildKey];
+
+ // A rolling window, not a running total. Probes are only issued when
+ // the context changes, so a record that is never re-based would add
+ // up occasional flakes weeks apart and read them as one outage.
+ // Retention rather than the grace window, so that a deployment
+ // probing less often than the grace window can still reach the bound.
+ const failures = previousFailures
+ && now - previousFailures.lastFailureAt < PROBE_FAILURE_RETENTION_MS
+ ? previousFailures
+ : {
+ count: 0, firstFailureAt: now, lastFailureAt: now, incidentStartedAt: now,
+ };
+
+ // Requests that arrived together and failed on the same blink of a
+ // dependency are one refusal, not one each. The gap is measured
+ // against the last refusal seen rather than the last one counted, so
+ // that a continuous stream stays one incident — which is only sound
+ // because an incident is also bounded by its own duration below.
+ if (
+ failures.count === 0 ||
+ now - failures.lastFailureAt >= PROBE_FAILURE_COALESCE_MS
+ ) {
+ failures.count += 1;
+ failures.incidentStartedAt = now;
+ }
+
+ failures.lastFailureAt = now;
+ driverProbeFailures[rebuildKey] = failures;
+
+ const failingForMs = now - failures.firstFailureAt;
+
+ // Two shapes of sustained refusal, because either alone leaves a
+ // traffic profile uncovered. Repeated incidents catch a deployment
+ // whose probes are sparse enough that each refusal stands alone; one
+ // unbroken incident catches a busy deployment, where refusals arrive
+ // faster than the coalescing window and would otherwise count once
+ // however long the credential stayed dead.
+ const sustainedIncident = now - failures.incidentStartedAt >= PROBE_FAILURE_GRACE_MS;
+ const repeatedIncidents = failures.count >= MAX_PROBE_FAILURE_INCIDENTS
+ && failingForMs >= PROBE_FAILURE_GRACE_MS;
+
+ // Transient, as far as anything here can tell. Reuse, exactly as
+ // before this bound existed.
+ if (!sustainedIncident && !repeatedIncidents) {
+ return cached;
+ }
+
+ recordDriverRebuild(
+ 'repeated staleness check failures',
+ `driverFactory has failed every staleness check for ${
+ Math.round(failingForMs / 1000)
+ }s. Releasing the connection it built rather than serving queries on `
+ + 'a configuration it will no longer produce; the next request calls '
+ + 'the factory itself, so a factory that fails closed on an unusable '
+ + 'credential surfaces its own error.',
+ );
+
+ delete driverProbeFailures[rebuildKey];
+ replaceCachedDriver(cached);
+
+ // Falls through to the build below, which calls the factory itself:
+ // it either recovers, or throws where the caller can see it.
+ } else {
+ // Opens a fresh suppression window, so the next configuration change
+ // for this alias set waits it out rather than tearing down the pool
+ // this rebuild is about to stand up.
+ recordDriverRebuild(
+ staleness.reason,
+ `Replacing the connection — ${staleness.reason}.`,
+ );
+
+ delete driverProbeFailures[rebuildKey];
+ replaceCachedDriver(cached);
+
+ resolvedFactoryResult = staleness.factoryResult;
+ }
+ }
+
+ if (preAggregations && hasSeparatePreAggEnv && this.optsHandler.isCustomDriverFactory()) {
+ this.logger('Pre-aggregation driver conflict', {
+ error: 'Both driverFactory and PRE_AGGREGATIONS env vars are defined. driverFactory will take precedence.',
+ dataSource,
+ });
+ }
+
+ // Shared by reference across `aliasedKeys`, so every key describes the
+ // one driver they all resolve to. Starts empty: until the factory has
+ // been called there is nothing to compare against, and
+ // `resolveDriverStaleness` reads that as "reuse".
+ const origin: DriverOrigin = {
+ securityContextFingerprint: null,
+ configFingerprint: null,
+ expiresAt: undefined,
+ lifetimeIgnoredReported: false,
+ };
+
+ aliasedKeys.forEach((key) => {
+ driverOrigin[key] = origin;
+ });
+
+ const pending = (async () => {
+ let driver: BaseDriver | null = null;
+
+ try {
+ const currentDriverContext = driverContext();
+ const factoryResult = resolvedFactoryResult ?? {
+ value: await this.options.driverFactory(currentDriverContext),
+ securityContextFingerprint: fingerprint(currentDriverContext.securityContext),
+ };
- if (driver) {
- await driver.release();
+ const factoryConfig = isDriver(factoryResult.value)
+ ? undefined
+ : factoryResult.value;
+
+ origin.securityContextFingerprint = factoryResult.securityContextFingerprint;
+ origin.configFingerprint = factoryConfig
+ ? driverConfigFingerprint(factoryConfig)
+ : null;
+ origin.expiresAt = factoryConfig
+ ? this.resolveBuiltDriverExpiry(factoryConfig, dataSource, origin)
+ : undefined;
+
+ driver = await this.createDriverFromFactoryResult(
+ factoryResult.value,
+ currentDriverContext,
+ orchestratorOptions,
+ );
+
+ if (typeof driver === 'object' && driver != null) {
+ if (driver.setLogger) {
+ driver.setLogger(this.logger);
}
- throw e;
+ await driver.testConnection();
+
+ // Resolved a configuration and stood a connection up on it, so
+ // whatever the probes were failing on has passed.
+ delete driverProbeFailures[rebuildKey];
+
+ return driver;
+ }
+
+ throw new Error(
+ `Unexpected return type, driverFactory must return driver (dataSource: "${dataSource}"), actual: ${getRealType(driver)}`
+ );
+ } catch (e) {
+ // Only if this build still owns the keys. A concurrent rebuild
+ // installs its own `origin`, and its driver must not be evicted
+ // because ours failed.
+ if (driverOrigin[factoryKey] === origin) {
+ invalidate();
+ }
+
+ if (driver) {
+ await driver.release();
}
- })();
- // No separate pre-agg driver needed — share the same promise for both keys
- if (!preAggregations && !hasSeparatePreAggEnv) {
- driverPromise[`${dataSource}@pre_agg`] = driverPromise[factoryKey];
+ throw e;
}
+ })();
+
+ // No separate pre-agg driver needed — share the same promise across keys
+ aliasedKeys.forEach((key) => {
+ driverPromise[key] = pending;
+ });
- return driverPromise[factoryKey];
- },
+ return pending;
+ };
+
+ const orchestratorApi = this.createOrchestratorApi(
+ resolveDataSourceDriver,
{
+ // Deliberately outside the staleness check that `resolveDataSourceDriver`
+ // applies to `requestContextRef.current`: this and `contextToDbType`
+ // below keep resolving from `context`, the request that created the
+ // orchestrator, and resolve once for its lifetime.
+ //
+ // Both were pinned that way before rebuilding existed, and neither is
+ // the shape the rebuild is for. The external store is Cube Store or a
+ // shared pre-aggregation warehouse — one connection the deployment owns,
+ // not one derived from who is asking — and a data source's type does not
+ // change per user, only its credentials do. Widening the rebuild to
+ // cover them would mean tearing down the pre-aggregation store's pool on
+ // a per-user signal that says nothing about it.
externalDriverFactory: this.options.externalDriverFactory && (async () => {
if (externalPreAggregationsDriverPromise) {
return externalPreAggregationsDriverPromise;
@@ -713,6 +1301,7 @@ export class CubejsServerCore {
}
);
+ this.orchestratorRequestContexts.set(orchestratorApi, requestContextRef);
this.orchestratorStorage.set(orchestratorId, orchestratorApi);
return orchestratorApi;
@@ -877,11 +1466,31 @@ export class CubejsServerCore {
context: DriverContext,
options?: OrchestratorInitedOptions,
): Promise {
- const val = await this.options.driverFactory(context);
+ return this.createDriverFromFactoryResult(
+ await this.options.driverFactory(context),
+ context,
+ options,
+ );
+ }
+
+ /**
+ * Build a driver from whatever `driverFactory` returned. Split out of
+ * `resolveDriver` so a caller that has already invoked the factory — to
+ * compare its result against the cached driver's — can build from that same
+ * result instead of invoking a user-supplied function a second time.
+ */
+ protected async createDriverFromFactoryResult(
+ val: DriverConfig | BaseDriver,
+ context: DriverContext,
+ options?: OrchestratorInitedOptions,
+ ): Promise {
if (isDriver(val)) {
return val;
} else {
- const { type, ...rest } = val;
+ // Without the lifetime: it describes when to replace this driver, not
+ // how to connect, and every other key here is passed to the driver's own
+ // constructor.
+ const { type, ...rest } = withoutDriverExpiry(val);
const opts = Object.keys(rest).length
? rest
: {
@@ -895,6 +1504,214 @@ export class CubejsServerCore {
}
}
+ /**
+ * The lifetime to hold a driver to, given the configuration it was just built
+ * from — and `undefined` where that configuration named a deadline that had
+ * already passed.
+ *
+ * Rebuilding cannot fix a deadline the factory keeps re-asserting. Honouring
+ * one would find the new driver stale the moment its suppression window
+ * closed, tear down a pool it had just stood up, and resolve the same
+ * unusable deadline again, for the life of the process. A driver built from
+ * such a configuration is no worse than the one it replaced, so the
+ * connection is kept and the lifetime dropped — the operator gets a warning
+ * naming the field rather than churn that never resolves.
+ *
+ * Two deadlines are unusable, and they produce the same loop. One has already
+ * passed. The other is shorter than the interval replacements are rate-limited
+ * to: the driver is stale again the moment its suppression window closes, so
+ * the rate limiter can never let this mechanism honour it. Dropping it strands
+ * nothing — a credential that short is rotating, and rotation changes the
+ * configuration, which is caught by comparison rather than by lifetime.
+ *
+ * The documented recipe does not reach either: its `accessToken()` withholds a
+ * token that is already near expiry, so the configuration changes to the
+ * service account, which names no lifetime, and converges. A factory passing
+ * the provider's `accessTokenExpiresAt` straight through does reach them.
+ */
+ protected resolveBuiltDriverExpiry(
+ config: DriverConfig,
+ dataSource: string,
+ origin: DriverOrigin,
+ ): number | undefined {
+ const expiresAt = parseDriverExpiry(config.expiresAt);
+
+ if (expiresAt === undefined) {
+ return undefined;
+ }
+
+ // Already judged when it was installed. This also runs on every probe that
+ // carries an unchanged configuration over, where what is left of the
+ // deadline is a measure of time passing rather than of anything the factory
+ // stated. Measuring it there would drop a perfectly good deadline once it
+ // entered its final window — and leave the driver with no lifetime at all,
+ // in precisely the stretch the lifetime exists to cover.
+ if (expiresAt === origin.expiresAt) {
+ return expiresAt;
+ }
+
+ const remainingMs = expiresAt - Date.now();
+
+ if (remainingMs >= DRIVER_REBUILD_MIN_INTERVAL_MS) {
+ return expiresAt;
+ }
+
+ // Once per driver, not once per call: the carry-over path resolves this on
+ // every security context change, and the operator needs the field named
+ // once, not on every query that arrives with a fresh JWT.
+ if (!origin.lifetimeIgnoredReported) {
+ origin.lifetimeIgnoredReported = true;
+
+ this.logger('Driver lifetime ignored', {
+ dataSource,
+ expiresAt: new Date(expiresAt).toISOString(),
+ warning: remainingMs <= 0
+ ? 'driverFactory returned a configuration whose expiresAt has already '
+ + 'passed. Using the connection anyway and ignoring the lifetime: '
+ + 'replacing a driver cannot move a deadline the factory keeps '
+ + 're-asserting, and honouring it would rebuild the pool for the life '
+ + 'of the process. expiresAt must state when the credential being '
+ + 'returned stops being usable, in the future.'
+ : 'driverFactory returned a configuration whose expiresAt is less than '
+ + `${DRIVER_REBUILD_MIN_INTERVAL_MS / 1000}s away, which is shorter `
+ + 'than the interval replacements are rate-limited to. Honouring it '
+ + 'would replace the connection once per window for the life of the '
+ + 'process, so the lifetime is ignored; a credential rotating that '
+ + 'fast is picked up by its configuration changing instead.',
+ });
+ }
+
+ // Keep whatever was accepted, if anything. The newly stated deadline cannot
+ // be honoured, but one this driver is already held to can: an installed
+ // deadline is still in the future here, because an elapsed one returns
+ // `stale` from the lifetime check before the factory is ever asked. At the
+ // build path this is `undefined`, so nothing changes there.
+ return origin.expiresAt;
+ }
+
+ /**
+ * Decide whether a cached driver still reflects what `driverFactory` would
+ * resolve for the current request context.
+ *
+ * The check is deliberately layered so that deployments which cannot be
+ * affected never leave the fast path, and no user-supplied function is called
+ * more often than it has to be:
+ *
+ * 1. No custom `driverFactory`, or one that hands back a constructed driver
+ * rather than a config — nothing context-derived to compare. Reuse.
+ * 2. The security context is byte-for-byte what the cached driver was built
+ * from. Reuse, without calling the factory at all. This is the common
+ * case: `requestId` changes per request, credentials do not.
+ * 3. The security context changed, so ask the factory. Most factories ignore
+ * it and return an identical config — reuse, and remember the new context
+ * so step 2 short-circuits next time.
+ * 4. The config genuinely changed. Rebuild.
+ *
+ * Step 4 is what fixes a rotated per-user credential: previously the driver
+ * built from the first request's token was reused for the life of the
+ * process, so every new connection it opened failed to authenticate.
+ *
+ * A `stale: true` verdict is permission to rebuild, not an instruction to: the
+ * caller rate-limits rebuilds per data source, because the rate at which a
+ * configuration appears to change is a property of user code, while the cost
+ * of acting on it is a connection pool.
+ *
+ * Note this follows the documented contract of `contextToOrchestratorId` —
+ * that it is the cache key for database connections. Two contexts that
+ * resolve to different connections but share an orchestrator id are a
+ * misconfiguration; they were already sharing one user's connection before
+ * this change.
+ */
+ protected async resolveDriverStaleness(
+ origin: DriverOrigin | undefined,
+ context: DriverContext,
+ ): Promise {
+ if (!origin) {
+ return { stale: false };
+ }
+
+ // Checked first, and without asking the factory: a credential that has
+ // stopped rotating resolves to the same configuration indefinitely while
+ // the connection built from it is already dead. That is the one staleness a
+ // comparison cannot see, which is why a configuration may state its own
+ // lifetime.
+ if (origin.expiresAt !== undefined && Date.now() >= origin.expiresAt) {
+ return { stale: true, reason: 'lifetime elapsed' };
+ }
+
+ if (
+ origin.configFingerprint === null ||
+ !this.optsHandler.isCustomDriverFactory()
+ ) {
+ return { stale: false };
+ }
+
+ const securityContextFingerprint = fingerprint(context.securityContext);
+
+ if (
+ securityContextFingerprint === null ||
+ securityContextFingerprint === origin.securityContextFingerprint
+ ) {
+ return { stale: false };
+ }
+
+ let value: DriverConfig | BaseDriver;
+
+ try {
+ value = await this.options.driverFactory(context);
+ } catch (error) {
+ // This call is a probe, not the request's own resolution: a cache hit
+ // never used to invoke the factory at all, so letting a transient failure
+ // here propagate would fail a query the cached driver could have served.
+ // Degrade to reuse, as with anything else that cannot be compared — but
+ // report it, because a factory that keeps refusing is not transient and
+ // the caller gives the driver up once these stop being occasional.
+ this.logger('Driver staleness check error', {
+ dataSource: context.dataSource,
+ error: (error as Error).stack || (error as Error).toString(),
+ });
+
+ return { stale: false, probeFailed: true };
+ }
+
+ // `null` for a constructed driver, which carries no configuration to
+ // compare — and, like every other `null` here, is read as "assume
+ // unchanged". Nothing is released on that path: the value belongs to the
+ // factory, which may be handing out a singleton it expects to keep working.
+ //
+ // No factory can actually reach it. One that returns drivers consistently
+ // recorded a null config fingerprint on its first build and is rejected by
+ // the guard above before the factory is ever called; one that switches from
+ // configs to drivers is rejected by `OptsHandler.assertDriverFactoryResult`,
+ // and that throw is caught above as a probe failure. It is handled because
+ // the type admits it, not because it happens.
+ const config = isDriver(value) ? undefined : value;
+ const configFingerprint = config ? driverConfigFingerprint(config) : null;
+
+ if (configFingerprint === null || configFingerprint === origin.configFingerprint) {
+ origin.securityContextFingerprint = securityContextFingerprint;
+
+ // The connection is unchanged, but its deadline may not be — the lifetime
+ // is excluded from the fingerprint, so a credential re-issued with the
+ // same value and a later expiry compares equal. Carrying the new deadline
+ // over is what keeps that from rebuilding on the old one, once per window,
+ // forever. Guarded like the build path, because a factory re-asserting an
+ // elapsed deadline would otherwise reinstate it here on the next context
+ // change, reopening the loop that guard exists to close.
+ if (config) {
+ origin.expiresAt = this.resolveBuiltDriverExpiry(config, context.dataSource, origin);
+ }
+
+ return { stale: false };
+ }
+
+ return {
+ stale: true,
+ reason: 'configuration change',
+ factoryResult: { value, securityContextFingerprint },
+ };
+ }
+
public async testConnections() {
return this.orchestratorStorage.testConnections();
}
diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts
index 3d1cfee7d79ee..5f2cba0b96514 100644
--- a/packages/cubejs-server-core/src/core/types.ts
+++ b/packages/cubejs-server-core/src/core/types.ts
@@ -159,6 +159,23 @@ export type DriverOptions = {
export type DriverConfig = {
type: DatabaseType,
+ /**
+ * When the connection this configuration describes stops being usable, as
+ * epoch milliseconds (or seconds), an ISO 8601 string, or a `Date`.
+ *
+ * A driver is resolved once and then cached, so a connection built from a
+ * rotating credential is replaced only when that credential *changes*. One
+ * that stops rotating — an expired token whose refresh is failing upstream —
+ * resolves to an identical configuration indefinitely while the connection
+ * built from it is already dead, and no comparison can see that. Setting this
+ * states the lifetime outright, so the driver is replaced when it elapses
+ * whether or not anything about the configuration changed.
+ *
+ * Optional, and deliberately not part of the configuration's identity: a
+ * factory may return a fresh deadline on every call without that reading as a
+ * changed connection.
+ */
+ expiresAt?: number | string | Date,
} & DriverOptions;
export type DriverFactoryFn = (context: DriverContext) =>
diff --git a/packages/cubejs-server-core/test/unit/driver-cache-invalidation.test.ts b/packages/cubejs-server-core/test/unit/driver-cache-invalidation.test.ts
new file mode 100644
index 0000000000000..c1881210fdb6d
--- /dev/null
+++ b/packages/cubejs-server-core/test/unit/driver-cache-invalidation.test.ts
@@ -0,0 +1,1217 @@
+/* eslint-disable @typescript-eslint/no-empty-function */
+import { BaseDriver } from '@cubejs-backend/query-orchestrator';
+import type { DriverFactoryByDataSource } from '@cubejs-backend/query-orchestrator';
+
+import { CreateOptions, CubejsServerCore } from '../../src';
+
+type FakeDriver = BaseDriver & {
+ builtFrom: any;
+ release: jest.Mock;
+ testConnection: jest.Mock;
+};
+
+/**
+ * Rebuilds are rate-limited per data source, so a test that rebuilds more than
+ * once has to say how much time passed in between. Patching `Date.now` rather
+ * than using fake timers keeps the real microtask scheduling these tests depend
+ * on to interleave concurrent resolutions.
+ */
+function fakeClock() {
+ let now = Date.parse('2026-01-01T00:00:00Z');
+ const spy = jest.spyOn(Date, 'now').mockImplementation(() => now);
+
+ return {
+ /** Longer than any rebuild interval the server enforces. */
+ advancePastRebuildInterval: () => {
+ now += 60 * 1000;
+ },
+ advance: (ms: number) => {
+ now += ms;
+ },
+ restore: () => spy.mockRestore(),
+ };
+}
+
+/**
+ * Stands in for real driver construction so these tests exercise the caching
+ * decisions without needing a database. Everything else — the driver factory
+ * closure, the fingerprinting, the rebuild — is the production code path.
+ */
+class TestServerCore extends CubejsServerCore {
+ public builtDrivers: FakeDriver[] = [];
+
+ /** Set to fail the next driver construction, as a bad credential would. */
+ public failNextBuild = false;
+
+ /**
+ * Runs before each staleness probe, standing in for another caller that wins
+ * the race while this one is awaiting the factory. Re-entrant probes skip it,
+ * so the hook can drive the driver factory itself.
+ */
+ public onStalenessProbe: (() => Promise) | undefined;
+
+ private inStalenessHook = false;
+
+ protected async resolveDriverStaleness(origin: any, context: any): Promise {
+ if (this.onStalenessProbe && !this.inStalenessHook) {
+ this.inStalenessHook = true;
+
+ try {
+ await this.onStalenessProbe();
+ } finally {
+ this.inStalenessHook = false;
+ }
+ }
+
+ return super.resolveDriverStaleness(origin, context);
+ }
+
+ protected async createDriverFromFactoryResult(
+ val: any,
+ context: any,
+ options?: any,
+ ): Promise {
+ // A factory that hands back a constructed driver takes the real path — that
+ // branch is exactly what one of these tests is about.
+ if (val instanceof BaseDriver) {
+ return super.createDriverFromFactoryResult(val, context, options);
+ }
+
+ if (this.failNextBuild) {
+ this.failNextBuild = false;
+
+ throw new Error('driver construction failed');
+ }
+
+ const driver = {
+ builtFrom: val,
+ release: jest.fn(async () => {}),
+ testConnection: jest.fn(async () => {}),
+ setLogger: () => {},
+ } as unknown as FakeDriver;
+
+ this.builtDrivers.push(driver);
+
+ return driver;
+ }
+}
+
+/**
+ * Boot a core, resolve its orchestrator once, and hand back the driver factory
+ * the orchestrator was created with — the same closure the query orchestrator
+ * calls for every query.
+ */
+async function createCore(options: CreateOptions, securityContext: unknown) {
+ const logger = jest.fn();
+ const core = new TestServerCore({
+ contextToOrchestratorId: () => 'ORCHESTRATOR',
+ logger,
+ ...options,
+ });
+ const spy = jest.spyOn(core, 'createOrchestratorApi');
+
+ await core.getOrchestratorApi({ requestId: 'req-1', securityContext });
+
+ const driverFactory = spy.mock.calls[0][0];
+
+ return {
+ core,
+ driverFactory,
+ logger,
+ /** Log messages, with the params each was reported with. */
+ logged: (message: string) => logger.mock.calls.filter(([msg]) => msg === message).map(([, params]) => params),
+ /** Serve another request through the cached orchestrator. */
+ request: (nextSecurityContext: unknown, requestId = 'req-n') => core.getOrchestratorApi({ requestId, securityContext: nextSecurityContext }),
+ };
+}
+
+describe('driver cache invalidation', () => {
+ let clock: ReturnType;
+
+ beforeAll(() => {
+ process.env.CUBEJS_API_SECRET = 'api-secret';
+ });
+
+ // Frozen by default, so a test that rebuilds twice has to be explicit about
+ // the time in between rather than passing on whatever the wall clock did.
+ beforeEach(() => {
+ clock = fakeClock();
+ });
+
+ afterEach(() => {
+ clock.restore();
+ });
+
+ // The CUB-3599 regression: the orchestrator closed over the context of the
+ // request that created it, so a driver built from a per-user credential was
+ // reused for the life of the process. Every connection it opened after the
+ // token rotated failed to authenticate.
+ test('rebuilds the driver when a context-derived credential changes', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+ expect(first.builtFrom).toMatchObject({ password: 'token-a' });
+
+ await request({ token: 'token-b' });
+ const second = await driverFactory('default');
+
+ expect(second).not.toBe(first);
+ expect(second.builtFrom).toMatchObject({ password: 'token-b' });
+ expect(core.builtDrivers).toHaveLength(2);
+ });
+
+ test('releases the driver it replaced, so its pool is drained', async () => {
+ const { driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ await request({ token: 'token-b' });
+ await driverFactory('default');
+
+ // Released off the request path, so give the detached promise a tick.
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(first.release).toHaveBeenCalledTimes(1);
+ });
+
+ test('reuses the driver when the security context is unchanged', async () => {
+ const factory = jest.fn((ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }));
+ const { driverFactory, request } = await createCore({ driverFactory: factory }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ // A different request, same user, same credential.
+ await request({ token: 'token-a' }, 'req-2');
+
+ expect(await driverFactory('default')).toBe(first);
+ // Not re-invoked: an unchanged security context short-circuits before the
+ // user's factory is called at all.
+ expect(factory).toHaveBeenCalledTimes(1);
+ });
+
+ test('reuses the driver when the factory ignores the security context', async () => {
+ const factory = jest.fn(() => ({ type: 'postgres', password: 'from-env' }));
+ const { core, driverFactory, request } = await createCore({ driverFactory: factory }, { user: 'a' });
+
+ const first = await driverFactory('default');
+
+ await request({ user: 'b' });
+ const second = await driverFactory('default');
+
+ expect(second).toBe(first);
+ expect(core.builtDrivers).toHaveLength(1);
+ // Asked once more because the context changed, but the answer matched, so
+ // nothing was rebuilt.
+ expect(factory).toHaveBeenCalledTimes(2);
+ // ...and that answer is remembered, so a third request with the same
+ // context does not ask again.
+ await request({ user: 'b' }, 'req-3');
+ await driverFactory('default');
+ expect(factory).toHaveBeenCalledTimes(2);
+ });
+
+ test('never rebuilds when the factory returns a constructed driver', async () => {
+ class ConstructedDriver extends BaseDriver {
+ public release = jest.fn(async () => {});
+
+ public testConnection = jest.fn(async () => {});
+
+ public async query(): Promise {
+ return [];
+ }
+ }
+
+ const driver = new ConstructedDriver();
+ const factory = jest.fn(() => driver);
+ const { driverFactory, request } = await createCore({ driverFactory: factory }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ await request({ token: 'token-b' });
+
+ // A constructed driver carries no configuration to compare, so the previous
+ // resolve-once behaviour is preserved rather than guessed at.
+ expect(await driverFactory('default')).toBe(first);
+ expect(driver.release).not.toHaveBeenCalled();
+ expect(factory).toHaveBeenCalledTimes(1);
+ });
+
+ test('keeps the pre-aggregation alias pointing at the rebuilt driver', async () => {
+ const { driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ await driverFactory('default');
+ await request({ token: 'token-b' });
+
+ const rebuilt = await driverFactory('default');
+ const preAgg = await driverFactory('default', true);
+
+ expect(preAgg).toBe(rebuilt);
+ expect(preAgg.builtFrom).toMatchObject({ password: 'token-b' });
+ });
+
+ // The `default` and `default@pre_agg` keys share one driver when the data
+ // source has no separate pre-aggregation credentials. A pre-aggregation build
+ // can be the first caller to observe a rotation, so invalidation has to clear
+ // both keys whichever one asked: clearing only the requested key left the
+ // other serving the drained driver, released it twice, and then built a
+ // second pool for what should be a single shared driver.
+ test('rebuilds once when a pre-aggregation build observes the rotation first', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ await request({ token: 'token-b' });
+
+ const preAgg = await driverFactory('default', true);
+
+ // Past the rebuild interval, so it is the alias bookkeeping that keeps these
+ // two on one driver rather than the rate limit masking a second rebuild.
+ clock.advancePastRebuildInterval();
+
+ const regular = await driverFactory('default');
+
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(preAgg).toBe(regular);
+ expect(regular.builtFrom).toMatchObject({ password: 'token-b' });
+ expect(core.builtDrivers).toHaveLength(2);
+ // Exactly once — a second release would run against an already-drained pool.
+ expect(first.release).toHaveBeenCalledTimes(1);
+ });
+
+ test('a failed rebuild does not leave a poisoned cache entry', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ await driverFactory('default');
+
+ // The rotation is detected, but building the replacement fails.
+ core.failNextBuild = true;
+ await request({ token: 'token-b' });
+ await expect(driverFactory('default')).rejects.toThrow('driver construction failed');
+
+ // The next attempt resolves from scratch rather than serving the failure.
+ const recovered = await driverFactory('default');
+ expect(recovered.builtFrom).toMatchObject({ password: 'token-b' });
+ });
+
+ // The staleness check calls the factory speculatively, on a path that used to
+ // be a pure cache hit. A factory that reads a secret store can fail
+ // transiently, and that must not fail a query the cached driver can serve.
+ test('reuses the cached driver when the staleness probe throws', async () => {
+ let shouldFail = false;
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => {
+ if (shouldFail) {
+ throw new Error('secret store unreachable');
+ }
+
+ return { type: 'postgres', password: ctx.securityContext.token };
+ },
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ shouldFail = true;
+ await request({ token: 'token-b' });
+
+ expect(await driverFactory('default')).toBe(first);
+ expect(core.builtDrivers).toHaveLength(1);
+
+ // Once the factory recovers, the rotation is picked up as usual.
+ shouldFail = false;
+ const rebuilt = await driverFactory('default');
+ expect(rebuilt).not.toBe(first);
+ expect(rebuilt.builtFrom).toMatchObject({ password: 'token-b' });
+ });
+
+ // Two queries in flight when a rotation lands both see the cached driver as
+ // stale. Only one may rebuild: the loser must not release the driver the
+ // winner has already handed to its caller, nor stand up a second pool.
+ test('concurrent callers rebuild once and release once', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ await request({ token: 'token-b' });
+
+ const [a, b] = await Promise.all([
+ driverFactory('default'),
+ driverFactory('default'),
+ ]);
+
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(a).toBe(b);
+ expect(a.builtFrom).toMatchObject({ password: 'token-b' });
+ expect(core.builtDrivers).toHaveLength(2);
+ expect(first.release).toHaveBeenCalledTimes(1);
+ // The driver handed back is usable — not one whose pool is being drained.
+ expect(a.release).not.toHaveBeenCalled();
+ });
+
+ // Both rebuild logs have to survive the default log level, which drops any
+ // message carrying neither `error` nor `warning`. Without that param the
+ // rebuild — a connection pool being torn down — is invisible in production.
+ test('reports every rebuild, and escalates once it looks like a misconfiguration', async () => {
+ const { driverFactory, request, logged } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-0' });
+
+ await driverFactory('default');
+
+ for (let i = 1; i <= 50; i++) {
+ // Each rotation is its own, well clear of the previous rebuild, so all 50
+ // are acted on rather than rate-limited.
+ clock.advancePastRebuildInterval();
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token: `token-${i}` }, `req-${i}`);
+ // eslint-disable-next-line no-await-in-loop
+ await driverFactory('default');
+ }
+
+ const rebuilds = logged('Rebuilding driver');
+
+ expect(rebuilds).toHaveLength(50);
+ expect(rebuilds.every((params) => params.warning)).toBe(true);
+ expect(rebuilds.every((params) => params.reason === 'configuration change')).toBe(true);
+ expect(rebuilds[0]).toMatchObject({ dataSource: 'default', rebuildCount: 1 });
+ // Counted per alias set, so the 50th rotation reads as 50, not as a pair of
+ // separate counters for `default` and `default@pre_agg`.
+ expect(rebuilds[49]).toMatchObject({ rebuildCount: 50 });
+
+ const escalations = logged('Driver rebuilt repeatedly');
+
+ expect(escalations).toHaveLength(1);
+ expect(escalations[0]).toMatchObject({ rebuildCount: 50 });
+ expect(escalations[0].warning).toContain('contextToOrchestratorId');
+ });
+
+ // Losing the race enough times to exhaust the retry bound, where the winner's
+ // own build then failed, leaves the key invalidated rather than replaced. The
+ // driver this caller started from has already been released by that winner, so
+ // it can be neither handed back nor released again — the only safe move is to
+ // build. Reaching it needs four lost races and a failed build, hence the hook.
+ test('builds instead of reusing a released driver when the retry bound is exhausted', async () => {
+ const { core, driverFactory, request, logged } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ let round = 0;
+
+ core.onStalenessProbe = async () => {
+ round += 1;
+
+ // A different context takes the orchestrator over, then rebuilds — which
+ // replaces the key for the first three rounds. On the fourth that rebuild
+ // fails, so it invalidates the key and releases what it replaced.
+ await request({ token: `concurrent-${round}` }, `req-${round}`);
+
+ if (round === 4) {
+ core.failNextBuild = true;
+ }
+
+ await Promise.resolve(driverFactory('default')).catch(() => {});
+
+ // Each round's rebuild has to sit outside the rate-limiting window by the
+ // time the caller that lost the race retries — otherwise that retry is a
+ // cache hit rather than another probe, and the bound is never reached.
+ // Which is the rate limit working: displacement this rapid is what it
+ // exists to stop, so provoking the bound means stepping past it.
+ clock.advancePastRebuildInterval();
+ };
+
+ const resolved = await driverFactory('default');
+
+ core.onStalenessProbe = undefined;
+ await new Promise((resolve) => setImmediate(resolve));
+
+ // Four rounds, so the bound was genuinely exhausted rather than short-circuited.
+ expect(round).toBe(4);
+ expect(resolved).not.toBe(first);
+ // The returned driver is usable: not one some other caller already drained.
+ expect(resolved.release).not.toHaveBeenCalled();
+ // And nothing was released twice on the way there.
+ expect(core.builtDrivers.every((driver) => driver.release.mock.calls.length <= 1)).toBe(true);
+ expect(logged('Driver release error')).toHaveLength(0);
+ });
+
+ test('reuses the driver when the security context cannot be fingerprinted', async () => {
+ const circular: any = { token: 'token-a' };
+ circular.self = circular;
+
+ const factory = jest.fn((ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }));
+ const { core, driverFactory, request } = await createCore({ driverFactory: factory }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ // A circular security context fingerprints as null, which every caller must
+ // read as "assume unchanged" rather than rebuilding blindly.
+ await request(circular);
+
+ expect(await driverFactory('default')).toBe(first);
+ expect(core.builtDrivers).toHaveLength(1);
+ });
+
+ // A `driverFactory` whose configuration is not stable across calls — a
+ // credential minted per call, a nonce — resolves as changed every time. That
+ // deployment works today, because the driver is resolved once and the
+ // difference is never noticed, and it must not become a pool teardown per
+ // query. Rebuilds are therefore rate-limited per data source, and inside the
+ // window the cached driver is served exactly as it was before.
+ test('rate-limits rebuilds, reporting the suppression once per window', async () => {
+ let nonce = 0;
+ const factory = jest.fn(() => {
+ nonce += 1;
+
+ return { type: 'postgres', password: `token-${nonce}` };
+ });
+ const { core, driverFactory, request, logged } = await createCore(
+ { driverFactory: factory },
+ { user: 'a' },
+ );
+
+ await driverFactory('default');
+
+ clock.advancePastRebuildInterval();
+ await request({ user: 'b' });
+ const rebuilt = await driverFactory('default');
+
+ expect(core.builtDrivers).toHaveLength(2);
+
+ const callsBeforeSuppression = factory.mock.calls.length;
+
+ // Three more contexts inside the window. Each would resolve a different
+ // configuration, and none may replace the driver.
+ for (const user of ['c', 'd', 'e']) {
+ // eslint-disable-next-line no-await-in-loop
+ await request({ user });
+ // eslint-disable-next-line no-await-in-loop
+ expect(await driverFactory('default')).toBe(rebuilt);
+ }
+
+ expect(core.builtDrivers).toHaveLength(2);
+ expect(rebuilt.release).not.toHaveBeenCalled();
+ // Not even asked: acting on the answer is what is rate-limited, and calling
+ // user code per query to discard the result would be its own cost.
+ expect(factory).toHaveBeenCalledTimes(callsBeforeSuppression);
+
+ const suppressions = logged('Driver rebuild suppressed');
+
+ // Once per window, not once per query.
+ expect(suppressions).toHaveLength(1);
+ expect(suppressions[0]).toMatchObject({ dataSource: 'default', rebuildCount: 1 });
+ expect(suppressions[0].warning).toContain('driverFactory');
+ });
+
+ test('rebuilds again once the interval has passed', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ await driverFactory('default');
+
+ clock.advancePastRebuildInterval();
+ await request({ token: 'token-b' });
+ const rebuilt = await driverFactory('default');
+
+ // A second rotation inside the window is held back...
+ await request({ token: 'token-c' });
+ expect(await driverFactory('default')).toBe(rebuilt);
+
+ // ...and picked up by the first resolution after it closes, so the rate
+ // limit delays a rotation rather than dropping it.
+ clock.advancePastRebuildInterval();
+ const latest = await driverFactory('default');
+
+ expect(latest).not.toBe(rebuilt);
+ expect(latest.builtFrom).toMatchObject({ password: 'token-c' });
+ expect(core.builtDrivers).toHaveLength(3);
+ });
+
+ // The probe can never observe a factory switching from configs to a
+ // constructed driver — `OptsHandler` rejects the second shape — but what that
+ // rejection must not do is fail a query. It surfaces as a probe failure, and
+ // the driver the deployment is already using keeps serving. Pinned because the
+ // staleness check is what put user code on a path that used to be a pure cache
+ // hit, and the driver it hands back must never be one nobody owns.
+ test('keeps serving the cached driver when the factory changes its return shape', async () => {
+ class ConstructedDriver extends BaseDriver {
+ public release = jest.fn(async () => {});
+
+ public testConnection = jest.fn(async () => {});
+
+ public async query(): Promise {
+ return [];
+ }
+ }
+
+ const constructed = new ConstructedDriver();
+ let returnDriver = false;
+ const factory = jest.fn((ctx: any) => (returnDriver
+ ? constructed
+ : { type: 'postgres', password: ctx.securityContext.token }));
+
+ const { core, driverFactory, request, logged } = await createCore(
+ { driverFactory: factory },
+ { token: 'token-a' },
+ );
+
+ const built = await driverFactory('default');
+
+ returnDriver = true;
+ clock.advancePastRebuildInterval();
+ await request({ token: 'token-b' });
+
+ expect(await driverFactory('default')).toBe(built);
+
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(core.builtDrivers).toHaveLength(1);
+ expect(built.release).not.toHaveBeenCalled();
+ // Not touched either: whatever the factory constructed belongs to the
+ // factory, which may be handing out a singleton it expects to keep working.
+ expect(constructed.release).not.toHaveBeenCalled();
+ expect(logged('Driver staleness check error')).toHaveLength(1);
+ });
+
+ // The refresh scheduler's default context carries no security context at all,
+ // so it shares an orchestrator with API traffic on a deployment that does not
+ // partition by user.
+ test('treats an absent security context as a change in both directions', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({
+ type: 'postgres',
+ password: ctx.securityContext?.token ?? 'service-account',
+ }),
+ }, { token: 'token-a' });
+
+ const user = await driverFactory('default');
+ expect(user.builtFrom).toMatchObject({ password: 'token-a' });
+
+ await request(undefined);
+ const scheduler = await driverFactory('default');
+
+ expect(scheduler).not.toBe(user);
+ expect(scheduler.builtFrom).toMatchObject({ password: 'service-account' });
+ expect(core.builtDrivers).toHaveLength(2);
+ });
+ // The other half of CUB-3599, and the half no comparison can reach: the
+ // credential stopped rotating instead of rotating to something new. The
+ // factory keeps resolving an identical configuration while the connection
+ // built from it is already dead, so only a stated lifetime can replace it.
+ test('rebuilds when the stated lifetime elapses, configuration unchanged', async () => {
+ const expiresAt = Date.now() + 60 * 60 * 1000;
+ const { core, driverFactory } = await createCore({
+ driverFactory: () => ({ type: 'postgres', password: 'frozen-token', expiresAt }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ // Same request context, same configuration — and past the deadline.
+ clock.advance(61 * 60 * 1000);
+
+ const second = await driverFactory('default');
+
+ expect(second).not.toBe(first);
+ expect(core.builtDrivers).toHaveLength(2);
+ expect(first.release).toHaveBeenCalled();
+ });
+
+ test('reports the lifetime rebuild as its own reason', async () => {
+ const expiresAt = Date.now() + 60 * 60 * 1000;
+ const { driverFactory, logged } = await createCore({
+ driverFactory: () => ({ type: 'postgres', password: 'frozen-token', expiresAt }),
+ }, { token: 'token-a' });
+
+ await driverFactory('default');
+ clock.advance(61 * 60 * 1000);
+ await driverFactory('default');
+
+ const rebuilds = logged('Rebuilding driver');
+
+ expect(rebuilds).toHaveLength(1);
+ expect(rebuilds[0]).toMatchObject({ reason: 'lifetime elapsed', rebuildCount: 1 });
+ });
+
+ test('keeps the driver until its lifetime elapses', async () => {
+ const expiresAt = Date.now() + 60 * 60 * 1000;
+ const { core, driverFactory } = await createCore({
+ driverFactory: () => ({ type: 'postgres', password: 'frozen-token', expiresAt }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ clock.advance(59 * 60 * 1000);
+
+ expect(await driverFactory('default')).toBe(first);
+ expect(core.builtDrivers).toHaveLength(1);
+ });
+
+ // A deadline recomputed from the current clock differs on every call. It is
+ // excluded from the configuration's identity precisely so that a factory
+ // written that way does not rebuild the pool on a timer.
+ test('a lifetime that changes on every call is not a configuration change', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: () => ({
+ type: 'postgres',
+ password: 'stable-token',
+ expiresAt: Date.now() + 60 * 60 * 1000,
+ }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ for (let i = 1; i <= 5; i++) {
+ clock.advancePastRebuildInterval();
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token: `token-${i}` }, `req-${i}`);
+ // eslint-disable-next-line no-await-in-loop
+ expect(await driverFactory('default')).toBe(first);
+ }
+
+ expect(core.builtDrivers).toHaveLength(1);
+ });
+
+ // The lifetime is not fingerprinted, so a credential re-issued with the same
+ // value and a later deadline compares equal. Carrying the new deadline over is
+ // what stops that driver from being rebuilt on the old one, once per window,
+ // for as long as the process runs.
+ test('carries a later deadline over when the configuration is unchanged', async () => {
+ let expiresAt = Date.now() + 60 * 60 * 1000;
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: () => ({ type: 'postgres', password: 'stable-token', expiresAt }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ // The credential is re-issued 30 minutes in: same token, later deadline.
+ clock.advance(30 * 60 * 1000);
+ expiresAt = Date.now() + 60 * 60 * 1000;
+ await request({ token: 'token-b' });
+
+ expect(await driverFactory('default')).toBe(first);
+
+ // Past the original deadline, inside the new one.
+ clock.advance(45 * 60 * 1000);
+
+ expect(await driverFactory('default')).toBe(first);
+ expect(core.builtDrivers).toHaveLength(1);
+ });
+
+ // A `driverFactory` that fails closed on an unusable credential is stating
+ // that the connection must not serve queries. Reusing the cached driver
+ // forever because the refusal arrives as a throw is how an expired credential
+ // goes on being served from a pool nobody rebuilds.
+ test('gives the driver up when the factory keeps refusing', async () => {
+ let shouldFail = false;
+ const { core, driverFactory, request, logged } = await createCore({
+ driverFactory: (ctx: any) => {
+ if (shouldFail) {
+ throw new Error('credential is unusable');
+ }
+
+ return { type: 'postgres', password: ctx.securityContext.token };
+ },
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ shouldFail = true;
+
+ // Two refusals inside the grace window: still transient as far as this can
+ // tell, so the cached driver is reused.
+ await request({ token: 'token-b' }, 'req-2');
+ expect(await driverFactory('default')).toBe(first);
+
+ clock.advance(4 * 60 * 1000);
+ await request({ token: 'token-c' }, 'req-3');
+ expect(await driverFactory('default')).toBe(first);
+
+ // Sustained past the grace window — and with no gap long enough to have
+ // aged the earlier refusals out. The driver is given up, and the caller
+ // sees the factory's own error rather than a connection it refused to build.
+ clock.advance(4 * 60 * 1000);
+ await request({ token: 'token-d' }, 'req-4');
+
+ await expect(driverFactory('default')).rejects.toThrow('credential is unusable');
+
+ await new Promise(process.nextTick);
+ expect((first).release).toHaveBeenCalled();
+ expect(core.builtDrivers).toHaveLength(1);
+
+ const released = logged('Rebuilding driver')
+ .filter((params: any) => params.reason === 'repeated staleness check failures');
+
+ expect(released).toHaveLength(1);
+ expect(released[0]).toMatchObject({ dataSource: 'default', rebuildCount: 1 });
+
+ // And once the credential is usable again, the next request rebuilds.
+ shouldFail = false;
+ await request({ token: 'token-e' }, 'req-5');
+
+ const rebuilt = await driverFactory('default');
+
+ expect(rebuilt.builtFrom).toMatchObject({ password: 'token-e' });
+ });
+
+ test('a recovered factory resets the refusal count', async () => {
+ let shouldFail = false;
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => {
+ if (shouldFail) {
+ throw new Error('secret store unreachable');
+ }
+
+ return { type: 'postgres', password: ctx.securityContext.token };
+ },
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ // Two refusals, then a probe that resolves — which clears the count, so the
+ // two refusals after it cannot reach the bound between them.
+ for (const token of ['token-b', 'token-c']) {
+ shouldFail = true;
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token }, `req-${token}`);
+ // eslint-disable-next-line no-await-in-loop
+ expect(await driverFactory('default')).toBe(first);
+ clock.advance(31 * 1000);
+ }
+
+ shouldFail = false;
+ await request({ token: 'token-a' }, 'req-recovered');
+ expect(await driverFactory('default')).toBe(first);
+
+ shouldFail = true;
+ clock.advance(31 * 1000);
+ await request({ token: 'token-d' }, 'req-d');
+
+ expect(await driverFactory('default')).toBe(first);
+ expect(core.builtDrivers).toHaveLength(1);
+ });
+
+ // Probes only run when the security context changes, so in a quiet deployment
+ // they can be hours apart. A counter that only ever went up would read three
+ // unrelated flakes on three different days as one sustained outage and tear
+ // down a working pool for it.
+ test('refusals spread beyond the window do not accumulate', async () => {
+ let shouldFail = false;
+ const { core, driverFactory, request, logged } = await createCore({
+ driverFactory: (ctx: any) => {
+ if (shouldFail) {
+ throw new Error('secret store unreachable');
+ }
+
+ return { type: 'postgres', password: ctx.securityContext.token };
+ },
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ shouldFail = true;
+
+ // Past retention between each, so every refusal opens a fresh window rather
+ // than extending the one before it. Six of them, twice the bound, and the
+ // driver is still there.
+ for (const token of ['token-b', 'token-c', 'token-d', 'token-e', 'token-f', 'token-g']) {
+ clock.advance(31 * 60 * 1000);
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token }, `req-${token}`);
+ // eslint-disable-next-line no-await-in-loop
+ expect(await driverFactory('default')).toBe(first);
+ }
+
+ expect(core.builtDrivers).toHaveLength(1);
+ expect((first).release).not.toHaveBeenCalled();
+ expect(logged('Rebuilding driver')).toHaveLength(0);
+ });
+
+ // Giving a driver up tears down the same pool a configuration change does, so
+ // it has to answer to the same rate limit. Bypassing it let a factory that
+ // fails probes but succeeds when called directly churn a pool every window,
+ // with nothing in the logs that read as churn.
+ test('rate-limits repeated give-ups', async () => {
+ let shouldFailProbe = false;
+ const { core, driverFactory, request, logged } = await createCore({
+ driverFactory: (ctx: any) => {
+ if (shouldFailProbe) {
+ throw new Error('secret store unreachable');
+ }
+
+ return { type: 'postgres', password: ctx.securityContext.token };
+ },
+ }, { token: 'token-a' });
+
+ await driverFactory('default');
+
+ // Three refusals spanning the grace window give the driver up. The build
+ // that follows succeeds, because it calls the factory directly.
+ shouldFailProbe = true;
+
+ for (const token of ['token-b', 'token-c']) {
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token }, `req-${token}`);
+ // eslint-disable-next-line no-await-in-loop
+ await driverFactory('default');
+ clock.advance(4 * 60 * 1000);
+ }
+
+ await request({ token: 'token-d' }, 'req-d');
+ shouldFailProbe = false;
+
+ const rebuilt = await driverFactory('default');
+
+ expect(logged('Rebuilding driver')).toHaveLength(1);
+ expect(core.builtDrivers).toHaveLength(2);
+
+ // The give-up opened a suppression window like any other replacement, so
+ // the next changed context reuses rather than tearing the new pool down.
+ await request({ token: 'token-e' }, 'req-e');
+
+ expect(await driverFactory('default')).toBe(rebuilt);
+ expect(core.builtDrivers).toHaveLength(2);
+ expect(logged('Driver rebuild suppressed')).toHaveLength(1);
+ });
+
+ // The other half of that contract. Probes only fire when the security context
+ // changes, so a few-user deployment may probe far slower than the grace
+ // window — and a dead credential there has to be given up eventually, which
+ // is what retention being longer than the grace window buys.
+ test('gives up a refusal that is sustained but slow', async () => {
+ let shouldFail = false;
+ const { driverFactory, request, logged } = await createCore({
+ driverFactory: (ctx: any) => {
+ if (shouldFail) {
+ throw new Error('credential is unusable');
+ }
+
+ return { type: 'postgres', password: ctx.securityContext.token };
+ },
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ shouldFail = true;
+
+ // Ten minutes apart: slower than the grace window, well inside retention.
+ for (const token of ['token-b', 'token-c']) {
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token }, `req-${token}`);
+ // eslint-disable-next-line no-await-in-loop
+ expect(await driverFactory('default')).toBe(first);
+ clock.advance(10 * 60 * 1000);
+ }
+
+ await request({ token: 'token-d' }, 'req-d');
+
+ await expect(driverFactory('default')).rejects.toThrow('credential is unusable');
+
+ await new Promise(process.nextTick);
+ expect((first).release).toHaveBeenCalled();
+ expect(logged('Rebuilding driver')
+ .filter((params: any) => params.reason === 'repeated staleness check failures'))
+ .toHaveLength(1);
+ });
+
+ // Retention outliving the grace window must not make the bound reachable
+ // across unrelated incidents. Concurrent probes all fail on one blink of a
+ // dependency, so a burst is one refusal — otherwise two brief outages half an
+ // hour apart would drain a working pool.
+ test('counts a burst of refusals as one incident', async () => {
+ let shouldFail = false;
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => {
+ if (shouldFail) {
+ throw new Error('secret store unreachable');
+ }
+
+ return { type: 'postgres', password: ctx.securityContext.token };
+ },
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ shouldFail = true;
+
+ // Three refusals at the same instant: one dependency blink under load.
+ for (const token of ['token-b', 'token-c', 'token-d']) {
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token }, `req-${token}`);
+ // eslint-disable-next-line no-await-in-loop
+ expect(await driverFactory('default')).toBe(first);
+ }
+
+ // A second, unrelated blink six minutes later. Two incidents, not four
+ // refusals, so the bound is not reached and the pool survives.
+ clock.advance(6 * 60 * 1000);
+ await request({ token: 'token-e' }, 'req-e');
+
+ expect(await driverFactory('default')).toBe(first);
+ expect((first).release).not.toHaveBeenCalled();
+ expect(core.builtDrivers).toHaveLength(1);
+ });
+
+ // The other end of the traffic range from the burst case. Under steady load
+ // refusals arrive faster than the coalescing window, so they are all one
+ // incident — and an incident that never ends must be caught by having lasted,
+ // or a permanently dead credential is never given up where it costs most.
+ test('gives up an unbroken refusal stream under steady traffic', async () => {
+ let shouldFail = false;
+ const { driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => {
+ if (shouldFail) {
+ throw new Error('credential is unusable');
+ }
+
+ return { type: 'postgres', password: ctx.securityContext.token };
+ },
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ shouldFail = true;
+
+ const startedAt = Date.now();
+ let gaveUpAfterMs: number | undefined;
+
+ // A refusal every 1.5s — inside the coalescing window, so nothing here ever
+ // starts a second incident.
+ for (let i = 0; i < 400 && gaveUpAfterMs === undefined; i++) {
+ clock.advance(1500);
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token: `token-${i}` }, `req-${i}`);
+
+ try {
+ // eslint-disable-next-line no-await-in-loop
+ await driverFactory('default');
+ } catch (error) {
+ gaveUpAfterMs = Date.now() - startedAt;
+ }
+ }
+
+ // Given up once the incident had itself run the grace window, not before.
+ expect(gaveUpAfterMs).toBeGreaterThanOrEqual(5 * 60 * 1000);
+ expect(gaveUpAfterMs).toBeLessThan(6 * 60 * 1000);
+
+ await new Promise(process.nextTick);
+ expect((first).release).toHaveBeenCalled();
+ });
+
+ // The other side of that boundary. An incident bounded by its duration has to
+ // be reused right up until the duration is reached, or a single dependency
+ // outage shorter than the grace window drains the pool after all.
+ test('reuses through an incident shorter than the grace window', async () => {
+ let shouldFail = false;
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => {
+ if (shouldFail) {
+ throw new Error('secret store unreachable');
+ }
+
+ return { type: 'postgres', password: ctx.securityContext.token };
+ },
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ shouldFail = true;
+
+ // Four minutes of refusals at 1.5s — one unbroken incident, under the bound.
+ for (let i = 0; i < 160; i++) {
+ clock.advance(1500);
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token: `token-${i}` }, `req-${i}`);
+ // eslint-disable-next-line no-await-in-loop
+ expect(await driverFactory('default')).toBe(first);
+ }
+
+ expect((first).release).not.toHaveBeenCalled();
+ expect(core.builtDrivers).toHaveLength(1);
+ });
+
+ // Replacing a driver cannot move a deadline the factory keeps re-asserting.
+ // Honouring one would find the new driver stale the moment its suppression
+ // window closed, for the life of the process.
+ test('ignores a lifetime that has already elapsed', async () => {
+ const { core, driverFactory, request, logged } = await createCore({
+ driverFactory: (ctx: any) => ({
+ type: 'postgres',
+ password: ctx.securityContext.token,
+ // A factory passing the provider's expiry straight through, on a
+ // credential whose refresh has already stopped.
+ expiresAt: Date.now() - 60 * 1000,
+ }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ expect(logged('Driver lifetime ignored')).toHaveLength(1);
+
+ // Past any suppression window: an honoured deadline would rebuild here, and
+ // again on every window after it.
+ clock.advancePastRebuildInterval();
+ await request({ token: 'token-a' }, 'req-2');
+
+ expect(await driverFactory('default')).toBe(first);
+
+ clock.advancePastRebuildInterval();
+ await request({ token: 'token-a' }, 'req-3');
+
+ expect(await driverFactory('default')).toBe(first);
+ expect(core.builtDrivers).toHaveLength(1);
+ expect(logged('Rebuilding driver')).toHaveLength(0);
+ });
+
+ // The deadline is resolved again on every probe, and probes run whenever the
+ // security context changes. Reporting per call would trade the churn the
+ // guard removes for a warning on every query that arrives with a fresh JWT.
+ test('reports an ignored lifetime once, not once per probe', async () => {
+ const { core, driverFactory, request, logged } = await createCore({
+ // Ignores the context: the configuration never changes, only the elapsed
+ // deadline it keeps re-asserting.
+ driverFactory: () => ({
+ type: 'postgres',
+ password: 'service-account',
+ expiresAt: Date.now() - 60 * 1000,
+ }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ // Each of these changes the fingerprint, so each one probes and each one
+ // carries the same unusable deadline over.
+ for (const token of ['token-b', 'token-c', 'token-d', 'token-e']) {
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token }, `req-${token}`);
+ // eslint-disable-next-line no-await-in-loop
+ expect(await driverFactory('default')).toBe(first);
+ }
+
+ expect(logged('Driver lifetime ignored')).toHaveLength(1);
+ expect(core.builtDrivers).toHaveLength(1);
+ });
+
+ // A lifetime shorter than the replacement interval is the same loop as an
+ // elapsed one: stale the moment the suppression window closes, every window,
+ // for the life of the process.
+ test('ignores a lifetime shorter than the replacement interval', async () => {
+ const { core, driverFactory, request, logged } = await createCore({
+ driverFactory: () => ({
+ type: 'postgres',
+ password: 'sts-credential',
+ // Genuinely in the future, and still too short for the rate limiter to
+ // ever let this mechanism act on it.
+ expiresAt: Date.now() + 10 * 1000,
+ }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ expect(logged('Driver lifetime ignored')).toHaveLength(1);
+
+ // Past the deadline and past any suppression window, twice over.
+ for (const requestId of ['req-2', 'req-3']) {
+ clock.advancePastRebuildInterval();
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token: 'token-a' }, requestId);
+ // eslint-disable-next-line no-await-in-loop
+ expect(await driverFactory('default')).toBe(first);
+ }
+
+ expect(core.builtDrivers).toHaveLength(1);
+ expect(logged('Rebuilding driver')).toHaveLength(0);
+ });
+
+ // The lower bound judges what the factory states, not how much of an accepted
+ // deadline is left. Re-judging on the carry-over would drop a good deadline as
+ // it entered its final window, switching the lifetime off in exactly the
+ // stretch it exists to cover.
+ test('keeps a deadline that a probe lands inside the final window of', async () => {
+ const expiresAt = Date.now() + 60 * 60 * 1000;
+ const { core, driverFactory, request, logged } = await createCore({
+ // Ignores the context, so the configuration compares equal and the probe
+ // takes the carry-over path.
+ driverFactory: () => ({ type: 'postgres', password: 'static', expiresAt }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ // A re-issued JWT ten seconds before the deadline: inside the replacement
+ // interval, but this deadline was judged an hour ago and accepted.
+ clock.advance(60 * 60 * 1000 - 10 * 1000);
+ await request({ token: 'token-b' }, 'req-2');
+
+ expect(await driverFactory('default')).toBe(first);
+ expect(logged('Driver lifetime ignored')).toHaveLength(0);
+
+ // And the deadline it kept still fires.
+ clock.advance(11 * 1000);
+ await request({ token: 'token-c' }, 'req-3');
+
+ expect(await driverFactory('default')).not.toBe(first);
+ expect(core.builtDrivers).toHaveLength(2);
+ });
+
+ // A deadline is excluded from the fingerprint, so an unchanged credential can
+ // arrive with a moved one. If that new deadline cannot be honoured, the one
+ // this driver was already held to still can — dropping to no deadline at all
+ // is the failure the stated-not-aged rule exists to prevent.
+ test('keeps the accepted deadline when a newly stated one cannot be honoured', async () => {
+ const accepted = Date.now() + 60 * 60 * 1000;
+ let expiresAt = accepted;
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: () => ({ type: 'postgres', password: 'static', expiresAt }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ // Ten seconds before the accepted deadline, the factory re-states the same
+ // credential with a slightly different deadline — inside the interval, so
+ // unhonourable as a new lifetime.
+ clock.advance(60 * 60 * 1000 - 10 * 1000);
+ expiresAt = accepted + 5 * 1000;
+ await request({ token: 'token-b' }, 'req-2');
+
+ expect(await driverFactory('default')).toBe(first);
+
+ // The accepted deadline still fires.
+ clock.advance(11 * 1000);
+ await request({ token: 'token-c' }, 'req-3');
+
+ expect(await driverFactory('default')).not.toBe(first);
+ expect(core.builtDrivers).toHaveLength(2);
+ });
+
+ // The other half of the lifetime contract: a deadline the rate limiter can
+ // honour is still honoured, so widening the guard did not disable the feature.
+ test('honours a lifetime longer than the replacement interval', async () => {
+ let expiresAt = Date.now() + 60 * 60 * 1000;
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: () => ({ type: 'postgres', password: 'static', expiresAt }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ clock.advance(61 * 60 * 1000);
+ expiresAt = Date.now() + 60 * 60 * 1000;
+ await request({ token: 'token-a' }, 'req-2');
+
+ expect(await driverFactory('default')).not.toBe(first);
+ expect(core.builtDrivers).toHaveLength(2);
+ });
+});
diff --git a/packages/cubejs-server-core/test/unit/driver-config-expiry.test.ts b/packages/cubejs-server-core/test/unit/driver-config-expiry.test.ts
new file mode 100644
index 0000000000000..a1a8215b374fd
--- /dev/null
+++ b/packages/cubejs-server-core/test/unit/driver-config-expiry.test.ts
@@ -0,0 +1,129 @@
+import { CreateOptions, CubejsServerCore } from '../../src';
+import { parseDriverExpiry, withoutDriverExpiry } from '../../src/core/driver-config-expiry';
+
+describe('parseDriverExpiry', () => {
+ const iso = '2026-08-17T22:29:31.136Z';
+ const epochMs = Date.parse(iso);
+
+ test('reads an ISO 8601 string, which is how a credential usually carries it', () => {
+ expect(parseDriverExpiry(iso)).toBe(epochMs);
+ expect(parseDriverExpiry('2026-08-17T22:29:31.136+00:00')).toBe(epochMs);
+ expect(parseDriverExpiry(` ${iso} `)).toBe(epochMs);
+ });
+
+ test('reads a Date', () => {
+ expect(parseDriverExpiry(new Date(epochMs))).toBe(epochMs);
+ });
+
+ test('reads epoch milliseconds and epoch seconds alike', () => {
+ expect(parseDriverExpiry(epochMs)).toBe(epochMs);
+ // What `time.time() + 3600` in a Python config produces. Read as seconds it
+ // is 2026; read as milliseconds it would be 1970, and every driver built
+ // from it would be born expired.
+ expect(parseDriverExpiry(Math.floor(epochMs / 1000))).toBe(Math.floor(epochMs / 1000) * 1000);
+ });
+
+ test('reads a stringified timestamp, which Date.parse would take for a year', () => {
+ expect(parseDriverExpiry(String(epochMs))).toBe(epochMs);
+ expect(parseDriverExpiry(String(Math.floor(epochMs / 1000)))).toBe(
+ Math.floor(epochMs / 1000) * 1000,
+ );
+ });
+
+ test('drops anything that is not a moment in time', () => {
+ // Dropped rather than thrown: the lifetime is an optimisation over
+ // comparing configurations, and failing a deployment's queries over a
+ // malformed one would be worse than the behaviour it had before the field
+ // existed.
+ expect(parseDriverExpiry(undefined)).toBeUndefined();
+ expect(parseDriverExpiry(null)).toBeUndefined();
+ expect(parseDriverExpiry('')).toBeUndefined();
+ expect(parseDriverExpiry(' ')).toBeUndefined();
+ expect(parseDriverExpiry('whenever')).toBeUndefined();
+ expect(parseDriverExpiry(new Date('nonsense'))).toBeUndefined();
+ expect(parseDriverExpiry(0)).toBeUndefined();
+ expect(parseDriverExpiry(-1)).toBeUndefined();
+ expect(parseDriverExpiry(NaN)).toBeUndefined();
+ expect(parseDriverExpiry(Infinity)).toBeUndefined();
+ expect(parseDriverExpiry(true)).toBeUndefined();
+ expect(parseDriverExpiry({ expiresAt: iso })).toBeUndefined();
+ expect(parseDriverExpiry([iso])).toBeUndefined();
+ });
+});
+
+describe('withoutDriverExpiry', () => {
+ test('strips the lifetime without mutating the input', () => {
+ const config = { type: 'postgres', password: 'secret', expiresAt: 1 };
+
+ expect(withoutDriverExpiry(config)).toEqual({ type: 'postgres', password: 'secret' });
+ expect(config.expiresAt).toBe(1);
+ });
+
+ test('returns the same object when there is nothing to strip', () => {
+ const config = { type: 'postgres' };
+
+ expect(withoutDriverExpiry(config)).toBe(config);
+ });
+});
+
+/**
+ * Every key of a `DriverConfig` other than `type` is passed to the driver's own
+ * constructor, so a lifetime left in would arrive as a connection option — and,
+ * for a configuration that names nothing else, would also displace the pool
+ * defaults that an otherwise-empty configuration is meant to get.
+ */
+describe('driver construction', () => {
+ class ExposedCore extends CubejsServerCore {
+ public build(val: any, context: any) {
+ return this.createDriverFromFactoryResult(val, context);
+ }
+ }
+
+ let created: jest.SpyInstance;
+
+ beforeAll(() => {
+ process.env.CUBEJS_API_SECRET = 'api-secret';
+ });
+
+ beforeEach(() => {
+ created = jest.spyOn(CubejsServerCore, 'createDriver').mockReturnValue({});
+ });
+
+ afterEach(() => {
+ created.mockRestore();
+ });
+
+ function core() {
+ return new ExposedCore({
+ driverFactory: () => ({ type: 'postgres' }),
+ logger: jest.fn(),
+ });
+ }
+
+ test('does not pass the lifetime to the driver', async () => {
+ await core().build(
+ { type: 'postgres', password: 'secret', expiresAt: '2026-08-17T22:29:31.136Z' },
+ { dataSource: 'default' },
+ );
+
+ expect(created).toHaveBeenCalledWith('postgres', expect.not.objectContaining({
+ expiresAt: expect.anything(),
+ }));
+ expect(created).toHaveBeenCalledWith('postgres', expect.objectContaining({
+ password: 'secret',
+ dataSource: 'default',
+ }));
+ });
+
+ test('a configuration carrying only a lifetime still gets the pool defaults', async () => {
+ await core().build(
+ { type: 'postgres', expiresAt: '2026-08-17T22:29:31.136Z' },
+ { dataSource: 'default' },
+ );
+
+ const [, opts] = created.mock.calls[0];
+
+ expect(opts).not.toHaveProperty('expiresAt');
+ expect(opts).toHaveProperty('maxPoolSize');
+ });
+});
diff --git a/packages/cubejs-server-core/test/unit/driver-config-fingerprint.test.ts b/packages/cubejs-server-core/test/unit/driver-config-fingerprint.test.ts
new file mode 100644
index 0000000000000..b0572385cfd2d
--- /dev/null
+++ b/packages/cubejs-server-core/test/unit/driver-config-fingerprint.test.ts
@@ -0,0 +1,65 @@
+import { fingerprint } from '../../src/core/driver-config-fingerprint';
+
+describe('fingerprint', () => {
+ test('is stable across property order', () => {
+ expect(fingerprint({ type: 'databricks-jdbc', token: 'a', url: 'u' }))
+ .toEqual(fingerprint({ url: 'u', token: 'a', type: 'databricks-jdbc' }));
+ });
+
+ test('changes when a nested value changes', () => {
+ expect(fingerprint({ type: 'postgres', options: { password: 'one' } }))
+ .not.toEqual(fingerprint({ type: 'postgres', options: { password: 'two' } }));
+ });
+
+ // The case this exists for: a rotated per-user OAuth token has to be visible
+ // as a different configuration, or the cached driver is never rebuilt.
+ test('changes when only the credential changes', () => {
+ const base = { type: 'databricks-jdbc', url: 'jdbc:databricks://host', acceptPolicy: true };
+
+ expect(fingerprint({ ...base, token: 'token-issued-at-09:00' }))
+ .not.toEqual(fingerprint({ ...base, token: 'token-issued-at-10:00' }));
+ });
+
+ test('treats an absent property and an undefined one as equal', () => {
+ expect(fingerprint({ type: 'postgres', catalog: undefined }))
+ .toEqual(fingerprint({ type: 'postgres' }));
+ });
+
+ test('distinguishes arrays by order', () => {
+ expect(fingerprint({ scopes: ['a', 'b'] })).not.toEqual(fingerprint({ scopes: ['b', 'a'] }));
+ });
+
+ test('handles dates, bigints and nested structures', () => {
+ const value = {
+ when: new Date('2026-07-31T12:00:00.000Z'),
+ big: BigInt(42),
+ nested: [{ a: 1 }, { b: [true, null] }],
+ };
+
+ expect(fingerprint(value)).toEqual(fingerprint({
+ nested: [{ a: 1 }, { b: [true, null] }],
+ big: BigInt(42),
+ when: new Date('2026-07-31T12:00:00.000Z'),
+ }));
+ expect(fingerprint(value)).not.toEqual(fingerprint({ ...value, big: BigInt(43) }));
+ });
+
+ test('does not expose the value it hashes', () => {
+ const digest = fingerprint({ type: 'postgres', password: 'super-secret' });
+
+ expect(digest).not.toContain('super-secret');
+ expect(digest).toMatch(/^[0-9a-f]{32}$/);
+ });
+
+ test('returns null for a circular structure rather than throwing', () => {
+ const circular: Record = { type: 'postgres' };
+ circular.self = circular;
+
+ expect(fingerprint(circular)).toBeNull();
+ });
+
+ test('returns a digest for null and undefined', () => {
+ expect(fingerprint(null)).toEqual(fingerprint(undefined));
+ expect(fingerprint(null)).not.toBeNull();
+ });
+});