feat: Add the file-based override source

packages/sdk/server-node/__tests__/LDClientNode.fileOverrides.test.ts

@@ -0,0 +1,135 @@
+import { mkdtemp, rm, unlink, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { FileOverrideSourceOptions } from '@launchdarkly/js-server-sdk-common';
+
+import LDClientNode from '../src/LDClientNode';
+import waitFor, { sleep } from './waitFor';
+
+const user = { key: 'user-key' };
+
+// The client never obtains data from LaunchDarkly: its only initializer reads a file that does
+// not exist. Overrides are served from the override layer alone.
+function makeClient(
+  directory: string,
+  overrides: Omit<FileOverrideSourceOptions, 'type'>,
+): LDClientNode {
+  return new LDClientNode('sdk-key-file-overrides', {
+    sendEvents: false,
+    diagnosticOptOut: true,
+    logger: { error: () => {}, warn: () => {}, info: () => {}, debug: () => {} },
+    dataSystem: {
+      dataSource: {
+        dataSourceOptionsType: 'custom',
+        initializers: [{ type: 'file', paths: [join(directory, 'no-launchdarkly-data.json')] }],
+        synchronizers: [],
+      },
+      overrides: { type: 'file', ...overrides },
+    },
+  });
+}
+
+const document = (value: string) => JSON.stringify({ flagValues: { 'overridden-flag': value } });
+
+describe('given a temporary directory of override files', () => {
+  let directory: string;
+  let client: LDClientNode | undefined;
+
+  beforeEach(async () => {
+    directory = await mkdtemp(join(tmpdir(), 'ld-file-overrides-'));
+  });
+
+  afterEach(async () => {
+    client?.close();
+    client = undefined;
+    await rm(directory, { recursive: true, force: true });
+  });
+
+  it('reads a YAML file with the built-in parser', async () => {
+    const path = join(directory, 'overrides.yaml');
+    await writeFile(path, 'flagValues:\n  yaml-flag: "override-value"\n');
+    client = makeClient(directory, { paths: [path] });
+
+    // Evaluation does not wait for the initial load, so wait for the override to be in effect.
+    await waitFor(
+      async () =>
+        (await client!.variationDetail('yaml-flag', user, 'default')).value === 'override-value',
+      10000,
+    );
+    const detail = await client.variationDetail('yaml-flag', user, 'default');
+
+    expect(detail.value).toEqual('override-value');
+    expect(detail.reason).toEqual({ kind: 'FALLTHROUGH', overrideAffected: true });
+    expect(client.initialized()).toBe(false);
+  });
+
+  it('reloads a changed file in polling mode', async () => {
+    const path = join(directory, 'overrides.json');
+    await writeFile(path, document('b'));
+    client = makeClient(directory, { paths: [path], changeDetection: 'polling', pollInterval: 1 });
+    await waitFor(
+      async () => (await client!.variation('overridden-flag', user, 'default')) === 'b',
+      10000,
+    );
+
+    await writeFile(path, document('c'));
+
+    await waitFor(
+      async () => (await client!.variation('overridden-flag', user, 'default')) === 'c',
+      10000,
+    );
+  });
+
+  it('picks up a file that does not exist yet in watching mode', async () => {
+    const path = join(directory, 'overrides.json');
+    client = makeClient(directory, { paths: [path], changeDetection: 'watching' });
+    const before = await client.variationDetail('overridden-flag', user, 'default');
+    expect(before.reason).toEqual({ kind: 'ERROR', errorKind: 'CLIENT_NOT_READY' });
+
+    await writeFile(path, document('b'));
+
+    await waitFor(
+      async () => (await client!.variation('overridden-flag', user, 'default')) === 'b',
+      10000,
+    );
+  });
+
+  it('removes the overrides of a deleted file', async () => {
+    const path = join(directory, 'overrides.json');
+    await writeFile(path, document('b'));
+    client = makeClient(directory, { paths: [path] });
+    await waitFor(
+      async () => (await client!.variation('overridden-flag', user, 'default')) === 'b',
+      10000,
+    );
+
+    await unlink(path);
+
+    await waitFor(async () => {
+      const detail = await client!.variationDetail('overridden-flag', user, 'default');
+      return detail.reason.errorKind === 'CLIENT_NOT_READY';
+    }, 10000);
+  });
+
+  it('keeps the last good overrides while the file is malformed', async () => {
+    const path = join(directory, 'overrides.json');
+    await writeFile(path, document('b'));
+    client = makeClient(directory, { paths: [path] });
+    await waitFor(
+      async () => (await client!.variation('overridden-flag', user, 'default')) === 'b',
+      10000,
+    );
+
+    await writeFile(path, '{"flagValues"');
+    await sleep(1500);
+    expect(await client.variation('overridden-flag', user, 'default')).toEqual('b');
+
+    await writeFile(path, document('c'));
+
+    await waitFor(
+      async () => (await client!.variation('overridden-flag', user, 'default')) === 'c',
+      10000,
+    );
+  });
+});

packages/sdk/server-node/__tests__/waitFor.ts

@@ -0,0 +1,28 @@
+/**
+ * Resolves after the given number of milliseconds.
+ */
+export function sleep(ms: number): Promise<void> {
+  return new Promise<void>((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+/**
+ * Polls a condition until it holds or the timeout elapses.
+ */
+export default async function waitFor(
+  condition: () => Promise<boolean> | boolean,
+  timeoutMs: number = 5000,
+  intervalMs: number = 20,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    // eslint-disable-next-line no-await-in-loop
+    if (await condition()) {
+      return;
+    }
+    // eslint-disable-next-line no-await-in-loop
+    await sleep(intervalMs);
+  }
+  throw new Error('timed out waiting for the condition');
+}

packages/sdk/server-node/package.json

@@ -49,7 +49,8 @@
   "dependencies": {
     "@launchdarkly/js-server-sdk-common": "2.24.1",
     "https-proxy-agent": "^7.0.6",
-    "launchdarkly-eventsource": "2.3.0"
+    "launchdarkly-eventsource": "2.3.0",
+    "yaml": "2.9.1"
   },
   "devDependencies": {
     "@types/jest": "^29.4.0",

packages/sdk/server-node/src/LDClientNode.ts

@@ -1,5 +1,6 @@
 import { EventEmitter } from 'events';
 import { format } from 'util';
+import { parse as parseYaml } from 'yaml';
 
 import {
   BasicLogger,
@@ -89,6 +90,8 @@ class LDClientNode extends LDClientImpl implements LDClient {
         getImplementationHooks: (environmentMetadata: LDPluginEnvironmentMetadata) =>
           internal.safeGetHooks(logger, environmentMetadata, plugins),
         instanceId,
+        // The Node platform reads YAML files without configuration from the application.
+        yamlParser: (data: string) => parseYaml(data),
       },
     );
 

packages/shared/sdk-server/__tests__/LDClientImpl.fileOverrides.test.ts

@@ -0,0 +1,131 @@
+import { FileOverrideSourceOptions } from '../src/api/options/LDDataSystemOptions';
+import { LDOptions } from '../src/api/options/LDOptions';
+import LDClientImpl from '../src/LDClientImpl';
+import MockFilesystem from './data_sources/filedata/MockFilesystem';
+import makeMockLogger from './mockLogger';
+import { makeCallbacks, makeFDv2Platform } from './overrides/overridesTestSupport';
+import waitFor from './waitFor';
+
+const user = { key: 'user-key' };
+const directory = '/etc/launchdarkly';
+const jsonPath = `${directory}/overrides.json`;
+const yamlPath = `${directory}/overrides.yaml`;
+
+describe('given a client with a file override source over a mock filesystem', () => {
+  let filesystem: MockFilesystem;
+  let client: LDClientImpl | undefined;
+  let platformYamlParser: jest.Mock;
+
+  const makeClient = (
+    overrides: Omit<FileOverrideSourceOptions, 'type'>,
+    options: LDOptions = {},
+  ) => {
+    client = new LDClientImpl(
+      'sdk-key-file-overrides',
+      { ...makeFDv2Platform(), fileSystem: filesystem },
+      {
+        sendEvents: false,
+        diagnosticOptOut: true,
+        logger: makeMockLogger(),
+        ...options,
+        dataSystem: {
+          dataSource: {
+            dataSourceOptionsType: 'custom',
+            initializers: [],
+            synchronizers: [{ type: 'polling', pollInterval: 1000 }],
+          },
+          overrides: { type: 'file', changeDetection: 'watching', ...overrides },
+        },
+      },
+      makeCallbacks(),
+      { yamlParser: platformYamlParser },
+    );
+    return client;
+  };
+
+  beforeEach(() => {
+    filesystem = new MockFilesystem();
+    platformYamlParser = jest.fn(() => ({ flagValues: { 'yaml-flag': 'from-platform-parser' } }));
+  });
+
+  afterEach(() => {
+    client?.close();
+    client = undefined;
+  });
+
+  it('serves an override from a file that is present when the client is created', async () => {
+    filesystem.set(jsonPath, '{"flagValues": {"overridden-flag": true}}');
+    makeClient({ paths: [jsonPath] });
+
+    // The initial load reads the file asynchronously and evaluation does not wait for it, so the
+    // test waits until the override shows.
+    await waitFor(
+      async () => (await client!.boolVariation('overridden-flag', user, false)) === true,
+    );
+    const detail = await client!.boolVariationDetail('overridden-flag', user, false);
+
+    expect(detail.value).toBe(true);
+    expect(detail.reason).toEqual({ kind: 'FALLTHROUGH', overrideAffected: true });
+    expect(client!.initialized()).toBe(false);
+  });
+
+  it('reads a YAML file with the parser the platform supplies', async () => {
+    filesystem.set(yamlPath, 'flagValues:\n  yaml-flag: from-platform-parser\n');
+    makeClient({ paths: [yamlPath] });
+
+    await waitFor(
+      async () =>
+        (await client!.variation('yaml-flag', user, 'default')) === 'from-platform-parser',
+    );
+    expect(platformYamlParser).toHaveBeenCalledWith(
+      'flagValues:\n  yaml-flag: from-platform-parser\n',
+    );
+  });
+
+  it('prefers a configured YAML parser over the one the platform supplies', async () => {
+    const yamlParser = jest.fn(() => ({ flagValues: { 'yaml-flag': 'from-configured-parser' } }));
+    filesystem.set(yamlPath, 'flagValues:\n  yaml-flag: x\n');
+    makeClient({ paths: [yamlPath], yamlParser });
+
+    await waitFor(
+      async () =>
+        (await client!.variation('yaml-flag', user, 'default')) === 'from-configured-parser',
+    );
+    expect(platformYamlParser).not.toHaveBeenCalled();
+  });
+
+  it('reloads when the directory reports a change', async () => {
+    filesystem.set(jsonPath, '{"flagValues": {"overridden-flag": "b"}}');
+    makeClient({ paths: [jsonPath] });
+    await waitFor(
+      async () => (await client!.variation('overridden-flag', user, 'default')) === 'b',
+    );
+
+    filesystem.set(jsonPath, '{"flagValues": {"overridden-flag": "c"}}');
+    filesystem.emit(directory);
+
+    await waitFor(
+      async () => (await client!.variation('overridden-flag', user, 'default')) === 'c',
+    );
+  });
+
+  it('contributes no overrides for a file that does not exist until it appears', async () => {
+    makeClient({ paths: [jsonPath] });
+
+    const before = await client!.variationDetail('overridden-flag', user, 'default');
+    expect(before.reason).toEqual({ kind: 'ERROR', errorKind: 'CLIENT_NOT_READY' });
+
+    filesystem.set(jsonPath, '{"flagValues": {"overridden-flag": "b"}}');
+    filesystem.emit(directory, 'rename');
+
+    await waitFor(
+      async () => (await client!.variation('overridden-flag', user, 'default')) === 'b',
+    );
+  });
+
+  it('fails client construction when no file paths are configured', () => {
+    expect(() => makeClient({ paths: [] })).toThrow(
+      'The file-based override source requires at least one file path',
+    );
+  });
+});

packages/shared/sdk-server/__tests__/Logger.ts

@@ -93,6 +93,13 @@ export default class TestLogger implements LDLogger {
     });
   }
 
+  /**
+   * The messages received at a level, in order.
+   */
+  getMessages(level: LogLevel): string[] {
+    return [...this._messages[level]];
+  }
+
   getCount(level?: LogLevel) {
     if (level === undefined) {
       return this._callCount;