fix: Notify overridden flags when the LaunchDarkly data they depend on changes

packages/shared/sdk-server/__tests__/LDClientImpl.overrideDependents.test.ts

@@ -0,0 +1,134 @@
+import { AsyncQueue } from 'launchdarkly-js-test-helpers';
+
+import { sleep } from '@launchdarkly/js-sdk-common';
+
+import LDClientImpl from '../src/LDClientImpl';
+import { createBasicPlatform } from './createBasicPlatform';
+import { makeCallbacks, makeFDv2Client } from './overrides/overridesTestSupport';
+import TestOverrideSource from './overrides/TestOverrideSource';
+
+const user = { key: 'user-key' };
+
+// A full FDv2 payload with one LaunchDarkly flag and one segment. The version parameter moves the
+// payload state forward so that a later poll is a new data set rather than a repeat.
+function payload(version: number, segmentVersion: number): string {
+  const events: any[] = [
+    {
+      event: 'server-intent',
+      data: { payloads: [{ intentCode: 'xfer-full', id: 'test-payload', target: version }] },
+    },
+    {
+      event: 'put-object',
+      data: {
+        kind: 'flag',
+        key: 'ld-flag',
+        version: 1,
+        object: {
+          key: 'ld-flag',
+          version: 1,
+          on: true,
+          fallthrough: { variation: 0 },
+          variations: [true, false],
+        },
+      },
+    },
+    {
+      event: 'put-object',
+      data: {
+        kind: 'segment',
+        key: 'segment1',
+        version: segmentVersion,
+        object: { key: 'segment1', version: segmentVersion, included: [user.key] },
+      },
+    },
+    { event: 'payload-transferred', data: { state: `test-state-${version}`, version } },
+  ];
+  return JSON.stringify({ events });
+}
+
+// An override flag whose only rule matches a LaunchDarkly segment.
+function overriddenFlagMatching(segmentKey: string): any {
+  return {
+    key: 'overridden',
+    version: 1,
+    on: true,
+    fallthrough: { variation: 1 },
+    variations: [true, false],
+    rules: [
+      {
+        id: 'rule',
+        variation: 0,
+        clauses: [{ attribute: '', op: 'segmentMatch', values: [segmentKey], negate: false }],
+      },
+    ],
+  };
+}
+
+describe('given an initialized FDv2 client with an override that references a LaunchDarkly segment', () => {
+  let body: string;
+  let client: LDClientImpl;
+  let queue: AsyncQueue<string>;
+
+  beforeEach(async () => {
+    body = payload(1, 1);
+    const platform = createBasicPlatform();
+    platform.requests.fetch = jest.fn(() =>
+      Promise.resolve({ status: 200, headers: new Headers(), text: async () => body }),
+    ) as any;
+    const source = new TestOverrideSource({ flags: [overriddenFlagMatching('segment1')] });
+    queue = new AsyncQueue<string>();
+    client = makeFDv2Client(
+      platform,
+      {
+        dataSystem: {
+          overrides: source,
+          // The poll interval is in seconds; the test waits for the second poll.
+          dataSource: {
+            dataSourceOptionsType: 'custom',
+            initializers: [],
+            synchronizers: [{ type: 'polling', pollInterval: 1 }],
+          },
+        },
+      },
+      makeCallbacks((key) => queue.add(key), true),
+    );
+    await client.waitForInitialization({ timeout: 10 });
+  });
+
+  afterEach(() => {
+    client.close();
+  });
+
+  // Takes every notification that has arrived, waiting briefly for stragglers, so that a later
+  // assertion sees only notifications caused by what the test did next.
+  async function drain(): Promise<string[]> {
+    const keys: string[] = [];
+    for (;;) {
+      while (!queue.isEmpty()) {
+        // eslint-disable-next-line no-await-in-loop
+        keys.push(await queue.take());
+      }
+      // eslint-disable-next-line no-await-in-loop
+      await sleep(100);
+      if (queue.isEmpty()) {
+        return keys;
+      }
+    }
+  }
+
+  it('notifies the overridden flag when the segment changes on the LaunchDarkly side', async () => {
+    // Startup notifies the override (from the override source) and the LaunchDarkly flag, and the
+    // segment in the initial data set reaches the override as well.
+    const initial = await drain();
+    expect(initial).toContain('ld-flag');
+    expect(initial).toContain('overridden');
+    expect(await client.boolVariation('overridden', user, false)).toBe(true);
+
+    // The next poll delivers a new data set in which only the segment changed. The LaunchDarkly
+    // flag is unchanged, so the override is the only flag notified.
+    body = payload(2, 2);
+    expect(await queue.take()).toEqual('overridden');
+    await sleep(100);
+    expect(queue.isEmpty()).toBeTruthy();
+  });
+});

packages/shared/sdk-server/__tests__/data_sources/DataSourceUpdates.test.ts

@@ -5,6 +5,7 @@ import { internal } from '@launchdarkly/js-sdk-common';
 import { LDTransactionalFeatureStore } from '../../src/api/subsystems';
 import promisify from '../../src/async/promisify';
 import DataSourceUpdates from '../../src/data_sources/TransactionalDataSourceUpdates';
+import { OverrideLayer } from '../../src/overrides';
 import InMemoryFeatureStore from '../../src/store/InMemoryFeatureStore';
 import VersionedDataKinds from '../../src/store/VersionedDataKinds';
 
@@ -213,3 +214,120 @@ describe.each([true, false])(
     });
   },
 );
+
+describe('given a DataSourceUpdates whose external dependencies come from an override layer', () => {
+  let store: LDTransactionalFeatureStore;
+  let layer: OverrideLayer;
+  let updates: DataSourceUpdates;
+  let queue: AsyncQueue<string>;
+
+  function overriddenFlagMatching(key: string, segmentKey: string): any {
+    return {
+      key,
+      version: 1,
+      on: true,
+      fallthrough: { variation: 0 },
+      variations: [true, false],
+      rules: [
+        {
+          id: 'rule',
+          variation: 0,
+          clauses: [{ attribute: '', op: 'segmentMatch', values: [segmentKey], negate: false }],
+        },
+      ],
+    };
+  }
+
+  async function takeSorted(count: number): Promise<string[]> {
+    const keys: string[] = [];
+    for (let i = 0; i < count; i += 1) {
+      // eslint-disable-next-line no-await-in-loop
+      keys.push(await queue.take());
+    }
+    return keys.sort();
+  }
+
+  beforeEach(() => {
+    queue = new AsyncQueue<string>();
+    store = new InMemoryFeatureStore();
+    layer = new OverrideLayer();
+    updates = new DataSourceUpdates(
+      store,
+      () => true,
+      (key) => queue.add(key),
+      layer,
+    );
+  });
+
+  it('notifies an overridden flag when the LaunchDarkly segment it references changes', async () => {
+    layer.setAll([overriddenFlagMatching('overridden', 'segment1')], []);
+    await promisify((cb) => {
+      updates.init(
+        {
+          features: { ld: { key: 'ld', version: 1 } },
+          segments: { segment1: { key: 'segment1', version: 1 } },
+        },
+        () => cb(undefined),
+      );
+    });
+    // The initial data set counts every item as modified; the segment reaches the override.
+    expect(await takeSorted(2)).toEqual(['ld', 'overridden']);
+
+    await promisify((cb) => {
+      updates.upsert(VersionedDataKinds.Segments, { key: 'segment1', version: 2 }, () =>
+        cb(undefined),
+      );
+    });
+    expect(await queue.take()).toEqual('overridden');
+    expect(queue.isEmpty()).toBeTruthy();
+  });
+
+  it('follows a chain that crosses from LaunchDarkly data into the override layer and back', async () => {
+    // parent is a LaunchDarkly flag whose prerequisite is overridden; overridden references the
+    // LaunchDarkly segment. A change to the segment reaches parent through the override entry.
+    layer.setAll([overriddenFlagMatching('overridden', 'segment1')], []);
+    await promisify((cb) => {
+      updates.init(
+        {
+          features: {
+            parent: {
+              key: 'parent',
+              version: 1,
+              prerequisites: [{ key: 'overridden', variation: 0 }],
+            },
+          },
+          segments: { segment1: { key: 'segment1', version: 1 } },
+        },
+        () => cb(undefined),
+      );
+    });
+    expect(await takeSorted(2)).toEqual(['overridden', 'parent']);
+
+    await promisify((cb) => {
+      updates.upsert(VersionedDataKinds.Segments, { key: 'segment1', version: 2 }, () =>
+        cb(undefined),
+      );
+    });
+    expect(await takeSorted(2)).toEqual(['overridden', 'parent']);
+    expect(queue.isEmpty()).toBeTruthy();
+  });
+
+  it('does not notify an overridden flag for an item it does not reference', async () => {
+    layer.setAll([overriddenFlagMatching('overridden', 'segment1')], []);
+    await promisify((cb) => {
+      updates.init({ features: {}, segments: { segment2: { key: 'segment2', version: 1 } } }, () =>
+        cb(undefined),
+      );
+    });
+    await promisify((cb) => {
+      updates.upsert(VersionedDataKinds.Segments, { key: 'segment2', version: 2 }, () =>
+        cb(undefined),
+      );
+    });
+    await promisify((cb) => {
+      updates.upsert(VersionedDataKinds.Features, { key: 'ld', version: 1 }, () => cb(undefined));
+    });
+    expect(await queue.take()).toEqual('ld');
+    expect(queue.isEmpty()).toBeTruthy();
+  });
+});

packages/shared/sdk-server/__tests__/data_sources/NamespacedDataSet.test.ts

@@ -0,0 +1,38 @@
+import DependencyTracker from '../../src/data_sources/DependencyTracker';
+import NamespacedDataSet from '../../src/data_sources/NamespacedDataSet';
+
+describe('given keys that are the names of properties of Object', () => {
+  const keys = ['constructor', 'toString', 'hasOwnProperty', '__proto__', 'valueOf'];
+
+  it('stores, finds, enumerates, and removes such entries like any other', () => {
+    const set = new NamespacedDataSet<number>();
+    keys.forEach((key, index) => set.set('features', key, index));
+    keys.forEach((key, index) => expect(set.get('features', key)).toBe(index));
+    expect(set.get('features', 'missing')).toBeUndefined();
+    expect(set.get('constructor', 'constructor')).toBeUndefined();
+
+    const seen: string[] = [];
+    set.enumerate((_namespace, key) => seen.push(key));
+    expect(seen.sort()).toEqual([...keys].sort());
+
+    set.remove('features', '__proto__');
+    expect(set.get('features', '__proto__')).toBeUndefined();
+    expect(set.get('features', 'constructor')).toBe(0);
+  });
+
+  it('tracks dependencies between items with such keys', () => {
+    // Before the maps lost their prototype, looking up "constructor" returned Object's
+    // constructor function and the tracker threw while updating.
+    const tracker = new DependencyTracker();
+    const deps = new NamespacedDataSet<boolean>();
+    deps.set('segments', '__proto__', true);
+    tracker.updateDependenciesFrom('features', 'constructor', deps);
+    tracker.updateDependenciesFrom('features', 'constructor', deps);
+
+    const affected = new NamespacedDataSet<boolean>();
+    tracker.updateModifiedItems(affected, 'segments', '__proto__');
+    const items: string[] = [];
+    affected.enumerate((namespace, key) => items.push(`${namespace}/${key}`));
+    expect(items.sort()).toEqual(['features/constructor', 'segments/__proto__']);
+  });
+});

packages/shared/sdk-server/__tests__/overrides/OverrideLayer.test.ts

@@ -2,6 +2,7 @@
 import { AttributeReference } from '@launchdarkly/js-sdk-common';
 
 import { makeFlagWithValue } from '../../src/data_sources/FileDataSource';
+import NamespacedDataSet from '../../src/data_sources/NamespacedDataSet';
 import { Flag } from '../../src/evaluation/data/Flag';
 import { Segment } from '../../src/evaluation/data/Segment';
 import TestData from '../../src/integrations/test_data/TestData';
@@ -485,3 +486,57 @@ describe('given keys that are the names of properties of Object', () => {
     expect((layer.get(features, 'flag1') as Flag).on).toBe(true);
   });
 });
+
+describe('given a layer whose entries reference other definitions', () => {
+  const features = VersionedDataKinds.Features.namespace;
+  const segments = VersionedDataKinds.Segments.namespace;
+
+  function flagReferencing(key: string, segmentKey: string, prerequisites: string[] = []): any {
+    return {
+      key,
+      version: 1,
+      on: true,
+      fallthrough: { variation: 0 },
+      variations: [true, false],
+      prerequisites: prerequisites.map((prerequisite) => ({ key: prerequisite, variation: 0 })),
+      rules: [
+        {
+          id: 'rule',
+          variation: 0,
+          clauses: [{ attribute: '', op: 'segmentMatch', values: [segmentKey], negate: false }],
+        },
+      ],
+    };
+  }
+
+  function reachedFrom(layer: OverrideLayer, namespace: string, key: string): string[] {
+    const set = new NamespacedDataSet<boolean>();
+    layer.updateModifiedItems(set, namespace, key);
+    const items: string[] = [];
+    set.enumerate((itemNamespace, itemKey) => items.push(`${itemNamespace}/${itemKey}`));
+    return items.sort();
+  }
+
+  it('reports the entries that depend on a segment or a prerequisite, transitively', () => {
+    const layer = new OverrideLayer();
+    layer.setAll([flagReferencing('a', 'seg'), flagReferencing('b', 'other', ['a'])], []);
+
+    // b does not reference seg itself; it depends on a, which does.
+    expect(reachedFrom(layer, segments, 'seg')).toEqual([
+      'features/a',
+      'features/b',
+      'segments/seg',
+    ]);
+    expect(reachedFrom(layer, features, 'a')).toEqual(['features/a', 'features/b']);
+    expect(reachedFrom(layer, segments, 'unreferenced')).toEqual(['segments/unreferenced']);
+  });
+
+  it('forgets the edges of the entries that a replacement removed', () => {
+    const layer = new OverrideLayer();
+    layer.setAll([flagReferencing('a', 'seg')], []);
+    layer.setAll([flagReferencing('c', 'seg2')], []);
+
+    expect(reachedFrom(layer, segments, 'seg')).toEqual(['segments/seg']);
+    expect(reachedFrom(layer, segments, 'seg2')).toEqual(['features/c', 'segments/seg2']);
+  });
+});