refactor: restructure rollout plan data format and improve data loading

Author: EmilianoSanchez

src/dtos/types.ts

@@ -259,7 +259,7 @@ export interface ISegmentChangesResponse {
   name: string,
   added: string[],
   removed: string[],
-  since: number,
+  since?: number,
   till: number
 }
 

src/storages/__tests__/dataLoader.spec.ts

@@ -6,7 +6,7 @@ import { IRBSegment, ISplit } from '../../dtos/types';
 
 import * as dataLoader from '../dataLoader';
 
-describe('setRolloutPlan & getRolloutPlan', () => {
+describe('getRolloutPlan & setRolloutPlan (client-side)', () => {
   jest.spyOn(dataLoader, 'setRolloutPlan');
   const onReadyFromCacheCb = jest.fn();
   const onReadyCb = jest.fn();
@@ -19,15 +19,52 @@ describe('setRolloutPlan & getRolloutPlan', () => {
   serverStorage.rbSegments.update([{ name: 'rbs1' } as IRBSegment], [], 321);
   serverStorage.segments.update('segment1', [fullSettings.core.key as string, otherKey], [], 123);
 
+  const expectedRolloutPlan = {
+    splitChanges: {
+      ff: { d: [{ name: 'split1' }], t: 123 },
+      rbs: { d: [{ name: 'rbs1' }], t: 321 }
+    },
+    memberships: {
+      [fullSettings.core.key as string]: { ms: { k: [{ n: 'segment1' }] }, ls: { k: [] } },
+      [otherKey]: { ms: { k: [{ n: 'segment1' }] }, ls: { k: [] } }
+    },
+    segmentChanges: [{
+      name: 'segment1',
+      added: [fullSettings.core.key as string, otherKey],
+      removed: [],
+      till: 123
+    }]
+  };
+
   afterEach(() => {
     jest.clearAllMocks();
   });
 
+  test('using preloaded data (no memberships, no segments)', () => {
+    const rolloutPlan = dataLoader.getRolloutPlan(loggerMock, serverStorage);
+
+    // Load client-side storage with preloaded data
+    const clientStorage = InMemoryStorageCSFactory({ settings: { ...fullSettings, initialRolloutPlan: rolloutPlan }, onReadyFromCacheCb, onReadyCb });
+    expect(dataLoader.setRolloutPlan).toBeCalledTimes(1);
+    expect(onReadyFromCacheCb).toBeCalledTimes(1);
+
+    // Shared client storage
+    const sharedClientStorage = clientStorage.shared!(otherKey);
+    expect(dataLoader.setRolloutPlan).toBeCalledTimes(2);
+
+    expect(clientStorage.segments.getRegisteredSegments()).toEqual([]);
+    expect(sharedClientStorage.segments.getRegisteredSegments()).toEqual([]);
+
+    // Get preloaded data from client-side storage
+    expect(dataLoader.getRolloutPlan(loggerMock, clientStorage)).toEqual(rolloutPlan);
+    expect(rolloutPlan).toEqual({ ...expectedRolloutPlan, memberships: undefined, segmentChanges: undefined });
+  });
+
   test('using preloaded data with memberships', () => {
-    const rolloutPlanData = dataLoader.getRolloutPlan(loggerMock, serverStorage, [fullSettings.core.key as string, otherKey]);
+    const rolloutPlan = dataLoader.getRolloutPlan(loggerMock, serverStorage, { keys: [fullSettings.core.key as string, otherKey] });
 
     // Load client-side storage with preloaded data
-    const clientStorage = InMemoryStorageCSFactory({ settings: { ...fullSettings, initialRolloutPlan: rolloutPlanData }, onReadyFromCacheCb, onReadyCb });
+    const clientStorage = InMemoryStorageCSFactory({ settings: { ...fullSettings, initialRolloutPlan: rolloutPlan }, onReadyFromCacheCb, onReadyCb });
     expect(dataLoader.setRolloutPlan).toBeCalledTimes(1);
     expect(onReadyFromCacheCb).toBeCalledTimes(1);
 
@@ -39,43 +76,43 @@ describe('setRolloutPlan & getRolloutPlan', () => {
     expect(sharedClientStorage.segments.getRegisteredSegments()).toEqual(['segment1']);
 
     // Get preloaded data from client-side storage
-    expect(dataLoader.getRolloutPlan(loggerMock, clientStorage, [fullSettings.core.key as string, otherKey])).toEqual(rolloutPlanData);
-    expect(rolloutPlanData).toEqual({
-      since: 123,
-      flags: [{ name: 'split1' }],
-      rbSince: 321,
-      rbSegments: [{ name: 'rbs1' }],
-      memberships: {
-        [fullSettings.core.key as string]: { ms: { k: [{ n: 'segment1' }] }, ls: { k: [] } },
-        [otherKey]: { ms: { k: [{ n: 'segment1' }] }, ls: { k: [] } }
-      },
-      segments: undefined
-    });
+    expect(dataLoader.getRolloutPlan(loggerMock, clientStorage, { keys: [fullSettings.core.key as string, otherKey] })).toEqual(rolloutPlan);
+    expect(rolloutPlan).toEqual({ ...expectedRolloutPlan, segmentChanges: undefined });
   });
 
   test('using preloaded data with segments', () => {
-    const rolloutPlanData = dataLoader.getRolloutPlan(loggerMock, serverStorage);
+    const rolloutPlan = dataLoader.getRolloutPlan(loggerMock, serverStorage, { exposeSegments: true });
 
     // Load client-side storage with preloaded data
-    const clientStorage = InMemoryStorageCSFactory({ settings: { ...fullSettings, initialRolloutPlan: rolloutPlanData }, onReadyFromCacheCb, onReadyCb });
+    const clientStorage = InMemoryStorageCSFactory({ settings: { ...fullSettings, initialRolloutPlan: rolloutPlan }, onReadyFromCacheCb, onReadyCb });
     expect(dataLoader.setRolloutPlan).toBeCalledTimes(1);
     expect(onReadyFromCacheCb).toBeCalledTimes(1);
 
     // Shared client storage
     const sharedClientStorage = clientStorage.shared!(otherKey);
     expect(dataLoader.setRolloutPlan).toBeCalledTimes(2);
+
     expect(clientStorage.segments.getRegisteredSegments()).toEqual(['segment1']);
     expect(sharedClientStorage.segments.getRegisteredSegments()).toEqual(['segment1']);
 
-    expect(rolloutPlanData).toEqual({
-      since: 123,
-      flags: [{ name: 'split1' }],
-      rbSince: 321,
-      rbSegments: [{ name: 'rbs1' }],
-      memberships: undefined,
-      segments: {
-        segment1: [fullSettings.core.key as string, otherKey]
-      }
-    });
+    expect(rolloutPlan).toEqual({ ...expectedRolloutPlan, memberships: undefined });
+  });
+
+  test('using preloaded data with memberships and segments', () => {
+    const rolloutPlan = dataLoader.getRolloutPlan(loggerMock, serverStorage, { keys: [fullSettings.core.key as string], exposeSegments: true });
+
+    // Load client-side storage with preloaded data
+    const clientStorage = InMemoryStorageCSFactory({ settings: { ...fullSettings, initialRolloutPlan: rolloutPlan }, onReadyFromCacheCb, onReadyCb });
+    expect(dataLoader.setRolloutPlan).toBeCalledTimes(1);
+    expect(onReadyFromCacheCb).toBeCalledTimes(1);
+
+    // Shared client storage
+    const sharedClientStorage = clientStorage.shared!(otherKey);
+    expect(dataLoader.setRolloutPlan).toBeCalledTimes(2);
+
+    expect(clientStorage.segments.getRegisteredSegments()).toEqual(['segment1']); // main client membership is set via the rollout plan `memberships` field
+    expect(sharedClientStorage.segments.getRegisteredSegments()).toEqual(['segment1']); // shared client membership is set via the rollout plan `segmentChanges` field
+
+    expect(rolloutPlan).toEqual({ ...expectedRolloutPlan, memberships: { [fullSettings.core.key as string]: expectedRolloutPlan.memberships![fullSettings.core.key as string] } });
   });
 });

src/storages/dataLoader.ts

@@ -2,74 +2,73 @@ import SplitIO from '../../types/splitio';
 import { IRBSegmentsCacheSync, ISegmentsCacheSync, ISplitsCacheSync, IStorageSync } from './types';
 import { setToArray } from '../utils/lang/sets';
 import { getMatching } from '../utils/key';
-import { IMembershipsResponse, IMySegmentsResponse, IRBSegment, ISplit } from '../dtos/types';
+import { IMembershipsResponse, IMySegmentsResponse, ISegmentChangesResponse, ISplitChangesResponse } from '../dtos/types';
 import { ILogger } from '../logger/types';
+import { isObject } from '../utils/lang';
 
 export type RolloutPlan = {
   /**
-   * Change number of feature flags.
+   * Feature flags and rule-based segments.
    */
-  since: number;
+  splitChanges: ISplitChangesResponse;
   /**
-   * List of feature flags.
-   */
-  flags: ISplit[];
-  /**
-   * Change number of rule-based segments.
-   */
-  rbSince?: number;
-  /**
-   * List of rule-based segments.
-   */
-  rbSegments?: IRBSegment[];
-  /**
-   * Optional map of user keys to their memberships.
+   * Optional map of matching keys to their memberships.
    */
   memberships?: {
-    [key: string]: IMembershipsResponse;
+    [matchingKey: string]: IMembershipsResponse;
   };
   /**
-   * Optional map of standard segments to their list of keys.
+   * Optional list of standard segments.
    * This property is ignored if `memberships` is provided.
    */
-  segments?: {
-    [segmentName: string]: string[];
-  };
+  segmentChanges?: ISegmentChangesResponse[];
 };
 
+/**
+ * Validates if the given rollout plan is valid.
+ */
+function validateRolloutPlan(rolloutPlan: unknown): rolloutPlan is RolloutPlan {
+  if (isObject(rolloutPlan) && isObject((rolloutPlan as any).splitChanges)) return true;
+
+  return false;
+}
+
 /**
  * Sets the given synchronous storage with the provided rollout plan snapshot.
  * If `matchingKey` is provided, the storage is handled as a client-side storage (segments and largeSegments are instances of MySegmentsCache).
  * Otherwise, the storage is handled as a server-side storage (segments is an instance of SegmentsCache).
  */
 export function setRolloutPlan(log: ILogger, rolloutPlan: RolloutPlan, storage: { splits?: ISplitsCacheSync, rbSegments?: IRBSegmentsCacheSync, segments: ISegmentsCacheSync, largeSegments?: ISegmentsCacheSync }, matchingKey?: string) {
   // Do not load data if current rollout plan is empty
-  if (Object.keys(rolloutPlan).length === 0) return;
+  if (!validateRolloutPlan(rolloutPlan)) {
+    log.error('storage: invalid rollout plan provided');
+    return;
+  }
 
   const { splits, rbSegments, segments, largeSegments } = storage;
+  const { splitChanges: { ff, rbs } } = rolloutPlan;
 
   log.debug(`storage: set feature flags and segments${matchingKey ? ` for key ${matchingKey}` : ''}`);
 
-  if (splits) {
+  if (splits && ff) {
     splits.clear();
-    splits.update(rolloutPlan.flags || [], [], rolloutPlan.since || -1);
+    splits.update(ff.d, [], ff.t);
   }
 
-  if (rbSegments) {
+  if (rbSegments && rbs) {
     rbSegments.clear();
-    rbSegments.update(rolloutPlan.rbSegments || [], [], rolloutPlan.rbSince || -1);
+    rbSegments.update(rbs.d, [], rbs.t);
   }
 
-  const segmentsData = rolloutPlan.segments || {};
+  const segmentChanges = rolloutPlan.segmentChanges;
   if (matchingKey) { // add memberships data (client-side)
     let memberships = rolloutPlan.memberships && rolloutPlan.memberships[matchingKey];
-    if (!memberships && segmentsData) {
+    if (!memberships && segmentChanges) {
       memberships = {
         ms: {
-          k: Object.keys(segmentsData).filter(segmentName => {
-            const segmentKeys = segmentsData[segmentName];
-            return segmentKeys.indexOf(matchingKey) > -1;
-          }).map(segmentName => ({ n: segmentName }))
+          k: segmentChanges.filter(segment => {
+            return segment.added.indexOf(matchingKey) > -1;
+          }).map(segment => ({ n: segment.name }))
         }
       };
     }
@@ -79,10 +78,11 @@ export function setRolloutPlan(log: ILogger, rolloutPlan: RolloutPlan, storage:
       if (memberships.ls && largeSegments) largeSegments.resetSegments(memberships.ls!);
     }
   } else { // add segments data (server-side)
-    Object.keys(segmentsData).forEach(segmentName => {
-      const segmentKeys = segmentsData[segmentName];
-      segments.update(segmentName, segmentKeys, [], -1);
-    });
+    if (segmentChanges) {
+      segmentChanges.forEach(segment => {
+        segments.update(segment.name, segment.added, segment.removed, segment.till);
+      });
+    }
   }
 }
 
@@ -91,21 +91,32 @@ export function setRolloutPlan(log: ILogger, rolloutPlan: RolloutPlan, storage:
  * If `keys` are provided, the memberships for those keys is returned, to protect segments data.
  * Otherwise, the segments data is returned.
  */
-export function getRolloutPlan(log: ILogger, storage: IStorageSync, keys?: SplitIO.SplitKey[]): RolloutPlan {
+export function getRolloutPlan(log: ILogger, storage: IStorageSync, options: SplitIO.RolloutPlanOptions = {}): RolloutPlan {
 
-  log.debug(`storage: get feature flags and segments${keys ? ` for keys ${keys}` : ''}`);
+  const { keys, exposeSegments } = options;
+  const { splits, segments, rbSegments } = storage;
+
+  log.debug(`storage: get feature flags${keys ? `, and memberships for keys ${keys}` : ''}${exposeSegments ? ', and segments' : ''}`);
 
   return {
-    since: storage.splits.getChangeNumber(),
-    flags: storage.splits.getAll(),
-    rbSince: storage.rbSegments.getChangeNumber(),
-    rbSegments: storage.rbSegments.getAll(),
-    segments: keys ?
-      undefined : // @ts-ignore accessing private prop
-      Object.keys(storage.segments.segmentCache).reduce((prev, cur) => { // @ts-ignore accessing private prop
-        prev[cur] = setToArray(storage.segments.segmentCache[cur] as Set<string>);
-        return prev;
-      }, {}),
+    splitChanges: {
+      ff: {
+        t: splits.getChangeNumber(),
+        d: splits.getAll(),
+      },
+      rbs: {
+        t: rbSegments.getChangeNumber(),
+        d: rbSegments.getAll(),
+      }
+    },
+    segmentChanges: exposeSegments ? // @ts-ignore accessing private prop
+      Object.keys(segments.segmentCache).map(segmentName => ({
+        name: segmentName, // @ts-ignore
+        added: setToArray(segments.segmentCache[segmentName] as Set<string>),
+        removed: [],
+        till: segments.getChangeNumber(segmentName)!
+      })) :
+      undefined,
     memberships: keys ?
       keys.reduce<Record<string, IMembershipsResponse>>((prev, key) => {
         if (storage.shared) {

types/splitio.d.ts

@@ -1031,6 +1031,23 @@ declare namespace SplitIO {
    * A JSON-serializable plain object that defines the format of rollout plan data to preload the SDK cache with feature flags and segments.
    */
   type RolloutPlan = Object;
+  /**
+   * Options for the `factory.getRolloutPlan` method.
+   */
+  type RolloutPlanOptions = {
+    /**
+     * Optional list of keys to generate the rollout plan snapshot with the memberships of the given keys.
+     *
+     * @defaultValue `undefined`
+     */
+    keys?: SplitKey[];
+    /**
+     * Optional flag to expose segments data in the rollout plan snapshot.
+     *
+     * @defaultValue `false`
+     */
+    exposeSegments?: boolean;
+  };
   /**
    * Impression listener interface. This is the interface that needs to be implemented
    * by the element you provide to the SDK as impression listener.
@@ -1580,7 +1597,7 @@ declare namespace SplitIO {
      * @param keys - Optional list of keys to generate the rollout plan snapshot with the memberships of the given keys, rather than the complete segments data.
      * @returns The current snapshot of the SDK rollout plan.
      */
-    getRolloutPlan(keys?: SplitKey[]): RolloutPlan;
+    getRolloutPlan(options?: RolloutPlanOptions): RolloutPlan;
   }
   /**
    * This represents the interface for the SDK instance for server-side with asynchronous storage.