array diffs

Author: pavelglac

packages/apollo-forest-run/src/diff/diffObject.ts

@@ -19,11 +19,14 @@ import type {
   ObjectDifference,
   ObjectDiffState,
   ValueDifference,
+  CompositeListLayoutChangeItemRemoved,
+  CompositeListLayoutChange,
 } from "./types";
 import * as Value from "../values";
 import * as Difference from "./difference";
 import { DiffErrorKind } from "./types";
 import { ValueKind } from "../values/types";
+import * as ChangeKind from "./itemChangeKind";
 
 /**
  * Compares base object version with model version and returns a normalized Difference object,
@@ -420,8 +423,15 @@ function diffCompositeListValue(
     return undefined;
   }
 
-  const layoutDiffResult =
-    diff?.layout ?? diffCompositeListLayout(context, base, model);
+  let layoutDiffResult: any = diff?.layout;
+  let itemsChanges: CompositeListLayoutChange[] = [];
+
+  if (!layoutDiffResult) {
+    const { deletedItems: newDeletedItems, layout: newLayout } =
+      diffCompositeListLayout(context, base, model);
+    itemsChanges = newDeletedItems;
+    layoutDiffResult = newLayout;
+  }
 
   if (layoutDiffResult === "BREAK") {
     // Fast-path, no further diffing necessary
@@ -432,6 +442,7 @@ function diffCompositeListValue(
   if (layoutDiffResult) {
     diff = diff ?? Difference.createCompositeListDifference();
     diff.layout = layoutDiffResult;
+    diff.itemsChanges = itemsChanges;
   }
 
   for (const index of itemQueue) {
@@ -478,7 +489,10 @@ function diffCompositeListLayout(
   context: DiffContext,
   base: CompositeListValue,
   model: CompositeListValue,
-): CompositeListLayoutDifference | undefined | "BREAK" {
+): {
+  deletedItems: CompositeListLayoutChange[];
+  layout: CompositeListLayoutDifference | undefined | "BREAK";
+} {
   // What constitutes layout change?
   // - Change of "keyed object" position in the list
   // - Change of list length
@@ -489,6 +503,11 @@ function diffCompositeListLayout(
 
   const baseChunk = Value.isAggregate(base) ? base.chunks[0] : base;
   const modelChunk = Value.isAggregate(model) ? model.chunks[0] : model;
+  const unusedBaseIndixes = new Set<number>();
+  const itemChanges: CompositeListLayoutChange[] = [];
+  for (let i = 0; i < baseLen; i++) {
+    unusedBaseIndixes.add(i);
+  }
 
   let itemDiffRequired = false;
   let firstDirtyIndex = -1;
@@ -508,17 +527,32 @@ function diffCompositeListLayout(
       firstDirtyIndex = i;
       break;
     }
+    unusedBaseIndixes.delete(i);
   }
   // Fast-path: no layout difference found
   if (firstDirtyIndex === -1) {
+    const deletedItems: CompositeListLayoutChangeItemRemoved[] = [];
+    for (const index of unusedBaseIndixes) {
+      deletedItems.push({
+        kind: ChangeKind.ItemRemove,
+        oldIndex: index,
+        data: baseChunk.data[index],
+      });
+    }
     if (baseLen > modelLen) {
       const layout: CompositeListLayoutDifference = [];
       for (let i = 0; i < modelLen; i++) {
         layout.push(i);
       }
-      return layout;
+      return {
+        deletedItems,
+        layout,
+      };
     }
-    return !itemDiffRequired ? "BREAK" : undefined;
+    return {
+      deletedItems,
+      layout: !itemDiffRequired ? "BREAK" : undefined,
+    };
   }
   // TODO: lastDirtyIndex to isolate changed segment (prepend case)
 
@@ -527,12 +561,19 @@ function diffCompositeListLayout(
     layout.push(i);
   }
   let plainObjectLookupStartIndex = firstDirtyIndex;
-  for (let i = firstDirtyIndex; i < modelLen; i++) {
-    if (modelChunk.data[i] === null) {
+  for (let index = firstDirtyIndex; index < modelLen; index++) {
+    if (modelChunk.data[index] === null) {
       layout.push(null);
+      if (baseChunk.data[index] !== null) {
+        itemChanges.push({
+          kind: ChangeKind.ItemAdd,
+          index,
+          data: null,
+        });
+      }
       continue;
     }
-    const modelKey = resolveItemKey(env, modelChunk, i);
+    const modelKey = resolveItemKey(env, modelChunk, index);
     const lookupStartIndex =
       modelKey === false ? plainObjectLookupStartIndex : 0; // TODO: should be firstDirtyIndex; (0 is necessary only for cases when array contains duplicates - we should detect such arrays when indexing and special-case it instead)
 
@@ -545,8 +586,17 @@ function diffCompositeListLayout(
     );
     if (baseIndex !== -1) {
       layout.push(baseIndex);
+      unusedBaseIndixes.delete(baseIndex);
+      if (index !== baseIndex) {
+        itemChanges.push({
+          kind: ChangeKind.ItemIndexChange,
+          index,
+          oldIndex: baseIndex,
+          data: baseChunk.data[baseIndex],
+        });
+      }
     } else {
-      const value = Value.aggregateListItemValue(model, i);
+      const value = Value.aggregateListItemValue(model, index);
       if (Value.isCompositeNullValue(value)) {
         layout.push(null);
       } else if (
@@ -556,13 +606,24 @@ function diffCompositeListLayout(
         layout.push(value);
       } else {
         throw new Error(
-          `Unexpected list item value at index #${i}\n` +
+          `Unexpected list item value at index #${index}\n` +
             `  original list: ${JSON.stringify(model.data)}`,
         );
       }
     }
   }
-  return layout;
+
+  for (const oldIndex of unusedBaseIndixes) {
+    itemChanges.push({
+      kind: ChangeKind.ItemRemove,
+      oldIndex,
+      data: baseChunk.data[oldIndex],
+    });
+  }
+  return {
+    deletedItems: itemChanges,
+    layout,
+  };
 }
 
 function resolveItemKey(

packages/apollo-forest-run/src/diff/difference.ts

@@ -78,6 +78,7 @@ export function createCompositeListDifference(): CompositeListDifference {
     dirtyItems: undefined,
     layout: undefined,
     deletedKeys: undefined,
+    itemsChanges: [],
     errors: undefined,
   };
 }

packages/apollo-forest-run/src/diff/itemChangeKind.ts

@@ -0,0 +1,3 @@
+export const ItemAdd = 0,
+  ItemRemove = 1,
+  ItemIndexChange = 2;

packages/apollo-forest-run/src/diff/types.ts

@@ -6,10 +6,13 @@ import {
   ObjectValue,
   SourceObject,
   SourceCompositeList,
+  NestedList,
+  MissingFieldsMap,
 } from "../values/types";
 import { FieldInfo, NormalizedFieldEntry } from "../descriptor/types";
 import * as DifferenceKind from "./differenceKind";
 import * as DiffErrorKind from "./diffErrorKind";
+import * as ChangeKind from "./itemChangeKind";
 
 export type DiffEnv = {
   allowMissingFields?: boolean;
@@ -90,13 +93,39 @@ export type FieldEntryDifference = {
   state: ValueDifference;
 };
 
+export type CompositeListLayoutItemAdded = {
+  kind: typeof ChangeKind.ItemAdd;
+  index: number;
+  missingFields?: MissingFieldsMap | undefined;
+  data?: SourceObject | NestedList<SourceObject> | null;
+};
+
+export type CompositeListLayoutChangeItemRemoved = {
+  kind: typeof ChangeKind.ItemRemove;
+  oldIndex: number;
+  data?: SourceObject | NestedList<SourceObject>;
+};
+
+export type CompositeListLayoutIndexChange = {
+  kind: typeof ChangeKind.ItemIndexChange;
+  index: number;
+  oldIndex: number;
+  data?: SourceObject | NestedList<SourceObject>;
+};
+
+export type CompositeListLayoutChange =
+  | CompositeListLayoutChangeItemRemoved
+  | CompositeListLayoutIndexChange
+  | CompositeListLayoutItemAdded;
+
 export type CompositeListDifference = {
   readonly kind: typeof DifferenceKind.CompositeListDifference;
   itemQueue: Set<number>;
   itemState: Map<number, ValueDifference>;
   dirtyItems?: Set<number>;
   layout?: CompositeListLayoutDifference;
   deletedKeys?: string[];
+  itemsChanges: CompositeListLayoutChange[];
   errors?: DiffError[];
 };
 

packages/apollo-forest-run/src/forest/indexTree.ts

@@ -105,7 +105,7 @@ export function indexTree(
 
   let history: HistoryArray | null = null;
   if (env.enableHistory) {
-    const historySize = operation.historySize ?? env.defaultHistorySize;
+    const historySize = operation.historySize ?? env.defaultHistorySize ?? 0;
     history =
       previousTreeState?.history ??
       new HistoryArray(historySize, env.enableHistory, env.enableDataHistory);

packages/apollo-forest-run/src/forest/types.ts

@@ -30,6 +30,7 @@ import { UpdateTreeStats } from "../telemetry/updateStats/types";
 import { UpdateLogger } from "../telemetry/updateStats/updateLogger";
 import { HistoryArray } from "../jsutils/historyArray";
 import type * as DifferenceKind from "../diff/differenceKind";
+import { CompositeListLayoutChange } from "../diff/types";
 
 export type IndexedTree = {
   operation: OperationDescriptor;
@@ -68,7 +69,7 @@ export type HistoryEntry = {
     operation?: OperationDescriptor | undefined;
   };
   updated: {
-    changes: ChangedChunksMap;
+    changes: any;
     result: OperationResult | undefined;
   };
 };
@@ -98,7 +99,7 @@ export type FieldChange = (
     }
   | {
       kind: typeof DifferenceKind.CompositeListDifference;
-      newSize: number | undefined;
+      itemChanges: CompositeListLayoutChange[] | undefined;
     }
 ) & {
   fieldInfo: FieldInfo;
@@ -127,6 +128,7 @@ export type UpdateTreeContext = {
   completeObject: CompleteObjectFn;
   findParent: ParentLocator;
   env: ForestEnv;
+  childChanges: any[];
   statsLogger?: UpdateLogger;
 };
 
@@ -175,7 +177,8 @@ export type ForestEnv = {
   logStaleOperations?: boolean;
 
   // History configuration
-  enableHistory: boolean;
-  enableDataHistory: boolean;
-  defaultHistorySize: number;
+  enableHistory?: boolean;
+  enableDataHistory?: boolean;
+  defaultHistorySize?: number;
+  storeFieldPaths?: boolean; // Store computed field paths in history (more memory, faster summary generation)
 };

packages/apollo-forest-run/src/forest/updateObject.ts

@@ -29,6 +29,7 @@ import type {
   UpdateTreeContext,
 } from "./types";
 import * as Difference from "../diff/difference";
+import * as ChangeKind from "../diff/itemChangeKind";
 import * as Value from "../values";
 import { ValueKind } from "../values/types";
 import { DifferenceKind } from "../diff/types";
@@ -64,7 +65,7 @@ function updateObjectValue(
   let copy = context.drafts.get(base.data);
   assert(!Array.isArray(copy));
   context.statsLogger?.copyChunkStats(base, copy);
-  let dirtyFields: FieldChange[] | undefined;
+  const changes: FieldChange[] = [];
 
   for (const fieldName of difference.dirtyFields) {
     const aliases = base.selection.fields.get(fieldName);
@@ -116,36 +117,35 @@ function updateObjectValue(
 
       switch (fieldDiff.kind) {
         case DifferenceKind.Filler:
-          dirtyFields ??= [];
-          dirtyFields.push({
+          changes.push({
             kind: fieldDiff.kind,
             fieldInfo,
             newValue: enableDataHistory ? updated : undefined,
           });
           break;
         case DifferenceKind.Replacement:
-          dirtyFields ??= [];
-          dirtyFields.push({
+          changes.push({
             kind: fieldDiff.kind,
             fieldInfo,
             oldValue: enableDataHistory ? fieldDiff.oldValue : undefined,
             newValue: enableDataHistory ? updated : undefined,
           });
           break;
         case DifferenceKind.CompositeListDifference:
-          dirtyFields ??= [];
-          dirtyFields.push({
+          changes.push({
             kind: fieldDiff.kind,
             fieldInfo,
-            newSize: fieldDiff.layout?.length,
+            itemChanges: enableDataHistory ? context.childChanges : undefined,
           });
+          context.childChanges = [];
           break;
       }
     }
   }
-  if (dirtyFields?.length) {
-    context.changes.set(base, dirtyFields);
+  if (changes.length) {
+    context.changes.set(base, changes);
   }
+
   return copy ?? base.data;
 }
 
@@ -193,7 +193,7 @@ function updateCompositeListValue(
   const layoutDiff = difference.layout;
   let dirty = false; // Only dirty on self changes - item replacement/filler, layout changes (ignores child changes)
   let copy = drafts.get(base.data);
-  const dirtyItemIndexes: any[] = [];
+  const arrayChanges = difference.itemsChanges;
   assert(Array.isArray(copy) || copy === undefined);
   statsLogger?.copyChunkStats(base, copy);
 
@@ -223,9 +223,6 @@ function updateCompositeListValue(
       dirty = true;
     }
   }
-  if (dirty) {
-    context.changes.set(base, null);
-  }
   if (!layoutDiff) {
     return copy ?? base.data;
   }
@@ -264,7 +261,8 @@ function updateCompositeListValue(
       assert(newValue.data);
       accumulateMissingFields(context, newValue);
       result[i] = newValue.data;
-      dirtyItemIndexes.push({
+      arrayChanges.push({
+        kind: ChangeKind.ItemAdd,
         index: i,
         data: newValue.data,
         missingFields: newValue.missingFields,
@@ -293,7 +291,7 @@ function updateCompositeListValue(
     dirty = true;
   }
   if (dirty) {
-    context.changes.set(base, dirtyItemIndexes);
+    context.childChanges = arrayChanges;
   }
 
   return copy ?? base.data;

packages/apollo-forest-run/src/forest/updateTree.ts

@@ -60,6 +60,7 @@ export function updateTree(
     completeObject: completeObject.bind(null, env, base, getNodeChunks),
     findParent: createParentLocator(base.dataMap),
     statsLogger: createUpdateLogger(env.logUpdateStats),
+    childChanges: [],
     env,
   };
   const {