diffing of attribution manager state

rollup.config.js

@@ -1,12 +1,3 @@
-const resolver = {
-  resolveId (importee) {
-    return
-    if (importee === 'yjs') {
-      return `${process.cwd()}/src/index.js`
-    }
-  }
-}
-
 export default [{
   // cjs output
   input: {
@@ -20,9 +11,6 @@ export default [{
     entryFileNames: '[name].cjs',
     sourcemap: true
   },
-  plugins: [
-    resolver
-  ],
   external: id => /^(lib0|y-protocols)\//.test(id)
 }, {
   // esm output
@@ -37,8 +25,5 @@ export default [{
     entryFileNames: '[name].mjs',
     sourcemap: true
   },
-  plugins: [
-    resolver
-  ],
   external: id => /^(lib0|y-protocols)\//.test(id)
 }]

src/utils/AttributionManager.js

@@ -1,6 +1,7 @@
 import {
+  _diffSet,
   findIndexInIdRanges,
-  ID // @eslint-disable-line
+  IdSet, ID // eslint-disable-line
 } from '../internals.js'
 
 import * as array from 'lib0/array'
@@ -50,12 +51,20 @@ export class AttrRange {
      */
     this.attrs = attrs
   }
+
+  /**
+   * @param {number} clock
+   * @param {number} len
+   */
+  copyWith (clock, len) {
+    return new AttrRange(clock, len, this.attrs)
+  }
 }
 
 /**
  * @template Attrs
  */
-class AttrRanges {
+export class AttrRanges {
   /**
    * @param {Array<AttrRange<Attrs>>} ids
    */
@@ -104,7 +113,7 @@ class AttrRanges {
        */
       for (let i = 0; i < ids.length - 1;) {
         const range = ids[i]
-        const nextRange = ids[i+1]
+        const nextRange = ids[i + 1]
         // find out how to split range. it must match with next range.
         // 1) we have space. Split if necessary.
         // 2) concat attributes in range to the next range. Split range and splice the remainder at
@@ -257,3 +266,14 @@ export class AttributionManager {
 }
 
 export const createAttributionManager = () => new AttributionManager()
+
+/**
+ * Remove all ranges from `exclude` from `ds`. The result is a fresh AttributionManager containing all ranges from `idSet` that are not
+ * in `exclude`.
+ *
+ * @template {AttributionManager<any>} Set
+ * @param {Set} set
+ * @param {IdSet | AttributionManager<any>} exclude
+ * @return {Set}
+ */
+export const diffAttributionManager = _diffSet

src/utils/IdSet.js

@@ -4,6 +4,8 @@ import {
   splitItem,
   iterateStructs,
   UpdateEncoderV2,
+  AttributionManager,
+  AttrRanges,
   AbstractStruct, DSDecoderV1, DSEncoderV1, DSDecoderV2, DSEncoderV2, Item, GC, StructStore, Transaction, ID // eslint-disable-line
 } from '../internals.js'
 
@@ -27,6 +29,14 @@ export class IdRange {
      */
     this.len = len
   }
+
+  /**
+   * @param {number} clock
+   * @param {number} len
+   */
+  copyWith (clock, len) {
+    return new IdRange(clock, len)
+  }
 }
 
 class IdRanges {
@@ -203,54 +213,58 @@ export const insertIntoIdSet = (dest, src) => {
   })
 }
 
-
 /**
  * Remove all ranges from `exclude` from `ds`. The result is a fresh IdSet containing all ranges from `idSet` that are not
  * in `exclude`.
  *
- * @param {IdSet} idSet
+ * @template {IdSet | AttributionManager<any>} Set
- * @param {IdSet} exclude
+ * @param {Set} set
- * @return {IdSet}
+ * @param {IdSet | AttributionManager<any>} exclude
+ * @return {Set}
  */
-export const diffIdSets = (idSet, exclude) => {
+export const _diffSet = (set, exclude) => {
-  const res = new IdSet()
+  /**
-  idSet.clients.forEach((_idRanges, client) => {
+   * @type {Set}
+   */
+  const res = /** @type {any } */ (set instanceof IdSet ? new IdSet() : new AttributionManager())
+  const Ranges = set instanceof IdSet ? IdRanges : AttrRanges
+  set.clients.forEach((_setRanges, client) => {
     /**
      * @type {Array<IdRange>}
      */
     let resRanges = []
     const _excludedRanges = exclude.clients.get(client)
-    const idRanges = _idRanges.getIds()
+    const setRanges = _setRanges.getIds()
     if (_excludedRanges == null) {
-      resRanges = idRanges.slice()
+      resRanges = setRanges.slice()
     } else {
       const excludedRanges = _excludedRanges.getIds()
       let i = 0; let j = 0
-      let currRange = idRanges[0]
+      let currRange = setRanges[0]
-      while (i < idRanges.length && j < excludedRanges.length) {
+      while (i < setRanges.length && j < excludedRanges.length) {
         const e = excludedRanges[j]
         if (currRange.clock + currRange.len <= e.clock) { // no overlapping, use next range item
           if (currRange.len > 0) resRanges.push(currRange)
-          currRange = idRanges[++i]
+          currRange = setRanges[++i]
         } else if (e.clock + e.len <= currRange.clock) { // no overlapping, use next excluded item
           j++
         } else if (e.clock <= currRange.clock) { // exclude laps into range (we already know that the ranges somehow collide)
           const newClock = e.clock + e.len
           const newLen = currRange.clock + currRange.len - newClock
           if (newLen > 0) {
-            currRange = new IdRange(newClock, newLen)
+            currRange = currRange.copyWith(newClock, newLen)
             j++
           } else {
             // this item is completely overwritten. len=0. We can jump to the next range
-            currRange = idRanges[++i]
+            currRange = setRanges[++i]
           }
         } else { // currRange.clock < e.clock -- range laps into exclude => adjust len
           // beginning can't be empty, add it to the result
           const nextLen = e.clock - currRange.clock
-          resRanges.push(new IdRange(currRange.clock, nextLen))
+          resRanges.push(currRange.copyWith(currRange.clock, nextLen))
           // retain the remaining length after exclude in currRange
-          currRange = new IdRange(currRange.clock + e.len + nextLen, math.max(currRange.len - e.len - nextLen, 0))
+          currRange = currRange.copyWith(currRange.clock + e.len + nextLen, math.max(currRange.len - e.len - nextLen, 0))
-          if (currRange.len === 0) currRange = idRanges[++i]
+          if (currRange.len === 0) currRange = setRanges[++i]
           else j++
         }
       }
@@ -258,15 +272,27 @@ export const diffIdSets = (idSet, exclude) => {
         resRanges.push(currRange)
       }
       i++
-      while (i < idRanges.length) {
+      while (i < setRanges.length) {
-        resRanges.push(idRanges[i++])
+        resRanges.push(setRanges[i++])
       }
     }
-    if (resRanges.length > 0) res.clients.set(client, new IdRanges(resRanges))
+    // @ts-ignore
+    if (resRanges.length > 0) res.clients.set(client, /** @type {any} */ (new Ranges(resRanges)))
   })
   return res
 }
 
+/**
+ * Remove all ranges from `exclude` from `ds`. The result is a fresh IdSet containing all ranges from `idSet` that are not
+ * in `exclude`.
+ *
+ * @template {IdSet} Set
+ * @param {Set} set
+ * @param {IdSet | AttributionManager<any>} exclude
+ * @return {Set}
+ */
+export const diffIdSet = _diffSet
+
 /**
  * @param {IdSet} idSet
  * @param {number} client

tests/AttributionManager.tests.js

@@ -1,8 +1,6 @@
 import * as t from 'lib0/testing'
 import * as am from '../src/utils/AttributionManager.js'
-import * as prng from 'lib0/prng'
+import { compareAttributionManagers, createAttributionManager, ID, createRandomIdSet, createRandomAttributionManager } from './testHelper.js'
-import * as math from 'lib0/math'
-import { compareAttributionManagers, createAttributionManager, ID } from './testHelper.js'
 
 /**
  * @template T
@@ -16,35 +14,6 @@ const simpleConstructAttrs = ops => {
   return attrs
 }
 
-/**
- * @template T
- * @param {prng.PRNG} gen
- * @param {number} clients
- * @param {number} clockRange (max clock - exclusive - by each client)
- * @param {Array<T>} attrChoices (max clock - exclusive - by each client)
- * @return {am.AttributionManager<T>}
- */
-const createRandomAttributionManager = (gen, clients, clockRange, attrChoices) => {
-  const maxOpLen = 5
-  const numOfOps = math.ceil((clients * clockRange) / maxOpLen)
-  const attrMngr = createAttributionManager()
-  for (let i = 0; i < numOfOps; i++) {
-    const client = prng.uint32(gen, 0, clients - 1)
-    const clockStart = prng.uint32(gen, 0, clockRange)
-    const len = prng.uint32(gen, 0, clockRange - clockStart)
-    const attrs = [prng.oneOf(gen, attrChoices)]
-    // maybe add another attr
-    if (prng.bool(gen)) {
-      const a = prng.oneOf(gen, attrChoices)
-      if (attrs.find((attr => attr === a)) == null) {
-        attrs.push(a)
-      }
-    }
-    attrMngr.add(client, clockStart, len, attrs)
-  }
-  return attrMngr
-}
-
 /**
  * @param {t.TestCase} _tc
  */
@@ -133,3 +102,35 @@ export const testRepeatMergingMultipleAttrManagers = tc => {
   compareAttributionManagers(merged, composed)
 }
 
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatRandomDiffing = tc => {
+  const clients = 4
+  const clockRange = 100
+  const attrs = [1, 2, 3]
+  const ds1 = createRandomAttributionManager(tc.prng, clients, clockRange, attrs)
+  const ds2 = createRandomAttributionManager(tc.prng, clients, clockRange, attrs)
+  const merged = am.mergeAttributionManagers([ds1, ds2])
+  const e1 = am.diffAttributionManager(ds1, ds2)
+  const e2 = am.diffAttributionManager(merged, ds2)
+  compareAttributionManagers(e1, e2)
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatRandomDiffing2 = tc => {
+  const clients = 4
+  const clockRange = 100
+  const attrs = [1, 2, 3]
+  const am1 = createRandomAttributionManager(tc.prng, clients, clockRange, attrs)
+  const am2 = createRandomAttributionManager(tc.prng, clients, clockRange, attrs)
+  const idsExclude = createRandomIdSet(tc.prng, clients, clockRange)
+  const merged = am.mergeAttributionManagers([am1, am2])
+  const mergedExcluded = am.diffAttributionManager(merged, idsExclude)
+  const e1 = am.diffAttributionManager(am1, idsExclude)
+  const e2 = am.diffAttributionManager(am2, idsExclude)
+  const excludedMerged = am.mergeAttributionManagers([e1, e2])
+  compareAttributionManagers(mergedExcluded, excludedMerged)
+}

tests/IdSet.tests.js

@@ -1,8 +1,6 @@
 import * as t from 'lib0/testing'
 import * as d from '../src/utils/IdSet.js'
-import * as prng from 'lib0/prng'
+import { compareIdSets, createRandomIdSet, ID } from './testHelper.js'
-import * as math from 'lib0/math'
-import { compareIdSets, ID } from './testHelper.js'
 
 /**
  * @param {Array<[number, number, number]>} ops
@@ -69,7 +67,7 @@ export const testIdsetMerge = _tc => {
 export const testDiffing = _tc => {
   t.group('simple case (1))', () => {
     compareIdSets(
-      d.diffIdSets(
+      d.diffIdSet(
         simpleConstructIdSet([[0, 1, 1], [0, 3, 1]]),
         simpleConstructIdSet([[0, 3, 1]])
       ),
@@ -78,7 +76,7 @@ export const testDiffing = _tc => {
   })
   t.group('subset left', () => {
     compareIdSets(
-      d.diffIdSets(
+      d.diffIdSet(
         simpleConstructIdSet([[0, 1, 3]]),
         simpleConstructIdSet([[0, 1, 1]])
       ),
@@ -87,7 +85,7 @@ export const testDiffing = _tc => {
   })
   t.group('subset right', () => {
     compareIdSets(
-      d.diffIdSets(
+      d.diffIdSet(
         simpleConstructIdSet([[0, 1, 3]]),
         simpleConstructIdSet([[0, 3, 1]])
       ),
@@ -96,7 +94,7 @@ export const testDiffing = _tc => {
   })
   t.group('subset middle', () => {
     compareIdSets(
-      d.diffIdSets(
+      d.diffIdSet(
         simpleConstructIdSet([[0, 1, 3]]),
         simpleConstructIdSet([[0, 2, 1]])
       ),
@@ -105,7 +103,7 @@ export const testDiffing = _tc => {
   })
   t.group('overlapping left', () => {
     compareIdSets(
-      d.diffIdSets(
+      d.diffIdSet(
         simpleConstructIdSet([[0, 1, 3]]),
         simpleConstructIdSet([[0, 0, 2]])
       ),
@@ -114,7 +112,7 @@ export const testDiffing = _tc => {
   })
   t.group('overlapping right', () => {
     compareIdSets(
-      d.diffIdSets(
+      d.diffIdSet(
         simpleConstructIdSet([[0, 1, 3]]),
         simpleConstructIdSet([[0, 3, 5]])
       ),
@@ -123,7 +121,7 @@ export const testDiffing = _tc => {
   })
   t.group('overlapping completely', () => {
     compareIdSets(
-      d.diffIdSets(
+      d.diffIdSet(
         simpleConstructIdSet([[0, 1, 3]]),
         simpleConstructIdSet([[0, 0, 5]])
       ),
@@ -132,7 +130,7 @@ export const testDiffing = _tc => {
   })
   t.group('overlapping into new range', () => {
     compareIdSets(
-      d.diffIdSets(
+      d.diffIdSet(
         simpleConstructIdSet([[0, 1, 3], [0, 5, 2]]),
         simpleConstructIdSet([[0, 0, 6]])
       ),
@@ -141,38 +139,17 @@ export const testDiffing = _tc => {
   })
 }
 
-/**
- * @param {prng.PRNG} gen
- * @param {number} clients
- * @param {number} clockRange (max clock - exclusive - by each client)
- */
-const createRandomDiffSet = (gen, clients, clockRange) => {
-  const maxOpLen = 5
-  const numOfOps = math.ceil((clients * clockRange) / maxOpLen)
-  const ds = d.createIdSet()
-  for (let i = 0; i < numOfOps; i++) {
-    const client = prng.uint32(gen, 0, clients - 1)
-    const clockStart = prng.uint32(gen, 0, clockRange)
-    const len = prng.uint32(gen, 0, clockRange - clockStart)
-    d.addToIdSet(ds, client, clockStart, len)
-  }
-  if (ds.clients.size === clients && clients > 1 && prng.bool(gen)) {
-    ds.clients.delete(prng.uint32(gen, 0, clients))
-  }
-  return ds
-}
-
 /**
  * @param {t.TestCase} tc
  */
 export const testRepeatRandomDiffing = tc => {
   const clients = 4
   const clockRange = 100
-  const ds1 = createRandomDiffSet(tc.prng, clients, clockRange)
+  const ds1 = createRandomIdSet(tc.prng, clients, clockRange)
-  const ds2 = createRandomDiffSet(tc.prng, clients, clockRange)
+  const ds2 = createRandomIdSet(tc.prng, clients, clockRange)
   const merged = d.mergeIdSets([ds1, ds2])
-  const e1 = d.diffIdSets(ds1, ds2)
+  const e1 = d.diffIdSet(ds1, ds2)
-  const e2 = d.diffIdSets(merged, ds2)
+  const e2 = d.diffIdSet(merged, ds2)
   compareIdSets(e1, e2)
 }
 
@@ -187,7 +164,7 @@ export const testRepeatMergingMultipleIdsets = tc => {
    */
   const idss = []
   for (let i = 0; i < 3; i++) {
-    idss.push(createRandomDiffSet(tc.prng, clients, clockRange))
+    idss.push(createRandomIdSet(tc.prng, clients, clockRange))
   }
   const merged = d.mergeIdSets(idss)
   const mergedReverse = d.mergeIdSets(idss.reverse())
@@ -206,3 +183,19 @@ export const testRepeatMergingMultipleIdsets = tc => {
   compareIdSets(merged, composed)
 }
 
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatRandomDiffing2 = tc => {
+  const clients = 4
+  const clockRange = 100
+  const ids1 = createRandomIdSet(tc.prng, clients, clockRange)
+  const ids2 = createRandomIdSet(tc.prng, clients, clockRange)
+  const idsExclude = createRandomIdSet(tc.prng, clients, clockRange)
+  const merged = d.mergeIdSets([ids1, ids2])
+  const mergedExcluded = d.diffIdSet(merged, idsExclude)
+  const e1 = d.diffIdSet(ids1, idsExclude)
+  const e2 = d.diffIdSet(ids2, idsExclude)
+  const excludedMerged = d.mergeIdSets([e1, e2])
+  compareIdSets(mergedExcluded, excludedMerged)
+}

tests/testHelper.js

@@ -6,7 +6,10 @@ import * as syncProtocol from 'y-protocols/sync'
 import * as object from 'lib0/object'
 import * as map from 'lib0/map'
 import * as Y from '../src/index.js'
-import { amAttrsEqual } from '../src/internals.js'
+import * as math from 'lib0/math'
+import {
+  amAttrsEqual, createIdSet, createAttributionManager, addToIdSet
+} from '../src/internals.js'
 
 export * from '../src/index.js'
 
@@ -344,7 +347,55 @@ export const compareAttributionManagers = (am1, am2) => {
   return true
 }
 
+/**
+ * @param {prng.PRNG} gen
+ * @param {number} clients
+ * @param {number} clockRange (max clock - exclusive - by each client)
+ */
+export const createRandomIdSet = (gen, clients, clockRange) => {
+  const maxOpLen = 5
+  const numOfOps = math.ceil((clients * clockRange) / maxOpLen)
+  const ds = createIdSet()
+  for (let i = 0; i < numOfOps; i++) {
+    const client = prng.uint32(gen, 0, clients - 1)
+    const clockStart = prng.uint32(gen, 0, clockRange)
+    const len = prng.uint32(gen, 0, clockRange - clockStart)
+    addToIdSet(ds, client, clockStart, len)
+  }
+  if (ds.clients.size === clients && clients > 1 && prng.bool(gen)) {
+    ds.clients.delete(prng.uint32(gen, 0, clients))
+  }
+  return ds
+}
 
+/**
+ * @template T
+ * @param {prng.PRNG} gen
+ * @param {number} clients
+ * @param {number} clockRange (max clock - exclusive - by each client)
+ * @param {Array<T>} attrChoices (max clock - exclusive - by each client)
+ * @return {Y.AttributionManager<T>}
+ */
+export const createRandomAttributionManager = (gen, clients, clockRange, attrChoices) => {
+  const maxOpLen = 5
+  const numOfOps = math.ceil((clients * clockRange) / maxOpLen)
+  const attrMngr = createAttributionManager()
+  for (let i = 0; i < numOfOps; i++) {
+    const client = prng.uint32(gen, 0, clients - 1)
+    const clockStart = prng.uint32(gen, 0, clockRange)
+    const len = prng.uint32(gen, 0, clockRange - clockStart)
+    const attrs = [prng.oneOf(gen, attrChoices)]
+    // maybe add another attr
+    if (prng.bool(gen)) {
+      const a = prng.oneOf(gen, attrChoices)
+      if (attrs.find(attr => attr === a) == null) {
+        attrs.push(a)
+      }
+    }
+    attrMngr.add(client, clockStart, len, attrs)
+  }
+  return attrMngr
+}
 
 /**
  * 1. reconnect and flush all