Work on AttributionManager

package-lock.json

@@ -9,7 +9,7 @@
       "version": "13.6.24",
       "license": "MIT",
       "dependencies": {
-        "lib0": "^0.2.101",
+        "lib0": "^0.2.103",
         "y-protocols": "^1.0.5"
       },
       "devDependencies": {
@@ -2773,9 +2773,9 @@
       }
     },
     "node_modules/lib0": {
-      "version": "0.2.101",
-      "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.101.tgz",
-      "integrity": "sha512-LljA6+Ehf0Z7YnxhgSAvspzWALjW4wlWdN/W4iGiqYc1KvXQgOVXWI0xwlwqozIL5WRdKeUW2gq0DLhFsY+Xlw==",
+      "version": "0.2.103",
+      "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.103.tgz",
+      "integrity": "sha512-1zT9KqSh54uEQZksnm8ONj0bclW3PrisT59nhgY2eOV4PaCZ5Pt9MV4y4KGkNIE/5vp6yNzpYX/+5/aGvfZS5Q==",
       "license": "MIT",
       "dependencies": {
         "isomorphic.js": "^0.2.4"

package.json

@@ -86,7 +86,7 @@
   },
   "homepage": "https://docs.yjs.dev",
   "dependencies": {
-    "lib0": "^0.2.101",
+    "lib0": "^0.2.103",
     "y-protocols": "^1.0.5"
   },
   "devDependencies": {

src/index.js

@@ -101,7 +101,9 @@ export {
   // idset
   IdSet,
   equalIdSets,
-  createDeleteSetFromStructStore
+  createDeleteSetFromStructStore,
+  AttributionManager,
+  createAttributionManager
 } from './internals.js'
 
 const glo = /** @type {any} */ (typeof globalThis !== 'undefined'

src/utils/AttributionManager.js

@@ -1,8 +1,256 @@
+import {
+  findIndexInIdRanges,
+  ID // @eslint-disable-line
+} from '../internals.js'
 
-export class AttributionManager {
+import * as array from 'lib0/array'
+
+/**
+ * @template T
+ * @param {Array<T>} attrs
+ * @param {T} attr
+ *
+ */
+const amAttrsHas = (attrs, attr) => attrs.find(a => a === attr)
+
+/**
+ * @template T
+ * @param {Array<T>} a
+ * @param {Array<T>} b
+ */
+export const amAttrsEqual = (a, b) => a.length === b.length && a.every(v => amAttrsHas(b, v))
+
+/**
+ * @template T
+ * @param {Array<T>} a
+ * @param {Array<T>} b
+ */
+const amAttrRangeJoin = (a, b) => a.concat(b.filter(attr => !amAttrsHas(a, attr)))
+
+/**
+ * @template Attrs
+ */
+export class AttrRange {
   /**
-   *
+   * @param {number} clock
+   * @param {number} len
+   * @param {Array<Attrs>} attrs
    */
-  constructor () {
+  constructor (clock, len, attrs) {
+    /**
+     * @readonly
+     */
+    this.clock = clock
+    /**
+     * @readonly
+     */
+    this.len = len
+    /**
+     * @readonly
+     */
+    this.attrs = attrs
   }
 }
+
+/**
+ * @template Attrs
+ */
+class AttrRanges {
+  /**
+   * @param {Array<AttrRange<Attrs>>} ids
+   */
+  constructor (ids) {
+    this.sorted = false
+    /**
+     * @private
+     */
+    this._ids = ids
+  }
+
+  /**
+   * @param {number} clock
+   * @param {number} length
+   * @param {Array<Attrs>} attrs
+   */
+  add (clock, length, attrs) {
+    this.sorted = false
+    this._ids.push(new AttrRange(clock, length, attrs))
+  }
+
+  /**
+   * Return the list of id ranges, sorted and merged.
+   */
+  getIds () {
+    const ids = this._ids
+    if (!this.sorted) {
+      this.sorted = true
+      ids.sort((a, b) => a.clock - b.clock)
+      /**
+       * algorithm thoughts:
+       * - sort (by clock AND by length), bigger length is to the right (or not, we can't make
+       *   assumptions abouth length after long length has been split)
+       * -- maybe better: sort by clock+length. Then split items from right to left. This way, items are always
+       *   in the right order. But I also need to swap if left items is smaller after split
+       *   --- thought: there is no way to go around swapping. Unless, for each item from left to
+       *   right, when I have to split because one of the look-ahead items is overlapping, i split
+       *   it and merge the attributes into the following ones (that I also need to split). Best is
+       *   probably left to right with lookahead.
+       * - left to right, split overlapping items so that we can make the assumption that either an
+       *   item is overlapping with the next 1-on-1 or it is not overlapping at all (when splitting,
+       *   we can already incorporate the attributes)
+       *   -- better: for each item, go left to right and add own attributes to overlapping items.
+       *   Split them if necessary. After split, i must insert the retainer at a valid position.
+       * - merge items if neighbor has same attributes
+       */
+      for (let i = 0; i < ids.length - 1; i++) {
+        const range = ids[i]
+        const nextRange = ids[i+1]
+        // find out how to split range. it must match with next range.
+        // 1) we have space. Split if necessary.
+        // 2) concat attributes in range to the next range. Split range and splice the remainder at
+        // the correct position.
+        if (range.clock < nextRange.clock) { // might need to split range
+          if (range.clock + range.len > nextRange.clock) {
+            // is overlapping
+            const diff = nextRange.clock - range.clock
+            ids[i] = new AttrRange(range.clock, diff, range.attrs)
+            ids.splice(i + 1, 0, new AttrRange(nextRange.clock, range.len - diff, range.attrs))
+          }
+          continue
+        }
+        // now we know that range.clock === nextRange.clock
+        // merge range with nextRange
+        const largerRange = range.len > nextRange.len ? range : nextRange
+        const smallerLen = range.len < nextRange.len ? range.len : nextRange.len
+        ids[i] = new AttrRange(range.clock, smallerLen, amAttrRangeJoin(range.attrs, nextRange.attrs))
+        if (range.len === nextRange.len) {
+          ids.splice(i + 1, 1)
+          i--
+        } else {
+          ids[i + 1] = new AttrRange(range.clock + smallerLen, largerRange.len - smallerLen, largerRange.attrs)
+          array.bubblesortItem(ids, i + 1, (a, b) => a.clock - b.clock)
+        }
+      }
+
+      // merge items without filtering or splicing the array.
+      // i is the current pointer
+      // j refers to the current insert position for the pointed item
+      // try to merge dels[i] into dels[j-1] or set dels[j]=dels[i]
+      let i, j
+      for (i = 1, j = 1; i < ids.length; i++) {
+        const left = ids[j - 1]
+        const right = ids[i]
+        if (left.clock + left.len === right.clock && amAttrsEqual(left.attrs, right.attrs)) {
+          ids[j - 1] = new AttrRange(left.clock, left.len + right.len, left.attrs)
+        } else if (right.len !== 0) {
+          if (j < i) {
+            ids[j] = right
+          }
+          j++
+        }
+      }
+      ids.length = ids[j - 1].len === 0 ? j - 1 : j
+    }
+    return ids
+  }
+}
+
+/**
+ * @template T
+ * @param {Array<AttributionManager<T>>} ams
+ * @return {AttributionManager<T>} A fresh IdSet
+ */
+export const mergeAttributionManagers = ams => {
+  const merged = createAttributionManager()
+  for (let amsI = 0; amsI < ams.length; amsI++) {
+    ams[amsI].clients.forEach((rangesLeft, client) => {
+      if (!merged.clients.has(client)) {
+        // Write all missing keys from current set and all following.
+        // If merged already contains `client` current ds has already been added.
+        const ids = rangesLeft.getIds().slice()
+        for (let i = amsI + 1; i < ams.length; i++) {
+          const nextIds = ams[i].clients.get(client)
+          if (nextIds) {
+            array.appendTo(ids, nextIds.getIds())
+          }
+        }
+        merged.clients.set(client, new AttrRanges(ids))
+      }
+    })
+  }
+  return merged
+}
+
+/**
+ * @template Attrs
+ */
+export class AttributionManager {
+  constructor () {
+    /**
+     * @type {Map<number,AttrRanges<Attrs>>}
+     */
+    this.clients = new Map()
+  }
+
+  /**
+   * @param {ID} id
+   * @return {boolean}
+   */
+  has (id) {
+    const dr = this.clients.get(id.client)
+    if (dr) {
+      return findIndexInIdRanges(dr.getIds(), id.clock) !== null
+    }
+    return false
+  }
+
+  /**
+   * @param {ID} id
+   * @param {number} len
+   * @return {Array<AttrRange<Attrs>>?}
+   */
+  slice (id, len) {
+    const dr = this.clients.get(id.client)
+    if (dr) {
+      /**
+       * @type {Array<AttrRange<Attrs>>}
+       */
+      const ranges = dr.getIds()
+      let index = findIndexInIdRanges(ranges, id.clock)
+      if (index !== null) {
+        const res = []
+        while (true) {
+          let r = ranges[index]
+          if (r.clock < id.clock) {
+            r = new AttrRange(id.clock, r.len - (id.clock - r.clock), r.attrs)
+          }
+          if (r.clock + r.len > id.clock + len) {
+            r = new AttrRange(r.clock, id.clock + len - r.clock, r.attrs)
+          }
+          if (r.len <= 0) break
+          res.push(r)
+          index++
+        }
+        return res
+      }
+    }
+    return null
+  }
+
+  /**
+   * @param {number} client
+   * @param {number} clock
+   * @param {number} len
+   * @param {Array<Attrs>} attrs
+   */
+  add (client, clock, len, attrs) {
+    const ranges = this.clients.get(client)
+    if (ranges == null) {
+      this.clients.set(client, new AttrRanges([new AttrRange(clock, len, attrs)]))
+    } else {
+      ranges.add(clock, len, attrs)
+    }
+  }
+}
+
+export const createAttributionManager = () => new AttributionManager()

src/utils/IdSet.js

@@ -19,12 +19,10 @@ export class IdRange {
    */
   constructor (clock, len) {
     /**
-     * @readonly
      * @type {number}
      */
     this.clock = clock
     /**
-     * @readonly
      * @type {number}
      */
     this.len = len
@@ -87,11 +85,7 @@ class IdRanges {
           j++
         }
       }
-      if (ids[j - 1].len === 0) {
-        ids.length = j - 1
-      } else {
-        ids.length = j
-      }
+      ids.length = ids[j - 1].len === 0 ? j - 1 : j
     }
     return ids
   }

tests/AttributionManager.tests.js

@@ -0,0 +1,131 @@
+import * as t from 'lib0/testing'
+import * as am from '../src/utils/AttributionManager.js'
+import * as prng from 'lib0/prng'
+import * as math from 'lib0/math'
+import { compareAttributionManagers, createAttributionManager, ID } from './testHelper.js'
+
+/**
+ * @template T
+ * @param {Array<[number, number, number, Array<T>]>} ops
+ */
+const simpleConstructAttrs = ops => {
+  const attrs = createAttributionManager()
+  ops.forEach(op => {
+    attrs.add(op[0], op[1], op[2], op[3])
+  })
+  return attrs
+}
+
+/**
+ * @template T
+ * @param {prng.PRNG} gen
+ * @param {number} clients
+ * @param {number} clockRange (max clock - exclusive - by each client)
+ * @param {Array<T>} attrChoices (max clock - exclusive - by each client)
+ * @return {am.AttributionManager<T>}
+ */
+const createRandomAttributionManager = (gen, clients, clockRange, attrChoices) => {
+  const maxOpLen = 5
+  const numOfOps = math.ceil((clients * clockRange) / maxOpLen)
+  const attrMngr = createAttributionManager()
+  for (let i = 0; i < numOfOps; i++) {
+    const client = prng.uint32(gen, 0, clients - 1)
+    const clockStart = prng.uint32(gen, 0, clockRange)
+    const len = prng.uint32(gen, 0, clockRange - clockStart)
+    const attrs = [prng.oneOf(gen, attrChoices)]
+    if (prng.bool(gen)) {
+      attrs.push(prng.oneOf(gen, attrChoices))
+    }
+    attrMngr.add(client, clockStart, len, attrs)
+  }
+  return attrMngr
+}
+
+/**
+ * @param {t.TestCase} _tc
+ */
+export const testAmMerge = _tc => {
+  const attrs = [42]
+  t.group('filter out empty items (1))', () => {
+    compareAttributionManagers(
+      simpleConstructAttrs([[0, 1, 0, attrs]]),
+      simpleConstructAttrs([])
+    )
+  })
+  t.group('filter out empty items (2))', () => {
+    compareAttributionManagers(
+      simpleConstructAttrs([[0, 1, 0, attrs], [0, 2, 0, attrs]]),
+      simpleConstructAttrs([])
+    )
+  })
+  t.group('filter out empty items (3 - end))', () => {
+    compareAttributionManagers(
+      simpleConstructAttrs([[0, 1, 1, attrs], [0, 2, 0, attrs]]),
+      simpleConstructAttrs([[0, 1, 1, attrs]])
+    )
+  })
+  t.group('filter out empty items (4 - middle))', () => {
+    compareAttributionManagers(
+      simpleConstructAttrs([[0, 1, 1, attrs], [0, 2, 0, attrs], [0, 3, 1, attrs]]),
+      simpleConstructAttrs([[0, 1, 1, attrs], [0, 3, 1, attrs]])
+    )
+  })
+  t.group('filter out empty items (5 - beginning))', () => {
+    compareAttributionManagers(
+      simpleConstructAttrs([[0, 1, 0, attrs], [0, 2, 1, attrs], [0, 3, 1, attrs]]),
+      simpleConstructAttrs([[0, 2, 1, attrs], [0, 3, 1, attrs]])
+    )
+  })
+  t.group('merge of overlapping id ranges', () => {
+    compareAttributionManagers(
+      simpleConstructAttrs([[0, 1, 2, attrs], [0, 0, 2, attrs]]),
+      simpleConstructAttrs([[0, 0, 3, attrs]])
+    )
+  })
+  t.group('construct without hole', () => {
+    compareAttributionManagers(
+      simpleConstructAttrs([[0, 1, 2, attrs], [0, 3, 1, attrs]]),
+      simpleConstructAttrs([[0, 1, 3, attrs]])
+    )
+  })
+  t.group('no merge of overlapping id ranges with different attributes', () => {
+    compareAttributionManagers(
+      simpleConstructAttrs([[0, 1, 2, [1]], [0, 0, 2, [2]]]),
+      simpleConstructAttrs([[0, 0, 1, [2]], [0, 1, 1, [1, 2]], [0, 2, 1, [2]]])
+    )
+  })
+}
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatMergingMultipleAttrManagers = tc => {
+  const clients = 4
+  const clockRange = 100
+  /**
+   * @type {Array<am.AttributionManager<number>>}
+   */
+  const sets = []
+  for (let i = 0; i < 3; i++) {
+    sets.push(createRandomAttributionManager(tc.prng, clients, clockRange, [1, 2, 3]))
+  }
+  const merged = am.mergeAttributionManagers(sets)
+  const mergedReverse = am.mergeAttributionManagers(sets.reverse())
+  compareAttributionManagers(merged, mergedReverse)
+  const composed = am.createAttributionManager()
+  for (let iclient = 0; iclient < clients; iclient++) {
+    for (let iclock = 0; iclock < clockRange + 42; iclock++) {
+      const mergedHas = merged.has(new ID(iclient, iclock))
+      const oneHas = sets.some(ids => ids.has(new ID(iclient, iclock)))
+      t.assert(mergedHas === oneHas)
+      const mergedAttrs = merged.slice(new ID(iclient, iclock), 1)
+      if (mergedAttrs) {
+        mergedAttrs.forEach(a => {
+          composed.add(iclient, a.clock, a.len, a.attrs)
+        })
+      }
+    }
+  }
+  compareAttributionManagers(merged, composed)
+}
+

tests/IdSet.tests.js

@@ -2,7 +2,7 @@ import * as t from 'lib0/testing'
 import * as d from '../src/utils/IdSet.js'
 import * as prng from 'lib0/prng'
 import * as math from 'lib0/math'
-import { compareIdSets } from './testHelper.js'
+import { compareIdSets, ID } from './testHelper.js'
 
 /**
  * @param {Array<[number, number, number]>} ops
@@ -175,3 +175,32 @@ export const testRepeatRandomDiffing = tc => {
   const e2 = d.diffIdSets(merged, ds2)
   compareIdSets(e1, e2)
 }
+
+/**
+ * @param {t.TestCase} tc
+ */
+export const testRepeatMergingMultipleIdsets = tc => {
+  const clients = 4
+  const clockRange = 100
+  /**
+   * @type {Array<d.IdSet>}
+   */
+  const idss = []
+  for (let i = 0; i < 3; i++) {
+    idss.push(createRandomDiffSet(tc.prng, clients, clockRange))
+  }
+  const merged = d.mergeIdSets(idss)
+  const mergedReverse = d.mergeIdSets(idss.reverse())
+  compareIdSets(merged, mergedReverse)
+  const composed = d.createIdSet()
+  for (let iclient = 0; iclient < clients; iclient++) {
+    for (let iclock = 0; iclock < clockRange + 42; iclock++) {
+      const mergedHas = merged.has(new ID(iclient, iclock))
+      const oneHas = idss.some(ids => ids.has(new ID(iclient, iclock)))
+      t.assert(mergedHas === oneHas)
+      d.addToIdSet(composed, iclient, iclock, 1)
+    }
+  }
+  compareIdSets(merged, composed)
+}
+

tests/index.js

@@ -13,6 +13,7 @@ import * as updates from './updates.tests.js'
 import * as relativePositions from './relativePositions.tests.js'
 import * as delta from './delta.tests.js'
 import * as idset from './IdSet.tests.js'
+import * as attributionManager from './AttributionManager.tests.js'
 
 import { runTests } from 'lib0/testing'
 import { isBrowser, isNode } from 'lib0/environment'
@@ -23,7 +24,7 @@ if (isBrowser) {
 }
 
 const tests = {
-  doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, updates, relativePositions, delta, idset
+  doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, updates, relativePositions, delta, idset, attributionManager
 }
 
 const run = async () => {

tests/testHelper.js

@@ -6,6 +6,7 @@ import * as syncProtocol from 'y-protocols/sync'
 import * as object from 'lib0/object'
 import * as map from 'lib0/map'
 import * as Y from '../src/index.js'
+import { amAttrsEqual } from '../src/internals.js'
 
 export * from '../src/index.js'
 
@@ -323,6 +324,27 @@ export const compareIdSets = (idSet1, idSet2) => {
   return true
 }
 
+/**
+ * @template T
+ * @param {Y.AttributionManager<T>} am1
+ * @param {Y.AttributionManager<T>} am2
+ */
+export const compareAttributionManagers = (am1, am2) => {
+  if (am1.clients.size !== am2.clients.size) return false
+  for (const [client, _items1] of am1.clients.entries()) {
+    const items1 = _items1.getIds()
+    const items2 = am2.clients.get(client)?.getIds()
+    t.assert(items2 !== undefined && items1.length === items2.length)
+    for (let i = 0; i < items1.length; i++) {
+      const di1 = items1[i]
+      const di2 = /** @type {Array<import('../src/utils/AttributionManager.js').AttrRange<T>>} */ (items2)[i]
+      t.assert(di1.clock === di2.clock && di1.len === di2.len && amAttrsEqual(di1.attrs, di2.attrs))
+    }
+  }
+  return true
+}
+
+
 
 /**
  * 1. reconnect and flush all