[readUpdateIdRanges] and refactors

src/index.js

@@ -76,8 +76,6 @@ export {
   logType,
   mergeUpdates,
   mergeUpdatesV2,
-  parseUpdateMeta,
-  parseUpdateMetaV2,
   encodeStateVectorFromUpdate,
   encodeStateVectorFromUpdateV2,
   encodeRelativePosition,
@@ -114,7 +112,9 @@ export {
   DiffAttributionManager,
   createIdSet,
   mergeIdSets,
-  cloneDoc
+  cloneDoc,
+  readUpdateIdRanges,
+  readUpdateIdRangesV2
 } from './internals.js'
 
 const glo = /** @type {any} */ (typeof globalThis !== 'undefined'

src/utils/Doc.js

@@ -50,8 +50,8 @@ export const generateNewClientId = random.uint32
  * @property {function(Doc):void} DocEvents.destroy
  * @property {function(Doc):void} DocEvents.load
  * @property {function(boolean, Doc):void} DocEvents.sync
- * @property {function(Uint8Array, any, Doc, Transaction):void} DocEvents.update
+ * @property {function(Uint8Array<ArrayBuffer>, any, Doc, Transaction):void} DocEvents.update
- * @property {function(Uint8Array, any, Doc, Transaction):void} DocEvents.updateV2
+ * @property {function(Uint8Array<ArrayBuffer>, any, Doc, Transaction):void} DocEvents.updateV2
  * @property {function(Doc):void} DocEvents.beforeAllTransactions
  * @property {function(Transaction, Doc):void} DocEvents.beforeTransaction
  * @property {function(Transaction, Doc):void} DocEvents.beforeObserverCalls

src/utils/IdSet.js

@@ -14,6 +14,7 @@ import * as array from 'lib0/array'
 import * as math from 'lib0/math'
 import * as encoding from 'lib0/encoding'
 import * as decoding from 'lib0/decoding'
+import * as traits from 'lib0/traits'
 
 export class IdRange {
   /**
@@ -157,6 +158,9 @@ export class IdRanges {
   }
 }
 
+/**
+ * @implements {traits.EqualityTrait}
+ */
 export class IdSet {
   constructor () {
     /**
@@ -270,6 +274,13 @@ export class IdSet {
   delete (client, clock, len) {
     _deleteRangeFromIdSet(this, client, clock, len)
   }
+
+  /**
+   * @param {any} other
+   */
+  [traits.EqualityTraitSymbol] (other) {
+    return equalIdSets(this, other)
+  }
 }
 
 /**

src/utils/updates.js

@@ -34,7 +34,8 @@ import {
   UpdateEncoderV2,
   writeIdSet,
   YXmlElement,
-  YXmlHook
+  YXmlHook,
+  createIdSet
 } from '../internals.js'
 
 /**
@@ -118,7 +119,6 @@ export const logUpdate = update => logUpdateV2(update, UpdateDecoderV1)
 /**
  * @param {Uint8Array} update
  * @param {typeof UpdateDecoderV2 | typeof UpdateDecoderV1} [YDecoder]
- *
  */
 export const logUpdateV2 = (update, YDecoder = UpdateDecoderV2) => {
   const structs = []
@@ -247,48 +247,40 @@ export const encodeStateVectorFromUpdate = update => encodeStateVectorFromUpdate
 
 /**
  * @param {Uint8Array} update
- * @param {typeof UpdateDecoderV1 | typeof UpdateDecoderV2} YDecoder
+ * @param {typeof UpdateDecoderV2 | typeof UpdateDecoderV1} [YDecoder]
- * @return {{ from: Map<number,number>, to: Map<number,number> }}
  */
-export const parseUpdateMetaV2 = (update, YDecoder = UpdateDecoderV2) => {
+export const readUpdateIdRangesV2 = (update, YDecoder = UpdateDecoderV2) => {
-  /**
+  const updateDecoder = new YDecoder(decoding.createDecoder(update))
-   * @type {Map<number, number>}
+  const lazyDecoder = new LazyStructReader(updateDecoder, true)
-   */
+  const inserts = createIdSet()
-  const from = new Map()
+  let lastClientId = -1
-  /**
+  let lastClock = 0
-   * @type {Map<number, number>}
+  let lastLen = 0
-   */
+  for (let curr = lazyDecoder.curr; curr !== null; curr = lazyDecoder.next()) {
-  const to = new Map()
+    const currId = curr.id
-  const updateDecoder = new LazyStructReader(new YDecoder(decoding.createDecoder(update)), false)
+    if (lastClientId === currId.client && lastClock + lastLen === currId.clock) {
-  let curr = updateDecoder.curr
+      // default case: extend prev entry
-  if (curr !== null) {
+      lastLen += curr.length
-    let currClient = curr.id.client
+    } else {
-    let currClock = curr.id.clock
+      if (lastClientId >= 0) {
-    // write the beginning to `from`
+        inserts.add(lastClientId, lastClock, lastLen)
-    from.set(currClient, currClock)
-    for (; curr !== null; curr = updateDecoder.next()) {
-      if (currClient !== curr.id.client) {
-        // We found a new client
-        // write the end to `to`
-        to.set(currClient, currClock)
-        // write the beginning to `from`
-        from.set(curr.id.client, curr.id.clock)
-        // update currClient
-        currClient = curr.id.client
       }
-      currClock = curr.id.clock + curr.length
+      lastClientId = currId.client
+      lastClock = currId.clock
+      lastLen = curr.length
     }
-    // write the end to `to`
-    to.set(currClient, currClock)
   }
-  return { from, to }
+  if (lastClientId >= 0) {
+    inserts.add(lastClientId, lastClock, lastLen)
+  }
+  const deletes = readIdSet(updateDecoder)
+  return { inserts, deletes }
 }
 
 /**
  * @param {Uint8Array} update
- * @return {{ from: Map<number,number>, to: Map<number,number> }}
  */
-export const parseUpdateMeta = update => parseUpdateMetaV2(update, UpdateDecoderV1)
+export const readUpdateIdRanges = update => readUpdateIdRangesV2(update, UpdateDecoderV1)
 
 /**
  * This method is intended to slice any kind of struct and retrieve the right part.

tests/IdMap.tests.js

@@ -205,7 +205,7 @@ export const testUserAttributionEncodingBenchmark = tc => {
    * @todo it should be possible to only use a single idmap and, in each attr entry, encode the diff
    * to the previous entries (e.g. remove a,b, insert c,d)
    */
-  let attributions = createIdMap()
+  const attributions = createIdMap()
   let currentTime = time.getUnixTime()
   const ydoc = new YY.Doc()
   ydoc.on('afterTransaction', tr => {
@@ -215,7 +215,7 @@ export const testUserAttributionEncodingBenchmark = tc => {
   })
   const ytext = ydoc.getText()
   const N = 10000
-  t.measureTime(`time to attribute ${N/1000}k changes`, () => {
+  t.measureTime(`time to attribute ${N / 1000}k changes`, () => {
     for (let i = 0; i < N; i++) {
       if (i % 2 > 0 && ytext.length > 0) {
         const pos = prng.int31(tc.prng, 0, ytext.length)
@@ -226,12 +226,12 @@ export const testUserAttributionEncodingBenchmark = tc => {
       }
     }
   })
-  t.measureTime(`time to encode attributions map`, () => {
+  t.measureTime('time to encode attributions map', () => {
     /**
      * @todo I can optimize size by encoding only the differences to the prev item.
      */
     const encAttributions = idmap.encodeIdMap(attributions)
     t.info('encoded size: ' + encAttributions.byteLength)
-    t.info('size per change: ' + math.floor((encAttributions.byteLength / N) * 100)/100 + ' bytes')
+    t.info('size per change: ' + math.floor((encAttributions.byteLength / N) * 100) / 100 + ' bytes')
   })
 }

tests/updates.tests.js

@@ -6,14 +6,15 @@ import * as encoding from 'lib0/encoding'
 import * as decoding from 'lib0/decoding'
 import * as object from 'lib0/object'
 import * as delta from 'lib0/delta'
+import * as array from 'lib0/array'
 
 /**
  * @typedef {Object} Enc
- * @property {function(Array<Uint8Array>):Uint8Array} Enc.mergeUpdates
+ * @property {function(Array<Uint8Array<ArrayBuffer>>):Uint8Array<ArrayBuffer>} Enc.mergeUpdates
- * @property {function(Y.Doc):Uint8Array} Enc.encodeStateAsUpdate
+ * @property {function(Y.Doc):Uint8Array<ArrayBuffer>} Enc.encodeStateAsUpdate
  * @property {function(Y.Doc, Uint8Array):void} Enc.applyUpdate
  * @property {function(Uint8Array):void} Enc.logUpdate
- * @property {function(Uint8Array):{from:Map<number,number>,to:Map<number,number>}} Enc.parseUpdateMeta
+ * @property {function(Uint8Array):{deletes:Y.IdSet,inserts:Y.IdSet}} Enc.readUpdateIdRanges
  * @property {function(Y.Doc):Uint8Array} Enc.encodeStateVector
  * @property {function(Uint8Array):Uint8Array} Enc.encodeStateVectorFromUpdate
  * @property {'update'|'updateV2'} Enc.updateEventName
@@ -29,7 +30,7 @@ const encV1 = {
   encodeStateAsUpdate: Y.encodeStateAsUpdate,
   applyUpdate: Y.applyUpdate,
   logUpdate: Y.logUpdate,
-  parseUpdateMeta: Y.parseUpdateMeta,
+  readUpdateIdRanges: Y.readUpdateIdRanges,
   encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdate,
   encodeStateVector: Y.encodeStateVector,
   updateEventName: 'update',
@@ -45,7 +46,7 @@ const encV2 = {
   encodeStateAsUpdate: Y.encodeStateAsUpdateV2,
   applyUpdate: Y.applyUpdateV2,
   logUpdate: Y.logUpdateV2,
-  parseUpdateMeta: Y.parseUpdateMetaV2,
+  readUpdateIdRanges: Y.readUpdateIdRangesV2,
   encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdateV2,
   encodeStateVector: Y.encodeStateVector,
   updateEventName: 'updateV2',
@@ -67,7 +68,7 @@ const encDoc = {
   encodeStateAsUpdate: Y.encodeStateAsUpdateV2,
   applyUpdate: Y.applyUpdateV2,
   logUpdate: Y.logUpdateV2,
-  parseUpdateMeta: Y.parseUpdateMetaV2,
+  readUpdateIdRanges: Y.readUpdateIdRangesV2,
   encodeStateVectorFromUpdate: Y.encodeStateVectorFromUpdateV2,
   encodeStateVector: Y.encodeStateVector,
   updateEventName: 'updateV2',
@@ -142,7 +143,7 @@ export const testKeyEncoding = tc => {
 
 /**
  * @param {Y.Doc} ydoc
- * @param {Array<Uint8Array>} updates - expecting at least 4 updates
+ * @param {Array<Uint8Array<ArrayBuffer>>} updates - expecting at least 4 updates
  * @param {Enc} enc
  * @param {boolean} hasDeletes
  */
@@ -188,11 +189,11 @@ const checkUpdateCases = (ydoc, updates, enc, hasDeletes) => {
     if (enc.updateEventName !== 'update') { // @todo should this also work on legacy updates?
       for (let j = 1; j < updates.length; j++) {
         const partMerged = enc.mergeUpdates(updates.slice(j))
-        const partMeta = enc.parseUpdateMeta(partMerged)
+        const partMeta = enc.readUpdateIdRanges(partMerged)
         const targetSV = enc.encodeStateVectorFromUpdate(enc.mergeUpdates(updates.slice(0, j)))
         const diffed = enc.diffUpdate(mergedUpdates, targetSV)
-        const diffedMeta = enc.parseUpdateMeta(diffed)
+        const diffedMeta = enc.readUpdateIdRanges(diffed)
-        t.compare(partMeta, diffedMeta)
+        t.compare(partMeta.inserts, diffedMeta.inserts)
         {
           // We can'd do the following
           //  - t.compare(diffed, mergedDeletes)
@@ -214,13 +215,13 @@ const checkUpdateCases = (ydoc, updates, enc, hasDeletes) => {
         }
       }
     }
-
+    const meta = enc.readUpdateIdRanges(mergedUpdates)
-    const meta = enc.parseUpdateMeta(mergedUpdates)
+    meta.inserts.clients.forEach(range => { t.assert(range.getIds()[0].clock === 0) })
-    meta.from.forEach((clock, _client) => t.assert(clock === 0))
+    meta.inserts.clients.forEach((range, client) => {
-    meta.to.forEach((clock, client) => {
       const structs = /** @type {Array<Y.Item>} */ (merged.store.clients.get(client))
       const lastStruct = structs[structs.length - 1]
-      t.assert(lastStruct.id.clock + lastStruct.length === clock)
+      const lastIdRange = array.last(range.getIds())
+      t.assert(lastStruct.id.clock + lastStruct.length === lastIdRange.clock + lastIdRange.len)
     })
   })
 }
@@ -232,15 +233,13 @@ export const testMergeUpdates1 = _tc => {
   encoders.forEach((enc) => {
     t.info(`Using encoder: ${enc.description}`)
     const ydoc = new Y.Doc({ gc: false })
-    const updates = /** @type {Array<Uint8Array>} */ ([])
+    const updates = /** @type {Array<Uint8Array<ArrayBuffer>>} */ ([])
     ydoc.on(enc.updateEventName, update => { updates.push(update) })
-
     const array = ydoc.getArray()
     array.insert(0, [1])
     array.insert(0, [2])
     array.insert(0, [3])
     array.insert(0, [4])
-
     checkUpdateCases(ydoc, updates, enc, false)
   })
 }
@@ -252,15 +251,13 @@ export const testMergeUpdates2 = _tc => {
   encoders.forEach((enc, _i) => {
     t.info(`Using encoder: ${enc.description}`)
     const ydoc = new Y.Doc({ gc: false })
-    const updates = /** @type {Array<Uint8Array>} */ ([])
+    const updates = /** @type {Array<Uint8Array<ArrayBuffer>>} */ ([])
     ydoc.on(enc.updateEventName, update => { updates.push(update) })
-
     const array = ydoc.getArray()
     array.insert(0, [1, 2])
     array.delete(1, 1)
     array.insert(0, [3, 4])
     array.delete(1, 2)
-
     checkUpdateCases(ydoc, updates, enc, true)
   })
 }