implement support for diffing deletesets

src/internals.js

@@ -40,3 +40,4 @@ export * from './structs/ContentString.js'
 export * from './structs/ContentType.js'
 export * from './structs/Item.js'
 export * from './structs/Skip.js'
+export * from './utils/AttributionManager.js'

src/types/YText.js

@@ -27,9 +27,12 @@ import {
   updateMarkerChanges,
   ContentType,
   warnPrematureAccess,
-  ArraySearchMarker, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line
+  ArraySearchMarker, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ID, Doc, Item, Snapshot, Transaction, AttributionManager,  // eslint-disable-line
+  snapshot
 } from '../internals.js'
 
+import * as delta from '../utils/Delta.js'
+
 import * as object from 'lib0/object'
 import * as map from 'lib0/map'
 import * as error from 'lib0/error'
@@ -996,6 +999,96 @@ export class YText extends AbstractType {
     }
   }
 
+  /**
+   * Render the difference to another ydoc (which can be empty) and highlight the differences with
+   * attributions.
+   *
+   * Note that deleted content that was not deleted in prevYdoc is rendered as an insertion with the
+   * attribution `{ isDeleted: true, .. }`.
+   *
+   * @param {AttributionManager} [attributionManager]
+   * @param {Doc} [prevYdoc]
+   * @return {import('../utils/Delta.js').Delta} The Delta representation of this type.
+   *
+   * @public
+   */
+  getContent (attributionManager, prevYdoc) {
+    this.doc ?? warnPrematureAccess()
+    const prevSnapshot = prevYdoc ? snapshot(prevYdoc) : null
+    const d = delta.create()
+    /**
+     * @type {{ [key: string]: any }}
+     */
+    const currentAttributes = {}
+    const doc = /** @type {Doc} */ (this.doc)
+    const computeContent = () => {
+      let n = this._start
+      while (n !== null) {
+        switch (n.content.constructor) {
+          case ContentString: {
+            const cur = currentAttributes.get('ychange')
+            if (snapshot !== undefined && !isVisible(n, snapshot)) {
+              if (cur === undefined || cur.user !== n.id.client || cur.type !== 'removed') {
+                packStr()
+                currentAttributes.set('ychange', computeYChange ? computeYChange('removed', n.id) : { type: 'removed' })
+              }
+            } else if (prevSnapshot !== undefined && !isVisible(n, prevSnapshot)) {
+              if (cur === undefined || cur.user !== n.id.client || cur.type !== 'added') {
+                packStr()
+                currentAttributes.set('ychange', computeYChange ? computeYChange('added', n.id) : { type: 'added' })
+              }
+            } else if (cur !== undefined) {
+              packStr()
+              currentAttributes.delete('ychange')
+            }
+            str += /** @type {ContentString} */ (n.content).str
+            break
+          }
+          case ContentType:
+          case ContentEmbed: {
+            packStr()
+            /**
+             * @type {Object<string,any>}
+             */
+            const op = {
+              insert: n.content.getContent()[0]
+            }
+            if (currentAttributes.size > 0) {
+              const attrs = /** @type {Object<string,any>} */ ({})
+              op.attributes = attrs
+              currentAttributes.forEach((value, key) => {
+                attrs[key] = value
+              })
+            }
+            ops.push(op)
+            break
+          }
+          case ContentFormat:
+            if (isVisible(n, snapshot)) {
+              packStr()
+              updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (n.content))
+            }
+            break
+        }
+        n = n.right
+      }
+    }
+    if (prevSnapshot) {
+      // snapshots are merged again after the transaction, so we need to keep the
+      // transaction alive until we are done
+      transact(doc, transaction => {
+        if (prevSnapshot) {
+          splitSnapshotAffectedStructs(transaction, prevSnapshot)
+        }
+        computeContent()
+      }, 'cleanup')
+    } else {
+      computeContent()
+    }
+    return d.done()
+  }
+
+
   /**
    * Returns the Delta representation of this YText type.
    *

src/utils/AttributionManager.js

@@ -0,0 +1,8 @@
+
+export class AttributionManager {
+  /**
+   * 
+   */
+  constructor () {
+  }
+}

src/utils/DeleteSet.js

@@ -20,10 +20,12 @@ export class DeleteItem {
    */
   constructor (clock, len) {
     /**
+     * @readonly
      * @type {number}
      */
     this.clock = clock
     /**
+     * @readonly
      * @type {number}
      */
     this.len = len
@@ -111,7 +113,7 @@ export const isDeleted = (ds, id) => {
  * @function
  */
 export const sortAndMergeDeleteSet = ds => {
-  ds.clients.forEach(dels => {
+  ds.clients.forEach((dels, client) => {
     dels.sort((a, b) => a.clock - b.clock)
     // merge items without filtering or splicing the array
     // i is the current pointer
@@ -122,7 +124,12 @@ export const sortAndMergeDeleteSet = ds => {
       const left = dels[j - 1]
       const right = dels[i]
       if (left.clock + left.len >= right.clock) {
-        left.len = math.max(left.len, right.clock + right.len - left.clock)
+        const r = right.clock + right.len - left.clock
+        if (left.len < r) {
+          dels[j - 1] = new DeleteItem(left.clock, r)
+        }
+      } else if (left.len === 0) {
+        dels[j - 1] = right
       } else {
         if (j < i) {
           dels[j] = right
@@ -130,7 +137,14 @@ export const sortAndMergeDeleteSet = ds => {
         j++
       }
     }
-    dels.length = j
+    if (dels[j - 1].len === 0) {
+      dels.length = j - 1
+    } else {
+      dels.length = j
+    }
+    if (dels.length === 0) {
+      ds.clients.delete(client)
+    }
   })
 }
 
@@ -160,6 +174,63 @@ export const mergeDeleteSets = dss => {
   return merged
 }
 
+/**
+ * Remove all ranges from `exclude` from `ds`. The result will contain all ranges from `ds` that are not
+ * in `exclude`.
+ *
+ * @param {DeleteSet} ds
+ * @param {DeleteSet} exclude
+ * @return {DeleteSet}
+ */
+export const diffDeleteSet = (ds, exclude) => {
+  const res = new DeleteSet()
+  ds.clients.forEach((ranges, client) => {
+    /**
+     * @type {Array<DeleteItem>}
+     */
+    const resRanges = []
+    const excludedRanges = exclude.clients.get(client) ?? []
+    let i = 0, j = 0
+    let currRange = ranges[0]
+    while (i < ranges.length && j < excludedRanges.length) {
+      const e = excludedRanges[j]
+      if (currRange.clock + currRange.len <= e.clock) { // no overlapping, use next range item
+        if (currRange.len > 0) resRanges.push(currRange)
+        currRange = ranges[++i]
+      } else if (e.clock + e.len <= currRange.clock) { // no overlapping, use next excluded item
+        j++
+      } else if (e.clock <= currRange.clock) { // exclude laps into range (we already know that the ranges somehow collide)
+        const newClock = e.clock + e.len
+        const newLen = currRange.clock + currRange.len - newClock
+        if (newLen > 0) {
+          currRange = new DeleteItem(newClock, newLen)
+          j++
+        } else {
+          // this item is completely overwritten. len=0. We can jump to the next range
+          currRange = ranges[++i]
+        }
+      } else { // currRange.clock < e.clock -- range laps into exclude => adjust len
+        // beginning can't be empty, add it to the result
+        const nextLen = e.clock - currRange.clock
+        resRanges.push(new DeleteItem(currRange.clock, nextLen))
+        // retain the remaining length after exclude in currRange
+        currRange = new DeleteItem(currRange.clock + e.len + nextLen, math.max(currRange.len - e.len - nextLen, 0))
+        if (currRange.len === 0) currRange = ranges[++i]
+        j++
+      }
+    }
+    if (currRange != null) {
+      resRanges.push(currRange)
+    }
+    i++
+    while (i < ranges.length) {
+      resRanges.push(ranges[i++])
+    }
+    if (resRanges.length > 0) res.clients.set(client, resRanges)
+  })
+  return res
+}
+
 /**
  * @param {DeleteSet} ds
  * @param {number} client

src/utils/Delta.js

@@ -1,5 +1,4 @@
 import * as object from 'lib0/object'
-import * as array from 'lib0/array'
 
 /**
  * @typedef {InsertOp|RetainOp|DeleteOp} DeltaOp
@@ -29,11 +28,11 @@ export class InsertOp {
     this.attribution = attribution
   }
   toJSON () {
-    return object.assign({ insert: this.insert }, this.attributes ? { attributes: this.attributes } : {}, this.attribution ? { attribution: this.attribution } : {})
+    return object.assign({ insert: this.insert }, this.attributes ? { attributes: this.attributes } : ({}), this.attribution ? { attribution: this.attribution } : ({}))
   }
 }
 
-class DeleteOp {
+export class DeleteOp {
   /**
    * @param {number} len
    */
@@ -45,7 +44,7 @@ class DeleteOp {
   }
 }
 
-class RetainOp {
+export class RetainOp {
   /**
    * @param {number} retain
    * @param {FormattingAttributes|null} attributes
@@ -133,7 +132,7 @@ export class DeltaBuilder extends Delta {
     if (attributes === null && attribution === null && this._lastOp instanceof InsertOp) {
       this._lastOp.insert += insert
     } else {
-      this.ops.push(this._lastOp = new InsertOp(insert, mergeAttrs(this.useAttributes, attributes), mergeAttrs(this._useAttribution, attribution)))
+      this.ops.push(this._lastOp = new InsertOp(insert, mergeAttrs(this._useAttributes, attributes), mergeAttrs(this._useAttribution, attribution)))
     }
     return this
   }
@@ -148,7 +147,7 @@ export class DeltaBuilder extends Delta {
     if (attributes === null && attribution === null && this._lastOp instanceof RetainOp) {
       this._lastOp.retain += retain
     } else {
-      this.ops.push(this._lastOp = new RetainOp(retain, mergeAttrs(this.useAttributes, attributes), mergeAttrs(this._useAttribution, attribution)))
+      this.ops.push(this._lastOp = new RetainOp(retain, mergeAttrs(this._useAttributes, attributes), mergeAttrs(this._useAttribution, attribution)))
     }
     return this
   }

tests/deleteset.tests.js

@@ -0,0 +1,157 @@
+import * as t from 'lib0/testing'
+import * as d from '../src/utils/DeleteSet.js'
+
+/**
+ * @param {Array<[number, number, number]>} ops
+ */
+const simpleConstructDs = ops => {
+  const ds = new d.DeleteSet()
+  ops.forEach(op => {
+    d.addToDeleteSet(ds, op[0], op[1], op[2])
+  })
+  d.sortAndMergeDeleteSet(ds)
+  return ds
+}
+
+/**
+ * @param {d.DeleteSet} ds1
+ * @param {d.DeleteSet} ds2
+ */
+const compareDs = (ds1, ds2) => {
+  t.assert(ds1.clients.size === ds2.clients.size)
+  ds1.clients.forEach((ranges1, clientid) => {
+    const ranges2 = ds2.clients.get(clientid) ?? []
+    t.assert(ranges1.length === ranges2?.length)
+    for (let i = 0; i < ranges1.length; i++) {
+      const d1 = ranges1[i]
+      const d2 = ranges2[i]
+      t.assert(d1.len === d2.len && d1.clock == d2.clock)
+    }
+  })
+}
+
+/**
+ * @param {t.TestCase} _tc
+ */
+export const testDeletesetMerge = _tc => {
+  t.group('filter out empty items (1))', () => {
+    compareDs(
+      simpleConstructDs([[0, 1, 0]]),
+      simpleConstructDs([])
+    )
+  })
+  t.group('filter out empty items (2))', () => {
+    compareDs(
+      simpleConstructDs([[0, 1, 0], [0, 2, 0]]),
+      simpleConstructDs([])
+    )
+  })
+  t.group('filter out empty items (3 - end))', () => {
+    compareDs(
+      simpleConstructDs([[0, 1, 1], [0, 2, 0]]),
+      simpleConstructDs([[0, 1, 1]])
+    )
+  })
+  t.group('filter out empty items (4 - middle))', () => {
+    compareDs(
+      simpleConstructDs([[0, 1, 1], [0, 2, 0], [0, 3, 1]]),
+      simpleConstructDs([[0, 1, 1], [0, 3, 1]])
+    )
+  })
+  t.group('filter out empty items (5 - beginning))', () => {
+    compareDs(
+      simpleConstructDs([[0, 1, 0], [0, 2, 1], [0, 3, 1]]),
+      simpleConstructDs([[0, 2, 1], [0, 3, 1]])
+    )
+  })
+  t.group('merge of overlapping deletes', () => {
+    compareDs(
+      simpleConstructDs([[0, 1, 2], [0, 0, 2]]),
+      simpleConstructDs([[0, 0, 3]])
+    )
+  })
+  t.group('construct without hole', () => {
+    compareDs(
+      simpleConstructDs([[0, 1, 2], [0, 3, 1]]),
+      simpleConstructDs([[0, 1, 3]])
+    )
+  })
+}
+
+/**
+ * @param {t.TestCase} _tc
+ */
+export const testDeletesetDiffing = _tc => {
+  t.group('simple case (1))', () => {
+    compareDs(
+      d.diffDeleteSet(
+        simpleConstructDs([[0, 1, 1], [0, 3, 1]]),
+        simpleConstructDs([[0, 3, 1]])
+      ),
+      simpleConstructDs([[0, 1, 1]])
+    )
+  })
+  t.group('subset left', () => {
+    compareDs(
+      d.diffDeleteSet(
+        simpleConstructDs([[0, 1, 3]]),
+        simpleConstructDs([[0, 1, 1]])
+      ),
+      simpleConstructDs([[0, 2, 2]])
+    )
+  })
+  t.group('subset right', () => {
+    compareDs(
+      d.diffDeleteSet(
+        simpleConstructDs([[0, 1, 3]]),
+        simpleConstructDs([[0, 3, 1]])
+      ),
+      simpleConstructDs([[0, 1, 2]])
+    )
+  })
+  t.group('subset middle', () => {
+    compareDs(
+      d.diffDeleteSet(
+        simpleConstructDs([[0, 1, 3]]),
+        simpleConstructDs([[0, 2, 1]])
+      ),
+      simpleConstructDs([[0, 1, 1], [0, 3, 1]])
+    )
+  })
+  t.group('overlapping left', () => {
+    compareDs(
+      d.diffDeleteSet(
+        simpleConstructDs([[0, 1, 3]]),
+        simpleConstructDs([[0, 0, 2]])
+      ),
+      simpleConstructDs([[0, 2, 2]])
+    )
+  })
+  t.group('overlapping right', () => {
+    compareDs(
+      d.diffDeleteSet(
+        simpleConstructDs([[0, 1, 3]]),
+        simpleConstructDs([[0, 3, 5]])
+      ),
+      simpleConstructDs([[0, 1, 2]])
+    )
+  })
+  t.group('overlapping completely', () => {
+    compareDs(
+      d.diffDeleteSet(
+        simpleConstructDs([[0, 1, 3]]),
+        simpleConstructDs([[0, 0, 5]])
+      ),
+      simpleConstructDs([])
+    )
+  })
+  t.group('overlapping into new range', () => {
+    compareDs(
+      d.diffDeleteSet(
+        simpleConstructDs([[0, 1, 3], [0, 5, 2]]),
+        simpleConstructDs([[0, 0, 6]])
+      ),
+      simpleConstructDs([[0, 6, 1]])
+    )
+  })
+}

tests/delta.tests.js

@@ -6,6 +6,6 @@ import * as delta from '../src/utils/Delta.js'
  */
 export const testDelta = _tc => {
   const d = delta.create().insert('hello').insert(' ').useAttributes({ bold: true }).insert('world').useAttribution({ creator: 'tester' }).insert('!').done()
-  t.compare(d.toJSON().ops, [{ insert: 'hello' }, { insert: ' world', attributes: { bold: true } }, { insert: '!', attributes: { bold: true }, attribution: { creator: 'tester' } }])
+  t.compare(d.toJSON().ops, [{ insert: 'hello ' }, { insert: 'world', attributes: { bold: true } }, { insert: '!', attributes: { bold: true }, attribution: { creator: 'tester' } }])
 }
 

tests/index.js

@@ -11,7 +11,8 @@ import * as doc from './doc.tests.js'
 import * as snapshot from './snapshot.tests.js'
 import * as updates from './updates.tests.js'
 import * as relativePositions from './relativePositions.tests.js'
-// import * as delta from './delta.tests.js'
+import * as delta from './delta.tests.js'
+import * as deleteset from './deleteset.tests.js'
 
 import { runTests } from 'lib0/testing'
 import { isBrowser, isNode } from 'lib0/environment'
@@ -21,11 +22,8 @@ if (isBrowser) {
   log.createVConsole(document.body)
 }
 
-/**
- * @type {any}
- */
 const tests = {
-  doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, updates, relativePositions, delta
+  doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, updates, relativePositions, delta, deleteset
 }
 
 const run = async () => {