From da8aad3615cb8abe360c4a2cb64775ac0b5a955c Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 27 Mar 2025 16:24:11 +0100 Subject: [PATCH] implement support for diffing deletesets --- src/internals.js | 1 + src/types/YText.js | 95 ++++++++++++++++++- src/utils/AttributionManager.js | 8 ++ src/utils/DeleteSet.js | 77 +++++++++++++++- src/utils/Delta.js | 11 +-- tests/deleteset.tests.js | 157 ++++++++++++++++++++++++++++++++ tests/delta.tests.js | 2 +- tests/index.js | 8 +- 8 files changed, 343 insertions(+), 16 deletions(-) create mode 100644 src/utils/AttributionManager.js create mode 100644 tests/deleteset.tests.js diff --git a/src/internals.js b/src/internals.js index cb2fcac8..b9fe33bf 100644 --- a/src/internals.js +++ b/src/internals.js @@ -40,3 +40,4 @@ export * from './structs/ContentString.js' export * from './structs/ContentType.js' export * from './structs/Item.js' export * from './structs/Skip.js' +export * from './utils/AttributionManager.js' diff --git a/src/types/YText.js b/src/types/YText.js index 75bf5dd6..56aec77f 100644 --- a/src/types/YText.js +++ b/src/types/YText.js @@ -27,9 +27,12 @@ import { updateMarkerChanges, ContentType, warnPrematureAccess, - ArraySearchMarker, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ID, Doc, Item, Snapshot, Transaction // eslint-disable-line + ArraySearchMarker, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ID, Doc, Item, Snapshot, Transaction, AttributionManager, // eslint-disable-line + snapshot } from '../internals.js' +import * as delta from '../utils/Delta.js' + import * as object from 'lib0/object' import * as map from 'lib0/map' import * as error from 'lib0/error' @@ -996,6 +999,96 @@ export class YText extends AbstractType { } } + /** + * Render the difference to another ydoc (which can be empty) and highlight the differences with + * attributions. + * + * Note that deleted content that was not deleted in prevYdoc is rendered as an insertion with the + * attribution `{ isDeleted: true, .. }`. + * + * @param {AttributionManager} [attributionManager] + * @param {Doc} [prevYdoc] + * @return {import('../utils/Delta.js').Delta} The Delta representation of this type. + * + * @public + */ + getContent (attributionManager, prevYdoc) { + this.doc ?? warnPrematureAccess() + const prevSnapshot = prevYdoc ? snapshot(prevYdoc) : null + const d = delta.create() + /** + * @type {{ [key: string]: any }} + */ + const currentAttributes = {} + const doc = /** @type {Doc} */ (this.doc) + const computeContent = () => { + let n = this._start + while (n !== null) { + switch (n.content.constructor) { + case ContentString: { + const cur = currentAttributes.get('ychange') + if (snapshot !== undefined && !isVisible(n, snapshot)) { + if (cur === undefined || cur.user !== n.id.client || cur.type !== 'removed') { + packStr() + currentAttributes.set('ychange', computeYChange ? computeYChange('removed', n.id) : { type: 'removed' }) + } + } else if (prevSnapshot !== undefined && !isVisible(n, prevSnapshot)) { + if (cur === undefined || cur.user !== n.id.client || cur.type !== 'added') { + packStr() + currentAttributes.set('ychange', computeYChange ? computeYChange('added', n.id) : { type: 'added' }) + } + } else if (cur !== undefined) { + packStr() + currentAttributes.delete('ychange') + } + str += /** @type {ContentString} */ (n.content).str + break + } + case ContentType: + case ContentEmbed: { + packStr() + /** + * @type {Object} + */ + const op = { + insert: n.content.getContent()[0] + } + if (currentAttributes.size > 0) { + const attrs = /** @type {Object} */ ({}) + op.attributes = attrs + currentAttributes.forEach((value, key) => { + attrs[key] = value + }) + } + ops.push(op) + break + } + case ContentFormat: + if (isVisible(n, snapshot)) { + packStr() + updateCurrentAttributes(currentAttributes, /** @type {ContentFormat} */ (n.content)) + } + break + } + n = n.right + } + } + if (prevSnapshot) { + // snapshots are merged again after the transaction, so we need to keep the + // transaction alive until we are done + transact(doc, transaction => { + if (prevSnapshot) { + splitSnapshotAffectedStructs(transaction, prevSnapshot) + } + computeContent() + }, 'cleanup') + } else { + computeContent() + } + return d.done() + } + + /** * Returns the Delta representation of this YText type. * diff --git a/src/utils/AttributionManager.js b/src/utils/AttributionManager.js new file mode 100644 index 00000000..205826c4 --- /dev/null +++ b/src/utils/AttributionManager.js @@ -0,0 +1,8 @@ + +export class AttributionManager { + /** + * + */ + constructor () { + } +} diff --git a/src/utils/DeleteSet.js b/src/utils/DeleteSet.js index fe07b7ce..d91fefa5 100644 --- a/src/utils/DeleteSet.js +++ b/src/utils/DeleteSet.js @@ -20,10 +20,12 @@ export class DeleteItem { */ constructor (clock, len) { /** + * @readonly * @type {number} */ this.clock = clock /** + * @readonly * @type {number} */ this.len = len @@ -111,7 +113,7 @@ export const isDeleted = (ds, id) => { * @function */ export const sortAndMergeDeleteSet = ds => { - ds.clients.forEach(dels => { + ds.clients.forEach((dels, client) => { dels.sort((a, b) => a.clock - b.clock) // merge items without filtering or splicing the array // i is the current pointer @@ -122,7 +124,12 @@ export const sortAndMergeDeleteSet = ds => { const left = dels[j - 1] const right = dels[i] if (left.clock + left.len >= right.clock) { - left.len = math.max(left.len, right.clock + right.len - left.clock) + const r = right.clock + right.len - left.clock + if (left.len < r) { + dels[j - 1] = new DeleteItem(left.clock, r) + } + } else if (left.len === 0) { + dels[j - 1] = right } else { if (j < i) { dels[j] = right @@ -130,7 +137,14 @@ export const sortAndMergeDeleteSet = ds => { j++ } } - dels.length = j + if (dels[j - 1].len === 0) { + dels.length = j - 1 + } else { + dels.length = j + } + if (dels.length === 0) { + ds.clients.delete(client) + } }) } @@ -160,6 +174,63 @@ export const mergeDeleteSets = dss => { return merged } +/** + * Remove all ranges from `exclude` from `ds`. The result will contain all ranges from `ds` that are not + * in `exclude`. + * + * @param {DeleteSet} ds + * @param {DeleteSet} exclude + * @return {DeleteSet} + */ +export const diffDeleteSet = (ds, exclude) => { + const res = new DeleteSet() + ds.clients.forEach((ranges, client) => { + /** + * @type {Array} + */ + const resRanges = [] + const excludedRanges = exclude.clients.get(client) ?? [] + let i = 0, j = 0 + let currRange = ranges[0] + while (i < ranges.length && j < excludedRanges.length) { + const e = excludedRanges[j] + if (currRange.clock + currRange.len <= e.clock) { // no overlapping, use next range item + if (currRange.len > 0) resRanges.push(currRange) + currRange = ranges[++i] + } else if (e.clock + e.len <= currRange.clock) { // no overlapping, use next excluded item + j++ + } else if (e.clock <= currRange.clock) { // exclude laps into range (we already know that the ranges somehow collide) + const newClock = e.clock + e.len + const newLen = currRange.clock + currRange.len - newClock + if (newLen > 0) { + currRange = new DeleteItem(newClock, newLen) + j++ + } else { + // this item is completely overwritten. len=0. We can jump to the next range + currRange = ranges[++i] + } + } else { // currRange.clock < e.clock -- range laps into exclude => adjust len + // beginning can't be empty, add it to the result + const nextLen = e.clock - currRange.clock + resRanges.push(new DeleteItem(currRange.clock, nextLen)) + // retain the remaining length after exclude in currRange + currRange = new DeleteItem(currRange.clock + e.len + nextLen, math.max(currRange.len - e.len - nextLen, 0)) + if (currRange.len === 0) currRange = ranges[++i] + j++ + } + } + if (currRange != null) { + resRanges.push(currRange) + } + i++ + while (i < ranges.length) { + resRanges.push(ranges[i++]) + } + if (resRanges.length > 0) res.clients.set(client, resRanges) + }) + return res +} + /** * @param {DeleteSet} ds * @param {number} client diff --git a/src/utils/Delta.js b/src/utils/Delta.js index 81306e51..e306ab81 100644 --- a/src/utils/Delta.js +++ b/src/utils/Delta.js @@ -1,5 +1,4 @@ import * as object from 'lib0/object' -import * as array from 'lib0/array' /** * @typedef {InsertOp|RetainOp|DeleteOp} DeltaOp @@ -29,11 +28,11 @@ export class InsertOp { this.attribution = attribution } toJSON () { - return object.assign({ insert: this.insert }, this.attributes ? { attributes: this.attributes } : {}, this.attribution ? { attribution: this.attribution } : {}) + return object.assign({ insert: this.insert }, this.attributes ? { attributes: this.attributes } : ({}), this.attribution ? { attribution: this.attribution } : ({})) } } -class DeleteOp { +export class DeleteOp { /** * @param {number} len */ @@ -45,7 +44,7 @@ class DeleteOp { } } -class RetainOp { +export class RetainOp { /** * @param {number} retain * @param {FormattingAttributes|null} attributes @@ -133,7 +132,7 @@ export class DeltaBuilder extends Delta { if (attributes === null && attribution === null && this._lastOp instanceof InsertOp) { this._lastOp.insert += insert } else { - this.ops.push(this._lastOp = new InsertOp(insert, mergeAttrs(this.useAttributes, attributes), mergeAttrs(this._useAttribution, attribution))) + this.ops.push(this._lastOp = new InsertOp(insert, mergeAttrs(this._useAttributes, attributes), mergeAttrs(this._useAttribution, attribution))) } return this } @@ -148,7 +147,7 @@ export class DeltaBuilder extends Delta { if (attributes === null && attribution === null && this._lastOp instanceof RetainOp) { this._lastOp.retain += retain } else { - this.ops.push(this._lastOp = new RetainOp(retain, mergeAttrs(this.useAttributes, attributes), mergeAttrs(this._useAttribution, attribution))) + this.ops.push(this._lastOp = new RetainOp(retain, mergeAttrs(this._useAttributes, attributes), mergeAttrs(this._useAttribution, attribution))) } return this } diff --git a/tests/deleteset.tests.js b/tests/deleteset.tests.js new file mode 100644 index 00000000..eee764ae --- /dev/null +++ b/tests/deleteset.tests.js @@ -0,0 +1,157 @@ +import * as t from 'lib0/testing' +import * as d from '../src/utils/DeleteSet.js' + +/** + * @param {Array<[number, number, number]>} ops + */ +const simpleConstructDs = ops => { + const ds = new d.DeleteSet() + ops.forEach(op => { + d.addToDeleteSet(ds, op[0], op[1], op[2]) + }) + d.sortAndMergeDeleteSet(ds) + return ds +} + +/** + * @param {d.DeleteSet} ds1 + * @param {d.DeleteSet} ds2 + */ +const compareDs = (ds1, ds2) => { + t.assert(ds1.clients.size === ds2.clients.size) + ds1.clients.forEach((ranges1, clientid) => { + const ranges2 = ds2.clients.get(clientid) ?? [] + t.assert(ranges1.length === ranges2?.length) + for (let i = 0; i < ranges1.length; i++) { + const d1 = ranges1[i] + const d2 = ranges2[i] + t.assert(d1.len === d2.len && d1.clock == d2.clock) + } + }) +} + +/** + * @param {t.TestCase} _tc + */ +export const testDeletesetMerge = _tc => { + t.group('filter out empty items (1))', () => { + compareDs( + simpleConstructDs([[0, 1, 0]]), + simpleConstructDs([]) + ) + }) + t.group('filter out empty items (2))', () => { + compareDs( + simpleConstructDs([[0, 1, 0], [0, 2, 0]]), + simpleConstructDs([]) + ) + }) + t.group('filter out empty items (3 - end))', () => { + compareDs( + simpleConstructDs([[0, 1, 1], [0, 2, 0]]), + simpleConstructDs([[0, 1, 1]]) + ) + }) + t.group('filter out empty items (4 - middle))', () => { + compareDs( + simpleConstructDs([[0, 1, 1], [0, 2, 0], [0, 3, 1]]), + simpleConstructDs([[0, 1, 1], [0, 3, 1]]) + ) + }) + t.group('filter out empty items (5 - beginning))', () => { + compareDs( + simpleConstructDs([[0, 1, 0], [0, 2, 1], [0, 3, 1]]), + simpleConstructDs([[0, 2, 1], [0, 3, 1]]) + ) + }) + t.group('merge of overlapping deletes', () => { + compareDs( + simpleConstructDs([[0, 1, 2], [0, 0, 2]]), + simpleConstructDs([[0, 0, 3]]) + ) + }) + t.group('construct without hole', () => { + compareDs( + simpleConstructDs([[0, 1, 2], [0, 3, 1]]), + simpleConstructDs([[0, 1, 3]]) + ) + }) +} + +/** + * @param {t.TestCase} _tc + */ +export const testDeletesetDiffing = _tc => { + t.group('simple case (1))', () => { + compareDs( + d.diffDeleteSet( + simpleConstructDs([[0, 1, 1], [0, 3, 1]]), + simpleConstructDs([[0, 3, 1]]) + ), + simpleConstructDs([[0, 1, 1]]) + ) + }) + t.group('subset left', () => { + compareDs( + d.diffDeleteSet( + simpleConstructDs([[0, 1, 3]]), + simpleConstructDs([[0, 1, 1]]) + ), + simpleConstructDs([[0, 2, 2]]) + ) + }) + t.group('subset right', () => { + compareDs( + d.diffDeleteSet( + simpleConstructDs([[0, 1, 3]]), + simpleConstructDs([[0, 3, 1]]) + ), + simpleConstructDs([[0, 1, 2]]) + ) + }) + t.group('subset middle', () => { + compareDs( + d.diffDeleteSet( + simpleConstructDs([[0, 1, 3]]), + simpleConstructDs([[0, 2, 1]]) + ), + simpleConstructDs([[0, 1, 1], [0, 3, 1]]) + ) + }) + t.group('overlapping left', () => { + compareDs( + d.diffDeleteSet( + simpleConstructDs([[0, 1, 3]]), + simpleConstructDs([[0, 0, 2]]) + ), + simpleConstructDs([[0, 2, 2]]) + ) + }) + t.group('overlapping right', () => { + compareDs( + d.diffDeleteSet( + simpleConstructDs([[0, 1, 3]]), + simpleConstructDs([[0, 3, 5]]) + ), + simpleConstructDs([[0, 1, 2]]) + ) + }) + t.group('overlapping completely', () => { + compareDs( + d.diffDeleteSet( + simpleConstructDs([[0, 1, 3]]), + simpleConstructDs([[0, 0, 5]]) + ), + simpleConstructDs([]) + ) + }) + t.group('overlapping into new range', () => { + compareDs( + d.diffDeleteSet( + simpleConstructDs([[0, 1, 3], [0, 5, 2]]), + simpleConstructDs([[0, 0, 6]]) + ), + simpleConstructDs([[0, 6, 1]]) + ) + }) +} diff --git a/tests/delta.tests.js b/tests/delta.tests.js index e3eaf8dd..01cd08d2 100644 --- a/tests/delta.tests.js +++ b/tests/delta.tests.js @@ -6,6 +6,6 @@ import * as delta from '../src/utils/Delta.js' */ export const testDelta = _tc => { const d = delta.create().insert('hello').insert(' ').useAttributes({ bold: true }).insert('world').useAttribution({ creator: 'tester' }).insert('!').done() - t.compare(d.toJSON().ops, [{ insert: 'hello' }, { insert: ' world', attributes: { bold: true } }, { insert: '!', attributes: { bold: true }, attribution: { creator: 'tester' } }]) + t.compare(d.toJSON().ops, [{ insert: 'hello ' }, { insert: 'world', attributes: { bold: true } }, { insert: '!', attributes: { bold: true }, attribution: { creator: 'tester' } }]) } diff --git a/tests/index.js b/tests/index.js index 2799d745..6e7dc6de 100644 --- a/tests/index.js +++ b/tests/index.js @@ -11,7 +11,8 @@ import * as doc from './doc.tests.js' import * as snapshot from './snapshot.tests.js' import * as updates from './updates.tests.js' import * as relativePositions from './relativePositions.tests.js' -// import * as delta from './delta.tests.js' +import * as delta from './delta.tests.js' +import * as deleteset from './deleteset.tests.js' import { runTests } from 'lib0/testing' import { isBrowser, isNode } from 'lib0/environment' @@ -21,11 +22,8 @@ if (isBrowser) { log.createVConsole(document.body) } -/** - * @type {any} - */ const tests = { - doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, updates, relativePositions, delta + doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, updates, relativePositions, delta, deleteset } const run = async () => {