From a6ae65d32c9f01c2761343ee17b8ecc29497ffa0 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Sat, 12 Apr 2025 14:44:37 +0200 Subject: [PATCH] Work on AttributionManager --- package-lock.json | 8 +- package.json | 2 +- src/index.js | 4 +- src/utils/AttributionManager.js | 254 +++++++++++++++++++++++++++++- src/utils/IdSet.js | 8 +- tests/AttributionManager.tests.js | 131 +++++++++++++++ tests/IdSet.tests.js | 31 +++- tests/index.js | 3 +- tests/testHelper.js | 22 +++ 9 files changed, 445 insertions(+), 18 deletions(-) create mode 100644 tests/AttributionManager.tests.js diff --git a/package-lock.json b/package-lock.json index 1d07c178..1f9e9fb2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "13.6.27", "license": "MIT", "dependencies": { - "lib0": "^0.2.101", + "lib0": "^0.2.103", "y-protocols": "^1.0.5" }, "devDependencies": { @@ -2773,9 +2773,9 @@ } }, "node_modules/lib0": { - "version": "0.2.101", - "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.101.tgz", - "integrity": "sha512-LljA6+Ehf0Z7YnxhgSAvspzWALjW4wlWdN/W4iGiqYc1KvXQgOVXWI0xwlwqozIL5WRdKeUW2gq0DLhFsY+Xlw==", + "version": "0.2.103", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.103.tgz", + "integrity": "sha512-1zT9KqSh54uEQZksnm8ONj0bclW3PrisT59nhgY2eOV4PaCZ5Pt9MV4y4KGkNIE/5vp6yNzpYX/+5/aGvfZS5Q==", "license": "MIT", "dependencies": { "isomorphic.js": "^0.2.4" diff --git a/package.json b/package.json index 448c39be..fd761b8f 100644 --- a/package.json +++ b/package.json @@ -86,7 +86,7 @@ }, "homepage": "https://docs.yjs.dev", "dependencies": { - "lib0": "^0.2.101", + "lib0": "^0.2.103", "y-protocols": "^1.0.5" }, "devDependencies": { diff --git a/src/index.js b/src/index.js index eba8727b..5adfac56 100644 --- a/src/index.js +++ b/src/index.js @@ -101,7 +101,9 @@ export { // idset IdSet, equalIdSets, - createDeleteSetFromStructStore + createDeleteSetFromStructStore, + AttributionManager, + createAttributionManager } from './internals.js' const glo = /** @type {any} */ (typeof globalThis !== 'undefined' diff --git a/src/utils/AttributionManager.js b/src/utils/AttributionManager.js index f174e836..b06023e8 100644 --- a/src/utils/AttributionManager.js +++ b/src/utils/AttributionManager.js @@ -1,8 +1,256 @@ +import { + findIndexInIdRanges, + ID // @eslint-disable-line +} from '../internals.js' -export class AttributionManager { +import * as array from 'lib0/array' + +/** + * @template T + * @param {Array} attrs + * @param {T} attr + * + */ +const amAttrsHas = (attrs, attr) => attrs.find(a => a === attr) + +/** + * @template T + * @param {Array} a + * @param {Array} b + */ +export const amAttrsEqual = (a, b) => a.length === b.length && a.every(v => amAttrsHas(b, v)) + +/** + * @template T + * @param {Array} a + * @param {Array} b + */ +const amAttrRangeJoin = (a, b) => a.concat(b.filter(attr => !amAttrsHas(a, attr))) + +/** + * @template Attrs + */ +export class AttrRange { /** - * + * @param {number} clock + * @param {number} len + * @param {Array} attrs */ - constructor () { + constructor (clock, len, attrs) { + /** + * @readonly + */ + this.clock = clock + /** + * @readonly + */ + this.len = len + /** + * @readonly + */ + this.attrs = attrs } } + +/** + * @template Attrs + */ +class AttrRanges { + /** + * @param {Array>} ids + */ + constructor (ids) { + this.sorted = false + /** + * @private + */ + this._ids = ids + } + + /** + * @param {number} clock + * @param {number} length + * @param {Array} attrs + */ + add (clock, length, attrs) { + this.sorted = false + this._ids.push(new AttrRange(clock, length, attrs)) + } + + /** + * Return the list of id ranges, sorted and merged. + */ + getIds () { + const ids = this._ids + if (!this.sorted) { + this.sorted = true + ids.sort((a, b) => a.clock - b.clock) + /** + * algorithm thoughts: + * - sort (by clock AND by length), bigger length is to the right (or not, we can't make + * assumptions abouth length after long length has been split) + * -- maybe better: sort by clock+length. Then split items from right to left. This way, items are always + * in the right order. But I also need to swap if left items is smaller after split + * --- thought: there is no way to go around swapping. Unless, for each item from left to + * right, when I have to split because one of the look-ahead items is overlapping, i split + * it and merge the attributes into the following ones (that I also need to split). Best is + * probably left to right with lookahead. + * - left to right, split overlapping items so that we can make the assumption that either an + * item is overlapping with the next 1-on-1 or it is not overlapping at all (when splitting, + * we can already incorporate the attributes) + * -- better: for each item, go left to right and add own attributes to overlapping items. + * Split them if necessary. After split, i must insert the retainer at a valid position. + * - merge items if neighbor has same attributes + */ + for (let i = 0; i < ids.length - 1; i++) { + const range = ids[i] + const nextRange = ids[i+1] + // find out how to split range. it must match with next range. + // 1) we have space. Split if necessary. + // 2) concat attributes in range to the next range. Split range and splice the remainder at + // the correct position. + if (range.clock < nextRange.clock) { // might need to split range + if (range.clock + range.len > nextRange.clock) { + // is overlapping + const diff = nextRange.clock - range.clock + ids[i] = new AttrRange(range.clock, diff, range.attrs) + ids.splice(i + 1, 0, new AttrRange(nextRange.clock, range.len - diff, range.attrs)) + } + continue + } + // now we know that range.clock === nextRange.clock + // merge range with nextRange + const largerRange = range.len > nextRange.len ? range : nextRange + const smallerLen = range.len < nextRange.len ? range.len : nextRange.len + ids[i] = new AttrRange(range.clock, smallerLen, amAttrRangeJoin(range.attrs, nextRange.attrs)) + if (range.len === nextRange.len) { + ids.splice(i + 1, 1) + i-- + } else { + ids[i + 1] = new AttrRange(range.clock + smallerLen, largerRange.len - smallerLen, largerRange.attrs) + array.bubblesortItem(ids, i + 1, (a, b) => a.clock - b.clock) + } + } + + // merge items without filtering or splicing the array. + // i is the current pointer + // j refers to the current insert position for the pointed item + // try to merge dels[i] into dels[j-1] or set dels[j]=dels[i] + let i, j + for (i = 1, j = 1; i < ids.length; i++) { + const left = ids[j - 1] + const right = ids[i] + if (left.clock + left.len === right.clock && amAttrsEqual(left.attrs, right.attrs)) { + ids[j - 1] = new AttrRange(left.clock, left.len + right.len, left.attrs) + } else if (right.len !== 0) { + if (j < i) { + ids[j] = right + } + j++ + } + } + ids.length = ids[j - 1].len === 0 ? j - 1 : j + } + return ids + } +} + +/** + * @template T + * @param {Array>} ams + * @return {AttributionManager} A fresh IdSet + */ +export const mergeAttributionManagers = ams => { + const merged = createAttributionManager() + for (let amsI = 0; amsI < ams.length; amsI++) { + ams[amsI].clients.forEach((rangesLeft, client) => { + if (!merged.clients.has(client)) { + // Write all missing keys from current set and all following. + // If merged already contains `client` current ds has already been added. + const ids = rangesLeft.getIds().slice() + for (let i = amsI + 1; i < ams.length; i++) { + const nextIds = ams[i].clients.get(client) + if (nextIds) { + array.appendTo(ids, nextIds.getIds()) + } + } + merged.clients.set(client, new AttrRanges(ids)) + } + }) + } + return merged +} + +/** + * @template Attrs + */ +export class AttributionManager { + constructor () { + /** + * @type {Map>} + */ + this.clients = new Map() + } + + /** + * @param {ID} id + * @return {boolean} + */ + has (id) { + const dr = this.clients.get(id.client) + if (dr) { + return findIndexInIdRanges(dr.getIds(), id.clock) !== null + } + return false + } + + /** + * @param {ID} id + * @param {number} len + * @return {Array>?} + */ + slice (id, len) { + const dr = this.clients.get(id.client) + if (dr) { + /** + * @type {Array>} + */ + const ranges = dr.getIds() + let index = findIndexInIdRanges(ranges, id.clock) + if (index !== null) { + const res = [] + while (true) { + let r = ranges[index] + if (r.clock < id.clock) { + r = new AttrRange(id.clock, r.len - (id.clock - r.clock), r.attrs) + } + if (r.clock + r.len > id.clock + len) { + r = new AttrRange(r.clock, id.clock + len - r.clock, r.attrs) + } + if (r.len <= 0) break + res.push(r) + index++ + } + return res + } + } + return null + } + + /** + * @param {number} client + * @param {number} clock + * @param {number} len + * @param {Array} attrs + */ + add (client, clock, len, attrs) { + const ranges = this.clients.get(client) + if (ranges == null) { + this.clients.set(client, new AttrRanges([new AttrRange(clock, len, attrs)])) + } else { + ranges.add(clock, len, attrs) + } + } +} + +export const createAttributionManager = () => new AttributionManager() diff --git a/src/utils/IdSet.js b/src/utils/IdSet.js index 6135bf20..60638670 100644 --- a/src/utils/IdSet.js +++ b/src/utils/IdSet.js @@ -19,12 +19,10 @@ export class IdRange { */ constructor (clock, len) { /** - * @readonly * @type {number} */ this.clock = clock /** - * @readonly * @type {number} */ this.len = len @@ -87,11 +85,7 @@ class IdRanges { j++ } } - if (ids[j - 1].len === 0) { - ids.length = j - 1 - } else { - ids.length = j - } + ids.length = ids[j - 1].len === 0 ? j - 1 : j } return ids } diff --git a/tests/AttributionManager.tests.js b/tests/AttributionManager.tests.js new file mode 100644 index 00000000..1a9cce9c --- /dev/null +++ b/tests/AttributionManager.tests.js @@ -0,0 +1,131 @@ +import * as t from 'lib0/testing' +import * as am from '../src/utils/AttributionManager.js' +import * as prng from 'lib0/prng' +import * as math from 'lib0/math' +import { compareAttributionManagers, createAttributionManager, ID } from './testHelper.js' + +/** + * @template T + * @param {Array<[number, number, number, Array]>} ops + */ +const simpleConstructAttrs = ops => { + const attrs = createAttributionManager() + ops.forEach(op => { + attrs.add(op[0], op[1], op[2], op[3]) + }) + return attrs +} + +/** + * @template T + * @param {prng.PRNG} gen + * @param {number} clients + * @param {number} clockRange (max clock - exclusive - by each client) + * @param {Array} attrChoices (max clock - exclusive - by each client) + * @return {am.AttributionManager} + */ +const createRandomAttributionManager = (gen, clients, clockRange, attrChoices) => { + const maxOpLen = 5 + const numOfOps = math.ceil((clients * clockRange) / maxOpLen) + const attrMngr = createAttributionManager() + for (let i = 0; i < numOfOps; i++) { + const client = prng.uint32(gen, 0, clients - 1) + const clockStart = prng.uint32(gen, 0, clockRange) + const len = prng.uint32(gen, 0, clockRange - clockStart) + const attrs = [prng.oneOf(gen, attrChoices)] + if (prng.bool(gen)) { + attrs.push(prng.oneOf(gen, attrChoices)) + } + attrMngr.add(client, clockStart, len, attrs) + } + return attrMngr +} + +/** + * @param {t.TestCase} _tc + */ +export const testAmMerge = _tc => { + const attrs = [42] + t.group('filter out empty items (1))', () => { + compareAttributionManagers( + simpleConstructAttrs([[0, 1, 0, attrs]]), + simpleConstructAttrs([]) + ) + }) + t.group('filter out empty items (2))', () => { + compareAttributionManagers( + simpleConstructAttrs([[0, 1, 0, attrs], [0, 2, 0, attrs]]), + simpleConstructAttrs([]) + ) + }) + t.group('filter out empty items (3 - end))', () => { + compareAttributionManagers( + simpleConstructAttrs([[0, 1, 1, attrs], [0, 2, 0, attrs]]), + simpleConstructAttrs([[0, 1, 1, attrs]]) + ) + }) + t.group('filter out empty items (4 - middle))', () => { + compareAttributionManagers( + simpleConstructAttrs([[0, 1, 1, attrs], [0, 2, 0, attrs], [0, 3, 1, attrs]]), + simpleConstructAttrs([[0, 1, 1, attrs], [0, 3, 1, attrs]]) + ) + }) + t.group('filter out empty items (5 - beginning))', () => { + compareAttributionManagers( + simpleConstructAttrs([[0, 1, 0, attrs], [0, 2, 1, attrs], [0, 3, 1, attrs]]), + simpleConstructAttrs([[0, 2, 1, attrs], [0, 3, 1, attrs]]) + ) + }) + t.group('merge of overlapping id ranges', () => { + compareAttributionManagers( + simpleConstructAttrs([[0, 1, 2, attrs], [0, 0, 2, attrs]]), + simpleConstructAttrs([[0, 0, 3, attrs]]) + ) + }) + t.group('construct without hole', () => { + compareAttributionManagers( + simpleConstructAttrs([[0, 1, 2, attrs], [0, 3, 1, attrs]]), + simpleConstructAttrs([[0, 1, 3, attrs]]) + ) + }) + t.group('no merge of overlapping id ranges with different attributes', () => { + compareAttributionManagers( + simpleConstructAttrs([[0, 1, 2, [1]], [0, 0, 2, [2]]]), + simpleConstructAttrs([[0, 0, 1, [2]], [0, 1, 1, [1, 2]], [0, 2, 1, [2]]]) + ) + }) +} + +/** + * @param {t.TestCase} tc + */ +export const testRepeatMergingMultipleAttrManagers = tc => { + const clients = 4 + const clockRange = 100 + /** + * @type {Array>} + */ + const sets = [] + for (let i = 0; i < 3; i++) { + sets.push(createRandomAttributionManager(tc.prng, clients, clockRange, [1, 2, 3])) + } + const merged = am.mergeAttributionManagers(sets) + const mergedReverse = am.mergeAttributionManagers(sets.reverse()) + compareAttributionManagers(merged, mergedReverse) + const composed = am.createAttributionManager() + for (let iclient = 0; iclient < clients; iclient++) { + for (let iclock = 0; iclock < clockRange + 42; iclock++) { + const mergedHas = merged.has(new ID(iclient, iclock)) + const oneHas = sets.some(ids => ids.has(new ID(iclient, iclock))) + t.assert(mergedHas === oneHas) + const mergedAttrs = merged.slice(new ID(iclient, iclock), 1) + if (mergedAttrs) { + mergedAttrs.forEach(a => { + composed.add(iclient, a.clock, a.len, a.attrs) + }) + } + } + } + compareAttributionManagers(merged, composed) +} + diff --git a/tests/IdSet.tests.js b/tests/IdSet.tests.js index c9ca5617..4601ad1f 100644 --- a/tests/IdSet.tests.js +++ b/tests/IdSet.tests.js @@ -2,7 +2,7 @@ import * as t from 'lib0/testing' import * as d from '../src/utils/IdSet.js' import * as prng from 'lib0/prng' import * as math from 'lib0/math' -import { compareIdSets } from './testHelper.js' +import { compareIdSets, ID } from './testHelper.js' /** * @param {Array<[number, number, number]>} ops @@ -175,3 +175,32 @@ export const testRepeatRandomDiffing = tc => { const e2 = d.diffIdSets(merged, ds2) compareIdSets(e1, e2) } + +/** + * @param {t.TestCase} tc + */ +export const testRepeatMergingMultipleIdsets = tc => { + const clients = 4 + const clockRange = 100 + /** + * @type {Array} + */ + const idss = [] + for (let i = 0; i < 3; i++) { + idss.push(createRandomDiffSet(tc.prng, clients, clockRange)) + } + const merged = d.mergeIdSets(idss) + const mergedReverse = d.mergeIdSets(idss.reverse()) + compareIdSets(merged, mergedReverse) + const composed = d.createIdSet() + for (let iclient = 0; iclient < clients; iclient++) { + for (let iclock = 0; iclock < clockRange + 42; iclock++) { + const mergedHas = merged.has(new ID(iclient, iclock)) + const oneHas = idss.some(ids => ids.has(new ID(iclient, iclock))) + t.assert(mergedHas === oneHas) + d.addToIdSet(composed, iclient, iclock, 1) + } + } + compareIdSets(merged, composed) +} + diff --git a/tests/index.js b/tests/index.js index 0103bc09..fd129a27 100644 --- a/tests/index.js +++ b/tests/index.js @@ -13,6 +13,7 @@ import * as updates from './updates.tests.js' import * as relativePositions from './relativePositions.tests.js' import * as delta from './delta.tests.js' import * as idset from './IdSet.tests.js' +import * as attributionManager from './AttributionManager.tests.js' import { runTests } from 'lib0/testing' import { isBrowser, isNode } from 'lib0/environment' @@ -23,7 +24,7 @@ if (isBrowser) { } const tests = { - doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, updates, relativePositions, delta, idset + doc, map, array, text, xml, encoding, undoredo, compatibility, snapshot, updates, relativePositions, delta, idset, attributionManager } const run = async () => { diff --git a/tests/testHelper.js b/tests/testHelper.js index 60c323f7..acf38b93 100644 --- a/tests/testHelper.js +++ b/tests/testHelper.js @@ -6,6 +6,7 @@ import * as syncProtocol from 'y-protocols/sync' import * as object from 'lib0/object' import * as map from 'lib0/map' import * as Y from '../src/index.js' +import { amAttrsEqual } from '../src/internals.js' export * from '../src/index.js' @@ -323,6 +324,27 @@ export const compareIdSets = (idSet1, idSet2) => { return true } +/** + * @template T + * @param {Y.AttributionManager} am1 + * @param {Y.AttributionManager} am2 + */ +export const compareAttributionManagers = (am1, am2) => { + if (am1.clients.size !== am2.clients.size) return false + for (const [client, _items1] of am1.clients.entries()) { + const items1 = _items1.getIds() + const items2 = am2.clients.get(client)?.getIds() + t.assert(items2 !== undefined && items1.length === items2.length) + for (let i = 0; i < items1.length; i++) { + const di1 = items1[i] + const di2 = /** @type {Array>} */ (items2)[i] + t.assert(di1.clock === di2.clock && di1.len === di2.len && amAttrsEqual(di1.attrs, di2.attrs)) + } + } + return true +} + + /** * 1. reconnect and flush all