be able to encode partial state with holes correctly

src/structs/AbstractStruct.js

@@ -51,6 +51,7 @@ export class AbstractStruct {
 
   /**
    * @param {number} diff
+   * @return {import('../internals.js').GC|import('../internals.js').Item}
    */
   splice (diff) {
     throw error.methodUnimplemented()

src/structs/ContentAny.js

@@ -82,11 +82,12 @@ export class ContentAny {
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
    * @param {number} offset
+   * @param {number} offsetEnd
    */
-  write (encoder, offset) {
-    const len = this.arr.length
-    encoder.writeLen(len - offset)
-    for (let i = offset; i < len; i++) {
+  write (encoder, offset, offsetEnd) {
+    const end = this.arr.length - offsetEnd
+    encoder.writeLen(end - offset)
+    for (let i = offset; i < end; i++) {
       const c = this.arr[i]
       encoder.writeAny(c)
     }

src/structs/ContentBinary.js

@@ -71,9 +71,10 @@ export class ContentBinary {
   gc (_tr) {}
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
-   * @param {number} offset
+   * @param {number} _offset
+   * @param {number} _offsetEnd
    */
-  write (encoder, offset) {
+  write (encoder, _offset, _offsetEnd) {
     encoder.writeBuf(this.content)
   }
 

src/structs/ContentDeleted.js

@@ -78,9 +78,10 @@ export class ContentDeleted {
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
    * @param {number} offset
+   * @param {number} offsetEnd
    */
-  write (encoder, offset) {
-    encoder.writeLen(this.len - offset)
+  write (encoder, offset, offsetEnd) {
+    encoder.writeLen(this.len - offset - offsetEnd)
   }
 
   /**

src/structs/ContentDoc.js

@@ -116,9 +116,10 @@ export class ContentDoc {
 
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
-   * @param {number} offset
+   * @param {number} _offset
+   * @param {number} _offsetEnd
    */
-  write (encoder, offset) {
+  write (encoder, _offset, _offsetEnd) {
     encoder.writeString(this.doc.guid)
     encoder.writeAny(this.opts)
   }

src/structs/ContentEmbed.js

@@ -74,9 +74,10 @@ export class ContentEmbed {
   gc (_tr) {}
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
-   * @param {number} offset
+   * @param {number} _offset
+   * @param {number} _offsetEnd
    */
-  write (encoder, offset) {
+  write (encoder, _offset, _offsetEnd) {
     encoder.writeJSON(this.embed)
   }
 

src/structs/ContentFormat.js

@@ -83,8 +83,9 @@ export class ContentFormat {
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
    * @param {number} _offset
+   * @param {number} _offsetEnd
    */
-  write (encoder, _offset) {
+  write (encoder, _offset, _offsetEnd) {
     encoder.writeKey(this.key)
     encoder.writeJSON(this.value)
   }

src/structs/ContentJSON.js

@@ -79,11 +79,12 @@ export class ContentJSON {
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
    * @param {number} offset
+   * @param {number} offsetEnd
    */
-  write (encoder, offset) {
-    const len = this.arr.length
-    encoder.writeLen(len - offset)
-    for (let i = offset; i < len; i++) {
+  write (encoder, offset, offsetEnd) {
+    const end = this.arr.length - offsetEnd
+    encoder.writeLen(end - offset)
+    for (let i = offset; i < end; i++) {
       const c = this.arr[i]
       encoder.writeString(c === undefined ? 'undefined' : JSON.stringify(c))
     }

src/structs/ContentString.js

@@ -90,9 +90,10 @@ export class ContentString {
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
    * @param {number} offset
+   * @param {number} offsetEnd
    */
-  write (encoder, offset) {
-    encoder.writeString(offset === 0 ? this.str : this.str.slice(offset))
+  write (encoder, offset, offsetEnd) {
+    encoder.writeString((offset === 0 && offsetEnd === 0) ? this.str : this.str.slice(offset, this.str.length - offsetEnd))
   }
 
   /**

src/structs/ContentType.js

@@ -149,8 +149,9 @@ export class ContentType {
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
    * @param {number} _offset
+   * @param {number} _offsetEnd
    */
-  write (encoder, _offset) {
+  write (encoder, _offset, _offsetEnd) {
     this.type._write(encoder)
   }
 

src/structs/GC.js

@@ -3,6 +3,7 @@ import {
   addStruct,
   addStructToIdSet,
   addToIdSet,
+  createID,
   UpdateEncoderV1, UpdateEncoderV2, StructStore, Transaction // eslint-disable-line
 } from '../internals.js'
 
@@ -47,10 +48,11 @@ export class GC extends AbstractStruct {
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
    * @param {number} offset
+   * @param {number} offsetEnd
    */
-  write (encoder, offset) {
+  write (encoder, offset, offsetEnd) {
     encoder.writeInfo(structGCRefNumber)
-    encoder.writeLen(this.length - offset)
+    encoder.writeLen(this.length - offset - offsetEnd)
   }
 
   /**
@@ -68,9 +70,11 @@ export class GC extends AbstractStruct {
    * If this feature is required in the future, then need to try to merge this struct after
    * transaction.
    *
-   * @param {number} _diff
+   * @param {number} diff
    */
-  splice (_diff) {
-    return this
+  splice (diff) {
+    const other = new GC(createID(this.id.client, this.id.clock + diff), this.length - diff)
+    this.length = diff
+    return other
   }
 }

src/structs/Item.js

@@ -22,7 +22,6 @@ import {
   readContentType,
   addChangedTypeToTransaction,
   addStructToIdSet,
-  Skip,
   IdSet, StackItem, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction, // eslint-disable-line
 } from '../internals.js'
 
@@ -126,6 +125,26 @@ export const splitItem = (transaction, leftItem, diff) => {
   return rightItem
 }
 
+/**
+ * More generalized version of splitItem. Split leftStruct into two structs
+ * @param {Transaction?} transaction
+ * @param {AbstractStruct} leftStruct
+ * @param {number} diff
+ * @return {GC|Item}
+ *
+ * @function
+ * @private
+ */
+export const splitStruct = (transaction, leftStruct, diff) => {
+  if (leftStruct instanceof Item) {
+    return splitItem(transaction, leftStruct, diff)
+  } else {
+    const rightItem = leftStruct.splice(diff)
+    transaction?._mergeStructs.push(rightItem)
+    return rightItem
+  }
+}
+
 /**
  * @param {Array<StackItem>} stack
  * @param {ID} id
@@ -409,11 +428,6 @@ export class Item extends AbstractStruct {
         this.parent = /** @type {ContentType} */ (parentItem.content).type
       }
     }
-    // @todo remove thgis
-    if (this.left instanceof Skip || this.right instanceof Skip || this.parent instanceof Skip) {
-      debugger
-      throw new Error('dtruinae')
-    }
     return null
   }
 
@@ -634,7 +648,7 @@ export class Item extends AbstractStruct {
   }
 
   /**
-   * @param {Transaction} tr 
+   * @param {Transaction} tr
    * @param {boolean} parentGCd
    */
   gc (tr, parentGCd) {
@@ -657,8 +671,9 @@ export class Item extends AbstractStruct {
    *
    * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder The encoder to write data to.
    * @param {number} offset
+   * @param {number} offsetEnd
    */
-  write (encoder, offset) {
+  write (encoder, offset, offsetEnd) {
     const origin = offset > 0 ? createID(this.id.client, this.id.clock + offset - 1) : this.origin
     const rightOrigin = this.rightOrigin
     const parentSub = this.parentSub
@@ -700,7 +715,7 @@ export class Item extends AbstractStruct {
         encoder.writeString(parentSub)
       }
     }
-    this.content.write(encoder, offset)
+    this.content.write(encoder, offset, offsetEnd)
   }
 }
 
@@ -808,8 +823,9 @@ export class AbstractContent {
   /**
    * @param {UpdateEncoderV1 | UpdateEncoderV2} _encoder
    * @param {number} _offset
+   * @param {number} _offsetEnd
    */
-  write (_encoder, _offset) {
+  write (_encoder, _offset, _offsetEnd) {
     throw error.methodUnimplemented()
   }
 

src/utils/Doc.js

@@ -62,7 +62,7 @@ export class Doc extends ObservableV2 {
   /**
    * @param {DocOpts} opts configuration
    */
-  constructor ({ guid = random.uuidv4(), collectionid = null, gc = true, gcFilter = () => true, meta = null, autoLoad = false, shouldLoad = true, isSuggestionDoc = false} = {}) {
+  constructor ({ guid = random.uuidv4(), collectionid = null, gc = true, gcFilter = () => true, meta = null, autoLoad = false, shouldLoad = true, isSuggestionDoc = false } = {}) {
     super()
     this.gc = gc
     this.gcFilter = gcFilter

src/utils/Snapshot.js

@@ -184,7 +184,7 @@ export const createDocFromSnapshot = (originDoc, snapshot, newDoc = new Doc()) =
       // first clock written is 0
       encoding.writeVarUint(encoder.restEncoder, 0)
       for (let i = 0; i <= lastStructIndex; i++) {
-        structs[i].write(encoder, 0)
+        structs[i].write(encoder, 0, 0)
       }
     }
     writeIdSet(encoder, ds)

src/utils/StructSet.js

@@ -83,6 +83,7 @@ export const readStructSet = (decoder, doc) => {
  * @param {IdSet} exclude
  */
 export const removeRangesFromStructSet = (ss, exclude) => {
+  // @todo walk through ss instead to reduce iterations
   exclude.clients.forEach((range, client) => {
     const structs = /** @type {StructRange} */ (ss.clients.get(client))?.refs
     if (structs != null) {

src/utils/StructStore.js

@@ -5,7 +5,8 @@ import {
   createIdSet,
   Transaction, ID, Item, // eslint-disable-line
   Skip,
-  createID
+  createID,
+  splitStruct
 } from '../internals.js'
 
 import * as math from 'lib0/math'
@@ -198,8 +199,8 @@ export const getItem = /** @type {function(StructStore,ID):Item} */ (find)
 export const findIndexCleanStart = (transaction, structs, clock) => {
   const index = findIndexSS(structs, clock)
   const struct = structs[index]
-  if (struct.id.clock < clock && struct.constructor !== GC) {
-    structs.splice(index + 1, 0, struct instanceof Item ? splitItem(transaction, struct, clock - struct.id.clock) : struct.splice(clock - struct.id.clock))
+  if (struct.id.clock < clock) {
+    structs.splice(index + 1, 0, splitStruct(transaction, struct, clock - struct.id.clock))
     return index + 1
   }
   return index

src/utils/Transaction.js

@@ -10,8 +10,7 @@ import {
   generateNewClientId,
   createID,
   cleanupYTextAfterTransaction,
-  IdSet, UpdateEncoderV1, UpdateEncoderV2, GC, StructStore, AbstractType, AbstractStruct, YEvent, Doc
-  // insertIntoIdSet
+  IdSet, UpdateEncoderV1, UpdateEncoderV2, GC, StructStore, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line
 } from '../internals.js'
 
 import * as error from 'lib0/error'

src/utils/encoding.js

@@ -37,7 +37,8 @@ import {
   removeRangesFromStructSet,
   createIdSet,
   StructSet, IdSet, DSDecoderV2, Doc, Transaction, GC, Item, StructStore, // eslint-disable-line
-  createID
+  createID,
+  IdRange
 } from '../internals.js'
 
 import * as encoding from 'lib0/encoding'
@@ -50,24 +51,57 @@ import * as array from 'lib0/array'
  * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
  * @param {Array<GC|Item>} structs All structs by `client`
  * @param {number} client
- * @param {number} clock write structs starting with `ID(client,clock)`
+ * @param {Array<IdRange>} idranges
  *
  * @function
  */
-const writeStructs = (encoder, structs, client, clock) => {
-  // write first id
-  clock = math.max(clock, structs[0].id.clock) // make sure the first id exists
-  const startNewStructs = findIndexSS(structs, clock)
+const writeStructs = (encoder, structs, client, idranges) => {
+  let structsToWrite = 0 // this accounts for the skips
+  /**
+   * @type {Array<{ start: number, end: number, startClock: number, endClock: number }>}
+   */
+  const indexRanges = []
+  const firstPossibleClock = structs[0].id.clock
+  const lastStruct = array.last(structs)
+  const lastPossibleClock = lastStruct.id.clock + lastStruct.length
+  idranges.forEach(idrange => {
+    const startClock = math.max(idrange.clock, firstPossibleClock)
+    const endClock = math.min(idrange.clock + idrange.len, lastPossibleClock)
+    if (startClock >= endClock) return // structs for this range do not exist
+    // inclusive start
+    const start = findIndexSS(structs, startClock)
+    // exclusive end
+    const end = findIndexSS(structs, endClock - 1) + 1
+    structsToWrite += end - start
+    indexRanges.push({
+      start,
+      end,
+      startClock,
+      endClock
+    })
+  })
+  structsToWrite += idranges.length - 1
+  // start writing with this clock. this is updated to the next clock that we expect to write
+  let clock = indexRanges[0].startClock
   // write # encoded structs
-  encoding.writeVarUint(encoder.restEncoder, structs.length - startNewStructs)
+  encoding.writeVarUint(encoder.restEncoder, structsToWrite)
   encoder.writeClient(client)
+  // write clock
   encoding.writeVarUint(encoder.restEncoder, clock)
-  const firstStruct = structs[startNewStructs]
-  // write first struct with an offset
-  firstStruct.write(encoder, clock - firstStruct.id.clock)
-  for (let i = startNewStructs + 1; i < structs.length; i++) {
-    structs[i].write(encoder, 0)
-  }
+  indexRanges.forEach(indexRange => {
+    const skipLen = indexRange.startClock - clock
+    if (skipLen > 0) {
+      new Skip(createID(client, clock), skipLen).write(encoder, 0)
+      clock += skipLen
+    }
+    for (let i = indexRange.start; i < indexRange.end; i++) {
+      const struct = structs[i]
+      const structEnd = struct.id.clock + struct.length
+      const offsetEnd = math.max(structEnd - indexRange.endClock, 0)
+      struct.write(encoder, clock - struct.id.clock, offsetEnd)
+      clock = structEnd - offsetEnd
+    }
+  })
 }
 
 /**
@@ -97,7 +131,9 @@ export const writeClientsStructs = (encoder, store, _sm) => {
   // Write items with higher client ids first
   // This heavily improves the conflict algorithm.
   array.from(sm.entries()).sort((a, b) => b[0] - a[0]).forEach(([client, clock]) => {
-    writeStructs(encoder, /** @type {Array<GC|Item>} */ (store.clients.get(client)), client, clock)
+    const structs = /** @type {Array<GC|Item>} */ (store.clients.get(client))
+    const lastStruct = structs[structs.length - 1]
+    writeStructs(encoder, structs, client, [new IdRange(clock, lastStruct.id.clock + lastStruct.length - clock)])
   })
 }
 
@@ -117,7 +153,9 @@ export const writeStructsFromIdSet = (encoder, store, idset) => {
   // Write items with higher client ids first
   // This heavily improves the conflict algorithm.
   array.from(idset.clients.entries()).sort((a, b) => b[0] - a[0]).forEach(([client, ids]) => {
-    writeStructs(encoder, /** @type {Array<GC|Item>} */ (store.clients.get(client)), client, ids.getIds()[0].clock)
+    const idRanges = ids.getIds()
+    const structs = /** @type {Array<GC|Item>} */ (store.clients.get(client))
+    writeStructs(encoder, structs, client, idRanges)
   })
 }
 

src/utils/updates.js

@@ -534,7 +534,7 @@ const writeStructToLazyStructWriter = (lazyWriter, struct, offset) => {
     // write startClock
     encoding.writeVarUint(lazyWriter.encoder.restEncoder, struct.id.clock + offset)
   }
-  struct.write(lazyWriter.encoder, offset)
+  struct.write(lazyWriter.encoder, offset, 0)
   lazyWriter.written++
 }
 /**

tests/updates.tests.js

@@ -180,7 +180,6 @@ const checkUpdateCases = (ydoc, updates, enc, hasDeletes) => {
       for (let j = 1; j < updates.length; j++) {
         const partMerged = enc.mergeUpdates(updates.slice(j))
         const partMeta = enc.parseUpdateMeta(partMerged)
-        
         const targetSV = enc.encodeStateVectorFromUpdate(enc.mergeUpdates(updates.slice(0, j)))
         const diffed = enc.diffUpdate(mergedUpdates, targetSV)
         const diffedMeta = enc.parseUpdateMeta(diffed)