Implement delta updates for documents

desktop/package-lock.json

@@ -64,6 +64,7 @@
         "clsx": "^2.1.1",
         "cmdk": "^1.0.0",
         "date-fns": "^3.6.0",
+        "diff": "^7.0.0",
         "electron-squirrel-startup": "^1.0.1",
         "form-data": "^4.0.1",
         "fractional-indexing-jittered": "^0.9.1",
@@ -105,6 +106,7 @@
         "@electron-forge/plugin-vite": "^7.5.0",
         "@electron/fuses": "^1.8.0",
         "@types/better-sqlite3": "^7.6.11",
+        "@types/diff": "^5.2.3",
         "@types/is-hotkey": "^0.1.10",
         "@types/lodash": "^4.17.12",
         "@types/mime-types": "^2.1.4",
@@ -3670,6 +3672,13 @@
         "@types/responselike": "^1.0.0"
       }
     },
+    "node_modules/@types/diff": {
+      "version": "5.2.3",
+      "resolved": "https://registry.npmjs.org/@types/diff/-/diff-5.2.3.tgz",
+      "integrity": "sha512-K0Oqlrq3kQMaO2RhfrNQX5trmt+XLyom88zS0u84nnIcLvFnRUMRRHmrGny5GSM+kNO9IZLARsdQHDzkhAgmrQ==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@types/estree": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
@@ -6150,10 +6159,9 @@
       "license": "Apache-2.0"
     },
     "node_modules/diff": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
-      "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
-      "devOptional": true,
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz",
+      "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==",
       "license": "BSD-3-Clause",
       "engines": {
         "node": ">=0.3.1"
@@ -13524,6 +13532,16 @@
         }
       }
     },
+    "node_modules/ts-node/node_modules/diff": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
+      "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
+      "devOptional": true,
+      "license": "BSD-3-Clause",
+      "engines": {
+        "node": ">=0.3.1"
+      }
+    },
     "node_modules/ts-pattern": {
       "version": "5.5.0",
       "resolved": "https://registry.npmjs.org/ts-pattern/-/ts-pattern-5.5.0.tgz",

desktop/package.json

@@ -23,6 +23,7 @@
     "@electron-forge/plugin-vite": "^7.5.0",
     "@electron/fuses": "^1.8.0",
     "@types/better-sqlite3": "^7.6.11",
+    "@types/diff": "^5.2.3",
     "@types/is-hotkey": "^0.1.10",
     "@types/lodash": "^4.17.12",
     "@types/mime-types": "^2.1.4",
@@ -107,6 +108,7 @@
     "clsx": "^2.1.1",
     "cmdk": "^1.0.0",
     "date-fns": "^3.6.0",
+    "diff": "^7.0.0",
     "electron-squirrel-startup": "^1.0.1",
     "form-data": "^4.0.1",
     "fractional-indexing-jittered": "^0.9.1",

desktop/src/lib/editor.ts

@@ -0,0 +1,370 @@
+import { EditorNodeTypes } from '@/lib/constants';
+import { generateId, getIdTypeFromNode } from '@/lib/id';
+import { generateNodeIndex } from '@/lib/nodes';
+import { compareString } from '@/lib/utils';
+import { NodeBlock, NodeBlockContent } from '@/types/nodes';
+import { JSONContent } from '@tiptap/core';
+import { isEqual } from 'lodash';
+import * as Y from 'yjs';
+import { diffChars } from 'diff';
+
+const leafBlockTypes = new Set([
+  EditorNodeTypes.Paragraph,
+  EditorNodeTypes.Heading1,
+  EditorNodeTypes.Heading2,
+  EditorNodeTypes.Heading3,
+  EditorNodeTypes.HorizontalRule,
+  EditorNodeTypes.CodeBlock,
+]);
+
+export const mapContentsToBlocks = (
+  parentId: string,
+  contents: JSONContent[],
+  blocksMap: Map<string, NodeBlock>,
+): NodeBlock[] => {
+  const blocks: NodeBlock[] = [];
+  mapAndPushContentsToBlocks(contents, parentId, blocks, blocksMap);
+  validateBlocksIndexes(blocks);
+  return blocks;
+};
+
+const mapAndPushContentsToBlocks = (
+  contents: JSONContent[] | null | undefined,
+  parentId: string,
+  blocks: NodeBlock[],
+  blocksMap: Map<string, NodeBlock>,
+): void => {
+  if (!contents) {
+    return;
+  }
+  contents.map((content) => {
+    mapAndPushContentToBlock(content, parentId, blocks, blocksMap);
+  });
+};
+
+const mapAndPushContentToBlock = (
+  content: JSONContent,
+  parentId: string,
+  blocks: NodeBlock[],
+  blocksMap: Map<string, NodeBlock>,
+): void => {
+  const id = getIdFromContent(content);
+  const index = blocksMap.get(id)?.index;
+  const attrs =
+    (content.attrs &&
+      Object.entries(content.attrs).filter(([key]) => key !== 'id')) ??
+    [];
+
+  const isLeafBlock = leafBlockTypes.has(content.type);
+  const blockContent = isLeafBlock
+    ? mapContentsToNodeBlockContents(content.type, content.content)
+    : null;
+
+  blocks.push({
+    id: id,
+    index: index,
+    attrs: attrs.length > 0 ? Object.fromEntries(attrs) : null,
+    parentId: parentId,
+    type: content.type,
+    content: blockContent,
+  });
+
+  if (!isLeafBlock && content.content) {
+    mapAndPushContentsToBlocks(content.content, id, blocks, blocksMap);
+  }
+};
+
+const mapContentsToNodeBlockContents = (
+  type: string,
+  contents?: JSONContent[],
+): NodeBlockContent[] | null => {
+  if (!leafBlockTypes.has(type) || contents == null) {
+    return null;
+  }
+
+  const nodeBlocks: NodeBlockContent[] = [];
+  for (const content of contents) {
+    nodeBlocks.push({
+      type: content.type,
+      text: content.text,
+      marks:
+        content.marks?.map((mark) => {
+          return {
+            type: mark.type,
+            attrs: mark.attrs,
+          };
+        }) ?? null,
+    });
+  }
+  return nodeBlocks;
+};
+
+export const mapBlocksToContents = (
+  parentId: string,
+  blocks: NodeBlock[],
+): JSONContent[] => {
+  const contents: JSONContent[] = [];
+  const children = blocks
+    .filter((block) => block.parentId === parentId)
+    .sort((a, b) => compareString(a.index, b.index));
+
+  for (const child of children) {
+    contents.push(mapBlockToContent(child, blocks));
+  }
+
+  return contents;
+};
+
+const mapBlockToContent = (
+  block: NodeBlock,
+  blocks: NodeBlock[],
+): JSONContent => {
+  return {
+    type: block.type,
+    attrs: {
+      id: block.id,
+      ...block.attrs,
+    },
+    content: leafBlockTypes.has(block.type)
+      ? mapNodeBlockContentsToContents(block.content)
+      : mapBlocksToContents(block.id, blocks),
+  };
+};
+
+const mapNodeBlockContentsToContents = (
+  nodeBlocks: NodeBlockContent[] | null,
+): JSONContent[] | null => {
+  if (nodeBlocks == null) {
+    return null;
+  }
+  const contents: JSONContent[] = [];
+  for (const nodeBlock of nodeBlocks) {
+    contents.push({
+      type: nodeBlock.type,
+      text: nodeBlock.text,
+      marks:
+        nodeBlock.marks?.map((mark) => {
+          return {
+            type: mark.type,
+            attrs: mark.attrs,
+          };
+        }) ?? null,
+    });
+  }
+  return contents;
+};
+
+const validateBlocksIndexes = (blocks: NodeBlock[]) => {
+  //group by parentId
+  const groupedBlocks: { [key: string]: NodeBlock[] } = {};
+  for (const block of blocks) {
+    if (!groupedBlocks[block.parentId]) {
+      groupedBlocks[block.parentId] = [];
+    }
+    groupedBlocks[block.parentId].push(block);
+  }
+  for (const parentId in groupedBlocks) {
+    const blocks = groupedBlocks[parentId];
+    for (let i = 0; i < blocks.length; i++) {
+      const currentIndex = blocks[i].index;
+      const beforeIndex = i === 0 ? null : blocks[i - 1].index;
+      // find the lowest index after the current node
+      // we do this because sometimes nodes can be ordered in such a way that
+      // the current node's index is higher than one of its siblings
+      // after the next sibling
+      // for example:  1, {current}, 4, 3
+      let afterIndex = i === blocks.length - 1 ? null : blocks[i + 1].index;
+      for (let j = i + 1; j < blocks.length; j++) {
+        if (blocks[j].index < afterIndex) {
+          afterIndex = blocks[j].index;
+          break;
+        }
+      }
+      // extra check to make sure that the beforeIndex is less than the afterIndex
+      // because otherwise the fractional index library will throw an error
+      if (afterIndex < beforeIndex) {
+        afterIndex = generateNodeIndex(null, beforeIndex);
+      } else if (beforeIndex === afterIndex) {
+        afterIndex = generateNodeIndex(beforeIndex, null);
+      }
+      if (
+        !currentIndex ||
+        currentIndex <= beforeIndex ||
+        currentIndex > afterIndex
+      ) {
+        blocks[i].index = generateNodeIndex(beforeIndex, afterIndex);
+      }
+    }
+  }
+};
+
+const getIdFromContent = (content: JSONContent): string => {
+  return content.attrs?.id ?? generateId(getIdTypeFromNode(content.type));
+};
+
+export const applyChangeToYDoc = (doc: Y.Doc, blocks: NodeBlock[]) => {
+  const attributesMap = doc.getMap('attributes');
+  if (!attributesMap.has('content')) {
+    attributesMap.set('content', new Y.Map());
+  }
+
+  const contentMap = attributesMap.get('content') as Y.Map<any>;
+  for (const block of blocks) {
+    if (!contentMap.has(block.id)) {
+      contentMap.set(block.id, new Y.Map());
+    }
+
+    const blockMap = contentMap.get(block.id) as Y.Map<any>;
+    applyBlockChangesToYDoc(blockMap, block);
+  }
+};
+
+const applyBlockChangesToYDoc = (blockMap: Y.Map<any>, block: NodeBlock) => {
+  if (blockMap.get('id') !== block.id) {
+    blockMap.set('id', block.id);
+  }
+
+  if (blockMap.get('type') !== block.type) {
+    blockMap.set('type', block.type);
+  }
+
+  if (blockMap.get('index') !== block.index) {
+    blockMap.set('index', block.index);
+  }
+
+  if (blockMap.get('parentId') !== block.parentId) {
+    blockMap.set('parentId', block.parentId);
+  }
+
+  applyBlockAttrsChangesToYDoc(blockMap, block);
+  applyBlockContentChangesToYDoc(blockMap, block);
+};
+
+const applyBlockAttrsChangesToYDoc = (
+  blockMap: Y.Map<any>,
+  block: NodeBlock,
+) => {
+  if (block.attrs === null) {
+    if (blockMap.has('attrs')) {
+      blockMap.delete('attrs');
+    }
+
+    return;
+  }
+
+  if (!blockMap.has('attrs')) {
+    blockMap.set('attrs', new Y.Map());
+  }
+
+  const attrsMap = blockMap.get('attrs') as Y.Map<any>;
+  for (const [key, value] of Object.entries(block.attrs)) {
+    const existingValue = attrsMap.get(key);
+    if (!isEqual(existingValue, value)) {
+      attrsMap.set(key, value);
+    }
+  }
+};
+
+const applyBlockContentChangesToYDoc = (
+  blockMap: Y.Map<any>,
+  block: NodeBlock,
+) => {
+  if (block.content === null || block.content.length === 0) {
+    if (blockMap.has('content')) {
+      blockMap.delete('content');
+    }
+
+    return;
+  }
+
+  if (!blockMap.has('content')) {
+    blockMap.set('content', new Y.Array());
+  }
+
+  const contentArray = blockMap.get('content') as Y.Array<any>;
+  for (let i = 0; i < block.content.length; i++) {
+    const blockContent = block.content[i];
+    if (contentArray.length > i) {
+      const blockContentMap = contentArray.get(i) as Y.Map<any>;
+      applyBlockContentItemChangesToYDoc(blockContentMap, blockContent);
+    } else {
+      const blockContentMap = new Y.Map<any>();
+      contentArray.insert(i, [blockContentMap]);
+      applyBlockContentItemChangesToYDoc(blockContentMap, blockContent);
+    }
+  }
+};
+
+const applyBlockContentItemChangesToYDoc = (
+  blockContentMap: Y.Map<any>,
+  blockContent: NodeBlockContent,
+) => {
+  if (blockContentMap.get('type') !== blockContent.type) {
+    blockContentMap.set('type', blockContent.type);
+  }
+
+  applyBlockContentTextChangesToYDoc(blockContentMap, blockContent);
+  applyBlockContentMarksChangesToYDoc(blockContentMap, blockContent);
+};
+
+const applyBlockContentTextChangesToYDoc = (
+  blockContentMap: Y.Map<any>,
+  blockContent: NodeBlockContent,
+) => {
+  if (
+    blockContent.text === null ||
+    blockContent.text === undefined ||
+    blockContent.text === ''
+  ) {
+    if (blockContentMap.has('text')) {
+      blockContentMap.delete('text');
+    }
+
+    return;
+  }
+
+  if (!blockContentMap.has('text')) {
+    blockContentMap.set('text', new Y.Text(blockContent.text));
+    return;
+  }
+
+  const yText = blockContentMap.get('text') as Y.Text;
+  const currentText = yText.toString();
+  const newText = blockContent.text;
+
+  if (currentText === newText) {
+    return;
+  }
+
+  const diffs = diffChars(currentText, newText);
+  let index = 0;
+  for (const diff of diffs) {
+    if (diff.added) {
+      yText.insert(index, diff.value);
+      index += diff.value.length;
+    } else if (diff.removed) {
+      yText.delete(index, diff.value.length);
+      index -= diff.value.length;
+    } else {
+      index += diff.value.length;
+    }
+  }
+};
+
+const applyBlockContentMarksChangesToYDoc = (
+  blockContentMap: Y.Map<any>,
+  blockContent: NodeBlockContent,
+) => {
+  if (blockContent.marks === null || blockContent.marks.length === 0) {
+    if (blockContentMap.has('marks')) {
+      blockContentMap.delete('marks');
+    }
+
+    return;
+  }
+
+  const existingMarks = blockContentMap.get('marks');
+  if (!isEqual(existingMarks, blockContent.marks)) {
+    blockContentMap.set('marks', blockContent.marks);
+  }
+};

desktop/src/main/handlers/mutations/document-save.ts

@@ -1,11 +1,12 @@
 import * as Y from 'yjs';
 import { fromUint8Array, toUint8Array } from 'js-base64';
-import { mapNode } from '@/lib/nodes';
 import { databaseManager } from '@/main/data/database-manager';
 import { MutationHandler, MutationResult } from '@/operations/mutations';
 import { DocumentSaveMutationInput } from '@/operations/mutations/document-save';
 import { generateId, IdType } from '@/lib/id';
 import { LocalUpdateNodeChangeData } from '@/types/sync';
+import { applyChangeToYDoc, mapContentsToBlocks } from '@/lib/editor';
+import { LocalNodeAttributes, NodeBlock } from '@/types/nodes';
 
 export class DocumentSaveMutationHandler
   implements MutationHandler<DocumentSaveMutationInput>
@@ -32,29 +33,53 @@ export class DocumentSaveMutationHandler
       };
     }
 
-    const node = mapNode(document);
     const versionId = generateId(IdType.Version);
     const updatedAt = new Date().toISOString();
     const updates: string[] = [];
 
     const doc = new Y.Doc({
-      guid: node.id,
+      guid: document.id,
     });
-    Y.applyUpdate(doc, toUint8Array(node.state));
+    Y.applyUpdate(doc, toUint8Array(document.state));
 
     doc.on('update', (update) => {
       updates.push(fromUint8Array(update));
     });
 
-    const attributesMap = doc.getMap('attributes');
-    attributesMap.set('content', input.content.content);
+    const attributes = JSON.parse(document.attributes) as LocalNodeAttributes;
+    const blocksMap = new Map<string, NodeBlock>();
+    if (attributes.content) {
+      for (const [key, value] of Object.entries(attributes.content)) {
+        blocksMap.set(key, value);
+      }
+    }
 
-    const attributes = JSON.stringify(attributesMap.toJSON());
+    const blocks = mapContentsToBlocks(
+      document.id,
+      input.content.content,
+      blocksMap,
+    );
+
+    doc.transact(() => {
+      applyChangeToYDoc(doc, blocks);
+    });
+
+    if (updates.length === 0) {
+      return {
+        output: {
+          success: true,
+        },
+        changes: [],
+      };
+    }
+
+    const attributesMap = doc.getMap('attributes');
+    const attributesJson = JSON.stringify(attributesMap.toJSON());
     const encodedState = fromUint8Array(Y.encodeStateAsUpdate(doc));
 
     const changeData: LocalUpdateNodeChangeData = {
       type: 'node_update',
-      id: node.id,
+      id: document.id,
       updatedAt: updatedAt,
       updatedBy: input.userId,
       versionId: versionId,
@@ -65,7 +90,7 @@ export class DocumentSaveMutationHandler
       await trx
         .updateTable('nodes')
         .set({
-          attributes: attributes,
+          attributes: attributesJson,
           state: encodedState,
           updated_at: updatedAt,
           updated_by: input.userId,

desktop/src/main/handlers/queries/document-get.ts

@@ -12,8 +12,9 @@ import {
 } from '@/operations/queries';
 
 import { SelectNode } from '@/main/data/workspace/schema';
-import { mapNode } from '@/lib/nodes';
 import { MutationChange } from '@/operations/mutations';
+import { LocalNodeAttributes } from '@/types/nodes';
+import { mapBlocksToContents } from '@/lib/editor';
 
 export class DocumentGetQueryHandler
   implements QueryHandler<DocumentGetQueryInput>
@@ -88,8 +89,10 @@ export class DocumentGetQueryHandler
       };
     }
 
-    const node = mapNode(document);
-    const contents = node.attributes?.content ?? [];
+    const attributes = JSON.parse(document.attributes) as LocalNodeAttributes;
+    const nodeBlocks = Object.values(attributes?.content ?? {});
+    const contents = mapBlocksToContents(document.id, nodeBlocks);
+
     if (!contents.length) {
       contents.push({
         type: 'paragraph',

desktop/src/main/handlers/queries/message-list.ts

@@ -13,6 +13,7 @@ import { mapNode } from '@/lib/nodes';
 import { UserNode } from '@/types/users';
 import { compareString } from '@/lib/utils';
 import { isEqual } from 'lodash';
+import { mapBlocksToContents } from '@/lib/editor';
 
 export class MessageListQueryHandler
   implements QueryHandler<MessageListQueryInput>
@@ -159,10 +160,13 @@ export class MessageListQueryHandler
         author: author ?? {
           id: messageNode.createdBy,
           name: 'Unknown User',
-          email: 'unknown@neuron.com',
+          email: 'unknown@colanode.com',
           avatar: null,
         },
-        content: messageNode.attributes.content,
+        content: mapBlocksToContents(
+          messageNode.id,
+          Object.values(messageNode.attributes.content ?? {}),
+        ),
         reactionCounts,
       };
 

desktop/src/types/editor.ts

@@ -1,5 +1,4 @@
 import { Editor, type Range } from '@tiptap/core';
-import { LocalNodeAttributes } from '@/types/nodes';
 import { FC } from 'react';
 
 export type EditorCommandProps = {
@@ -22,8 +21,3 @@ export type EditorCommand = {
   handler: (props: EditorCommandProps) => void | Promise<void>;
   disabled?: boolean;
 };
-
-export type EditorNode = {
-  id: string;
-  attributes: LocalNodeAttributes;
-};

desktop/src/types/nodes.ts

@@ -1,5 +1,3 @@
-import { JSONContent } from '@tiptap/core';
-
 export type LocalNode = {
   id: string;
   parentId: string | null;
@@ -21,19 +19,28 @@ export type LocalNodeAttributes = {
   type: string;
   parentId?: string | null;
   index?: string | null;
-  content?: JSONContent[] | null;
+  content?: Record<string, NodeBlock> | null;
   [key: string]: any;
 };
 
 export type NodeBlock = {
+  id: string;
   type: string;
-  text?: string | null;
-  marks?: NodeBlockMark[];
+  index: string;
+  parentId: string;
+  content: NodeBlockContent[] | null;
+  attrs: Record<string, any> | null;
 };
 
-export type NodeBlockMark = {
+export type NodeBlockContent = {
   type: string;
-  attrs: any;
+  text?: string | null;
+  marks?: NodeBlocContentkMark[];
+};
+
+export type NodeBlocContentkMark = {
+  type: string;
+  attrs: Record<string, any>;
 };
 
 export type LocalNodeWithChildren = LocalNode & {
@@ -60,7 +67,7 @@ export type ServerNodeAttributes = {
   type: string;
   parentId?: string | null;
   index?: string | null;
-  content?: NodeBlock[] | null;
+  content?: NodeBlockContent[] | null;
   [key: string]: any;
 };