From 639b01c092caabfc4ed7201d1c8a22ab6905f918 Mon Sep 17 00:00:00 2001 From: Hakan Shehu Date: Mon, 28 Oct 2024 09:27:08 +0100 Subject: [PATCH] Implement delta updates for documents --- desktop/package-lock.json | 26 +- desktop/package.json | 2 + desktop/src/lib/editor.ts | 370 ++++++++++++++++++ .../main/handlers/mutations/document-save.ts | 43 +- .../src/main/handlers/queries/document-get.ts | 9 +- .../src/main/handlers/queries/message-list.ts | 8 +- desktop/src/types/editor.ts | 6 - desktop/src/types/nodes.ts | 23 +- 8 files changed, 455 insertions(+), 32 deletions(-) create mode 100644 desktop/src/lib/editor.ts diff --git a/desktop/package-lock.json b/desktop/package-lock.json index 940df84c..c79682c0 100644 --- a/desktop/package-lock.json +++ b/desktop/package-lock.json @@ -64,6 +64,7 @@ "clsx": "^2.1.1", "cmdk": "^1.0.0", "date-fns": "^3.6.0", + "diff": "^7.0.0", "electron-squirrel-startup": "^1.0.1", "form-data": "^4.0.1", "fractional-indexing-jittered": "^0.9.1", @@ -105,6 +106,7 @@ "@electron-forge/plugin-vite": "^7.5.0", "@electron/fuses": "^1.8.0", "@types/better-sqlite3": "^7.6.11", + "@types/diff": "^5.2.3", "@types/is-hotkey": "^0.1.10", "@types/lodash": "^4.17.12", "@types/mime-types": "^2.1.4", @@ -3670,6 +3672,13 @@ "@types/responselike": "^1.0.0" } }, + "node_modules/@types/diff": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/diff/-/diff-5.2.3.tgz", + "integrity": "sha512-K0Oqlrq3kQMaO2RhfrNQX5trmt+XLyom88zS0u84nnIcLvFnRUMRRHmrGny5GSM+kNO9IZLARsdQHDzkhAgmrQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", @@ -6150,10 +6159,9 @@ "license": "Apache-2.0" }, "node_modules/diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "devOptional": true, + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" @@ -13524,6 +13532,16 @@ } } }, + "node_modules/ts-node/node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "devOptional": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/ts-pattern": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/ts-pattern/-/ts-pattern-5.5.0.tgz", diff --git a/desktop/package.json b/desktop/package.json index 6a049683..f5d829d3 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -23,6 +23,7 @@ "@electron-forge/plugin-vite": "^7.5.0", "@electron/fuses": "^1.8.0", "@types/better-sqlite3": "^7.6.11", + "@types/diff": "^5.2.3", "@types/is-hotkey": "^0.1.10", "@types/lodash": "^4.17.12", "@types/mime-types": "^2.1.4", @@ -107,6 +108,7 @@ "clsx": "^2.1.1", "cmdk": "^1.0.0", "date-fns": "^3.6.0", + "diff": "^7.0.0", "electron-squirrel-startup": "^1.0.1", "form-data": "^4.0.1", "fractional-indexing-jittered": "^0.9.1", diff --git a/desktop/src/lib/editor.ts b/desktop/src/lib/editor.ts new file mode 100644 index 00000000..032e6c10 --- /dev/null +++ b/desktop/src/lib/editor.ts @@ -0,0 +1,370 @@ +import { EditorNodeTypes } from '@/lib/constants'; +import { generateId, getIdTypeFromNode } from '@/lib/id'; +import { generateNodeIndex } from '@/lib/nodes'; +import { compareString } from '@/lib/utils'; +import { NodeBlock, NodeBlockContent } from '@/types/nodes'; +import { JSONContent } from '@tiptap/core'; +import { isEqual } from 'lodash'; +import * as Y from 'yjs'; +import { diffChars } from 'diff'; + +const leafBlockTypes = new Set([ + EditorNodeTypes.Paragraph, + EditorNodeTypes.Heading1, + EditorNodeTypes.Heading2, + EditorNodeTypes.Heading3, + EditorNodeTypes.HorizontalRule, + EditorNodeTypes.CodeBlock, +]); + +export const mapContentsToBlocks = ( + parentId: string, + contents: JSONContent[], + blocksMap: Map, +): NodeBlock[] => { + const blocks: NodeBlock[] = []; + mapAndPushContentsToBlocks(contents, parentId, blocks, blocksMap); + validateBlocksIndexes(blocks); + return blocks; +}; + +const mapAndPushContentsToBlocks = ( + contents: JSONContent[] | null | undefined, + parentId: string, + blocks: NodeBlock[], + blocksMap: Map, +): void => { + if (!contents) { + return; + } + contents.map((content) => { + mapAndPushContentToBlock(content, parentId, blocks, blocksMap); + }); +}; + +const mapAndPushContentToBlock = ( + content: JSONContent, + parentId: string, + blocks: NodeBlock[], + blocksMap: Map, +): void => { + const id = getIdFromContent(content); + const index = blocksMap.get(id)?.index; + const attrs = + (content.attrs && + Object.entries(content.attrs).filter(([key]) => key !== 'id')) ?? + []; + + const isLeafBlock = leafBlockTypes.has(content.type); + const blockContent = isLeafBlock + ? mapContentsToNodeBlockContents(content.type, content.content) + : null; + + blocks.push({ + id: id, + index: index, + attrs: attrs.length > 0 ? Object.fromEntries(attrs) : null, + parentId: parentId, + type: content.type, + content: blockContent, + }); + + if (!isLeafBlock && content.content) { + mapAndPushContentsToBlocks(content.content, id, blocks, blocksMap); + } +}; + +const mapContentsToNodeBlockContents = ( + type: string, + contents?: JSONContent[], +): NodeBlockContent[] | null => { + if (!leafBlockTypes.has(type) || contents == null) { + return null; + } + + const nodeBlocks: NodeBlockContent[] = []; + for (const content of contents) { + nodeBlocks.push({ + type: content.type, + text: content.text, + marks: + content.marks?.map((mark) => { + return { + type: mark.type, + attrs: mark.attrs, + }; + }) ?? null, + }); + } + return nodeBlocks; +}; + +export const mapBlocksToContents = ( + parentId: string, + blocks: NodeBlock[], +): JSONContent[] => { + const contents: JSONContent[] = []; + const children = blocks + .filter((block) => block.parentId === parentId) + .sort((a, b) => compareString(a.index, b.index)); + + for (const child of children) { + contents.push(mapBlockToContent(child, blocks)); + } + + return contents; +}; + +const mapBlockToContent = ( + block: NodeBlock, + blocks: NodeBlock[], +): JSONContent => { + return { + type: block.type, + attrs: { + id: block.id, + ...block.attrs, + }, + content: leafBlockTypes.has(block.type) + ? mapNodeBlockContentsToContents(block.content) + : mapBlocksToContents(block.id, blocks), + }; +}; + +const mapNodeBlockContentsToContents = ( + nodeBlocks: NodeBlockContent[] | null, +): JSONContent[] | null => { + if (nodeBlocks == null) { + return null; + } + const contents: JSONContent[] = []; + for (const nodeBlock of nodeBlocks) { + contents.push({ + type: nodeBlock.type, + text: nodeBlock.text, + marks: + nodeBlock.marks?.map((mark) => { + return { + type: mark.type, + attrs: mark.attrs, + }; + }) ?? null, + }); + } + return contents; +}; + +const validateBlocksIndexes = (blocks: NodeBlock[]) => { + //group by parentId + const groupedBlocks: { [key: string]: NodeBlock[] } = {}; + for (const block of blocks) { + if (!groupedBlocks[block.parentId]) { + groupedBlocks[block.parentId] = []; + } + groupedBlocks[block.parentId].push(block); + } + for (const parentId in groupedBlocks) { + const blocks = groupedBlocks[parentId]; + for (let i = 0; i < blocks.length; i++) { + const currentIndex = blocks[i].index; + const beforeIndex = i === 0 ? null : blocks[i - 1].index; + // find the lowest index after the current node + // we do this because sometimes nodes can be ordered in such a way that + // the current node's index is higher than one of its siblings + // after the next sibling + // for example: 1, {current}, 4, 3 + let afterIndex = i === blocks.length - 1 ? null : blocks[i + 1].index; + for (let j = i + 1; j < blocks.length; j++) { + if (blocks[j].index < afterIndex) { + afterIndex = blocks[j].index; + break; + } + } + // extra check to make sure that the beforeIndex is less than the afterIndex + // because otherwise the fractional index library will throw an error + if (afterIndex < beforeIndex) { + afterIndex = generateNodeIndex(null, beforeIndex); + } else if (beforeIndex === afterIndex) { + afterIndex = generateNodeIndex(beforeIndex, null); + } + if ( + !currentIndex || + currentIndex <= beforeIndex || + currentIndex > afterIndex + ) { + blocks[i].index = generateNodeIndex(beforeIndex, afterIndex); + } + } + } +}; + +const getIdFromContent = (content: JSONContent): string => { + return content.attrs?.id ?? generateId(getIdTypeFromNode(content.type)); +}; + +export const applyChangeToYDoc = (doc: Y.Doc, blocks: NodeBlock[]) => { + const attributesMap = doc.getMap('attributes'); + if (!attributesMap.has('content')) { + attributesMap.set('content', new Y.Map()); + } + + const contentMap = attributesMap.get('content') as Y.Map; + for (const block of blocks) { + if (!contentMap.has(block.id)) { + contentMap.set(block.id, new Y.Map()); + } + + const blockMap = contentMap.get(block.id) as Y.Map; + applyBlockChangesToYDoc(blockMap, block); + } +}; + +const applyBlockChangesToYDoc = (blockMap: Y.Map, block: NodeBlock) => { + if (blockMap.get('id') !== block.id) { + blockMap.set('id', block.id); + } + + if (blockMap.get('type') !== block.type) { + blockMap.set('type', block.type); + } + + if (blockMap.get('index') !== block.index) { + blockMap.set('index', block.index); + } + + if (blockMap.get('parentId') !== block.parentId) { + blockMap.set('parentId', block.parentId); + } + + applyBlockAttrsChangesToYDoc(blockMap, block); + applyBlockContentChangesToYDoc(blockMap, block); +}; + +const applyBlockAttrsChangesToYDoc = ( + blockMap: Y.Map, + block: NodeBlock, +) => { + if (block.attrs === null) { + if (blockMap.has('attrs')) { + blockMap.delete('attrs'); + } + + return; + } + + if (!blockMap.has('attrs')) { + blockMap.set('attrs', new Y.Map()); + } + + const attrsMap = blockMap.get('attrs') as Y.Map; + for (const [key, value] of Object.entries(block.attrs)) { + const existingValue = attrsMap.get(key); + if (!isEqual(existingValue, value)) { + attrsMap.set(key, value); + } + } +}; + +const applyBlockContentChangesToYDoc = ( + blockMap: Y.Map, + block: NodeBlock, +) => { + if (block.content === null || block.content.length === 0) { + if (blockMap.has('content')) { + blockMap.delete('content'); + } + + return; + } + + if (!blockMap.has('content')) { + blockMap.set('content', new Y.Array()); + } + + const contentArray = blockMap.get('content') as Y.Array; + for (let i = 0; i < block.content.length; i++) { + const blockContent = block.content[i]; + if (contentArray.length > i) { + const blockContentMap = contentArray.get(i) as Y.Map; + applyBlockContentItemChangesToYDoc(blockContentMap, blockContent); + } else { + const blockContentMap = new Y.Map(); + contentArray.insert(i, [blockContentMap]); + applyBlockContentItemChangesToYDoc(blockContentMap, blockContent); + } + } +}; + +const applyBlockContentItemChangesToYDoc = ( + blockContentMap: Y.Map, + blockContent: NodeBlockContent, +) => { + if (blockContentMap.get('type') !== blockContent.type) { + blockContentMap.set('type', blockContent.type); + } + + applyBlockContentTextChangesToYDoc(blockContentMap, blockContent); + applyBlockContentMarksChangesToYDoc(blockContentMap, blockContent); +}; + +const applyBlockContentTextChangesToYDoc = ( + blockContentMap: Y.Map, + blockContent: NodeBlockContent, +) => { + if ( + blockContent.text === null || + blockContent.text === undefined || + blockContent.text === '' + ) { + if (blockContentMap.has('text')) { + blockContentMap.delete('text'); + } + + return; + } + + if (!blockContentMap.has('text')) { + blockContentMap.set('text', new Y.Text(blockContent.text)); + return; + } + + const yText = blockContentMap.get('text') as Y.Text; + const currentText = yText.toString(); + const newText = blockContent.text; + + if (currentText === newText) { + return; + } + + const diffs = diffChars(currentText, newText); + let index = 0; + for (const diff of diffs) { + if (diff.added) { + yText.insert(index, diff.value); + index += diff.value.length; + } else if (diff.removed) { + yText.delete(index, diff.value.length); + index -= diff.value.length; + } else { + index += diff.value.length; + } + } +}; + +const applyBlockContentMarksChangesToYDoc = ( + blockContentMap: Y.Map, + blockContent: NodeBlockContent, +) => { + if (blockContent.marks === null || blockContent.marks.length === 0) { + if (blockContentMap.has('marks')) { + blockContentMap.delete('marks'); + } + + return; + } + + const existingMarks = blockContentMap.get('marks'); + if (!isEqual(existingMarks, blockContent.marks)) { + blockContentMap.set('marks', blockContent.marks); + } +}; diff --git a/desktop/src/main/handlers/mutations/document-save.ts b/desktop/src/main/handlers/mutations/document-save.ts index cd49c3bd..5f54ddf6 100644 --- a/desktop/src/main/handlers/mutations/document-save.ts +++ b/desktop/src/main/handlers/mutations/document-save.ts @@ -1,11 +1,12 @@ import * as Y from 'yjs'; import { fromUint8Array, toUint8Array } from 'js-base64'; -import { mapNode } from '@/lib/nodes'; import { databaseManager } from '@/main/data/database-manager'; import { MutationHandler, MutationResult } from '@/operations/mutations'; import { DocumentSaveMutationInput } from '@/operations/mutations/document-save'; import { generateId, IdType } from '@/lib/id'; import { LocalUpdateNodeChangeData } from '@/types/sync'; +import { applyChangeToYDoc, mapContentsToBlocks } from '@/lib/editor'; +import { LocalNodeAttributes, NodeBlock } from '@/types/nodes'; export class DocumentSaveMutationHandler implements MutationHandler @@ -32,29 +33,53 @@ export class DocumentSaveMutationHandler }; } - const node = mapNode(document); const versionId = generateId(IdType.Version); const updatedAt = new Date().toISOString(); const updates: string[] = []; const doc = new Y.Doc({ - guid: node.id, + guid: document.id, }); - Y.applyUpdate(doc, toUint8Array(node.state)); + Y.applyUpdate(doc, toUint8Array(document.state)); doc.on('update', (update) => { updates.push(fromUint8Array(update)); }); - const attributesMap = doc.getMap('attributes'); - attributesMap.set('content', input.content.content); + const attributes = JSON.parse(document.attributes) as LocalNodeAttributes; + const blocksMap = new Map(); + if (attributes.content) { + for (const [key, value] of Object.entries(attributes.content)) { + blocksMap.set(key, value); + } + } - const attributes = JSON.stringify(attributesMap.toJSON()); + const blocks = mapContentsToBlocks( + document.id, + input.content.content, + blocksMap, + ); + + doc.transact(() => { + applyChangeToYDoc(doc, blocks); + }); + + if (updates.length === 0) { + return { + output: { + success: true, + }, + changes: [], + }; + } + + const attributesMap = doc.getMap('attributes'); + const attributesJson = JSON.stringify(attributesMap.toJSON()); const encodedState = fromUint8Array(Y.encodeStateAsUpdate(doc)); const changeData: LocalUpdateNodeChangeData = { type: 'node_update', - id: node.id, + id: document.id, updatedAt: updatedAt, updatedBy: input.userId, versionId: versionId, @@ -65,7 +90,7 @@ export class DocumentSaveMutationHandler await trx .updateTable('nodes') .set({ - attributes: attributes, + attributes: attributesJson, state: encodedState, updated_at: updatedAt, updated_by: input.userId, diff --git a/desktop/src/main/handlers/queries/document-get.ts b/desktop/src/main/handlers/queries/document-get.ts index c45e50bd..eda7981b 100644 --- a/desktop/src/main/handlers/queries/document-get.ts +++ b/desktop/src/main/handlers/queries/document-get.ts @@ -12,8 +12,9 @@ import { } from '@/operations/queries'; import { SelectNode } from '@/main/data/workspace/schema'; -import { mapNode } from '@/lib/nodes'; import { MutationChange } from '@/operations/mutations'; +import { LocalNodeAttributes } from '@/types/nodes'; +import { mapBlocksToContents } from '@/lib/editor'; export class DocumentGetQueryHandler implements QueryHandler @@ -88,8 +89,10 @@ export class DocumentGetQueryHandler }; } - const node = mapNode(document); - const contents = node.attributes?.content ?? []; + const attributes = JSON.parse(document.attributes) as LocalNodeAttributes; + const nodeBlocks = Object.values(attributes?.content ?? {}); + const contents = mapBlocksToContents(document.id, nodeBlocks); + if (!contents.length) { contents.push({ type: 'paragraph', diff --git a/desktop/src/main/handlers/queries/message-list.ts b/desktop/src/main/handlers/queries/message-list.ts index 7ab9a584..ac6f5d7a 100644 --- a/desktop/src/main/handlers/queries/message-list.ts +++ b/desktop/src/main/handlers/queries/message-list.ts @@ -13,6 +13,7 @@ import { mapNode } from '@/lib/nodes'; import { UserNode } from '@/types/users'; import { compareString } from '@/lib/utils'; import { isEqual } from 'lodash'; +import { mapBlocksToContents } from '@/lib/editor'; export class MessageListQueryHandler implements QueryHandler @@ -159,10 +160,13 @@ export class MessageListQueryHandler author: author ?? { id: messageNode.createdBy, name: 'Unknown User', - email: 'unknown@neuron.com', + email: 'unknown@colanode.com', avatar: null, }, - content: messageNode.attributes.content, + content: mapBlocksToContents( + messageNode.id, + Object.values(messageNode.attributes.content ?? {}), + ), reactionCounts, }; diff --git a/desktop/src/types/editor.ts b/desktop/src/types/editor.ts index 0fe7d497..c4d6e7e0 100644 --- a/desktop/src/types/editor.ts +++ b/desktop/src/types/editor.ts @@ -1,5 +1,4 @@ import { Editor, type Range } from '@tiptap/core'; -import { LocalNodeAttributes } from '@/types/nodes'; import { FC } from 'react'; export type EditorCommandProps = { @@ -22,8 +21,3 @@ export type EditorCommand = { handler: (props: EditorCommandProps) => void | Promise; disabled?: boolean; }; - -export type EditorNode = { - id: string; - attributes: LocalNodeAttributes; -}; diff --git a/desktop/src/types/nodes.ts b/desktop/src/types/nodes.ts index 9fa78ab8..e32a257c 100644 --- a/desktop/src/types/nodes.ts +++ b/desktop/src/types/nodes.ts @@ -1,5 +1,3 @@ -import { JSONContent } from '@tiptap/core'; - export type LocalNode = { id: string; parentId: string | null; @@ -21,19 +19,28 @@ export type LocalNodeAttributes = { type: string; parentId?: string | null; index?: string | null; - content?: JSONContent[] | null; + content?: Record | null; [key: string]: any; }; export type NodeBlock = { + id: string; type: string; - text?: string | null; - marks?: NodeBlockMark[]; + index: string; + parentId: string; + content: NodeBlockContent[] | null; + attrs: Record | null; }; -export type NodeBlockMark = { +export type NodeBlockContent = { type: string; - attrs: any; + text?: string | null; + marks?: NodeBlocContentkMark[]; +}; + +export type NodeBlocContentkMark = { + type: string; + attrs: Record; }; export type LocalNodeWithChildren = LocalNode & { @@ -60,7 +67,7 @@ export type ServerNodeAttributes = { type: string; parentId?: string | null; index?: string | null; - content?: NodeBlock[] | null; + content?: NodeBlockContent[] | null; [key: string]: any; };