diff --git a/packages/core/src/api/sync/__tests__/merger.test.js b/packages/core/src/api/sync/__tests__/merger.test.js new file mode 100644 index 000000000..c70d5f629 --- /dev/null +++ b/packages/core/src/api/sync/__tests__/merger.test.js @@ -0,0 +1,409 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import { + TEST_NOTE, + databaseTest, + loginFakeUser +} from "../../../../__tests__/utils"; +import { expect, describe, vi } from "vitest"; +import Merger from "../merger"; + +describe.concurrent("merge item synchronously", (test) => { + test("accept remote item if no local item is found", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const merged = merger.mergeItem( + { + type: "color" + }, + undefined + ); + + expect(merged).toBeDefined(); + expect(merged.type).toBe("color"); + })); + + test("accept remote item if it is newer than local item", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const merged = merger.mergeItem( + { + type: "color", + dateModified: Date.now() + }, + { + type: "color", + dateModified: Date.now() - 1000 + } + ); + + expect(merged).toBeDefined(); + expect(merged.type).toBe("color"); + })); + + test("accept local item if it is newer than remote item", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const merged = merger.mergeItem( + { + type: "color", + dateModified: Date.now() - 1000 + }, + { + type: "color", + dateModified: Date.now() + } + ); + + expect(merged).toBeUndefined(); + })); +}); + +describe.concurrent("merge content", (test) => { + test("do nothing if local item is localOnly", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const merged = await merger.mergeContent( + { + type: "tiptap", + data: "Hello" + }, + { + type: "tiptap", + localOnly: true + } + ); + + expect(merged).toBeUndefined(); + })); + + test("accept remote item if local item is not defined", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const merged = await merger.mergeContent( + { + type: "tiptap" + }, + undefined + ); + + expect(merged).toBeDefined(); + expect(merged.type).toBe("tiptap"); + })); + + test("accept remote item if it is newer than local item", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const merged = await merger.mergeContent( + { + type: "tiptap", + data: "Remote", + dateEdited: Date.now() + }, + { + type: "tiptap", + data: "Local", + dateEdited: Date.now() - 1000 + } + ); + + expect(merged).toBeDefined(); + expect(merged.data).toBe("Remote"); + })); + + test("trigger conflict if local item dateEdited is newer", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const noteId = await db.notes.add(TEST_NOTE); + const merged = await merger.mergeContent( + { + type: "tiptap", + data: "Remote", + noteId, + dateEdited: Date.now() - 60000 + }, + { + type: "tiptap", + data: "Local", + noteId, + + dateEdited: Date.now() + } + ); + + expect(merged).toBeDefined(); + expect(merged.data).toBe("Local"); + expect(merged.conflicted).toBeDefined(); + expect(merged.conflicted.data).toBe("Remote"); + expect(await db.notes.conflicted.has(noteId)).toBe(true); + })); + + test("merge conflicts if local item is already conflicted", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const noteId = await db.notes.add({ ...TEST_NOTE, conflicted: true }); + const merged = await merger.mergeContent( + { + type: "tiptap", + data: "Conflicted remote 2", + noteId + }, + { + type: "tiptap", + data: "Local", + noteId, + conflicted: { type: "tiptap", data: "Conflicted remote" } + } + ); + + expect(merged).toBeDefined(); + expect(merged.data).toBe("Local"); + expect(merged.conflicted).toBeDefined(); + expect(merged.conflicted.data).toBe("Conflicted remote 2"); + expect(await db.notes.conflicted.has(noteId)).toBe(true); + })); + + describe("auto resolve conflict", () => { + describe("edits under the conflict threshold", (test) => { + test("keep remote if it is newer", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const noteId = await db.notes.add(TEST_NOTE); + const merged = await merger.mergeContent( + { + type: "tiptap", + data: "Remote", + noteId, + dateEdited: Date.now() - 3000, + dateModified: Date.now() + }, + { + type: "tiptap", + data: "Local", + noteId, + dateEdited: Date.now(), + dateModified: Date.now() - 6000 + } + ); + + expect(merged).toBeDefined(); + expect(merged.data).toBe("Remote"); + })); + + test("keep local if it is newer", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const noteId = await db.notes.add(TEST_NOTE); + const merged = await merger.mergeContent( + { + type: "tiptap", + data: "Remote", + noteId, + dateEdited: Date.now() - 3000, + dateModified: Date.now() - 6000 + }, + { + type: "tiptap", + data: "Local", + noteId, + dateEdited: Date.now(), + dateModified: Date.now() + } + ); + + expect(merged).toBeUndefined(); + })); + }); + + describe("edits are equal", (test) => { + test("keep remote if it is newer", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const noteId = await db.notes.add(TEST_NOTE); + const merged = await merger.mergeContent( + { + type: "tiptap", + data: "Remote", + noteId, + dateEdited: Date.now() - 60000, + dateModified: Date.now() + }, + { + type: "tiptap", + data: "Remote", + noteId, + dateEdited: Date.now(), + dateModified: Date.now() - 6000 + } + ); + + expect(merged).toBeDefined(); + expect(merged.data).toBe("Remote"); + })); + + test("keep local if it is newer", () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const noteId = await db.notes.add(TEST_NOTE); + const merged = await merger.mergeContent( + { + type: "tiptap", + data: "Remote", + noteId, + dateEdited: Date.now() - 60000, + dateModified: Date.now() - 6000 + }, + { + type: "tiptap", + data: "Remote", + noteId, + dateEdited: Date.now(), + dateModified: Date.now() + } + ); + + expect(merged).toBeUndefined(); + })); + }); + }); +}); + +describe.concurrent("merge attachment", () => { + describe("accept remote item", (test) => { + const cases = [ + { + name: "local item is undefined", + remote: { type: "attachment" }, + local: undefined + }, + { + name: "local item is deleted (remote is newer)", + remote: { type: "attachment", dateModified: Date.now() }, + local: { + type: "attachment", + deleted: true, + dateModified: Date.now() - 1000 + } + }, + { + name: "remote item is deleted (remote is newer)", + remote: { type: "attachment", deleted: true, dateModified: Date.now() }, + local: { type: "attachment", dateModified: Date.now() - 1000 } + }, + { + name: "remote item's dateUploaded is more recent", + remote: { + type: "attachment", + dateUploaded: Date.now() + }, + local: { type: "attachment", dateUploaded: Date.now() - 1000 } + } + ]; + + for (const testCase of cases) { + test(testCase.name, () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + db.attachments.remove = vi.fn(() => true); + + const merged = await merger.mergeAttachment( + testCase.remote, + testCase.local + ); + + expect(merged).toBeDefined(); + expect(merged).toStrictEqual(testCase.remote); + }) + ); + } + }); + + describe("accept local item", (test) => { + const cases = [ + { + name: "local item is deleted (local is newer)", + remote: { type: "attachment", dateModified: Date.now() - 1000 }, + local: { + type: "attachment", + deleted: true, + dateModified: Date.now() + } + }, + { + name: "remote item is deleted (local is newer)", + remote: { + type: "attachment", + deleted: true, + dateModified: Date.now() - 1000 + }, + local: { type: "attachment", dateModified: Date.now() } + }, + { + name: "local item's dateUploaded is more recent", + remote: { + type: "attachment", + dateUploaded: Date.now() - 1000 + }, + local: { type: "attachment", dateUploaded: Date.now() } + } + ]; + + for (const testCase of cases) { + test(testCase.name, () => + databaseTest().then(async (db) => { + await loginFakeUser(db); + const merger = new Merger(db); + + const merged = await merger.mergeAttachment( + testCase.remote, + testCase.local + ); + + expect(merged).toBeUndefined(); + }) + ); + } + }); +}); diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.ts index 0101824ce..ae6342efb 100644 --- a/packages/core/src/api/sync/index.ts +++ b/packages/core/src/api/sync/index.ts @@ -286,8 +286,6 @@ class Sync { async processChunk(chunk: SyncTransferItem, key: SerializedKey) { const itemType = chunk.type; - if (itemType === "settings") return; - const decrypted = await this.db.storage().decryptMulti(key, chunk.items); const deserialized: MaybeDeletedItem[] = []; @@ -313,11 +311,11 @@ class Sync { itemType === "attachment" ? await Promise.all( deserialized.map((item) => - this.merger.mergeItemAsync(item, localItems[item.id], itemType) + this.merger.mergeAttachment(item, localItems[item.id]) ) ) : deserialized.map((item) => - this.merger.mergeItemSync(item, localItems[item.id], itemType) + this.merger.mergeItem(item, localItems[item.id]) ); } diff --git a/packages/core/src/api/sync/merger.ts b/packages/core/src/api/sync/merger.ts index 2f2990195..2e2262421 100644 --- a/packages/core/src/api/sync/merger.ts +++ b/packages/core/src/api/sync/merger.ts @@ -20,7 +20,13 @@ along with this program. If not, see . import { logger } from "../../logger"; import { isHTMLEqual } from "../../utils/html-diff"; import Database from ".."; -import { ContentItem, Item, MaybeDeletedItem, isDeleted } from "../../types"; +import { + Attachment, + ContentItem, + Item, + MaybeDeletedItem, + isDeleted +} from "../../types"; const THRESHOLD = process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000; class Merger { @@ -31,33 +37,12 @@ class Merger { // return type in SYNC_COLLECTIONS_MAP; // } - mergeItemSync( + mergeItem( remoteItem: MaybeDeletedItem, - localItem: MaybeDeletedItem | undefined, - type: - | "shortcut" - | "reminder" - | "tag" - | "color" - | "note" - | "relation" - | "notebook" - | "settingitem" + localItem: MaybeDeletedItem | undefined ) { - switch (type) { - case "shortcut": - case "reminder": - case "tag": - case "color": - case "note": - case "relation": - case "notebook": - case "settingitem": { - if (!localItem || remoteItem.dateModified > localItem.dateModified) { - return remoteItem; - } - break; - } + if (!localItem || remoteItem.dateModified > localItem.dateModified) { + return remoteItem; } } @@ -78,9 +63,7 @@ class Merger { !localItem.data || !remoteItem.data ) { - if (!localItem || remoteItem.dateModified > localItem.dateModified) - return remoteItem; - return; + return this.mergeItem(remoteItem, localItem); } else { // it's possible that the local item already has a conflict so // we can just replace the conflicted content @@ -101,48 +84,41 @@ class Merger { } } - async mergeItemAsync( - remoteItem: MaybeDeletedItem, - localItem: MaybeDeletedItem | undefined, - type: "attachment" + async mergeAttachment( + remoteItem: MaybeDeletedItem, + localItem: MaybeDeletedItem | undefined ) { - switch (type) { - case "attachment": { - if (!localItem) return remoteItem; - if ( - isDeleted(localItem) || - isDeleted(remoteItem) || - remoteItem.type !== "attachment" || - localItem.type !== "attachment" - ) { - if (remoteItem.dateModified > localItem.dateModified) - return remoteItem; - return; - } - - if (localItem.dateUploaded !== remoteItem.dateUploaded) { - const isRemoved = await this.db.attachments.remove( - localItem.hash, - true - ); - if (!isRemoved) - throw new Error( - "Conflict could not be resolved in one of the attachments." - ); - } - return remoteItem; - } + if ( + !localItem || + isDeleted(localItem) || + isDeleted(remoteItem) || + !localItem.dateUploaded || + !remoteItem.dateUploaded + ) { + return this.mergeItem(remoteItem, localItem); } + + if (localItem.dateUploaded > remoteItem.dateUploaded) return; + + const isRemoved = await this.db.attachments.remove(localItem.hash, true); + if (!isRemoved) + throw new Error( + "Conflict could not be resolved in one of the attachments." + ); + return remoteItem; } } export default Merger; -function isContentConflicted( +export function isContentConflicted( localItem: ContentItem, remoteItem: ContentItem, conflictThreshold: number ) { - const isResolved = localItem.dateResolved === remoteItem.dateModified; + const isResolved = + localItem.dateResolved && + remoteItem.dateModified && + localItem.dateResolved === remoteItem.dateModified; const isEdited = // the local item is edited if it was changed/edited after the remote // note and it also wasn't synced yet. diff --git a/packages/core/src/api/sync/types.ts b/packages/core/src/api/sync/types.ts index 50ec5f2b6..574b9c179 100644 --- a/packages/core/src/api/sync/types.ts +++ b/packages/core/src/api/sync/types.ts @@ -19,38 +19,12 @@ along with this program. If not, see . import { Cipher } from "@notesnook/crypto"; -export type SyncableItemType = - | "note" - | "shortcut" - | "notebook" - | "content" - | "attachment" - | "reminder" - | "relation" - | "color" - | "tag" - | "settings" - | "settingitem"; - export type SyncItem = { id: string; v: number; } & Cipher<"base64">; -// export const SYNC_COLLECTIONS_MAP = { -// note: "notes", -// notebook: "notebooks", -// shortcut: "shortcuts", -// reminder: "reminders", -// relation: "relations", -// tag: "tags", -// color: "colors", -// settingitem: "settings" -// } as const; - -// export const ASYNC_COLLECTIONS_MAP = { -// content: "content" -// } as const; +export type SyncableItemType = keyof typeof SYNC_COLLECTIONS_MAP; export const SYNC_COLLECTIONS_MAP = { settingitem: "settings", @@ -62,21 +36,13 @@ export const SYNC_COLLECTIONS_MAP = { relation: "relations", tag: "tags", color: "colors", - note: "notes" + note: "notes", + vault: "vaults" } as const; -export const SYNC_ITEM_TYPES = [ - "settingitem", - "attachment", - "content", - "notebook", - "shortcut", - "reminder", - "relation", - "tag", - "color", - "note" -] as const; +export const SYNC_ITEM_TYPES = Object.keys( + SYNC_COLLECTIONS_MAP +) as SyncableItemType[]; export type SyncTransferItem = { items: SyncItem[]; diff --git a/packages/core/src/models/note.ts b/packages/core/src/models/note.ts deleted file mode 100644 index 68914a28f..000000000 --- a/packages/core/src/models/note.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* -This file is part of the Notesnook project (https://notesnook.com/) - -Copyright (C) 2023 Streetwriters (Private) Limited - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -import Database from "../api"; -import { isDeleted, type Note } from "../types"; - -export function createNoteModel(note: Note, db: Database) { - return { - ...note, - data: note, - async content() { - if (!note.contentId) return null; - const content = await db.content.get(note.contentId); - return content && !isDeleted(content) ? content.data : null; - }, - synced() { - return !note.contentId || db.content.exists(note.contentId); - }, - localOnly() { - return toggleProperty(db, note, "localOnly"); - }, - favorite() { - return toggleProperty(db, note, "favorite"); - }, - pin() { - return toggleProperty(db, note, "pinned"); - }, - readonly() { - return toggleProperty(db, note, "readonly"); - } - }; -} - -function toggleProperty(db: Database, note: Note, property: keyof Note) { - return db.notes.add({ id: note.id, [property]: !note[property] }); -} diff --git a/packages/core/src/models/notebook.ts b/packages/core/src/models/notebook.ts deleted file mode 100644 index e26b64305..000000000 --- a/packages/core/src/models/notebook.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* -This file is part of the Notesnook project (https://notesnook.com/) - -Copyright (C) 2023 Streetwriters (Private) Limited - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -import Database from "../api"; -import { Notebook } from "../types"; - -export function createNotebookModel(notebook: Notebook, db: Database) { - return { - ...notebook, - /** - * @deprecated please use `notebook` directly instead - */ - data: notebook, - /** - * @deprecated please use `db.notebooks.totalNotes()` instead - */ - totalNotes: (function () { - return db.notebooks.totalNotes(notebook.id); - })(), - /** - * @deprecated please use `db.notebooks.pin()` & `db.notebooks.unpin()` instead. - */ - pin() { - return db.notebooks?.add({ - id: notebook.id, - pinned: !notebook.pinned - }); - } - }; -} diff --git a/packages/core/src/utils/buffer-pool.ts b/packages/core/src/utils/buffer-pool.ts deleted file mode 100644 index dfae03a3e..000000000 --- a/packages/core/src/utils/buffer-pool.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* -This file is part of the Notesnook project (https://notesnook.com/) - -Copyright (C) 2023 Streetwriters (Private) Limited - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -export class BufferPool { - private freeBuffers: Buffer[] = []; - constructor(private readonly size: number) {} - - alloc() { - return this.freeBuffers.pop() || this.allocNew(); - } - - private allocNew() { - return Buffer.alloc(this.size); - } - - free(buf: Buffer) { - this.freeBuffers.push(buf); - return true; - } -}