core: add tests for sync merger

packages/core/src/api/sync/__tests__/merger.test.js

@@ -0,0 +1,409 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+  TEST_NOTE,
+  databaseTest,
+  loginFakeUser
+} from "../../../../__tests__/utils";
+import { expect, describe, vi } from "vitest";
+import Merger from "../merger";
+
+describe.concurrent("merge item synchronously", (test) => {
+  test("accept remote item if no local item is found", () =>
+    databaseTest().then(async (db) => {
+      await loginFakeUser(db);
+      const merger = new Merger(db);
+
+      const merged = merger.mergeItem(
+        {
+          type: "color"
+        },
+        undefined
+      );
+
+      expect(merged).toBeDefined();
+      expect(merged.type).toBe("color");
+    }));
+
+  test("accept remote item if it is newer than local item", () =>
+    databaseTest().then(async (db) => {
+      await loginFakeUser(db);
+      const merger = new Merger(db);
+
+      const merged = merger.mergeItem(
+        {
+          type: "color",
+          dateModified: Date.now()
+        },
+        {
+          type: "color",
+          dateModified: Date.now() - 1000
+        }
+      );
+
+      expect(merged).toBeDefined();
+      expect(merged.type).toBe("color");
+    }));
+
+  test("accept local item if it is newer than remote item", () =>
+    databaseTest().then(async (db) => {
+      await loginFakeUser(db);
+      const merger = new Merger(db);
+
+      const merged = merger.mergeItem(
+        {
+          type: "color",
+          dateModified: Date.now() - 1000
+        },
+        {
+          type: "color",
+          dateModified: Date.now()
+        }
+      );
+
+      expect(merged).toBeUndefined();
+    }));
+});
+
+describe.concurrent("merge content", (test) => {
+  test("do nothing if local item is localOnly", () =>
+    databaseTest().then(async (db) => {
+      await loginFakeUser(db);
+      const merger = new Merger(db);
+
+      const merged = await merger.mergeContent(
+        {
+          type: "tiptap",
+          data: "Hello"
+        },
+        {
+          type: "tiptap",
+          localOnly: true
+        }
+      );
+
+      expect(merged).toBeUndefined();
+    }));
+
+  test("accept remote item if local item is not defined", () =>
+    databaseTest().then(async (db) => {
+      await loginFakeUser(db);
+      const merger = new Merger(db);
+
+      const merged = await merger.mergeContent(
+        {
+          type: "tiptap"
+        },
+        undefined
+      );
+
+      expect(merged).toBeDefined();
+      expect(merged.type).toBe("tiptap");
+    }));
+
+  test("accept remote item if it is newer than local item", () =>
+    databaseTest().then(async (db) => {
+      await loginFakeUser(db);
+      const merger = new Merger(db);
+
+      const merged = await merger.mergeContent(
+        {
+          type: "tiptap",
+          data: "Remote",
+          dateEdited: Date.now()
+        },
+        {
+          type: "tiptap",
+          data: "Local",
+          dateEdited: Date.now() - 1000
+        }
+      );
+
+      expect(merged).toBeDefined();
+      expect(merged.data).toBe("Remote");
+    }));
+
+  test("trigger conflict if local item dateEdited is newer", () =>
+    databaseTest().then(async (db) => {
+      await loginFakeUser(db);
+      const merger = new Merger(db);
+
+      const noteId = await db.notes.add(TEST_NOTE);
+      const merged = await merger.mergeContent(
+        {
+          type: "tiptap",
+          data: "Remote",
+          noteId,
+          dateEdited: Date.now() - 60000
+        },
+        {
+          type: "tiptap",
+          data: "Local",
+          noteId,
+
+          dateEdited: Date.now()
+        }
+      );
+
+      expect(merged).toBeDefined();
+      expect(merged.data).toBe("Local");
+      expect(merged.conflicted).toBeDefined();
+      expect(merged.conflicted.data).toBe("Remote");
+      expect(await db.notes.conflicted.has(noteId)).toBe(true);
+    }));
+
+  test("merge conflicts if local item is already conflicted", () =>
+    databaseTest().then(async (db) => {
+      await loginFakeUser(db);
+      const merger = new Merger(db);
+
+      const noteId = await db.notes.add({ ...TEST_NOTE, conflicted: true });
+      const merged = await merger.mergeContent(
+        {
+          type: "tiptap",
+          data: "Conflicted remote 2",
+          noteId
+        },
+        {
+          type: "tiptap",
+          data: "Local",
+          noteId,
+          conflicted: { type: "tiptap", data: "Conflicted remote" }
+        }
+      );
+
+      expect(merged).toBeDefined();
+      expect(merged.data).toBe("Local");
+      expect(merged.conflicted).toBeDefined();
+      expect(merged.conflicted.data).toBe("Conflicted remote 2");
+      expect(await db.notes.conflicted.has(noteId)).toBe(true);
+    }));
+
+  describe("auto resolve conflict", () => {
+    describe("edits under the conflict threshold", (test) => {
+      test("keep remote if it is newer", () =>
+        databaseTest().then(async (db) => {
+          await loginFakeUser(db);
+          const merger = new Merger(db);
+
+          const noteId = await db.notes.add(TEST_NOTE);
+          const merged = await merger.mergeContent(
+            {
+              type: "tiptap",
+              data: "Remote",
+              noteId,
+              dateEdited: Date.now() - 3000,
+              dateModified: Date.now()
+            },
+            {
+              type: "tiptap",
+              data: "Local",
+              noteId,
+              dateEdited: Date.now(),
+              dateModified: Date.now() - 6000
+            }
+          );
+
+          expect(merged).toBeDefined();
+          expect(merged.data).toBe("Remote");
+        }));
+
+      test("keep local if it is newer", () =>
+        databaseTest().then(async (db) => {
+          await loginFakeUser(db);
+          const merger = new Merger(db);
+
+          const noteId = await db.notes.add(TEST_NOTE);
+          const merged = await merger.mergeContent(
+            {
+              type: "tiptap",
+              data: "Remote",
+              noteId,
+              dateEdited: Date.now() - 3000,
+              dateModified: Date.now() - 6000
+            },
+            {
+              type: "tiptap",
+              data: "Local",
+              noteId,
+              dateEdited: Date.now(),
+              dateModified: Date.now()
+            }
+          );
+
+          expect(merged).toBeUndefined();
+        }));
+    });
+
+    describe("edits are equal", (test) => {
+      test("keep remote if it is newer", () =>
+        databaseTest().then(async (db) => {
+          await loginFakeUser(db);
+          const merger = new Merger(db);
+
+          const noteId = await db.notes.add(TEST_NOTE);
+          const merged = await merger.mergeContent(
+            {
+              type: "tiptap",
+              data: "Remote",
+              noteId,
+              dateEdited: Date.now() - 60000,
+              dateModified: Date.now()
+            },
+            {
+              type: "tiptap",
+              data: "Remote",
+              noteId,
+              dateEdited: Date.now(),
+              dateModified: Date.now() - 6000
+            }
+          );
+
+          expect(merged).toBeDefined();
+          expect(merged.data).toBe("Remote");
+        }));
+
+      test("keep local if it is newer", () =>
+        databaseTest().then(async (db) => {
+          await loginFakeUser(db);
+          const merger = new Merger(db);
+
+          const noteId = await db.notes.add(TEST_NOTE);
+          const merged = await merger.mergeContent(
+            {
+              type: "tiptap",
+              data: "Remote",
+              noteId,
+              dateEdited: Date.now() - 60000,
+              dateModified: Date.now() - 6000
+            },
+            {
+              type: "tiptap",
+              data: "Remote",
+              noteId,
+              dateEdited: Date.now(),
+              dateModified: Date.now()
+            }
+          );
+
+          expect(merged).toBeUndefined();
+        }));
+    });
+  });
+});
+
+describe.concurrent("merge attachment", () => {
+  describe("accept remote item", (test) => {
+    const cases = [
+      {
+        name: "local item is undefined",
+        remote: { type: "attachment" },
+        local: undefined
+      },
+      {
+        name: "local item is deleted (remote is newer)",
+        remote: { type: "attachment", dateModified: Date.now() },
+        local: {
+          type: "attachment",
+          deleted: true,
+          dateModified: Date.now() - 1000
+        }
+      },
+      {
+        name: "remote item is deleted (remote is newer)",
+        remote: { type: "attachment", deleted: true, dateModified: Date.now() },
+        local: { type: "attachment", dateModified: Date.now() - 1000 }
+      },
+      {
+        name: "remote item's dateUploaded is more recent",
+        remote: {
+          type: "attachment",
+          dateUploaded: Date.now()
+        },
+        local: { type: "attachment", dateUploaded: Date.now() - 1000 }
+      }
+    ];
+
+    for (const testCase of cases) {
+      test(testCase.name, () =>
+        databaseTest().then(async (db) => {
+          await loginFakeUser(db);
+          const merger = new Merger(db);
+          db.attachments.remove = vi.fn(() => true);
+
+          const merged = await merger.mergeAttachment(
+            testCase.remote,
+            testCase.local
+          );
+
+          expect(merged).toBeDefined();
+          expect(merged).toStrictEqual(testCase.remote);
+        })
+      );
+    }
+  });
+
+  describe("accept local item", (test) => {
+    const cases = [
+      {
+        name: "local item is deleted (local is newer)",
+        remote: { type: "attachment", dateModified: Date.now() - 1000 },
+        local: {
+          type: "attachment",
+          deleted: true,
+          dateModified: Date.now()
+        }
+      },
+      {
+        name: "remote item is deleted (local is newer)",
+        remote: {
+          type: "attachment",
+          deleted: true,
+          dateModified: Date.now() - 1000
+        },
+        local: { type: "attachment", dateModified: Date.now() }
+      },
+      {
+        name: "local item's dateUploaded is more recent",
+        remote: {
+          type: "attachment",
+          dateUploaded: Date.now() - 1000
+        },
+        local: { type: "attachment", dateUploaded: Date.now() }
+      }
+    ];
+
+    for (const testCase of cases) {
+      test(testCase.name, () =>
+        databaseTest().then(async (db) => {
+          await loginFakeUser(db);
+          const merger = new Merger(db);
+
+          const merged = await merger.mergeAttachment(
+            testCase.remote,
+            testCase.local
+          );
+
+          expect(merged).toBeUndefined();
+        })
+      );
+    }
+  });
+});

packages/core/src/api/sync/index.ts

@@ -286,8 +286,6 @@ class Sync {
 
   async processChunk(chunk: SyncTransferItem, key: SerializedKey) {
     const itemType = chunk.type;
-    if (itemType === "settings") return;
-
     const decrypted = await this.db.storage().decryptMulti(key, chunk.items);
 
     const deserialized: MaybeDeletedItem<Item>[] = [];
@@ -313,11 +311,11 @@ class Sync {
         itemType === "attachment"
           ? await Promise.all(
               deserialized.map((item) =>
-                this.merger.mergeItemAsync(item, localItems[item.id], itemType)
+                this.merger.mergeAttachment(item, localItems[item.id])
               )
             )
           : deserialized.map((item) =>
-              this.merger.mergeItemSync(item, localItems[item.id], itemType)
+              this.merger.mergeItem(item, localItems[item.id])
             );
     }
 

packages/core/src/api/sync/merger.ts

@@ -20,7 +20,13 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { logger } from "../../logger";
 import { isHTMLEqual } from "../../utils/html-diff";
 import Database from "..";
-import { ContentItem, Item, MaybeDeletedItem, isDeleted } from "../../types";
+import {
+  Attachment,
+  ContentItem,
+  Item,
+  MaybeDeletedItem,
+  isDeleted
+} from "../../types";
 
 const THRESHOLD = process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000;
 class Merger {
@@ -31,33 +37,12 @@ class Merger {
   //   return type in SYNC_COLLECTIONS_MAP;
   // }
 
-  mergeItemSync(
+  mergeItem(
     remoteItem: MaybeDeletedItem<Item>,
-    localItem: MaybeDeletedItem<Item> | undefined,
-    type:
-      | "shortcut"
-      | "reminder"
-      | "tag"
-      | "color"
-      | "note"
-      | "relation"
-      | "notebook"
-      | "settingitem"
+    localItem: MaybeDeletedItem<Item> | undefined
   ) {
-    switch (type) {
-      case "shortcut":
-      case "reminder":
-      case "tag":
-      case "color":
-      case "note":
-      case "relation":
-      case "notebook":
-      case "settingitem": {
-        if (!localItem || remoteItem.dateModified > localItem.dateModified) {
-          return remoteItem;
-        }
-        break;
-      }
+    if (!localItem || remoteItem.dateModified > localItem.dateModified) {
+      return remoteItem;
     }
   }
 
@@ -78,9 +63,7 @@ class Merger {
       !localItem.data ||
       !remoteItem.data
     ) {
-      if (!localItem || remoteItem.dateModified > localItem.dateModified)
-        return remoteItem;
-      return;
+      return this.mergeItem(remoteItem, localItem);
     } else {
       // it's possible that the local item already has a conflict so
       // we can just replace the conflicted content
@@ -101,48 +84,41 @@ class Merger {
     }
   }
 
-  async mergeItemAsync(
-    remoteItem: MaybeDeletedItem<Item>,
-    localItem: MaybeDeletedItem<Item> | undefined,
-    type: "attachment"
+  async mergeAttachment(
+    remoteItem: MaybeDeletedItem<Attachment>,
+    localItem: MaybeDeletedItem<Attachment> | undefined
   ) {
-    switch (type) {
-      case "attachment": {
-        if (!localItem) return remoteItem;
-        if (
-          isDeleted(localItem) ||
-          isDeleted(remoteItem) ||
-          remoteItem.type !== "attachment" ||
-          localItem.type !== "attachment"
-        ) {
-          if (remoteItem.dateModified > localItem.dateModified)
-            return remoteItem;
-          return;
-        }
-
-        if (localItem.dateUploaded !== remoteItem.dateUploaded) {
-          const isRemoved = await this.db.attachments.remove(
-            localItem.hash,
-            true
-          );
-          if (!isRemoved)
-            throw new Error(
-              "Conflict could not be resolved in one of the attachments."
-            );
-        }
-        return remoteItem;
-      }
+    if (
+      !localItem ||
+      isDeleted(localItem) ||
+      isDeleted(remoteItem) ||
+      !localItem.dateUploaded ||
+      !remoteItem.dateUploaded
+    ) {
+      return this.mergeItem(remoteItem, localItem);
     }
+
+    if (localItem.dateUploaded > remoteItem.dateUploaded) return;
+
+    const isRemoved = await this.db.attachments.remove(localItem.hash, true);
+    if (!isRemoved)
+      throw new Error(
+        "Conflict could not be resolved in one of the attachments."
+      );
+    return remoteItem;
   }
 }
 export default Merger;
 
-function isContentConflicted(
+export function isContentConflicted(
   localItem: ContentItem,
   remoteItem: ContentItem,
   conflictThreshold: number
 ) {
-  const isResolved = localItem.dateResolved === remoteItem.dateModified;
+  const isResolved =
+    localItem.dateResolved &&
+    remoteItem.dateModified &&
+    localItem.dateResolved === remoteItem.dateModified;
   const isEdited =
     // the local item is edited if it was changed/edited after the remote
     // note and it also wasn't synced yet.

packages/core/src/api/sync/types.ts

@@ -19,38 +19,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { Cipher } from "@notesnook/crypto";
 
-export type SyncableItemType =
-  | "note"
-  | "shortcut"
-  | "notebook"
-  | "content"
-  | "attachment"
-  | "reminder"
-  | "relation"
-  | "color"
-  | "tag"
-  | "settings"
-  | "settingitem";
-
 export type SyncItem = {
   id: string;
   v: number;
 } & Cipher<"base64">;
 
-// export const SYNC_COLLECTIONS_MAP = {
-//   note: "notes",
-//   notebook: "notebooks",
-//   shortcut: "shortcuts",
-//   reminder: "reminders",
-//   relation: "relations",
-//   tag: "tags",
-//   color: "colors",
-//   settingitem: "settings"
-// } as const;
-
-// export const ASYNC_COLLECTIONS_MAP = {
-//   content: "content"
-// } as const;
+export type SyncableItemType = keyof typeof SYNC_COLLECTIONS_MAP;
 
 export const SYNC_COLLECTIONS_MAP = {
   settingitem: "settings",
@@ -62,21 +36,13 @@ export const SYNC_COLLECTIONS_MAP = {
   relation: "relations",
   tag: "tags",
   color: "colors",
-  note: "notes"
+  note: "notes",
+  vault: "vaults"
 } as const;
 
-export const SYNC_ITEM_TYPES = [
-  "settingitem",
-  "attachment",
-  "content",
-  "notebook",
-  "shortcut",
-  "reminder",
-  "relation",
-  "tag",
-  "color",
-  "note"
-] as const;
+export const SYNC_ITEM_TYPES = Object.keys(
+  SYNC_COLLECTIONS_MAP
+) as SyncableItemType[];
 
 export type SyncTransferItem = {
   items: SyncItem[];

packages/core/src/models/note.ts

@@ -1,52 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import Database from "../api";
-import { isDeleted, type Note } from "../types";
-
-export function createNoteModel(note: Note, db: Database) {
-  return {
-    ...note,
-    data: note,
-    async content() {
-      if (!note.contentId) return null;
-      const content = await db.content.get(note.contentId);
-      return content && !isDeleted(content) ? content.data : null;
-    },
-    synced() {
-      return !note.contentId || db.content.exists(note.contentId);
-    },
-    localOnly() {
-      return toggleProperty(db, note, "localOnly");
-    },
-    favorite() {
-      return toggleProperty(db, note, "favorite");
-    },
-    pin() {
-      return toggleProperty(db, note, "pinned");
-    },
-    readonly() {
-      return toggleProperty(db, note, "readonly");
-    }
-  };
-}
-
-function toggleProperty(db: Database, note: Note, property: keyof Note) {
-  return db.notes.add({ id: note.id, [property]: !note[property] });
-}

packages/core/src/models/notebook.ts

@@ -1,46 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import Database from "../api";
-import { Notebook } from "../types";
-
-export function createNotebookModel(notebook: Notebook, db: Database) {
-  return {
-    ...notebook,
-    /**
-     * @deprecated please use `notebook` directly instead
-     */
-    data: notebook,
-    /**
-     * @deprecated please use `db.notebooks.totalNotes()` instead
-     */
-    totalNotes: (function () {
-      return db.notebooks.totalNotes(notebook.id);
-    })(),
-    /**
-     * @deprecated please use `db.notebooks.pin()` & `db.notebooks.unpin()` instead.
-     */
-    pin() {
-      return db.notebooks?.add({
-        id: notebook.id,
-        pinned: !notebook.pinned
-      });
-    }
-  };
-}

packages/core/src/utils/buffer-pool.ts

@@ -1,36 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-export class BufferPool {
-  private freeBuffers: Buffer[] = [];
-  constructor(private readonly size: number) {}
-
-  alloc() {
-    return this.freeBuffers.pop() || this.allocNew();
-  }
-
-  private allocNew() {
-    return Buffer.alloc(this.size);
-  }
-
-  free(buf: Buffer) {
-    this.freeBuffers.push(buf);
-    return true;
-  }
-}