core: improve merging performance

packages/core/src/api/settings.js

@@ -45,24 +45,24 @@ class Settings {
     return this._settings;
   }
 
-  async merge(item) {
-    if (this._settings.dateModified > (await this._db.lastSynced())) {
-      this._settings.id = item.id;
+  async merge(remoteItem, lastSynced) {
+    if (this._settings.dateModified > lastSynced) {
+      this._settings.id = remoteItem.id;
       this._settings.groupOptions = {
         ...this._settings.groupOptions,
-        ...item.groupOptions
+        ...remoteItem.groupOptions
       };
       this._settings.toolbarConfig = {
         ...this._settings.toolbarConfig,
-        ...item.toolbarConfig
+        ...remoteItem.toolbarConfig
       };
       this._settings.aliases = {
         ...this._settings.aliases,
-        ...item.aliases
+        ...remoteItem.aliases
       };
       this._settings.dateModified = Date.now();
     } else {
-      this._initSettings(item);
+      this._initSettings(remoteItem);
     }
     await this._saveSettings(false);
   }
@@ -215,7 +215,11 @@ class Settings {
     }
 
     await this._db.storage.write("settings", this._settings);
-    this._db.eventManager.publish(EVENTS.databaseUpdated, this._settings);
+    this._db.eventManager.publish(
+      EVENTS.databaseUpdated,
+      this._settings.id,
+      this._settings
+    );
   }
 }
 export default Settings;

packages/core/src/api/sync/index.js

@@ -282,11 +282,27 @@ class Sync {
         })
       );
 
-      const items = await Promise.all(
-        deserialized.map((item) =>
-          this.merger.mergeItem(chunk.type, item, dbLastSynced)
-        )
-      );
+      let items = [];
+      if (this.merger.isSyncCollection(chunk.type)) {
+        items = deserialized.map((item) =>
+          this.merger.mergeItemSync(item, chunk.type, dbLastSynced)
+        );
+      } else if (chunk.type === "content") {
+        const localItems = await this.db.content.multi(
+          chunk.items.map((i) => i.id)
+        );
+        items = await Promise.all(
+          deserialized.map((item) =>
+            this.merger.mergeContent(item, localItems, dbLastSynced)
+          )
+        );
+      } else {
+        items = await Promise.all(
+          deserialized.map((item) =>
+            this.merger.mergeItem(item, chunk.type, dbLastSynced)
+          )
+        );
+      }
 
       const collection = typeToCollection[chunk.type];
       if (collection) await collection._collection.setItems(items);

packages/core/src/api/sync/merger.js

@@ -30,181 +30,155 @@ class Merger {
     this._db = db;
     this.logger = logger.scope("Merger");
 
-    this._mergeDefinition = {
-      settings: {
-        threshold: 1000,
-        get: () => this._db.settings.raw,
-        set: (item) => this._db.settings.merge(item),
-        conflict: (_local, remote) => this._db.settings.merge(remote)
-      },
-      note: {
-        get: (id) => this._db.notes._collection.getItem(id),
-        set: (item) => this._db.notes.merge(item)
-      },
-      shortcut: {
-        get: (id) => this._db.shortcuts.shortcut(id),
-        set: (item) => this._db.shortcuts.merge(item)
-      },
-      reminder: {
-        get: (id) => this._db.reminders.reminder(id),
-        set: (item) => this._db.reminders.merge(item)
-      },
-      relation: {
-        get: (id) => this._db.relations.relation(id),
-        set: (item) => this._db.relations.merge(item)
-      },
-      notebook: {
-        threshold: 1000,
-        get: (id) => this._db.notebooks._collection.getItem(id),
-        set: (item) => this._db.notebooks.merge(item),
-        conflict: (_local, remote) => this._db.notebooks.merge(remote)
-      },
-      content: {
-        threshold: process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000,
-        get: (id) => this._db.content.raw(id),
-        set: (item) => this._db.content.add(item),
-        conflict: async (local, remote) => {
-          let note = this._db.notes.note(local.noteId);
-          if (!note || !note.data) return;
-          note = note.data;
-
-          // if hashes are equal do nothing
-          if (
-            !note.locked &&
-            (!remote ||
-              !local ||
-              !local.data ||
-              !remote.data ||
-              remote.data === "undefined" || //TODO not sure about this
-              isHTMLEqual(local.data, remote.data))
-          )
-            return;
-
-          if (remote.deleted || local.deleted || note.locked) {
-            // if note is locked or content is deleted we keep the most recent version.
-            if (remote.dateModified > local.dateModified)
-              await this._db.content.add({ id: local.id, ...remote });
-          } else {
-            // otherwise we trigger the conflicts
-            await this._db.content.add({ ...local, conflicted: remote });
-            await this._db.notes.add({ id: local.noteId, conflicted: true });
-            await this._db.storage.write("hasConflicts", true);
-          }
-        }
-      },
-      attachment: {
-        set: async (remoteAttachment) => {
-          if (remoteAttachment.deleted) {
-            return await this._db.attachments.merge(remoteAttachment);
-          }
-
-          const localAttachment = this._db.attachments.attachment(
-            remoteAttachment.metadata.hash
-          );
-          if (
-            localAttachment &&
-            localAttachment.dateUploaded !== remoteAttachment.dateUploaded
-          ) {
-            const noteIds = localAttachment.noteIds.slice();
-            const isRemoved = await this._db.attachments.remove(
-              localAttachment.metadata.hash,
-              true
-            );
-            if (!isRemoved)
-              throw new Error(
-                "Conflict could not be resolved in one of the attachments."
-              );
-            remoteAttachment.noteIds = setManipulator.union(
-              remoteAttachment.noteIds,
-              noteIds
-            );
-          }
-          return await this._db.attachments.merge(remoteAttachment);
-        }
-      },
-      vaultKey: {
-        set: async (vaultKey) => this._db.vault._setKey(vaultKey)
-      }
+    this.syncCollectionMap = {
+      note: "notes",
+      shortcut: "shortcuts",
+      reminder: "reminders",
+      relation: "relations",
+      notebook: "notebooks"
     };
   }
 
-  async _mergeItem(remoteItem, get, add) {
-    let localItem = await get(remoteItem.id);
-    if (!localItem || remoteItem.dateModified > localItem.dateModified) {
-      return await add(remoteItem);
+  isSyncCollection(type) {
+    return !!this.syncCollectionMap[type];
+  }
+
+  isConflicted(localItem, remoteItem, lastSynced, conflictThreshold) {
+    const isResolved = localItem.dateResolved === remoteItem.dateModified;
+    const isModified =
+      // the local item is modified if it was changed/modified after the last
+      // sync i.e. it wasn't synced yet.
+      // However, in case a sync is interrupted the local item's date modified
+      // will be ahead of last sync. In that case, we also have to check if the
+      // synced flag is false (it is only false if a user makes edits on the
+      // local device).
+      localItem.dateModified > lastSynced && !localItem.synced;
+    if (isModified && !isResolved) {
+      // If time difference between local item's edits & remote item's edits
+      // is less than threshold, we shouldn't trigger a merge conflict; instead
+      // we will keep the most recently changed item.
+      const timeDiff =
+        Math.max(remoteItem.dateModified, localItem.dateModified) -
+        Math.min(remoteItem.dateModified, localItem.dateModified);
+
+      if (timeDiff < conflictThreshold) {
+        if (remoteItem.dateModified > localItem.dateModified) {
+          return "merge";
+        }
+        return;
+      }
+
+      return "conflict";
+    } else if (!isResolved) {
+      return "merge";
     }
   }
 
-  async _mergeItemWithConflicts(
-    remoteItem,
-    get,
-    add,
-    markAsConflicted,
-    threshold
-  ) {
-    let localItem = await get(remoteItem.id);
-
-    if (!localItem) {
-      return await add(remoteItem);
-    } else {
-      const isResolved = localItem.dateResolved === remoteItem.dateModified;
-      const isModified =
-        // the local item is modified if it was changed/modified after the last sync
-        // i.e. it wasn't synced yet.
-        // However, in case a sync is interrupted the local item's date modified will
-        // be ahead of last sync. In that case, we also have to check if the synced flag
-        // is false (it is only false if a user makes edits on the local device).
-        localItem.dateModified > this._lastSynced && !localItem.synced;
-      if (isModified && !isResolved) {
-        // If time difference between local item's edits & remote item's edits
-        // is less than threshold, we shouldn't trigger a merge conflict; instead
-        // we will keep the most recently changed item.
-        const timeDiff =
-          Math.max(remoteItem.dateModified, localItem.dateModified) -
-          Math.min(remoteItem.dateModified, localItem.dateModified);
-
-        if (timeDiff < threshold) {
-          if (remoteItem.dateModified > localItem.dateModified) {
-            return await add(remoteItem);
-          }
-          return;
+  mergeItemSync(remoteItem, type, lastSynced) {
+    switch (type) {
+      case "note":
+      case "shortcut":
+      case "reminder":
+      case "relation": {
+        const localItem = this._db[
+          this.syncCollectionMap[type]
+        ]._collection.getItem(remoteItem.id);
+        if (!localItem || remoteItem.dateModified > localItem.dateModified) {
+          return remoteItem;
         }
-
-        this.logger.info("Conflict detected", {
-          itemId: remoteItem.id,
-          isResolved,
-          isModified,
-          timeDiff,
-          remote: remoteItem.dateModified,
-          local: localItem.dateModified,
-          lastSynced: this._lastSynced
-        });
-
-        await markAsConflicted(localItem, remoteItem);
-      } else if (!isResolved) {
-        return await add(remoteItem);
+        break;
+      }
+      case "notebook": {
+        const THRESHOLD = 1000;
+        const localItem = this._db.notebooks._collection.getItem(remoteItem.id);
+        if (
+          !localItem ||
+          this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD)
+        ) {
+          return this._db.notebooks.merge(localItem, remoteItem, lastSynced);
+        }
+        break;
       }
     }
   }
 
-  async mergeItem(type, item, lastSynced) {
-    this._lastSynced = lastSynced;
+  async mergeContent(remoteItem, localItems, lastSynced) {
+    const THRESHOLD = process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000;
+    const localItem = localItems[remoteItem.id];
+    const conflicted =
+      localItem &&
+      this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD);
+    if (!localItem || conflicted === "merge") {
+      return remoteItem;
+    } else if (conflicted === "conflict") {
+      const note = this._db.notes._collection.getItem(localItem.noteId);
+      if (!note || note.deleted) return;
 
-    const definition = this._mergeDefinition[type];
-    if (!type || !item || !definition) return;
+      // if hashes are equal do nothing
+      if (
+        !note.locked &&
+        (!remoteItem ||
+          !remoteItem ||
+          !localItem.data ||
+          !remoteItem.data ||
+          isHTMLEqual(localItem.data, remoteItem.data))
+      )
+        return;
 
-    if (definition.conflict) {
-      return await this._mergeItemWithConflicts(
-        item,
-        definition.get,
-        definition.set,
-        definition.conflict,
-        definition.threshold
-      );
-    } else if (definition.get && definition.set) {
-      return await this._mergeItem(item, definition.get, definition.set);
-    } else if (!definition.get && definition.set) {
-      return await definition.set(item);
+      if (remoteItem.deleted || localItem.deleted || note.locked) {
+        // if note is locked or content is deleted we keep the most recent version.
+        if (remoteItem.dateModified > localItem.dateModified) return remoteItem;
+      } else {
+        // otherwise we trigger the conflicts
+        await this._db.notes.add({
+          id: localItem.noteId,
+          conflicted: true
+        });
+        await this._db.storage.write("hasConflicts", true);
+        return {
+          ...localItem,
+          conflicted: remoteItem
+        };
+      }
+    }
+  }
+
+  async mergeItem(remoteItem, type, lastSynced) {
+    switch (type) {
+      case "settings": {
+        const localItem = this._db.settings.raw;
+        if (
+          !localItem ||
+          this.isConflicted(localItem, remoteItem, lastSynced, 1000)
+        ) {
+          await this._db.settings.merge(remoteItem, lastSynced);
+        }
+        break;
+      }
+      case "attachment": {
+        if (remoteItem.deleted)
+          return this._db.attachments.merge(null, remoteItem);
+
+        const localItem = this._db.attachments.attachment(
+          remoteItem.metadata.hash
+        );
+        if (localItem && localItem.dateUploaded !== remoteItem.dateUploaded) {
+          const noteIds = localItem.noteIds.slice();
+          const isRemoved = await this._db.attachments.remove(
+            localItem.metadata.hash,
+            true
+          );
+          if (!isRemoved)
+            throw new Error(
+              "Conflict could not be resolved in one of the attachments."
+            );
+          remoteItem.noteIds = setManipulator.union(
+            remoteItem.noteIds,
+            noteIds
+          );
+        }
+        return this._db.attachments.merge(localItem, remoteItem);
+      }
     }
   }
 }

packages/core/src/collections/attachments.js

@@ -36,12 +36,9 @@ export default class Attachments extends Collection {
     this.key = null;
   }
 
-  merge(remoteAttachment) {
+  merge(localAttachment, remoteAttachment) {
     if (remoteAttachment.deleted) return remoteAttachment;
 
-    const id = remoteAttachment.id;
-    let localAttachment = this._collection.getItem(id);
-
     if (localAttachment && localAttachment.noteIds) {
       remoteAttachment.noteIds = setManipulator.union(
         remoteAttachment.noteIds,

packages/core/src/collections/content.js

@@ -26,8 +26,11 @@ export default class Content extends Collection {
   async add(content) {
     if (!content) return;
 
-    if (typeof content.data === "object") {
-      if (typeof content.data.data === "string")
+    if (
+      !!content.data &&
+      (!!content.data.data || (!content.data.iv && !content.data.cipher))
+    ) {
+      if (content.data.data && content.data.data.length)
         content.data = content.data.data;
       else if (!content.data.iv && !content.data.cipher)
         content.data = `<p>Content is invalid: ${JSON.stringify(
@@ -35,9 +38,6 @@ export default class Content extends Collection {
         )}</p>`;
     }
 
-    if (content.remote || content.deleted)
-      return await this.extractAttachments(content);
-
     const oldContent = await this.raw(content.id);
     if (content.id && oldContent) {
       content = {

packages/core/src/collections/notebooks.js

@@ -24,14 +24,9 @@ import { CHECK_IDS, checkIsUserPremium } from "../common";
 import qclone from "qclone";
 
 export default class Notebooks extends Collection {
-  async merge(remoteNotebook) {
+  merge(localNotebook, remoteNotebook, lastSyncedTimestamp) {
     if (remoteNotebook.deleted) return remoteNotebook;
-
-    const id = remoteNotebook.id || id();
-    let localNotebook = this._collection.getItem(id);
-
     if (localNotebook && localNotebook.topics?.length) {
-      const lastSyncedTimestamp = await this._db.lastSynced();
       let isChanged = false;
       // merge new and old topics
       for (let oldTopic of localNotebook.topics) {