core: add tests for migrations

packages/core/__tests__/migrations.test.ts

@@ -0,0 +1,663 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { test, expect, describe } from "vitest";
+import { migrateItem } from "../src/migrations";
+import { databaseTest } from "./utils";
+import { getId, makeId } from "../src/utils/id";
+import { LegacySettingsItem } from "../src/types";
+
+describe("[5.2] replace date edited with date modified", () => {
+  const itemsWithDateEdited = ["note", "notebook", "trash", "tiny"] as const;
+  const itemsWithoutDateEdited = ["tag", "attachment", "settings"] as const;
+  for (const type of itemsWithDateEdited) {
+    test(type, () =>
+      databaseTest().then(async (db) => {
+        const item = { type, dateEdited: Date.now() };
+        expect(await migrateItem(item, 5.2, 5.3, type, db, "local")).toBe(true);
+        expect(item.dateModified).toBeDefined();
+        expect(item.dateEdited).toBeDefined();
+      })
+    );
+  }
+
+  for (const type of itemsWithoutDateEdited) {
+    test(type, () =>
+      databaseTest().then(async (db) => {
+        const item = { type, dateEdited: Date.now() };
+        expect(await migrateItem(item, 5.2, 5.3, type, db, "local")).toBe(true);
+        expect(item.dateModified).toBeDefined();
+        expect(item.dateEdited).toBeUndefined();
+      })
+    );
+  }
+});
+
+test("[5.2] remove tox class from checklist", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      type: "tiny",
+      dateEdited: Date.now(),
+      data: `<p>hello</p><ul class="tox-checklist"><li class="tox-checklist--checked">world</li></ul>`
+    };
+    expect(await migrateItem(item, 5.2, 5.3, "tiny", db, "local")).toBe(true);
+    expect(item.data).toBe(
+      `<p>hello</p><ul class="checklist"><li class="checked">world</li></ul>`
+    );
+  }));
+
+test("[5.2] wrap tables with div", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      type: "tiny",
+      dateEdited: Date.now(),
+      data: `<p>hello</p><table></table>`
+    };
+    expect(await migrateItem(item, 5.2, 5.3, "tiny", db, "local")).toBe(true);
+    expect(item.data).toBe(
+      `<p>hello</p><div class="table-container" contenteditable="false"><table contenteditable="true"></table></div>`
+    );
+  }));
+
+test("[5.3] decode wrapped table html entities", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      type: "tiny",
+      dateEdited: Date.now(),
+      data: `<p>hello</p>&lt;div class="table-container" contenteditable="false"&gt;&lt;table contenteditable="true"&gt;&lt;/table&gt;&lt;/div&gt;`
+    };
+    expect(await migrateItem(item, 5.3, 5.4, "tiny", db, "local")).toBe(true);
+    expect(item.data).toBe(
+      `<p>hello</p><div class="table-container" contenteditable="false"><table contenteditable="true"></table></div>`
+    );
+  }));
+
+describe("[5.4] convert tiny to tiptap", () => {
+  const cases = [
+    {
+      name: "preserve newlines in code blocks",
+      from: `<p>Function</p>\n<p>Hello</p>\n<pre class="hljs language-javascript" spellcheck="false"><span class="hljs-keyword">function</span> <span class="hljs-title function_">google</span>() {<br><span class="hljs-keyword">var</span> <span class="hljs-keyword">function</span> <span class="hljs-title function_">google</span>();<br><span class="hljs-title class_">Function</span> <span class="hljs-title function_">google</span>();<br>}</pre>\n<p>Hh</p>\n<p>&nbsp;</p>`,
+      to: `<p>Function</p><br><p>Hello</p><br><pre class="hljs language-javascript" spellcheck="false">function google() {<br>var function google();<br>Function google();<br>}</pre><br><p>Hh</p><br><p>&nbsp;</p>`
+    },
+    {
+      name: "replace table container with table",
+      from: `<p>hello</p><div class="table-container" contenteditable="false"><table contenteditable="true"></table></div>`,
+      to: `<p>hello</p><table></table>`
+    },
+    {
+      name: "move images out of paragraphs",
+      from: `<p><img src="hello.jpg" /></p>`,
+      to: `<img src="hello.jpg">`
+    },
+    {
+      name: "replace [data-mce-bogus] elements",
+      from: `<p><br data-mce-bogus="" /></p><br data-mce-bogus="" />`,
+      to: `<p></p>`
+    },
+    {
+      name: "remove [data-mce-href] & [data-mce-flag] attributes",
+      from: `<p>Hello <a data-mce-href="#" href="#">world</a><span data-mce-flag="true">2</span></p>`,
+      to: `<p>Hello <a href="#">world</a><span>2</span></p>`
+    }
+  ];
+
+  for (const testCase of cases) {
+    test(testCase.name, () =>
+      databaseTest().then(async (db) => {
+        const item = {
+          type: "tiny",
+          dateEdited: Date.now(),
+          data: testCase.from
+        };
+        expect(await migrateItem(item, 5.4, 5.5, "tiny", db, "local")).toBe(
+          true
+        );
+        expect(item.data).toBe(testCase.to);
+      })
+    );
+  }
+});
+
+test("[5.6] remove note ids from topics", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      type: "notebook",
+      topics: [{ notes: ["helloworld"] }, { notes: ["helloworld2"] }]
+    };
+    expect(await migrateItem(item, 5.6, 5.7, "notebook", db, "local")).toBe(
+      true
+    );
+    expect(item.topics.every((t) => !t.notes)).toBe(true);
+  }));
+
+test("[5.6] move pins to shortcuts", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      type: "settings",
+      pins: [
+        {
+          type: "topic",
+          data: {
+            id: "hello",
+            notebookId: "world"
+          }
+        },
+        {
+          type: "notebook",
+          data: { id: "world" }
+        },
+        {
+          type: "tag",
+          data: { id: "tag" }
+        }
+      ]
+    };
+    expect(await migrateItem(item, 5.6, 5.7, "settings", db, "local")).toBe(
+      true
+    );
+    expect(item.pins).toBeUndefined();
+    expect(
+      db.shortcuts.all.find(
+        (s) =>
+          s.item.type === "topic" &&
+          s.item.id === "hello" &&
+          s.item.notebookId === "world"
+      )
+    ).toBeDefined();
+    expect(
+      db.shortcuts.all.find(
+        (s) => s.item.type === "notebook" && s.item.id === "world"
+      )
+    ).toBeDefined();
+    expect(
+      db.shortcuts.all.find((s) => s.item.type === "tag" && s.item.id === "tag")
+    ).toBeDefined();
+  }));
+
+test("[5.7] change session content type from tiny to tiptap", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      id: "hello_content",
+      type: "tiny",
+      data: "<p>hello world</p>"
+    };
+    expect(await migrateItem(item, 5.7, 5.8, "tiny", db, "local")).toBe(true);
+    expect(item.type).toBe("sessioncontent");
+    expect(item.contentType).toBe("tiptap");
+  }));
+
+test("[5.7] change content item type to tiptap", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      id: "hello_content",
+      type: "content",
+      data: "<p>hello world</p>"
+    };
+    expect(await migrateItem(item, 5.7, 5.8, "content", db, "local")).toBe(
+      true
+    );
+    expect(item.type).toBe("tiptap");
+  }));
+
+test("[5.7] change shortcut item id to be same as its reference id", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      id: "something",
+      type: "shortcut",
+      item: {
+        type: "notebook",
+        id: "world"
+      }
+    };
+    expect(await migrateItem(item, 5.7, 5.8, "shortcut", db, "local")).toBe(
+      true
+    );
+    expect(item.id).toBe("world");
+  }));
+
+test("[5.7] add type to session content", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      id: "hello_content",
+      type: "tiptap",
+      data: "<p>hello world</p>"
+    };
+    expect(await migrateItem(item, 5.7, 5.8, "tiptap", db, "local")).toBe(true);
+    expect(item.type).toBe("sessioncontent");
+    expect(item.contentType).toBe("tiptap");
+  }));
+
+test("[5.7] change notehistory type to session", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      type: "notehistory"
+    };
+    expect(await migrateItem(item, 5.7, 5.8, "notehistory", db, "local")).toBe(
+      true
+    );
+    expect(item.type).toBe("session");
+  }));
+
+test("[5.8] remove remote property from items", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      type: "note",
+      remote: true
+    };
+    expect(await migrateItem(item, 5.8, 5.9, "note", db, "local")).toBe(true);
+    expect(item.remote).toBeUndefined();
+  }));
+
+test("[5.8] do nothing if backup type is not local", () =>
+  databaseTest().then(async (db) => {
+    const item = {
+      type: "note",
+      remote: true
+    };
+    expect(await migrateItem(item, 5.8, 5.9, "note", db, "backup")).toBe(false);
+    expect(await migrateItem(item, 5.8, 5.9, "note", db, "sync")).toBe(false);
+  }));
+
+describe("[5.9] make tags syncable", () => {
+  test("create tags inside notes & link to them using relations", () =>
+    databaseTest().then(async (db) => {
+      const noteId = getId();
+      const tags = ["hello", "world", "i am here"];
+      await db.notes.collection.add({
+        type: "note",
+        title: "I am a note",
+        tags,
+        id: noteId
+      });
+
+      for (const tag of tags) {
+        await db.tags.collection.add({
+          id: makeId(tag),
+          noteIds: [noteId],
+          type: "tag",
+          title: tag
+        });
+      }
+      await db.storage().write("settings", {
+        ...db.legacySettings.raw,
+        aliases: {
+          [makeId(tags[1])]: "I AM GOOD!"
+        }
+      });
+      await db.legacySettings.init();
+
+      const note = db.notes.note(noteId);
+      if (!note) throw new Error("Failed to find note.");
+
+      expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(
+        true
+      );
+
+      const resolvedTags = db.relations
+        .to({ type: "note", id: noteId }, "tag")
+        .resolved()
+        .sort((a, b) => a.title.localeCompare(b.title));
+
+      expect(note.data.tags).toBeUndefined();
+      expect(db.tags.all).toHaveLength(3);
+      expect(resolvedTags).toHaveLength(3);
+      expect(resolvedTags[0].title).toBe("hello");
+      expect(resolvedTags[1].title).toBe("I AM GOOD!");
+      expect(resolvedTags[2].title).toBe("i am here");
+      expect(
+        tags.every((t) => !db.tags.collection.exists(makeId(t)))
+      ).toBeTruthy();
+    }));
+
+  test("migrate old tag item to new one", () =>
+    databaseTest().then(async (db) => {
+      const tag = {
+        id: makeId("oldone"),
+        noteIds: [],
+        type: "tag",
+        title: "oldone"
+      };
+      expect(await migrateItem(tag, 5.9, 6.0, "tag", db, "backup")).toBe(true);
+      expect(tag.id).not.toBe(makeId("oldone"));
+      expect(tag.noteIds).toBeUndefined();
+      expect(tag.alias).toBeUndefined();
+      expect(tag.title).toBe("oldone");
+    }));
+
+  test("migrate old tag item with alias to new one", () =>
+    databaseTest().then(async (db) => {
+      await db.storage().write("settings", {
+        ...db.legacySettings.raw,
+        aliases: {
+          [makeId("oldone")]: "alias"
+        }
+      });
+      await db.legacySettings.init();
+
+      const tag = {
+        id: makeId("oldone"),
+        noteIds: [],
+        type: "tag",
+        title: "oldone"
+      };
+      expect(await migrateItem(tag, 5.9, 6.0, "tag", db, "backup")).toBe(true);
+      expect(tag.id).not.toBe(makeId("oldone"));
+      expect(tag.noteIds).toBeUndefined();
+      expect(tag.alias).toBeUndefined();
+      expect(tag.title).toBe("alias");
+    }));
+
+  test("migrate tags before notes", () =>
+    databaseTest().then(async (db) => {
+      const noteId = getId();
+      const tags = ["hello", "world", "i am here"];
+      await db.notes.collection.add({
+        type: "note",
+        title: "I am a note",
+        tags,
+        id: noteId
+      });
+      await db.storage().write("settings", {
+        ...db.legacySettings.raw,
+        aliases: {
+          [makeId(tags[1])]: "I AM GOOD!"
+        }
+      });
+      await db.legacySettings.init();
+      for (const tag of tags) {
+        const item = {
+          id: makeId(tag),
+          noteIds: [noteId],
+          type: "tag",
+          title: tag
+        };
+        await migrateItem(item, 5.9, 6.0, "tag", db, "backup");
+        await db.tags.collection.add(item);
+      }
+
+      const note = db.notes.note(noteId);
+      if (!note) throw new Error("Failed to find note.");
+
+      expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(
+        true
+      );
+
+      const resolvedTags = db.relations
+        .to({ type: "note", id: noteId }, "tag")
+        .resolved()
+        .sort((a, b) => a.title.localeCompare(b.title));
+
+      expect(note.data.tags).toBeUndefined();
+      expect(db.tags.all).toHaveLength(3);
+      expect(resolvedTags).toHaveLength(3);
+      expect(resolvedTags[0].title).toBe("hello");
+      expect(resolvedTags[1].title).toBe("I AM GOOD!");
+      expect(resolvedTags[2].title).toBe("i am here");
+      expect(
+        tags.every((t) => !db.tags.collection.exists(makeId(t)))
+      ).toBeTruthy();
+    }));
+});
+
+describe("[5.9] make colors syncable", () => {
+  test("create colors from notes & link to them using relations", () =>
+    databaseTest().then(async (db) => {
+      const noteId = getId();
+      await db.notes.collection.add({
+        type: "note",
+        title: "I am a note",
+        color: "blue",
+        id: noteId
+      });
+
+      await db.colors.collection.add({
+        id: makeId("blue"),
+        noteIds: [noteId],
+        type: "tag",
+        title: "blue"
+      });
+
+      const note = db.notes.note(noteId);
+      if (!note) throw new Error("Failed to find note.");
+
+      expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(
+        true
+      );
+
+      const resolvedColors = db.relations
+        .to({ type: "note", id: noteId }, "color")
+        .resolved();
+
+      expect(note.data.color).toBeUndefined();
+      expect(db.colors.all).toHaveLength(1);
+      expect(resolvedColors).toHaveLength(1);
+      expect(resolvedColors[0].title).toBe("blue");
+      expect(resolvedColors[0].colorCode).toBe("#2196F3");
+      expect(db.colors.collection.exists(makeId("blue"))).toBeFalsy();
+    }));
+
+  test("migrate old color item to new one", () =>
+    databaseTest().then(async (db) => {
+      const color = {
+        id: makeId("blue"),
+        noteIds: [],
+        type: "tag",
+        title: "blue"
+      };
+      expect(await migrateItem(color, 5.9, 6.0, "tag", db, "backup")).toBe(
+        true
+      );
+      expect(color.id).not.toBe(makeId("oldone"));
+      expect(color.noteIds).toBeUndefined();
+      expect(color.alias).toBeUndefined();
+      expect(color.title).toBe("blue");
+      expect(color.type).toBe("color");
+      expect(color.colorCode).toBe("#2196F3");
+    }));
+
+  test("migrate old color item with alias to new one", () =>
+    databaseTest().then(async (db) => {
+      await db.storage().write("settings", {
+        ...db.legacySettings.raw,
+        aliases: {
+          [makeId("blue")]: "very important"
+        }
+      });
+      await db.legacySettings.init();
+
+      const color = {
+        id: makeId("blue"),
+        noteIds: [],
+        type: "tag",
+        title: "blue"
+      };
+      expect(await migrateItem(color, 5.9, 6.0, "tag", db, "backup")).toBe(
+        true
+      );
+      expect(color.id).not.toBe(makeId("oldone"));
+      expect(color.noteIds).toBeUndefined();
+      expect(color.alias).toBeUndefined();
+      expect(color.title).toBe("very important");
+      expect(color.type).toBe("color");
+      expect(color.colorCode).toBe("#2196F3");
+    }));
+
+  test("migrate color before notes", () =>
+    databaseTest().then(async (db) => {
+      const noteId = getId();
+      await db.notes.collection.add({
+        type: "note",
+        title: "I am a note",
+        color: "blue",
+        id: noteId
+      });
+      await db.storage().write("settings", {
+        ...db.legacySettings.raw,
+        aliases: {
+          [makeId("blue")]: "I AM GOOD!"
+        }
+      });
+      await db.legacySettings.init();
+
+      const color = {
+        id: makeId("blue"),
+        noteIds: [noteId],
+        type: "tag",
+        title: "blue"
+      };
+      await migrateItem(color, 5.9, 6.0, "tag", db, "backup");
+      await db.colors.collection.add(color);
+
+      const note = db.notes.note(noteId);
+      if (!note) throw new Error("Failed to find note.");
+
+      expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(
+        true
+      );
+
+      const resolvedColors = db.relations
+        .to({ type: "note", id: noteId }, "color")
+        .resolved();
+
+      expect(note.data.color).toBeUndefined();
+      expect(db.colors.all).toHaveLength(1);
+      expect(resolvedColors).toHaveLength(1);
+      expect(resolvedColors[0].title).toBe("I AM GOOD!");
+      expect(resolvedColors[0].colorCode).toBe("#2196F3");
+      expect(db.colors.collection.exists(makeId("blue"))).toBeFalsy();
+    }));
+});
+
+test("[5.9] move attachments.noteIds to relations", () =>
+  databaseTest().then(async (db) => {
+    const attachment = {
+      id: "ATTACHMENT_ID",
+      type: "attachment",
+      noteIds: ["HELLO_NOTE_ID"]
+    };
+    await migrateItem(attachment, 5.9, 6.0, "attachment", db, "backup");
+
+    const linkedNotes = db.relations.from(
+      { type: "attachment", id: "ATTACHMENT_ID" },
+      "note"
+    );
+    expect(attachment.noteIds).toBeUndefined();
+    expect(linkedNotes).toHaveLength(1);
+    expect(linkedNotes[0].to.id).toBe("HELLO_NOTE_ID");
+  }));
+
+describe("[5.9] move topics out of notebooks & use relations", () => {
+  test("convert topics to subnotebooks", () =>
+    databaseTest().then(async (db) => {
+      const notebook = {
+        id: "parent_notebook",
+        type: "notebook",
+        topics: [
+          { id: "topics1", title: "Topic 1" },
+          { id: "topics2", title: "Topic 2" }
+        ]
+      };
+      await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup");
+
+      const linkedNotebooks = db.relations
+        .from({ type: "notebook", id: "parent_notebook" }, "notebook")
+        .sort((a, b) => a.to.id.localeCompare(b.to.id));
+      expect(notebook.topics).toBeUndefined();
+      expect(linkedNotebooks).toHaveLength(2);
+      expect(linkedNotebooks[0].to.id).toBe("topics1");
+      expect(linkedNotebooks[1].to.id).toBe("topics2");
+      expect(db.notebooks.all).toHaveLength(2);
+      expect(db.notebooks.notebook("topics1")).toBeDefined();
+      expect(db.notebooks.notebook("topics2")).toBeDefined();
+    }));
+
+  test("convert topic shortcuts to notebook shortcuts", () =>
+    databaseTest().then(async (db) => {
+      const shortcut = {
+        id: "shortcut1",
+        type: "shortcut",
+        item: {
+          type: "topic",
+          id: "topics1"
+        }
+      };
+      await migrateItem(shortcut, 5.9, 6.0, "shortcut", db, "backup");
+
+      expect(shortcut.item.type).toBe("notebook");
+      expect(shortcut.item.id).toBe("topics1");
+    }));
+
+  test("convert topic links in note to relations", () =>
+    databaseTest().then(async (db) => {
+      const note = {
+        id: "note1",
+        type: "note",
+        notebooks: [{ id: "notebook1", topics: ["topic1", "topic2"] }]
+      };
+      await migrateItem(note, 5.9, 6.0, "note", db, "backup");
+
+      const linkedNotebooks = db.relations
+        .to({ type: "note", id: "note1" }, "notebook")
+        .sort();
+      expect(note.notebooks).toBeUndefined();
+      expect(linkedNotebooks).toHaveLength(2);
+      expect(linkedNotebooks[0].from.id).toBe("topic1");
+      expect(linkedNotebooks[1].from.id).toBe("topic2");
+    }));
+});
+
+test("[5.9] migrate settings to its own collection", () =>
+  databaseTest().then(async (db) => {
+    const settings: LegacySettingsItem = {
+      type: "settings",
+      id: "settings",
+      dateCreated: Date.now(),
+      dateModified: Date.now(),
+      dateFormat: "CUSTOM_DATE_FORMAT!",
+      defaultNotebook: { id: "notebook1", topic: "topic1" },
+      groupOptions: {
+        favorites: {
+          groupBy: "abc",
+          sortBy: "dateCreated",
+          sortDirection: "asc"
+        }
+      },
+      timeFormat: "24-hour",
+      titleFormat: "I AM TITLE FORMAT",
+      toolbarConfig: { desktop: { preset: "custom" } },
+      trashCleanupInterval: 365
+    };
+    await migrateItem(settings, 5.9, 6.0, "settings", db, "backup");
+
+    expect(db.settings.getDateFormat()).toBe(settings.dateFormat);
+    expect(db.settings.getDefaultNotebook()).toBe(
+      settings.defaultNotebook?.topic
+    );
+    expect(db.settings.getGroupOptions("favorites")).toMatchObject(
+      settings.groupOptions?.favorites || {}
+    );
+    expect(db.settings.getTitleFormat()).toBe(settings.titleFormat);
+    expect(db.settings.getTimeFormat()).toBe(settings.timeFormat);
+    expect(db.settings.getToolbarConfig("desktop")).toMatchObject(
+      settings.toolbarConfig?.desktop || {}
+    );
+    expect(db.settings.getTrashCleanupInterval()).toBe(
+      settings.trashCleanupInterval
+    );
+  }));

packages/core/src/api/migrations.ts

@@ -45,6 +45,14 @@ class Migrations {
       await this.db.notes.init();
 
       const collections: MigratableCollections = [
+        {
+          items: () => [this.db.legacySettings.raw],
+          type: "settings"
+        },
+        {
+          items: () => this.db.settings.raw,
+          type: "settingsv2"
+        },
         {
           items: () => this.db.attachments.all,
           type: "attachments"
@@ -65,10 +73,6 @@ class Migrations {
           iterate: true,
           type: "content"
         },
-        {
-          items: () => [this.db.settings.raw],
-          type: "settings"
-        },
         {
           items: () => this.db.shortcuts.raw,
           type: "shortcuts"

packages/core/src/api/sync/index.ts

@@ -19,6 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import {
   checkSyncStatus,
+  CURRENT_DATABASE_VERSION,
   EV,
   EVENTS,
   sendSyncProgressEvent,
@@ -43,19 +44,11 @@ import {
   Notebook,
   TrashOrItem
 } from "../../types";
-import { SyncableItemType, SyncTransferItem } from "./types";
-
-const ITEM_TYPE_TO_COLLECTION_TYPE = {
-  note: "notes",
-  notebook: "notebooks",
-  content: "content",
-  attachment: "attachments",
-  relation: "relations",
-  reminder: "reminders",
-  shortcut: "shortcuts",
-  tag: "tags",
-  color: "colors"
-};
+import {
+  MERGE_COLLECTIONS_MAP,
+  SyncableItemType,
+  SyncTransferItem
+} from "./types";
 
 export type SyncOptions = {
   type: "full" | "fetch" | "send";
@@ -388,6 +381,9 @@ class Sync {
     dbLastSynced: number,
     notify = false
   ) {
+    const itemType = chunk.type;
+    if (itemType === "settings") return;
+
     const decrypted = await this.db.storage().decryptMulti(key, chunk.items);
 
     const deserialized = await Promise.all(
@@ -396,7 +392,6 @@ class Sync {
       )
     );
 
-    const itemType = chunk.type;
     let items: (
       | MaybeDeletedItem<
           ItemMap[SyncableItemType] | TrashOrItem<Note> | TrashOrItem<Notebook>
@@ -412,9 +407,6 @@ class Sync {
           this.merger.mergeContent(item, localItems[item.id], dbLastSynced)
         )
       );
-    } else if (itemType === "settings") {
-      await this.merger.mergeItem(deserialized[0], itemType, dbLastSynced);
-      return;
     } else {
       items = this.merger.isSyncCollection(itemType)
         ? deserialized.map((item) =>
@@ -427,7 +419,7 @@ class Sync {
           );
     }
 
-    const collectionType = ITEM_TYPE_TO_COLLECTION_TYPE[itemType];
+    const collectionType = MERGE_COLLECTIONS_MAP[itemType];
     await this.db[collectionType].collection.setItems(items as any);
 
     if (
@@ -498,6 +490,7 @@ async function deserializeItem(
     await migrateItem(
       deserialized,
       version,
+      CURRENT_DATABASE_VERSION,
       deserialized.type,
       database,
       "sync"

packages/core/src/api/sync/types.ts

@@ -48,6 +48,12 @@ export const SYNC_COLLECTIONS_MAP = {
   settingitem: "settings"
 } as const;
 
+export const MERGE_COLLECTIONS_MAP = {
+  ...SYNC_COLLECTIONS_MAP,
+  attachment: "attachments",
+  content: "content"
+} as const;
+
 export type SyncTransferItem = {
   items: SyncItem[];
   type: SyncableItemType;

packages/core/src/collections/settings.ts

@@ -20,7 +20,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { makeId } from "../utils/id";
 import Database from "../api";
 import {
-  DefaultNotebook,
   GroupOptions,
   GroupingKey,
   SettingItem,
@@ -138,7 +137,7 @@ export class Settings implements ICollection {
     return this.get("trashCleanupInterval");
   }
 
-  setDefaultNotebook(item: DefaultNotebook | undefined) {
+  setDefaultNotebook(item: string | undefined) {
     return this.set("defaultNotebook", item);
   }
 

packages/core/src/database/backup.ts

@@ -137,6 +137,7 @@ const itemTypeToCollectionKey = {
   notehistory: "notehistory",
   content: "content",
   shortcut: "shortcuts",
+  settingitem: "settingsv2",
 
   // to make ts happy
   topic: "topics"
@@ -343,13 +344,21 @@ export default class Backup {
       if ("sessionContentId" in item && item.type !== "session")
         (item as any).type = "notehistory";
 
-      await migrateItem(item, version, item.type, this.db, "backup");
+      await migrateItem(
+        item,
+        version,
+        CURRENT_DATABASE_VERSION,
+        item.type,
+        this.db,
+        "backup"
+      );
       // since items in trash can have their own set of migrations,
       // we have to run the migration again to account for that.
       if (item.type === "trash" && item.itemType)
         await migrateItem(
           item as unknown as Note | Notebook,
           version,
+          CURRENT_DATABASE_VERSION,
           item.itemType,
           this.db,
           "backup"

packages/core/src/database/migrator.ts

@@ -18,7 +18,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import Database from "../api";
-import { sendMigrationProgressEvent } from "../common";
+import {
+  CURRENT_DATABASE_VERSION,
+  sendMigrationProgressEvent
+} from "../common";
 import { migrateCollection, migrateItem } from "../migrations";
 import {
   CollectionType,
@@ -99,17 +102,31 @@ class Migrator {
       }
 
       const itemId = item.id;
-      const migrated = await migrateItem(
+      let migrated = await migrateItem(
         item,
         version,
+        CURRENT_DATABASE_VERSION,
         item.type || type,
         db,
         "local"
       );
 
+      // trash item is also a notebook or a note so we have to migrate it separately.
+      if (isTrashItem(item)) {
+        migrated = await migrateItem(
+          item as any,
+          version,
+          CURRENT_DATABASE_VERSION,
+          item.itemType,
+          db,
+          "local"
+        );
+      }
+
       if (migrated) {
-        if (item.type === "settings") {
-          await db.settings.merge(item, Infinity);
+        if (item.type !== "settings") {
+          // we are removing the old settings.
+          await db.storage().remove("settings");
         } else toAdd.push(item);
 
         // if id changed after migration, we need to delete the old one.

packages/core/src/migrations.ts

@@ -76,7 +76,7 @@ const migrations: Migration[] = [
       notebook: replaceDateEditedWithDateModified(false),
       tag: replaceDateEditedWithDateModified(true),
       attachment: replaceDateEditedWithDateModified(true),
-      trash: replaceDateEditedWithDateModified(),
+      trash: replaceDateEditedWithDateModified(false),
       tiny: (item) => {
         replaceDateEditedWithDateModified(false)(item);
 
@@ -147,12 +147,10 @@ const migrations: Migration[] = [
     version: 5.7,
     items: {
       tiny: (item) => {
-        if (!item.data || isCipher(item.data)) return false;
         item.type = "tiptap";
         return changeSessionContentType(item);
       },
       content: (item) => {
-        if (!item.data || isCipher(item.data)) return false;
         const oldType = item.type;
         item.type = "tiptap";
         return oldType !== item.type;
@@ -190,9 +188,18 @@ const migrations: Migration[] = [
     version: 5.9,
     items: {
       tag: async (item, db) => {
+        const oldTagId = makeId(item.title);
         const alias = db.legacySettings.getAlias(item.id);
-        item.title = alias || item.title;
-        item.id = getId(item.dateCreated);
+        if (
+          !alias &&
+          (db.tags.all.find(
+            (t) => item.title === t.title && t.id !== oldTagId
+          ) ||
+            db.colors.all.find(
+              (t) => item.title === t.title && t.id !== oldTagId
+            ))
+        )
+          return false;
 
         const colorCode = ColorToHexCode[item.title];
         if (colorCode) {
@@ -200,6 +207,9 @@ const migrations: Migration[] = [
           (item as unknown as Color).colorCode = colorCode;
         }
 
+        item.title = alias || item.title;
+        item.id = getId(item.dateCreated);
+
         delete item.localOnly;
         delete item.noteIds;
         delete item.alias;
@@ -229,9 +239,9 @@ const migrations: Migration[] = [
 
         if (item.color) {
           const oldColorId = makeId(item.color);
-          const oldColor = db.tags.tag(oldColorId);
+          const oldColor = db.colors.color(oldColorId);
           const alias = db.legacySettings.getAlias(oldColorId);
-          const newColor = db.tags.all.find(
+          const newColor = db.colors.all.find(
             (t) => [alias, item.color].includes(t.title) && t.id !== oldColorId
           );
           const newColorId =
@@ -257,6 +267,7 @@ const migrations: Migration[] = [
           }
         }
 
+        delete item.notebooks;
         delete item.tags;
         delete item.color;
         return true;
@@ -296,7 +307,11 @@ const migrations: Migration[] = [
         if (item.trashCleanupInterval)
           await db.settings.setTrashCleanupInterval(item.trashCleanupInterval);
         if (item.defaultNotebook)
-          await db.settings.setDefaultNotebook(item.defaultNotebook);
+          await db.settings.setDefaultNotebook(
+            item.defaultNotebook
+              ? item.defaultNotebook.topic || item.defaultNotebook.id
+              : undefined
+          );
 
         if (item.titleFormat)
           await db.settings.setTitleFormat(item.titleFormat);
@@ -332,15 +347,18 @@ const migrations: Migration[] = [
 
 export async function migrateItem<TItemType extends MigrationItemType>(
   item: MigrationItemMap[TItemType],
-  version: number,
+  itemVersion: number,
+  databaseVersion: number,
   type: TItemType,
   database: Database,
   migrationType: MigrationType
 ) {
-  let migrationStartIndex = migrations.findIndex((m) => m.version === version);
+  let migrationStartIndex = migrations.findIndex(
+    (m) => m.version === itemVersion
+  );
   if (migrationStartIndex <= -1) {
     throw new Error(
-      version > CURRENT_DATABASE_VERSION
+      itemVersion > databaseVersion
         ? `Please update the app to the latest version.`
         : `You seem to be on a very outdated version. Please update the app to the latest version.`
     );
@@ -349,7 +367,7 @@ export async function migrateItem<TItemType extends MigrationItemType>(
   let count = 0;
   for (; migrationStartIndex < migrations.length; ++migrationStartIndex) {
     const migration = migrations[migrationStartIndex];
-    if (migration.version === CURRENT_DATABASE_VERSION) break;
+    if (migration.version === databaseVersion) break;
 
     if (
       migration.items.all &&

packages/core/src/types.ts

@@ -86,7 +86,7 @@ export type GroupableItem = ValueOf<
     | "session"
     | "sessioncontent"
     | "settings"
-    | "settingsv2"
+    | "settingitem"
   >
 >;
 
@@ -324,7 +324,7 @@ export type DefaultNotebook = { id: string; topic?: string };
  */
 export interface LegacySettingsItem extends BaseItem<"settings"> {
   groupOptions?: Partial<Record<GroupingKey, GroupOptions>>;
-  toolbarConfig?: Record<ToolbarConfigPlatforms, ToolbarConfig>;
+  toolbarConfig?: Partial<Record<ToolbarConfigPlatforms, ToolbarConfig>>;
   trashCleanupInterval?: TrashCleanupInterval;
   titleFormat?: string;
   timeFormat?: TimeFormat;
@@ -350,7 +350,7 @@ export type SettingItemMap = {
   titleFormat: string;
   timeFormat: TimeFormat;
   dateFormat: string;
-  defaultNotebook: DefaultNotebook | undefined;
+  defaultNotebook: string | undefined;
 } & Record<`groupOptions:${GroupingKey}`, GroupOptions> &
   Record<`toolbarConfig:${ToolbarConfigPlatforms}`, ToolbarConfig | undefined>;
 
@@ -389,9 +389,7 @@ export function isDeleted<T extends BaseItem<ItemType>>(
   return "deleted" in item;
 }
 
-export function isTrashItem(
-  item: MaybeDeletedItem<TrashOrItem<BaseItem<"note" | "notebook">>>
-): item is TrashItem {
+export function isTrashItem(item: MaybeDeletedItem<Item>): item is TrashItem {
   return !isDeleted(item) && item.type === "trash";
 }