core: add migrations for notebooks & settings (#1020)

2025-12-23 23:19:40 +01:00 · 2022-09-21 11:47:09 +05:00
parent 6c613c6f1e
commit 7c99c3acb9
9 changed files with 244 additions and 16 deletions
--- a/packages/core/tests/fixtures/backup.v5.6.json
+++ b/packages/core/tests/fixtures/backup.v5.6.json
@@ -0,0 +1,180 @@
+{
+  "version": 5.6,
+  "type": "web",
+  "date": 1663740570849,
+  "data": {
+    "632aaa64bdbdbd6fde58585a": {
+      "id": "632aaa64bdbdbd6fde58585a",
+      "contentId": "632aaa67bdbdbd6fde58585b",
+      "type": "note",
+      "title": "note",
+      "headline": "note 2",
+      "tags": [],
+      "notebooks": [
+        {
+          "id": "632aaa6fbdbdbd6fde58585c",
+          "topics": ["632aaa7cbdbdbd6fde585865"]
+        }
+      ],
+      "pinned": false,
+      "locked": false,
+      "favorite": false,
+      "localOnly": false,
+      "conflicted": false,
+      "readonly": false,
+      "dateCreated": 1663740516168,
+      "dateEdited": 1663740519352,
+      "dateModified": 1663740547431,
+      "synced": false
+    },
+    "632aaa64bdbdbd6fde58585a_1663740501150": {
+      "id": "632aaa64bdbdbd6fde58585a_1663740501150",
+      "sessionContentId": "632aaa64bdbdbd6fde58585a_1663740501150_content",
+      "noteId": "632aaa64bdbdbd6fde58585a",
+      "dateCreated": 1663740519346,
+      "localOnly": true,
+      "dateModified": 1663740519346,
+      "synced": false
+    },
+    "632aaa64bdbdbd6fde58585a_1663740501150_content": {
+      "id": "632aaa64bdbdbd6fde58585a_1663740501150_content",
+      "data": "H4sIAGeqKmMAAwUAQQoAEOwra3cpZ/xlTFKyNfN/ZQVeVnC7ITA5havU15kFWV7bA+sRH5By1PoBfnNzbi0AAAA=",
+      "type": "tiptap",
+      "compressed": true,
+      "localOnly": true,
+      "dateCreated": 1663740519350,
+      "dateModified": 1663740519350,
+      "synced": false
+    },
+    "632aaa67bdbdbd6fde58585b": {
+      "noteId": "632aaa64bdbdbd6fde58585a",
+      "id": "632aaa67bdbdbd6fde58585b",
+      "type": "tiptap",
+      "data": "<p dir=\"ltr\" data-spacing=\"double\">note 2</p>",
+      "dateCreated": 1663740519342,
+      "dateModified": 1663740519342,
+      "localOnly": false,
+      "synced": false
+    },
+    "632aaa6fbdbdbd6fde58585c": {
+      "id": "632aaa6fbdbdbd6fde58585c",
+      "type": "notebook",
+      "title": "1241",
+      "description": "214",
+      "pinned": false,
+      "topics": [
+        {
+          "type": "topic",
+          "id": "632aaa7cbdbdbd6fde585865",
+          "notebookId": "632aaa6fbdbdbd6fde58585c",
+          "title": "12412412412",
+          "dateCreated": 1663740540759,
+          "dateEdited": 1663740547435,
+          "notes": ["632aaa64bdbdbd6fde58585a"]
+        },
+        {
+          "type": "topic",
+          "id": "632aaa7bbdbdbd6fde585864",
+          "notebookId": "632aaa6fbdbdbd6fde58585c",
+          "title": "124124",
+          "dateCreated": 1663740539034,
+          "dateEdited": 1663740539034,
+          "notes": []
+        }
+      ],
+      "dateCreated": 1663740527478,
+      "dateModified": 1663740547435,
+      "dateEdited": 1663740547435,
+      "synced": false
+    },
+    "632aaa77bdbdbd6fde58585d": {
+      "id": "632aaa77bdbdbd6fde58585d",
+      "type": "notebook",
+      "title": "124124",
+      "description": "12412412",
+      "pinned": false,
+      "topics": [
+        {
+          "type": "topic",
+          "id": "632aaa77bdbdbd6fde58585e",
+          "notebookId": "632aaa77bdbdbd6fde58585d",
+          "title": "21412412412",
+          "dateCreated": 1663740535717,
+          "dateEdited": 1663740555660,
+          "notes": ["632aaa86bdbdbd6fde585866"]
+        },
+        {
+          "type": "topic",
+          "id": "632aaa77bdbdbd6fde58585f",
+          "notebookId": "632aaa77bdbdbd6fde58585d",
+          "title": "1124124124",
+          "dateCreated": 1663740535717,
+          "dateEdited": 1663740535717,
+          "notes": []
+        },
+        {
+          "type": "topic",
+          "id": "632aaa77bdbdbd6fde585860",
+          "notebookId": "632aaa77bdbdbd6fde58585d",
+          "title": "214124124",
+          "dateCreated": 1663740535717,
+          "dateEdited": 1663740535717,
+          "notes": []
+        }
+      ],
+      "dateCreated": 1663740535713,
+      "dateModified": 1663740555660,
+      "dateEdited": 1663740555660,
+      "synced": false
+    },
+    "632aaa86bdbdbd6fde585866": {
+      "id": "632aaa86bdbdbd6fde585866",
+      "type": "note",
+      "title": "12412",
+      "tags": [],
+      "notebooks": [
+        {
+          "id": "632aaa77bdbdbd6fde58585d",
+          "topics": ["632aaa77bdbdbd6fde58585e"]
+        }
+      ],
+      "pinned": false,
+      "locked": false,
+      "favorite": false,
+      "localOnly": false,
+      "conflicted": false,
+      "readonly": false,
+      "dateCreated": 1663740550136,
+      "dateEdited": 1663740550376,
+      "dateModified": 1663740555650,
+      "synced": false
+    },
+    "content": ["632aaa67bdbdbd6fde58585b"],
+    "notebooks": ["632aaa6fbdbdbd6fde58585c", "632aaa77bdbdbd6fde58585d"],
+    "notehistory": ["632aaa64bdbdbd6fde58585a_1663740501150"],
+    "notes": ["632aaa64bdbdbd6fde58585a", "632aaa86bdbdbd6fde585866"],
+    "sessioncontent": ["632aaa64bdbdbd6fde58585a_1663740501150_content"],
+    "settings": {
+      "type": "settings",
+      "id": "632aaa4e606060f1e0141415",
+      "pins": [
+        {
+          "type": "topic",
+          "data": {
+            "id": "632aaa77bdbdbd6fde58585e",
+            "notebookId": "632aaa77bdbdbd6fde58585d"
+          }
+        },
+        { "type": "notebook", "data": { "id": "632aaa77bdbdbd6fde58585d" } }
+      ],
+      "groupOptions": {},
+      "toolbarConfig": {},
+      "aliases": {},
+      "dateModified": 1663740567578,
+      "dateCreated": 0,
+      "synced": false
+    }
+  },
+  "hash": "28b0b1c1711fc4b5f3eea054ce98b030",
+  "hash_type": "md5"
+}
--- a/packages/core/tests/backup.test.js
+++ b/packages/core/tests/backup.test.js
@@ -25,6 +25,7 @@ import {
 } from "./utils";
 import v52Backup from "./__fixtures__/backup.v5.2.json";
 import v52BackupCopy from "./__fixtures__/backup.v5.2.copy.json";
+import v56BackupCopy from "./__fixtures__/backup.v5.6.json";
 import { qclone } from "qclone";

 beforeEach(() => {
@@ -85,7 +86,8 @@ test("import tempered backup", () =>

 describe.each([
  ["v5.2", v52Backup],
-  ["v5.2 copy", v52BackupCopy]
+  ["v5.2 copy", v52BackupCopy],
+  ["v5.6", v56BackupCopy]
 ])("testing backup version: %s", (version, data) => {
  test(`import ${version} backup`, () => {
    return databaseTest().then(async (db) => {
@@ -94,10 +96,11 @@ describe.each([
      expect(db.settings.raw.id).toBeDefined();
      expect(db.settings.raw.dateModified).toBeDefined();
      expect(db.settings.raw.dateEdited).toBeUndefined();
+      expect(db.settings.raw.pins).toBeUndefined();

      expect(
        db.notes.all.every((v) => {
-          const doesNotHaveContent = v.contentId && !v.content;
+          const doesNotHaveContent = !v.content;
          const doesNotHaveColors = !v.colors && (!v.color || v.color.length);
          const hasTopicsInAllNotebooks =
            !v.notebooks ||
@@ -117,10 +120,16 @@ describe.each([
        db.notebooks.all.every((v) => v.title != null && v.dateModified > 0)
      ).toBeTruthy();

+      expect(
+        db.notebooks.all.every((v) => v.topics.every((topic) => !topic.notes))
+      ).toBeTruthy();
+
      expect(
        db.attachments.all.every((v) => v.dateModified > 0 && !v.dateEdited)
      ).toBeTruthy();

+      expect(db.shortcuts.all).toHaveLength(data.data.settings.pins.length);
+
      const allContent = await db.content.all();
      expect(
        allContent.every((v) => v.type === "tiptap" || v.deleted)
--- a/packages/core/api/migrations.js
+++ b/packages/core/api/migrations.js
@@ -84,7 +84,12 @@ class Migrations {
        dbCollection: this._db.notes
      }
    ];
-    await this._migrator.migrate(collections, (item) => item, this.dbVersion);
+    await this._migrator.migrate(
+      this._db,
+      collections,
+      (item) => item,
+      this.dbVersion
+    );
    await this._db.storage.write("v", CURRENT_DATABASE_VERSION);
    this.dbVersion = CURRENT_DATABASE_VERSION;
  }
--- a/packages/core/api/sync/merger.js
+++ b/packages/core/api/sync/merger.js
@@ -125,7 +125,7 @@ class Merger {
    };
  }

-  _migrate(deserialized, version) {
+  async _migrate(deserialized, version) {
    // it is a locked note, bail out.
    if (deserialized.alg && deserialized.cipher) return deserialized;

@@ -146,7 +146,7 @@ class Merger {

    const migrate = migrations[version][type];
    if (migrate) {
-      return migrate(deserialized);
+      return await migrate(deserialized, this._db);
    }
    return deserialized;
  }
@@ -161,7 +161,7 @@ class Merger {
    deserialized.remote = true;
    deserialized.synced = true;
    if (!migrate) return deserialized;
-    return this._migrate(deserialized, item.v);
+    return await this._migrate(deserialized, item.v);
  }

  async _mergeItem(remoteItem, get, add) {
--- a/packages/core/collections/shortcuts.js
+++ b/packages/core/collections/shortcuts.js
@@ -139,10 +139,10 @@ export default class Shortcuts extends Collection {
  }

  async remove(...shortcutIds) {
-    const shortcuts = this.all.filter((shortcut) =>
-      shortcutIds.includes(
-        shortcut.item.id || shortcutIds.includes(shortcut.id)
-      )
+    const shortcuts = this.all.filter(
+      (shortcut) =>
+        shortcutIds.includes(shortcut.item.id) ||
+        shortcutIds.includes(shortcut.id)
    );
    for (const { id } of shortcuts) {
      await this._collection.removeItem(id);
--- a/packages/core/common.js
+++ b/packages/core/common.js
@@ -83,4 +83,4 @@ export const EVENTS = {
  systemTimeInvalid: "system:invalidTime"
 };

-export const CURRENT_DATABASE_VERSION = 5.6;
+export const CURRENT_DATABASE_VERSION = 5.7;
--- a/packages/core/database/backup.js
+++ b/packages/core/database/backup.js
@@ -132,6 +132,7 @@ export default class Backup {

    switch (version) {
      case CURRENT_DATABASE_VERSION:
+      case 5.6:
      case 5.5:
      case 5.4:
      case 5.3:
@@ -185,7 +186,12 @@ export default class Backup {
    ];

    await this._db.syncer.acquireLock(async () => {
-      await this._migrator.migrate(collections, (id) => data[id], version);
+      await this._migrator.migrate(
+        this._db,
+        collections,
+        (id) => data[id],
+        version
+      );
    });
  }

--- a/packages/core/database/migrator.js
+++ b/packages/core/database/migrator.js
@@ -20,7 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { migrations } from "../migrations";

 class Migrator {
-  async migrate(collections, get, version) {
+  async migrate(db, collections, get, version) {
    for (let collection of collections) {
      if (!collection.index || !collection.dbCollection) continue;
      for (var i = 0; i < collection.index.length; ++i) {
@@ -35,7 +35,7 @@ class Migrator {
          continue;
        }
        const migrate = migrations[version][item.type || collection.type];
-        if (migrate) item = migrate(item);
+        if (migrate) item = await migrate(item, db);

        if (collection.dbCollection.merge) {
          await collection.dbCollection.merge(item);
--- a/packages/core/migrations.js
+++ b/packages/core/migrations.js
@@ -25,7 +25,10 @@ export const migrations = {
  5.1: {},
  5.2: {
    note: replaceDateEditedWithDateModified(),
-    notebook: replaceDateEditedWithDateModified(),
+    notebook: (item) => {
+      item = replaceDateEditedWithDateModified()(item);
+      return migrations["5.6"].notebook(item);
+    },
    tag: replaceDateEditedWithDateModified(true),
    attachment: replaceDateEditedWithDateModified(true),
    trash: replaceDateEditedWithDateModified(),
@@ -37,7 +40,10 @@ export const migrations = {
      item.data = removeToxClassFromChecklist(wrapTablesWithDiv(item.data));
      return migrations["5.3"].tiny(item);
    },
-    settings: replaceDateEditedWithDateModified(true)
+    settings: (item) => {
+      item = replaceDateEditedWithDateModified(true)(item);
+      return migrations["5.6"].settings(item);
+    }
  },
  5.3: {
    tiny: (item) => {
@@ -56,6 +62,28 @@ export const migrations = {
  },
  5.5: {},
  5.6: {
+    notebook: (item) => {
+      item.topics = item.topics.map((topic) => {
+        delete topic.notes;
+        return topic;
+      });
+      return item;
+    },
+    settings: async (item, db) => {
+      for (const pin of item.pins) {
+        await db.shortcuts.add({
+          item: {
+            type: pin.type,
+            id: pin.data.id,
+            notebookId: pin.data.notebookId
+          }
+        });
+      }
+      delete item.pins;
+      return item;
+    }
+  },
+  5.7: {
    note: false,
    shortcut: false,
    notebook: false,