core: move index migrations to main database migrations

this is better design wise as we won't have to keep checking if the
indices have been migrated or not. We'll just check the database version
and do the appropriate migrations based on that.

packages/core/api/migrations.js

@@ -42,48 +42,60 @@ class Migrations {
   }
 
   async migrate() {
+    try {
       if (!this.required() || this._isMigrating) return;
       this._isMigrating = true;
 
       await this._db.notes.init();
-    const content = await this._db.content.all();
 
       const collections = [
-      { index: this._db.attachments.all, dbCollection: this._db.attachments },
         {
-        index: this._db.notebooks.raw,
+          index: () => this._db.attachments.all,
+          dbCollection: this._db.attachments
+        },
+        {
+          index: () => this._db.notebooks.raw,
           dbCollection: this._db.notebooks
         },
         {
-        index: this._db.tags.raw,
+          index: () => this._db.tags.raw,
           dbCollection: this._db.tags
         },
         {
-        index: this._db.colors.raw,
+          index: () => this._db.colors.raw,
           dbCollection: this._db.colors
         },
         {
-        index: this._db.trash.raw,
+          index: () => this._db.trash.raw,
           dbCollection: this._db.trash
         },
         {
-        index: content,
+          index: () => this._db.content.all(),
           dbCollection: this._db.content
         },
         {
-        index: [this._db.settings.raw],
+          index: () => [this._db.settings.raw],
           dbCollection: this._db.settings,
           type: "settings"
         },
         {
-        index: this._db.shortcuts.raw,
+          index: () => this._db.shortcuts.raw,
           dbCollection: this._db.shortcuts
         },
         {
-        index: this._db.notes.raw,
+          index: () => this._db.noteHistory.sessionContent.all(),
+          dbCollection: this._db.noteHistory
+        },
+        {
+          index: () => this._db.noteHistory.sessionContent.all(),
+          dbCollection: this._db.noteHistory.sessionContent
+        },
+        {
+          index: () => this._db.notes.raw,
           dbCollection: this._db.notes
         }
       ];
+
       await this._migrator.migrate(
         this._db,
         collections,
@@ -92,6 +104,9 @@ class Migrations {
       );
       await this._db.storage.write("v", CURRENT_DATABASE_VERSION);
       this.dbVersion = CURRENT_DATABASE_VERSION;
+    } finally {
+      this._isMigrating = false;
+    }
   }
 }
 export default Migrations;

packages/core/collections/note-history.js

@@ -51,6 +51,16 @@ export default class NoteHistory extends Collection {
     );
   }
 
+  async merge(item) {
+    await this._collection.addItem(item);
+  }
+
+  async all() {
+    return Object.values(
+      await this._collection.getItems(this._collection.indexer.indices)
+    );
+  }
+
   /**
    * Get complete session history of a note.
    * @param noteId id of the note
@@ -84,6 +94,7 @@ export default class NoteHistory extends Collection {
     let locked = this._db.notes.note(noteId)?.data?.locked;
 
     let session = {
+      type: "session",
       id: sessionId,
       sessionContentId: makeSessionContentId(sessionId),
       noteId,
@@ -180,14 +191,14 @@ export default class NoteHistory extends Collection {
       await this._db.content.add({
         id: note.contentId,
         data: content.data,
-        type: content.type
+        type: content.contentType
       });
     } else {
       await this._db.notes.add({
         id: session.noteId,
         content: {
           data: content.data,
-          type: content.type
+          type: content.contentType
         }
       });
     }

packages/core/collections/session-content.js

@@ -17,11 +17,16 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+import { tinyToTiptap } from "../migrations";
 import { compress, decompress } from "../utils/compression";
 import { makeSessionContentId } from "../utils/id";
 import Collection from "./collection";
 
 export default class SessionContent extends Collection {
+  async merge(item) {
+    await this._collection.addItem(item);
+  }
+
   /**
    *
    * @param {string} sessionId
@@ -29,13 +34,14 @@ export default class SessionContent extends Collection {
    */
   async add(sessionId, content, locked) {
     if (!sessionId || !content) return;
-    let compressed = locked ? null : compress(content.data);
+    let data = locked ? content.data : compress(content.data);
 
     await this._collection.addItem({
+      type: "sessioncontent",
       id: makeSessionContentId(sessionId),
-      data: compressed || content.data,
-      type: content.type,
-      compressed: !!compressed,
+      data,
+      contentType: content.type,
+      compressed: !locked,
       localOnly: true,
       locked
     });
@@ -49,9 +55,16 @@ export default class SessionContent extends Collection {
   async get(sessionContentId) {
     if (!sessionContentId) return;
     let session = await this._collection.getItem(sessionContentId);
+
+    if (session.contentType === "tiny" && session.compressed) {
+      session.compressed = compress(tinyToTiptap(decompress(session.data)));
+      session.contentType = "tiptap";
+      await this._collection.addItem(session);
+    }
+
     return {
       data: session.compressed ? decompress(session.data) : session.data,
-      type: session.type
+      type: session.contentType
     };
   }
 
@@ -64,7 +77,7 @@ export default class SessionContent extends Collection {
   }
 
   async all() {
-    let indices = await this._collection.indexer.getIndices();
+    let indices = this._collection.indexer.getIndices();
     let items = await this._collection.getItems(indices);
 
     return Object.values(items);

packages/core/database/backup.js

@@ -154,32 +154,39 @@ export default class Backup {
         "This backup was made from a newer version of Notesnook. Cannot migrate."
       );
 
-    // we have to reindex to make sure we handle all the items
-    // properly.
-    reindex(data);
     const collections = [
       {
-        index: data["attachments"],
+        index: () => data["attachments"],
         dbCollection: this._db.attachments
       },
       {
-        index: data["notebooks"],
+        index: () => data["notebooks"],
         dbCollection: this._db.notebooks
       },
       {
-        index: data["content"],
+        index: () => data["content"],
         dbCollection: this._db.content
       },
       {
-        index: data["shortcuts"],
+        index: () => data["shortcuts"],
         dbCollection: this._db.shortcuts
       },
       {
-        index: data["notes"],
+        index: () => data["notehistory"],
+        dbCollection: this._db.noteHistory,
+        type: "notehistory"
+      },
+      {
+        index: () => data["sessioncontent"],
+        dbCollection: this._db.noteHistory.sessionContent,
+        type: "sessioncontent"
+      },
+      {
+        index: () => data["notes"],
         dbCollection: this._db.notes
       },
       {
-        index: ["settings"],
+        index: () => ["settings"],
         dbCollection: this._db.settings,
         type: "settings"
       }

packages/core/database/indexer.js

@@ -28,7 +28,6 @@ export default class Indexer extends Storage {
 
   async init() {
     this.indices = (await super.read(this.type, true)) || [];
-    await this.migrateIndices();
   }
 
   exists(key) {
@@ -89,7 +88,7 @@ export default class Indexer extends Storage {
 
     // remove old ids once they have been moved
     for (const id of keys) {
-      await this.remove(id);
+      await super.remove(id);
     }
   }
 

packages/core/database/migrator.js

@@ -17,14 +17,18 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { migrateItem } from "../migrations";
+import { migrateCollection, migrateItem } from "../migrations";
 
 class Migrator {
   async migrate(db, collections, get, version) {
     for (let collection of collections) {
       if (!collection.index || !collection.dbCollection) continue;
-      for (var i = 0; i < collection.index.length; ++i) {
-        let id = collection.index[i];
+
+      await migrateCollection(collection.dbCollection, version);
+
+      const index = (await collection.index()) || [];
+      for (var i = 0; i < index.length; ++i) {
+        let id = index[i];
         let item = get(id);
         if (!item) {
           continue;
@@ -35,19 +39,24 @@ class Migrator {
           await collection.dbCollection?._collection?.addItem(item);
           continue;
         }
-
+        const itemId = item.id;
         item = await migrateItem(
           item,
           version,
-          item.type || collection.type,
+          item.type || collection.type || collection.dbCollection.type,
           db
         );
 
         if (collection.dbCollection.merge) {
           await collection.dbCollection.merge(item);
-        } else {
+        } else if (collection.dbCollection.add) {
           await collection.dbCollection.add(item);
         }
+
+        // if id changed after migration, we need to delete the old one.
+        if (item.id !== itemId) {
+          await collection.dbCollection?._collection?.deleteItem(itemId);
+        }
       }
     }
     return true;

packages/core/migrations.js

@@ -104,12 +104,25 @@ const migrations = [
       tiny: (item) => {
         if (!item.data || item.data.iv) return item;
         item.type = "tiptap";
-        return item;
+        return changeSessionContentType(item);
       },
       content: (item) => {
         if (!item.data || item.data.iv) return item;
         item.type = "tiptap";
         return item;
+    },
+      tiptap: (item) => {
+        return changeSessionContentType(item);
+      },
+      notehistory: (item) => {
+        item.type = "session";
+        return item;
+      }
+    },
+    collection: async (collection) => {
+      if (collection._collection) {
+        const indexer = collection._collection.indexer;
+        await indexer.migrateIndices();
       }
     }
   },
@@ -137,6 +150,28 @@ export async function migrateItem(item, version, type, database) {
   return item;
 }
 
+export async function migrateCollection(collection, version) {
+  let migrationStartIndex = migrations.findIndex((m) => m.version === version);
+  if (migrationStartIndex <= -1) {
+    throw new Error(
+      version > CURRENT_DATABASE_VERSION
+        ? `Please update the app to the latest version.`
+        : `You seem to be on a very outdated version. Please update the app to the latest version.`
+    );
+  }
+
+  for (; migrationStartIndex < migrations.length; ++migrationStartIndex) {
+    const migration = migrations[migrationStartIndex];
+    if (migration.version === CURRENT_DATABASE_VERSION) break;
+
+    if (!migration.collection) continue;
+    await migration.collection(collection);
+
+    if (collection._collection && collection._collection.init)
+      await collection._collection.init();
+  }
+}
+
 function replaceDateEditedWithDateModified(removeDateEditedProperty = false) {
   return function (item) {
     item.dateModified = item.dateEdited;
@@ -235,3 +270,11 @@ export function tinyToTiptap(html) {
 
   return document.body.innerHTML;
 }
+
+function changeSessionContentType(item) {
+  if (item.id.endsWith("_content")) {
+    item.contentType = item.type;
+    item.type = "sessioncontent";
+  }
+  return item;
+}