From c5352c2a73176aedd27bf16799a8f34965783df7 Mon Sep 17 00:00:00 2001 From: Abdullah Atta Date: Tue, 12 Sep 2023 15:55:07 +0500 Subject: [PATCH] core: add support for restoring new nnbackupz format --- packages/core/__tests__/backup.test.js | 85 +++++++---- packages/core/src/database/backup.js | 198 ++++++++++++------------- packages/core/src/database/migrator.js | 11 +- 3 files changed, 160 insertions(+), 134 deletions(-) diff --git a/packages/core/__tests__/backup.test.js b/packages/core/__tests__/backup.test.js index 1d4b3c177..9a1357b39 100644 --- a/packages/core/__tests__/backup.test.js +++ b/packages/core/__tests__/backup.test.js @@ -33,10 +33,19 @@ import { test, expect, describe } from "vitest"; test("export backup", () => noteTest().then(() => notebookTest().then(async ({ db }) => { - const exp = await db.backup.export("node"); - let backup = JSON.parse(exp); + const exp = []; + for await (const file of db.backup.export("node", false)) { + exp.push(file); + } + + let backup = JSON.parse(exp[1].data); + expect(exp.length).toBe(2); + expect(exp[0].path).toBe(".nnbackup"); expect(backup.type).toBe("node"); expect(backup.date).toBeGreaterThan(0); + expect(backup.data).toBeTypeOf("string"); + expect(backup.compressed).toBe(true); + expect(backup.encrypted).toBe(false); }) )); @@ -44,19 +53,34 @@ test("export encrypted backup", () => notebookTest().then(async ({ db }) => { await loginFakeUser(db); await db.notes.add(TEST_NOTE); - const exp = await db.backup.export("node", true); - let backup = JSON.parse(exp); + + const exp = []; + for await (const file of db.backup.export("node", true)) { + exp.push(file); + } + + const backup = JSON.parse(exp[1].data); + expect(exp.length).toBe(2); + expect(exp[0].path).toBe(".nnbackup"); expect(backup.type).toBe("node"); expect(backup.date).toBeGreaterThan(0); expect(backup.data.iv).not.toBeUndefined(); + expect(backup.data).toBeTypeOf("object"); + expect(backup.compressed).toBe(true); + expect(backup.encrypted).toBe(true); })); test("import backup", () => notebookTest().then(async ({ db, id }) => { await db.notes.add(TEST_NOTE); - const exp = await db.backup.export("node"); + + const exp = []; + for await (const file of db.backup.export("node", false)) { + exp.push(file); + } + await db.storage.clear(); - await db.backup.import(JSON.parse(exp)); + await db.backup.import(JSON.parse(exp[1].data)); expect(db.notebooks.notebook(id).data.id).toBe(id); })); @@ -64,19 +88,29 @@ test("import encrypted backup", () => notebookTest().then(async ({ db, id }) => { await loginFakeUser(db); await db.notes.add(TEST_NOTE); - const exp = await db.backup.export("node", true); + + const exp = []; + for await (const file of db.backup.export("node", true)) { + exp.push(file); + } + await db.storage.clear(); - await db.backup.import(JSON.parse(exp), "password"); + await db.backup.import(JSON.parse(exp[1].data), "password"); expect(db.notebooks.notebook(id).data.id).toBe(id); })); test("import tempered backup", () => notebookTest().then(async ({ db }) => { await db.notes.add(TEST_NOTE); - const exp = await db.backup.export("node"); + + const exp = []; + for await (const file of db.backup.export("node", false)) { + exp.push(file); + } + await db.storage.clear(); - const backup = JSON.parse(exp); - backup.data.hello = "world"; + const backup = JSON.parse(exp[1].data); + backup.data += "hello"; await expect(db.backup.import(backup)).rejects.toThrow(/tempered/); })); @@ -147,21 +181,20 @@ describe.each([ return databaseTest().then(async (db) => { await db.backup.import(qclone(data)); - verifyIndex(data, db, "notes", "notes"); - verifyIndex(data, db, "notebooks", "notebooks"); - verifyIndex(data, db, "content", "content"); - verifyIndex(data, db, "attachments", "attachments"); - // verifyIndex(data, db, "trash", "trash"); + const keys = await db.storage.getAllKeys(); + for (let key in data.data) { + const item = data.data[key]; + if (item && !item.type && item.deleted) continue; + if ( + key.startsWith("_uk_") || + key === "hasConflicts" || + key === "monographs" || + key === "token" + ) + continue; + + expect(keys.some((k) => k.startsWith(key))).toBeTruthy(); + } }); }); }); - -function verifyIndex(backup, db, backupCollection, collection) { - if (!backup.data[backupCollection]) return; - - expect( - backup.data[backupCollection].every( - (v) => db[collection]._collection.indexer.indices.indexOf(v) > -1 - ) - ).toBeTruthy(); -} diff --git a/packages/core/src/database/backup.js b/packages/core/src/database/backup.js index 7b37ce1b6..597c12867 100644 --- a/packages/core/src/database/backup.js +++ b/packages/core/src/database/backup.js @@ -21,6 +21,8 @@ import SparkMD5 from "spark-md5"; import { CURRENT_DATABASE_VERSION } from "../common.js"; import Migrator from "./migrator.js"; import { toChunks } from "../utils/array.js"; +import { migrateItem } from "../migrations.js"; +import Indexer from "./indexer.js"; const invalidKeys = [ "user", @@ -39,9 +41,30 @@ const invalidKeys = [ "reminders", "sessioncontent", "notehistory", - "shortcuts" + "shortcuts", + "vaultKey", + "hasConflict", + "token", + "monographs" ]; -const invalidIndices = ["tags", "colors"]; + +const itemTypeToCollectionKey = { + note: "notes", + notebook: "notebooks", + tiptap: "content", + tiny: "content", + tag: "tags", + color: "colors", + attachment: "attachments", + relation: "relations", + reminder: "reminders", + sessioncontent: "sessioncontent", + session: "notehistory", + notehistory: "notehistory", + content: "content", + shortcut: "shortcuts" +}; + const validTypes = ["mobile", "web", "node"]; export default class Backup { /** @@ -71,6 +94,11 @@ export default class Backup { if (encrypt && !(await this._db.user.getUser())) throw new Error("Please login to create encrypted backups."); + yield { + path: ".nnbackup", + data: "" + }; + let keys = await this._db.storage.getAllKeys(); const key = await this._db.user.getEncryptionKey(); const chunks = toChunks(keys, 20); @@ -78,8 +106,25 @@ export default class Backup { let bufferLength = 0; const MAX_CHUNK_SIZE = 10 * 1024 * 1024; let chunkIndex = 0; - for (const chunk of chunks) { - if (bufferLength >= MAX_CHUNK_SIZE) { + + while (chunks.length > 0) { + const chunk = chunks.pop(); + + const items = await this._db.storage.readMulti(chunk); + items.forEach(([id, item]) => { + if ( + invalidKeys.includes(id) || + (item.deleted && !item.type) || + id.startsWith("_uk_") + ) + return; + + const data = JSON.stringify(item); + buffer.push(data); + bufferLength += data.length; + }); + + if (bufferLength >= MAX_CHUNK_SIZE || chunks.length === 0) { let itemsJSON = `[${buffer.join(",")}]`; buffer = []; @@ -89,37 +134,26 @@ export default class Backup { const hash = SparkMD5.hash(itemsJSON); - if (encrypt) - itemsJSON = JSON.stringify( - await this._db.storage.encrypt(key, itemsJSON) - ); + if (encrypt) itemsJSON = await this._db.storage.encrypt(key, itemsJSON); yield { path: `${chunkIndex++}-${encrypt ? "encrypted" : "plain"}-${hash}`, data: `{ - "version": ${CURRENT_DATABASE_VERSION}, - "type": "${type}", - "date": ${Date.now()}, - "data": ${itemsJSON}, - "hash": "${hash}", - "hash_type": "md5", +"version": ${CURRENT_DATABASE_VERSION}, +"type": "${type}", +"date": ${Date.now()}, +"data": ${JSON.stringify(itemsJSON)}, +"hash": "${hash}", +"hash_type": "md5", +"compressed": true, +"encrypted": ${encrypt ? "true" : "false"} }` }; } - - const items = await this._db.storage.readMulti(chunk); - items.forEach(([id, item]) => { - if (invalidKeys.includes(id) || item.deleted) return; - const data = JSON.stringify(item); - buffer.push(data); - bufferLength += data.length; - }); } - yield { - path: ".nnbackup", - data: "" - }; + if (bufferLength > 0 || buffer.length > 0) + throw new Error("Buffer not empty."); await this.updateBackupTime(); } @@ -137,7 +171,7 @@ export default class Backup { let db = backup.data; const isEncrypted = db.salt && db.iv && db.cipher; - if (isEncrypted) { + if (backup.encrypted || isEncrypted) { if (!password) throw new Error( "Please provide a password to decrypt this backup & restore it." @@ -148,8 +182,7 @@ export default class Backup { throw new Error("Could not generate encryption key for backup."); try { - const decrypted = await this._db.storage.decrypt(key, db); - backup.data = JSON.parse(decrypted); + backup.data = await this._db.storage.decrypt(key, db); } catch (e) { if ( e.message.includes("ciphertext cannot be decrypted") || @@ -159,9 +192,16 @@ export default class Backup { throw new Error(`Could not decrypt backup: ${e.message}`); } - } else if (!this._verify(backup)) + } + + if (backup.hash && !this._verify(backup)) throw new Error("Backup file has been tempered, aborting..."); + if (backup.compressed) + backup.data = await this._db.compressor.decompress(backup.data); + backup.data = + typeof backup.data === "string" ? JSON.parse(backup.data) : backup.data; + await this._migrateData(backup); } @@ -169,7 +209,7 @@ export default class Backup { const { version = 0 } = backup; if (version > CURRENT_DATABASE_VERSION) throw new Error( - "This backup was made from a newer version of Notesnook. Cannot migrate." + "This backup was made from a newer version of Notesnook. Cannot restore." ); switch (version) { @@ -193,66 +233,32 @@ export default class Backup { async _migrateData(backup) { const { data, version = 0 } = backup; - if (version > CURRENT_DATABASE_VERSION) - throw new Error( - "This backup was made from a newer version of Notesnook. Cannot migrate." - ); + const toAdd = {}; + for (const item of Array.isArray(data) ? data : Object.values(data)) { + // we do not want to restore deleted items + if (!item.type && item.deleted) continue; + // in v5.6 of the database, we did not set note history session's type + if (!item.type && item.sessionContentId) item.type = "notehistory"; - const collections = [ - { - index: () => data["attachments"], - dbCollection: this._db.attachments - }, - { - index: () => data["notebooks"], - dbCollection: this._db.notebooks - }, - { - index: () => data["content"], - dbCollection: this._db.content - }, - { - index: () => data["shortcuts"], - dbCollection: this._db.shortcuts - }, - { - index: () => data["reminders"], - dbCollection: this._db.reminders - }, - { - index: () => data["relations"], - dbCollection: this._db.relations - }, - { - index: () => data["notehistory"], - dbCollection: this._db.noteHistory, - type: "notehistory" - }, - { - index: () => data["sessioncontent"], - dbCollection: this._db.noteHistory.sessionContent, - type: "sessioncontent" - }, - { - index: () => data["notes"], - dbCollection: this._db.notes - }, - { - index: () => ["settings"], - dbCollection: this._db.settings, - type: "settings" - } - ]; + await migrateItem(item, version, item.type, this._db); + // since items in trash can have their own set of migrations, + // we have to run the migration again to account for that. + if (item.type === "trash" && item.itemType) + await migrateItem(item, version, item.itemType, this._db); - await this._db.syncer.acquireLock(async () => { - await this._migrator.migrate( - this._db, - collections, - (id, type) => (version < 5.8 ? data[id] : data[`${id}_${type}`]), - version, - true + const collectionKey = itemTypeToCollectionKey[item.itemType || item.type]; + if (collectionKey) { + toAdd[collectionKey] = toAdd[collectionKey] || []; + toAdd[collectionKey].push([item.id, item]); + } else if (item.type === "settings") + await this._db.storage.write("settings", item); + } + + for (const collectionKey in toAdd) { + await new Indexer(this._db.storage, collectionKey).writeMulti( + toAdd[collectionKey] ); - }); + } } _validate(backup) { @@ -265,10 +271,10 @@ export default class Backup { } _verify(backup) { - const { hash, hash_type, data: db } = backup; + const { compressed, hash, hash_type, data: db } = backup; switch (hash_type) { case "md5": { - return hash === SparkMD5.hash(JSON.stringify(db)); + return hash === SparkMD5.hash(compressed ? db : JSON.stringify(db)); } default: { return false; @@ -276,15 +282,3 @@ export default class Backup { } } } - -function filterData(data) { - let skippedKeys = [...invalidKeys, ...invalidIndices]; - invalidIndices.forEach((key) => { - const index = data[key]; - if (!index) return; - skippedKeys.push(...index); - }); - - skippedKeys.forEach((key) => delete data[key]); - return data; -} diff --git a/packages/core/src/database/migrator.js b/packages/core/src/database/migrator.js index d514d0c1f..352edd533 100644 --- a/packages/core/src/database/migrator.js +++ b/packages/core/src/database/migrator.js @@ -21,7 +21,7 @@ import { sendMigrationProgressEvent } from "../common"; import { migrateCollection, migrateItem } from "../migrations"; class Migrator { - async migrate(db, collections, get, version, restore = false) { + async migrate(db, collections, get, version) { for (let collection of collections) { if ( (!collection.iterate && !collection.index) || @@ -45,21 +45,20 @@ class Migrator { collection, collection.index(), get, - version, - restore + version ); } else if (collection.iterate) { for await (const index of collection.dbCollection._collection.iterate( 100 )) { - await this.migrateItems(db, collection, index, get, version, restore); + await this.migrateItems(db, collection, index, get, version); } } } return true; } - async migrateItems(db, collection, index, get, version, restore) { + async migrateItems(db, collection, index, get, version) { const toAdd = []; for (var i = 0; i < index.length; ++i) { let id = index[i]; @@ -82,7 +81,7 @@ class Migrator { db ); - if (migrated || restore) { + if (migrated) { if (collection.type === "settings") { await collection.dbCollection.merge(item); } else if (item.type === "note") {