Mirrors/notesnook
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook.git synced 2026-02-24 04:00:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

core: add support for restoring new nnbackupz format

Browse Source
This commit is contained in:
Abdullah Atta
2023-09-12 15:55:07 +05:00
committed by Abdullah Atta
parent dd6410c33e
commit c5352c2a73
3 changed files with 160 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
packages/core/__tests__/backup.test.js
View File

@@ -33,10 +33,19 @@ import { test, expect, describe } from "vitest";
test("export backup", () =>
noteTest().then(() =>
notebookTest().then(async ({ db }) => {
const exp = await db.backup.export("node");
let backup = JSON.parse(exp);
const exp = [];
for await (const file of db.backup.export("node", false)) {
exp.push(file);
}
let backup = JSON.parse(exp[1].data);
expect(exp.length).toBe(2);
expect(exp[0].path).toBe(".nnbackup");
expect(backup.type).toBe("node");
expect(backup.date).toBeGreaterThan(0);
expect(backup.data).toBeTypeOf("string");
expect(backup.compressed).toBe(true);
expect(backup.encrypted).toBe(false);
})
));
@@ -44,19 +53,34 @@ test("export encrypted backup", () =>
notebookTest().then(async ({ db }) => {
await loginFakeUser(db);
await db.notes.add(TEST_NOTE);
const exp = await db.backup.export("node", true);
let backup = JSON.parse(exp);
const exp = [];
for await (const file of db.backup.export("node", true)) {
exp.push(file);
}
const backup = JSON.parse(exp[1].data);
expect(exp.length).toBe(2);
expect(exp[0].path).toBe(".nnbackup");
expect(backup.type).toBe("node");
expect(backup.date).toBeGreaterThan(0);
expect(backup.data.iv).not.toBeUndefined();
expect(backup.data).toBeTypeOf("object");
expect(backup.compressed).toBe(true);
expect(backup.encrypted).toBe(true);
}));
test("import backup", () =>
notebookTest().then(async ({ db, id }) => {
await db.notes.add(TEST_NOTE);
const exp = await db.backup.export("node");
const exp = [];
for await (const file of db.backup.export("node", false)) {
exp.push(file);
}
await db.storage.clear();
await db.backup.import(JSON.parse(exp));
await db.backup.import(JSON.parse(exp[1].data));
expect(db.notebooks.notebook(id).data.id).toBe(id);
}));
@@ -64,19 +88,29 @@ test("import encrypted backup", () =>
notebookTest().then(async ({ db, id }) => {
await loginFakeUser(db);
await db.notes.add(TEST_NOTE);
const exp = await db.backup.export("node", true);
const exp = [];
for await (const file of db.backup.export("node", true)) {
exp.push(file);
}
await db.storage.clear();
await db.backup.import(JSON.parse(exp), "password");
await db.backup.import(JSON.parse(exp[1].data), "password");
expect(db.notebooks.notebook(id).data.id).toBe(id);
}));
test("import tempered backup", () =>
notebookTest().then(async ({ db }) => {
await db.notes.add(TEST_NOTE);
const exp = await db.backup.export("node");
const exp = [];
for await (const file of db.backup.export("node", false)) {
exp.push(file);
}
await db.storage.clear();
const backup = JSON.parse(exp);
backup.data.hello = "world";
const backup = JSON.parse(exp[1].data);
backup.data += "hello";
await expect(db.backup.import(backup)).rejects.toThrow(/tempered/);
}));
@@ -147,21 +181,20 @@ describe.each([
return databaseTest().then(async (db) => {
await db.backup.import(qclone(data));
verifyIndex(data, db, "notes", "notes");
verifyIndex(data, db, "notebooks", "notebooks");
verifyIndex(data, db, "content", "content");
verifyIndex(data, db, "attachments", "attachments");
// verifyIndex(data, db, "trash", "trash");
const keys = await db.storage.getAllKeys();
for (let key in data.data) {
const item = data.data[key];
if (item && !item.type && item.deleted) continue;
if (
key.startsWith("_uk_") ||
key === "hasConflicts" ||
key === "monographs" ||
key === "token"
)
continue;
expect(keys.some((k) => k.startsWith(key))).toBeTruthy();
}
});
});
});
function verifyIndex(backup, db, backupCollection, collection) {
if (!backup.data[backupCollection]) return;
expect(
backup.data[backupCollection].every(
(v) => db[collection]._collection.indexer.indices.indexOf(v) > -1
)
).toBeTruthy();
}

198
packages/core/src/database/backup.js
View File

@@ -21,6 +21,8 @@ import SparkMD5 from "spark-md5";
import { CURRENT_DATABASE_VERSION } from "../common.js";
import Migrator from "./migrator.js";
import { toChunks } from "../utils/array.js";
import { migrateItem } from "../migrations.js";
import Indexer from "./indexer.js";
const invalidKeys = [
"user",
@@ -39,9 +41,30 @@ const invalidKeys = [
"reminders",
"sessioncontent",
"notehistory",
"shortcuts"
"shortcuts",
"vaultKey",
"hasConflict",
"token",
"monographs"
];
const invalidIndices = ["tags", "colors"];
const itemTypeToCollectionKey = {
note: "notes",
notebook: "notebooks",
tiptap: "content",
tiny: "content",
tag: "tags",
color: "colors",
attachment: "attachments",
relation: "relations",
reminder: "reminders",
sessioncontent: "sessioncontent",
session: "notehistory",
notehistory: "notehistory",
content: "content",
shortcut: "shortcuts"
};
const validTypes = ["mobile", "web", "node"];
export default class Backup {
/**
@@ -71,6 +94,11 @@ export default class Backup {
if (encrypt && !(await this._db.user.getUser()))
throw new Error("Please login to create encrypted backups.");
yield {
path: ".nnbackup",
data: ""
};
let keys = await this._db.storage.getAllKeys();
const key = await this._db.user.getEncryptionKey();
const chunks = toChunks(keys, 20);
@@ -78,8 +106,25 @@ export default class Backup {
let bufferLength = 0;
const MAX_CHUNK_SIZE = 10 * 1024 * 1024;
let chunkIndex = 0;
for (const chunk of chunks) {
if (bufferLength >= MAX_CHUNK_SIZE) {
while (chunks.length > 0) {
const chunk = chunks.pop();
const items = await this._db.storage.readMulti(chunk);
items.forEach(([id, item]) => {
if (
invalidKeys.includes(id) ||
(item.deleted && !item.type) ||
id.startsWith("_uk_")
)
return;
const data = JSON.stringify(item);
buffer.push(data);
bufferLength += data.length;
});
if (bufferLength >= MAX_CHUNK_SIZE || chunks.length === 0) {
let itemsJSON = `[${buffer.join(",")}]`;
buffer = [];
@@ -89,37 +134,26 @@ export default class Backup {
const hash = SparkMD5.hash(itemsJSON);
if (encrypt)
itemsJSON = JSON.stringify(
await this._db.storage.encrypt(key, itemsJSON)
);
if (encrypt) itemsJSON = await this._db.storage.encrypt(key, itemsJSON);
yield {
path: `${chunkIndex++}-${encrypt ? "encrypted" : "plain"}-${hash}`,
data: `{
"version": ${CURRENT_DATABASE_VERSION},
"type": "${type}",
"date": ${Date.now()},
"data": ${itemsJSON},
"hash": "${hash}",
"hash_type": "md5",
"version": ${CURRENT_DATABASE_VERSION},
"type": "${type}",
"date": ${Date.now()},
"data": ${JSON.stringify(itemsJSON)},
"hash": "${hash}",
"hash_type": "md5",
"compressed": true,
"encrypted": ${encrypt ? "true" : "false"}
}`
};
}
const items = await this._db.storage.readMulti(chunk);
items.forEach(([id, item]) => {
if (invalidKeys.includes(id) || item.deleted) return;
const data = JSON.stringify(item);
buffer.push(data);
bufferLength += data.length;
});
}
yield {
path: ".nnbackup",
data: ""
};
if (bufferLength > 0 || buffer.length > 0)
throw new Error("Buffer not empty.");
await this.updateBackupTime();
}
@@ -137,7 +171,7 @@ export default class Backup {
let db = backup.data;
const isEncrypted = db.salt && db.iv && db.cipher;
if (isEncrypted) {
if (backup.encrypted || isEncrypted) {
if (!password)
throw new Error(
"Please provide a password to decrypt this backup & restore it."
@@ -148,8 +182,7 @@ export default class Backup {
throw new Error("Could not generate encryption key for backup.");
try {
const decrypted = await this._db.storage.decrypt(key, db);
backup.data = JSON.parse(decrypted);
backup.data = await this._db.storage.decrypt(key, db);
} catch (e) {
if (
e.message.includes("ciphertext cannot be decrypted") ||
@@ -159,9 +192,16 @@ export default class Backup {
throw new Error(`Could not decrypt backup: ${e.message}`);
}
} else if (!this._verify(backup))
}
if (backup.hash && !this._verify(backup))
throw new Error("Backup file has been tempered, aborting...");
if (backup.compressed)
backup.data = await this._db.compressor.decompress(backup.data);
backup.data =
typeof backup.data === "string" ? JSON.parse(backup.data) : backup.data;
await this._migrateData(backup);
}
@@ -169,7 +209,7 @@ export default class Backup {
const { version = 0 } = backup;
if (version > CURRENT_DATABASE_VERSION)
throw new Error(
"This backup was made from a newer version of Notesnook. Cannot migrate."
"This backup was made from a newer version of Notesnook. Cannot restore."
);
switch (version) {
@@ -193,66 +233,32 @@ export default class Backup {
async _migrateData(backup) {
const { data, version = 0 } = backup;
if (version > CURRENT_DATABASE_VERSION)
throw new Error(
"This backup was made from a newer version of Notesnook. Cannot migrate."
);
const toAdd = {};
for (const item of Array.isArray(data) ? data : Object.values(data)) {
// we do not want to restore deleted items
if (!item.type && item.deleted) continue;
// in v5.6 of the database, we did not set note history session's type
if (!item.type && item.sessionContentId) item.type = "notehistory";
const collections = [
{
index: () => data["attachments"],
dbCollection: this._db.attachments
},
{
index: () => data["notebooks"],
dbCollection: this._db.notebooks
},
{
index: () => data["content"],
dbCollection: this._db.content
},
{
index: () => data["shortcuts"],
dbCollection: this._db.shortcuts
},
{
index: () => data["reminders"],
dbCollection: this._db.reminders
},
{
index: () => data["relations"],
dbCollection: this._db.relations
},
{
index: () => data["notehistory"],
dbCollection: this._db.noteHistory,
type: "notehistory"
},
{
index: () => data["sessioncontent"],
dbCollection: this._db.noteHistory.sessionContent,
type: "sessioncontent"
},
{
index: () => data["notes"],
dbCollection: this._db.notes
},
{
index: () => ["settings"],
dbCollection: this._db.settings,
type: "settings"
}
];
await migrateItem(item, version, item.type, this._db);
// since items in trash can have their own set of migrations,
// we have to run the migration again to account for that.
if (item.type === "trash" && item.itemType)
await migrateItem(item, version, item.itemType, this._db);
await this._db.syncer.acquireLock(async () => {
await this._migrator.migrate(
this._db,
collections,
(id, type) => (version < 5.8 ? data[id] : data[`${id}_${type}`]),
version,
true
const collectionKey = itemTypeToCollectionKey[item.itemType || item.type];
if (collectionKey) {
toAdd[collectionKey] = toAdd[collectionKey] || [];
toAdd[collectionKey].push([item.id, item]);
} else if (item.type === "settings")
await this._db.storage.write("settings", item);
}
for (const collectionKey in toAdd) {
await new Indexer(this._db.storage, collectionKey).writeMulti(
toAdd[collectionKey]
);
});
}
}
_validate(backup) {
@@ -265,10 +271,10 @@ export default class Backup {
}
_verify(backup) {
const { hash, hash_type, data: db } = backup;
const { compressed, hash, hash_type, data: db } = backup;
switch (hash_type) {
case "md5": {
return hash === SparkMD5.hash(JSON.stringify(db));
return hash === SparkMD5.hash(compressed ? db : JSON.stringify(db));
}
default: {
return false;
@@ -276,15 +282,3 @@ export default class Backup {
}
}
}
function filterData(data) {
let skippedKeys = [...invalidKeys, ...invalidIndices];
invalidIndices.forEach((key) => {
const index = data[key];
if (!index) return;
skippedKeys.push(...index);
});
skippedKeys.forEach((key) => delete data[key]);
return data;
}

11
packages/core/src/database/migrator.js
View File

@@ -21,7 +21,7 @@ import { sendMigrationProgressEvent } from "../common";
import { migrateCollection, migrateItem } from "../migrations";
class Migrator {
async migrate(db, collections, get, version, restore = false) {
async migrate(db, collections, get, version) {
for (let collection of collections) {
if (
(!collection.iterate && !collection.index) ||
@@ -45,21 +45,20 @@ class Migrator {
collection,
collection.index(),
get,
version,
restore
version
);
} else if (collection.iterate) {
for await (const index of collection.dbCollection._collection.iterate(
100
)) {
await this.migrateItems(db, collection, index, get, version, restore);
await this.migrateItems(db, collection, index, get, version);
}
}
}
return true;
}
async migrateItems(db, collection, index, get, version, restore) {
async migrateItems(db, collection, index, get, version) {
const toAdd = [];
for (var i = 0; i < index.length; ++i) {
let id = index[i];
@@ -82,7 +81,7 @@ class Migrator {
db
);
if (migrated || restore) {
if (migrated) {
if (collection.type === "settings") {
await collection.dbCollection.merge(item);
} else if (item.type === "note") {