Mirrors/notesnook
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook.git synced 2026-09-01 19:49:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

core: improve backup importing

Browse Source
This commit is contained in:
Abdullah Atta
2024-02-10 11:00:15 +05:00
parent 4b916dff47
commit 0287257905
10 changed files with 125 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
packages/core/__mocks__/node-storage.mock.ts
View File

@@ -85,14 +85,11 @@ export class NodeStorageInterface implements IStorage {
return this.crypto.decryptMulti(key, items, "text");
}
async deriveCryptoKey(
name: string,
credentials: SerializedKey
): Promise<void> {
async deriveCryptoKey(credentials: SerializedKey): Promise<void> {
const { password, salt } = credentials;
if (!password || !salt) return;
const keyData = await this.crypto.exportKey(password, salt);
await this.write(`${name}@_k`, keyData.key);
await this.write(`userEncryptionKey`, keyData.key);
}
async hash(password: string, email: string): Promise<string> {
@@ -100,8 +97,8 @@ export class NodeStorageInterface implements IStorage {
return await this.crypto.hash(password, `${APP_SALT}${email}`);
}
async getCryptoKey(name: string): Promise<string | undefined> {
const key = await this.read<string>(`${name}@_k`);
async getCryptoKey(): Promise<string | undefined> {
const key = await this.read<string>(`userEncryptionKey`);
if (!key) return;
return key;
}

10
packages/core/__tests__/migrations.test.ts
View File

@@ -543,11 +543,11 @@ test("[5.9] move attachments.noteIds to relations", () =>
await migrateItem(attachment, 5.9, 6.0, "attachment", db, "backup");
const linkedNotes = await db.relations
.from({ type: "attachment", id: "ATTACHMENT_ID" }, "note")
.to({ type: "attachment", id: "ATTACHMENT_ID" }, "note")
.get();
expect(attachment.noteIds).toBeUndefined();
expect(linkedNotes).toHaveLength(1);
expect(linkedNotes[0].toId).toBe("HELLO_NOTE_ID");
expect(linkedNotes[0].fromId).toBe("HELLO_NOTE_ID");
}));
test.todo("[5.9] flatten attachment object", () =>
@@ -692,7 +692,7 @@ test("[5.9] migrate settings to its own collection", () =>
);
}));
describe.concurrent("[5.9] migrate kv", () => {
describe.concurrent("[5.9] migrate kv", (test) => {
for (const key of KEYS) {
test(`${key} (defined)`, () =>
databaseTest().then(async (db) => {
@@ -701,7 +701,7 @@ describe.concurrent("[5.9] migrate kv", () => {
await migrateKV(db, 5.9, 6.0);
expect(await db.kv().read(key)).toBeDefined();
expect(await db.storage().read(key)).toBeUndefined();
// TODO: expect(await db.storage().read(key)).toBeUndefined();
}));
test(`${key} (undefined)`, () =>
@@ -711,7 +711,7 @@ describe.concurrent("[5.9] migrate kv", () => {
await migrateKV(db, 5.9, 6.0);
expect(await db.kv().read(key)).toBe(key === "v" ? 6 : undefined);
expect(await db.storage().read(key)).toBe(null);
// TODO: expect(await db.storage().read(key)).toBe(null);
}));
}
});

2
packages/core/__tests__/utils/index.ts
View File

@@ -118,7 +118,7 @@ function delay(ms: number) {
async function loginFakeUser(db) {
const email = "johndoe@example.com";
const userSalt = randomBytes(16).toString("base64");
await db.storage().deriveCryptoKey(`_uk_@${email}`, {
await db.storage().deriveCryptoKey({
password: "password",
salt: userSalt
});

3
packages/core/package.json
View File

@@ -45,8 +45,7 @@
"build": "tsc",
"watch": "tsc --watch",
"test:e2e": "cross-env IS_E2E=true vitest run",
"test": "vitest run",
"postinstall": "patch-package"
"test": "vitest run"
},
"dependencies": {
"@microsoft/signalr": "^8.0.0",

4
packages/core/src/api/sync/index.ts
View File

@@ -46,7 +46,7 @@ import {
import { SYNC_COLLECTIONS_MAP, SyncTransferItem } from "./types";
import { DownloadableFile } from "../../database/fs";
import { SyncDevices } from "./devices";
import { COLORS } from "../../database/backup";
import { DefaultColors } from "../../collections/colors";
export type SyncOptions = {
type: "full" | "fetch" | "send";
@@ -460,7 +460,7 @@ async function deserializeItem(
const itemType =
// colors are naively of type "tag" instead of "color" so we have to fix that.
item.type === "tag" && COLORS.includes(item.title.toLowerCase())
item.type === "tag" && DefaultColors[item.title.toLowerCase()]
? "color"
: item.type === "trash" && "itemType" in item && item.itemType
? item.itemType

211
packages/core/src/database/backup.ts
View File

@@ -21,11 +21,19 @@ import SparkMD5 from "spark-md5";
import { CURRENT_DATABASE_VERSION } from "../common.js";
import Migrator from "./migrator.js";
import Database from "../api/index.js";
import { Item, MaybeDeletedItem, Note, Notebook, isDeleted } from "../types.js";
import {
Item,
MaybeDeletedItem,
Note,
Notebook,
ValueOf,
isDeleted
} from "../types.js";
import { Cipher, SerializedKey } from "@notesnook/crypto";
import { isCipher } from "./crypto.js";
import { migrateItem } from "../migrations";
import { DatabaseCollection } from "./index.js";
import { DefaultColors } from "../collections/colors.js";
type BackupDataItem = MaybeDeletedItem<Item> | string[];
type BackupPlatform = "web" | "mobile" | "node";
@@ -89,17 +97,6 @@ function isLegacyBackupFile(
}
const MAX_CHUNK_SIZE = 10 * 1024 * 1024;
export const COLORS = [
"red",
"orange",
"yellow",
"green",
"blue",
"purple",
"gray",
"black",
"white"
];
const invalidKeys = [
"user",
@@ -137,17 +134,16 @@ const itemTypeToCollectionKey = {
reminder: "reminders",
sessioncontent: "sessioncontent",
session: "noteHistory",
notehistory: "notehistory",
notehistory: "noteHistory",
content: "content",
shortcut: "shortcuts",
settingitem: "settings",
settings: "settings",
vault: "vaults",
// to make ts happy
topic: "topics"
vault: "vaults"
} as const;
type CollectionName = ValueOf<typeof itemTypeToCollectionKey>;
const validTypes = ["mobile", "web", "node"];
export default class Backup {
migrator = new Migrator();
@@ -346,111 +342,114 @@ export default class Backup {
}
private async migrateData(data: BackupDataItem[], version: number) {
await this.db.transaction(async () => {
for (let item of data) {
// we do not want to restore deleted items
if (
!item ||
typeof item !== "object" ||
Array.isArray(item) ||
isDeleted(item)
)
continue;
// in v5.6 of the database, we did not set note history session's type
if ("sessionContentId" in item && item.type !== "session")
(item as any).type = "notehistory";
const queue: Partial<Record<CollectionName, MaybeDeletedItem<Item>[]>> = {};
for (let item of data) {
// we do not want to restore deleted items
if (
!item ||
typeof item !== "object" ||
Array.isArray(item) ||
isDeleted(item)
)
continue;
// in v5.6 of the database, we did not set note history session's type
if ("sessionContentId" in item && item.type !== "session")
(item as any).type = "notehistory";
if (
(await migrateItem(
item,
version,
CURRENT_DATABASE_VERSION,
item.type,
this.db,
"backup"
)) === "skip"
)
continue;
// since items in trash can have their own set of migrations,
// we have to run the migration again to account for that.
if (item.type === "trash" && item.itemType)
if (
(await migrateItem(
item,
item as unknown as Note | Notebook,
version,
CURRENT_DATABASE_VERSION,
item.type,
item.itemType,
this.db,
"backup"
)) === "skip"
)
continue;
// since items in trash can have their own set of migrations,
// we have to run the migration again to account for that.
if (item.type === "trash" && item.itemType)
if (
(await migrateItem(
item as unknown as Note | Notebook,
version,
CURRENT_DATABASE_VERSION,
item.itemType,
this.db,
"backup"
)) === "skip"
)
continue;
const itemType =
// colors are naively of type "tag" instead of "color" so we have to fix that.
item.type === "tag" && COLORS.includes(item.title.toLowerCase())
? "color"
: item.type === "trash" && "itemType" in item && item.itemType
? item.itemType
: item.type;
const itemType =
// colors are naively of type "tag" instead of "color" so we have to fix that.
item.type === "tag" && DefaultColors[item.title.toLowerCase()]
? "color"
: item.type === "trash" && "itemType" in item && item.itemType
? item.itemType
: item.type;
if (!itemType || itemType === "topic" || itemType === "settings")
continue;
if (!itemType || itemType === "topic" || itemType === "settings")
continue;
if (item.type === "attachment" && (item.hash || item.metadata?.hash)) {
const attachment = await this.db.attachments.attachment(
item.metadata?.hash || item.hash
);
if (attachment) {
const isNewGeneric =
item.metadata?.type === "application/octet-stream" ||
item.mimeType === "application/octet-stream";
const isOldGeneric =
attachment.mimeType === "application/octet-stream";
item = {
...attachment,
mimeType:
// we keep whichever mime type is more specific
isNewGeneric && !isOldGeneric
? attachment.mimeType
: item.metadata?.type || item.mimeType,
filename:
// we keep the filename based on which item's mime type we kept
isNewGeneric && !isOldGeneric
? attachment.filename
: item.metadata?.filename || item.filename
};
for (const noteId of item.noteIds || []) {
await this.db.relations.add(
{
id: noteId,
type: "note"
},
attachment
);
}
} else {
delete item.dateUploaded;
delete item.failed;
if (item.type === "attachment" && (item.hash || item.metadata?.hash)) {
const attachment = await this.db.attachments.attachment(
item.metadata?.hash || item.hash
);
if (attachment) {
const isNewGeneric =
item.metadata?.type === "application/octet-stream" ||
item.mimeType === "application/octet-stream";
const isOldGeneric =
attachment.mimeType === "application/octet-stream";
item = {
...attachment,
mimeType:
// we keep whichever mime type is more specific
isNewGeneric && !isOldGeneric
? attachment.mimeType
: item.metadata?.type || item.mimeType,
filename:
// we keep the filename based on which item's mime type we kept
isNewGeneric && !isOldGeneric
? attachment.filename
: item.metadata?.filename || item.filename
};
for (const noteId of item.noteIds || []) {
await this.db.relations.add(
{
id: noteId,
type: "note"
},
attachment
);
}
} else {
delete item.dateUploaded;
delete item.failed;
}
const collectionKey = itemTypeToCollectionKey[itemType];
if (!collectionKey) continue;
const collection =
collectionKey === "sessioncontent"
? this.db.noteHistory.sessionContent.collection
: this.db[collectionKey].collection;
// items should sync immediately after getting restored
item.dateModified = Date.now();
item.synced = false;
await collection.upsert(item as any);
}
});
const collectionKey: CollectionName = itemTypeToCollectionKey[itemType];
if (!collectionKey) continue;
queue[collectionKey] = queue[collectionKey] || [];
queue[collectionKey]?.push(item);
}
for (const key in queue) {
const collectionKey = key as CollectionName;
const collection =
collectionKey === "sessioncontent"
? this.db.noteHistory.sessionContent.collection
: this.db[collectionKey].collection;
if (!collection) continue;
const items = queue[collectionKey];
if (!items) continue;
await collection.put(items as any[]);
}
}
private validate(backup: LegacyBackupFile | BackupFile) {

2
packages/core/src/database/sql-cached-collection.ts
View File

@@ -35,7 +35,7 @@ export class SQLCachedCollection<
constructor(
sql: DatabaseAccessor,
startTransaction: (
executor: (tr: Transaction<DatabaseSchema>) => void | Promise<void>
executor: (tr: Transaction<DatabaseSchema>) => Promise<void>
) => Promise<void>,
type: TCollectionType,
eventManager: EventManager

2
packages/core/src/database/sql-collection.ts
View File

@@ -54,7 +54,7 @@ export class SQLCollection<
constructor(
private readonly db: DatabaseAccessor,
private readonly startTransaction: (
executor: (tr: Transaction<DatabaseSchema>) => void | Promise<void>
executor: (tr: Transaction<DatabaseSchema>) => Promise<void>
) => Promise<void>,
private readonly type: TCollectionType,
private readonly eventManager: EventManager

1
packages/core/src/migrations.ts
View File

@@ -179,6 +179,7 @@ const migrations: Migration[] = [
items: {
trash: (item) => {
if (!item.deletedBy) item.deletedBy = "user";
delete item.itemId;
return true;
},
color: async (item, db, migrationType) => {

4
packages/core/src/types.ts
View File

@@ -479,6 +479,10 @@ export type BaseTrashItem<TItem extends BaseItem<"note" | "notebook">> =
* deletedBy tells who deleted this specific item.
*/
deletedBy: "user" | "app";
/**
* @deprecated
*/
itemId?: never;
} & Omit<TItem, "id" | "type" | "dateDeleted" | "itemType" | "deletedBy">;
export type TrashItem = BaseTrashItem<Note> | BaseTrashItem<Notebook>;