From dc3caa16d0efbb8397faecc0b6b72d45643c4c33 Mon Sep 17 00:00:00 2001 From: Abdullah Atta Date: Sat, 3 Feb 2024 16:24:07 +0500 Subject: [PATCH] core: migrate vault to a database collection --- .../__snapshots__/pricing.test.js.snap | 9 + packages/core/__tests__/vault.test.js | 103 ++++++++- packages/core/src/api/index.ts | 3 +- packages/core/src/api/sync/index.ts | 49 ++--- packages/core/src/api/vault.ts | 202 ++++++++---------- packages/core/src/collections/content.ts | 18 +- packages/core/src/collections/notes.ts | 12 -- packages/core/src/collections/relations.ts | 3 +- packages/core/src/collections/tags.ts | 5 +- packages/core/src/collections/vaults.ts | 83 +++++++ packages/core/src/database/backup.ts | 2 + packages/core/src/database/index.ts | 7 +- packages/core/src/database/migrations.ts | 8 + packages/core/src/database/migrator.ts | 7 +- packages/core/src/migrations.ts | 53 +++-- packages/core/src/types.ts | 13 +- packages/core/vitest.config.ts | 4 +- 17 files changed, 390 insertions(+), 191 deletions(-) create mode 100644 packages/core/src/collections/vaults.ts diff --git a/packages/core/__e2e__/__snapshots__/pricing.test.js.snap b/packages/core/__e2e__/__snapshots__/pricing.test.js.snap index 49e608e4c..b92262c58 100644 --- a/packages/core/__e2e__/__snapshots__/pricing.test.js.snap +++ b/packages/core/__e2e__/__snapshots__/pricing.test.js.snap @@ -72,6 +72,15 @@ exports[`get web pricing tier > get yearly web tier > yearly-web-pricing 1`] = ` } `; +exports[`get yearly price > monthly-pricing 1`] = ` +{ + "country": Any, + "countryCode": Any, + "discount": Any, + "price": Any, +} +`; + exports[`get yearly price > yearly-pricing 1`] = ` { "country": Any, diff --git a/packages/core/__tests__/vault.test.js b/packages/core/__tests__/vault.test.js index e23a7c9c7..baa5076a7 100644 --- a/packages/core/__tests__/vault.test.js +++ b/packages/core/__tests__/vault.test.js @@ -17,17 +17,18 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -import { databaseTest, noteTest, TEST_NOTE } from "./utils"; +import { VAULT_ERRORS } from "../src/api/vault"; +import { databaseTest, delay, noteTest, TEST_NOTE } from "./utils"; import { test, expect } from "vitest"; test("create vault", () => databaseTest().then(async (db) => { await expect(db.vault.create("password")).resolves.toBe(true); - const vaultKey = await db.storage().read("vaultKey"); - expect(vaultKey).toBeDefined(); - expect(vaultKey.iv).toBeDefined(); - expect(vaultKey.cipher).toBeDefined(); - expect(vaultKey.length).toBeDefined(); + const vault = await db.vaults.default(); + expect(vault).toBeDefined(); + expect(vault.key.iv).toBeDefined(); + expect(vault.key.cipher).toBeDefined(); + expect(vault.key.length).toBeDefined(); })); test("unlock vault", () => @@ -57,15 +58,20 @@ test("lock a note when no vault has been created", () => test("lock a note", () => noteTest().then(async ({ db, id }) => { await db.vault.create("password"); + await db.vault.add(id); + const note = await db.notes.note(id); + const content = await db.content.get(note.contentId); + const vault = await db.vaults.default(); expect(note.headline).toBe(""); - const content = await db.content.get(note.contentId); expect(content.noteId).toBeDefined(); expect(content.data.iv).toBeDefined(); expect(content.data.cipher).toBeDefined(); + + expect(await db.relations.from(vault, "note").has(id)).toBe(true); })); test("locked note is not favorited", () => @@ -81,23 +87,32 @@ test("unlock a note", () => noteTest().then(async ({ db, id }) => { await db.vault.create("password"); await db.vault.add(id); + const note = await db.vault.open(id, "password"); + + const vault = await db.vaults.default(); expect(note.id).toBe(id); expect(note.content.data).toBeDefined(); expect(note.content.type).toBe(TEST_NOTE.content.type); + expect(await db.relations.from(vault, "note").has(id)).toBe(true); })); test("unlock a note permanently", () => noteTest().then(async ({ db, id }) => { await db.vault.create("password"); await db.vault.add(id); + await db.vault.remove(id, "password"); + const note = await db.notes.note(id); + const content = await db.content.get(note.contentId); + const vault = await db.vaults.default(); + expect(note.id).toBe(id); expect(note.headline).not.toBe(""); - const content = await db.content.get(note.contentId); expect(content.data).toBeDefined(); expect(typeof content.data).toBe("string"); + expect(await db.relations.from(vault, "note").has(id)).toBe(false); })); test("save a locked note", () => @@ -142,3 +157,75 @@ test("change vault password", () => await expect(db.vault.open(id, "password")).rejects.toThrow(); await expect(db.vault.open(id, "newPassword")).resolves.toBeDefined(); })); + +test("changing vault password without a vault should throw", () => + noteTest().then(async ({ db }) => { + await expect( + db.vault.changePassword("password", "newPassword") + ).rejects.toThrow(VAULT_ERRORS.noVault); + })); + +test("clear vault", () => + noteTest().then(async ({ db, id }) => { + await db.vault.create("password"); + await db.vault.add(id); + + await db.vault.clear("password"); + + const vault = await db.vaults.default(); + expect(await db.relations.from(vault, "note").has(id)).toBe(false); + })); + +test("delete vault without deleting all locked notes", () => + noteTest().then(async ({ db, id }) => { + await db.vault.create("password"); + await db.vault.add(id); + const vault = await db.vaults.default(); + + await db.vault.delete(); + + expect(await db.relations.from(vault, "note").has(id)).toBe(false); + expect(await db.vaults.default()).toBeUndefined(); + })); + +test("delete vault and delete all locked notes", () => + noteTest().then(async ({ db, id }) => { + await db.vault.create("password"); + await db.vault.add(id); + const vault = await db.vaults.default(); + + await db.vault.delete(true); + + expect(await db.relations.from(vault, "note").has(id)).toBe(false); + expect(await db.notes.exists(id)).toBe(false); + expect(await db.vaults.default()).toBeUndefined(); + })); + +test("vault password is cleared after specified time", () => + databaseTest().then(async (db) => { + await expect(db.vault.create("password")).resolves.toBe(true); + db.vault.eraseTime = 1000; + await expect(db.vault.unlock("password")).resolves.toBe(true); + expect(db.vault.unlocked).toBe(true); + await delay(1500); + expect(db.vault.unlocked).toBe(false); + })); + +// test("remove note from vault if it isn't encrypted", () => +// noteTest().then(async ({ db, id }) => { +// await db.vault.create("password"); +// const vault = await db.vaults.default(); + +// await db.relations.add(vault, { id, type: "note" }); + +// const decryptedContent = await db.vault.decryptContent( +// { +// data: "

hello world

", +// type: "tiptap" +// }, +// "password" +// ); + +// expect(await db.relations.from(vault, "note").has(id)).toBe(false); +// expect(decryptedContent.data).toBe("

hello world

"); +// })); diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts index 9a2d60481..c32e6a168 100644 --- a/packages/core/src/api/index.ts +++ b/packages/core/src/api/index.ts @@ -68,6 +68,7 @@ import { } from "../database"; import { Kysely, Transaction, sql } from "kysely"; import { CachedCollection } from "../database/cached-collection"; +import { Vaults } from "../collections/vaults"; type EventSourceConstructor = new ( uri: string, @@ -183,6 +184,7 @@ class Database { reminders = new Reminders(this); relations = new Relations(this); notes = new Notes(this); + vaults = new Vaults(this); /** * @deprecated only kept here for migration purposes @@ -255,7 +257,6 @@ class Database { ); EV.subscribe(EVENTS.attachmentDeleted, async (attachment: Attachment) => { await this.fs().cancel(attachment.hash); - await this.fs().cancel(attachment.hash); }); EV.subscribe(EVENTS.userLoggedOut, async () => { await this.monographs.clear(); diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.ts index 4e0762cf9..0101824ce 100644 --- a/packages/core/src/api/sync/index.ts +++ b/packages/core/src/api/sync/index.ts @@ -34,7 +34,7 @@ import { AutoSync } from "./auto-sync"; import { logger } from "../../logger"; import { Mutex } from "async-mutex"; import Database from ".."; -import { migrateItem } from "../../migrations"; +import { migrateItem, migrateVaultKey } from "../../migrations"; import { SerializedKey } from "@notesnook/crypto"; import { Item, MaybeDeletedItem, Note, Notebook } from "../../types"; import { SYNC_COLLECTIONS_MAP, SyncTransferItem } from "./types"; @@ -189,6 +189,26 @@ class Sync { let count = 0; this.connection?.off("SendItems"); + this.connection?.off("SendVaultKey"); + + this.connection?.on("SendVaultKey", async (vaultKey) => { + if (this.connection?.state !== signalr.HubConnectionState.Connected) + return false; + + if ( + vaultKey && + vaultKey.cipher !== null && + vaultKey.iv !== null && + vaultKey.salt !== null && + vaultKey.length > 0 + ) { + const vault = await this.db.vaults.default(); + if (!vault) await migrateVaultKey(this.db, vaultKey, 5.9); + } + + return true; + }); + this.connection?.on("SendItems", async (chunk) => { if (this.connection?.state !== signalr.HubConnectionState.Connected) return false; @@ -200,39 +220,17 @@ class Sync { return true; }); - const serverResponse = await this.connection?.invoke( - "RequestFetch", - deviceId - ); - - if ( - serverResponse.vaultKey && - serverResponse.vaultKey.cipher !== null && - serverResponse.vaultKey.iv !== null && - serverResponse.vaultKey.salt !== null && - serverResponse.vaultKey.length > 0 - ) { - await this.db.vault.setKey(serverResponse.vaultKey); - } + await this.connection?.invoke("RequestFetch", deviceId); this.connection?.off("SendItems"); + this.connection?.off("SendVaultKey"); } async send(deviceId: string, isForceSync?: boolean) { await this.uploadAttachments(); - let isSyncInitialized = false; let done = 0; for await (const item of this.collector.collect(100, isForceSync)) { - if (!isSyncInitialized) { - const vaultKey = await this.db.vault.getKey(); - await this.connection?.send("InitializePush", { - vaultKey, - synced: false - }); - isSyncInitialized = true; - } - const result = await this.pushItem(deviceId, item); if (result) { done += item.items.length; @@ -245,7 +243,6 @@ class Sync { ); } } - if (!isSyncInitialized) return false; await this.connection?.send("PushCompleted"); return true; } diff --git a/packages/core/src/api/vault.ts b/packages/core/src/api/vault.ts index 3f33beead..dd00d2d0e 100644 --- a/packages/core/src/api/vault.ts +++ b/packages/core/src/api/vault.ts @@ -22,11 +22,6 @@ import Database from "."; import { CHECK_IDS, EV, EVENTS, checkIsUserPremium } from "../common"; import { tinyToTiptap } from "../migrations"; import { isCipher } from "../database/crypto"; -import { Note } from "../types"; -import { - isEncryptedContent, - isUnencryptedContent -} from "../collections/content"; import { NoteContent } from "../collections/session-content"; export const VAULT_ERRORS = { @@ -35,8 +30,8 @@ export const VAULT_ERRORS = { wrongPassword: "ERR_WRONG_PASSWORD" }; -const ERASE_TIME = 1000 * 60 * 30; export default class Vault { + eraseTime = 1000 * 60 * 30; private vaultPassword?: string; private erasureTimeout = 0; private key = "svvaads1212#2123"; @@ -57,7 +52,7 @@ export default class Vault { this.erasureTimeout = setTimeout(() => { this.password = undefined; EV.publish(EVENTS.vaultLocked); - }, ERASE_TIME) as unknown as number; + }, this.eraseTime) as unknown as number; } constructor(private readonly db: Database) { @@ -99,63 +94,73 @@ export default class Vault { } async changePassword(oldPassword: string, newPassword: string) { + const vault = await this.db.vaults.default(); + if (!vault) throw new Error(VAULT_ERRORS.noVault); + if (await this.unlock(oldPassword)) { - const contentItems = []; - for await (const note of this.db.notes.locked) { - if (!note.contentId) continue; - const encryptedContent = await this.db.content.get(note.contentId); - if (!encryptedContent || !isEncryptedContent(encryptedContent)) + const relations = await this.db.relations.from(vault, "note").get(); + for (const { toId: noteId } of relations) { + const content = await this.db.content.findByNoteId(noteId); + if (!content || !content.locked) { + await this.db.relations.unlink(vault, { id: noteId, type: "note" }); continue; + } try { - const content = await this.decryptContent( - encryptedContent, - note.id, + const decryptedContent = await this.decryptContent( + content, oldPassword ); - contentItems.push({ - ...content, - id: note.contentId, - noteId: note.id - }); + + await this.encryptContent( + decryptedContent, + noteId, + newPassword, + `${Date.now()}` + ); } catch (e) { console.error(e); throw new Error( - `Could not decrypt content of note ${note.id}. Error: ${ + `Could not decrypt content of note ${noteId}. Error: ${ (e as Error).message }` ); } } - for (const content of contentItems) { - await this.encryptContent( - content, - content.noteId, - newPassword, - content.id - ); - } - - await this.db.storage().remove("vaultKey"); - await this.create(newPassword); + await this.db.vaults.add({ + id: vault.id, + key: await this.db + .storage() + .encrypt({ password: newPassword }, this.key) + }); } } async clear(password: string) { + const vault = await this.db.vaults.default(); + if (!vault) return; + if (await this.unlock(password)) { - for await (const note of this.db.notes.locked) { - await this.unlockNote(note, password, true); + const relations = await this.db.relations.from(vault, "note").get(); + for (const { toId: noteId } of relations) { + await this.unlockNote(noteId, password, true); + await this.db.relations.unlink(vault, { id: noteId, type: "note" }); } } } async delete(deleteAllLockedNotes = false) { + const vault = await this.db.vaults.default(); + if (!vault) return; + if (deleteAllLockedNotes) { - const lockedIds = await this.db.notes.locked.ids(); + const relations = await this.db.relations.from(vault, "note").get(); + const lockedIds = relations.map((r) => r.toId); await this.db.notes.remove(...lockedIds); } - await this.db.storage().remove("vaultKey"); + + await this.db.vaults.remove(vault.id); this.password = undefined; } @@ -173,11 +178,11 @@ export default class Vault { * Permanently unlocks (remove from vault) a note */ async remove(noteId: string, password: string) { - const note = await this.db.notes.note(noteId); - if (!note) return; - await this.unlockNote(note, password, true); + await this.unlockNote(noteId, password, true); - if (!(await this.exists())) await this.create(password); + const vault = await this.db.vaults.default(); + if (!vault) await this.create(password); + else await this.db.relations.unlink(vault, { id: noteId, type: "note" }); } /** @@ -187,23 +192,22 @@ export default class Vault { const note = await this.db.notes.note(noteId); if (!note) return; - const unlockedNote = await this.unlockNote(note, password, false); + const content = await this.unlockNote(noteId, password, false); if (password) { this.password = password; if (!(await this.exists())) await this.create(password); } - - return unlockedNote; + return { ...note, ...content }; } /** * Saves a note in the vault */ - async save( - note: Partial; sessionId: string }> & { - id: string; - } - ) { + async save(note: { + content?: NoteContent; + sessionId?: string; + id: string; + }) { if (!note) return; // roll over erase timer this.startEraser(); @@ -233,7 +237,6 @@ export default class Vault { content: NoteContent, noteId: string, password: string, - contentId?: string, sessionId?: string ) { const encryptedContent = await this.db @@ -241,7 +244,6 @@ export default class Vault { .encrypt({ password }, JSON.stringify(content.data)); await this.db.content.add({ - id: contentId, noteId, sessionId, data: encryptedContent, @@ -249,23 +251,11 @@ export default class Vault { }); } - async decryptContent( - encryptedContent: NoteContent, - noteId: string, - password?: string - ) { + async decryptContent(encryptedContent: NoteContent, password?: string) { if (!password) password = await this.getVaultPassword(); - if ( - typeof encryptedContent.data !== "object" && - !isCipher(encryptedContent.data) - ) { - await this.db.notes.add({ - id: noteId, - locked: false - }); - return { data: encryptedContent.data, type: encryptedContent.type }; - } + if (!isCipher(encryptedContent.data)) + return encryptedContent as unknown as NoteContent; const decryptedContent = await this.db .storage() @@ -286,33 +276,30 @@ export default class Vault { } private async lockNote( - item: Partial; sessionId: string }> & { - id: string; - }, + item: { content?: NoteContent; sessionId?: string; id: string }, password: string ) { - const { id, content, sessionId, title } = item; + const vault = await this.db.vaults.default(); + if (!vault) throw new Error(VAULT_ERRORS.noVault); + + const { id, content, sessionId } = item; let { type, data } = content || {}; - const note = await this.db.notes.note(id); - if (!note) return; - - const contentId = note.contentId; - // if (!contentId) throw new Error("Cannot lock note because it is empty."); + const locked = await this.db.relations.from(vault, "note").has(id); // Case: when note is being newly locked - if (!note.locked && (!data || !type) && !!contentId) { - const rawContent = await this.db.content.get(contentId); - if (!rawContent || !isUnencryptedContent(rawContent)) - return await this.db.notes.add({ + if (!locked && (!data || !type)) { + const content = await this.db.content.findByNoteId(id); + if (!content || content.locked) + return await this.db.relations.add(vault, { id, - locked: true + type: "note" }); // NOTE: // At this point, the note already has all the attachments extracted // so we should just encrypt it as normal. - data = rawContent.data; - type = rawContent.type; + data = content.data; + type = content.type; } else if (data && type) { data = await this.db.content.extractAttachments({ data, @@ -322,60 +309,47 @@ export default class Vault { } if (data && type) - await this.encryptContent( - { data, type }, - id, - password, - contentId, - sessionId - ); + await this.encryptContent({ data, type }, id, password, sessionId); - return await this.db.notes.add({ + await this.db.notes.add({ id, - locked: true, - headline: "", - title: title || note.title, - favorite: note.favorite, - localOnly: note.localOnly, - readonly: note.readonly, - dateEdited: Date.now() + headline: "" + }); + + await this.db.relations.add(vault, { + id, + type: "note" }); } - private async unlockNote(note: Note, password?: string, perm = false) { - if (!note.contentId) return; - - const encryptedContent = await this.db.content.get(note.contentId); - if (!encryptedContent || !isEncryptedContent(encryptedContent)) return; - const content = await this.decryptContent( - encryptedContent, - note.id, - password - ); + private async unlockNote(noteId: string, password?: string, perm = false) { + const content = await this.db.content.findByNoteId(noteId); + if (!content || !content.locked) return; + const decryptedContent = await this.decryptContent(content, password); if (perm) { await this.db.notes.add({ - id: note.id, - locked: false, - headline: note.headline, - contentId: note.contentId, - content + id: noteId, + contentId: content.id, + content: decryptedContent }); // await this.db.content.add({ id: note.contentId, data: content }); return; } return { - ...note, content }; } async getKey() { - return await this.db.storage().read>("vaultKey"); + const vault = await this.db.vaults.default(); + return vault?.key; } async setKey(vaultKey: Cipher<"base64">) { - await this.db.storage().write("vaultKey", vaultKey); + const vault = await this.db.vaults.default(); + if (vault) return; + await this.db.vaults.add({ title: "Default", key: vaultKey }); } } diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.ts index 698f6893d..504a9d243 100644 --- a/packages/core/src/collections/content.ts +++ b/packages/core/src/collections/content.ts @@ -200,13 +200,17 @@ export class Content implements ICollection { .execute(); } - // async findByNoteId(noteId: string) { - // await this.db - // .sql() - // .selectFrom("content") - // .where("noteId", "==", noteId) - // .execute(); - // } + async findByNoteId(noteId: string) { + const content = (await this.db + .sql() + .selectFrom("content") + .where("noteId", "==", noteId) + .selectAll() + .executeTakeFirst()) as ContentItem; + if (!content || isDeleted(content)) return; + return content; + } + // multi(ids: string[]) { // return this.collection.getItems(ids); // } diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.ts index f13372d2a..aba891ba3 100644 --- a/packages/core/src/collections/notes.ts +++ b/packages/core/src/collections/notes.ts @@ -121,7 +121,6 @@ export class Notes implements ICollection { contentId, pinned: item.pinned, - locked: item.locked, favorite: item.favorite, localOnly: item.localOnly, conflicted: item.conflicted, @@ -228,17 +227,6 @@ export class Notes implements ICollection { ); } - get locked() { - return this.collection.createFilter( - (qb) => - qb - .where(isFalse("dateDeleted")) - .where(isFalse("deleted")) - .where("locked", "==", true), - this.db.options?.batchSize - ); - } - exists(id: string) { return this.collection.exists(id); } diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.ts index 80fb6a366..81471731a 100644 --- a/packages/core/src/collections/relations.ts +++ b/packages/core/src/collections/relations.ts @@ -194,7 +194,8 @@ const TABLE_MAP = { reminder: "reminders", tag: "tags", color: "colors", - attachment: "attachments" + attachment: "attachments", + vault: "vaults" } as const; type RelatableTable = typeof TABLE_MAP; diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.ts index 59a351f9e..833c0d74f 100644 --- a/packages/core/src/collections/tags.ts +++ b/packages/core/src/collections/tags.ts @@ -49,8 +49,6 @@ export class Tags implements ICollection { } async add(item: Partial) { - if (item.remote) - throw new Error("Please use db.tags.merge to merge remote tags."); item.title = item.title ? Tags.sanitize(item.title) : item.title; const id = item.id || getId(item.dateCreated); @@ -68,8 +66,7 @@ export class Tags implements ICollection { dateCreated: item.dateCreated || oldTag?.dateCreated || Date.now(), dateModified: item.dateModified || oldTag?.dateModified || Date.now(), title: item.title || oldTag?.title || "", - type: "tag", - remote: false + type: "tag" }); return id; } diff --git a/packages/core/src/collections/vaults.ts b/packages/core/src/collections/vaults.ts new file mode 100644 index 000000000..468b9c718 --- /dev/null +++ b/packages/core/src/collections/vaults.ts @@ -0,0 +1,83 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import Database from "../api"; +import { Vault } from "../types"; +import { ICollection } from "./collection"; +import { SQLCollection } from "../database/sql-collection"; +import { getId } from "../utils/id"; +import { isFalse } from "../database"; + +export class Vaults implements ICollection { + name = "vaults"; + readonly collection: SQLCollection<"vaults", Vault>; + constructor(private readonly db: Database) { + this.collection = new SQLCollection( + db.sql, + db.transaction, + "vaults", + db.eventManager + ); + } + + async init() {} + + async add(item: Partial) { + const id = item.id || getId(); + const oldVault = item.id ? await this.vault(item.id) : undefined; + + if (!item.title && !oldVault?.title) throw new Error("Title is required."); + + await this.collection.upsert({ + id, + dateCreated: item.dateCreated || oldVault?.dateCreated || Date.now(), + dateModified: item.dateModified || oldVault?.dateModified || Date.now(), + title: item.title || oldVault?.title || "", + key: item.key, + type: "vault" + }); + return id; + } + + async remove(id: string) { + await this.db.transaction(async () => { + await this.db.relations.unlinkOfType("vault", [id]); + await this.collection.softDelete([id]); + }); + } + + vault(id: string) { + return this.collection.get(id); + } + + /** + * This is temporary until we add proper support for multiple vaults + * @deprecated + */ + async default() { + return (await this.all.items()).at(0); + } + + get all() { + return this.collection.createFilter( + (qb) => qb.where(isFalse("deleted")), + this.db.options?.batchSize + ); + } +} diff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.ts index 4f376f93b..9a42b4781 100644 --- a/packages/core/src/database/backup.ts +++ b/packages/core/src/database/backup.ts @@ -142,6 +142,7 @@ const itemTypeToCollectionKey = { shortcut: "shortcuts", settingitem: "settings", settings: "settings", + vault: "vaults", // to make ts happy topic: "topics" @@ -198,6 +199,7 @@ export default class Backup { yield* this.backupCollection(this.db.reminders.collection, backupState); yield* this.backupCollection(this.db.relations.collection, backupState); yield* this.backupCollection(this.db.attachments.collection, backupState); + yield* this.backupCollection(this.db.vaults.collection, backupState); if (backupState.buffer.length > 0) yield* this.bufferToFile(backupState); diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.ts index 424f70f0a..40dfb065b 100644 --- a/packages/core/src/database/index.ts +++ b/packages/core/src/database/index.ts @@ -53,7 +53,8 @@ import { Shortcut, Tag, TrashOrItem, - ValueOf + ValueOf, + Vault } from "../types"; import { NNMigrationProvider } from "./migrations"; import { createTriggers } from "./triggers"; @@ -85,6 +86,7 @@ export interface DatabaseSchema { notehistory: SQLiteItem; sessioncontent: SQLiteItem; shortcuts: SQLiteItem; + vaults: SQLiteItem; } export type DatabaseSchemaWithFTS = DatabaseSchema & { @@ -194,6 +196,9 @@ const DataMappers: Partial void>> = { }, attachment: (row) => { if (row.key) row.key = JSON.parse(row.key); + }, + vault: (row) => { + if (row.key) row.key = JSON.parse(row.key); } }; diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.ts index b028870d3..133385622 100644 --- a/packages/core/src/database/migrations.ts +++ b/packages/core/src/database/migrations.ts @@ -123,6 +123,14 @@ export class NNMigrationProvider implements MigrationProvider { .addColumn("colorCode", "text") .execute(); + await db.schema + .createTable("vaults") + .modifyEnd(sql`without rowid`) + .$call(addBaseColumns) + .addColumn("title", "text", COLLATE_NOCASE) + .addColumn("key", "text") + .execute(); + await db.schema .createTable("relations") .modifyEnd(sql`without rowid`) diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.ts index aa796f09b..3708f6a90 100644 --- a/packages/core/src/database/migrator.ts +++ b/packages/core/src/database/migrator.ts @@ -17,13 +17,14 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +import { Cipher } from "@notesnook/crypto"; import { DatabaseSchema } from "."; import Database from "../api"; import { CURRENT_DATABASE_VERSION, sendMigrationProgressEvent } from "../common"; -import { migrateCollection, migrateItem } from "../migrations"; +import { migrateCollection, migrateItem, migrateVaultKey } from "../migrations"; import { CollectionType, Collections, @@ -50,6 +51,9 @@ class Migrator { collections: MigratableCollections, version: number ) { + const vaultKey = await db.storage().read>("vaultKey"); + if (vaultKey) await migrateVaultKey(db, vaultKey, version); + for (const collection of collections) { sendMigrationProgressEvent(db.eventManager, collection.name, 0, 0); @@ -79,6 +83,7 @@ class Migrator { ); } } + await db.initCollections(); return true; } diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.ts index 8a3211709..e11c057b2 100644 --- a/packages/core/src/migrations.ts +++ b/packages/core/src/migrations.ts @@ -34,18 +34,8 @@ import { } from "./types"; import { isCipher } from "./database/crypto"; import { IndexedCollection } from "./database/indexed-collection"; - -const ColorToHexCode: Record = { - red: "#f44336", - orange: "#FF9800", - yellow: "#FFD600", - green: "#4CAF50", - blue: "#2196F3", - purple: "#673AB7", - gray: "#9E9E9E", - black: "#000000", - white: "#ffffff" -}; +import { DefaultColors } from "./collections/colors"; +import { Cipher } from "@notesnook/crypto"; type MigrationType = "local" | "sync" | "backup"; type MigrationItemType = ItemType | "notehistory" | "content" | "all"; @@ -64,6 +54,7 @@ type Migration = { ) => "skip" | boolean | Promise | void; }; collection?: (collection: IndexedCollection) => Promise | void; + vaultKey?: (db: Database, key: Cipher<"base64">) => Promise | void; }; const migrations: Migration[] = [ @@ -198,7 +189,7 @@ const migrations: Migration[] = [ ) return "skip"; - const colorCode = ColorToHexCode[item.title]; + const colorCode = DefaultColors[item.title]; if (colorCode) { const newColor = await db.colors.all.find((eb) => eb.or([eb("title", "in", [alias, item.title])]) @@ -258,7 +249,7 @@ const migrations: Migration[] = [ dateCreated: oldColor?.dateCreated, dateModified: oldColor?.dateModified, title: alias || item.color, - colorCode: ColorToHexCode[item.color], + colorCode: DefaultColors[item.color], type: "color" })); if (newColorId) { @@ -275,6 +266,13 @@ const migrations: Migration[] = [ } } + if (item.locked) { + const vault = await db.vaults.default(); + if (vault) + await db.relations.add({ type: "vault", id: vault.id }, item); + } + + delete item.locked; delete item.notebooks; delete item.tags; delete item.color; @@ -380,6 +378,10 @@ const migrations: Migration[] = [ return true; }, all: () => true + }, + async vaultKey(db, key) { + await db.vaults.add({ title: "Default", key }); + await db.storage().remove("vaultKey"); } }, { @@ -453,6 +455,29 @@ export async function migrateCollection( } } +export async function migrateVaultKey( + db: Database, + vaultKey: Cipher<"base64">, + version: number +) { + let migrationStartIndex = migrations.findIndex((m) => m.version === version); + if (migrationStartIndex <= -1) { + throw new Error( + version > CURRENT_DATABASE_VERSION + ? `Please update the app to the latest version.` + : `You seem to be on a very outdated version. Please update the app to the latest version.` + ); + } + + for (; migrationStartIndex < migrations.length; ++migrationStartIndex) { + const migration = migrations[migrationStartIndex]; + if (migration.version === CURRENT_DATABASE_VERSION) break; + + if (!migration.vaultKey) continue; + await migration.vaultKey(db, vaultKey); + } +} + function replaceDateEditedWithDateModified(removeDateEditedProperty = false) { return function (item: any) { item.dateModified = item.dateEdited; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index ee7cfe32d..82f44044c 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -73,6 +73,7 @@ export type Collections = { notehistory: "session"; sessioncontent: "sessioncontent"; settingsv2: "settingitem"; + vaults: "vault"; /** * @deprecated only kept here for migration purposes @@ -102,6 +103,7 @@ export type GroupableItem = ValueOf< | "sessioncontent" | "settings" | "settingitem" + | "vault" > >; @@ -121,6 +123,7 @@ export type ItemMap = { session: HistorySession; sessioncontent: SessionContentItem; settingitem: SettingItem; + vault: Vault; /** * @deprecated only kept here for migration purposes @@ -171,9 +174,12 @@ export interface Note extends BaseItem<"note"> { * @deprecated only kept here for migration purposes. */ notebooks?: NotebookReference[]; + /** + * @deprecated + */ + locked?: boolean; pinned: boolean; - locked: boolean; favorite: boolean; localOnly: boolean; conflicted: boolean; @@ -432,6 +438,11 @@ export interface SettingItem< value: SettingItemMap[TKey]; } +export interface Vault extends BaseItem<"vault"> { + title: string; + key: Cipher<"base64">; +} + export interface DeletedItem { id: string; deleted: true; diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts index 3a4fb3a65..0d6334888 100644 --- a/packages/core/vitest.config.ts +++ b/packages/core/vitest.config.ts @@ -25,7 +25,9 @@ export default defineConfig({ test: { setupFiles: ["./globals.setup.js"], coverage: { - reporter: ["text", "html"] + reporter: ["text", "html"], + exclude: ["src/utils/templates/html/languages/*.js"], + include: ["src/**/*.ts"] }, retry: 1, exclude: ["__benches__/**/*.bench.ts"],