diff --git a/packages/core/__tests__/migrations.test.ts b/packages/core/__tests__/migrations.test.ts index e62094b88..6748a7bfa 100644 --- a/packages/core/__tests__/migrations.test.ts +++ b/packages/core/__tests__/migrations.test.ts @@ -18,12 +18,13 @@ along with this program. If not, see . */ import { test, expect, describe } from "vitest"; -import { migrateItem } from "../src/migrations"; +import { migrateItem, migrateKV, migrateVaultKey } from "../src/migrations"; import { databaseTest } from "./utils"; import { getId, makeId } from "../src/utils/id"; import { LegacySettingsItem } from "../src/types"; +import { KEYS } from "../src/database/kv"; -describe("[5.2] replace date edited with date modified", () => { +describe.concurrent("[5.2] replace date edited with date modified", () => { const itemsWithDateEdited = ["note", "notebook", "trash", "tiny"] as const; const itemsWithoutDateEdited = ["tag", "attachment", "settings"] as const; for (const type of itemsWithDateEdited) { @@ -88,7 +89,7 @@ test("[5.3] decode wrapped table html entities", () => ); })); -describe("[5.4] convert tiny to tiptap", () => { +describe.concurrent("[5.4] convert tiny to tiptap", () => { const cases = [ { name: "preserve newlines in code blocks", @@ -268,7 +269,7 @@ test("[5.8] do nothing if backup type is not local", () => expect(await migrateItem(item, 5.8, 5.9, "note", db, "sync")).toBe(false); })); -describe("[5.9] make tags syncable", () => { +describe.concurrent("[5.9] make tags syncable", () => { test("create tags inside notes & link to them using relations", () => databaseTest().then(async (db) => { const noteId = getId(); @@ -403,7 +404,7 @@ describe("[5.9] make tags syncable", () => { })); }); -describe("[5.9] make colors syncable", () => { +describe.concurrent("[5.9] make colors syncable", () => { test("create colors from notes & link to them using relations", () => databaseTest().then(async (db) => { const noteId = getId(); @@ -566,65 +567,90 @@ test.todo("[5.9] flatten attachment object", () => }) ); -describe("[5.9] move topics out of notebooks & use relations", () => { - test("convert topics to subnotebooks", () => - databaseTest().then(async (db) => { - const notebook = { - id: "parent_notebook", - type: "notebook", - topics: [ - { id: "topics1", title: "Topic 1" }, - { id: "topics2", title: "Topic 2" } - ] - }; - await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup"); +describe.concurrent( + "[5.9] move topics out of notebooks & use relations", + () => { + test("move topics of deleted notebook to trash after migration", () => + databaseTest().then(async (db) => { + const notebook = { + id: "parent_notebook", + type: "trash", + itemType: "notebook", + dateDeleted: Date.now(), + topics: [ + { id: "topics1", title: "Topic 1" }, + { id: "topics2", title: "Topic 2" } + ] + }; - const linkedNotebooks = await db.relations - .from({ type: "notebook", id: "parent_notebook" }, "notebook") - .get(); - expect(notebook.topics).toBeUndefined(); - expect(linkedNotebooks).toHaveLength(2); - expect(linkedNotebooks.some((a) => a.toId === "topics1")).toBeTruthy(); - expect(linkedNotebooks.some((a) => a.toId === "topics2")).toBeTruthy(); - expect(await db.notebooks.all.count()).toBe(2); - expect(await db.notebooks.notebook("topics1")).toBeDefined(); - expect(await db.notebooks.notebook("topics2")).toBeDefined(); - })); + await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup"); - test("convert topic shortcuts to notebook shortcuts", () => - databaseTest().then(async (db) => { - const shortcut = { - id: "shortcut1", - type: "shortcut", - item: { - type: "topic", - id: "topics1" - } - }; - await migrateItem(shortcut, 5.9, 6.0, "shortcut", db, "backup"); + const trash = await db.trash.all(); + expect(await db.notebooks.notebook("topics1")).toBeUndefined(); + expect(await db.notebooks.notebook("topics2")).toBeUndefined(); + expect(trash.find((t) => t.title === "Topic 1")).toBeDefined(); + expect(trash.find((t) => t.title === "Topic 2")).toBeDefined(); + })); - expect(shortcut.itemType).toBe("notebook"); - expect(shortcut.itemId).toBe("topics1"); - })); + test("convert topics to subnotebooks", () => + databaseTest().then(async (db) => { + const notebook = { + id: "parent_notebook", + type: "notebook", + topics: [ + { id: "topics1", title: "Topic 1" }, + { id: "topics2", title: "Topic 2" } + ] + }; + await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup"); - test("convert topic links in note to relations", () => - databaseTest().then(async (db) => { - const note = { - id: "note1", - type: "note", - notebooks: [{ id: "notebook1", topics: ["topic1", "topic2"] }] - }; - await migrateItem(note, 5.9, 6.0, "note", db, "backup"); + const linkedNotebooks = await db.relations + .from({ type: "notebook", id: "parent_notebook" }, "notebook") + .get(); + expect(notebook.topics).toBeUndefined(); + expect(linkedNotebooks).toHaveLength(2); + expect(linkedNotebooks.some((a) => a.toId === "topics1")).toBeTruthy(); + expect(linkedNotebooks.some((a) => a.toId === "topics2")).toBeTruthy(); + expect(await db.notebooks.all.count()).toBe(2); + expect(await db.notebooks.notebook("topics1")).toBeDefined(); + expect(await db.notebooks.notebook("topics2")).toBeDefined(); + })); - const linkedNotebooks = await db.relations - .to({ type: "note", id: "note1" }, "notebook") - .get(); - expect(note.notebooks).toBeUndefined(); - expect(linkedNotebooks).toHaveLength(2); - expect(linkedNotebooks.some((a) => a.fromId === "topic1")).toBeTruthy(); - expect(linkedNotebooks.some((a) => a.fromId === "topic2")).toBeTruthy(); - })); -}); + test("convert topic shortcuts to notebook shortcuts", () => + databaseTest().then(async (db) => { + const shortcut = { + id: "shortcut1", + type: "shortcut", + item: { + type: "topic", + id: "topics1" + } + }; + await migrateItem(shortcut, 5.9, 6.0, "shortcut", db, "backup"); + + expect(shortcut.itemType).toBe("notebook"); + expect(shortcut.itemId).toBe("topics1"); + })); + + test("convert topic links in note to relations", () => + databaseTest().then(async (db) => { + const note = { + id: "note1", + type: "note", + notebooks: [{ id: "notebook1", topics: ["topic1", "topic2"] }] + }; + await migrateItem(note, 5.9, 6.0, "note", db, "backup"); + + const linkedNotebooks = await db.relations + .to({ type: "note", id: "note1" }, "notebook") + .get(); + expect(note.notebooks).toBeUndefined(); + expect(linkedNotebooks).toHaveLength(2); + expect(linkedNotebooks.some((a) => a.fromId === "topic1")).toBeTruthy(); + expect(linkedNotebooks.some((a) => a.fromId === "topic2")).toBeTruthy(); + })); + } +); test("[5.9] migrate settings to its own collection", () => databaseTest().then(async (db) => { @@ -665,3 +691,40 @@ test("[5.9] migrate settings to its own collection", () => settings.trashCleanupInterval ); })); + +describe.concurrent("[5.9] migrate kv", () => { + for (const key of KEYS) { + test(`${key} (defined)`, () => + databaseTest().then(async (db) => { + await db.storage().write(key, "test"); + + await migrateKV(db, 5.9, 6.0); + + expect(await db.kv().read(key)).toBeDefined(); + expect(await db.storage().read(key)).toBeUndefined(); + })); + + test(`${key} (undefined)`, () => + databaseTest().then(async (db) => { + await db.storage().write(key, null); + + await migrateKV(db, 5.9, 6.0); + + expect(await db.kv().read(key)).toBe(key === "v" ? 6 : undefined); + expect(await db.storage().read(key)).toBe(null); + })); + } +}); + +test("[5.9] migrate vaultKey", () => + databaseTest().then(async (db) => { + const key = await db.storage().encrypt({ password: "hello" }, "world"); + await db.storage().write("vaultKey", key); + + await migrateVaultKey(db, key, 5.9, 6.0); + + expect(await db.storage().read("vaultKey")).toBeUndefined(); + expect(await db.vaults.default()).toBeDefined(); + expect((await db.vaults.default())?.key).toStrictEqual(key); + expect((await db.vaults.default())?.key).toStrictEqual(key); + })); diff --git a/packages/core/__tests__/trash.test.ts b/packages/core/__tests__/trash.test.ts index ab3e91bfe..60ae8fe59 100644 --- a/packages/core/__tests__/trash.test.ts +++ b/packages/core/__tests__/trash.test.ts @@ -245,10 +245,11 @@ test("deleting a notebook should not re-delete already deleted subnotebooks", () await db.notebooks.moveToTrash(child3); await db.notebooks.moveToTrash(parent); - expect((await db.trash.all()).some((a) => a.id === child3)).toBe(true); - expect((await db.trash.all()).some((a) => a.id === parent)).toBe(true); - expect((await db.trash.all()).some((a) => a.id === child2)).toBe(false); - expect((await db.trash.all()).some((a) => a.id === child)).toBe(false); + const trash = await db.trash.all("user"); + expect(trash.some((a) => a.id === child3)).toBe(true); + expect(trash.some((a) => a.id === parent)).toBe(true); + expect(trash.some((a) => a.id === child2)).toBe(false); + expect(trash.some((a) => a.id === child)).toBe(false); })); test("restoring a deleted notebook should also restore all its subnotebooks", () => @@ -355,8 +356,9 @@ test("permanently deleting a notebook should not delete independently deleted su await db.trash.delete(parent); - expect((await db.trash.all()).some((a) => a.id === child3)).toBe(true); - expect((await db.trash.all()).some((a) => a.id === parent)).toBe(false); - expect((await db.trash.all()).some((a) => a.id === child2)).toBe(false); - expect((await db.trash.all()).some((a) => a.id === child)).toBe(false); + const trash = await db.trash.all("user"); + expect(trash.some((a) => a.id === child3)).toBe(true); + expect(trash.some((a) => a.id === parent)).toBe(false); + expect(trash.some((a) => a.id === child2)).toBe(false); + expect(trash.some((a) => a.id === child)).toBe(false); })); diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts index c32e6a168..65033c550 100644 --- a/packages/core/src/api/index.ts +++ b/packages/core/src/api/index.ts @@ -29,7 +29,6 @@ import Vault from "./vault"; import Lookup from "./lookup"; import { Content } from "../collections/content"; import Backup from "../database/backup"; -import Session from "./session"; import Hosts from "../utils/constants"; import { EV, EVENTS } from "../common"; import { LegacySettings } from "../collections/legacy-settings"; @@ -55,6 +54,7 @@ import { ICompressor, IFileStorage, IStorage, + KVStorageAccessor, StorageAccessor } from "../interfaces"; import TokenManager from "./token-manager"; @@ -69,6 +69,7 @@ import { import { Kysely, Transaction, sql } from "kysely"; import { CachedCollection } from "../database/cached-collection"; import { Vaults } from "../collections/vaults"; +import { KVStorage } from "../database/kv"; type EventSourceConstructor = new ( uri: string, @@ -103,7 +104,7 @@ class Database { throw new Error( "Database not initialized. Did you forget to call db.setup()?" ); - return new FileStorage(this.options.fs, this.storage); + return new FileStorage(this.options.fs, this.tokenManager); }; crypto: CryptoAccessor = () => { @@ -133,6 +134,9 @@ class Database { return this._sql; }; + private _kv?: KVStorage; + kv: KVStorageAccessor = () => this._kv || new KVStorage(this.sql); + private _transaction?: Transaction; private transactionMutex = new Mutex(); transaction = ( @@ -156,9 +160,8 @@ class Database { EventSource?: EventSourceConstructor; eventSource?: EventSource | null; - session = new Session(this.storage); - mfa = new MFAManager(this.storage); - tokenManager = new TokenManager(this.storage); + tokenManager = new TokenManager(this.kv); + mfa = new MFAManager(this.tokenManager); subscriptions = new Subscriptions(this.tokenManager); offers = new Offers(); debug = new Debug(); @@ -189,19 +192,15 @@ class Database { /** * @deprecated only kept here for migration purposes */ - legacyTags = new CachedCollection(this.storage, "tags", this.eventManager); + legacyTags = new CachedCollection(this.storage, "tags"); /** * @deprecated only kept here for migration purposes */ - legacyColors = new CachedCollection( - this.storage, - "colors", - this.eventManager - ); + legacyColors = new CachedCollection(this.storage, "colors"); /** * @deprecated only kept here for migration purposes */ - legacyNotes = new CachedCollection(this.storage, "notes", this.eventManager); + legacyNotes = new CachedCollection(this.storage, "notes"); /** * @deprecated only kept here for migration purposes */ @@ -216,15 +215,6 @@ class Database { this.options = options; } - async _validate() { - if (!(await this.session.valid())) { - throw new Error( - "Your system clock is not setup correctly. Please adjust your date and time and then retry." - ); - } - await this.session.set(); - } - async reset() { await this.storage().clear(); @@ -269,8 +259,6 @@ class Database { this.options.sqliteOptions )) as unknown as Kysely; - await this._validate(); - await this.initCollections(); await this.migrations.init(); @@ -383,7 +371,11 @@ class Database { } async lastSynced() { - return (await this.storage().read("lastSynced")) || 0; + return (await this.kv().read("lastSynced")) || 0; + } + + setLastSynced(lastSynced: number) { + return this.kv().write("lastSynced", lastSynced); } sync(options: SyncOptions) { diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.ts index 5dd3fbfa9..49f519827 100644 --- a/packages/core/src/api/lookup.ts +++ b/packages/core/src/api/lookup.ts @@ -20,7 +20,7 @@ along with this program. If not, see . import { match } from "fuzzyjs"; import Database from "."; import { Item, Note, TrashItem } from "../types"; -import { DatabaseSchema, DatabaseSchemaWithFTS, isFalse } from "../database"; +import { DatabaseSchema, RawDatabaseSchema, isFalse } from "../database"; import { AnyColumnWithTable, Kysely, sql } from "kysely"; import { FilteredSelector } from "../database/sql-collection"; import { VirtualizedGrouping } from "../utils/virtualized-grouping"; @@ -43,7 +43,7 @@ export default class Lookup { return this.toSearchResults(async (limit) => { if (query.length <= 3) return []; - const db = this.db.sql() as Kysely; + const db = this.db.sql() as Kysely; query = query.replace(/"/, '""'); const result = await db .with("matching", (eb) => diff --git a/packages/core/src/api/mfa-manager.ts b/packages/core/src/api/mfa-manager.ts index f9c2890d9..cb8bfda73 100644 --- a/packages/core/src/api/mfa-manager.ts +++ b/packages/core/src/api/mfa-manager.ts @@ -20,7 +20,6 @@ along with this program. If not, see . import http from "../utils/http"; import constants from "../utils/constants"; import TokenManager from "./token-manager"; -import { StorageAccessor } from "../interfaces"; const ENDPOINTS = { setup: "/mfa", @@ -31,10 +30,7 @@ const ENDPOINTS = { }; class MFAManager { - tokenManager: TokenManager; - constructor(private readonly storage: StorageAccessor) { - this.tokenManager = new TokenManager(storage); - } + constructor(private readonly tokenManager: TokenManager) {} async setup(type: "app" | "sms" | "email", phoneNumber?: string) { const token = await this.tokenManager.getAccessToken(); diff --git a/packages/core/src/api/migrations.ts b/packages/core/src/api/migrations.ts index 42da09168..af772f4bd 100644 --- a/packages/core/src/api/migrations.ts +++ b/packages/core/src/api/migrations.ts @@ -85,7 +85,7 @@ class Migrations { async init() { this.version = (await this.db.storage().read("v")) || CURRENT_DATABASE_VERSION; - this.db.storage().write("v", this.version); + this.db.kv().write("v", this.version); } required() { @@ -100,7 +100,7 @@ class Migrations { await this.db.notes.init(); await this.migrator.migrate(this.db, collections, this.version); - await this.db.storage().write("v", CURRENT_DATABASE_VERSION); + await this.db.kv().write("v", CURRENT_DATABASE_VERSION); this.version = CURRENT_DATABASE_VERSION; } finally { this.migrating = false; diff --git a/packages/core/src/api/monographs.ts b/packages/core/src/api/monographs.ts index 13b18636f..2070189c7 100644 --- a/packages/core/src/api/monographs.ts +++ b/packages/core/src/api/monographs.ts @@ -46,7 +46,7 @@ export class Monographs { async clear() { this.monographs = []; - await this.db.storage().write("monographs", this.monographs); + await this.db.kv().write("monographs", this.monographs); } async refresh() { @@ -59,7 +59,7 @@ export class Monographs { `${Constants.API_HOST}/monographs`, token ); - await this.db.storage().write("monographs", monographs); + await this.db.kv().write("monographs", monographs); if (monographs) this.monographs = monographs; } catch (e) { console.error(e); diff --git a/packages/core/src/api/session.ts b/packages/core/src/api/session.ts deleted file mode 100644 index 6cb989a7a..000000000 --- a/packages/core/src/api/session.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* -This file is part of the Notesnook project (https://notesnook.com/) - -Copyright (C) 2023 Streetwriters (Private) Limited - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -import { StorageAccessor } from "../interfaces"; - -class Session { - constructor(private readonly storage: StorageAccessor) {} - - get() { - return this.storage().read("t"); - } - - set() { - return this.storage().write("t", Date.now()); - } - - async valid() { - const t = await this.get(); - return !t || t < Date.now(); - } -} -export default Session; diff --git a/packages/core/src/api/sync/devices.ts b/packages/core/src/api/sync/devices.ts index 449340815..20e55e912 100644 --- a/packages/core/src/api/sync/devices.ts +++ b/packages/core/src/api/sync/devices.ts @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -import { StorageAccessor } from "../../interfaces"; +import { KVStorageAccessor } from "../../interfaces"; import hosts from "../../utils/constants"; import http from "../../utils/http"; import { getId } from "../../utils/id"; @@ -25,7 +25,7 @@ import TokenManager from "../token-manager"; export class SyncDevices { constructor( - private readonly storage: StorageAccessor, + private readonly kv: KVStorageAccessor, private readonly tokenManager: TokenManager ) {} @@ -35,20 +35,18 @@ export class SyncDevices { const token = await this.tokenManager.getAccessToken(); return http .post(url, null, token) - .then(() => this.storage().write("deviceId", deviceId)); + .then(() => this.kv().write("deviceId", deviceId)); } async unregister() { - const deviceId = await this.storage().read("deviceId"); + const deviceId = await this.kv().read("deviceId"); if (!deviceId) return; const url = `${hosts.API_HOST}/devices?deviceId=${deviceId}`; const token = await this.tokenManager.getAccessToken(); - return http - .delete(url, token) - .then(() => this.storage().remove("deviceId")); + return http.delete(url, token).then(() => this.kv().delete("deviceId")); } get() { - return this.storage().read("deviceId"); + return this.kv().read("deviceId"); } } diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.ts index 2d2ea1d08..5e5c38a29 100644 --- a/packages/core/src/api/sync/index.ts +++ b/packages/core/src/api/sync/index.ts @@ -113,7 +113,7 @@ class Sync { logger = logger.scope("Sync"); syncConnectionMutex = new Mutex(); connection?: signalr.HubConnection; - devices = new SyncDevices(this.db.storage, this.db.tokenManager); + devices = new SyncDevices(this.db.kv, this.db.tokenManager); constructor(private readonly db: Database) { EV.subscribe(EVENTS.userLoggedOut, async () => { @@ -209,7 +209,13 @@ class Sync { vaultKey.length > 0 ) { const vault = await this.db.vaults.default(); - if (!vault) await migrateVaultKey(this.db, vaultKey, 5.9); + if (!vault) + await migrateVaultKey( + this.db, + vaultKey, + 5.9, + CURRENT_DATABASE_VERSION + ); } return true; @@ -258,7 +264,7 @@ class Sync { await this.db.monographs.refresh(); this.logger.info("Stopping sync"); - await this.db.storage().write("lastSynced", Date.now()); + await this.db.setLastSynced(Date.now()); this.db.eventManager.publish(EVENTS.syncCompleted); } @@ -345,7 +351,7 @@ class Sync { private createConnection() { if (this.connection) return; - const tokenManager = new TokenManager(this.db.storage); + const tokenManager = new TokenManager(this.db.kv); this.connection = new signalr.HubConnectionBuilder() .withUrl(`${Constants.API_HOST}/hubs/sync/v2`, { accessTokenFactory: async () => { diff --git a/packages/core/src/api/token-manager.ts b/packages/core/src/api/token-manager.ts index 1d9acc9f4..de612bda3 100644 --- a/packages/core/src/api/token-manager.ts +++ b/packages/core/src/api/token-manager.ts @@ -22,9 +22,9 @@ import constants from "../utils/constants"; import { EV, EVENTS } from "../common"; import { withTimeout, Mutex } from "async-mutex"; import { logger } from "../logger"; -import { StorageAccessor } from "../interfaces"; +import { KVStorageAccessor } from "../interfaces"; -type Token = { +export type Token = { access_token: string; t: number; expires_in: number; @@ -42,10 +42,10 @@ const ENDPOINTS = { class TokenManager { mutex = withTimeout(new Mutex(), 10 * 1000); logger = logger.scope("TokenManager"); - constructor(private readonly storage: StorageAccessor) {} + constructor(private readonly storage: KVStorageAccessor) {} async getToken(renew = true, forceRenew = false): Promise { - const token = await this.storage().read("token"); + const token = await this.storage().read("token"); if (!token || !token.access_token) return; this.logger.info("Access token requested", { @@ -119,7 +119,7 @@ class TokenManager { if (!token) return; const { access_token } = token; - await this.storage().remove("token"); + await this.storage().delete("token"); await http.post( `${constants.AUTH_HOST}${ENDPOINTS.logout}`, null, diff --git a/packages/core/src/api/user-manager.ts b/packages/core/src/api/user-manager.ts index 04d9da4ea..8362a7c58 100644 --- a/packages/core/src/api/user-manager.ts +++ b/packages/core/src/api/user-manager.ts @@ -68,7 +68,7 @@ const ENDPOINTS = { class UserManager { private tokenManager: TokenManager; constructor(private readonly db: Database) { - this.tokenManager = new TokenManager(this.db.storage); + this.tokenManager = new TokenManager(this.db.kv); EV.subscribe(EVENTS.userUnauthorized, async (url: string) => { if (url.includes("/connect/token") || !(await HealthCheck.auth())) return; @@ -175,7 +175,7 @@ class UserManager { if (!user) throw new Error("Unauthorized."); if (!sessionExpired) { - await this.db.storage().write("lastSynced", 0); + await this.db.setLastSynced(0); await this.db.syncer.devices.register(); } @@ -228,7 +228,7 @@ class UserManager { password, salt: user.salt }); - await this.db.storage().write("lastSynced", 0); + await this.db.setLastSynced(0); await this.db.syncer.devices.register(); EV.publish(EVENTS.userLoggedIn, user); @@ -276,11 +276,11 @@ class UserManager { } setUser(user: User) { - return this.db.storage().write("user", user); + return this.db.kv().write("user", user); } getUser() { - return this.db.storage().read("user"); + return this.db.kv().read("user"); } async resetUser(removeAttachments = true) { diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.ts index fc5f9851d..e538532a8 100644 --- a/packages/core/src/collections/notebooks.ts +++ b/packages/core/src/collections/notebooks.ts @@ -24,6 +24,7 @@ import { ICollection } from "./collection"; import { SQLCollection } from "../database/sql-collection"; import { isFalse } from "../database"; import { sql } from "kysely"; +import { deleteItems } from "../utils/array"; export class Notebooks implements ICollection { name = "notebooks"; @@ -240,7 +241,7 @@ export class Notebooks implements ICollection { } async moveToTrash(...ids: string[]) { - this.db.transaction(async (tr) => { + await this.db.transaction(async (tr) => { const query = tr .withRecursive(`subNotebooks(id)`, (eb) => eb @@ -265,7 +266,9 @@ export class Notebooks implements ICollection { .select("id"); const subNotebookIds = (await query.execute()).map((ref) => ref.id); - await this.db.trash.add("notebook", subNotebookIds, "app"); + deleteItems(subNotebookIds, ...ids); + if (subNotebookIds.length > 0) + await this.db.trash.add("notebook", subNotebookIds, "app"); await this.db.trash.add("notebook", ids, "user"); }); } diff --git a/packages/core/src/collections/trash.ts b/packages/core/src/collections/trash.ts index 9c67653a9..6efb5dbfb 100644 --- a/packages/core/src/collections/trash.ts +++ b/packages/core/src/collections/trash.ts @@ -201,31 +201,37 @@ export default class Trash { // } else return true; // } - async all() { + async all(deletedBy?: TrashItem["deletedBy"]) { return [ - ...(await this.trashedNotes(this.cache.notes)), - ...(await this.trashedNotebooks(this.cache.notebooks)) + ...(await this.trashedNotes(this.cache.notes, deletedBy)), + ...(await this.trashedNotebooks(this.cache.notebooks, deletedBy)) ] as TrashItem[]; } - private async trashedNotes(ids: string[]) { + private async trashedNotes( + ids: string[], + deletedBy?: TrashItem["deletedBy"] + ) { return (await this.db .sql() .selectFrom("notes") .where("type", "==", "trash") .where("id", "in", ids) - .where("deletedBy", "==", "user") + .$if(!!deletedBy, (eb) => eb.where("deletedBy", "==", deletedBy)) .selectAll() .execute()) as TrashItem[]; } - private async trashedNotebooks(ids: string[]) { + private async trashedNotebooks( + ids: string[], + deletedBy?: TrashItem["deletedBy"] + ) { return (await this.db .sql() .selectFrom("notebooks") .where("type", "==", "trash") .where("id", "in", ids) - .where("deletedBy", "==", "user") + .$if(!!deletedBy, (eb) => eb.where("deletedBy", "==", deletedBy)) .selectAll() .execute()) as TrashItem[]; } @@ -249,10 +255,12 @@ export default class Trash { const items = [ ...(await this.trashedNotes( - this.cache.notes.slice(notesRange[0], notesRange[1]) + this.cache.notes.slice(notesRange[0], notesRange[1]), + "user" )), ...(await this.trashedNotebooks( - this.cache.notebooks.slice(notebooksRange[0], notebooksRange[1]) + this.cache.notebooks.slice(notebooksRange[0], notebooksRange[1]), + "user" )) ]; items.sort(selector); @@ -314,6 +322,10 @@ export default class Trash { .selectFrom("subNotebooks") .select("id") .execute(); - return ids.map((ref) => ref.id); + + return deleteItems( + ids.map((ref) => ref.id), + ...notebookIds + ); } } diff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.ts index 9a42b4781..0de75a9de 100644 --- a/packages/core/src/database/backup.ts +++ b/packages/core/src/database/backup.ts @@ -154,11 +154,11 @@ export default class Backup { constructor(private readonly db: Database) {} lastBackupTime() { - return this.db.storage().read("lastBackupTime"); + return this.db.kv().read("lastBackupTime"); } async updateBackupTime() { - await this.db.storage().write("lastBackupTime", Date.now()); + await this.db.kv().write("lastBackupTime", Date.now()); } async *export(type: BackupPlatform, encrypt = false) { diff --git a/packages/core/src/database/cached-collection.ts b/packages/core/src/database/cached-collection.ts index 796ae609c..b04143fb6 100644 --- a/packages/core/src/database/cached-collection.ts +++ b/packages/core/src/database/cached-collection.ts @@ -26,8 +26,6 @@ import { isDeleted } from "../types"; import { StorageAccessor } from "../interfaces"; -import EventManager from "../utils/event-manager"; -import { chunkedIterate } from "../utils/array"; /** * @deprecated only kept here for migration purposes @@ -40,12 +38,8 @@ export class CachedCollection< private cache = new Map>(); private cachedItems?: T[]; - constructor( - storage: StorageAccessor, - type: TCollectionType, - eventManager: EventManager - ) { - this.collection = new IndexedCollection(storage, type, eventManager); + constructor(storage: StorageAccessor, type: TCollectionType) { + this.collection = new IndexedCollection(storage, type); } async init() { @@ -68,28 +62,6 @@ export class CachedCollection< this.invalidateCache(); } - async update(item: T) { - await this.collection.updateItem(item); - this.cache.set(item.id, item); - this.invalidateCache(); - } - - async delete(id: string) { - this.cache.delete(id); - await this.collection.deleteItem(id); - this.invalidateCache(); - } - - async remove(id: string) { - this.cache.set(id, { - id, - deleted: true, - dateModified: Date.now() - }); - await this.collection.removeItem(id); - this.invalidateCache(); - } - exists(id: string) { const item = this.cache.get(id); return this.collection.exists(id) && !!item && !isDeleted(item); @@ -103,19 +75,10 @@ export class CachedCollection< return this.cache.size; } - get(id: string) { - const item = this.cache.get(id); - if (!item || isDeleted(item)) return; - return item; - } - - getRaw(id: string) { - const item = this.cache.get(id); - return item; - } - - raw() { - return Array.from(this.cache.values()); + async delete(id: string) { + this.cache.delete(id); + await this.collection.deleteItem(id); + this.invalidateCache(); } items(map?: (item: T) => T | undefined) { @@ -133,19 +96,10 @@ export class CachedCollection< return this.cachedItems; } - async setItems(items: (MaybeDeletedItem | undefined)[]) { - await this.collection.setItems(items); - for (const item of items) { - if (item) { - this.cache.set(item.id, item); - } - } - - this.invalidateCache(); - } - - *iterateSync(chunkSize: number) { - yield* chunkedIterate(Array.from(this.cache.values()), chunkSize); + get(id: string) { + const item = this.cache.get(id); + if (!item || isDeleted(item)) return; + return item; } invalidateCache() { diff --git a/packages/core/src/database/fs.ts b/packages/core/src/database/fs.ts index f635d1d4a..d7bea463f 100644 --- a/packages/core/src/database/fs.ts +++ b/packages/core/src/database/fs.ts @@ -21,8 +21,7 @@ import hosts from "../utils/constants"; import TokenManager from "../api/token-manager"; import { FileEncryptionMetadataWithOutputType, - IFileStorage, - StorageAccessor + IFileStorage } from "../interfaces"; import { DataFormat, SerializedKey } from "@notesnook/crypto/dist/src/types"; import { EV, EVENTS } from "../common"; @@ -38,12 +37,12 @@ export type QueueItem = DownloadableFile & { }; export class FileStorage { - private readonly tokenManager: TokenManager; downloads = new Map(); uploads = new Map(); - constructor(private readonly fs: IFileStorage, storage: StorageAccessor) { - this.tokenManager = new TokenManager(storage); - } + constructor( + private readonly fs: IFileStorage, + private readonly tokenManager: TokenManager + ) {} async queueDownloads( files: DownloadableFile[], diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.ts index 40dfb065b..95a86ab71 100644 --- a/packages/core/src/database/index.ts +++ b/packages/core/src/database/index.ts @@ -89,7 +89,13 @@ export interface DatabaseSchema { vaults: SQLiteItem; } -export type DatabaseSchemaWithFTS = DatabaseSchema & { +export type RawDatabaseSchema = DatabaseSchema & { + kv: { + key: string; + value?: string | null; + dateModified?: number | null; + }; + notes_fts: SQLiteItemWithRowID<{ notes_fts: string; title: string; @@ -136,9 +142,9 @@ export interface DatabaseCollection { ): IsAsync extends true ? AsyncIterableIterator : IterableIterator; } -export type DatabaseAccessor = () => - | Kysely - | Transaction; +export type DatabaseAccessor = () => + | Kysely + | Transaction; type FilterBooleanProperties = keyof { [K in keyof T as T[K] extends Type ? K : never]: T[K]; @@ -212,7 +218,7 @@ export type SQLiteOptions = { pageSize?: number; }; export async function createDatabase(name: string, options: SQLiteOptions) { - const db = new Kysely({ + const db = new Kysely({ dialect: options.dialect(name), plugins: [new SqliteBooleanPlugin()] }); diff --git a/packages/core/src/database/indexed-collection.ts b/packages/core/src/database/indexed-collection.ts index 99168e291..aa60e4928 100644 --- a/packages/core/src/database/indexed-collection.ts +++ b/packages/core/src/database/indexed-collection.ts @@ -17,30 +17,26 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -import { EVENTS } from "../common"; import { toChunks } from "../utils/array"; import { StorageAccessor } from "../interfaces"; import { CollectionType, Collections, ItemMap, - MaybeDeletedItem, - isDeleted + MaybeDeletedItem } from "../types"; -import EventManager from "../utils/event-manager"; import Indexer from "./indexer"; +/** + * @deprecated only kept here for migration purposes + */ export class IndexedCollection< TCollectionType extends CollectionType = CollectionType, T extends ItemMap[Collections[TCollectionType]] = ItemMap[Collections[TCollectionType]] > { readonly indexer: Indexer; - constructor( - storage: StorageAccessor, - type: TCollectionType, - private readonly eventManager: EventManager - ) { + constructor(storage: StorageAccessor, type: TCollectionType) { this.indexer = new Indexer(storage, type); } @@ -48,82 +44,24 @@ export class IndexedCollection< return this.indexer.clear(); } + async deleteItem(id: string) { + await this.indexer.deindex(id); + return await this.indexer.remove(id); + } + async init() { await this.indexer.init(); } async addItem(item: MaybeDeletedItem) { - if (!item.id) throw new Error("The item must contain the id field."); - - const exists = this.exists(item.id); - if (!exists && !isDeleted(item)) - item.dateCreated = item.dateCreated || Date.now(); - await this.updateItem(item); - if (!exists) { - await this.indexer.index(item.id); - } - } - - async updateItem(item: MaybeDeletedItem) { - if (!item.id) throw new Error("The item must contain the id field."); - this.eventManager.publish(EVENTS.databaseUpdated, item.id, item); - - // if item is newly synced, remote will be true. - if (!item.remote) { - item.dateModified = Date.now(); - item.synced = false; - } - // the item has become local now, so remove the flags - delete item.remote; await this.indexer.write(item.id, item); - } - - removeItem(id: string) { - this.eventManager.publish(EVENTS.databaseUpdated, id); - return this.indexer.write(id, { - id, - deleted: true, - dateModified: Date.now() - }); - } - - async deleteItem(id: string) { - this.eventManager.publish(EVENTS.databaseUpdated, id); - await this.indexer.deindex(id); - return await this.indexer.remove(id); + await this.indexer.index(item.id); } exists(id: string) { return this.indexer.exists(id); } - async getItem(id: string) { - const item = await this.indexer.read(id); - if (!item) return; - return item; - } - - async getItems(indices: string[]) { - const data = await this.indexer.readMulti(indices); - return Object.fromEntries(data); - } - - setItems(items: (MaybeDeletedItem | undefined)[]) { - const entries = items.reduce((array, item) => { - if (!item) return array; - - if (!item.remote) { - item.dateModified = Date.now(); - item.synced = false; - } - delete item.remote; - - array.push([item.id, item]); - return array; - }, [] as [string, MaybeDeletedItem][]); - return this.indexer.writeMulti(entries); - } - async *iterate(chunkSize: number) { const chunks = toChunks(this.indexer.indices, chunkSize); for (const chunk of chunks) { diff --git a/packages/core/src/database/kv.ts b/packages/core/src/database/kv.ts new file mode 100644 index 000000000..7dbb305bd --- /dev/null +++ b/packages/core/src/database/kv.ts @@ -0,0 +1,75 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import { DatabaseAccessor, RawDatabaseSchema } from "."; +import { Token } from "../api/token-manager"; +import { User } from "../api/user-manager"; + +interface KV { + v: number; + lastSynced: number; + user: User; + token: Token; + monographs: string[]; + deviceId: string; + lastBackupTime: number; +} + +export const KEYS: (keyof KV)[] = [ + "v", + "lastSynced", + "user", + "token", + "monographs", + "deviceId", + "lastBackupTime" +]; + +export class KVStorage { + private readonly db: DatabaseAccessor; + constructor(db: DatabaseAccessor) { + this.db = db as unknown as DatabaseAccessor; + } + + async read(key: T): Promise { + const result = await this.db() + .selectFrom("kv") + .where("key", "==", key) + .select("value") + .limit(1) + .executeTakeFirst(); + if (!result?.value) return; + return JSON.parse(result.value) as KV[T]; + } + + async write(key: T, value: KV[T]) { + await this.db() + .replaceInto("kv") + .values({ + key, + value: JSON.stringify(value), + dateModified: Date.now() + }) + .execute(); + } + + async delete(key: T) { + await this.db().deleteFrom("kv").where("key", "==", key).execute(); + } +} diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.ts index 7949816cb..087f2bbe6 100644 --- a/packages/core/src/database/migrations.ts +++ b/packages/core/src/database/migrations.ts @@ -33,6 +33,14 @@ export class NNMigrationProvider implements MigrationProvider { return { "1": { async up(db) { + await db.schema + .createTable("kv") + .modifyEnd(sql`without rowid`) + .addColumn("key", "text", (c) => c.primaryKey().unique().notNull()) + .addColumn("value", "text") + .addColumn("dateModified", "integer") + .execute(); + await db.schema .createTable("notes") // .modifyEnd(sql`without rowid`) diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.ts index 3708f6a90..707ff64f5 100644 --- a/packages/core/src/database/migrator.ts +++ b/packages/core/src/database/migrator.ts @@ -24,7 +24,12 @@ import { CURRENT_DATABASE_VERSION, sendMigrationProgressEvent } from "../common"; -import { migrateCollection, migrateItem, migrateVaultKey } from "../migrations"; +import { + migrateCollection, + migrateItem, + migrateKV, + migrateVaultKey +} from "../migrations"; import { CollectionType, Collections, @@ -52,15 +57,16 @@ class Migrator { version: number ) { const vaultKey = await db.storage().read>("vaultKey"); - if (vaultKey) await migrateVaultKey(db, vaultKey, version); + if (vaultKey) + await migrateVaultKey(db, vaultKey, version, CURRENT_DATABASE_VERSION); + await migrateKV(db, version, CURRENT_DATABASE_VERSION); for (const collection of collections) { sendMigrationProgressEvent(db.eventManager, collection.name, 0, 0); const indexedCollection = new IndexedCollection( db.storage, - collection.name, - db.eventManager + collection.name ); const table = new SQLCollection( db.sql, diff --git a/packages/core/src/database/triggers.ts b/packages/core/src/database/triggers.ts index 284dc89b6..4e70d5466 100644 --- a/packages/core/src/database/triggers.ts +++ b/packages/core/src/database/triggers.ts @@ -18,9 +18,9 @@ along with this program. If not, see . */ import { Kysely, sql } from "kysely"; -import { DatabaseSchemaWithFTS } from "."; +import { RawDatabaseSchema } from "."; -export async function createTriggers(db: Kysely) { +export async function createTriggers(db: Kysely) { // content triggers await db.schema .createTrigger("content_after_insert_content_fts") diff --git a/packages/core/src/interfaces.ts b/packages/core/src/interfaces.ts index 2b5c47dac..5d792e312 100644 --- a/packages/core/src/interfaces.ts +++ b/packages/core/src/interfaces.ts @@ -18,6 +18,7 @@ along with this program. If not, see . */ import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto"; +import { KVStorage } from "./database/kv"; export type Output = TOutputFormat extends Omit ? string : Uint8Array; @@ -111,4 +112,5 @@ export interface IFileStorage { } export type StorageAccessor = () => IStorage; +export type KVStorageAccessor = () => KVStorage; export type CompressorAccessor = () => ICompressor; diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.ts index c74a07546..407698948 100644 --- a/packages/core/src/migrations.ts +++ b/packages/core/src/migrations.ts @@ -36,6 +36,7 @@ import { isCipher } from "./database/crypto"; import { IndexedCollection } from "./database/indexed-collection"; import { DefaultColors } from "./collections/colors"; import { Cipher } from "@notesnook/crypto"; +import { KEYS } from "./database/kv"; type MigrationType = "local" | "sync" | "backup"; type MigrationItemType = ItemType | "notehistory" | "content" | "all"; @@ -55,6 +56,7 @@ type Migration = { }; collection?: (collection: IndexedCollection) => Promise | void; vaultKey?: (db: Database, key: Cipher<"base64">) => Promise | void; + kv?: (db: Database) => Promise | void; }; const migrations: Migration[] = [ @@ -317,7 +319,7 @@ const migrations: Migration[] = [ await db.relations.add(item, { id: subNotebookId, type: "notebook" }); // if the parent notebook is deleted, we should delete the newly // created notebooks too - if (item.dateDeleted !== null) { + if (item.dateDeleted) { await db.trash.add("notebook", [subNotebookId], "app"); } } @@ -391,6 +393,14 @@ const migrations: Migration[] = [ async vaultKey(db, key) { await db.vaults.add({ title: "Default", key }); await db.storage().remove("vaultKey"); + }, + async kv(db) { + for (const key of KEYS) { + const value = await db.storage().read(key); + if (value === undefined || value === null) continue; + await db.kv().write(key, value as any); + await db.storage().remove(key); + } } }, { @@ -467,12 +477,13 @@ export async function migrateCollection( export async function migrateVaultKey( db: Database, vaultKey: Cipher<"base64">, - version: number + version: number, + databaseVersion: number ) { let migrationStartIndex = migrations.findIndex((m) => m.version === version); if (migrationStartIndex <= -1) { throw new Error( - version > CURRENT_DATABASE_VERSION + version > databaseVersion ? `Please update the app to the latest version.` : `You seem to be on a very outdated version. Please update the app to the latest version.` ); @@ -480,13 +491,36 @@ export async function migrateVaultKey( for (; migrationStartIndex < migrations.length; ++migrationStartIndex) { const migration = migrations[migrationStartIndex]; - if (migration.version === CURRENT_DATABASE_VERSION) break; + if (migration.version === databaseVersion) break; if (!migration.vaultKey) continue; await migration.vaultKey(db, vaultKey); } } +export async function migrateKV( + db: Database, + version: number, + databaseVersion: number +) { + let migrationStartIndex = migrations.findIndex((m) => m.version === version); + if (migrationStartIndex <= -1) { + throw new Error( + version > databaseVersion + ? `Please update the app to the latest version.` + : `You seem to be on a very outdated version. Please update the app to the latest version.` + ); + } + + for (; migrationStartIndex < migrations.length; ++migrationStartIndex) { + const migration = migrations[migrationStartIndex]; + if (migration.version === databaseVersion) break; + + if (!migration.kv) continue; + await migration.kv(db); + } +} + function replaceDateEditedWithDateModified(removeDateEditedProperty = false) { return function (item: any) { item.dateModified = item.dateEdited; diff --git a/packages/core/src/utils/array.ts b/packages/core/src/utils/array.ts index fa92c2955..7a9608193 100644 --- a/packages/core/src/utils/array.ts +++ b/packages/core/src/utils/array.ts @@ -39,6 +39,7 @@ export function deleteItems(array: T[], ...items: T[]) { for (const item of items) { deleteItem(array, item); } + return array; } export function findById(array: T[], id: string) {