Mirrors/notesnook
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook.git synced 2026-09-02 04:01:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

core: migrate key value storage to sqlite

Browse Source
This commit is contained in:
Abdullah Atta
2024-02-05 22:05:55 +05:00
parent 7048c7e8db
commit 7bddfa096f
26 changed files with 383 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
packages/core/__tests__/migrations.test.ts
View File

@@ -18,12 +18,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { test, expect, describe } from "vitest";
import { migrateItem } from "../src/migrations";
import { migrateItem, migrateKV, migrateVaultKey } from "../src/migrations";
import { databaseTest } from "./utils";
import { getId, makeId } from "../src/utils/id";
import { LegacySettingsItem } from "../src/types";
import { KEYS } from "../src/database/kv";
describe("[5.2] replace date edited with date modified", () => {
describe.concurrent("[5.2] replace date edited with date modified", () => {
const itemsWithDateEdited = ["note", "notebook", "trash", "tiny"] as const;
const itemsWithoutDateEdited = ["tag", "attachment", "settings"] as const;
for (const type of itemsWithDateEdited) {
@@ -88,7 +89,7 @@ test("[5.3] decode wrapped table html entities", () =>
);
}));
describe("[5.4] convert tiny to tiptap", () => {
describe.concurrent("[5.4] convert tiny to tiptap", () => {
const cases = [
{
name: "preserve newlines in code blocks",
@@ -268,7 +269,7 @@ test("[5.8] do nothing if backup type is not local", () =>
expect(await migrateItem(item, 5.8, 5.9, "note", db, "sync")).toBe(false);
}));
describe("[5.9] make tags syncable", () => {
describe.concurrent("[5.9] make tags syncable", () => {
test("create tags inside notes & link to them using relations", () =>
databaseTest().then(async (db) => {
const noteId = getId();
@@ -403,7 +404,7 @@ describe("[5.9] make tags syncable", () => {
}));
});
describe("[5.9] make colors syncable", () => {
describe.concurrent("[5.9] make colors syncable", () => {
test("create colors from notes & link to them using relations", () =>
databaseTest().then(async (db) => {
const noteId = getId();
@@ -566,65 +567,90 @@ test.todo("[5.9] flatten attachment object", () =>
})
);
describe("[5.9] move topics out of notebooks & use relations", () => {
test("convert topics to subnotebooks", () =>
databaseTest().then(async (db) => {
const notebook = {
id: "parent_notebook",
type: "notebook",
topics: [
{ id: "topics1", title: "Topic 1" },
{ id: "topics2", title: "Topic 2" }
]
};
await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup");
describe.concurrent(
"[5.9] move topics out of notebooks & use relations",
() => {
test("move topics of deleted notebook to trash after migration", () =>
databaseTest().then(async (db) => {
const notebook = {
id: "parent_notebook",
type: "trash",
itemType: "notebook",
dateDeleted: Date.now(),
topics: [
{ id: "topics1", title: "Topic 1" },
{ id: "topics2", title: "Topic 2" }
]
};
const linkedNotebooks = await db.relations
.from({ type: "notebook", id: "parent_notebook" }, "notebook")
.get();
expect(notebook.topics).toBeUndefined();
expect(linkedNotebooks).toHaveLength(2);
expect(linkedNotebooks.some((a) => a.toId === "topics1")).toBeTruthy();
expect(linkedNotebooks.some((a) => a.toId === "topics2")).toBeTruthy();
expect(await db.notebooks.all.count()).toBe(2);
expect(await db.notebooks.notebook("topics1")).toBeDefined();
expect(await db.notebooks.notebook("topics2")).toBeDefined();
}));
await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup");
test("convert topic shortcuts to notebook shortcuts", () =>
databaseTest().then(async (db) => {
const shortcut = {
id: "shortcut1",
type: "shortcut",
item: {
type: "topic",
id: "topics1"
}
};
await migrateItem(shortcut, 5.9, 6.0, "shortcut", db, "backup");
const trash = await db.trash.all();
expect(await db.notebooks.notebook("topics1")).toBeUndefined();
expect(await db.notebooks.notebook("topics2")).toBeUndefined();
expect(trash.find((t) => t.title === "Topic 1")).toBeDefined();
expect(trash.find((t) => t.title === "Topic 2")).toBeDefined();
}));
expect(shortcut.itemType).toBe("notebook");
expect(shortcut.itemId).toBe("topics1");
}));
test("convert topics to subnotebooks", () =>
databaseTest().then(async (db) => {
const notebook = {
id: "parent_notebook",
type: "notebook",
topics: [
{ id: "topics1", title: "Topic 1" },
{ id: "topics2", title: "Topic 2" }
]
};
await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup");
test("convert topic links in note to relations", () =>
databaseTest().then(async (db) => {
const note = {
id: "note1",
type: "note",
notebooks: [{ id: "notebook1", topics: ["topic1", "topic2"] }]
};
await migrateItem(note, 5.9, 6.0, "note", db, "backup");
const linkedNotebooks = await db.relations
.from({ type: "notebook", id: "parent_notebook" }, "notebook")
.get();
expect(notebook.topics).toBeUndefined();
expect(linkedNotebooks).toHaveLength(2);
expect(linkedNotebooks.some((a) => a.toId === "topics1")).toBeTruthy();
expect(linkedNotebooks.some((a) => a.toId === "topics2")).toBeTruthy();
expect(await db.notebooks.all.count()).toBe(2);
expect(await db.notebooks.notebook("topics1")).toBeDefined();
expect(await db.notebooks.notebook("topics2")).toBeDefined();
}));
const linkedNotebooks = await db.relations
.to({ type: "note", id: "note1" }, "notebook")
.get();
expect(note.notebooks).toBeUndefined();
expect(linkedNotebooks).toHaveLength(2);
expect(linkedNotebooks.some((a) => a.fromId === "topic1")).toBeTruthy();
expect(linkedNotebooks.some((a) => a.fromId === "topic2")).toBeTruthy();
}));
});
test("convert topic shortcuts to notebook shortcuts", () =>
databaseTest().then(async (db) => {
const shortcut = {
id: "shortcut1",
type: "shortcut",
item: {
type: "topic",
id: "topics1"
}
};
await migrateItem(shortcut, 5.9, 6.0, "shortcut", db, "backup");
expect(shortcut.itemType).toBe("notebook");
expect(shortcut.itemId).toBe("topics1");
}));
test("convert topic links in note to relations", () =>
databaseTest().then(async (db) => {
const note = {
id: "note1",
type: "note",
notebooks: [{ id: "notebook1", topics: ["topic1", "topic2"] }]
};
await migrateItem(note, 5.9, 6.0, "note", db, "backup");
const linkedNotebooks = await db.relations
.to({ type: "note", id: "note1" }, "notebook")
.get();
expect(note.notebooks).toBeUndefined();
expect(linkedNotebooks).toHaveLength(2);
expect(linkedNotebooks.some((a) => a.fromId === "topic1")).toBeTruthy();
expect(linkedNotebooks.some((a) => a.fromId === "topic2")).toBeTruthy();
}));
}
);
test("[5.9] migrate settings to its own collection", () =>
databaseTest().then(async (db) => {
@@ -665,3 +691,40 @@ test("[5.9] migrate settings to its own collection", () =>
settings.trashCleanupInterval
);
}));
describe.concurrent("[5.9] migrate kv", () => {
for (const key of KEYS) {
test(`${key} (defined)`, () =>
databaseTest().then(async (db) => {
await db.storage().write(key, "test");
await migrateKV(db, 5.9, 6.0);
expect(await db.kv().read(key)).toBeDefined();
expect(await db.storage().read(key)).toBeUndefined();
}));
test(`${key} (undefined)`, () =>
databaseTest().then(async (db) => {
await db.storage().write(key, null);
await migrateKV(db, 5.9, 6.0);
expect(await db.kv().read(key)).toBe(key === "v" ? 6 : undefined);
expect(await db.storage().read(key)).toBe(null);
}));
}
});
test("[5.9] migrate vaultKey", () =>
databaseTest().then(async (db) => {
const key = await db.storage().encrypt({ password: "hello" }, "world");
await db.storage().write("vaultKey", key);
await migrateVaultKey(db, key, 5.9, 6.0);
expect(await db.storage().read("vaultKey")).toBeUndefined();
expect(await db.vaults.default()).toBeDefined();
expect((await db.vaults.default())?.key).toStrictEqual(key);
expect((await db.vaults.default())?.key).toStrictEqual(key);
}));

18
packages/core/__tests__/trash.test.ts
View File

@@ -245,10 +245,11 @@ test("deleting a notebook should not re-delete already deleted subnotebooks", ()
await db.notebooks.moveToTrash(child3);
await db.notebooks.moveToTrash(parent);
expect((await db.trash.all()).some((a) => a.id === child3)).toBe(true);
expect((await db.trash.all()).some((a) => a.id === parent)).toBe(true);
expect((await db.trash.all()).some((a) => a.id === child2)).toBe(false);
expect((await db.trash.all()).some((a) => a.id === child)).toBe(false);
const trash = await db.trash.all("user");
expect(trash.some((a) => a.id === child3)).toBe(true);
expect(trash.some((a) => a.id === parent)).toBe(true);
expect(trash.some((a) => a.id === child2)).toBe(false);
expect(trash.some((a) => a.id === child)).toBe(false);
}));
test("restoring a deleted notebook should also restore all its subnotebooks", () =>
@@ -355,8 +356,9 @@ test("permanently deleting a notebook should not delete independently deleted su
await db.trash.delete(parent);
expect((await db.trash.all()).some((a) => a.id === child3)).toBe(true);
expect((await db.trash.all()).some((a) => a.id === parent)).toBe(false);
expect((await db.trash.all()).some((a) => a.id === child2)).toBe(false);
expect((await db.trash.all()).some((a) => a.id === child)).toBe(false);
const trash = await db.trash.all("user");
expect(trash.some((a) => a.id === child3)).toBe(true);
expect(trash.some((a) => a.id === parent)).toBe(false);
expect(trash.some((a) => a.id === child2)).toBe(false);
expect(trash.some((a) => a.id === child)).toBe(false);
}));

40
packages/core/src/api/index.ts
View File

@@ -29,7 +29,6 @@ import Vault from "./vault";
import Lookup from "./lookup";
import { Content } from "../collections/content";
import Backup from "../database/backup";
import Session from "./session";
import Hosts from "../utils/constants";
import { EV, EVENTS } from "../common";
import { LegacySettings } from "../collections/legacy-settings";
@@ -55,6 +54,7 @@ import {
ICompressor,
IFileStorage,
IStorage,
KVStorageAccessor,
StorageAccessor
} from "../interfaces";
import TokenManager from "./token-manager";
@@ -69,6 +69,7 @@ import {
import { Kysely, Transaction, sql } from "kysely";
import { CachedCollection } from "../database/cached-collection";
import { Vaults } from "../collections/vaults";
import { KVStorage } from "../database/kv";
type EventSourceConstructor = new (
uri: string,
@@ -103,7 +104,7 @@ class Database {
throw new Error(
"Database not initialized. Did you forget to call db.setup()?"
);
return new FileStorage(this.options.fs, this.storage);
return new FileStorage(this.options.fs, this.tokenManager);
};
crypto: CryptoAccessor = () => {
@@ -133,6 +134,9 @@ class Database {
return this._sql;
};
private _kv?: KVStorage;
kv: KVStorageAccessor = () => this._kv || new KVStorage(this.sql);
private _transaction?: Transaction<DatabaseSchema>;
private transactionMutex = new Mutex();
transaction = (
@@ -156,9 +160,8 @@ class Database {
EventSource?: EventSourceConstructor;
eventSource?: EventSource | null;
session = new Session(this.storage);
mfa = new MFAManager(this.storage);
tokenManager = new TokenManager(this.storage);
tokenManager = new TokenManager(this.kv);
mfa = new MFAManager(this.tokenManager);
subscriptions = new Subscriptions(this.tokenManager);
offers = new Offers();
debug = new Debug();
@@ -189,19 +192,15 @@ class Database {
/**
* @deprecated only kept here for migration purposes
*/
legacyTags = new CachedCollection(this.storage, "tags", this.eventManager);
legacyTags = new CachedCollection(this.storage, "tags");
/**
* @deprecated only kept here for migration purposes
*/
legacyColors = new CachedCollection(
this.storage,
"colors",
this.eventManager
);
legacyColors = new CachedCollection(this.storage, "colors");
/**
* @deprecated only kept here for migration purposes
*/
legacyNotes = new CachedCollection(this.storage, "notes", this.eventManager);
legacyNotes = new CachedCollection(this.storage, "notes");
/**
* @deprecated only kept here for migration purposes
*/
@@ -216,15 +215,6 @@ class Database {
this.options = options;
}
async _validate() {
if (!(await this.session.valid())) {
throw new Error(
"Your system clock is not setup correctly. Please adjust your date and time and then retry."
);
}
await this.session.set();
}
async reset() {
await this.storage().clear();
@@ -269,8 +259,6 @@ class Database {
this.options.sqliteOptions
)) as unknown as Kysely<DatabaseSchema>;
await this._validate();
await this.initCollections();
await this.migrations.init();
@@ -383,7 +371,11 @@ class Database {
}
async lastSynced() {
return (await this.storage().read<number | undefined>("lastSynced")) || 0;
return (await this.kv().read("lastSynced")) || 0;
}
setLastSynced(lastSynced: number) {
return this.kv().write("lastSynced", lastSynced);
}
sync(options: SyncOptions) {

4
packages/core/src/api/lookup.ts
View File

@@ -20,7 +20,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { match } from "fuzzyjs";
import Database from ".";
import { Item, Note, TrashItem } from "../types";
import { DatabaseSchema, DatabaseSchemaWithFTS, isFalse } from "../database";
import { DatabaseSchema, RawDatabaseSchema, isFalse } from "../database";
import { AnyColumnWithTable, Kysely, sql } from "kysely";
import { FilteredSelector } from "../database/sql-collection";
import { VirtualizedGrouping } from "../utils/virtualized-grouping";
@@ -43,7 +43,7 @@ export default class Lookup {
return this.toSearchResults(async (limit) => {
if (query.length <= 3) return [];
const db = this.db.sql() as Kysely<DatabaseSchemaWithFTS>;
const db = this.db.sql() as Kysely<RawDatabaseSchema>;
query = query.replace(/"/, '""');
const result = await db
.with("matching", (eb) =>

6
packages/core/src/api/mfa-manager.ts
View File

@@ -20,7 +20,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import http from "../utils/http";
import constants from "../utils/constants";
import TokenManager from "./token-manager";
import { StorageAccessor } from "../interfaces";
const ENDPOINTS = {
setup: "/mfa",
@@ -31,10 +30,7 @@ const ENDPOINTS = {
};
class MFAManager {
tokenManager: TokenManager;
constructor(private readonly storage: StorageAccessor) {
this.tokenManager = new TokenManager(storage);
}
constructor(private readonly tokenManager: TokenManager) {}
async setup(type: "app" | "sms" | "email", phoneNumber?: string) {
const token = await this.tokenManager.getAccessToken();

4
packages/core/src/api/migrations.ts
View File

@@ -85,7 +85,7 @@ class Migrations {
async init() {
this.version =
(await this.db.storage().read("v")) || CURRENT_DATABASE_VERSION;
this.db.storage().write("v", this.version);
this.db.kv().write("v", this.version);
}
required() {
@@ -100,7 +100,7 @@ class Migrations {
await this.db.notes.init();
await this.migrator.migrate(this.db, collections, this.version);
await this.db.storage().write("v", CURRENT_DATABASE_VERSION);
await this.db.kv().write("v", CURRENT_DATABASE_VERSION);
this.version = CURRENT_DATABASE_VERSION;
} finally {
this.migrating = false;

4
packages/core/src/api/monographs.ts
View File

@@ -46,7 +46,7 @@ export class Monographs {
async clear() {
this.monographs = [];
await this.db.storage().write("monographs", this.monographs);
await this.db.kv().write("monographs", this.monographs);
}
async refresh() {
@@ -59,7 +59,7 @@ export class Monographs {
`${Constants.API_HOST}/monographs`,
token
);
await this.db.storage().write("monographs", monographs);
await this.db.kv().write("monographs", monographs);
if (monographs) this.monographs = monographs;
} catch (e) {
console.error(e);

38
packages/core/src/api/session.ts
View File

@@ -1,38 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { StorageAccessor } from "../interfaces";
class Session {
constructor(private readonly storage: StorageAccessor) {}
get() {
return this.storage().read<number>("t");
}
set() {
return this.storage().write("t", Date.now());
}
async valid() {
const t = await this.get();
return !t || t < Date.now();
}
}
export default Session;

14
packages/core/src/api/sync/devices.ts
View File

@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { StorageAccessor } from "../../interfaces";
import { KVStorageAccessor } from "../../interfaces";
import hosts from "../../utils/constants";
import http from "../../utils/http";
import { getId } from "../../utils/id";
@@ -25,7 +25,7 @@ import TokenManager from "../token-manager";
export class SyncDevices {
constructor(
private readonly storage: StorageAccessor,
private readonly kv: KVStorageAccessor,
private readonly tokenManager: TokenManager
) {}
@@ -35,20 +35,18 @@ export class SyncDevices {
const token = await this.tokenManager.getAccessToken();
return http
.post(url, null, token)
.then(() => this.storage().write("deviceId", deviceId));
.then(() => this.kv().write("deviceId", deviceId));
}
async unregister() {
const deviceId = await this.storage().read("deviceId");
const deviceId = await this.kv().read("deviceId");
if (!deviceId) return;
const url = `${hosts.API_HOST}/devices?deviceId=${deviceId}`;
const token = await this.tokenManager.getAccessToken();
return http
.delete(url, token)
.then(() => this.storage().remove("deviceId"));
return http.delete(url, token).then(() => this.kv().delete("deviceId"));
}
get() {
return this.storage().read<string>("deviceId");
return this.kv().read("deviceId");
}
}

14
packages/core/src/api/sync/index.ts
View File

@@ -113,7 +113,7 @@ class Sync {
logger = logger.scope("Sync");
syncConnectionMutex = new Mutex();
connection?: signalr.HubConnection;
devices = new SyncDevices(this.db.storage, this.db.tokenManager);
devices = new SyncDevices(this.db.kv, this.db.tokenManager);
constructor(private readonly db: Database) {
EV.subscribe(EVENTS.userLoggedOut, async () => {
@@ -209,7 +209,13 @@ class Sync {
vaultKey.length > 0
) {
const vault = await this.db.vaults.default();
if (!vault) await migrateVaultKey(this.db, vaultKey, 5.9);
if (!vault)
await migrateVaultKey(
this.db,
vaultKey,
5.9,
CURRENT_DATABASE_VERSION
);
}
return true;
@@ -258,7 +264,7 @@ class Sync {
await this.db.monographs.refresh();
this.logger.info("Stopping sync");
await this.db.storage().write("lastSynced", Date.now());
await this.db.setLastSynced(Date.now());
this.db.eventManager.publish(EVENTS.syncCompleted);
}
@@ -345,7 +351,7 @@ class Sync {
private createConnection() {
if (this.connection) return;
const tokenManager = new TokenManager(this.db.storage);
const tokenManager = new TokenManager(this.db.kv);
this.connection = new signalr.HubConnectionBuilder()
.withUrl(`${Constants.API_HOST}/hubs/sync/v2`, {
accessTokenFactory: async () => {

10
packages/core/src/api/token-manager.ts
View File

@@ -22,9 +22,9 @@ import constants from "../utils/constants";
import { EV, EVENTS } from "../common";
import { withTimeout, Mutex } from "async-mutex";
import { logger } from "../logger";
import { StorageAccessor } from "../interfaces";
import { KVStorageAccessor } from "../interfaces";
type Token = {
export type Token = {
access_token: string;
t: number;
expires_in: number;
@@ -42,10 +42,10 @@ const ENDPOINTS = {
class TokenManager {
mutex = withTimeout(new Mutex(), 10 * 1000);
logger = logger.scope("TokenManager");
constructor(private readonly storage: StorageAccessor) {}
constructor(private readonly storage: KVStorageAccessor) {}
async getToken(renew = true, forceRenew = false): Promise<Token | undefined> {
const token = await this.storage().read<Token>("token");
const token = await this.storage().read("token");
if (!token || !token.access_token) return;
this.logger.info("Access token requested", {
@@ -119,7 +119,7 @@ class TokenManager {
if (!token) return;
const { access_token } = token;
await this.storage().remove("token");
await this.storage().delete("token");
await http.post(
`${constants.AUTH_HOST}${ENDPOINTS.logout}`,
null,

10
packages/core/src/api/user-manager.ts
View File

@@ -68,7 +68,7 @@ const ENDPOINTS = {
class UserManager {
private tokenManager: TokenManager;
constructor(private readonly db: Database) {
this.tokenManager = new TokenManager(this.db.storage);
this.tokenManager = new TokenManager(this.db.kv);
EV.subscribe(EVENTS.userUnauthorized, async (url: string) => {
if (url.includes("/connect/token") || !(await HealthCheck.auth())) return;
@@ -175,7 +175,7 @@ class UserManager {
if (!user) throw new Error("Unauthorized.");
if (!sessionExpired) {
await this.db.storage().write("lastSynced", 0);
await this.db.setLastSynced(0);
await this.db.syncer.devices.register();
}
@@ -228,7 +228,7 @@ class UserManager {
password,
salt: user.salt
});
await this.db.storage().write("lastSynced", 0);
await this.db.setLastSynced(0);
await this.db.syncer.devices.register();
EV.publish(EVENTS.userLoggedIn, user);
@@ -276,11 +276,11 @@ class UserManager {
}
setUser(user: User) {
return this.db.storage().write("user", user);
return this.db.kv().write("user", user);
}
getUser() {
return this.db.storage().read<User>("user");
return this.db.kv().read("user");
}
async resetUser(removeAttachments = true) {

7
packages/core/src/collections/notebooks.ts
View File

@@ -24,6 +24,7 @@ import { ICollection } from "./collection";
import { SQLCollection } from "../database/sql-collection";
import { isFalse } from "../database";
import { sql } from "kysely";
import { deleteItems } from "../utils/array";
export class Notebooks implements ICollection {
name = "notebooks";
@@ -240,7 +241,7 @@ export class Notebooks implements ICollection {
}
async moveToTrash(...ids: string[]) {
this.db.transaction(async (tr) => {
await this.db.transaction(async (tr) => {
const query = tr
.withRecursive(`subNotebooks(id)`, (eb) =>
eb
@@ -265,7 +266,9 @@ export class Notebooks implements ICollection {
.select("id");
const subNotebookIds = (await query.execute()).map((ref) => ref.id);
await this.db.trash.add("notebook", subNotebookIds, "app");
deleteItems(subNotebookIds, ...ids);
if (subNotebookIds.length > 0)
await this.db.trash.add("notebook", subNotebookIds, "app");
await this.db.trash.add("notebook", ids, "user");
});
}

32
packages/core/src/collections/trash.ts
View File

@@ -201,31 +201,37 @@ export default class Trash {
// } else return true;
// }
async all() {
async all(deletedBy?: TrashItem["deletedBy"]) {
return [
...(await this.trashedNotes(this.cache.notes)),
...(await this.trashedNotebooks(this.cache.notebooks))
...(await this.trashedNotes(this.cache.notes, deletedBy)),
...(await this.trashedNotebooks(this.cache.notebooks, deletedBy))
] as TrashItem[];
}
private async trashedNotes(ids: string[]) {
private async trashedNotes(
ids: string[],
deletedBy?: TrashItem["deletedBy"]
) {
return (await this.db
.sql()
.selectFrom("notes")
.where("type", "==", "trash")
.where("id", "in", ids)
.where("deletedBy", "==", "user")
.$if(!!deletedBy, (eb) => eb.where("deletedBy", "==", deletedBy))
.selectAll()
.execute()) as TrashItem[];
}
private async trashedNotebooks(ids: string[]) {
private async trashedNotebooks(
ids: string[],
deletedBy?: TrashItem["deletedBy"]
) {
return (await this.db
.sql()
.selectFrom("notebooks")
.where("type", "==", "trash")
.where("id", "in", ids)
.where("deletedBy", "==", "user")
.$if(!!deletedBy, (eb) => eb.where("deletedBy", "==", deletedBy))
.selectAll()
.execute()) as TrashItem[];
}
@@ -249,10 +255,12 @@ export default class Trash {
const items = [
...(await this.trashedNotes(
this.cache.notes.slice(notesRange[0], notesRange[1])
this.cache.notes.slice(notesRange[0], notesRange[1]),
"user"
)),
...(await this.trashedNotebooks(
this.cache.notebooks.slice(notebooksRange[0], notebooksRange[1])
this.cache.notebooks.slice(notebooksRange[0], notebooksRange[1]),
"user"
))
];
items.sort(selector);
@@ -314,6 +322,10 @@ export default class Trash {
.selectFrom("subNotebooks")
.select("id")
.execute();
return ids.map((ref) => ref.id);
return deleteItems(
ids.map((ref) => ref.id),
...notebookIds
);
}
}

4
packages/core/src/database/backup.ts
View File

@@ -154,11 +154,11 @@ export default class Backup {
constructor(private readonly db: Database) {}
lastBackupTime() {
return this.db.storage().read<number>("lastBackupTime");
return this.db.kv().read("lastBackupTime");
}
async updateBackupTime() {
await this.db.storage().write("lastBackupTime", Date.now());
await this.db.kv().write("lastBackupTime", Date.now());
}
async *export(type: BackupPlatform, encrypt = false) {

66
packages/core/src/database/cached-collection.ts
View File

@@ -26,8 +26,6 @@ import {
isDeleted
} from "../types";
import { StorageAccessor } from "../interfaces";
import EventManager from "../utils/event-manager";
import { chunkedIterate } from "../utils/array";
/**
* @deprecated only kept here for migration purposes
@@ -40,12 +38,8 @@ export class CachedCollection<
private cache = new Map<string, MaybeDeletedItem<T>>();
private cachedItems?: T[];
constructor(
storage: StorageAccessor,
type: TCollectionType,
eventManager: EventManager
) {
this.collection = new IndexedCollection(storage, type, eventManager);
constructor(storage: StorageAccessor, type: TCollectionType) {
this.collection = new IndexedCollection(storage, type);
}
async init() {
@@ -68,28 +62,6 @@ export class CachedCollection<
this.invalidateCache();
}
async update(item: T) {
await this.collection.updateItem(item);
this.cache.set(item.id, item);
this.invalidateCache();
}
async delete(id: string) {
this.cache.delete(id);
await this.collection.deleteItem(id);
this.invalidateCache();
}
async remove(id: string) {
this.cache.set(id, {
id,
deleted: true,
dateModified: Date.now()
});
await this.collection.removeItem(id);
this.invalidateCache();
}
exists(id: string) {
const item = this.cache.get(id);
return this.collection.exists(id) && !!item && !isDeleted(item);
@@ -103,19 +75,10 @@ export class CachedCollection<
return this.cache.size;
}
get(id: string) {
const item = this.cache.get(id);
if (!item || isDeleted(item)) return;
return item;
}
getRaw(id: string) {
const item = this.cache.get(id);
return item;
}
raw() {
return Array.from(this.cache.values());
async delete(id: string) {
this.cache.delete(id);
await this.collection.deleteItem(id);
this.invalidateCache();
}
items(map?: (item: T) => T | undefined) {
@@ -133,19 +96,10 @@ export class CachedCollection<
return this.cachedItems;
}
async setItems(items: (MaybeDeletedItem<T> | undefined)[]) {
await this.collection.setItems(items);
for (const item of items) {
if (item) {
this.cache.set(item.id, item);
}
}
this.invalidateCache();
}
*iterateSync(chunkSize: number) {
yield* chunkedIterate(Array.from(this.cache.values()), chunkSize);
get(id: string) {
const item = this.cache.get(id);
if (!item || isDeleted(item)) return;
return item;
}
invalidateCache() {

11
packages/core/src/database/fs.ts
View File

@@ -21,8 +21,7 @@ import hosts from "../utils/constants";
import TokenManager from "../api/token-manager";
import {
FileEncryptionMetadataWithOutputType,
IFileStorage,
StorageAccessor
IFileStorage
} from "../interfaces";
import { DataFormat, SerializedKey } from "@notesnook/crypto/dist/src/types";
import { EV, EVENTS } from "../common";
@@ -38,12 +37,12 @@ export type QueueItem = DownloadableFile & {
};
export class FileStorage {
private readonly tokenManager: TokenManager;
downloads = new Map<string, QueueItem[]>();
uploads = new Map<string, QueueItem[]>();
constructor(private readonly fs: IFileStorage, storage: StorageAccessor) {
this.tokenManager = new TokenManager(storage);
}
constructor(
private readonly fs: IFileStorage,
private readonly tokenManager: TokenManager
) {}
async queueDownloads(
files: DownloadableFile[],

16
packages/core/src/database/index.ts
View File

@@ -89,7 +89,13 @@ export interface DatabaseSchema {
vaults: SQLiteItem<Vault>;
}
export type DatabaseSchemaWithFTS = DatabaseSchema & {
export type RawDatabaseSchema = DatabaseSchema & {
kv: {
key: string;
value?: string | null;
dateModified?: number | null;
};
notes_fts: SQLiteItemWithRowID<{
notes_fts: string;
title: string;
@@ -136,9 +142,9 @@ export interface DatabaseCollection<T, IsAsync extends boolean> {
): IsAsync extends true ? AsyncIterableIterator<T> : IterableIterator<T>;
}
export type DatabaseAccessor = () =>
| Kysely<DatabaseSchema>
| Transaction<DatabaseSchema>;
export type DatabaseAccessor<TSchema = DatabaseSchema> = () =>
| Kysely<TSchema>
| Transaction<TSchema>;
type FilterBooleanProperties<T, Type> = keyof {
[K in keyof T as T[K] extends Type ? K : never]: T[K];
@@ -212,7 +218,7 @@ export type SQLiteOptions = {
pageSize?: number;
};
export async function createDatabase(name: string, options: SQLiteOptions) {
const db = new Kysely<DatabaseSchemaWithFTS>({
const db = new Kysely<RawDatabaseSchema>({
dialect: options.dialect(name),
plugins: [new SqliteBooleanPlugin()]
});

84
packages/core/src/database/indexed-collection.ts
View File

@@ -17,30 +17,26 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { EVENTS } from "../common";
import { toChunks } from "../utils/array";
import { StorageAccessor } from "../interfaces";
import {
CollectionType,
Collections,
ItemMap,
MaybeDeletedItem,
isDeleted
MaybeDeletedItem
} from "../types";
import EventManager from "../utils/event-manager";
import Indexer from "./indexer";
/**
* @deprecated only kept here for migration purposes
*/
export class IndexedCollection<
TCollectionType extends CollectionType = CollectionType,
T extends ItemMap[Collections[TCollectionType]] = ItemMap[Collections[TCollectionType]]
> {
readonly indexer: Indexer<T>;
constructor(
storage: StorageAccessor,
type: TCollectionType,
private readonly eventManager: EventManager
) {
constructor(storage: StorageAccessor, type: TCollectionType) {
this.indexer = new Indexer(storage, type);
}
@@ -48,82 +44,24 @@ export class IndexedCollection<
return this.indexer.clear();
}
async deleteItem(id: string) {
await this.indexer.deindex(id);
return await this.indexer.remove(id);
}
async init() {
await this.indexer.init();
}
async addItem(item: MaybeDeletedItem<T>) {
if (!item.id) throw new Error("The item must contain the id field.");
const exists = this.exists(item.id);
if (!exists && !isDeleted(item))
item.dateCreated = item.dateCreated || Date.now();
await this.updateItem(item);
if (!exists) {
await this.indexer.index(item.id);
}
}
async updateItem(item: MaybeDeletedItem<T>) {
if (!item.id) throw new Error("The item must contain the id field.");
this.eventManager.publish(EVENTS.databaseUpdated, item.id, item);
// if item is newly synced, remote will be true.
if (!item.remote) {
item.dateModified = Date.now();
item.synced = false;
}
// the item has become local now, so remove the flags
delete item.remote;
await this.indexer.write(item.id, item);
}
removeItem(id: string) {
this.eventManager.publish(EVENTS.databaseUpdated, id);
return this.indexer.write(id, {
id,
deleted: true,
dateModified: Date.now()
});
}
async deleteItem(id: string) {
this.eventManager.publish(EVENTS.databaseUpdated, id);
await this.indexer.deindex(id);
return await this.indexer.remove(id);
await this.indexer.index(item.id);
}
exists(id: string) {
return this.indexer.exists(id);
}
async getItem(id: string) {
const item = await this.indexer.read(id);
if (!item) return;
return item;
}
async getItems(indices: string[]) {
const data = await this.indexer.readMulti(indices);
return Object.fromEntries(data);
}
setItems(items: (MaybeDeletedItem<T> | undefined)[]) {
const entries = items.reduce((array, item) => {
if (!item) return array;
if (!item.remote) {
item.dateModified = Date.now();
item.synced = false;
}
delete item.remote;
array.push([item.id, item]);
return array;
}, [] as [string, MaybeDeletedItem<T>][]);
return this.indexer.writeMulti(entries);
}
async *iterate(chunkSize: number) {
const chunks = toChunks(this.indexer.indices, chunkSize);
for (const chunk of chunks) {

75
packages/core/src/database/kv.ts Normal file
View File

@@ -0,0 +1,75 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { DatabaseAccessor, RawDatabaseSchema } from ".";
import { Token } from "../api/token-manager";
import { User } from "../api/user-manager";
interface KV {
v: number;
lastSynced: number;
user: User;
token: Token;
monographs: string[];
deviceId: string;
lastBackupTime: number;
}
export const KEYS: (keyof KV)[] = [
"v",
"lastSynced",
"user",
"token",
"monographs",
"deviceId",
"lastBackupTime"
];
export class KVStorage {
private readonly db: DatabaseAccessor<RawDatabaseSchema>;
constructor(db: DatabaseAccessor) {
this.db = db as unknown as DatabaseAccessor<RawDatabaseSchema>;
}
async read<T extends keyof KV>(key: T): Promise<KV[T] | undefined> {
const result = await this.db()
.selectFrom("kv")
.where("key", "==", key)
.select("value")
.limit(1)
.executeTakeFirst();
if (!result?.value) return;
return JSON.parse(result.value) as KV[T];
}
async write<T extends keyof KV>(key: T, value: KV[T]) {
await this.db()
.replaceInto("kv")
.values({
key,
value: JSON.stringify(value),
dateModified: Date.now()
})
.execute();
}
async delete<T extends keyof KV>(key: T) {
await this.db().deleteFrom("kv").where("key", "==", key).execute();
}
}

8
packages/core/src/database/migrations.ts
View File

@@ -33,6 +33,14 @@ export class NNMigrationProvider implements MigrationProvider {
return {
"1": {
async up(db) {
await db.schema
.createTable("kv")
.modifyEnd(sql`without rowid`)
.addColumn("key", "text", (c) => c.primaryKey().unique().notNull())
.addColumn("value", "text")
.addColumn("dateModified", "integer")
.execute();
await db.schema
.createTable("notes")
// .modifyEnd(sql`without rowid`)

14
packages/core/src/database/migrator.ts
View File

@@ -24,7 +24,12 @@ import {
CURRENT_DATABASE_VERSION,
sendMigrationProgressEvent
} from "../common";
import { migrateCollection, migrateItem, migrateVaultKey } from "../migrations";
import {
migrateCollection,
migrateItem,
migrateKV,
migrateVaultKey
} from "../migrations";
import {
CollectionType,
Collections,
@@ -52,15 +57,16 @@ class Migrator {
version: number
) {
const vaultKey = await db.storage().read<Cipher<"base64">>("vaultKey");
if (vaultKey) await migrateVaultKey(db, vaultKey, version);
if (vaultKey)
await migrateVaultKey(db, vaultKey, version, CURRENT_DATABASE_VERSION);
await migrateKV(db, version, CURRENT_DATABASE_VERSION);
for (const collection of collections) {
sendMigrationProgressEvent(db.eventManager, collection.name, 0, 0);
const indexedCollection = new IndexedCollection(
db.storage,
collection.name,
db.eventManager
collection.name
);
const table = new SQLCollection(
db.sql,

4
packages/core/src/database/triggers.ts
View File

@@ -18,9 +18,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Kysely, sql } from "kysely";
import { DatabaseSchemaWithFTS } from ".";
import { RawDatabaseSchema } from ".";
export async function createTriggers(db: Kysely<DatabaseSchemaWithFTS>) {
export async function createTriggers(db: Kysely<RawDatabaseSchema>) {
// content triggers
await db.schema
.createTrigger("content_after_insert_content_fts")

2
packages/core/src/interfaces.ts
View File

@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";
import { KVStorage } from "./database/kv";
export type Output<TOutputFormat extends DataFormat> =
TOutputFormat extends Omit<DataFormat, "uint8array"> ? string : Uint8Array;
@@ -111,4 +112,5 @@ export interface IFileStorage {
}
export type StorageAccessor = () => IStorage;
export type KVStorageAccessor = () => KVStorage;
export type CompressorAccessor = () => ICompressor;

42
packages/core/src/migrations.ts
View File

@@ -36,6 +36,7 @@ import { isCipher } from "./database/crypto";
import { IndexedCollection } from "./database/indexed-collection";
import { DefaultColors } from "./collections/colors";
import { Cipher } from "@notesnook/crypto";
import { KEYS } from "./database/kv";
type MigrationType = "local" | "sync" | "backup";
type MigrationItemType = ItemType | "notehistory" | "content" | "all";
@@ -55,6 +56,7 @@ type Migration = {
};
collection?: (collection: IndexedCollection) => Promise<void> | void;
vaultKey?: (db: Database, key: Cipher<"base64">) => Promise<void> | void;
kv?: (db: Database) => Promise<void> | void;
};
const migrations: Migration[] = [
@@ -317,7 +319,7 @@ const migrations: Migration[] = [
await db.relations.add(item, { id: subNotebookId, type: "notebook" });
// if the parent notebook is deleted, we should delete the newly
// created notebooks too
if (item.dateDeleted !== null) {
if (item.dateDeleted) {
await db.trash.add("notebook", [subNotebookId], "app");
}
}
@@ -391,6 +393,14 @@ const migrations: Migration[] = [
async vaultKey(db, key) {
await db.vaults.add({ title: "Default", key });
await db.storage().remove("vaultKey");
},
async kv(db) {
for (const key of KEYS) {
const value = await db.storage().read(key);
if (value === undefined || value === null) continue;
await db.kv().write(key, value as any);
await db.storage().remove(key);
}
}
},
{
@@ -467,12 +477,13 @@ export async function migrateCollection(
export async function migrateVaultKey(
db: Database,
vaultKey: Cipher<"base64">,
version: number
version: number,
databaseVersion: number
) {
let migrationStartIndex = migrations.findIndex((m) => m.version === version);
if (migrationStartIndex <= -1) {
throw new Error(
version > CURRENT_DATABASE_VERSION
version > databaseVersion
? `Please update the app to the latest version.`
: `You seem to be on a very outdated version. Please update the app to the latest version.`
);
@@ -480,13 +491,36 @@ export async function migrateVaultKey(
for (; migrationStartIndex < migrations.length; ++migrationStartIndex) {
const migration = migrations[migrationStartIndex];
if (migration.version === CURRENT_DATABASE_VERSION) break;
if (migration.version === databaseVersion) break;
if (!migration.vaultKey) continue;
await migration.vaultKey(db, vaultKey);
}
}
export async function migrateKV(
db: Database,
version: number,
databaseVersion: number
) {
let migrationStartIndex = migrations.findIndex((m) => m.version === version);
if (migrationStartIndex <= -1) {
throw new Error(
version > databaseVersion
? `Please update the app to the latest version.`
: `You seem to be on a very outdated version. Please update the app to the latest version.`
);
}
for (; migrationStartIndex < migrations.length; ++migrationStartIndex) {
const migration = migrations[migrationStartIndex];
if (migration.version === databaseVersion) break;
if (!migration.kv) continue;
await migration.kv(db);
}
}
function replaceDateEditedWithDateModified(removeDateEditedProperty = false) {
return function (item: any) {
item.dateModified = item.dateEdited;

1
packages/core/src/utils/array.ts
View File

@@ -39,6 +39,7 @@ export function deleteItems<T>(array: T[], ...items: T[]) {
for (const item of items) {
deleteItem(array, item);
}
return array;
}
export function findById<T extends { id: string }>(array: T[], id: string) {