From 44846cc7b54cb1efaeb097b950467d5703cf2dba Mon Sep 17 00:00:00 2001 From: Abdullah Atta Date: Thu, 12 Oct 2023 10:29:11 +0500 Subject: [PATCH] core: fix database migration --- packages/core/__tests__/notes.test.ts | 38 +++--- packages/core/src/api/index.ts | 4 +- packages/core/src/api/lookup.ts | 8 +- packages/core/src/api/migrations.ts | 110 +++++++++--------- packages/core/src/api/sync/index.ts | 2 +- packages/core/src/collections/note-history.ts | 2 +- packages/core/src/content-types/tiptap.ts | 4 +- packages/core/src/database/migrator.ts | 46 ++++---- packages/core/src/database/sql-collection.ts | 33 +++--- packages/core/src/models/note.ts | 2 +- .../core/src/utils/virtualized-grouping.ts | 2 +- 11 files changed, 126 insertions(+), 125 deletions(-) diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.ts index 85755b5ab..132ce6140 100644 --- a/packages/core/__tests__/notes.test.ts +++ b/packages/core/__tests__/notes.test.ts @@ -23,7 +23,6 @@ import { groupArray } from "../src/utils/grouping"; import { databaseTest, noteTest, - groupedTest, TEST_NOTE, TEST_NOTEBOOK, IMG_CONTENT, @@ -394,17 +393,20 @@ test("get grouped notes by abc", () => title: `Conflicted`, conflicted: true }); - const { grouping, ids } = await db.notes.all.grouped({ + const grouping = await db.notes.all.grouped({ groupBy: "abc", sortDirection: "asc", sortBy: "title" }); - expect((await grouping.item(ids[0]))?.group?.title).toBe("Conflicted"); - expect((await grouping.item(ids[1]))?.group?.title).toBe("Pinned"); + expect((await grouping.item(grouping.ids[0]))?.group?.title).toBe( + "Conflicted" + ); + expect((await grouping.item(grouping.ids[1]))?.group?.title).toBe("Pinned"); for (let i = 0; i < alphabet.length; ++i) { expect( - (await grouping.item(ids[i * alphabet.length + 2]))?.group?.title + (await grouping.item(grouping.ids[i * alphabet.length + 2]))?.group + ?.title ).toBe(alphabet[i]); } })); @@ -421,16 +423,16 @@ test("get grouped notes by month", () => } } - const { grouping, ids } = await db.notes.all.grouped({ + const grouping = await db.notes.all.grouped({ groupBy: "month", sortDirection: "desc", sortBy: "dateCreated" }); for (let month = 11; month >= 0; --month) { - expect((await grouping.item(ids[(11 - month) * 5]))?.group?.title).toBe( - MONTHS_FULL[month] - ); + expect( + (await grouping.item(grouping.ids[(11 - month) * 5]))?.group?.title + ).toBe(MONTHS_FULL[month]); } })); @@ -446,16 +448,16 @@ test("get grouped notes by year", () => } } - const { grouping, ids } = await db.notes.all.grouped({ + const grouping = await db.notes.all.grouped({ groupBy: "year", sortDirection: "desc", sortBy: "dateCreated" }); for (let year = 2020; year <= 2025; ++year) { - expect((await grouping.item(ids[(2025 - year) * 5]))?.group?.title).toBe( - year.toString() - ); + expect( + (await grouping.item(grouping.ids[(2025 - year) * 5]))?.group?.title + ).toBe(year.toString()); } })); @@ -477,7 +479,7 @@ test("get grouped notes by week", () => } } - const { grouping, ids } = await db.notes.all.grouped({ + const grouping = await db.notes.all.grouped({ groupBy: "week", sortDirection: "desc", sortBy: "dateCreated" @@ -491,7 +493,7 @@ test("get grouped notes by week", () => "20 - 26 Feb, 2023" ]; for (let i = 1; i <= 5; ++i) { - expect((await grouping.item(ids[i * 4]))?.group?.title).toBe( + expect((await grouping.item(grouping.ids[i * 4]))?.group?.title).toBe( weeks[i - 1] ); } @@ -516,7 +518,7 @@ test("get grouped notes default", () => } } - const { grouping, ids } = await db.notes.all.grouped({ + const grouping = await db.notes.all.grouped({ groupBy: "default", sortDirection: "desc", sortBy: "dateCreated" @@ -524,7 +526,9 @@ test("get grouped notes default", () => let i = 0; for (const key in ranges) { - expect((await grouping.item(ids[i * 7]))?.group?.title).toBe(key); + expect((await grouping.item(grouping.ids[i * 7]))?.group?.title).toBe( + key + ); ++i; } })); diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts index d3f261953..d9029b815 100644 --- a/packages/core/src/api/index.ts +++ b/packages/core/src/api/index.ts @@ -23,8 +23,8 @@ import { FileStorage, FileStorageAccessor } from "../database/fs"; import { Notebooks } from "../collections/notebooks"; import Trash from "../collections/trash"; import Sync, { SyncOptions } from "./sync"; -import { Tags } from "../collections/tags"; -import { Colors, } from "../collections/colors"; +import { Tags } from "../collections/tags"; +import { Colors } from "../collections/colors"; import Vault from "./vault"; import Lookup from "./lookup"; import { Content } from "../collections/content"; diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.ts index 723ed53c6..c5d153875 100644 --- a/packages/core/src/api/lookup.ts +++ b/packages/core/src/api/lookup.ts @@ -28,8 +28,8 @@ import { Tag, TrashItem } from "../types"; -import { isFalse } from "../database"; -import { sql } from "kysely"; +import { DatabaseSchemaWithFTS, isFalse } from "../database"; +import { Kysely, sql } from "kysely"; export default class Lookup { constructor(private readonly db: Database) {} @@ -38,8 +38,8 @@ export default class Lookup { query: string, ids?: string[] ): Promise { - return (await this.db - .sql() + const db = this.db.sql() as Kysely; + return (await db .with("matching", (eb) => eb .selectFrom("content_fts") diff --git a/packages/core/src/api/migrations.ts b/packages/core/src/api/migrations.ts index da88de8d5..42da09168 100644 --- a/packages/core/src/api/migrations.ts +++ b/packages/core/src/api/migrations.ts @@ -21,6 +21,61 @@ import Database from "."; import { CURRENT_DATABASE_VERSION } from "../common"; import Migrator, { MigratableCollections } from "../database/migrator"; +const collections: MigratableCollections = [ + { + name: "settings", + table: "settings" + }, + { + name: "settingsv2", + table: "settings" + }, + { + name: "attachments", + table: "attachments" + }, + { + name: "notebooks", + table: "notebooks" + }, + { + name: "tags", + table: "tags" + }, + { + name: "colors", + table: "colors" + }, + { + name: "content", + table: "content" + }, + { + name: "shortcuts", + table: "shortcuts" + }, + { + name: "reminders", + table: "reminders" + }, + { + name: "relations", + table: "relations" + }, + { + name: "notehistory", + table: "relations" + }, + { + name: "sessioncontent", + table: "sessioncontent" + }, + { + name: "notes", + table: "notes" + } +]; + class Migrations { private readonly migrator = new Migrator(); private migrating = false; @@ -44,61 +99,6 @@ class Migrations { await this.db.notes.init(); - const collections: MigratableCollections = [ - { - items: () => [this.db.legacySettings.raw], - type: "settings" - }, - { - items: () => this.db.settings.raw, - type: "settingsv2" - }, - { - items: () => this.db.attachments.all, - type: "attachments" - }, - { - items: () => this.db.notebooks.raw, - type: "notebooks" - }, - { - items: () => this.db.tags.raw, - type: "tags" - }, - { - items: () => this.db.colors.raw, - type: "colors" - }, - { - iterate: true, - type: "content" - }, - { - items: () => this.db.shortcuts.raw, - type: "shortcuts" - }, - { - items: () => this.db.reminders.raw, - type: "reminders" - }, - { - items: () => this.db.relations.raw, - type: "relations" - }, - { - iterate: true, - type: "notehistory" - }, - { - iterate: true, - type: "sessioncontent" - }, - { - items: () => this.db.notes.raw, - type: "notes" - } - ]; - await this.migrator.migrate(this.db, collections, this.version); await this.db.storage().write("v", CURRENT_DATABASE_VERSION); this.version = CURRENT_DATABASE_VERSION; diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.ts index 978daeba5..ece0d4bb9 100644 --- a/packages/core/src/api/sync/index.ts +++ b/packages/core/src/api/sync/index.ts @@ -59,7 +59,7 @@ export default class SyncManager { } catch (e) { const isHubException = (e as Error).message.includes("HubException:"); if (isHubException) { - var actualError = /HubException: (.*)/gm.exec((e as Error).message); + const actualError = /HubException: (.*)/gm.exec((e as Error).message); const errorText = actualError && actualError.length > 1 ? actualError[1] diff --git a/packages/core/src/collections/note-history.ts b/packages/core/src/collections/note-history.ts index 87769871d..404b53f35 100644 --- a/packages/core/src/collections/note-history.ts +++ b/packages/core/src/collections/note-history.ts @@ -141,7 +141,7 @@ export class NoteHistory implements ICollection { if (!session || isDeleted(session)) return; const content = await this.sessionContent.get(session.sessionContentId); - const note = this.db.notes.note(session.noteId); + const note = await this.db.notes.note(session.noteId); if (!note || !content) return; if (session.locked && isCipher(content.data)) { diff --git a/packages/core/src/content-types/tiptap.ts b/packages/core/src/content-types/tiptap.ts index b584208b8..620fa8702 100644 --- a/packages/core/src/content-types/tiptap.ts +++ b/packages/core/src/content-types/tiptap.ts @@ -181,9 +181,9 @@ export class Tiptap { const images: Record = {}; for (const image of sources) { try { - const { data, mime } = dataurl.toObject(image.src); + const { data, mimeType } = dataurl.toObject(image.src); if (!data) continue; - const hash = await store(data, mime, image.filename); + const hash = await store(data, mimeType, image.filename); if (!hash) continue; images[image.id] = hash; diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.ts index cabeba08d..8a0ab4a41 100644 --- a/packages/core/src/database/migrator.ts +++ b/packages/core/src/database/migrator.ts @@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +import { DatabaseSchema } from "."; import Database from "../api"; import { CURRENT_DATABASE_VERSION, @@ -32,12 +33,14 @@ import { isTrashItem } from "../types"; import { IndexedCollection } from "./indexed-collection"; +import { SQLCollection } from "./sql-collection"; export type RawItem = MaybeDeletedItem; type MigratableCollection = { - iterate?: boolean; - items?: () => (RawItem | undefined)[]; - type: CollectionType; + // iterate?: boolean; + // items?: () => (RawItem | undefined)[]; + name: CollectionType; + table: keyof DatabaseSchema; }; export type MigratableCollections = MigratableCollection[]; @@ -48,35 +51,31 @@ class Migrator { version: number ) { for (const collection of collections) { - sendMigrationProgressEvent(db.eventManager, collection.type, 0, 0); + sendMigrationProgressEvent(db.eventManager, collection.name, 0, 0); const indexedCollection = new IndexedCollection( db.storage, - collection.type, + collection.name, + db.eventManager + ); + const table = new SQLCollection( + db.sql, + collection.table, db.eventManager ); await migrateCollection(indexedCollection, version); - if (collection.items) { + await indexedCollection.init(); + await table.init(); + for await (const entries of indexedCollection.iterate(100)) { await this.migrateItems( db, - collection.type, - indexedCollection, - collection.items(), + table, + collection.name, + entries.map((i) => i[1]), version ); - } else if (collection.iterate) { - await indexedCollection.init(); - for await (const entries of indexedCollection.iterate(100)) { - await this.migrateItems( - db, - collection.type, - indexedCollection, - entries.map((i) => i[1]), - version - ); - } } } await db.initCollections(); @@ -85,8 +84,8 @@ class Migrator { async migrateItems( db: Database, + table: SQLCollection, type: keyof Collections, - collection: IndexedCollection, items: (RawItem | undefined)[], version: number ) { @@ -131,13 +130,14 @@ class Migrator { // if id changed after migration, we need to delete the old one. if (item.id !== itemId) { - await collection.deleteItem(itemId); + // await collection.deleteItem(itemId); } } } if (toAdd.length > 0) { - await collection.setItems(toAdd); + await table.put(toAdd as any); + // await collection.setItems(toAdd); sendMigrationProgressEvent( db.eventManager, type, diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.ts index b3be4baab..347a08c4e 100644 --- a/packages/core/src/database/sql-collection.ts +++ b/packages/core/src/database/sql-collection.ts @@ -308,25 +308,22 @@ export class FilteredSelector { async grouped(options: GroupOptions) { const ids = await this.ids(options); - return { + return new VirtualizedGrouping( ids, - grouping: new VirtualizedGrouping( - ids, - this.batchSize, - async (ids) => { - const results = await this.filter - .where("id", "in", ids) - .selectAll() - .execute(); - const items: Record = {}; - for (const item of results) { - items[item.id] = item as T; - } - return items; - }, - (ids, items) => groupArray(ids, items, options) - ) - }; + this.batchSize, + async (ids) => { + const results = await this.filter + .where("id", "in", ids) + .selectAll() + .execute(); + const items: Record = {}; + for (const item of results) { + items[item.id] = item as T; + } + return items; + }, + (ids, items) => groupArray(ids, items, options) + ); } private buildSortExpression(options: GroupOptions) { diff --git a/packages/core/src/models/note.ts b/packages/core/src/models/note.ts index f1d8e7a27..68914a28f 100644 --- a/packages/core/src/models/note.ts +++ b/packages/core/src/models/note.ts @@ -26,7 +26,7 @@ export function createNoteModel(note: Note, db: Database) { data: note, async content() { if (!note.contentId) return null; - const content = await db.content.raw(note.contentId); + const content = await db.content.get(note.contentId); return content && !isDeleted(content) ? content.data : null; }, synced() { diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.ts index c748c253d..afdbdbc38 100644 --- a/packages/core/src/utils/virtualized-grouping.ts +++ b/packages/core/src/utils/virtualized-grouping.ts @@ -26,7 +26,7 @@ export class VirtualizedGrouping { private groups: Map = new Map(); constructor( - private ids: string[], + public ids: string[], private readonly batchSize: number, private readonly fetchItems: (ids: string[]) => Promise>, private readonly groupItems: (