Mirrors/notesnook
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook.git synced 2025-12-25 16:09:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

core: fix database migration

Browse Source
This commit is contained in:
Abdullah Atta
2023-10-12 10:29:11 +05:00
parent 96e36a6f73
commit 44846cc7b5
11 changed files with 126 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
packages/core/__tests__/notes.test.ts
View File

@@ -23,7 +23,6 @@ import { groupArray } from "../src/utils/grouping";
import {
databaseTest,
noteTest,
groupedTest,
TEST_NOTE,
TEST_NOTEBOOK,
IMG_CONTENT,
@@ -394,17 +393,20 @@ test("get grouped notes by abc", () =>
title: `Conflicted`,
conflicted: true
});
const { grouping, ids } = await db.notes.all.grouped({
const grouping = await db.notes.all.grouped({
groupBy: "abc",
sortDirection: "asc",
sortBy: "title"
});
expect((await grouping.item(ids[0]))?.group?.title).toBe("Conflicted");
expect((await grouping.item(ids[1]))?.group?.title).toBe("Pinned");
expect((await grouping.item(grouping.ids[0]))?.group?.title).toBe(
"Conflicted"
);
expect((await grouping.item(grouping.ids[1]))?.group?.title).toBe("Pinned");
for (let i = 0; i < alphabet.length; ++i) {
expect(
(await grouping.item(ids[i * alphabet.length + 2]))?.group?.title
(await grouping.item(grouping.ids[i * alphabet.length + 2]))?.group
?.title
).toBe(alphabet[i]);
}
}));
@@ -421,16 +423,16 @@ test("get grouped notes by month", () =>
}
}
const { grouping, ids } = await db.notes.all.grouped({
const grouping = await db.notes.all.grouped({
groupBy: "month",
sortDirection: "desc",
sortBy: "dateCreated"
});
for (let month = 11; month >= 0; --month) {
expect((await grouping.item(ids[(11 - month) * 5]))?.group?.title).toBe(
MONTHS_FULL[month]
);
expect(
(await grouping.item(grouping.ids[(11 - month) * 5]))?.group?.title
).toBe(MONTHS_FULL[month]);
}
}));
@@ -446,16 +448,16 @@ test("get grouped notes by year", () =>
}
}
const { grouping, ids } = await db.notes.all.grouped({
const grouping = await db.notes.all.grouped({
groupBy: "year",
sortDirection: "desc",
sortBy: "dateCreated"
});
for (let year = 2020; year <= 2025; ++year) {
expect((await grouping.item(ids[(2025 - year) * 5]))?.group?.title).toBe(
year.toString()
);
expect(
(await grouping.item(grouping.ids[(2025 - year) * 5]))?.group?.title
).toBe(year.toString());
}
}));
@@ -477,7 +479,7 @@ test("get grouped notes by week", () =>
}
}
const { grouping, ids } = await db.notes.all.grouped({
const grouping = await db.notes.all.grouped({
groupBy: "week",
sortDirection: "desc",
sortBy: "dateCreated"
@@ -491,7 +493,7 @@ test("get grouped notes by week", () =>
"20 - 26 Feb, 2023"
];
for (let i = 1; i <= 5; ++i) {
expect((await grouping.item(ids[i * 4]))?.group?.title).toBe(
expect((await grouping.item(grouping.ids[i * 4]))?.group?.title).toBe(
weeks[i - 1]
);
}
@@ -516,7 +518,7 @@ test("get grouped notes default", () =>
}
}
const { grouping, ids } = await db.notes.all.grouped({
const grouping = await db.notes.all.grouped({
groupBy: "default",
sortDirection: "desc",
sortBy: "dateCreated"
@@ -524,7 +526,9 @@ test("get grouped notes default", () =>
let i = 0;
for (const key in ranges) {
expect((await grouping.item(ids[i * 7]))?.group?.title).toBe(key);
expect((await grouping.item(grouping.ids[i * 7]))?.group?.title).toBe(
key
);
++i;
}
}));

4
packages/core/src/api/index.ts
View File

@@ -23,8 +23,8 @@ import { FileStorage, FileStorageAccessor } from "../database/fs";
import { Notebooks } from "../collections/notebooks";
import Trash from "../collections/trash";
import Sync, { SyncOptions } from "./sync";
import { Tags } from "../collections/tags";
import { Colors, } from "../collections/colors";
import { Tags } from "../collections/tags";
import { Colors } from "../collections/colors";
import Vault from "./vault";
import Lookup from "./lookup";
import { Content } from "../collections/content";

8
packages/core/src/api/lookup.ts
View File

@@ -28,8 +28,8 @@ import {
Tag,
TrashItem
} from "../types";
import { isFalse } from "../database";
import { sql } from "kysely";
import { DatabaseSchemaWithFTS, isFalse } from "../database";
import { Kysely, sql } from "kysely";
export default class Lookup {
constructor(private readonly db: Database) {}
@@ -38,8 +38,8 @@ export default class Lookup {
query: string,
ids?: string[]
): Promise<Note & { rank: number }[]> {
return (await this.db
.sql()
const db = this.db.sql() as Kysely<DatabaseSchemaWithFTS>;
return (await db
.with("matching", (eb) =>
eb
.selectFrom("content_fts")

110
packages/core/src/api/migrations.ts
View File

@@ -21,6 +21,61 @@ import Database from ".";
import { CURRENT_DATABASE_VERSION } from "../common";
import Migrator, { MigratableCollections } from "../database/migrator";
const collections: MigratableCollections = [
{
name: "settings",
table: "settings"
},
{
name: "settingsv2",
table: "settings"
},
{
name: "attachments",
table: "attachments"
},
{
name: "notebooks",
table: "notebooks"
},
{
name: "tags",
table: "tags"
},
{
name: "colors",
table: "colors"
},
{
name: "content",
table: "content"
},
{
name: "shortcuts",
table: "shortcuts"
},
{
name: "reminders",
table: "reminders"
},
{
name: "relations",
table: "relations"
},
{
name: "notehistory",
table: "relations"
},
{
name: "sessioncontent",
table: "sessioncontent"
},
{
name: "notes",
table: "notes"
}
];
class Migrations {
private readonly migrator = new Migrator();
private migrating = false;
@@ -44,61 +99,6 @@ class Migrations {
await this.db.notes.init();
const collections: MigratableCollections = [
{
items: () => [this.db.legacySettings.raw],
type: "settings"
},
{
items: () => this.db.settings.raw,
type: "settingsv2"
},
{
items: () => this.db.attachments.all,
type: "attachments"
},
{
items: () => this.db.notebooks.raw,
type: "notebooks"
},
{
items: () => this.db.tags.raw,
type: "tags"
},
{
items: () => this.db.colors.raw,
type: "colors"
},
{
iterate: true,
type: "content"
},
{
items: () => this.db.shortcuts.raw,
type: "shortcuts"
},
{
items: () => this.db.reminders.raw,
type: "reminders"
},
{
items: () => this.db.relations.raw,
type: "relations"
},
{
iterate: true,
type: "notehistory"
},
{
iterate: true,
type: "sessioncontent"
},
{
items: () => this.db.notes.raw,
type: "notes"
}
];
await this.migrator.migrate(this.db, collections, this.version);
await this.db.storage().write("v", CURRENT_DATABASE_VERSION);
this.version = CURRENT_DATABASE_VERSION;

2
packages/core/src/api/sync/index.ts
View File

@@ -59,7 +59,7 @@ export default class SyncManager {
} catch (e) {
const isHubException = (e as Error).message.includes("HubException:");
if (isHubException) {
var actualError = /HubException: (.*)/gm.exec((e as Error).message);
const actualError = /HubException: (.*)/gm.exec((e as Error).message);
const errorText =
actualError && actualError.length > 1
? actualError[1]

2
packages/core/src/collections/note-history.ts
View File

@@ -141,7 +141,7 @@ export class NoteHistory implements ICollection {
if (!session || isDeleted(session)) return;
const content = await this.sessionContent.get(session.sessionContentId);
const note = this.db.notes.note(session.noteId);
const note = await this.db.notes.note(session.noteId);
if (!note || !content) return;
if (session.locked && isCipher(content.data)) {

4
packages/core/src/content-types/tiptap.ts
View File

@@ -181,9 +181,9 @@ export class Tiptap {
const images: Record<string, string | false> = {};
for (const image of sources) {
try {
const { data, mime } = dataurl.toObject(image.src);
const { data, mimeType } = dataurl.toObject(image.src);
if (!data) continue;
const hash = await store(data, mime, image.filename);
const hash = await store(data, mimeType, image.filename);
if (!hash) continue;
images[image.id] = hash;

46
packages/core/src/database/migrator.ts
View File

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { DatabaseSchema } from ".";
import Database from "../api";
import {
CURRENT_DATABASE_VERSION,
@@ -32,12 +33,14 @@ import {
isTrashItem
} from "../types";
import { IndexedCollection } from "./indexed-collection";
import { SQLCollection } from "./sql-collection";
export type RawItem = MaybeDeletedItem<Item>;
type MigratableCollection = {
iterate?: boolean;
items?: () => (RawItem | undefined)[];
type: CollectionType;
// iterate?: boolean;
// items?: () => (RawItem | undefined)[];
name: CollectionType;
table: keyof DatabaseSchema;
};
export type MigratableCollections = MigratableCollection[];
@@ -48,35 +51,31 @@ class Migrator {
version: number
) {
for (const collection of collections) {
sendMigrationProgressEvent(db.eventManager, collection.type, 0, 0);
sendMigrationProgressEvent(db.eventManager, collection.name, 0, 0);
const indexedCollection = new IndexedCollection(
db.storage,
collection.type,
collection.name,
db.eventManager
);
const table = new SQLCollection(
db.sql,
collection.table,
db.eventManager
);
await migrateCollection(indexedCollection, version);
if (collection.items) {
await indexedCollection.init();
await table.init();
for await (const entries of indexedCollection.iterate(100)) {
await this.migrateItems(
db,
collection.type,
indexedCollection,
collection.items(),
table,
collection.name,
entries.map((i) => i[1]),
version
);
} else if (collection.iterate) {
await indexedCollection.init();
for await (const entries of indexedCollection.iterate(100)) {
await this.migrateItems(
db,
collection.type,
indexedCollection,
entries.map((i) => i[1]),
version
);
}
}
}
await db.initCollections();
@@ -85,8 +84,8 @@ class Migrator {
async migrateItems(
db: Database,
table: SQLCollection<keyof DatabaseSchema>,
type: keyof Collections,
collection: IndexedCollection,
items: (RawItem | undefined)[],
version: number
) {
@@ -131,13 +130,14 @@ class Migrator {
// if id changed after migration, we need to delete the old one.
if (item.id !== itemId) {
await collection.deleteItem(itemId);
// await collection.deleteItem(itemId);
}
}
}
if (toAdd.length > 0) {
await collection.setItems(toAdd);
await table.put(toAdd as any);
// await collection.setItems(toAdd);
sendMigrationProgressEvent(
db.eventManager,
type,

33
packages/core/src/database/sql-collection.ts
View File

@@ -308,25 +308,22 @@ export class FilteredSelector<T extends Item> {
async grouped(options: GroupOptions) {
const ids = await this.ids(options);
return {
return new VirtualizedGrouping<T>(
ids,
grouping: new VirtualizedGrouping<T>(
ids,
this.batchSize,
async (ids) => {
const results = await this.filter
.where("id", "in", ids)
.selectAll()
.execute();
const items: Record<string, T> = {};
for (const item of results) {
items[item.id] = item as T;
}
return items;
},
(ids, items) => groupArray(ids, items, options)
)
};
this.batchSize,
async (ids) => {
const results = await this.filter
.where("id", "in", ids)
.selectAll()
.execute();
const items: Record<string, T> = {};
for (const item of results) {
items[item.id] = item as T;
}
return items;
},
(ids, items) => groupArray(ids, items, options)
);
}
private buildSortExpression(options: GroupOptions) {

2
packages/core/src/models/note.ts
View File

@@ -26,7 +26,7 @@ export function createNoteModel(note: Note, db: Database) {
data: note,
async content() {
if (!note.contentId) return null;
const content = await db.content.raw(note.contentId);
const content = await db.content.get(note.contentId);
return content && !isDeleted(content) ? content.data : null;
},
synced() {

2
packages/core/src/utils/virtualized-grouping.ts
View File

@@ -26,7 +26,7 @@ export class VirtualizedGrouping<T> {
private groups: Map<number, VirtualizedGroupHeader[]> = new Map();
constructor(
private ids: string[],
public ids: string[],
private readonly batchSize: number,
private readonly fetchItems: (ids: string[]) => Promise<Record<string, T>>,
private readonly groupItems: (