diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.ts index a1a065215..f6b5624ea 100644 --- a/packages/core/__tests__/notes.test.ts +++ b/packages/core/__tests__/notes.test.ts @@ -161,7 +161,7 @@ test("update note", () => title: "I am a new title", content: { type: TEST_NOTE.content.type, - data: "


" + data: '


' }, pinned: true, favorite: true diff --git a/packages/core/__tests__/utils/index.ts b/packages/core/__tests__/utils/index.ts index 7f22b889b..2018f46bf 100644 --- a/packages/core/__tests__/utils/index.ts +++ b/packages/core/__tests__/utils/index.ts @@ -75,7 +75,7 @@ const notebookTest = (notebook = TEST_NOTEBOOK) => const TEST_NOTE: { content: NoteContent } = { content: { type: "tiptap", - data: `

Hello This is colorful

` + data: `

Hello This is colorful

` } }; diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts index 2f59fae13..2b9f1b18b 100644 --- a/packages/core/src/api/index.ts +++ b/packages/core/src/api/index.ts @@ -72,6 +72,7 @@ import { CachedCollection } from "../database/cached-collection"; import { Vaults } from "../collections/vaults"; import { KVStorage } from "../database/kv"; import { QueueValue } from "../utils/queue-value"; +import { Sanitizer } from "../database/sanitizer"; type EventSourceConstructor = new ( uri: string, @@ -178,10 +179,10 @@ class Database { vault = new Vault(this); lookup = new Lookup(this); backup = new Backup(this); - settings = new Settings(this); migrations = new Migrations(this); monographs = new Monographs(this); trash = new Trash(this); + sanitizer = new Sanitizer(this.sql); notebooks = new Notebooks(this); tags = new Tags(this); @@ -194,6 +195,7 @@ class Database { relations = new Relations(this); notes = new Notes(this); vaults = new Vaults(this); + settings = new Settings(this); /** * @deprecated only kept here for migration purposes @@ -270,6 +272,8 @@ class Database { this.options.sqliteOptions )) as unknown as Kysely; + await this.sanitizer.init(); + await this.initCollections(); await this.migrations.init(); @@ -294,6 +298,7 @@ class Database { await this.reminders.init(); await this.relations.init(); await this.notes.init(); + await this.vaults.init(); await this.trash.init(); diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.ts index 6d15c3502..6ebef6b3b 100644 --- a/packages/core/src/collections/attachments.ts +++ b/packages/core/src/collections/attachments.ts @@ -40,7 +40,8 @@ export class Attachments implements ICollection { db.sql, db.transaction, "attachments", - db.eventManager + db.eventManager, + db.sanitizer ); this.key = null; diff --git a/packages/core/src/collections/colors.ts b/packages/core/src/collections/colors.ts index 2a5e2a39b..4932af94e 100644 --- a/packages/core/src/collections/colors.ts +++ b/packages/core/src/collections/colors.ts @@ -43,7 +43,8 @@ export class Colors implements ICollection { db.sql, db.transaction, "colors", - db.eventManager + db.eventManager, + db.sanitizer ); } diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.ts index dde305c71..986002f8f 100644 --- a/packages/core/src/collections/content.ts +++ b/packages/core/src/collections/content.ts @@ -56,7 +56,8 @@ export class Content implements ICollection { db.sql, db.transaction, "content", - db.eventManager + db.eventManager, + db.sanitizer ); } diff --git a/packages/core/src/collections/note-history.ts b/packages/core/src/collections/note-history.ts index bb1dc2729..116d18368 100644 --- a/packages/core/src/collections/note-history.ts +++ b/packages/core/src/collections/note-history.ts @@ -35,7 +35,8 @@ export class NoteHistory implements ICollection { db.sql, db.transaction, "notehistory", - db.eventManager + db.eventManager, + db.sanitizer ); } diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.ts index 4ca7df77e..abcc5109b 100644 --- a/packages/core/src/collections/notebooks.ts +++ b/packages/core/src/collections/notebooks.ts @@ -37,7 +37,8 @@ export class Notebooks implements ICollection { db.sql, db.transaction, "notebooks", - db.eventManager + db.eventManager, + db.sanitizer ); } diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.ts index 0d5ea11f2..17300f5de 100644 --- a/packages/core/src/collections/notes.ts +++ b/packages/core/src/collections/notes.ts @@ -51,7 +51,8 @@ export class Notes implements ICollection { db.sql, db.transaction, "notes", - db.eventManager + db.eventManager, + db.sanitizer ); } diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.ts index 81471731a..c094211b1 100644 --- a/packages/core/src/collections/relations.ts +++ b/packages/core/src/collections/relations.ts @@ -34,13 +34,14 @@ export class Relations implements ICollection { db.sql, db.transaction, "relations", - db.eventManager + db.eventManager, + db.sanitizer ); } async init() { // await this.buildCache(); - // return this.collection.init(); + await this.collection.init(); } async add(from: ItemReference, to: ItemReference) { diff --git a/packages/core/src/collections/reminders.ts b/packages/core/src/collections/reminders.ts index 1e42bd793..e547112f8 100644 --- a/packages/core/src/collections/reminders.ts +++ b/packages/core/src/collections/reminders.ts @@ -43,7 +43,8 @@ export class Reminders implements ICollection { db.sql, db.transaction, "reminders", - db.eventManager + db.eventManager, + db.sanitizer ); } diff --git a/packages/core/src/collections/session-content.ts b/packages/core/src/collections/session-content.ts index 78ec482b3..d8d3743f7 100644 --- a/packages/core/src/collections/session-content.ts +++ b/packages/core/src/collections/session-content.ts @@ -39,7 +39,8 @@ export class SessionContent implements ICollection { db.sql, db.transaction, "sessioncontent", - db.eventManager + db.eventManager, + db.sanitizer ); } diff --git a/packages/core/src/collections/settings.ts b/packages/core/src/collections/settings.ts index 813ea1a3f..72faf6039 100644 --- a/packages/core/src/collections/settings.ts +++ b/packages/core/src/collections/settings.ts @@ -82,7 +82,8 @@ export class Settings implements ICollection { db.sql, db.transaction, "settings", - db.eventManager + db.eventManager, + db.sanitizer ); } diff --git a/packages/core/src/collections/shortcuts.ts b/packages/core/src/collections/shortcuts.ts index 84bdef8a2..fea418be0 100644 --- a/packages/core/src/collections/shortcuts.ts +++ b/packages/core/src/collections/shortcuts.ts @@ -38,7 +38,8 @@ export class Shortcuts implements ICollection { db.sql, db.transaction, "shortcuts", - db.eventManager + db.eventManager, + db.sanitizer ); } diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.ts index 833c0d74f..45dad046c 100644 --- a/packages/core/src/collections/tags.ts +++ b/packages/core/src/collections/tags.ts @@ -32,7 +32,8 @@ export class Tags implements ICollection { db.sql, db.transaction, "tags", - db.eventManager + db.eventManager, + db.sanitizer ); } diff --git a/packages/core/src/collections/vaults.ts b/packages/core/src/collections/vaults.ts index a8e761eb8..4c4f45d16 100644 --- a/packages/core/src/collections/vaults.ts +++ b/packages/core/src/collections/vaults.ts @@ -32,11 +32,14 @@ export class Vaults implements ICollection { db.sql, db.transaction, "vaults", - db.eventManager + db.eventManager, + db.sanitizer ); } - async init() {} + async init() { + await this.collection.init(); + } async add(item: Partial) { const id = item.id || getId(); diff --git a/packages/core/src/content-types/__tests__/tiptap.test.js b/packages/core/src/content-types/__tests__/tiptap.test.js index 63f1f347d..8c1405d5b 100644 --- a/packages/core/src/content-types/__tests__/tiptap.test.js +++ b/packages/core/src/content-types/__tests__/tiptap.test.js @@ -26,7 +26,7 @@ import { test, expect } from "vitest"; test("img src is empty after extract attachments", async () => { const tiptap = new Tiptap(IMG_CONTENT_WITHOUT_HASH); - const result = await tiptap.extractAttachments(async () => { + const result = await tiptap.postProcess(async () => { return "helloworld"; }); expect(result.hashes).toHaveLength(1); @@ -37,7 +37,7 @@ test("img src is empty after extract attachments", async () => { test("img src is present after insert attachments", async () => { const tiptap = new Tiptap(IMG_CONTENT); - const result = await tiptap.extractAttachments(async () => { + const result = await tiptap.postProcess(async () => { return { key: "hello", metadata: {} }; }); const tiptap2 = new Tiptap(result.data); diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.ts index 1fcb393d7..211c87eeb 100644 --- a/packages/core/src/database/migrator.ts +++ b/packages/core/src/database/migrator.ts @@ -71,7 +71,8 @@ class Migrator { db.sql, db.transaction, collection.table, - db.eventManager + db.eventManager, + db.sanitizer ); if (version <= 5.9) { if (collection.name === "settings") { diff --git a/packages/core/src/database/sanitizer.ts b/packages/core/src/database/sanitizer.ts new file mode 100644 index 000000000..0148836d7 --- /dev/null +++ b/packages/core/src/database/sanitizer.ts @@ -0,0 +1,67 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import { DatabaseAccessor } from "."; +import { logger } from "../logger"; + +export class Sanitizer { + tables: Record> = {}; + logger = logger.scope("sanitizer"); + constructor(private readonly db: DatabaseAccessor) {} + + async init() { + const metadata = await this.db().introspection.getTables({ + withInternalKyselyTables: false + }); + for (const table of metadata) { + this.tables[table.name] = new Set(table.columns.map((c) => c.name)); + } + } + + /** + * Sanitization is done based on the latest table schema in the database. All + * unrecognized keys are removed + */ + sanitize(table: string, item: any) { + const schema = this.tables[table]; + if (!schema) { + if (process.env.NODE_ENV === "test") + throw new Error( + `Invalid table: ${table} (expected one of ${Object.keys( + this.tables + ).join(", ")})` + ); + return false; + } + + for (const key in item) { + if (schema.has(key)) continue; + if (process.env.NODE_ENV === "test") + throw new Error(`Found invalid key in item ${key} (${table})`); + else + this.logger.debug("Found invalid key in item", { + table, + key, + value: item[key] + }); + delete item[key]; + } + return true; + } +} diff --git a/packages/core/src/database/sql-cached-collection.ts b/packages/core/src/database/sql-cached-collection.ts index ae87775c4..042dd9a07 100644 --- a/packages/core/src/database/sql-cached-collection.ts +++ b/packages/core/src/database/sql-cached-collection.ts @@ -22,6 +22,7 @@ import EventManager from "../utils/event-manager"; import { DatabaseAccessor, DatabaseCollection, DatabaseSchema } from "."; import { SQLCollection } from "./sql-collection"; import { Transaction } from "kysely"; +import { Sanitizer } from "./sanitizer"; export class SQLCachedCollection< TCollectionType extends keyof DatabaseSchema, @@ -38,13 +39,15 @@ export class SQLCachedCollection< executor: (tr: Transaction) => Promise ) => Promise, type: TCollectionType, - eventManager: EventManager + eventManager: EventManager, + sanitizer: Sanitizer ) { this.collection = new SQLCollection( sql, startTransaction, type, - eventManager + eventManager, + sanitizer ); } diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.ts index 8b15ece25..4fe7971af 100644 --- a/packages/core/src/database/sql-collection.ts +++ b/packages/core/src/database/sql-collection.ts @@ -45,6 +45,7 @@ import { import { VirtualizedGrouping } from "../utils/virtualized-grouping"; import { groupArray } from "../utils/grouping"; import { toChunks } from "../utils/array"; +import { Sanitizer } from "./sanitizer"; const formats = { month: "%Y-%m", @@ -66,7 +67,8 @@ export class SQLCollection< executor: (tr: Transaction) => Promise ) => Promise, private readonly type: TCollectionType, - private readonly eventManager: EventManager + private readonly eventManager: EventManager, + private readonly sanitizer: Sanitizer ) {} async clear() { @@ -87,6 +89,8 @@ export class SQLCollection< // the item has become local now, so remove the flags delete item.remote; + if (!this.sanitizer.sanitize(this.type, item)) return; + await this.db() .replaceInto(this.type) .values(item) @@ -163,6 +167,9 @@ export class SQLCollection< item.synced = false; } delete item.remote; + + if (!this.sanitizer.sanitize(this.type, item)) return array; + array.push(item); return array; }, [] as SQLiteItem[]); @@ -185,6 +192,7 @@ export class SQLCollection< partial: Partial>, options: { sendEvent: boolean } = { sendEvent: true } ) { + if (!this.sanitizer.sanitize(this.type, partial)) return; await this.db() .updateTable(this.type) .where("id", "in", ids)