core: fix table x has no column y errors

packages/core/__tests__/notes.test.ts

@@ -161,7 +161,7 @@ test("update note", () =>
       title: "I am a new title",
       content: {
         type: TEST_NOTE.content.type,
-        data: "<p><br/></p>"
+        data: '<p data-block-id="p1"><br/></p>'
       },
       pinned: true,
       favorite: true

packages/core/__tests__/utils/index.ts

@@ -75,7 +75,7 @@ const notebookTest = (notebook = TEST_NOTEBOOK) =>
 const TEST_NOTE: { content: NoteContent<false> } = {
   content: {
     type: "tiptap",
-    data: `<p>Hello <span style="color:#f00">This is colorful</span></p>`
+    data: `<p data-block-id="p1">Hello <span style="color:#f00">This is colorful</span></p>`
   }
 };
 

packages/core/src/api/index.ts

@@ -72,6 +72,7 @@ import { CachedCollection } from "../database/cached-collection";
 import { Vaults } from "../collections/vaults";
 import { KVStorage } from "../database/kv";
 import { QueueValue } from "../utils/queue-value";
+import { Sanitizer } from "../database/sanitizer";
 
 type EventSourceConstructor = new (
   uri: string,
@@ -178,10 +179,10 @@ class Database {
   vault = new Vault(this);
   lookup = new Lookup(this);
   backup = new Backup(this);
-  settings = new Settings(this);
   migrations = new Migrations(this);
   monographs = new Monographs(this);
   trash = new Trash(this);
+  sanitizer = new Sanitizer(this.sql);
 
   notebooks = new Notebooks(this);
   tags = new Tags(this);
@@ -194,6 +195,7 @@ class Database {
   relations = new Relations(this);
   notes = new Notes(this);
   vaults = new Vaults(this);
+  settings = new Settings(this);
 
   /**
    * @deprecated only kept here for migration purposes
@@ -270,6 +272,8 @@ class Database {
       this.options.sqliteOptions
     )) as unknown as Kysely<DatabaseSchema>;
 
+    await this.sanitizer.init();
+
     await this.initCollections();
 
     await this.migrations.init();
@@ -294,6 +298,7 @@ class Database {
     await this.reminders.init();
     await this.relations.init();
     await this.notes.init();
+    await this.vaults.init();
 
     await this.trash.init();
 

packages/core/src/collections/attachments.ts

@@ -40,7 +40,8 @@ export class Attachments implements ICollection {
       db.sql,
       db.transaction,
       "attachments",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
     this.key = null;
 

packages/core/src/collections/colors.ts

@@ -43,7 +43,8 @@ export class Colors implements ICollection {
       db.sql,
       db.transaction,
       "colors",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 

packages/core/src/collections/content.ts

@@ -56,7 +56,8 @@ export class Content implements ICollection {
       db.sql,
       db.transaction,
       "content",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 

packages/core/src/collections/note-history.ts

@@ -35,7 +35,8 @@ export class NoteHistory implements ICollection {
       db.sql,
       db.transaction,
       "notehistory",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 

packages/core/src/collections/notebooks.ts

@@ -37,7 +37,8 @@ export class Notebooks implements ICollection {
       db.sql,
       db.transaction,
       "notebooks",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 

packages/core/src/collections/notes.ts

@@ -51,7 +51,8 @@ export class Notes implements ICollection {
       db.sql,
       db.transaction,
       "notes",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 

packages/core/src/collections/relations.ts

@@ -34,13 +34,14 @@ export class Relations implements ICollection {
       db.sql,
       db.transaction,
       "relations",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 
   async init() {
     // await this.buildCache();
-    // return this.collection.init();
+    await this.collection.init();
   }
 
   async add(from: ItemReference, to: ItemReference) {

packages/core/src/collections/reminders.ts

@@ -43,7 +43,8 @@ export class Reminders implements ICollection {
       db.sql,
       db.transaction,
       "reminders",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 

packages/core/src/collections/session-content.ts

@@ -39,7 +39,8 @@ export class SessionContent implements ICollection {
       db.sql,
       db.transaction,
       "sessioncontent",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 

packages/core/src/collections/settings.ts

@@ -82,7 +82,8 @@ export class Settings implements ICollection {
       db.sql,
       db.transaction,
       "settings",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 

packages/core/src/collections/shortcuts.ts

@@ -38,7 +38,8 @@ export class Shortcuts implements ICollection {
       db.sql,
       db.transaction,
       "shortcuts",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 

packages/core/src/collections/tags.ts

@@ -32,7 +32,8 @@ export class Tags implements ICollection {
       db.sql,
       db.transaction,
       "tags",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 

packages/core/src/collections/vaults.ts

@@ -32,11 +32,14 @@ export class Vaults implements ICollection {
       db.sql,
       db.transaction,
       "vaults",
-      db.eventManager
+      db.eventManager,
+      db.sanitizer
     );
   }
 
-  async init() {}
+  async init() {
+    await this.collection.init();
+  }
 
   async add(item: Partial<Vault>) {
     const id = item.id || getId();

packages/core/src/content-types/__tests__/tiptap.test.js

@@ -26,7 +26,7 @@ import { test, expect } from "vitest";
 
 test("img src is empty after extract attachments", async () => {
   const tiptap = new Tiptap(IMG_CONTENT_WITHOUT_HASH);
-  const result = await tiptap.extractAttachments(async () => {
+  const result = await tiptap.postProcess(async () => {
     return "helloworld";
   });
   expect(result.hashes).toHaveLength(1);
@@ -37,7 +37,7 @@ test("img src is empty after extract attachments", async () => {
 
 test("img src is present after insert attachments", async () => {
   const tiptap = new Tiptap(IMG_CONTENT);
-  const result = await tiptap.extractAttachments(async () => {
+  const result = await tiptap.postProcess(async () => {
     return { key: "hello", metadata: {} };
   });
   const tiptap2 = new Tiptap(result.data);

packages/core/src/database/migrator.ts

@@ -71,7 +71,8 @@ class Migrator {
         db.sql,
         db.transaction,
         collection.table,
-        db.eventManager
+        db.eventManager,
+        db.sanitizer
       );
       if (version <= 5.9) {
         if (collection.name === "settings") {

packages/core/src/database/sanitizer.ts

@@ -0,0 +1,67 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { DatabaseAccessor } from ".";
+import { logger } from "../logger";
+
+export class Sanitizer {
+  tables: Record<string, Set<string>> = {};
+  logger = logger.scope("sanitizer");
+  constructor(private readonly db: DatabaseAccessor) {}
+
+  async init() {
+    const metadata = await this.db().introspection.getTables({
+      withInternalKyselyTables: false
+    });
+    for (const table of metadata) {
+      this.tables[table.name] = new Set(table.columns.map((c) => c.name));
+    }
+  }
+
+  /**
+   * Sanitization is done based on the latest table schema in the database. All
+   * unrecognized keys are removed
+   */
+  sanitize(table: string, item: any) {
+    const schema = this.tables[table];
+    if (!schema) {
+      if (process.env.NODE_ENV === "test")
+        throw new Error(
+          `Invalid table: ${table} (expected one of ${Object.keys(
+            this.tables
+          ).join(", ")})`
+        );
+      return false;
+    }
+
+    for (const key in item) {
+      if (schema.has(key)) continue;
+      if (process.env.NODE_ENV === "test")
+        throw new Error(`Found invalid key in item ${key} (${table})`);
+      else
+        this.logger.debug("Found invalid key in item", {
+          table,
+          key,
+          value: item[key]
+        });
+      delete item[key];
+    }
+    return true;
+  }
+}

packages/core/src/database/sql-cached-collection.ts

@@ -22,6 +22,7 @@ import EventManager from "../utils/event-manager";
 import { DatabaseAccessor, DatabaseCollection, DatabaseSchema } from ".";
 import { SQLCollection } from "./sql-collection";
 import { Transaction } from "kysely";
+import { Sanitizer } from "./sanitizer";
 
 export class SQLCachedCollection<
   TCollectionType extends keyof DatabaseSchema,
@@ -38,13 +39,15 @@ export class SQLCachedCollection<
       executor: (tr: Transaction<DatabaseSchema>) => Promise<void>
     ) => Promise<void>,
     type: TCollectionType,
-    eventManager: EventManager
+    eventManager: EventManager,
+    sanitizer: Sanitizer
   ) {
     this.collection = new SQLCollection(
       sql,
       startTransaction,
       type,
-      eventManager
+      eventManager,
+      sanitizer
     );
   }
 

packages/core/src/database/sql-collection.ts

@@ -45,6 +45,7 @@ import {
 import { VirtualizedGrouping } from "../utils/virtualized-grouping";
 import { groupArray } from "../utils/grouping";
 import { toChunks } from "../utils/array";
+import { Sanitizer } from "./sanitizer";
 
 const formats = {
   month: "%Y-%m",
@@ -66,7 +67,8 @@ export class SQLCollection<
       executor: (tr: Transaction<DatabaseSchema>) => Promise<void>
     ) => Promise<void>,
     private readonly type: TCollectionType,
-    private readonly eventManager: EventManager
+    private readonly eventManager: EventManager,
+    private readonly sanitizer: Sanitizer
   ) {}
 
   async clear() {
@@ -87,6 +89,8 @@ export class SQLCollection<
     // the item has become local now, so remove the flags
     delete item.remote;
 
+    if (!this.sanitizer.sanitize(this.type, item)) return;
+
     await this.db()
       .replaceInto<keyof DatabaseSchema>(this.type)
       .values(item)
@@ -163,6 +167,9 @@ export class SQLCollection<
         item.synced = false;
       }
       delete item.remote;
+
+      if (!this.sanitizer.sanitize(this.type, item)) return array;
+
       array.push(item);
       return array;
     }, [] as SQLiteItem<T>[]);
@@ -185,6 +192,7 @@ export class SQLCollection<
     partial: Partial<SQLiteItem<T>>,
     options: { sendEvent: boolean } = { sendEvent: true }
   ) {
+    if (!this.sanitizer.sanitize(this.type, partial)) return;
     await this.db()
       .updateTable<keyof DatabaseSchema>(this.type)
       .where("id", "in", ids)