diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.ts
index a1a065215..f6b5624ea 100644
--- a/packages/core/__tests__/notes.test.ts
+++ b/packages/core/__tests__/notes.test.ts
@@ -161,7 +161,7 @@ test("update note", () =>
title: "I am a new title",
content: {
type: TEST_NOTE.content.type,
- data: "
"
+ data: '
'
},
pinned: true,
favorite: true
diff --git a/packages/core/__tests__/utils/index.ts b/packages/core/__tests__/utils/index.ts
index 7f22b889b..2018f46bf 100644
--- a/packages/core/__tests__/utils/index.ts
+++ b/packages/core/__tests__/utils/index.ts
@@ -75,7 +75,7 @@ const notebookTest = (notebook = TEST_NOTEBOOK) =>
const TEST_NOTE: { content: NoteContent } = {
content: {
type: "tiptap",
- data: `Hello This is colorful
`
+ data: `Hello This is colorful
`
}
};
diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts
index 2f59fae13..2b9f1b18b 100644
--- a/packages/core/src/api/index.ts
+++ b/packages/core/src/api/index.ts
@@ -72,6 +72,7 @@ import { CachedCollection } from "../database/cached-collection";
import { Vaults } from "../collections/vaults";
import { KVStorage } from "../database/kv";
import { QueueValue } from "../utils/queue-value";
+import { Sanitizer } from "../database/sanitizer";
type EventSourceConstructor = new (
uri: string,
@@ -178,10 +179,10 @@ class Database {
vault = new Vault(this);
lookup = new Lookup(this);
backup = new Backup(this);
- settings = new Settings(this);
migrations = new Migrations(this);
monographs = new Monographs(this);
trash = new Trash(this);
+ sanitizer = new Sanitizer(this.sql);
notebooks = new Notebooks(this);
tags = new Tags(this);
@@ -194,6 +195,7 @@ class Database {
relations = new Relations(this);
notes = new Notes(this);
vaults = new Vaults(this);
+ settings = new Settings(this);
/**
* @deprecated only kept here for migration purposes
@@ -270,6 +272,8 @@ class Database {
this.options.sqliteOptions
)) as unknown as Kysely;
+ await this.sanitizer.init();
+
await this.initCollections();
await this.migrations.init();
@@ -294,6 +298,7 @@ class Database {
await this.reminders.init();
await this.relations.init();
await this.notes.init();
+ await this.vaults.init();
await this.trash.init();
diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.ts
index 6d15c3502..6ebef6b3b 100644
--- a/packages/core/src/collections/attachments.ts
+++ b/packages/core/src/collections/attachments.ts
@@ -40,7 +40,8 @@ export class Attachments implements ICollection {
db.sql,
db.transaction,
"attachments",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
this.key = null;
diff --git a/packages/core/src/collections/colors.ts b/packages/core/src/collections/colors.ts
index 2a5e2a39b..4932af94e 100644
--- a/packages/core/src/collections/colors.ts
+++ b/packages/core/src/collections/colors.ts
@@ -43,7 +43,8 @@ export class Colors implements ICollection {
db.sql,
db.transaction,
"colors",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.ts
index dde305c71..986002f8f 100644
--- a/packages/core/src/collections/content.ts
+++ b/packages/core/src/collections/content.ts
@@ -56,7 +56,8 @@ export class Content implements ICollection {
db.sql,
db.transaction,
"content",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
diff --git a/packages/core/src/collections/note-history.ts b/packages/core/src/collections/note-history.ts
index bb1dc2729..116d18368 100644
--- a/packages/core/src/collections/note-history.ts
+++ b/packages/core/src/collections/note-history.ts
@@ -35,7 +35,8 @@ export class NoteHistory implements ICollection {
db.sql,
db.transaction,
"notehistory",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.ts
index 4ca7df77e..abcc5109b 100644
--- a/packages/core/src/collections/notebooks.ts
+++ b/packages/core/src/collections/notebooks.ts
@@ -37,7 +37,8 @@ export class Notebooks implements ICollection {
db.sql,
db.transaction,
"notebooks",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.ts
index 0d5ea11f2..17300f5de 100644
--- a/packages/core/src/collections/notes.ts
+++ b/packages/core/src/collections/notes.ts
@@ -51,7 +51,8 @@ export class Notes implements ICollection {
db.sql,
db.transaction,
"notes",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.ts
index 81471731a..c094211b1 100644
--- a/packages/core/src/collections/relations.ts
+++ b/packages/core/src/collections/relations.ts
@@ -34,13 +34,14 @@ export class Relations implements ICollection {
db.sql,
db.transaction,
"relations",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
async init() {
// await this.buildCache();
- // return this.collection.init();
+ await this.collection.init();
}
async add(from: ItemReference, to: ItemReference) {
diff --git a/packages/core/src/collections/reminders.ts b/packages/core/src/collections/reminders.ts
index 1e42bd793..e547112f8 100644
--- a/packages/core/src/collections/reminders.ts
+++ b/packages/core/src/collections/reminders.ts
@@ -43,7 +43,8 @@ export class Reminders implements ICollection {
db.sql,
db.transaction,
"reminders",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
diff --git a/packages/core/src/collections/session-content.ts b/packages/core/src/collections/session-content.ts
index 78ec482b3..d8d3743f7 100644
--- a/packages/core/src/collections/session-content.ts
+++ b/packages/core/src/collections/session-content.ts
@@ -39,7 +39,8 @@ export class SessionContent implements ICollection {
db.sql,
db.transaction,
"sessioncontent",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
diff --git a/packages/core/src/collections/settings.ts b/packages/core/src/collections/settings.ts
index 813ea1a3f..72faf6039 100644
--- a/packages/core/src/collections/settings.ts
+++ b/packages/core/src/collections/settings.ts
@@ -82,7 +82,8 @@ export class Settings implements ICollection {
db.sql,
db.transaction,
"settings",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
diff --git a/packages/core/src/collections/shortcuts.ts b/packages/core/src/collections/shortcuts.ts
index 84bdef8a2..fea418be0 100644
--- a/packages/core/src/collections/shortcuts.ts
+++ b/packages/core/src/collections/shortcuts.ts
@@ -38,7 +38,8 @@ export class Shortcuts implements ICollection {
db.sql,
db.transaction,
"shortcuts",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.ts
index 833c0d74f..45dad046c 100644
--- a/packages/core/src/collections/tags.ts
+++ b/packages/core/src/collections/tags.ts
@@ -32,7 +32,8 @@ export class Tags implements ICollection {
db.sql,
db.transaction,
"tags",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
diff --git a/packages/core/src/collections/vaults.ts b/packages/core/src/collections/vaults.ts
index a8e761eb8..4c4f45d16 100644
--- a/packages/core/src/collections/vaults.ts
+++ b/packages/core/src/collections/vaults.ts
@@ -32,11 +32,14 @@ export class Vaults implements ICollection {
db.sql,
db.transaction,
"vaults",
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
}
- async init() {}
+ async init() {
+ await this.collection.init();
+ }
async add(item: Partial) {
const id = item.id || getId();
diff --git a/packages/core/src/content-types/__tests__/tiptap.test.js b/packages/core/src/content-types/__tests__/tiptap.test.js
index 63f1f347d..8c1405d5b 100644
--- a/packages/core/src/content-types/__tests__/tiptap.test.js
+++ b/packages/core/src/content-types/__tests__/tiptap.test.js
@@ -26,7 +26,7 @@ import { test, expect } from "vitest";
test("img src is empty after extract attachments", async () => {
const tiptap = new Tiptap(IMG_CONTENT_WITHOUT_HASH);
- const result = await tiptap.extractAttachments(async () => {
+ const result = await tiptap.postProcess(async () => {
return "helloworld";
});
expect(result.hashes).toHaveLength(1);
@@ -37,7 +37,7 @@ test("img src is empty after extract attachments", async () => {
test("img src is present after insert attachments", async () => {
const tiptap = new Tiptap(IMG_CONTENT);
- const result = await tiptap.extractAttachments(async () => {
+ const result = await tiptap.postProcess(async () => {
return { key: "hello", metadata: {} };
});
const tiptap2 = new Tiptap(result.data);
diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.ts
index 1fcb393d7..211c87eeb 100644
--- a/packages/core/src/database/migrator.ts
+++ b/packages/core/src/database/migrator.ts
@@ -71,7 +71,8 @@ class Migrator {
db.sql,
db.transaction,
collection.table,
- db.eventManager
+ db.eventManager,
+ db.sanitizer
);
if (version <= 5.9) {
if (collection.name === "settings") {
diff --git a/packages/core/src/database/sanitizer.ts b/packages/core/src/database/sanitizer.ts
new file mode 100644
index 000000000..0148836d7
--- /dev/null
+++ b/packages/core/src/database/sanitizer.ts
@@ -0,0 +1,67 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+*/
+
+import { DatabaseAccessor } from ".";
+import { logger } from "../logger";
+
+export class Sanitizer {
+ tables: Record> = {};
+ logger = logger.scope("sanitizer");
+ constructor(private readonly db: DatabaseAccessor) {}
+
+ async init() {
+ const metadata = await this.db().introspection.getTables({
+ withInternalKyselyTables: false
+ });
+ for (const table of metadata) {
+ this.tables[table.name] = new Set(table.columns.map((c) => c.name));
+ }
+ }
+
+ /**
+ * Sanitization is done based on the latest table schema in the database. All
+ * unrecognized keys are removed
+ */
+ sanitize(table: string, item: any) {
+ const schema = this.tables[table];
+ if (!schema) {
+ if (process.env.NODE_ENV === "test")
+ throw new Error(
+ `Invalid table: ${table} (expected one of ${Object.keys(
+ this.tables
+ ).join(", ")})`
+ );
+ return false;
+ }
+
+ for (const key in item) {
+ if (schema.has(key)) continue;
+ if (process.env.NODE_ENV === "test")
+ throw new Error(`Found invalid key in item ${key} (${table})`);
+ else
+ this.logger.debug("Found invalid key in item", {
+ table,
+ key,
+ value: item[key]
+ });
+ delete item[key];
+ }
+ return true;
+ }
+}
diff --git a/packages/core/src/database/sql-cached-collection.ts b/packages/core/src/database/sql-cached-collection.ts
index ae87775c4..042dd9a07 100644
--- a/packages/core/src/database/sql-cached-collection.ts
+++ b/packages/core/src/database/sql-cached-collection.ts
@@ -22,6 +22,7 @@ import EventManager from "../utils/event-manager";
import { DatabaseAccessor, DatabaseCollection, DatabaseSchema } from ".";
import { SQLCollection } from "./sql-collection";
import { Transaction } from "kysely";
+import { Sanitizer } from "./sanitizer";
export class SQLCachedCollection<
TCollectionType extends keyof DatabaseSchema,
@@ -38,13 +39,15 @@ export class SQLCachedCollection<
executor: (tr: Transaction) => Promise
) => Promise,
type: TCollectionType,
- eventManager: EventManager
+ eventManager: EventManager,
+ sanitizer: Sanitizer
) {
this.collection = new SQLCollection(
sql,
startTransaction,
type,
- eventManager
+ eventManager,
+ sanitizer
);
}
diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.ts
index 8b15ece25..4fe7971af 100644
--- a/packages/core/src/database/sql-collection.ts
+++ b/packages/core/src/database/sql-collection.ts
@@ -45,6 +45,7 @@ import {
import { VirtualizedGrouping } from "../utils/virtualized-grouping";
import { groupArray } from "../utils/grouping";
import { toChunks } from "../utils/array";
+import { Sanitizer } from "./sanitizer";
const formats = {
month: "%Y-%m",
@@ -66,7 +67,8 @@ export class SQLCollection<
executor: (tr: Transaction) => Promise
) => Promise,
private readonly type: TCollectionType,
- private readonly eventManager: EventManager
+ private readonly eventManager: EventManager,
+ private readonly sanitizer: Sanitizer
) {}
async clear() {
@@ -87,6 +89,8 @@ export class SQLCollection<
// the item has become local now, so remove the flags
delete item.remote;
+ if (!this.sanitizer.sanitize(this.type, item)) return;
+
await this.db()
.replaceInto(this.type)
.values(item)
@@ -163,6 +167,9 @@ export class SQLCollection<
item.synced = false;
}
delete item.remote;
+
+ if (!this.sanitizer.sanitize(this.type, item)) return array;
+
array.push(item);
return array;
}, [] as SQLiteItem[]);
@@ -185,6 +192,7 @@ export class SQLCollection<
partial: Partial>,
options: { sendEvent: boolean } = { sendEvent: true }
) {
+ if (!this.sanitizer.sanitize(this.type, partial)) return;
await this.db()
.updateTable(this.type)
.where("id", "in", ids)