core: fix FTS5 SQLite search

packages/core/src/api/lookup.ts

@@ -19,7 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { match } from "fuzzyjs";
 import Database from ".";
-import { Item, TrashItem } from "../types";
+import { Item, Note, TrashItem } from "../types";
 import { DatabaseSchema, DatabaseSchemaWithFTS, isFalse } from "../database";
 import { AnyColumnWithTable, Kysely, sql } from "kysely";
 import { FilteredSelector } from "../database/sql-collection";
@@ -39,8 +39,10 @@ type FuzzySearchField<T> = {
 export default class Lookup {
   constructor(private readonly db: Database) {}
 
-  notes(query: string, noteIds?: string[]) {
+  notes(query: string, notes?: FilteredSelector<Note>): SearchResults<Note> {
     return this.toSearchResults(async (limit) => {
+      if (query.length <= 3) return [];
+
       const db = this.db.sql() as Kysely<DatabaseSchemaWithFTS>;
       query = query.replace(/"/, '""');
       const result = await db
@@ -58,8 +60,8 @@ export default class Lookup {
             )
         )
         .selectFrom("notes")
-        .$if(!!noteIds && noteIds.length > 0, (eb) =>
-          eb.where("notes.id", "in", noteIds!)
+        .$if(!!notes, (eb) =>
+          eb.where("notes.id", "in", notes!.filter.select("id"))
         )
         .$if(!!limit, (eb) => eb.limit(limit!))
         .where(isFalse("notes.deleted"))
@@ -67,11 +69,11 @@ export default class Lookup {
         .innerJoin("matching", (eb) =>
           eb.onRef("notes.id", "==", "matching.id")
         )
-        .orderBy("matching.rank")
+        .orderBy("matching.rank desc")
         .select(["notes.id"])
         .execute();
       return result.map((id) => id.id);
-    }, this.db.notes.all);
+    }, notes || this.db.notes.all);
   }
 
   notebooks(query: string) {

packages/core/src/collections/content.ts

@@ -103,6 +103,7 @@ export class Content implements ICollection {
         localOnly: content.localOnly,
         conflicted: content.conflicted,
         dateResolved: content.dateResolved,
+        noteId: content.noteId,
         ...contentData
       });
 

packages/core/src/collections/notes.ts

@@ -67,75 +67,80 @@ export class Notes implements ICollection {
       throw new Error("Please use db.notes.merge to merge remote notes.");
 
     const id = item.id || getId();
+    const isUpdating = item.id && (await this.exists(item.id));
 
-    const oldNote = await this.note(id);
-
-    const note = {
-      ...oldNote,
-      ...item
-    };
-
-    let dateEdited = note.dateEdited || note.dateCreated || Date.now();
-    if (oldNote) note.contentId = oldNote.contentId;
-
-    if (!oldNote && !item.content && !item.contentId && !item.title)
+    if (!isUpdating && !item.content && !item.contentId && !item.title)
       throw new Error("Note must have a title or content.");
 
     await this.db.transaction(async () => {
-      if (item.content && item.content.data && item.content.type) {
-        if (oldNote) dateEdited = Date.now();
+      let contentId = item.contentId;
+      let dateEdited = item.dateEdited;
+      let headline = "";
 
+      if (item.content && item.content.data && item.content.type) {
         const { type, data } = item.content;
 
         const content = getContentFromData(type, data);
         if (!content) throw new Error("Invalid content type.");
 
-        note.contentId = await this.db.content.add({
+        headline = item.locked ? "" : getNoteHeadline(content);
+        dateEdited = Date.now();
+        contentId = await this.db.content.add({
           noteId: id,
-          sessionId: note.sessionId,
-          id: note.contentId,
+          sessionId: item.sessionId,
+          id: contentId,
           dateEdited,
           type,
           data,
-          localOnly: !!note.localOnly
+          ...(item.localOnly !== undefined ? { localOnly: item.localOnly } : {})
         });
-
-        note.headline = note.locked ? "" : getNoteHeadline(content);
-      }
-
-      if (note.contentId && item.localOnly !== undefined) {
+      } else if (contentId && item.localOnly !== undefined) {
         await this.db.content.add({
-          id: note.contentId,
+          id: contentId,
           localOnly: !!item.localOnly
         });
       }
 
-      const noteTitle = await this.getNoteTitle(note, oldNote, note.headline);
-      if (oldNote && oldNote.title !== noteTitle) dateEdited = Date.now();
+      if (item.title) {
+        item.title = this.getNoteTitle(item.title, headline);
+        dateEdited = Date.now();
+      }
 
-      await this.collection.upsert({
-        id,
-        contentId: note.contentId,
-        type: "note",
+      if (isUpdating) {
+        await this.collection.update([id], {
+          title: item.title,
+          headline,
 
-        title: noteTitle,
-        headline: note.headline,
+          pinned: item.pinned,
+          locked: item.locked,
+          favorite: item.favorite,
+          localOnly: item.localOnly,
+          conflicted: item.conflicted,
+          readonly: item.readonly,
 
-        notebooks: note.notebooks || undefined,
+          dateEdited: item.dateEdited || dateEdited
+        });
+      } else {
+        await this.collection.upsert({
+          id,
+          type: "note",
+          contentId,
 
-        pinned: !!note.pinned,
-        locked: !!note.locked,
-        favorite: !!note.favorite,
-        localOnly: !!note.localOnly,
-        conflicted: !!note.conflicted,
-        readonly: !!note.readonly,
+          title: item.title,
+          headline: headline,
 
-        dateCreated: note.dateCreated || Date.now(),
-        dateEdited: item.dateEdited || dateEdited,
-        dateModified: note.dateModified || Date.now()
-      });
+          pinned: item.pinned,
+          locked: item.locked,
+          favorite: item.favorite,
+          localOnly: item.localOnly,
+          conflicted: item.conflicted,
+          readonly: item.readonly,
 
-      if (!oldNote) this.totalNotes++;
+          dateCreated: item.dateCreated || Date.now(),
+          dateEdited: item.dateEdited || dateEdited || Date.now()
+        });
+        this.totalNotes++;
+      }
     });
     return id;
   }
@@ -386,20 +391,9 @@ export class Notes implements ICollection {
     });
   }
 
-  private async getNoteTitle(
-    note: Partial<Note>,
-    oldNote?: Note,
-    headline?: string
-  ) {
-    if (note.title && note.title.trim().length > 0) {
-      return note.title.replace(NEWLINE_STRIP_REGEX, " ");
-    } else if (
-      oldNote &&
-      oldNote.title &&
-      oldNote.title.trim().length > 0 &&
-      (note.title === undefined || note.title === null)
-    ) {
-      return oldNote.title.replace(NEWLINE_STRIP_REGEX, " ");
+  private getNoteTitle(title: string, headline?: string) {
+    if (title.trim().length > 0) {
+      return title.replace(NEWLINE_STRIP_REGEX, " ");
     }
 
     return formatTitle(

packages/core/src/database/index.ts

@@ -56,6 +56,7 @@ import {
   ValueOf
 } from "../types";
 import { NNMigrationProvider } from "./migrations";
+import { createTriggers } from "./triggers";
 
 // type FilteredKeys<T, U> = {
 //   [P in keyof T]: T[P] extends U ? P : never;
@@ -69,7 +70,7 @@ export type SQLiteItem<T> = {
   [P in keyof T]?: T[P] | null;
 } & { id: string };
 
-export type SQLiteItemWithRowID<T> = SQLiteItem<T> & { rowid: number };
+export type SQLiteItemWithRowID<T> = SQLiteItem<T> & { rowid?: number };
 
 export interface DatabaseSchema {
   notes: SQLiteItem<TrashOrItem<Note>>;
@@ -206,7 +207,7 @@ export type SQLiteOptions = {
   pageSize?: number;
 };
 export async function createDatabase(name: string, options: SQLiteOptions) {
-  const db = new Kysely<DatabaseSchema>({
+  const db = new Kysely<DatabaseSchemaWithFTS>({
     dialect: options.dialect(name),
     plugins: [new SqliteBooleanPlugin()]
   });
@@ -224,6 +225,10 @@ export async function createDatabase(name: string, options: SQLiteOptions) {
     options.synchronous || "normal"
   )}`.execute(db);
 
+  // recursive_triggers are required so that SQLite fires DELETE trigger on
+  // REPLACE INTO statements
+  await sql`PRAGMA recursive_triggers = true`.execute(db);
+
   if (options.pageSize)
     await sql`PRAGMA page_size = ${sql.raw(
       options.pageSize.toString()
@@ -242,7 +247,22 @@ export async function createDatabase(name: string, options: SQLiteOptions) {
       db
     );
 
-  await migrator.migrateToLatest();
+  const { error, results } = await migrator.migrateToLatest();
+
+  results?.forEach((it) => {
+    if (it.status === "Success") {
+      console.log(`migration "${it.migrationName}" was executed successfully`);
+    } else if (it.status === "Error") {
+      console.error(`failed to execute migration "${it.migrationName}"`);
+    }
+  });
+
+  if (error) {
+    console.error("failed to run `migrateToLatest`");
+    console.error(error);
+  }
+
+  await createTriggers(db);
 
   return db;
 }

packages/core/src/database/migrations.ts

@@ -20,7 +20,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import {
   ColumnBuilderCallback,
   CreateTableBuilder,
-  Kysely,
   Migration,
   MigrationProvider,
   sql
@@ -50,7 +49,12 @@ export class NNMigrationProvider implements MigrationProvider {
             .addColumn("readonly", "boolean")
             .addColumn("dateEdited", "integer")
             .execute();
-          await createFTS5Table(db, "notes", ["title"]);
+
+          await createFTS5Table(
+            "notes_fts",
+            [{ name: "id" }, { name: "title" }],
+            { contentTable: "notes", tokenizer: ["porter", "trigram"] }
+          ).execute(db);
 
           await db.schema
             .createTable("content")
@@ -65,13 +69,12 @@ export class NNMigrationProvider implements MigrationProvider {
             .addColumn("dateEdited", "integer")
             .addColumn("dateResolved", "integer")
             .execute();
+
           await createFTS5Table(
-            db,
-            "content",
-            ["data"],
-            ["noteId"],
-            ["(new.locked is null or new.locked == 0)"]
-          );
+            "content_fts",
+            [{ name: "id" }, { name: "noteId" }, { name: "data" }],
+            { contentTable: "content", tokenizer: ["porter", "trigram"] }
+          ).execute(db);
 
           await db.schema
             .createTable("notehistory")
@@ -285,50 +288,38 @@ const addTrashColumns = <T extends string, C extends string = never>(
     .addColumn("itemType", "text");
 };
 
-async function createFTS5Table(
-  db: Kysely<any>,
-  table: string,
-  indexedColumns: string[],
-  unindexedColumns: string[] = [],
-  insertConditions: string[] = []
+type Tokenizer = "porter" | "trigram" | "unicode61" | "ascii";
+function createFTS5Table(
+  name: string,
+  columns: {
+    name: string;
+    unindexed?: boolean;
+  }[],
+  options: {
+    contentTable?: string;
+    contentTableRowId?: string;
+    tokenizer?: Tokenizer[];
+    prefix?: number[];
+    columnSize?: 0 | 1;
+    detail?: "full" | "column" | "none";
+  } = {}
 ) {
-  const ref = sql.raw(table);
-  const ref_fts = sql.raw(table + "_fts");
-  const ref_ai = sql.raw(table + "_ai");
-  const ref_ad = sql.raw(table + "_ad");
-  const ref_au = sql.raw(table + "_au");
-  const indexed_cols = sql.raw(indexedColumns.join(", "));
-  const unindexed_cols =
-    unindexedColumns.length > 0
-      ? sql.raw(unindexedColumns.join(" UNINDEXED,") + " UNINDEXED,")
-      : sql.raw("");
-  const new_indexed_cols = sql.raw(indexedColumns.join(", new."));
-  const old_indexed_cols = sql.raw(indexedColumns.join(", old."));
-  await sql`CREATE VIRTUAL TABLE ${ref_fts} USING fts5(
-    id UNINDEXED, ${unindexed_cols} ${indexed_cols}, content='${sql.raw(
-    table
-  )}', tokenize='porter trigram'
-  )`.execute(db);
-  insertConditions = [
-    "(new.deleted is null or new.deleted == 0)",
-    ...insertConditions
-  ];
-  await sql`CREATE TRIGGER ${ref_ai} AFTER INSERT ON ${ref} WHEN ${sql.raw(
-    insertConditions.join(" AND ")
-  )}
-  BEGIN
-    INSERT INTO ${ref_fts}(rowid, id, ${indexed_cols}) VALUES (new.rowid, new.id, new.${new_indexed_cols});
-  END;`.execute(db);
-  await sql`CREATE TRIGGER ${ref_ad} AFTER DELETE ON ${ref}
-  BEGIN
-    INSERT INTO ${ref_fts} (${ref_fts}, rowid, id, ${indexed_cols})
-    VALUES ('delete', old.rowid, old.id, old.${old_indexed_cols});
-  END;`.execute(db);
-  await sql`CREATE TRIGGER ${ref_au} AFTER UPDATE ON ${ref}
-  BEGIN
-    INSERT INTO ${ref_fts} (${ref_fts}, rowid, id, ${indexed_cols})
-    VALUES ('delete', old.rowid, old.id, old.${old_indexed_cols});
-    INSERT INTO ${ref_fts} (rowid, id, ${indexed_cols})
-    VALUES (new.rowid, new.id, new.${new_indexed_cols});
-  END;`.execute(db);
+  const _options: string[] = [];
+  if (options.contentTable) _options.push(`content='${options.contentTable}'`);
+  if (options.contentTableRowId)
+    _options.push(`content_rowid='${options.contentTableRowId}'`);
+  if (options.tokenizer)
+    _options.push(`tokenize='${options.tokenizer.join(" ")}'`);
+  if (options.prefix) _options.push(`prefix='${options.prefix.join(" ")}'`);
+  if (options.columnSize) _options.push(`columnsize='${options.columnSize}'`);
+  if (options.detail) _options.push(`detail='${options.detail}'`);
+
+  const args = sql.join([
+    sql.join(
+      columns.map((c) => sql.ref(`${c.name}${c.unindexed ? " UNINDEXED" : ""}`))
+    ),
+    sql.join(_options.map((o) => sql.raw(o)))
+  ]);
+
+  return sql`CREATE VIRTUAL TABLE ${sql.raw(name)} USING fts5(${args})`;
 }

packages/core/src/database/triggers.ts

@@ -0,0 +1,144 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Kysely, sql } from "kysely";
+import { DatabaseSchemaWithFTS } from ".";
+
+export async function createTriggers(db: Kysely<DatabaseSchemaWithFTS>) {
+  // content triggers
+  await db.schema
+    .createTrigger("content_after_insert_content_fts")
+    .temporary()
+    .onTable("content", "main")
+    .after()
+    .addEvent("insert")
+    .when((eb) =>
+      eb.and([
+        eb.or([eb("new.deleted", "is", null), eb("new.deleted", "==", false)]),
+        eb.or([eb("new.locked", "is", null), eb("new.locked", "==", false)])
+      ])
+    )
+    .addQuery((c) =>
+      c.insertInto("content_fts").values({
+        id: sql`new.id`,
+        data: sql`new.data`,
+        noteId: sql`new.noteId`
+      })
+    )
+    .execute();
+
+  await db.schema
+    .createTrigger("content_after_delete_content_fts")
+    .temporary()
+    .onTable("content", "main")
+    .after()
+    .addEvent("delete")
+    .addQuery((c) =>
+      c.insertInto("content_fts").values({
+        content_fts: sql.lit("delete"),
+        id: sql.ref("old.id"),
+        rowid: sql.ref("old.rowid"),
+        data: sql.ref("old.data"),
+        noteId: sql.ref("old.noteId")
+      })
+    )
+    .execute();
+
+  await db.schema
+    .createTrigger("content_after_update_content_fts")
+    .temporary()
+    .onTable("content", "main")
+    .after()
+    .addEvent("update", ["data"])
+    .addQuery((c) =>
+      c.insertInto("content_fts").values({
+        content_fts: sql.lit("delete"),
+        id: sql.ref("old.id"),
+        rowid: sql.ref("old.rowid"),
+        data: sql.ref("old.data"),
+        noteId: sql.ref("old.noteId")
+      })
+    )
+    .addQuery((c) =>
+      c.insertInto("content_fts").values({
+        id: sql`new.id`,
+        data: sql`IIF(new.locked == 1, "", new.data)`,
+        noteId: sql`new.noteId`
+      })
+    )
+    .execute();
+
+  // notes triggers
+  await db.schema
+    .createTrigger("notes_after_insert_notes_fts")
+    .temporary()
+    .onTable("notes", "main")
+    .after()
+    .addEvent("insert")
+    .when((eb) =>
+      eb.and([
+        eb.or([eb("new.deleted", "is", null), eb("new.deleted", "==", false)])
+      ])
+    )
+    .addQuery((c) =>
+      c.insertInto("notes_fts").values({
+        id: sql.ref("new.id"),
+        title: sql.ref("new.title")
+      })
+    )
+    .execute();
+
+  await db.schema
+    .createTrigger("notes_after_delete_notes_fts")
+    .temporary()
+    .onTable("notes", "main")
+    .after()
+    .addEvent("delete")
+    .addQuery((c) =>
+      c.insertInto("notes_fts").values({
+        notes_fts: sql.lit("delete"),
+        id: sql.ref("old.id"),
+        rowid: sql.ref("old.rowid"),
+        title: sql.ref("old.title")
+      })
+    )
+    .execute();
+
+  await db.schema
+    .createTrigger("notes_after_update_notes_fts")
+    .temporary()
+    .onTable("notes", "main")
+    .after()
+    .addEvent("update", ["title"])
+    .addQuery((c) =>
+      c.insertInto("notes_fts").values({
+        notes_fts: sql.lit("delete"),
+        id: sql.ref("old.id"),
+        rowid: sql.ref("old.rowid"),
+        title: sql.ref("old.title")
+      })
+    )
+    .addQuery((c) =>
+      c.insertInto("notes_fts").values({
+        id: sql.ref("new.id"),
+        title: sql.ref("new.title")
+      })
+    )
+    .execute();
+}