core: ignore diacritics when searching (requires rebuilding search index)

packages/core/__tests__/lookup.test.js

@@ -41,6 +41,18 @@ test("search notes", () =>
     expect(filtered).toHaveLength(1);
   }));
 
+test("search notes (remove diacritics)", () =>
+  noteTest({
+    content: {
+      type: "tiptap",
+      data: "<p>hello i am à la maison</p>"
+    }
+  }).then(async ({ db }) => {
+    await db.notes.add(TEST_NOTE);
+    let filtered = await db.lookup.notes("a la maison").ids();
+    expect(filtered).toHaveLength(1);
+  }));
+
 test("search notes with a locked note", () =>
   noteTest({
     content: content

packages/core/src/database/migrations.ts

@@ -330,6 +330,33 @@ export class NNMigrationProvider implements MigrationProvider {
             )
             .execute();
         }
+      },
+      "6": {
+        async up(db) {
+          await db.transaction().execute(async (tx) => {
+            await tx.schema.dropTable("content_fts").execute();
+            await tx.schema.dropTable("notes_fts").execute();
+
+            await createFTS5Table(
+              "notes_fts",
+              [{ name: "id" }, { name: "title" }],
+              {
+                contentTable: "notes",
+                tokenizer: ["porter", "trigram", "remove_diacritics 1"]
+              }
+            ).execute(tx);
+
+            await createFTS5Table(
+              "content_fts",
+              [{ name: "id" }, { name: "noteId" }, { name: "data" }],
+              {
+                contentTable: "content",
+                tokenizer: ["porter", "trigram", "remove_diacritics 1"]
+              }
+            ).execute(tx);
+          });
+          await rebuildSearchIndex(db);
+        }
       }
     };
   }
@@ -357,7 +384,7 @@ const addTrashColumns = <T extends string, C extends string = never>(
     .addColumn("deletedBy", "text");
 };
 
-type Tokenizer = "porter" | "trigram" | "unicode61" | "ascii";
+type Tokenizer = "porter" | "trigram" | "unicode61" | "ascii" | (string & {});
 function createFTS5Table(
   name: string,
   columns: {