core: workaround for tokens shorter than 3 chars

packages/core/__tests__/lookup.test.js

@@ -37,8 +37,13 @@ test("search notes", () =>
     content: content
   }).then(async ({ db }) => {
     await db.notes.add(TEST_NOTE);
-    let filtered = await db.lookup.notes("note of the world").ids();
-    expect(filtered).toHaveLength(1);
+    await db.notes.add({
+      content: { data: "<p>hb <b>kb</b> ch</p>", type: "tiptap" },
+      title: "hello"
+    });
+
+    expect(await db.lookup.notes("note of the world").ids()).toHaveLength(1);
+    expect(await db.lookup.notes("hb kb ch").ids()).toHaveLength(1);
   }));
 
 test("search notes (remove diacritics)", () =>

packages/core/package-lock.json

@@ -48,7 +48,7 @@
         "@types/ws": "^8.5.5",
         "@vitest/coverage-v8": "^1.0.1",
         "abortcontroller-polyfill": "^1.7.3",
-        "better-sqlite3-multiple-ciphers": "^9.4.0",
+        "better-sqlite3-multiple-ciphers": "^11.5.0",
         "bson-objectid": "^2.0.4",
         "dotenv": "^16.0.1",
         "event-source-polyfill": "^1.0.31",
@@ -2486,11 +2486,12 @@
       ]
     },
     "node_modules/better-sqlite3-multiple-ciphers": {
-      "version": "9.4.0",
-      "resolved": "https://registry.npmjs.org/better-sqlite3-multiple-ciphers/-/better-sqlite3-multiple-ciphers-9.4.0.tgz",
-      "integrity": "sha512-uQoBdShnpCdAPUmCPuLSrHtfSHAxjIFQvLnMbXHsvbWkfBP3sPfnnIaUTz7KHYRqgWNLlcQfs+9F9uqvMqau6Q==",
+      "version": "11.5.0",
+      "resolved": "https://registry.npmjs.org/better-sqlite3-multiple-ciphers/-/better-sqlite3-multiple-ciphers-11.5.0.tgz",
+      "integrity": "sha512-t2RpIBaw6DYk8RNZjrqCLRoznBcIqownpd90spHpHVrJa+YYN/NOLoTlj1iLBS754yiYWL6uXgx4x+0E1Z5q8Q==",
       "dev": true,
       "hasInstallScript": true,
+      "license": "MIT",
       "dependencies": {
         "bindings": "^1.5.0",
         "prebuild-install": "^7.1.1"

packages/core/package.json

@@ -35,7 +35,7 @@
     "@types/ws": "^8.5.5",
     "@vitest/coverage-v8": "^1.0.1",
     "abortcontroller-polyfill": "^1.7.3",
-    "better-sqlite3-multiple-ciphers": "^9.4.0",
+    "better-sqlite3-multiple-ciphers": "^11.5.0",
     "bson-objectid": "^2.0.4",
     "dotenv": "^16.0.1",
     "event-source-polyfill": "^1.0.31",

packages/core/src/utils/__tests__/query-transformer.test.ts

@@ -20,6 +20,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { expect, test } from "vitest";
 import { transformQuery } from "../query-transformer.js";
 
+function lt2(str: string) {
+  return `(">${str}" OR ${str} OR "${str}<")`;
+}
+
 const TRANSFORM_QUERY_TESTS = [
   ["hello world", `hello AND world`],
   ["hello world OR bar", `hello AND world OR bar`],
@@ -29,40 +33,55 @@ const TRANSFORM_QUERY_TESTS = [
   ["hello world -foo", `hello AND world AND "-foo"`],
   ["hello world phrase-with-dash", `hello AND world AND "phrase-with-dash"`],
   ["hello world phrase-with-dash*", 'hello AND world AND "phrase-with-dash*"'],
-  ["example + foo + bar", `example AND + AND foo AND + AND bar`],
+  [
+    "example + foo + bar",
+    `example AND ${lt2("+")} AND foo AND ${lt2("+")} AND bar`
+  ],
   ["example OR foo NOT bar", `example OR foo NOT bar`],
   [
     'example "quoted phrase" "another quoted phrase"',
     `example AND "quoted phrase" AND "another quoted phrase"`
   ],
   ['"phrase-with-dash*"', `"phrase-with-dash*"`],
-  ['-foo + bar OR "quoted-phrase"', '"-foo" AND + AND bar OR "quoted-phrase"'],
+  [
+    '-foo + bar OR "quoted-phrase"',
+    `"-foo" AND ${lt2("+")} AND bar OR "quoted-phrase"`
+  ],
   [
     'phrase-with-dash* + "quoted-phrase"',
-    '"phrase-with-dash*" AND + AND "quoted-phrase"'
+    `"phrase-with-dash*" AND ${lt2("+")} AND "quoted-phrase"`
   ],
   [
     'example -foo + bar + "quoted-dash-phrase*" OR "another-quoted-phrase"',
-    'example AND "-foo" AND + AND bar AND + AND "quoted-dash-phrase*" OR "another-quoted-phrase"'
+    `example AND "-foo" AND ${lt2("+")} AND bar AND ${lt2(
+      "+"
+    )} AND "quoted-dash-phrase*" OR "another-quoted-phrase"`
   ],
   ["", ""],
   ["foo", `foo`],
   ['"quoted"', '"quoted"'],
   ["-foo -bar", `"-foo" AND "-bar"`],
-  ["foo + + bar", `foo AND + AND + AND bar`],
-  ["foo + OR", `foo AND +`],
+  ["foo + + bar", `foo AND ${lt2("+")} AND ${lt2("+")} AND bar`],
+  ["foo + OR", `foo AND ${lt2("+")}`],
   ['"special -phrase*"', '"special -phrase*"'],
-  ["foo* + bar*", `"foo*" AND + AND "bar*"`],
-  ["(foo + bar) -baz", `(foo AND + AND bar) AND "-baz"`],
+  ["foo* + bar*", `"foo*" AND ${lt2("+")} AND "bar*"`],
+  ["(foo + bar) -baz", `(foo AND ${lt2("+")} AND bar) AND "-baz"`],
   ['"phrase with "quotes""', '"phrase with ""quotes"""'],
-  ['foo + "bar -baz" OR "qux*"', 'foo AND + AND "bar -baz" OR "qux*"'],
-  ["foo + bar + ", `foo AND + AND bar AND +`],
+  [
+    'foo + "bar -baz" OR "qux*"',
+    `foo AND ${lt2("+")} AND "bar -baz" OR "qux*"`
+  ],
+  ["foo + bar + ", `foo AND ${lt2("+")} AND bar AND ${lt2("+")}`],
   ["+foo bar", `+foo AND bar`],
   ["foo*bar*", `"foo*bar*"`],
   ['"escaped "quotes""', '"escaped ""quotes"""'],
   ["-hello-world", `"-hello-world"`],
   ["-hello-world*", '"-hello-world*"'],
-  ["*helo*", `"*helo*"`]
+  ["*helo*", `"*helo*"`],
+  [">he", `">he"`],
+  ["something<hello", `"something<hello"`],
+  ["<", `"<"`],
+  [">", `">"`]
 ];
 
 for (const [input, expectedOutput] of TRANSFORM_QUERY_TESTS) {

packages/core/src/utils/query-transformer.ts

@@ -40,7 +40,10 @@ function escapeSQLString(str: string): string {
   }
 
   const maybeColspec =
-    str.startsWith("-") || str.endsWith(":") || str.includes("-");
+    str.includes(":") ||
+    str.includes(">") ||
+    str.includes("<") ||
+    str.includes("-");
   const isWildcard =
     str.startsWith("*") ||
     str.endsWith("*") ||
@@ -150,7 +153,20 @@ function generateSQL(ast: QueryNode): string {
   return ast.children
     .map((child) => {
       if (child.type === "phrase") {
-        return child.value.join(" AND ");
+        const result: string[] = [];
+        for (const value of child.value) {
+          if (value.length === 1 || value.length === 2) {
+            result.push(`(">${value}"`, "OR", value, "OR", `"${value}<")`);
+            result.push("AND");
+            continue;
+          }
+
+          result.push(value);
+          result.push("AND");
+        }
+        result.pop();
+        return result.join(" ");
+        // return child.value.join(" AND ");
       }
       if (child.type === "AND" || child.type === "OR" || child.type === "NOT") {
         return child.type;