core: filter out stop words in search query

packages/core/src/utils/query-transformer.ts

@@ -17,6 +17,8 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+import { EN_STOPWORDS } from "./stop-words";
+
 type ASTNode = QueryNode | PhraseNode | OperatorNode;
 
 type QueryNode = {
@@ -161,7 +163,9 @@ function generateSQL(ast: QueryNode): string {
 }
 
 export function transformQuery(query: string) {
-  const tokens = tokenize(query);
+  const tokens = tokenize(query).filter(
+    (token) => !EN_STOPWORDS.includes(token)
+  );
   const largeTokens = tokens.filter(
     (token) => token.length >= 3 || token === "OR"
   );

packages/core/src/utils/stop-words.ts

@@ -0,0 +1,321 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+export const EN_STOPWORDS = [
+  "a",
+  "about",
+  "above",
+  "across",
+  "actually",
+  "after",
+  "again",
+  "against",
+  "all",
+  "almost",
+  "alone",
+  "along",
+  "already",
+  "also",
+  "although",
+  "always",
+  "among",
+  "an",
+  "and",
+  "another",
+  "any",
+  "anybody",
+  "anyone",
+  "anything",
+  "anywhere",
+  "are",
+  "around",
+  "as",
+  "ask",
+  "asked",
+  "asking",
+  "asks",
+  "at",
+  "away",
+  "b",
+  "be",
+  "became",
+  "because",
+  "become",
+  "becomes",
+  "becoming",
+  "been",
+  "before",
+  "behind",
+  "being",
+  "best",
+  "better",
+  "between",
+  "both",
+  "but",
+  "by",
+  "c",
+  "came",
+  "can",
+  "certain",
+  "certainly",
+  "clearly",
+  "come",
+  "consider",
+  "considering",
+  "could",
+  "d",
+  "did",
+  "different",
+  "do",
+  "does",
+  "doing",
+  "done",
+  "down",
+  "downwards",
+  "during",
+  "e",
+  "each",
+  "eg",
+  "eight",
+  "either",
+  "enough",
+  "even",
+  "ever",
+  "every",
+  "everybody",
+  "everyone",
+  "everything",
+  "everywhere",
+  "ex",
+  "exactly",
+  "example",
+  "except",
+  "f",
+  "fact",
+  "facts",
+  "far",
+  "few",
+  "first",
+  "five",
+  "for",
+  "four",
+  "from",
+  "further",
+  "g",
+  "get",
+  "gets",
+  "given",
+  "gives",
+  "go",
+  "going",
+  "got",
+  "h",
+  "had",
+  "has",
+  "have",
+  "having",
+  "he",
+  "her",
+  "here",
+  "herself",
+  "him",
+  "himself",
+  "his",
+  "how",
+  "however",
+  "i",
+  "ie",
+  "if",
+  "in",
+  "into",
+  "is",
+  "it",
+  "its",
+  "itself",
+  "j",
+  "just",
+  "k",
+  "keep",
+  "keeps",
+  "knew",
+  "know",
+  "known",
+  "knows",
+  "l",
+  "last",
+  "later",
+  "least",
+  "less",
+  "let",
+  "like",
+  "likely",
+  "m",
+  "many",
+  "may",
+  "me",
+  "might",
+  "more",
+  "most",
+  "mostly",
+  "much",
+  "must",
+  "my",
+  "myself",
+  "n",
+  "necessary",
+  "need",
+  "needs",
+  "never",
+  "new",
+  "next",
+  "nine",
+  "no",
+  "nobody",
+  "non",
+  "not",
+  "nothing",
+  "now",
+  "nowhere",
+  "o",
+  "of",
+  "off",
+  "often",
+  "old",
+  "on",
+  "once",
+  "one",
+  "only",
+  "or",
+  "other",
+  "others",
+  "our",
+  "out",
+  "over",
+  "p",
+  "per",
+  "perhaps",
+  "please",
+  "possible",
+  "put",
+  "q",
+  "quite",
+  "r",
+  "rather",
+  "really",
+  "right",
+  "s",
+  "said",
+  "same",
+  "saw",
+  "say",
+  "says",
+  "second",
+  "see",
+  "seem",
+  "seemed",
+  "seems",
+  "seven",
+  "several",
+  "shall",
+  "she",
+  "should",
+  "since",
+  "six",
+  "so",
+  "some",
+  "somebody",
+  "someone",
+  "something",
+  "somewhere",
+  "still",
+  "such",
+  "sure",
+  "t",
+  "take",
+  "taken",
+  "ten",
+  "than",
+  "that",
+  "the",
+  "their",
+  "them",
+  "then",
+  "there",
+  "therefore",
+  "therein",
+  "thereupon",
+  "these",
+  "they",
+  "think",
+  "third",
+  "this",
+  "those",
+  "though",
+  "three",
+  "through",
+  "thus",
+  "to",
+  "together",
+  "too",
+  "took",
+  "toward",
+  "two",
+  "u",
+  "under",
+  "until",
+  "up",
+  "upon",
+  "us",
+  "use",
+  "used",
+  "uses",
+  "v",
+  "very",
+  "w",
+  "want",
+  "wanted",
+  "wants",
+  "was",
+  "way",
+  "we",
+  "well",
+  "went",
+  "were",
+  "what",
+  "when",
+  "where",
+  "whether",
+  "which",
+  "while",
+  "who",
+  "whole",
+  "whose",
+  "why",
+  "will",
+  "with",
+  "within",
+  "without",
+  "would",
+  "x",
+  "y",
+  "yet",
+  "you",
+  "your",
+  "yours",
+  "z"
+];