/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
import { match } from "fuzzyjs";
import Database from "./index.js";
import {
Item,
Note,
Notebook,
Reminder,
SortOptions,
TrashItem
} from "../types.js";
import { DatabaseSchema, RawDatabaseSchema } from "../database/index.js";
import { AnyColumnWithTable, Kysely, sql } from "@streetwriters/kysely";
import { FilteredSelector } from "../database/sql-collection.js";
import { VirtualizedGrouping } from "../utils/virtualized-grouping.js";
import { logger } from "../logger.js";
import { rebuildSearchIndex } from "../database/fts.js";
import { transformQuery } from "../utils/query-transformer.js";
import { getSortSelectors } from "../utils/grouping.js";
type SearchResults = {
sorted: (limit?: number) => Promise>;
items: (limit?: number) => Promise;
ids: () => Promise;
};
type FuzzySearchField = {
weight?: number;
name: keyof T;
column: AnyColumnWithTable;
};
export default class Lookup {
constructor(private readonly db: Database) {}
notes(
query: string,
notes?: FilteredSelector,
opts?: { titleOnly?: boolean }
): SearchResults {
return this.toSearchResults(async (limit, sortOptions) => {
const db = this.db.sql() as unknown as Kysely;
const excludedIds = this.db.trash.cache.notes;
query = transformQuery(query);
const results = await db
.selectFrom((eb) =>
eb
.selectFrom("notes_fts")
.$if(!!notes, (eb) =>
eb.where("id", "in", notes!.filter.select("id"))
)
.$if(excludedIds.length > 0, (eb) =>
eb.where("id", "not in", excludedIds)
)
.where("title", "match", query)
.select(["id", sql`rank * 10`.as("rank")])
.$if(!opts?.titleOnly, (eb) =>
eb.unionAll((eb) =>
eb
.selectFrom("content_fts")
.$if(!!notes, (eb) =>
eb.where("noteId", "in", notes!.filter.select("id"))
)
.$if(excludedIds.length > 0, (eb) =>
eb.where("id", "not in", excludedIds)
)
.where("data", "match", query)
.select(["noteId as id", "rank"])
.$castTo<{
id: string;
rank: number;
}>()
)
)
.as("results")
)
.select(["results.id"])
.groupBy("results.id")
.orderBy(sql`SUM(results.rank)`, sortOptions?.sortDirection || "desc")
.$if(!!limit, (eb) => eb.limit(limit!))
// filter out ids that have no note against them
.where(
"results.id",
"in",
(notes || this.db.notes.all).filter.select("id")
)
.execute()
.catch((e) => {
logger.error(e, `Error while searching`, { query });
return [];
});
return results.map((r) => r.id);
}, notes || this.db.notes.all);
}
notebooks(query: string, opts: { titleOnly?: boolean } = {}) {
const fields: FuzzySearchField[] = [
{ name: "id", column: "notebooks.id", weight: -100 },
{ name: "title", column: "notebooks.title", weight: 10 }
];
if (!opts.titleOnly) {
fields.push({
name: "description",
column: "notebooks.description"
});
}
return this.search(this.db.notebooks.all, query, fields);
}
tags(query: string) {
return this.search(this.db.tags.all, query, [
{ name: "id", column: "tags.id", weight: -100 },
{ name: "title", column: "tags.title" }
]);
}
reminders(query: string, opts: { titleOnly?: boolean } = {}) {
const fields: FuzzySearchField[] = [
{ name: "id", column: "reminders.id", weight: -100 },
{ name: "title", column: "reminders.title", weight: 10 }
];
if (!opts.titleOnly) {
fields.push({
name: "description",
column: "reminders.description"
});
}
return this.search(this.db.reminders.all, query, fields);
}
trash(query: string): SearchResults {
const sortOptions: SortOptions = {
sortBy: "dateDeleted",
sortDirection: "desc"
};
return {
sorted: async (limit?: number) => {
const { ids, items } = await this.filterTrash(
query,
limit,
sortOptions
);
return new VirtualizedGrouping(
ids.length,
this.db.options.batchSize,
() => Promise.resolve(ids),
async (start, end) => {
return {
ids: ids.slice(start, end),
items: items.slice(start, end)
};
}
);
},
items: async (limit?: number) => {
const { items } = await this.filterTrash(query, limit, sortOptions);
return items;
},
ids: () => this.filterTrash(query).then(({ ids }) => ids)
};
}
attachments(query: string) {
return this.search(this.db.attachments.all, query, [
{ name: "id", column: "attachments.id", weight: -100 },
{ name: "filename", column: "attachments.filename", weight: 5 },
{ name: "mimeType", column: "attachments.mimeType" },
{ name: "hash", column: "attachments.hash" }
]);
}
private search(
selector: FilteredSelector,
query: string,
fields: FuzzySearchField[]
) {
return this.toSearchResults(
(limit, sortOptions) =>
this.filter(selector, query, fields, limit, sortOptions),
selector
);
}
private async filter(
selector: FilteredSelector,
query: string,
fields: FuzzySearchField[],
limit?: number,
sortOptions?: SortOptions
) {
const results: Map = new Map();
const columns = fields.map((f) => f.column);
for await (const item of selector.fields(columns).iterate()) {
if (limit && results.size >= limit) break;
for (const field of fields) {
const result = match(query, `${item[field.name]}`);
if (result.match) {
const oldScore = results.get(item.id) || 0;
results.set(item.id, oldScore + result.score * (field.weight || 1));
}
}
}
selector.fields([]);
const sorted = Array.from(results.entries());
if (!sortOptions)
// || sortOptions.sortBy === "relevance")
sorted.sort(
// sortOptions?.sortDirection === "desc"
// ? (a, b) => a[1] - b[1]
// :
(a, b) => b[1] - a[1]
);
return sorted.map((a) => a[0]);
}
private toSearchResults(
ids: (limit?: number, sortOptions?: SortOptions) => Promise,
selector: FilteredSelector
): SearchResults {
const sortOptions: SortOptions = {
sortBy: "dateCreated",
sortDirection: "desc"
};
return {
sorted: async (limit?: number) =>
this.toVirtualizedGrouping(
await ids(limit, sortOptions),
selector,
sortOptions
),
items: async (limit?: number) =>
this.toItems(await ids(limit, sortOptions), selector, sortOptions),
ids
};
}
private async filterTrash(
query: string,
limit?: number,
sortOptions?: SortOptions
) {
const items = await this.db.trash.all();
const results: Map = new Map();
for (const item of items) {
if (limit && results.size >= limit) break;
const result = match(query, item.title);
if (result.match) {
results.set(item.id, { rank: result.score, item });
}
}
const sorted = Array.from(results.entries());
if (!sortOptions)
// || sortOptions.sortBy === "relevance")
sorted.sort(
// sortOptions?.sortDirection === "desc"
// ? (a, b) => a[1].rank - b[1].rank
// :
(a, b) => b[1].rank - a[1].rank
);
else {
const selector = getSortSelectors(sortOptions)[sortOptions.sortDirection];
sorted.sort((a, b) => selector(a[1].item, b[1].item));
}
return {
ids: sorted.map((a) => a[0]),
items: sorted.map((a) => a[1].item)
};
}
private toVirtualizedGrouping(
ids: string[],
selector: FilteredSelector,
sortOptions?: SortOptions
) {
// if (sortOptions?.sortBy === "relevance") sortOptions = undefined;
return new VirtualizedGrouping(
ids.length,
this.db.options.batchSize,
() => Promise.resolve(ids),
async (start, end) => {
const items = await selector.items(ids.slice(start, end), sortOptions);
return {
ids: ids.slice(start, end),
items
};
}
// (items) => groupArray(items, () => `${items.length} results`)
);
}
private toItems(
ids: string[],
selector: FilteredSelector,
sortOptions?: SortOptions
) {
if (!ids.length) return [];
// if (sortOptions?.sortBy === "relevance") sortOptions = undefined;
return selector.items(ids, sortOptions);
}
async rebuild() {
const db = this.db.sql() as unknown as Kysely;
await rebuildSearchIndex(db);
}
}