diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.ts index 66d8fb27c..85755b5ab 100644 --- a/packages/core/__tests__/notes.test.ts +++ b/packages/core/__tests__/notes.test.ts @@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +import dayjs from "dayjs"; import Database from "../src/api"; import { groupArray } from "../src/utils/grouping"; import { @@ -29,6 +30,7 @@ import { loginFakeUser } from "./utils"; import { test, expect } from "vitest"; +import { MONTHS_FULL } from "../src/utils/date"; async function createAndAddNoteToNotebook( db: Database, @@ -192,16 +194,6 @@ test("get pinned notes", () => expect(await db.notes.pinned.count()).toBeGreaterThan(0); })); -test.todo("get grouped notes by abc", () => groupedTest("abc")); - -test.todo("get grouped notes by month", () => groupedTest("month")); - -test.todo("get grouped notes by year", () => groupedTest("year")); - -test.todo("get grouped notes by weak", () => groupedTest("week")); - -test.todo("get grouped notes default", () => groupedTest("default")); - test("pin note", () => noteTest().then(async ({ db, id }) => { await db.notes.pin(true, id); @@ -388,3 +380,151 @@ test("adding a note with an invalid tag should clean the tag array", () => await db.relations.to({ id: "helloworld", type: "note" }, "tag").count() ).toBe(0); })); + +test("get grouped notes by abc", () => + databaseTest().then(async (db) => { + const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + for (const letter of alphabet) { + for (const letter2 of alphabet) { + await db.notes.add({ title: `${letter}${letter2}` }); + } + } + await db.notes.add({ title: `Pinned note`, pinned: true }); + await db.notes.add({ + title: `Conflicted`, + conflicted: true + }); + const { grouping, ids } = await db.notes.all.grouped({ + groupBy: "abc", + sortDirection: "asc", + sortBy: "title" + }); + + expect((await grouping.item(ids[0]))?.group?.title).toBe("Conflicted"); + expect((await grouping.item(ids[1]))?.group?.title).toBe("Pinned"); + for (let i = 0; i < alphabet.length; ++i) { + expect( + (await grouping.item(ids[i * alphabet.length + 2]))?.group?.title + ).toBe(alphabet[i]); + } + })); + +test("get grouped notes by month", () => + databaseTest().then(async (db) => { + for (let month = 0; month <= 11; ++month) { + for (let i = 0; i < 5; ++i) { + const date = dayjs().month(month).toDate().getTime(); + await db.notes.add({ + title: `Note in ${month} - ${i}`, + dateCreated: date + }); + } + } + + const { grouping, ids } = await db.notes.all.grouped({ + groupBy: "month", + sortDirection: "desc", + sortBy: "dateCreated" + }); + + for (let month = 11; month >= 0; --month) { + expect((await grouping.item(ids[(11 - month) * 5]))?.group?.title).toBe( + MONTHS_FULL[month] + ); + } + })); + +test("get grouped notes by year", () => + databaseTest().then(async (db) => { + for (let year = 2020; year <= 2025; ++year) { + for (let i = 0; i < 5; ++i) { + const date = dayjs().year(year).toDate().getTime(); + await db.notes.add({ + title: `Note in ${year} - ${i}`, + dateCreated: date + }); + } + } + + const { grouping, ids } = await db.notes.all.grouped({ + groupBy: "year", + sortDirection: "desc", + sortBy: "dateCreated" + }); + + for (let year = 2020; year <= 2025; ++year) { + expect((await grouping.item(ids[(2025 - year) * 5]))?.group?.title).toBe( + year.toString() + ); + } + })); + +test("get grouped notes by week", () => + databaseTest().then(async (db) => { + for (let i = 1; i <= 6; ++i) { + for (let week = 1; week <= 4; ++week) { + const date = dayjs() + .month(i) + .date(week * 7) + .year(2023) + .startOf("week") + .toDate() + .getTime(); + await db.notes.add({ + title: `Note in ${week} - ${i}`, + dateCreated: date + }); + } + } + + const { grouping, ids } = await db.notes.all.grouped({ + groupBy: "week", + sortDirection: "desc", + sortBy: "dateCreated" + }); + + const weeks = [ + "19 - 25 Jun, 2023", + "22 - 28 May, 2023", + "17 - 23 Apr, 2023", + "20 - 26 Mar, 2023", + "20 - 26 Feb, 2023" + ]; + for (let i = 1; i <= 5; ++i) { + expect((await grouping.item(ids[i * 4]))?.group?.title).toBe( + weeks[i - 1] + ); + } + })); + +test("get grouped notes default", () => + databaseTest().then(async (db) => { + const ranges = { + Recent: [0, 7], + "Last week": [7, 14], + Older: [14, 28] + }; + for (const key in ranges) { + const range = ranges[key]; + for (let i = range[0]; i < range[1]; i++) { + const date = dayjs().subtract(i, "days").toDate().getTime(); + + await db.notes.add({ + title: `Note in ${key} - ${i}`, + dateCreated: date + }); + } + } + + const { grouping, ids } = await db.notes.all.grouped({ + groupBy: "default", + sortDirection: "desc", + sortBy: "dateCreated" + }); + + let i = 0; + for (const key in ranges) { + expect((await grouping.item(ids[i * 7]))?.group?.title).toBe(key); + ++i; + } + })); diff --git a/packages/core/__tests__/utils.test.js b/packages/core/__tests__/utils.test.js deleted file mode 100644 index 7b100e7d9..000000000 --- a/packages/core/__tests__/utils.test.js +++ /dev/null @@ -1,41 +0,0 @@ -/* -This file is part of the Notesnook project (https://notesnook.com/) - -Copyright (C) 2023 Streetwriters (Private) Limited - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -import { groupArray } from "../src/utils/grouping"; -import { test, expect } from "vitest"; - -test("group alphabetically", () => { - const sortedAlphabet = "abcdefghijlmnopqrstuvwxyz" - .split("") - .map((a) => ({ title: a })); - const alphabet = "nopqrstuvwxyzabcdefghijlm" - .split("") - .map((a) => ({ title: a, item: true })); - let ret = groupArray(alphabet, { - groupBy: "abc", - sortDirection: "asc", - sortBy: "title" - }).filter((v) => !v.item); - - expect( - sortedAlphabet.every((alpha, index) => { - return ret[index].title === alpha.title.toUpperCase(); - }) - ).toBeTruthy(); -}); diff --git a/packages/core/incremental-grouping.md b/packages/core/incremental-grouping.md new file mode 100644 index 000000000..d2df4cdc7 --- /dev/null +++ b/packages/core/incremental-grouping.md @@ -0,0 +1,81 @@ +# Incremental grouping of list items + +## Requirements + +1. We do not want to bring all the items to the client side as that would take a lot of memory +2. We want to preserve & customize sorting +3. We want to avoid creating duplicate groups +4. Each batch of items should append and add groups as necesary + +## Approach + +1. Get all the IDs of all the items sorted using the required configuration +2. Request the first batch of `n` items. +3. Group the first batch +4. Bring the second batch and similarly group it. +5. If the last group of the previous batch & the first group of the new batch match up, we can remove the new group, otherwise keep it. + +Example: + +Items: A, Aa, B, Ba, Bb, C, Ca, Cb, Cc, Cd +Batch size: 2 + +Batch 1: [A, Aa] +Grouped: [A, [A, Aa]] + +Batch 2: [B, Ba] +Grouped: [B, [B, Ba]] + +Merged: [A, [A, Aa], B, [B, Ba]] + +Batch 3: [Bb, C] +Grouped: [B, [Bb], C, [C]] + +Merged: [A, [A, Aa], B, [B, Ba, Bb], C, [C]] + +This is a simple incremental grouping. However, what we require is windowed grouping. + +## Windowed incremental grouping of list items + +Windowed grouping means that we only keep `n` number of items in memory at any point in time. +As the user scrolls up & down, we fetch only the batch that is needed for rendering the items +in the current window. + +Requirements: + +1. Only keep N items in memory +2. Remove the previous window's items as soon as we don't need them +3. Respect sorting & grouping while doing append/prepend + +Approach: + +1. Get all the IDs of all the items sorted using the required configuration +2. Request the first batch of `n` items. +3. Group the first batch +4. If user is moving downward: + 1. When user moves closer to the the end of the first batch, load the next batch & group it accordingly + 2. Compare last & first groups of both batches + 3. If last & first groups are the same, remove the first group of the new batch + 4. Otherwise keep it & do nothing + 5. If user reaches the end of the second batch, remove the first batch +5. If user is moving upward: + Suppose we have batches 2 & 3 in memory, user is at the end of batch 2 and starts scrolling upwards: + 1. When user reaches closer to the start of the batch 2, load batch 1 & remove batch 3 + 2. For grouping compare the last & first groups of batch 1 & 2 respectively + 3. Remove first group of batch 2 if both match, otherwise do nothing + +Questions: + +1. What if new items are added at random places during sync or something else? + +The naive but stable approach would be to refetch everything and recalculate the current batch. +Another approach would be to diff the new & old set of IDs and determine which batches have +changed. + +We can then reload the batches only if they are currently cached, otherwise do nothing. + +2. What will be the responsibility of the core module? + +The core module will only manage the cache. It will also be responsible for loading the appropriate +batch based on if the item is found in the cache or not. This should be automatic for maximum +optimization opportunities. diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.ts index 48bd481c3..b3be4baab 100644 --- a/packages/core/src/database/sql-collection.ts +++ b/packages/core/src/database/sql-collection.ts @@ -18,7 +18,7 @@ along with this program. If not, see . */ import { EVENTS } from "../common"; -import { GroupOptions, MaybeDeletedItem, isDeleted } from "../types"; +import { GroupOptions, Item, MaybeDeletedItem, isDeleted } from "../types"; import EventManager from "../utils/event-manager"; import { DatabaseAccessor, @@ -28,6 +28,8 @@ import { isFalse } from "."; import { ExpressionOrFactory, SelectQueryBuilder, SqlBool } from "kysely"; +import { VirtualizedGrouping } from "../utils/virtualized-grouping"; +import { groupArray } from "../utils/grouping"; export class SQLCollection< TCollectionType extends keyof DatabaseSchema, @@ -164,16 +166,12 @@ export class SQLCollection< } async items( - ids: string[], - sortOptions?: GroupOptions + ids: string[] ): Promise | undefined>> { const results = await this.db() .selectFrom(this.type) .selectAll() .where("id", "in", ids) - .$if(!!sortOptions, (eb) => - eb.orderBy(sortOptions!.sortBy, sortOptions!.sortDirection) - ) .execute(); const items: Record> = {}; for (const item of results) { @@ -227,7 +225,7 @@ export class SQLCollection< } } - createFilter( + createFilter( selector: ( qb: SelectQueryBuilder ) => SelectQueryBuilder, @@ -240,7 +238,7 @@ export class SQLCollection< } } -export class FilteredSelector { +export class FilteredSelector { constructor( readonly filter: SelectQueryBuilder< DatabaseSchema, @@ -250,13 +248,23 @@ export class FilteredSelector { readonly batchSize: number ) {} - async ids() { - return (await this.filter.select("id").execute()).map((i) => i.id); + async ids(sortOptions?: GroupOptions) { + return ( + await this.filter + .$if(!!sortOptions, (eb) => + eb.$call(this.buildSortExpression(sortOptions!)) + ) + .select("id") + .execute() + ).map((i) => i.id); } - async items(ids?: string[]) { + async items(ids?: string[], sortOptions?: GroupOptions) { return (await this.filter .$if(!!ids && ids.length > 0, (eb) => eb.where("id", "in", ids!)) + .$if(!!sortOptions, (eb) => + eb.$call(this.buildSortExpression(sortOptions!)) + ) .selectAll() .execute()) as T[]; } @@ -298,6 +306,40 @@ export class FilteredSelector { } } + async grouped(options: GroupOptions) { + const ids = await this.ids(options); + return { + ids, + grouping: new VirtualizedGrouping( + ids, + this.batchSize, + async (ids) => { + const results = await this.filter + .where("id", "in", ids) + .selectAll() + .execute(); + const items: Record = {}; + for (const item of results) { + items[item.id] = item as T; + } + return items; + }, + (ids, items) => groupArray(ids, items, options) + ) + }; + } + + private buildSortExpression(options: GroupOptions) { + return ( + qb: SelectQueryBuilder + ) => { + return qb + .orderBy("conflicted desc") + .orderBy("pinned desc") + .orderBy(options.sortBy, options.sortDirection); + }; + } + async *[Symbol.asyncIterator]() { let index = 0; while (true) { diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 80792b0bc..f0d13021b 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -81,6 +81,7 @@ export type GroupableItem = ValueOf< | "shortcut" | "relation" | "tiny" + | "topic" | "tiptap" | "content" | "session" diff --git a/packages/core/src/utils/__tests__/virtualized-grouping.test.ts b/packages/core/src/utils/__tests__/virtualized-grouping.test.ts new file mode 100644 index 000000000..e85e77d6d --- /dev/null +++ b/packages/core/src/utils/__tests__/virtualized-grouping.test.ts @@ -0,0 +1,162 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import { expect, test, vi } from "vitest"; +import { VirtualizedGrouping } from "../virtualized-grouping"; + +function item(value: T) { + return { item: value }; +} +function createMock() { + return vi.fn(async (ids: string[]) => + Object.fromEntries(ids.map((id) => [id, id])) + ); +} +test("fetch items in batch if not found in cache", async (t) => { + const mocked = createMock(); + const grouping = new VirtualizedGrouping( + ["1", "2", "3", "4", "5", "6", "7"], + 3, + mocked + ); + t.expect(await grouping.item("4")).toStrictEqual(item("4")); + t.expect(mocked).toHaveBeenCalledOnce(); +}); + +test("do not fetch items in batch if found in cache", async (t) => { + const mocked = createMock(); + const grouping = new VirtualizedGrouping( + ["1", "2", "3", "4", "5", "6", "7"], + 3, + mocked + ); + t.expect(await grouping.item("4")).toStrictEqual(item("4")); + t.expect(await grouping.item("4")).toStrictEqual(item("4")); + t.expect(await grouping.item("4")).toStrictEqual(item("4")); + t.expect(await grouping.item("4")).toStrictEqual(item("4")); + t.expect(await grouping.item("4")).toStrictEqual(item("4")); + t.expect(mocked).toHaveBeenCalledOnce(); +}); + +test("clear old cached batches", async (t) => { + const mocked = createMock(); + const grouping = new VirtualizedGrouping( + ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], + 3, + mocked + ); + t.expect(await grouping.item("1")).toStrictEqual(item("1")); + t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); + t.expect(await grouping.item("4")).toStrictEqual(item("4")); + t.expect(mocked).toHaveBeenLastCalledWith(["4", "5", "6"]); + t.expect(await grouping.item("7")).toStrictEqual(item("7")); + t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]); + t.expect(await grouping.item("1")).toStrictEqual(item("1")); + t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); +}); + +test("clear old cached batches (random access)", async (t) => { + const mocked = createMock(); + const grouping = new VirtualizedGrouping( + ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], + 3, + mocked + ); + t.expect(await grouping.item("1")).toStrictEqual(item("1")); + t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); + + t.expect(await grouping.item("7")).toStrictEqual(item("7")); + t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]); + + t.expect(await grouping.item("11")).toStrictEqual(item("11")); + t.expect(mocked).toHaveBeenLastCalledWith(["10", "11", "12"]); + + t.expect(await grouping.item("1")).toStrictEqual(item("1")); + t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); + + t.expect(await grouping.item("7")).toStrictEqual(item("7")); + t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]); +}); + +test("reloading ids should clear all cached batches", async (t) => { + const mocked = createMock(); + const grouping = new VirtualizedGrouping( + ["1", "3", "4", "5", "7", "6", "50"], + 3, + mocked + ); + + t.expect(await grouping.item("1")).toStrictEqual(item("1")); + t.expect(mocked).toHaveBeenLastCalledWith(["1", "3", "4"]); + + grouping.refresh([ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12" + ]); + + t.expect(await grouping.item("1")).toStrictEqual(item("1")); + t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); +}); + +test("merge groups if last & first groups are the same (sequential)", async (t) => { + const mocked = createMock(); + const grouping = new VirtualizedGrouping( + ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], + 3, + mocked, + (ids) => [{ title: "Hello", id: ids[0] }] + ); + expect((await grouping.item("1"))?.group?.title).toBe("Hello"); + expect((await grouping.item("4"))?.group).toBeUndefined(); +}); + +test("merge groups if last & first groups are the same (random)", async (t) => { + const mocked = createMock(); + const grouping = new VirtualizedGrouping( + ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], + 3, + mocked, + (ids) => [{ title: "Hello", id: ids[0] }] + ); + expect((await grouping.item("1"))?.group?.title).toBe("Hello"); + expect((await grouping.item("7"))?.group).toBeUndefined(); +}); + +test("merge groups if last & first groups are the same (reverse)", async (t) => { + const mocked = createMock(); + const grouping = new VirtualizedGrouping( + ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], + 3, + mocked, + (ids) => [{ title: "Hello", id: ids[0] }] + ); + expect((await grouping.item("7"))?.group?.title).toBe("Hello"); + expect((await grouping.item("1"))?.group?.title).toBe("Hello"); + expect((await grouping.item("7"))?.group).toBeUndefined(); +}); diff --git a/packages/core/src/utils/grouping.ts b/packages/core/src/utils/grouping.ts index 905928ecb..853e811da 100644 --- a/packages/core/src/utils/grouping.ts +++ b/packages/core/src/utils/grouping.ts @@ -18,14 +18,9 @@ along with this program. If not, see . */ import { isReminderActive } from "../collections/reminders"; -import { - GroupedItems, - GroupOptions, - GroupableItem, - Item, - Reminder -} from "../types"; +import { GroupOptions, Item } from "../types"; import { getWeekGroupFromTimestamp, MONTHS_FULL } from "./date"; +import { VirtualizedGroupHeader } from "./virtualized-grouping"; type EvaluateKeyFunction = (item: T) => string; @@ -45,33 +40,19 @@ export const getSortValue = ( return item.dateCreated; }; -function getSortSelectors(options: GroupOptions) { - if (options.sortBy === "title") - return { - asc: (a: T, b: T) => - getTitle(a).localeCompare(getTitle(b), undefined, { numeric: true }), - desc: (a: T, b: T) => - getTitle(b).localeCompare(getTitle(a), undefined, { numeric: true }) - }; - - return { - asc: (a: T, b: T) => getSortValue(options, a) - getSortValue(options, b), - desc: (a: T, b: T) => getSortValue(options, b) - getSortValue(options, a) - }; -} - const MILLISECONDS_IN_DAY = 1000 * 60 * 60 * 24; const MILLISECONDS_IN_WEEK = MILLISECONDS_IN_DAY * 7; -function getKeySelector( - options: GroupOptions -): EvaluateKeyFunction { - return (item: T) => { +function getKeySelector(options: GroupOptions): EvaluateKeyFunction { + return (item: Item) => { if ("pinned" in item && item.pinned) return "Pinned"; else if ("conflicted" in item && item.conflicted) return "Conflicted"; const date = new Date(); - if (options.sortBy === "title") return getFirstCharacter(getTitle(item)); + if (item.type === "reminder") + return isReminderActive(item) ? "Active" : "Inactive"; + else if (options.sortBy === "title") + return getFirstCharacter(getTitle(item)); else { const value = getSortValue(options, item); switch (options.groupBy) { @@ -98,78 +79,37 @@ function getKeySelector( }; } -export function groupArray( - array: T[], +export function groupArray( + ids: string[], + items: Record, options: GroupOptions = { groupBy: "default", sortBy: "dateEdited", sortDirection: "desc" } -): GroupedItems { - if (options.sortBy && options.sortDirection) { - const selector = getSortSelectors(options)[options.sortDirection]; - array.sort(selector); - } - - if (options.groupBy === "none") { - const conflicted: T[] = []; - const pinned: T[] = []; - const others: T[] = []; - for (const item of array) { - if ("pinned" in item && item.pinned) { - pinned.push(item); - continue; - } else if ("conflicted" in item && item.conflicted) { - conflicted.push(item); - continue; - } else others.push(item); - } - const groups: GroupedItems = []; - if (conflicted.length > 0) - groups.push( - { title: "Conflicted", type: "header", id: "conflicted" }, - ...conflicted - ); - if (pinned.length > 0) - groups.push({ title: "Pinned", type: "header", id: "pinned" }, ...pinned); - if (others.length > 0) - groups.push({ title: "All", type: "header", id: "all" }, ...others); - return groups; - } - - const groups = new Map([ - ["Conflicted", []], - ["Pinned", []] +): VirtualizedGroupHeader[] { + const groups = new Map([ + ["Conflicted", { title: "Conflicted", id: "" }], + ["Pinned", { title: "Pinned", id: "" }], + ["Active", { title: "Active", id: "" }], + ["Inactive", { title: "Inactive", id: "" }] ]); const keySelector = getKeySelector(options); - array.forEach((item) => addToGroup(groups, keySelector(item), item)); + for (const id of ids) { + const item = items[id]; + if (!item) continue; - return flattenGroups(groups); -} + const groupTitle = keySelector(item); + const group = groups.get(groupTitle) || { + title: groupTitle, + id: "" + }; + if (group.id === "") group.id = id; + groups.set(groupTitle, group); + } -export function groupReminders(array: Reminder[]): GroupedItems { - const groups = new Map([ - ["Active", []], - ["Inactive", []] - ]); - - array.forEach((item) => { - const groupTitle = isReminderActive(item) ? "Active" : "Inactive"; - addToGroup(groups, groupTitle, item); - }); - - return flattenGroups(groups); -} - -function addToGroup( - groups: Map, - groupTitle: string, - item: T -) { - const group = groups.get(groupTitle) || []; - group.push(item); - groups.set(groupTitle, group); + return Array.from(groups.values()); } function getFirstCharacter(str: string) { @@ -179,21 +119,10 @@ function getFirstCharacter(str: string) { return str[0].toUpperCase(); } -function getTitle(item: T): string { - return item.type === "attachment" ? item.metadata.filename : item.title; -} - -function flattenGroups(groups: Map) { - const items: GroupedItems = []; - groups.forEach((groupItems, groupTitle) => { - if (groupItems.length <= 0) return; - items.push({ - title: groupTitle, - id: groupTitle.toLowerCase(), - type: "header" - }); - groupItems.forEach((item) => items.push(item)); - }); - - return items; +function getTitle(item: Item): string { + return item.type === "attachment" + ? item.filename + : "title" in item + ? item.title + : "Unknown"; } diff --git a/packages/core/src/utils/set.ts b/packages/core/src/utils/set.ts index 683253dbc..19e414178 100644 --- a/packages/core/src/utils/set.ts +++ b/packages/core/src/utils/set.ts @@ -85,14 +85,14 @@ class SetManipulator { // Symmetric difference. Items from either set that // are not in both sets. // Set.difference([1, 1, 2], [2, 3, 3]) => [1, 3] - difference(a: T[], b: T[], key: KeySelector) { - return this.process(a, b, key, (freq) => freq < 3); + difference(a: T[], b: T[], key?: KeySelector) { + return this.process(a, b, key, (freq) => freq < 3); } // Relative complement. Items from 'a' which are // not also in 'b'. // Set.complement([1, 2, 2], [2, 2, 3]) => [3] - complement(a: T[], b: T[], key: KeySelector) { + complement(a: T[], b: T[], key?: KeySelector) { return this.process(a, b, key, (freq) => freq === 1); } diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.ts new file mode 100644 index 000000000..c748c253d --- /dev/null +++ b/packages/core/src/utils/virtualized-grouping.ts @@ -0,0 +1,111 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +export type VirtualizedGroupHeader = { + title: string; + id: string; +}; +export class VirtualizedGrouping { + private cache: Map> = new Map(); + private groups: Map = new Map(); + + constructor( + private ids: string[], + private readonly batchSize: number, + private readonly fetchItems: (ids: string[]) => Promise>, + private readonly groupItems: ( + ids: string[], + items: Record + ) => VirtualizedGroupHeader[] = () => [] + ) { + this.ids = ids; + } + + /** + * Get item from cache or request the appropriate batch for caching + * and load it from there. + */ + async item(id: string) { + const index = this.ids.indexOf(id); + if (index <= -1) return; + + const batchIndex = Math.floor(index / this.batchSize); + const batch = this.cache.get(batchIndex) || (await this.load(batchIndex)); + const groups = this.groups.get(batchIndex); + + const group = groups?.find((g) => g.id === id); + if (group) + return { + group: { type: "header", id: group.title, title: group.title }, + item: batch[id] + }; + return { item: batch[id] }; + } + + /** + * Reload the cache + */ + refresh(ids: string[]) { + this.ids = ids; + this.cache.clear(); + } + + /** + * + * @param index + */ + private async load(batch: number) { + const start = batch * this.batchSize; + const end = start + this.batchSize; + const batchIds = this.ids.slice(start, end); + const items = await this.fetchItems(batchIds); + const groups = this.groupItems(batchIds, items); + + const lastBatchIndex = this.last; + const prevGroups = this.groups.get(lastBatchIndex); + if (prevGroups && prevGroups.length > 0 && groups.length > 0) { + const lastGroup = prevGroups[prevGroups.length - 1]; + if (lastGroup.title === groups[0].title) { + // if user is moving downwards, we remove the last group from the + // current batch, otherwise we remove the first group from the previous + // batch. + lastBatchIndex < batch ? groups.pop() : prevGroups.shift(); + } + } + + this.cache.set(batch, items); + this.groups.set(batch, groups); + this.clear(); + return items; + } + + private clear() { + if (this.cache.size <= 2) return; + for (const [key] of this.cache) { + this.cache.delete(key); + this.groups.delete(key); + if (this.cache.size === 2) break; + } + } + + private get last() { + const keys = Array.from(this.cache.keys()); + return keys[keys.length - 1]; + } +}