diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.ts index 3b3dc0040..bd2a8f8e8 100644 --- a/packages/core/__tests__/notes.test.ts +++ b/packages/core/__tests__/notes.test.ts @@ -19,7 +19,7 @@ along with this program. If not, see . import dayjs from "dayjs"; import Database from "../src/api"; -import { groupArray } from "../src/utils/grouping"; +import { createKeySelector, groupArray } from "../src/utils/grouping"; import { databaseTest, noteTest, @@ -346,11 +346,14 @@ test("grouping items where item.title is empty or undefined shouldn't throw", () expect( // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - groupArray([{ title: "" }], { - groupBy: "abc", - sortBy: "title", - sortDirection: "asc" - }) + groupArray( + [{ title: "" }], + createKeySelector({ + groupBy: "abc", + sortBy: "title", + sortDirection: "asc" + }) + ) ).toBeTruthy(); }); diff --git a/packages/core/src/collections/trash.ts b/packages/core/src/collections/trash.ts index 584cea888..b62e8d937 100644 --- a/packages/core/src/collections/trash.ts +++ b/packages/core/src/collections/trash.ts @@ -22,7 +22,11 @@ import Database from "../api"; import { deleteItems, toChunks } from "../utils/array"; import { GroupOptions, TrashItem } from "../types"; import { VirtualizedGrouping } from "../utils/virtualized-grouping"; -import { getSortSelectors, groupArray } from "../utils/grouping"; +import { + createKeySelector, + getSortSelectors, + groupArray +} from "../utils/grouping"; import { sql } from "kysely"; import { MAX_SQL_PARAMETERS } from "../database/sql-collection"; @@ -282,11 +286,13 @@ export default class Trash { items }; }, - (items) => groupArray(items, options), + (items) => groupArray(items, createKeySelector(options)), async () => { const items = await this.all(); items.sort(selector); - return Array.from(groupArray(items, options).values()); + return Array.from( + groupArray(items, createKeySelector(options)).values() + ); } ); } diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.ts index cd4c63537..88df76639 100644 --- a/packages/core/src/database/sql-collection.ts +++ b/packages/core/src/database/sql-collection.ts @@ -47,7 +47,7 @@ import { sql } from "kysely"; import { VirtualizedGrouping } from "../utils/virtualized-grouping"; -import { groupArray } from "../utils/grouping"; +import { createKeySelector, groupArray } from "../utils/grouping"; import { toChunks } from "../utils/array"; import { Sanitizer } from "./sanitizer"; import { @@ -450,7 +450,7 @@ export class FilteredSelector { items }; }, - (items) => groupArray(items as any, options), + (items) => groupArray(items as any, createKeySelector(options)), () => this.groups(options) ); } @@ -486,7 +486,7 @@ export class FilteredSelector { .select(fields) .$call(this.buildSortExpression(options, true)) .execute(), - options + createKeySelector(options) ).values() ); } diff --git a/packages/core/src/utils/__tests__/virtualized-grouping.test.ts b/packages/core/src/utils/__tests__/virtualized-grouping.test.ts index 1fc943f4a..ca0e7416a 100644 --- a/packages/core/src/utils/__tests__/virtualized-grouping.test.ts +++ b/packages/core/src/utils/__tests__/virtualized-grouping.test.ts @@ -17,110 +17,108 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -import { test, vi } from "vitest"; +import { test } from "vitest"; import { VirtualizedGrouping } from "../virtualized-grouping"; +import { groupArray } from "../grouping"; -function item(value: T) { - return { item: value }; +function generateItems(length: number, groupSize: number) { + const items: { group: string; id: string }[] = []; + const ids: string[] = []; + const divider = length / groupSize; + for (let i = 0; i < length; ++i) { + items.push({ group: `${i % divider}`, id: `${i}` }); + ids.push(`${i}`); + } + items.sort((a, b) => a.group.localeCompare(b.group)); + return { items, ids }; } -function createMock() { - return vi.fn(async (ids: string[]) => - Object.fromEntries(ids.map((id) => [id, id])) + +function createVirtualizedGrouping( + length: number, + groupSize: number, + batchSize: number +) { + const { ids, items } = generateItems(length, groupSize); + return new VirtualizedGrouping<{ group: string; id: string }>( + items.length, + batchSize, + () => Promise.resolve(ids), + async (start, end) => ({ + ids: ids.slice(start, end), + items: items.slice(start, end) + }), + (items) => groupArray(items, (item) => item.group) ); } -test.todo("renable virtualized grouping tests"); -// test("fetch items in batch if not found in cache", async (t) => { -// const mocked = createMock(); -// const grouping = new VirtualizedGrouping( -// ["1", "2", "3", "4", "5", "6", "7"], -// 3, -// mocked -// ); -// t.expect(await grouping.item("4")).toStrictEqual(item("4")); -// t.expect(mocked).toHaveBeenCalledOnce(); -// }); -// test("do not fetch items in batch if found in cache", async (t) => { -// const mocked = createMock(); -// const grouping = new VirtualizedGrouping( -// ["1", "2", "3", "4", "5", "6", "7"], -// 3, -// mocked -// ); -// t.expect(await grouping.item("4")).toStrictEqual(item("4")); -// t.expect(await grouping.item("4")).toStrictEqual(item("4")); -// t.expect(await grouping.item("4")).toStrictEqual(item("4")); -// t.expect(await grouping.item("4")).toStrictEqual(item("4")); -// t.expect(await grouping.item("4")).toStrictEqual(item("4")); -// t.expect(mocked).toHaveBeenCalledOnce(); -// }); +test("load first batch with a single group", async (t) => { + const grouping = createVirtualizedGrouping(100, 10, 10); -// test("clear old cached batches", async (t) => { -// const mocked = createMock(); -// const grouping = new VirtualizedGrouping( -// ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], -// 3, -// mocked -// ); -// t.expect(await grouping.item("1")).toStrictEqual(item("1")); -// t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); -// t.expect(await grouping.item("4")).toStrictEqual(item("4")); -// t.expect(mocked).toHaveBeenLastCalledWith(["4", "5", "6"]); -// t.expect(await grouping.item("7")).toStrictEqual(item("7")); -// t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]); -// t.expect(await grouping.item("1")).toStrictEqual(item("1")); -// t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); -// }); + t.expect((await grouping.item(0)).group?.title).toBe("0"); + for (let i = 1; i < 10; ++i) + t.expect(grouping.cacheItem(i)?.group?.title).toBeUndefined(); +}); -// test("clear old cached batches (random access)", async (t) => { -// const mocked = createMock(); -// const grouping = new VirtualizedGrouping( -// ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], -// 3, -// mocked -// ); -// t.expect(await grouping.item("1")).toStrictEqual(item("1")); -// t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); +test("load first batch with a multiple groups", async (t) => { + const grouping = createVirtualizedGrouping(100, 2, 10); -// t.expect(await grouping.item("7")).toStrictEqual(item("7")); -// t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]); + t.expect((await grouping.item(0)).group?.title).toBe(`0`); + t.expect(grouping.cacheItem(2)?.group?.title).toBe(`1`); + t.expect(grouping.cacheItem(4)?.group?.title).toBe(`10`); + t.expect(grouping.cacheItem(6)?.group?.title).toBe(`11`); + t.expect(grouping.cacheItem(8)?.group?.title).toBe(`12`); +}); -// t.expect(await grouping.item("11")).toStrictEqual(item("11")); -// t.expect(mocked).toHaveBeenLastCalledWith(["10", "11", "12"]); +test("load last batch with a single group", async (t) => { + const grouping = createVirtualizedGrouping(100, 10, 10); -// t.expect(await grouping.item("1")).toStrictEqual(item("1")); -// t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); + t.expect((await grouping.item(90)).group?.title).toBe("9"); + for (let i = 91; i < 100; ++i) + t.expect(grouping.cacheItem(i)?.group?.title).toBeUndefined(); +}); -// t.expect(await grouping.item("7")).toStrictEqual(item("7")); -// t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]); -// }); +test("load last batch with a multiple groups", async (t) => { + const grouping = createVirtualizedGrouping(100, 2, 10); -// test("reloading ids should clear all cached batches", async (t) => { -// const mocked = createMock(); -// const grouping = new VirtualizedGrouping( -// ["1", "3", "4", "5", "7", "6", "50"], -// 3, -// mocked -// ); + t.expect((await grouping.item(90)).group?.title).toBe(`5`); + t.expect(grouping.cacheItem(92)?.group?.title).toBe(`6`); + t.expect(grouping.cacheItem(94)?.group?.title).toBe(`7`); + t.expect(grouping.cacheItem(96)?.group?.title).toBe(`8`); + t.expect(grouping.cacheItem(98)?.group?.title).toBe(`9`); +}); -// t.expect(await grouping.item("1")).toStrictEqual(item("1")); -// t.expect(mocked).toHaveBeenLastCalledWith(["1", "3", "4"]); +test("group spanning multiple batches (down)", async (t) => { + const grouping = createVirtualizedGrouping(140, 14, 10); -// grouping.refresh([ -// "1", -// "2", -// "3", -// "4", -// "5", -// "6", -// "7", -// "8", -// "9", -// "10", -// "11", -// "12" -// ]); + t.expect((await grouping.item(0)).group?.title).toBe(`0`); + t.expect((await grouping.item(12)).group).toBeUndefined(); + t.expect((await grouping.item(14)).group?.title).toBe("1"); + t.expect((await grouping.item(24)).group).toBeUndefined(); + t.expect((await grouping.item(28)).group?.title).toBe("2"); +}); -// t.expect(await grouping.item("1")).toStrictEqual(item("1")); -// t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); -// }); +test("single group in all batches", async (t) => { + const grouping = createVirtualizedGrouping(100, 100, 10); + + t.expect((await grouping.item(0)).group?.title).toBe(`0`); + for (let i = 1; i < 100; ++i) { + t.expect((await grouping.item(i)).group).toBeUndefined(); + } +}); + +test("group at start of each batch", async (t) => { + const grouping = createVirtualizedGrouping(100, 10, 10); + + for (let i = 0; i < 100; i += 10) { + t.expect((await grouping.item(i)).group?.title).toBe(`${i / 10}`); + } +}); + +test("group spanning multiple batches (up)", async (t) => { + const grouping = createVirtualizedGrouping(140, 28, 10); + + t.expect((await grouping.item(130)).group).toBeUndefined(); + t.expect((await grouping.item(120)).group).toBeUndefined(); + t.expect((await grouping.item(140 - 28)).group).toBeDefined(); + t.expect((await grouping.item(110)).group).toBeUndefined(); +}); diff --git a/packages/core/src/utils/grouping.ts b/packages/core/src/utils/grouping.ts index 8a9659598..ce75608f5 100644 --- a/packages/core/src/utils/grouping.ts +++ b/packages/core/src/utils/grouping.ts @@ -30,7 +30,7 @@ type PartialGroupableItem = { dateEdited?: number | null; dateCreated?: number | null; }; -type EvaluateKeyFunction = (item: T) => string; +export type GroupKeySelectorFunction = (item: T) => string; export const getSortValue = ( options: GroupOptions | undefined, @@ -72,9 +72,13 @@ export function getSortSelectors( const MILLISECONDS_IN_DAY = 1000 * 60 * 60 * 24; const MILLISECONDS_IN_WEEK = MILLISECONDS_IN_DAY * 7; -function getKeySelector( - options: GroupOptions -): EvaluateKeyFunction { +export function createKeySelector( + options: GroupOptions = { + groupBy: "default", + sortBy: "dateEdited", + sortDirection: "desc" + } +): GroupKeySelectorFunction { return (item) => { if ("pinned" in item && item.pinned) return "Pinned"; else if ("conflicted" in item && item.conflicted) return "Conflicted"; @@ -110,20 +114,15 @@ function getKeySelector( }; } -export function groupArray( - items: PartialGroupableItem[], - options: GroupOptions = { - groupBy: "default", - sortBy: "dateEdited", - sortDirection: "desc" - } +export function groupArray( + items: T[], + keySelector: GroupKeySelectorFunction ): Map { const groups = new Map< string, [number, { index: number; group: GroupHeader }] >(); - const keySelector = getKeySelector(options); for (let i = 0; i < items.length; ++i) { const item = items[i]; const groupTitle = keySelector(item); diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.ts index 61a97b438..9ff1311ce 100644 --- a/packages/core/src/utils/virtualized-grouping.ts +++ b/packages/core/src/utils/virtualized-grouping.ts @@ -123,6 +123,9 @@ export class VirtualizedGrouping { const { ids, items } = await this.fetchItems(start, end); const groups = this.groupItems?.(items); + if (items.length > this.batchSize) + throw new Error("Got more items than the batch size."); + if (direction === "down") { const [, firstGroup] = groups ? firstInMap(groups) : []; const group = lastBatch?.groups