core: add tests for virtualized grouping

packages/core/src/collections/trash.ts

@@ -22,7 +22,11 @@ import Database from "../api";
 import { deleteItems, toChunks } from "../utils/array";
 import { GroupOptions, TrashItem } from "../types";
 import { VirtualizedGrouping } from "../utils/virtualized-grouping";
-import { getSortSelectors, groupArray } from "../utils/grouping";
+import {
+  createKeySelector,
+  getSortSelectors,
+  groupArray
+} from "../utils/grouping";
 import { sql } from "kysely";
 import { MAX_SQL_PARAMETERS } from "../database/sql-collection";
 
@@ -282,11 +286,13 @@ export default class Trash {
           items
         };
       },
-      (items) => groupArray(items, options),
+      (items) => groupArray(items, createKeySelector(options)),
       async () => {
         const items = await this.all();
         items.sort(selector);
-        return Array.from(groupArray(items, options).values());
+        return Array.from(
+          groupArray(items, createKeySelector(options)).values()
+        );
       }
     );
   }

packages/core/src/database/sql-collection.ts

@@ -47,7 +47,7 @@ import {
   sql
 } from "kysely";
 import { VirtualizedGrouping } from "../utils/virtualized-grouping";
-import { groupArray } from "../utils/grouping";
+import { createKeySelector, groupArray } from "../utils/grouping";
 import { toChunks } from "../utils/array";
 import { Sanitizer } from "./sanitizer";
 import {
@@ -450,7 +450,7 @@ export class FilteredSelector<T extends Item> {
           items
         };
       },
-      (items) => groupArray(items as any, options),
+      (items) => groupArray(items as any, createKeySelector(options)),
       () => this.groups(options)
     );
   }
@@ -486,7 +486,7 @@ export class FilteredSelector<T extends Item> {
           .select(fields)
           .$call(this.buildSortExpression(options, true))
           .execute(),
-        options
+        createKeySelector(options)
       ).values()
     );
   }

packages/core/src/utils/__tests__/virtualized-grouping.test.ts

@@ -17,110 +17,108 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { test, vi } from "vitest";
+import { test } from "vitest";
 import { VirtualizedGrouping } from "../virtualized-grouping";
+import { groupArray } from "../grouping";
 
-function item<T>(value: T) {
-  return { item: value };
+function generateItems(length: number, groupSize: number) {
+  const items: { group: string; id: string }[] = [];
+  const ids: string[] = [];
+  const divider = length / groupSize;
+  for (let i = 0; i < length; ++i) {
+    items.push({ group: `${i % divider}`, id: `${i}` });
+    ids.push(`${i}`);
+  }
+  items.sort((a, b) => a.group.localeCompare(b.group));
+  return { items, ids };
 }
-function createMock() {
-  return vi.fn(async (ids: string[]) =>
-    Object.fromEntries(ids.map((id) => [id, id]))
+
+function createVirtualizedGrouping(
+  length: number,
+  groupSize: number,
+  batchSize: number
+) {
+  const { ids, items } = generateItems(length, groupSize);
+  return new VirtualizedGrouping<{ group: string; id: string }>(
+    items.length,
+    batchSize,
+    () => Promise.resolve(ids),
+    async (start, end) => ({
+      ids: ids.slice(start, end),
+      items: items.slice(start, end)
+    }),
+    (items) => groupArray(items, (item) => item.group)
   );
 }
-test.todo("renable virtualized grouping tests");
-// test("fetch items in batch if not found in cache", async (t) => {
-//   const mocked = createMock();
-//   const grouping = new VirtualizedGrouping<string>(
-//     ["1", "2", "3", "4", "5", "6", "7"],
-//     3,
-//     mocked
-//   );
-//   t.expect(await grouping.item("4")).toStrictEqual(item("4"));
-//   t.expect(mocked).toHaveBeenCalledOnce();
-// });
 
-// test("do not fetch items in batch if found in cache", async (t) => {
-//   const mocked = createMock();
-//   const grouping = new VirtualizedGrouping<string>(
-//     ["1", "2", "3", "4", "5", "6", "7"],
-//     3,
-//     mocked
-//   );
-//   t.expect(await grouping.item("4")).toStrictEqual(item("4"));
-//   t.expect(await grouping.item("4")).toStrictEqual(item("4"));
-//   t.expect(await grouping.item("4")).toStrictEqual(item("4"));
-//   t.expect(await grouping.item("4")).toStrictEqual(item("4"));
-//   t.expect(await grouping.item("4")).toStrictEqual(item("4"));
-//   t.expect(mocked).toHaveBeenCalledOnce();
-// });
+test("load first batch with a single group", async (t) => {
+  const grouping = createVirtualizedGrouping(100, 10, 10);
 
-// test("clear old cached batches", async (t) => {
-//   const mocked = createMock();
-//   const grouping = new VirtualizedGrouping<string>(
-//     ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
-//     3,
-//     mocked
-//   );
-//   t.expect(await grouping.item("1")).toStrictEqual(item("1"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
-//   t.expect(await grouping.item("4")).toStrictEqual(item("4"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["4", "5", "6"]);
-//   t.expect(await grouping.item("7")).toStrictEqual(item("7"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
-//   t.expect(await grouping.item("1")).toStrictEqual(item("1"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
-// });
+  t.expect((await grouping.item(0)).group?.title).toBe("0");
+  for (let i = 1; i < 10; ++i)
+    t.expect(grouping.cacheItem(i)?.group?.title).toBeUndefined();
+});
 
-// test("clear old cached batches (random access)", async (t) => {
-//   const mocked = createMock();
-//   const grouping = new VirtualizedGrouping<string>(
-//     ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
-//     3,
-//     mocked
-//   );
-//   t.expect(await grouping.item("1")).toStrictEqual(item("1"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+test("load first batch with a multiple groups", async (t) => {
+  const grouping = createVirtualizedGrouping(100, 2, 10);
 
-//   t.expect(await grouping.item("7")).toStrictEqual(item("7"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
+  t.expect((await grouping.item(0)).group?.title).toBe(`0`);
+  t.expect(grouping.cacheItem(2)?.group?.title).toBe(`1`);
+  t.expect(grouping.cacheItem(4)?.group?.title).toBe(`10`);
+  t.expect(grouping.cacheItem(6)?.group?.title).toBe(`11`);
+  t.expect(grouping.cacheItem(8)?.group?.title).toBe(`12`);
+});
 
-//   t.expect(await grouping.item("11")).toStrictEqual(item("11"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["10", "11", "12"]);
+test("load last batch with a single group", async (t) => {
+  const grouping = createVirtualizedGrouping(100, 10, 10);
 
-//   t.expect(await grouping.item("1")).toStrictEqual(item("1"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+  t.expect((await grouping.item(90)).group?.title).toBe("9");
+  for (let i = 91; i < 100; ++i)
+    t.expect(grouping.cacheItem(i)?.group?.title).toBeUndefined();
+});
 
-//   t.expect(await grouping.item("7")).toStrictEqual(item("7"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
-// });
+test("load last batch with a multiple groups", async (t) => {
+  const grouping = createVirtualizedGrouping(100, 2, 10);
 
-// test("reloading ids should clear all cached batches", async (t) => {
-//   const mocked = createMock();
-//   const grouping = new VirtualizedGrouping<string>(
-//     ["1", "3", "4", "5", "7", "6", "50"],
-//     3,
-//     mocked
-//   );
+  t.expect((await grouping.item(90)).group?.title).toBe(`5`);
+  t.expect(grouping.cacheItem(92)?.group?.title).toBe(`6`);
+  t.expect(grouping.cacheItem(94)?.group?.title).toBe(`7`);
+  t.expect(grouping.cacheItem(96)?.group?.title).toBe(`8`);
+  t.expect(grouping.cacheItem(98)?.group?.title).toBe(`9`);
+});
 
-//   t.expect(await grouping.item("1")).toStrictEqual(item("1"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["1", "3", "4"]);
+test("group spanning multiple batches (down)", async (t) => {
+  const grouping = createVirtualizedGrouping(140, 14, 10);
 
-//   grouping.refresh([
-//     "1",
-//     "2",
-//     "3",
-//     "4",
-//     "5",
-//     "6",
-//     "7",
-//     "8",
-//     "9",
-//     "10",
-//     "11",
-//     "12"
-//   ]);
+  t.expect((await grouping.item(0)).group?.title).toBe(`0`);
+  t.expect((await grouping.item(12)).group).toBeUndefined();
+  t.expect((await grouping.item(14)).group?.title).toBe("1");
+  t.expect((await grouping.item(24)).group).toBeUndefined();
+  t.expect((await grouping.item(28)).group?.title).toBe("2");
+});
 
-//   t.expect(await grouping.item("1")).toStrictEqual(item("1"));
-//   t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
-// });
+test("single group in all batches", async (t) => {
+  const grouping = createVirtualizedGrouping(100, 100, 10);
+
+  t.expect((await grouping.item(0)).group?.title).toBe(`0`);
+  for (let i = 1; i < 100; ++i) {
+    t.expect((await grouping.item(i)).group).toBeUndefined();
+  }
+});
+
+test("group at start of each batch", async (t) => {
+  const grouping = createVirtualizedGrouping(100, 10, 10);
+
+  for (let i = 0; i < 100; i += 10) {
+    t.expect((await grouping.item(i)).group?.title).toBe(`${i / 10}`);
+  }
+});
+
+test("group spanning multiple batches (up)", async (t) => {
+  const grouping = createVirtualizedGrouping(140, 28, 10);
+
+  t.expect((await grouping.item(130)).group).toBeUndefined();
+  t.expect((await grouping.item(120)).group).toBeUndefined();
+  t.expect((await grouping.item(140 - 28)).group).toBeDefined();
+  t.expect((await grouping.item(110)).group).toBeUndefined();
+});

packages/core/src/utils/grouping.ts

@@ -30,7 +30,7 @@ type PartialGroupableItem = {
   dateEdited?: number | null;
   dateCreated?: number | null;
 };
-type EvaluateKeyFunction<T> = (item: T) => string;
+export type GroupKeySelectorFunction<T> = (item: T) => string;
 
 export const getSortValue = (
   options: GroupOptions | undefined,
@@ -72,9 +72,13 @@ export function getSortSelectors<T extends PartialGroupableItem>(
 const MILLISECONDS_IN_DAY = 1000 * 60 * 60 * 24;
 const MILLISECONDS_IN_WEEK = MILLISECONDS_IN_DAY * 7;
 
-function getKeySelector(
-  options: GroupOptions
-): EvaluateKeyFunction<PartialGroupableItem> {
+export function createKeySelector(
+  options: GroupOptions = {
+    groupBy: "default",
+    sortBy: "dateEdited",
+    sortDirection: "desc"
+  }
+): GroupKeySelectorFunction<PartialGroupableItem> {
   return (item) => {
     if ("pinned" in item && item.pinned) return "Pinned";
     else if ("conflicted" in item && item.conflicted) return "Conflicted";
@@ -110,20 +114,15 @@ function getKeySelector(
   };
 }
 
-export function groupArray(
-  items: PartialGroupableItem[],
-  options: GroupOptions = {
-    groupBy: "default",
-    sortBy: "dateEdited",
-    sortDirection: "desc"
-  }
+export function groupArray<T>(
+  items: T[],
+  keySelector: GroupKeySelectorFunction<T>
 ): Map<number, { index: number; group: GroupHeader }> {
   const groups = new Map<
     string,
     [number, { index: number; group: GroupHeader }]
   >();
 
-  const keySelector = getKeySelector(options);
   for (let i = 0; i < items.length; ++i) {
     const item = items[i];
     const groupTitle = keySelector(item);

packages/core/src/utils/virtualized-grouping.ts

@@ -123,6 +123,9 @@ export class VirtualizedGrouping<T> {
     const { ids, items } = await this.fetchItems(start, end);
     const groups = this.groupItems?.(items);
 
+    if (items.length > this.batchSize)
+      throw new Error("Got more items than the batch size.");
+
     if (direction === "down") {
       const [, firstGroup] = groups ? firstInMap(groups) : [];
       const group = lastBatch?.groups