core: add support for incremental grouping

2025-12-16 11:47:54 +01:00 · 2023-10-12 09:27:27 +05:00
parent 37ad3ea31a
commit 96e36a6f73
9 changed files with 596 additions and 171 deletions
--- a/packages/core/tests/notes.test.ts
+++ b/packages/core/tests/notes.test.ts
@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

+import dayjs from "dayjs";
 import Database from "../src/api";
 import { groupArray } from "../src/utils/grouping";
 import {
@@ -29,6 +30,7 @@ import {
  loginFakeUser
 } from "./utils";
 import { test, expect } from "vitest";
+import { MONTHS_FULL } from "../src/utils/date";

 async function createAndAddNoteToNotebook(
  db: Database,
@@ -192,16 +194,6 @@ test("get pinned notes", () =>
    expect(await db.notes.pinned.count()).toBeGreaterThan(0);
  }));

-test.todo("get grouped notes by abc", () => groupedTest("abc"));
-
-test.todo("get grouped notes by month", () => groupedTest("month"));
-
-test.todo("get grouped notes by year", () => groupedTest("year"));
-
-test.todo("get grouped notes by weak", () => groupedTest("week"));
-
-test.todo("get grouped notes default", () => groupedTest("default"));
-
 test("pin note", () =>
  noteTest().then(async ({ db, id }) => {
    await db.notes.pin(true, id);
@@ -388,3 +380,151 @@ test("adding a note with an invalid tag should clean the tag array", () =>
      await db.relations.to({ id: "helloworld", type: "note" }, "tag").count()
    ).toBe(0);
  }));
+
+test("get grouped notes by abc", () =>
+  databaseTest().then(async (db) => {
+    const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+    for (const letter of alphabet) {
+      for (const letter2 of alphabet) {
+        await db.notes.add({ title: `${letter}${letter2}` });
+      }
+    }
+    await db.notes.add({ title: `Pinned note`, pinned: true });
+    await db.notes.add({
+      title: `Conflicted`,
+      conflicted: true
+    });
+    const { grouping, ids } = await db.notes.all.grouped({
+      groupBy: "abc",
+      sortDirection: "asc",
+      sortBy: "title"
+    });
+
+    expect((await grouping.item(ids[0]))?.group?.title).toBe("Conflicted");
+    expect((await grouping.item(ids[1]))?.group?.title).toBe("Pinned");
+    for (let i = 0; i < alphabet.length; ++i) {
+      expect(
+        (await grouping.item(ids[i * alphabet.length + 2]))?.group?.title
+      ).toBe(alphabet[i]);
+    }
+  }));
+
+test("get grouped notes by month", () =>
+  databaseTest().then(async (db) => {
+    for (let month = 0; month <= 11; ++month) {
+      for (let i = 0; i < 5; ++i) {
+        const date = dayjs().month(month).toDate().getTime();
+        await db.notes.add({
+          title: `Note in ${month} - ${i}`,
+          dateCreated: date
+        });
+      }
+    }
+
+    const { grouping, ids } = await db.notes.all.grouped({
+      groupBy: "month",
+      sortDirection: "desc",
+      sortBy: "dateCreated"
+    });
+
+    for (let month = 11; month >= 0; --month) {
+      expect((await grouping.item(ids[(11 - month) * 5]))?.group?.title).toBe(
+        MONTHS_FULL[month]
+      );
+    }
+  }));
+
+test("get grouped notes by year", () =>
+  databaseTest().then(async (db) => {
+    for (let year = 2020; year <= 2025; ++year) {
+      for (let i = 0; i < 5; ++i) {
+        const date = dayjs().year(year).toDate().getTime();
+        await db.notes.add({
+          title: `Note in ${year} - ${i}`,
+          dateCreated: date
+        });
+      }
+    }
+
+    const { grouping, ids } = await db.notes.all.grouped({
+      groupBy: "year",
+      sortDirection: "desc",
+      sortBy: "dateCreated"
+    });
+
+    for (let year = 2020; year <= 2025; ++year) {
+      expect((await grouping.item(ids[(2025 - year) * 5]))?.group?.title).toBe(
+        year.toString()
+      );
+    }
+  }));
+
+test("get grouped notes by week", () =>
+  databaseTest().then(async (db) => {
+    for (let i = 1; i <= 6; ++i) {
+      for (let week = 1; week <= 4; ++week) {
+        const date = dayjs()
+          .month(i)
+          .date(week * 7)
+          .year(2023)
+          .startOf("week")
+          .toDate()
+          .getTime();
+        await db.notes.add({
+          title: `Note in ${week} - ${i}`,
+          dateCreated: date
+        });
+      }
+    }
+
+    const { grouping, ids } = await db.notes.all.grouped({
+      groupBy: "week",
+      sortDirection: "desc",
+      sortBy: "dateCreated"
+    });
+
+    const weeks = [
+      "19 - 25 Jun, 2023",
+      "22 - 28 May, 2023",
+      "17 - 23 Apr, 2023",
+      "20 - 26 Mar, 2023",
+      "20 - 26 Feb, 2023"
+    ];
+    for (let i = 1; i <= 5; ++i) {
+      expect((await grouping.item(ids[i * 4]))?.group?.title).toBe(
+        weeks[i - 1]
+      );
+    }
+  }));
+
+test("get grouped notes default", () =>
+  databaseTest().then(async (db) => {
+    const ranges = {
+      Recent: [0, 7],
+      "Last week": [7, 14],
+      Older: [14, 28]
+    };
+    for (const key in ranges) {
+      const range = ranges[key];
+      for (let i = range[0]; i < range[1]; i++) {
+        const date = dayjs().subtract(i, "days").toDate().getTime();
+
+        await db.notes.add({
+          title: `Note in ${key} - ${i}`,
+          dateCreated: date
+        });
+      }
+    }
+
+    const { grouping, ids } = await db.notes.all.grouped({
+      groupBy: "default",
+      sortDirection: "desc",
+      sortBy: "dateCreated"
+    });
+
+    let i = 0;
+    for (const key in ranges) {
+      expect((await grouping.item(ids[i * 7]))?.group?.title).toBe(key);
+      ++i;
+    }
+  }));
--- a/packages/core/tests/utils.test.js
+++ b/packages/core/tests/utils.test.js
@@ -1,41 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import { groupArray } from "../src/utils/grouping";
-import { test, expect } from "vitest";
-
-test("group alphabetically", () => {
-  const sortedAlphabet = "abcdefghijlmnopqrstuvwxyz"
-    .split("")
-    .map((a) => ({ title: a }));
-  const alphabet = "nopqrstuvwxyzabcdefghijlm"
-    .split("")
-    .map((a) => ({ title: a, item: true }));
-  let ret = groupArray(alphabet, {
-    groupBy: "abc",
-    sortDirection: "asc",
-    sortBy: "title"
-  }).filter((v) => !v.item);
-
-  expect(
-    sortedAlphabet.every((alpha, index) => {
-      return ret[index].title === alpha.title.toUpperCase();
-    })
-  ).toBeTruthy();
-});
--- a/packages/core/incremental-grouping.md
+++ b/packages/core/incremental-grouping.md
@@ -0,0 +1,81 @@
+# Incremental grouping of list items
+
+## Requirements
+
+1. We do not want to bring all the items to the client side as that would take a lot of memory
+2. We want to preserve & customize sorting
+3. We want to avoid creating duplicate groups
+4. Each batch of items should append and add groups as necesary
+
+## Approach
+
+1. Get all the IDs of all the items sorted using the required configuration
+2. Request the first batch of `n` items.
+3. Group the first batch
+4. Bring the second batch and similarly group it.
+5. If the last group of the previous batch & the first group of the new batch match up, we can remove the new group, otherwise keep it.
+
+Example:
+
+Items: A, Aa, B, Ba, Bb, C, Ca, Cb, Cc, Cd
+Batch size: 2
+
+Batch 1: [A, Aa]
+Grouped: [A, [A, Aa]]
+
+Batch 2: [B, Ba]
+Grouped: [B, [B, Ba]]
+
+Merged: [A, [A, Aa], B, [B, Ba]]
+
+Batch 3: [Bb, C]
+Grouped: [B, [Bb], C, [C]]
+
+Merged: [A, [A, Aa], B, [B, Ba, Bb], C, [C]]
+
+This is a simple incremental grouping. However, what we require is windowed grouping.
+
+## Windowed incremental grouping of list items
+
+Windowed grouping means that we only keep `n` number of items in memory at any point in time.
+As the user scrolls up & down, we fetch only the batch that is needed for rendering the items
+in the current window.
+
+Requirements:
+
+1. Only keep N items in memory
+2. Remove the previous window's items as soon as we don't need them
+3. Respect sorting & grouping while doing append/prepend
+
+Approach:
+
+1. Get all the IDs of all the items sorted using the required configuration
+2. Request the first batch of `n` items.
+3. Group the first batch
+4. If user is moving downward:
+   1. When user moves closer to the the end of the first batch, load the next batch & group it accordingly
+   2. Compare last & first groups of both batches
+   3. If last & first groups are the same, remove the first group of the new batch
+   4. Otherwise keep it & do nothing
+   5. If user reaches the end of the second batch, remove the first batch
+5. If user is moving upward:
+   Suppose we have batches 2 & 3 in memory, user is at the end of batch 2 and starts scrolling upwards:
+   1. When user reaches closer to the start of the batch 2, load batch 1 & remove batch 3
+   2. For grouping compare the last & first groups of batch 1 & 2 respectively
+   3. Remove first group of batch 2 if both match, otherwise do nothing
+
+Questions:
+
+1. What if new items are added at random places during sync or something else?
+
+The naive but stable approach would be to refetch everything and recalculate the current batch.
+Another approach would be to diff the new & old set of IDs and determine which batches have
+changed.
+
+We can then reload the batches only if they are currently cached, otherwise do nothing.
+
+2. What will be the responsibility of the core module?
+
+The core module will only manage the cache. It will also be responsible for loading the appropriate
+batch based on if the item is found in the cache or not. This should be automatic for maximum
+optimization opportunities.
--- a/packages/core/src/database/sql-collection.ts
+++ b/packages/core/src/database/sql-collection.ts
@@ -18,7 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

 import { EVENTS } from "../common";
-import { GroupOptions, MaybeDeletedItem, isDeleted } from "../types";
+import { GroupOptions, Item, MaybeDeletedItem, isDeleted } from "../types";
 import EventManager from "../utils/event-manager";
 import {
  DatabaseAccessor,
@@ -28,6 +28,8 @@ import {
  isFalse
 } from ".";
 import { ExpressionOrFactory, SelectQueryBuilder, SqlBool } from "kysely";
+import { VirtualizedGrouping } from "../utils/virtualized-grouping";
+import { groupArray } from "../utils/grouping";

 export class SQLCollection<
  TCollectionType extends keyof DatabaseSchema,
@@ -164,16 +166,12 @@ export class SQLCollection<
  }

  async items(
-    ids: string[],
-    sortOptions?: GroupOptions
+    ids: string[]
  ): Promise<Record<string, MaybeDeletedItem<T> | undefined>> {
    const results = await this.db()
      .selectFrom<keyof DatabaseSchema>(this.type)
      .selectAll()
      .where("id", "in", ids)
-      .$if(!!sortOptions, (eb) =>
-        eb.orderBy(sortOptions!.sortBy, sortOptions!.sortDirection)
-      )
      .execute();
    const items: Record<string, MaybeDeletedItem<T>> = {};
    for (const item of results) {
@@ -227,7 +225,7 @@ export class SQLCollection<
    }
  }

-  createFilter<T>(
+  createFilter<T extends Item>(
    selector: (
      qb: SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, unknown>
    ) => SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, unknown>,
@@ -240,7 +238,7 @@ export class SQLCollection<
  }
 }

-export class FilteredSelector<T> {
+export class FilteredSelector<T extends Item> {
  constructor(
    readonly filter: SelectQueryBuilder<
      DatabaseSchema,
@@ -250,13 +248,23 @@ export class FilteredSelector<T> {
    readonly batchSize: number
  ) {}

-  async ids() {
-    return (await this.filter.select("id").execute()).map((i) => i.id);
+  async ids(sortOptions?: GroupOptions) {
+    return (
+      await this.filter
+        .$if(!!sortOptions, (eb) =>
+          eb.$call(this.buildSortExpression(sortOptions!))
+        )
+        .select("id")
+        .execute()
+    ).map((i) => i.id);
  }

-  async items(ids?: string[]) {
+  async items(ids?: string[], sortOptions?: GroupOptions) {
    return (await this.filter
      .$if(!!ids && ids.length > 0, (eb) => eb.where("id", "in", ids!))
+      .$if(!!sortOptions, (eb) =>
+        eb.$call(this.buildSortExpression(sortOptions!))
+      )
      .selectAll()
      .execute()) as T[];
  }
@@ -298,6 +306,40 @@ export class FilteredSelector<T> {
    }
  }

+  async grouped(options: GroupOptions) {
+    const ids = await this.ids(options);
+    return {
+      ids,
+      grouping: new VirtualizedGrouping<T>(
+        ids,
+        this.batchSize,
+        async (ids) => {
+          const results = await this.filter
+            .where("id", "in", ids)
+            .selectAll()
+            .execute();
+          const items: Record<string, T> = {};
+          for (const item of results) {
+            items[item.id] = item as T;
+          }
+          return items;
+        },
+        (ids, items) => groupArray(ids, items, options)
+      )
+    };
+  }
+
+  private buildSortExpression(options: GroupOptions) {
+    return <T>(
+      qb: SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, T>
+    ) => {
+      return qb
+        .orderBy("conflicted desc")
+        .orderBy("pinned desc")
+        .orderBy(options.sortBy, options.sortDirection);
+    };
+  }
+
  async *[Symbol.asyncIterator]() {
    let index = 0;
    while (true) {
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -81,6 +81,7 @@ export type GroupableItem = ValueOf<
    | "shortcut"
    | "relation"
    | "tiny"
+    | "topic"
    | "tiptap"
    | "content"
    | "session"
--- a/packages/core/src/utils/tests/virtualized-grouping.test.ts
+++ b/packages/core/src/utils/tests/virtualized-grouping.test.ts
@@ -0,0 +1,162 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { expect, test, vi } from "vitest";
+import { VirtualizedGrouping } from "../virtualized-grouping";
+
+function item<T>(value: T) {
+  return { item: value };
+}
+function createMock() {
+  return vi.fn(async (ids: string[]) =>
+    Object.fromEntries(ids.map((id) => [id, id]))
+  );
+}
+test("fetch items in batch if not found in cache", async (t) => {
+  const mocked = createMock();
+  const grouping = new VirtualizedGrouping<string>(
+    ["1", "2", "3", "4", "5", "6", "7"],
+    3,
+    mocked
+  );
+  t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+  t.expect(mocked).toHaveBeenCalledOnce();
+});
+
+test("do not fetch items in batch if found in cache", async (t) => {
+  const mocked = createMock();
+  const grouping = new VirtualizedGrouping<string>(
+    ["1", "2", "3", "4", "5", "6", "7"],
+    3,
+    mocked
+  );
+  t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+  t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+  t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+  t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+  t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+  t.expect(mocked).toHaveBeenCalledOnce();
+});
+
+test("clear old cached batches", async (t) => {
+  const mocked = createMock();
+  const grouping = new VirtualizedGrouping<string>(
+    ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+    3,
+    mocked
+  );
+  t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+  t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["4", "5", "6"]);
+  t.expect(await grouping.item("7")).toStrictEqual(item("7"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
+  t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+});
+
+test("clear old cached batches (random access)", async (t) => {
+  const mocked = createMock();
+  const grouping = new VirtualizedGrouping<string>(
+    ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+    3,
+    mocked
+  );
+  t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+
+  t.expect(await grouping.item("7")).toStrictEqual(item("7"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
+
+  t.expect(await grouping.item("11")).toStrictEqual(item("11"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["10", "11", "12"]);
+
+  t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+
+  t.expect(await grouping.item("7")).toStrictEqual(item("7"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
+});
+
+test("reloading ids should clear all cached batches", async (t) => {
+  const mocked = createMock();
+  const grouping = new VirtualizedGrouping<string>(
+    ["1", "3", "4", "5", "7", "6", "50"],
+    3,
+    mocked
+  );
+
+  t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["1", "3", "4"]);
+
+  grouping.refresh([
+    "1",
+    "2",
+    "3",
+    "4",
+    "5",
+    "6",
+    "7",
+    "8",
+    "9",
+    "10",
+    "11",
+    "12"
+  ]);
+
+  t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+  t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+});
+
+test("merge groups if last & first groups are the same (sequential)", async (t) => {
+  const mocked = createMock();
+  const grouping = new VirtualizedGrouping<string>(
+    ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+    3,
+    mocked,
+    (ids) => [{ title: "Hello", id: ids[0] }]
+  );
+  expect((await grouping.item("1"))?.group?.title).toBe("Hello");
+  expect((await grouping.item("4"))?.group).toBeUndefined();
+});
+
+test("merge groups if last & first groups are the same (random)", async (t) => {
+  const mocked = createMock();
+  const grouping = new VirtualizedGrouping<string>(
+    ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+    3,
+    mocked,
+    (ids) => [{ title: "Hello", id: ids[0] }]
+  );
+  expect((await grouping.item("1"))?.group?.title).toBe("Hello");
+  expect((await grouping.item("7"))?.group).toBeUndefined();
+});
+
+test("merge groups if last & first groups are the same (reverse)", async (t) => {
+  const mocked = createMock();
+  const grouping = new VirtualizedGrouping<string>(
+    ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+    3,
+    mocked,
+    (ids) => [{ title: "Hello", id: ids[0] }]
+  );
+  expect((await grouping.item("7"))?.group?.title).toBe("Hello");
+  expect((await grouping.item("1"))?.group?.title).toBe("Hello");
+  expect((await grouping.item("7"))?.group).toBeUndefined();
+});
--- a/packages/core/src/utils/grouping.ts
+++ b/packages/core/src/utils/grouping.ts
@@ -18,14 +18,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

 import { isReminderActive } from "../collections/reminders";
-import {
-  GroupedItems,
-  GroupOptions,
-  GroupableItem,
-  Item,
-  Reminder
-} from "../types";
+import { GroupOptions, Item } from "../types";
 import { getWeekGroupFromTimestamp, MONTHS_FULL } from "./date";
+import { VirtualizedGroupHeader } from "./virtualized-grouping";

 type EvaluateKeyFunction<T> = (item: T) => string;

@@ -45,33 +40,19 @@ export const getSortValue = <T extends Item>(
  return item.dateCreated;
 };

-function getSortSelectors<T extends GroupableItem>(options: GroupOptions) {
-  if (options.sortBy === "title")
-    return {
-      asc: (a: T, b: T) =>
-        getTitle(a).localeCompare(getTitle(b), undefined, { numeric: true }),
-      desc: (a: T, b: T) =>
-        getTitle(b).localeCompare(getTitle(a), undefined, { numeric: true })
-    };
-
-  return {
-    asc: (a: T, b: T) => getSortValue(options, a) - getSortValue(options, b),
-    desc: (a: T, b: T) => getSortValue(options, b) - getSortValue(options, a)
-  };
-}
-
 const MILLISECONDS_IN_DAY = 1000 * 60 * 60 * 24;
 const MILLISECONDS_IN_WEEK = MILLISECONDS_IN_DAY * 7;

-function getKeySelector<T extends GroupableItem>(
-  options: GroupOptions
-): EvaluateKeyFunction<T> {
-  return (item: T) => {
+function getKeySelector(options: GroupOptions): EvaluateKeyFunction<Item> {
+  return (item: Item) => {
    if ("pinned" in item && item.pinned) return "Pinned";
    else if ("conflicted" in item && item.conflicted) return "Conflicted";

    const date = new Date();
-    if (options.sortBy === "title") return getFirstCharacter(getTitle(item));
+    if (item.type === "reminder")
+      return isReminderActive(item) ? "Active" : "Inactive";
+    else if (options.sortBy === "title")
+      return getFirstCharacter(getTitle(item));
    else {
      const value = getSortValue(options, item);
      switch (options.groupBy) {
@@ -98,78 +79,37 @@ function getKeySelector<T extends GroupableItem>(
  };
 }

-export function groupArray<T extends GroupableItem>(
-  array: T[],
+export function groupArray(
+  ids: string[],
+  items: Record<string, Item>,
  options: GroupOptions = {
    groupBy: "default",
    sortBy: "dateEdited",
    sortDirection: "desc"
  }
-): GroupedItems<T> {
-  if (options.sortBy && options.sortDirection) {
-    const selector = getSortSelectors(options)[options.sortDirection];
-    array.sort(selector);
-  }
-
-  if (options.groupBy === "none") {
-    const conflicted: T[] = [];
-    const pinned: T[] = [];
-    const others: T[] = [];
-    for (const item of array) {
-      if ("pinned" in item && item.pinned) {
-        pinned.push(item);
-        continue;
-      } else if ("conflicted" in item && item.conflicted) {
-        conflicted.push(item);
-        continue;
-      } else others.push(item);
-    }
-    const groups: GroupedItems<T> = [];
-    if (conflicted.length > 0)
-      groups.push(
-        { title: "Conflicted", type: "header", id: "conflicted" },
-        ...conflicted
-      );
-    if (pinned.length > 0)
-      groups.push({ title: "Pinned", type: "header", id: "pinned" }, ...pinned);
-    if (others.length > 0)
-      groups.push({ title: "All", type: "header", id: "all" }, ...others);
-    return groups;
-  }
-
-  const groups = new Map<string, T[]>([
-    ["Conflicted", []],
-    ["Pinned", []]
+): VirtualizedGroupHeader[] {
+  const groups = new Map<string, VirtualizedGroupHeader>([
+    ["Conflicted", { title: "Conflicted", id: "" }],
+    ["Pinned", { title: "Pinned", id: "" }],
+    ["Active", { title: "Active", id: "" }],
+    ["Inactive", { title: "Inactive", id: "" }]
  ]);

  const keySelector = getKeySelector(options);
-  array.forEach((item) => addToGroup(groups, keySelector(item), item));
+  for (const id of ids) {
+    const item = items[id];
+    if (!item) continue;

-  return flattenGroups(groups);
-}
+    const groupTitle = keySelector(item);
+    const group = groups.get(groupTitle) || {
+      title: groupTitle,
+      id: ""
+    };
+    if (group.id === "") group.id = id;
+    groups.set(groupTitle, group);
+  }

-export function groupReminders(array: Reminder[]): GroupedItems<Reminder> {
-  const groups = new Map([
-    ["Active", []],
-    ["Inactive", []]
-  ]);
-
-  array.forEach((item) => {
-    const groupTitle = isReminderActive(item) ? "Active" : "Inactive";
-    addToGroup(groups, groupTitle, item);
-  });
-
-  return flattenGroups(groups);
-}
-
-function addToGroup<T extends GroupableItem>(
-  groups: Map<string, T[]>,
-  groupTitle: string,
-  item: T
-) {
-  const group = groups.get(groupTitle) || [];
-  group.push(item);
-  groups.set(groupTitle, group);
+  return Array.from(groups.values());
 }

 function getFirstCharacter(str: string) {
@@ -179,21 +119,10 @@ function getFirstCharacter(str: string) {
  return str[0].toUpperCase();
 }

-function getTitle<T extends GroupableItem>(item: T): string {
-  return item.type === "attachment" ? item.metadata.filename : item.title;
-}
-
-function flattenGroups<T extends GroupableItem>(groups: Map<string, T[]>) {
-  const items: GroupedItems<T> = [];
-  groups.forEach((groupItems, groupTitle) => {
-    if (groupItems.length <= 0) return;
-    items.push({
-      title: groupTitle,
-      id: groupTitle.toLowerCase(),
-      type: "header"
-    });
-    groupItems.forEach((item) => items.push(item));
-  });
-
-  return items;
+function getTitle(item: Item): string {
+  return item.type === "attachment"
+    ? item.filename
+    : "title" in item
+    ? item.title
+    : "Unknown";
 }
--- a/packages/core/src/utils/set.ts
+++ b/packages/core/src/utils/set.ts
@@ -85,14 +85,14 @@ class SetManipulator {
  // Symmetric difference. Items from either set that
  // are not in both sets.
  // Set.difference([1, 1, 2], [2, 3, 3]) => [1, 3]
-  difference<T>(a: T[], b: T[], key: KeySelector<T>) {
-    return this.process(a, b, key, (freq) => freq < 3);
+  difference<T>(a: T[], b: T[], key?: KeySelector<T>) {
+    return <T[]>this.process(a, b, key, (freq) => freq < 3);
  }

  // Relative complement. Items from 'a' which are
  // not also in 'b'.
  // Set.complement([1, 2, 2], [2, 2, 3]) => [3]
-  complement<T>(a: T[], b: T[], key: KeySelector<T>) {
+  complement<T>(a: T[], b: T[], key?: KeySelector<T>) {
    return this.process(a, b, key, (freq) => freq === 1);
  }

--- a/packages/core/src/utils/virtualized-grouping.ts
+++ b/packages/core/src/utils/virtualized-grouping.ts
@@ -0,0 +1,111 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+export type VirtualizedGroupHeader = {
+  title: string;
+  id: string;
+};
+export class VirtualizedGrouping<T> {
+  private cache: Map<number, Record<string, T>> = new Map();
+  private groups: Map<number, VirtualizedGroupHeader[]> = new Map();
+
+  constructor(
+    private ids: string[],
+    private readonly batchSize: number,
+    private readonly fetchItems: (ids: string[]) => Promise<Record<string, T>>,
+    private readonly groupItems: (
+      ids: string[],
+      items: Record<string, T>
+    ) => VirtualizedGroupHeader[] = () => []
+  ) {
+    this.ids = ids;
+  }
+
+  /**
+   * Get item from cache or request the appropriate batch for caching
+   * and load it from there.
+   */
+  async item(id: string) {
+    const index = this.ids.indexOf(id);
+    if (index <= -1) return;
+
+    const batchIndex = Math.floor(index / this.batchSize);
+    const batch = this.cache.get(batchIndex) || (await this.load(batchIndex));
+    const groups = this.groups.get(batchIndex);
+
+    const group = groups?.find((g) => g.id === id);
+    if (group)
+      return {
+        group: { type: "header", id: group.title, title: group.title },
+        item: batch[id]
+      };
+    return { item: batch[id] };
+  }
+
+  /**
+   * Reload the cache
+   */
+  refresh(ids: string[]) {
+    this.ids = ids;
+    this.cache.clear();
+  }
+
+  /**
+   *
+   * @param index
+   */
+  private async load(batch: number) {
+    const start = batch * this.batchSize;
+    const end = start + this.batchSize;
+    const batchIds = this.ids.slice(start, end);
+    const items = await this.fetchItems(batchIds);
+    const groups = this.groupItems(batchIds, items);
+
+    const lastBatchIndex = this.last;
+    const prevGroups = this.groups.get(lastBatchIndex);
+    if (prevGroups && prevGroups.length > 0 && groups.length > 0) {
+      const lastGroup = prevGroups[prevGroups.length - 1];
+      if (lastGroup.title === groups[0].title) {
+        // if user is moving downwards, we remove the last group from the
+        // current batch, otherwise we remove the first group from the previous
+        // batch.
+        lastBatchIndex < batch ? groups.pop() : prevGroups.shift();
+      }
+    }
+
+    this.cache.set(batch, items);
+    this.groups.set(batch, groups);
+    this.clear();
+    return items;
+  }
+
+  private clear() {
+    if (this.cache.size <= 2) return;
+    for (const [key] of this.cache) {
+      this.cache.delete(key);
+      this.groups.delete(key);
+      if (this.cache.size === 2) break;
+    }
+  }
+
+  private get last() {
+    const keys = Array.from(this.cache.keys());
+    return keys[keys.length - 1];
+  }
+}