diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.ts
index 66d8fb27c..85755b5ab 100644
--- a/packages/core/__tests__/notes.test.ts
+++ b/packages/core/__tests__/notes.test.ts
@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
+import dayjs from "dayjs";
import Database from "../src/api";
import { groupArray } from "../src/utils/grouping";
import {
@@ -29,6 +30,7 @@ import {
loginFakeUser
} from "./utils";
import { test, expect } from "vitest";
+import { MONTHS_FULL } from "../src/utils/date";
async function createAndAddNoteToNotebook(
db: Database,
@@ -192,16 +194,6 @@ test("get pinned notes", () =>
expect(await db.notes.pinned.count()).toBeGreaterThan(0);
}));
-test.todo("get grouped notes by abc", () => groupedTest("abc"));
-
-test.todo("get grouped notes by month", () => groupedTest("month"));
-
-test.todo("get grouped notes by year", () => groupedTest("year"));
-
-test.todo("get grouped notes by weak", () => groupedTest("week"));
-
-test.todo("get grouped notes default", () => groupedTest("default"));
-
test("pin note", () =>
noteTest().then(async ({ db, id }) => {
await db.notes.pin(true, id);
@@ -388,3 +380,151 @@ test("adding a note with an invalid tag should clean the tag array", () =>
await db.relations.to({ id: "helloworld", type: "note" }, "tag").count()
).toBe(0);
}));
+
+test("get grouped notes by abc", () =>
+ databaseTest().then(async (db) => {
+ const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+ for (const letter of alphabet) {
+ for (const letter2 of alphabet) {
+ await db.notes.add({ title: `${letter}${letter2}` });
+ }
+ }
+ await db.notes.add({ title: `Pinned note`, pinned: true });
+ await db.notes.add({
+ title: `Conflicted`,
+ conflicted: true
+ });
+ const { grouping, ids } = await db.notes.all.grouped({
+ groupBy: "abc",
+ sortDirection: "asc",
+ sortBy: "title"
+ });
+
+ expect((await grouping.item(ids[0]))?.group?.title).toBe("Conflicted");
+ expect((await grouping.item(ids[1]))?.group?.title).toBe("Pinned");
+ for (let i = 0; i < alphabet.length; ++i) {
+ expect(
+ (await grouping.item(ids[i * alphabet.length + 2]))?.group?.title
+ ).toBe(alphabet[i]);
+ }
+ }));
+
+test("get grouped notes by month", () =>
+ databaseTest().then(async (db) => {
+ for (let month = 0; month <= 11; ++month) {
+ for (let i = 0; i < 5; ++i) {
+ const date = dayjs().month(month).toDate().getTime();
+ await db.notes.add({
+ title: `Note in ${month} - ${i}`,
+ dateCreated: date
+ });
+ }
+ }
+
+ const { grouping, ids } = await db.notes.all.grouped({
+ groupBy: "month",
+ sortDirection: "desc",
+ sortBy: "dateCreated"
+ });
+
+ for (let month = 11; month >= 0; --month) {
+ expect((await grouping.item(ids[(11 - month) * 5]))?.group?.title).toBe(
+ MONTHS_FULL[month]
+ );
+ }
+ }));
+
+test("get grouped notes by year", () =>
+ databaseTest().then(async (db) => {
+ for (let year = 2020; year <= 2025; ++year) {
+ for (let i = 0; i < 5; ++i) {
+ const date = dayjs().year(year).toDate().getTime();
+ await db.notes.add({
+ title: `Note in ${year} - ${i}`,
+ dateCreated: date
+ });
+ }
+ }
+
+ const { grouping, ids } = await db.notes.all.grouped({
+ groupBy: "year",
+ sortDirection: "desc",
+ sortBy: "dateCreated"
+ });
+
+ for (let year = 2020; year <= 2025; ++year) {
+ expect((await grouping.item(ids[(2025 - year) * 5]))?.group?.title).toBe(
+ year.toString()
+ );
+ }
+ }));
+
+test("get grouped notes by week", () =>
+ databaseTest().then(async (db) => {
+ for (let i = 1; i <= 6; ++i) {
+ for (let week = 1; week <= 4; ++week) {
+ const date = dayjs()
+ .month(i)
+ .date(week * 7)
+ .year(2023)
+ .startOf("week")
+ .toDate()
+ .getTime();
+ await db.notes.add({
+ title: `Note in ${week} - ${i}`,
+ dateCreated: date
+ });
+ }
+ }
+
+ const { grouping, ids } = await db.notes.all.grouped({
+ groupBy: "week",
+ sortDirection: "desc",
+ sortBy: "dateCreated"
+ });
+
+ const weeks = [
+ "19 - 25 Jun, 2023",
+ "22 - 28 May, 2023",
+ "17 - 23 Apr, 2023",
+ "20 - 26 Mar, 2023",
+ "20 - 26 Feb, 2023"
+ ];
+ for (let i = 1; i <= 5; ++i) {
+ expect((await grouping.item(ids[i * 4]))?.group?.title).toBe(
+ weeks[i - 1]
+ );
+ }
+ }));
+
+test("get grouped notes default", () =>
+ databaseTest().then(async (db) => {
+ const ranges = {
+ Recent: [0, 7],
+ "Last week": [7, 14],
+ Older: [14, 28]
+ };
+ for (const key in ranges) {
+ const range = ranges[key];
+ for (let i = range[0]; i < range[1]; i++) {
+ const date = dayjs().subtract(i, "days").toDate().getTime();
+
+ await db.notes.add({
+ title: `Note in ${key} - ${i}`,
+ dateCreated: date
+ });
+ }
+ }
+
+ const { grouping, ids } = await db.notes.all.grouped({
+ groupBy: "default",
+ sortDirection: "desc",
+ sortBy: "dateCreated"
+ });
+
+ let i = 0;
+ for (const key in ranges) {
+ expect((await grouping.item(ids[i * 7]))?.group?.title).toBe(key);
+ ++i;
+ }
+ }));
diff --git a/packages/core/__tests__/utils.test.js b/packages/core/__tests__/utils.test.js
deleted file mode 100644
index 7b100e7d9..000000000
--- a/packages/core/__tests__/utils.test.js
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see .
-*/
-
-import { groupArray } from "../src/utils/grouping";
-import { test, expect } from "vitest";
-
-test("group alphabetically", () => {
- const sortedAlphabet = "abcdefghijlmnopqrstuvwxyz"
- .split("")
- .map((a) => ({ title: a }));
- const alphabet = "nopqrstuvwxyzabcdefghijlm"
- .split("")
- .map((a) => ({ title: a, item: true }));
- let ret = groupArray(alphabet, {
- groupBy: "abc",
- sortDirection: "asc",
- sortBy: "title"
- }).filter((v) => !v.item);
-
- expect(
- sortedAlphabet.every((alpha, index) => {
- return ret[index].title === alpha.title.toUpperCase();
- })
- ).toBeTruthy();
-});
diff --git a/packages/core/incremental-grouping.md b/packages/core/incremental-grouping.md
new file mode 100644
index 000000000..d2df4cdc7
--- /dev/null
+++ b/packages/core/incremental-grouping.md
@@ -0,0 +1,81 @@
+# Incremental grouping of list items
+
+## Requirements
+
+1. We do not want to bring all the items to the client side as that would take a lot of memory
+2. We want to preserve & customize sorting
+3. We want to avoid creating duplicate groups
+4. Each batch of items should append and add groups as necesary
+
+## Approach
+
+1. Get all the IDs of all the items sorted using the required configuration
+2. Request the first batch of `n` items.
+3. Group the first batch
+4. Bring the second batch and similarly group it.
+5. If the last group of the previous batch & the first group of the new batch match up, we can remove the new group, otherwise keep it.
+
+Example:
+
+Items: A, Aa, B, Ba, Bb, C, Ca, Cb, Cc, Cd
+Batch size: 2
+
+Batch 1: [A, Aa]
+Grouped: [A, [A, Aa]]
+
+Batch 2: [B, Ba]
+Grouped: [B, [B, Ba]]
+
+Merged: [A, [A, Aa], B, [B, Ba]]
+
+Batch 3: [Bb, C]
+Grouped: [B, [Bb], C, [C]]
+
+Merged: [A, [A, Aa], B, [B, Ba, Bb], C, [C]]
+
+This is a simple incremental grouping. However, what we require is windowed grouping.
+
+## Windowed incremental grouping of list items
+
+Windowed grouping means that we only keep `n` number of items in memory at any point in time.
+As the user scrolls up & down, we fetch only the batch that is needed for rendering the items
+in the current window.
+
+Requirements:
+
+1. Only keep N items in memory
+2. Remove the previous window's items as soon as we don't need them
+3. Respect sorting & grouping while doing append/prepend
+
+Approach:
+
+1. Get all the IDs of all the items sorted using the required configuration
+2. Request the first batch of `n` items.
+3. Group the first batch
+4. If user is moving downward:
+ 1. When user moves closer to the the end of the first batch, load the next batch & group it accordingly
+ 2. Compare last & first groups of both batches
+ 3. If last & first groups are the same, remove the first group of the new batch
+ 4. Otherwise keep it & do nothing
+ 5. If user reaches the end of the second batch, remove the first batch
+5. If user is moving upward:
+ Suppose we have batches 2 & 3 in memory, user is at the end of batch 2 and starts scrolling upwards:
+ 1. When user reaches closer to the start of the batch 2, load batch 1 & remove batch 3
+ 2. For grouping compare the last & first groups of batch 1 & 2 respectively
+ 3. Remove first group of batch 2 if both match, otherwise do nothing
+
+Questions:
+
+1. What if new items are added at random places during sync or something else?
+
+The naive but stable approach would be to refetch everything and recalculate the current batch.
+Another approach would be to diff the new & old set of IDs and determine which batches have
+changed.
+
+We can then reload the batches only if they are currently cached, otherwise do nothing.
+
+2. What will be the responsibility of the core module?
+
+The core module will only manage the cache. It will also be responsible for loading the appropriate
+batch based on if the item is found in the cache or not. This should be automatic for maximum
+optimization opportunities.
diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.ts
index 48bd481c3..b3be4baab 100644
--- a/packages/core/src/database/sql-collection.ts
+++ b/packages/core/src/database/sql-collection.ts
@@ -18,7 +18,7 @@ along with this program. If not, see .
*/
import { EVENTS } from "../common";
-import { GroupOptions, MaybeDeletedItem, isDeleted } from "../types";
+import { GroupOptions, Item, MaybeDeletedItem, isDeleted } from "../types";
import EventManager from "../utils/event-manager";
import {
DatabaseAccessor,
@@ -28,6 +28,8 @@ import {
isFalse
} from ".";
import { ExpressionOrFactory, SelectQueryBuilder, SqlBool } from "kysely";
+import { VirtualizedGrouping } from "../utils/virtualized-grouping";
+import { groupArray } from "../utils/grouping";
export class SQLCollection<
TCollectionType extends keyof DatabaseSchema,
@@ -164,16 +166,12 @@ export class SQLCollection<
}
async items(
- ids: string[],
- sortOptions?: GroupOptions
+ ids: string[]
): Promise | undefined>> {
const results = await this.db()
.selectFrom(this.type)
.selectAll()
.where("id", "in", ids)
- .$if(!!sortOptions, (eb) =>
- eb.orderBy(sortOptions!.sortBy, sortOptions!.sortDirection)
- )
.execute();
const items: Record> = {};
for (const item of results) {
@@ -227,7 +225,7 @@ export class SQLCollection<
}
}
- createFilter(
+ createFilter(
selector: (
qb: SelectQueryBuilder
) => SelectQueryBuilder,
@@ -240,7 +238,7 @@ export class SQLCollection<
}
}
-export class FilteredSelector {
+export class FilteredSelector {
constructor(
readonly filter: SelectQueryBuilder<
DatabaseSchema,
@@ -250,13 +248,23 @@ export class FilteredSelector {
readonly batchSize: number
) {}
- async ids() {
- return (await this.filter.select("id").execute()).map((i) => i.id);
+ async ids(sortOptions?: GroupOptions) {
+ return (
+ await this.filter
+ .$if(!!sortOptions, (eb) =>
+ eb.$call(this.buildSortExpression(sortOptions!))
+ )
+ .select("id")
+ .execute()
+ ).map((i) => i.id);
}
- async items(ids?: string[]) {
+ async items(ids?: string[], sortOptions?: GroupOptions) {
return (await this.filter
.$if(!!ids && ids.length > 0, (eb) => eb.where("id", "in", ids!))
+ .$if(!!sortOptions, (eb) =>
+ eb.$call(this.buildSortExpression(sortOptions!))
+ )
.selectAll()
.execute()) as T[];
}
@@ -298,6 +306,40 @@ export class FilteredSelector {
}
}
+ async grouped(options: GroupOptions) {
+ const ids = await this.ids(options);
+ return {
+ ids,
+ grouping: new VirtualizedGrouping(
+ ids,
+ this.batchSize,
+ async (ids) => {
+ const results = await this.filter
+ .where("id", "in", ids)
+ .selectAll()
+ .execute();
+ const items: Record = {};
+ for (const item of results) {
+ items[item.id] = item as T;
+ }
+ return items;
+ },
+ (ids, items) => groupArray(ids, items, options)
+ )
+ };
+ }
+
+ private buildSortExpression(options: GroupOptions) {
+ return (
+ qb: SelectQueryBuilder
+ ) => {
+ return qb
+ .orderBy("conflicted desc")
+ .orderBy("pinned desc")
+ .orderBy(options.sortBy, options.sortDirection);
+ };
+ }
+
async *[Symbol.asyncIterator]() {
let index = 0;
while (true) {
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
index 80792b0bc..f0d13021b 100644
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -81,6 +81,7 @@ export type GroupableItem = ValueOf<
| "shortcut"
| "relation"
| "tiny"
+ | "topic"
| "tiptap"
| "content"
| "session"
diff --git a/packages/core/src/utils/__tests__/virtualized-grouping.test.ts b/packages/core/src/utils/__tests__/virtualized-grouping.test.ts
new file mode 100644
index 000000000..e85e77d6d
--- /dev/null
+++ b/packages/core/src/utils/__tests__/virtualized-grouping.test.ts
@@ -0,0 +1,162 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+*/
+
+import { expect, test, vi } from "vitest";
+import { VirtualizedGrouping } from "../virtualized-grouping";
+
+function item(value: T) {
+ return { item: value };
+}
+function createMock() {
+ return vi.fn(async (ids: string[]) =>
+ Object.fromEntries(ids.map((id) => [id, id]))
+ );
+}
+test("fetch items in batch if not found in cache", async (t) => {
+ const mocked = createMock();
+ const grouping = new VirtualizedGrouping(
+ ["1", "2", "3", "4", "5", "6", "7"],
+ 3,
+ mocked
+ );
+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+ t.expect(mocked).toHaveBeenCalledOnce();
+});
+
+test("do not fetch items in batch if found in cache", async (t) => {
+ const mocked = createMock();
+ const grouping = new VirtualizedGrouping(
+ ["1", "2", "3", "4", "5", "6", "7"],
+ 3,
+ mocked
+ );
+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+ t.expect(mocked).toHaveBeenCalledOnce();
+});
+
+test("clear old cached batches", async (t) => {
+ const mocked = createMock();
+ const grouping = new VirtualizedGrouping(
+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+ 3,
+ mocked
+ );
+ t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+ t.expect(await grouping.item("4")).toStrictEqual(item("4"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["4", "5", "6"]);
+ t.expect(await grouping.item("7")).toStrictEqual(item("7"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
+ t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+});
+
+test("clear old cached batches (random access)", async (t) => {
+ const mocked = createMock();
+ const grouping = new VirtualizedGrouping(
+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+ 3,
+ mocked
+ );
+ t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+
+ t.expect(await grouping.item("7")).toStrictEqual(item("7"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
+
+ t.expect(await grouping.item("11")).toStrictEqual(item("11"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["10", "11", "12"]);
+
+ t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+
+ t.expect(await grouping.item("7")).toStrictEqual(item("7"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
+});
+
+test("reloading ids should clear all cached batches", async (t) => {
+ const mocked = createMock();
+ const grouping = new VirtualizedGrouping(
+ ["1", "3", "4", "5", "7", "6", "50"],
+ 3,
+ mocked
+ );
+
+ t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "3", "4"]);
+
+ grouping.refresh([
+ "1",
+ "2",
+ "3",
+ "4",
+ "5",
+ "6",
+ "7",
+ "8",
+ "9",
+ "10",
+ "11",
+ "12"
+ ]);
+
+ t.expect(await grouping.item("1")).toStrictEqual(item("1"));
+ t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
+});
+
+test("merge groups if last & first groups are the same (sequential)", async (t) => {
+ const mocked = createMock();
+ const grouping = new VirtualizedGrouping(
+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+ 3,
+ mocked,
+ (ids) => [{ title: "Hello", id: ids[0] }]
+ );
+ expect((await grouping.item("1"))?.group?.title).toBe("Hello");
+ expect((await grouping.item("4"))?.group).toBeUndefined();
+});
+
+test("merge groups if last & first groups are the same (random)", async (t) => {
+ const mocked = createMock();
+ const grouping = new VirtualizedGrouping(
+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+ 3,
+ mocked,
+ (ids) => [{ title: "Hello", id: ids[0] }]
+ );
+ expect((await grouping.item("1"))?.group?.title).toBe("Hello");
+ expect((await grouping.item("7"))?.group).toBeUndefined();
+});
+
+test("merge groups if last & first groups are the same (reverse)", async (t) => {
+ const mocked = createMock();
+ const grouping = new VirtualizedGrouping(
+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
+ 3,
+ mocked,
+ (ids) => [{ title: "Hello", id: ids[0] }]
+ );
+ expect((await grouping.item("7"))?.group?.title).toBe("Hello");
+ expect((await grouping.item("1"))?.group?.title).toBe("Hello");
+ expect((await grouping.item("7"))?.group).toBeUndefined();
+});
diff --git a/packages/core/src/utils/grouping.ts b/packages/core/src/utils/grouping.ts
index 905928ecb..853e811da 100644
--- a/packages/core/src/utils/grouping.ts
+++ b/packages/core/src/utils/grouping.ts
@@ -18,14 +18,9 @@ along with this program. If not, see .
*/
import { isReminderActive } from "../collections/reminders";
-import {
- GroupedItems,
- GroupOptions,
- GroupableItem,
- Item,
- Reminder
-} from "../types";
+import { GroupOptions, Item } from "../types";
import { getWeekGroupFromTimestamp, MONTHS_FULL } from "./date";
+import { VirtualizedGroupHeader } from "./virtualized-grouping";
type EvaluateKeyFunction = (item: T) => string;
@@ -45,33 +40,19 @@ export const getSortValue = (
return item.dateCreated;
};
-function getSortSelectors(options: GroupOptions) {
- if (options.sortBy === "title")
- return {
- asc: (a: T, b: T) =>
- getTitle(a).localeCompare(getTitle(b), undefined, { numeric: true }),
- desc: (a: T, b: T) =>
- getTitle(b).localeCompare(getTitle(a), undefined, { numeric: true })
- };
-
- return {
- asc: (a: T, b: T) => getSortValue(options, a) - getSortValue(options, b),
- desc: (a: T, b: T) => getSortValue(options, b) - getSortValue(options, a)
- };
-}
-
const MILLISECONDS_IN_DAY = 1000 * 60 * 60 * 24;
const MILLISECONDS_IN_WEEK = MILLISECONDS_IN_DAY * 7;
-function getKeySelector(
- options: GroupOptions
-): EvaluateKeyFunction {
- return (item: T) => {
+function getKeySelector(options: GroupOptions): EvaluateKeyFunction- {
+ return (item: Item) => {
if ("pinned" in item && item.pinned) return "Pinned";
else if ("conflicted" in item && item.conflicted) return "Conflicted";
const date = new Date();
- if (options.sortBy === "title") return getFirstCharacter(getTitle(item));
+ if (item.type === "reminder")
+ return isReminderActive(item) ? "Active" : "Inactive";
+ else if (options.sortBy === "title")
+ return getFirstCharacter(getTitle(item));
else {
const value = getSortValue(options, item);
switch (options.groupBy) {
@@ -98,78 +79,37 @@ function getKeySelector(
};
}
-export function groupArray(
- array: T[],
+export function groupArray(
+ ids: string[],
+ items: Record,
options: GroupOptions = {
groupBy: "default",
sortBy: "dateEdited",
sortDirection: "desc"
}
-): GroupedItems {
- if (options.sortBy && options.sortDirection) {
- const selector = getSortSelectors(options)[options.sortDirection];
- array.sort(selector);
- }
-
- if (options.groupBy === "none") {
- const conflicted: T[] = [];
- const pinned: T[] = [];
- const others: T[] = [];
- for (const item of array) {
- if ("pinned" in item && item.pinned) {
- pinned.push(item);
- continue;
- } else if ("conflicted" in item && item.conflicted) {
- conflicted.push(item);
- continue;
- } else others.push(item);
- }
- const groups: GroupedItems = [];
- if (conflicted.length > 0)
- groups.push(
- { title: "Conflicted", type: "header", id: "conflicted" },
- ...conflicted
- );
- if (pinned.length > 0)
- groups.push({ title: "Pinned", type: "header", id: "pinned" }, ...pinned);
- if (others.length > 0)
- groups.push({ title: "All", type: "header", id: "all" }, ...others);
- return groups;
- }
-
- const groups = new Map([
- ["Conflicted", []],
- ["Pinned", []]
+): VirtualizedGroupHeader[] {
+ const groups = new Map([
+ ["Conflicted", { title: "Conflicted", id: "" }],
+ ["Pinned", { title: "Pinned", id: "" }],
+ ["Active", { title: "Active", id: "" }],
+ ["Inactive", { title: "Inactive", id: "" }]
]);
const keySelector = getKeySelector(options);
- array.forEach((item) => addToGroup(groups, keySelector(item), item));
+ for (const id of ids) {
+ const item = items[id];
+ if (!item) continue;
- return flattenGroups(groups);
-}
+ const groupTitle = keySelector(item);
+ const group = groups.get(groupTitle) || {
+ title: groupTitle,
+ id: ""
+ };
+ if (group.id === "") group.id = id;
+ groups.set(groupTitle, group);
+ }
-export function groupReminders(array: Reminder[]): GroupedItems {
- const groups = new Map([
- ["Active", []],
- ["Inactive", []]
- ]);
-
- array.forEach((item) => {
- const groupTitle = isReminderActive(item) ? "Active" : "Inactive";
- addToGroup(groups, groupTitle, item);
- });
-
- return flattenGroups(groups);
-}
-
-function addToGroup(
- groups: Map,
- groupTitle: string,
- item: T
-) {
- const group = groups.get(groupTitle) || [];
- group.push(item);
- groups.set(groupTitle, group);
+ return Array.from(groups.values());
}
function getFirstCharacter(str: string) {
@@ -179,21 +119,10 @@ function getFirstCharacter(str: string) {
return str[0].toUpperCase();
}
-function getTitle(item: T): string {
- return item.type === "attachment" ? item.metadata.filename : item.title;
-}
-
-function flattenGroups(groups: Map) {
- const items: GroupedItems = [];
- groups.forEach((groupItems, groupTitle) => {
- if (groupItems.length <= 0) return;
- items.push({
- title: groupTitle,
- id: groupTitle.toLowerCase(),
- type: "header"
- });
- groupItems.forEach((item) => items.push(item));
- });
-
- return items;
+function getTitle(item: Item): string {
+ return item.type === "attachment"
+ ? item.filename
+ : "title" in item
+ ? item.title
+ : "Unknown";
}
diff --git a/packages/core/src/utils/set.ts b/packages/core/src/utils/set.ts
index 683253dbc..19e414178 100644
--- a/packages/core/src/utils/set.ts
+++ b/packages/core/src/utils/set.ts
@@ -85,14 +85,14 @@ class SetManipulator {
// Symmetric difference. Items from either set that
// are not in both sets.
// Set.difference([1, 1, 2], [2, 3, 3]) => [1, 3]
- difference(a: T[], b: T[], key: KeySelector) {
- return this.process(a, b, key, (freq) => freq < 3);
+ difference(a: T[], b: T[], key?: KeySelector) {
+ return this.process(a, b, key, (freq) => freq < 3);
}
// Relative complement. Items from 'a' which are
// not also in 'b'.
// Set.complement([1, 2, 2], [2, 2, 3]) => [3]
- complement(a: T[], b: T[], key: KeySelector) {
+ complement(a: T[], b: T[], key?: KeySelector) {
return this.process(a, b, key, (freq) => freq === 1);
}
diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.ts
new file mode 100644
index 000000000..c748c253d
--- /dev/null
+++ b/packages/core/src/utils/virtualized-grouping.ts
@@ -0,0 +1,111 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+*/
+
+export type VirtualizedGroupHeader = {
+ title: string;
+ id: string;
+};
+export class VirtualizedGrouping {
+ private cache: Map> = new Map();
+ private groups: Map = new Map();
+
+ constructor(
+ private ids: string[],
+ private readonly batchSize: number,
+ private readonly fetchItems: (ids: string[]) => Promise>,
+ private readonly groupItems: (
+ ids: string[],
+ items: Record
+ ) => VirtualizedGroupHeader[] = () => []
+ ) {
+ this.ids = ids;
+ }
+
+ /**
+ * Get item from cache or request the appropriate batch for caching
+ * and load it from there.
+ */
+ async item(id: string) {
+ const index = this.ids.indexOf(id);
+ if (index <= -1) return;
+
+ const batchIndex = Math.floor(index / this.batchSize);
+ const batch = this.cache.get(batchIndex) || (await this.load(batchIndex));
+ const groups = this.groups.get(batchIndex);
+
+ const group = groups?.find((g) => g.id === id);
+ if (group)
+ return {
+ group: { type: "header", id: group.title, title: group.title },
+ item: batch[id]
+ };
+ return { item: batch[id] };
+ }
+
+ /**
+ * Reload the cache
+ */
+ refresh(ids: string[]) {
+ this.ids = ids;
+ this.cache.clear();
+ }
+
+ /**
+ *
+ * @param index
+ */
+ private async load(batch: number) {
+ const start = batch * this.batchSize;
+ const end = start + this.batchSize;
+ const batchIds = this.ids.slice(start, end);
+ const items = await this.fetchItems(batchIds);
+ const groups = this.groupItems(batchIds, items);
+
+ const lastBatchIndex = this.last;
+ const prevGroups = this.groups.get(lastBatchIndex);
+ if (prevGroups && prevGroups.length > 0 && groups.length > 0) {
+ const lastGroup = prevGroups[prevGroups.length - 1];
+ if (lastGroup.title === groups[0].title) {
+ // if user is moving downwards, we remove the last group from the
+ // current batch, otherwise we remove the first group from the previous
+ // batch.
+ lastBatchIndex < batch ? groups.pop() : prevGroups.shift();
+ }
+ }
+
+ this.cache.set(batch, items);
+ this.groups.set(batch, groups);
+ this.clear();
+ return items;
+ }
+
+ private clear() {
+ if (this.cache.size <= 2) return;
+ for (const [key] of this.cache) {
+ this.cache.delete(key);
+ this.groups.delete(key);
+ if (this.cache.size === 2) break;
+ }
+ }
+
+ private get last() {
+ const keys = Array.from(this.cache.keys());
+ return keys[keys.length - 1];
+ }
+}