editor: virtualize pages with the same machinery as blocks

packages/editor/src/extensions/paging/__tests__/paged-virtualization.test.ts

@@ -0,0 +1,128 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { describe, expect, test } from "vitest";
+import { Editor, Node } from "@tiptap/core";
+import StarterKit from "@tiptap/starter-kit";
+import { DecorationSet } from "@tiptap/pm/view";
+import { Page, Paging, countPages } from "../index.js";
+import { BlockId } from "../../block-id/block-id.js";
+import {
+  Virtualization,
+  virtualizationKey
+} from "../../virtualization/index.js";
+
+const PagedDocument = Node.create({
+  name: "doc",
+  topNode: true,
+  content: "(page | block)+"
+});
+
+const BLOCKS = 1000;
+const PAGE_SIZE = 100;
+
+function savedNoteHTML(n: number) {
+  let content = "";
+  for (let i = 0; i < n; i++)
+    content += `<p data-block-id="blk${i}">Paragraph number ${i}.</p>`;
+  return content;
+}
+
+function createEditor() {
+  return new Editor({
+    extensions: [
+      StarterKit.configure({ document: false }),
+      PagedDocument,
+      Page,
+      BlockId,
+      Paging.configure({
+        enabled: true,
+        pageSize: PAGE_SIZE,
+        thresholdBlocks: 10
+      }),
+      Virtualization.configure({ enabled: true, unit: "pages" })
+    ],
+    content: savedNoteHTML(BLOCKS)
+  });
+}
+
+async function created() {
+  await new Promise((resolve) => setTimeout(resolve, 0));
+}
+
+function decorations(editor: Editor): DecorationSet {
+  return (
+    virtualizationKey.getState(editor.state)?.decorations ?? DecorationSet.empty
+  );
+}
+
+describe("paged virtualization", () => {
+  test("renders pages, not blocks, as the top-level elements", async () => {
+    const editor = createEditor();
+    await created();
+
+    expect(countPages(editor.state.doc)).toBe(10);
+    expect(editor.view.dom.children).toHaveLength(10);
+    editor.destroy();
+  });
+
+  test("keeps off-screen pages as placeholders", async () => {
+    const editor = createEditor();
+    await created();
+
+    // pages 0 and 9 are edges and 0-1 hold the caret, so the rest are empty
+    const placeholders = editor.view.dom.querySelectorAll(
+      "[data-virtual-placeholder]"
+    );
+    expect(placeholders).toHaveLength(7);
+    editor.destroy();
+  });
+
+  test("decorates pages rather than every block", async () => {
+    const editor = createEditor();
+    await created();
+
+    const spans = decorations(editor).find(0, editor.state.doc.content.size);
+    expect(spans).toHaveLength(3);
+    editor.destroy();
+  });
+
+  test("a placeholder page keeps its blocks in the document", async () => {
+    const editor = createEditor();
+    await created();
+
+    expect(
+      editor.view.dom.querySelectorAll("[data-virtual-placeholder]").length
+    ).toBeGreaterThan(0);
+    const html = editor.getHTML();
+    expect(html.match(/<p/g)).toHaveLength(BLOCKS);
+    expect(html).not.toContain("data-page");
+    editor.destroy();
+  });
+
+  test("the page holding the caret is never a placeholder", async () => {
+    const editor = createEditor();
+    await created();
+
+    editor.commands.setTextSelection(editor.state.doc.content.size - 2);
+    const lastPage = editor.view.dom.children[9] as HTMLElement;
+    expect(lastPage.hasAttribute("data-virtual-placeholder")).toBe(false);
+    editor.destroy();
+  });
+});

packages/editor/src/extensions/paging/__tests__/paging.test.ts

@@ -23,6 +23,7 @@ import StarterKit from "@tiptap/starter-kit";
 import { Node } from "@tiptap/core";
 import { Page, Paging, countPages, flattenBlocks } from "../index.js";
 import { BlockId } from "../../block-id/block-id.js";
+import { getTableOfContents } from "../../../utils/toc.js";
 
 const PagedDocument = Node.create({
   name: "doc",
@@ -139,6 +140,43 @@ describe("paging", () => {
     editor.destroy();
   });
 
+  test("headings inside pages still reach the table of contents", async () => {
+    let content = "";
+    for (let i = 0; i < BLOCKS; i++)
+      content += `<h1 data-block-id="h${i}">Heading ${i}</h1>`;
+    const editor = createEditor(content);
+    await created();
+
+    expect(countPages(editor.state.doc)).toBe(3);
+    const toc = getTableOfContents(
+      editor.state.doc,
+      editor.view.dom as HTMLElement
+    );
+    expect(toc).toHaveLength(BLOCKS);
+    expect(toc[0].title).toBe("Heading 0");
+    editor.destroy();
+  });
+
+  test("blocks inside pages keep getting block ids", async () => {
+    let content = "";
+    for (let i = 0; i < BLOCKS; i++) content += `<p>Paragraph ${i}.</p>`;
+    const editor = createEditor(content);
+    await created();
+
+    editor.commands.setTextSelection(3);
+    editor.commands.insertContent("x");
+    await created();
+
+    const page = editor.state.doc.child(0);
+    let withIds = 0;
+    page.forEach((node) => {
+      if (node.attrs.blockId) withIds++;
+    });
+    expect(withIds).toBe(page.childCount);
+    expect(editor.state.doc.child(0).attrs.blockId).toBeTruthy();
+    editor.destroy();
+  });
+
   test("editing a paged document still serializes flat", async () => {
     const editor = createEditor(savedNoteHTML(BLOCKS));
     await created();

packages/editor/src/extensions/paging/split.ts

@@ -18,6 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import { Fragment, Node as ProsemirrorNode, Schema } from "@tiptap/pm/model";
+import { nanoid } from "nanoid";
 import { PAGE_NODE } from "./page.js";
 
 /** Blocks are grouped into pages of this size when a note is opened. */
@@ -43,9 +44,18 @@ export function toPages(
   const blocks = flattenBlocks(doc);
   if (!blocks.length) return doc.content;
 
+  // Pages are identified at creation rather than by the block id plugin: the
+  // viewport plugin needs an id the moment a page exists, and one assigned a
+  // transaction later would leave every page permanently materialized.
+  const identify = "blockId" in (pageType.spec.attrs ?? {});
   const pages: ProsemirrorNode[] = [];
   for (let i = 0; i < blocks.length; i += pageSize)
-    pages.push(pageType.create(null, blocks.slice(i, i + pageSize)));
+    pages.push(
+      pageType.create(
+        identify ? { blockId: nanoid(8) } : null,
+        blocks.slice(i, i + pageSize)
+      )
+    );
 
   return Fragment.fromArray(pages);
 }

packages/editor/src/extensions/virtualization/height-map.ts

@@ -44,6 +44,11 @@ export class HeightMap {
   private measured = new Map<string, number>();
 
   estimate(node: ProsemirrorNode): number {
+    if (node.type.name === "page") {
+      let total = 0;
+      node.forEach((child) => (total += this.heightFor(child)));
+      return total || FALLBACK_ESTIMATE;
+    }
     return DEFAULT_ESTIMATES[node.type.name] ?? FALLBACK_ESTIMATE;
   }
 

packages/editor/src/extensions/virtualization/index.ts

@@ -19,7 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { Editor, Extension } from "@tiptap/core";
 import { HeightMap } from "./height-map.js";
-import { withVirtualization } from "./node-views.js";
+import { VirtualizationUnit, withVirtualization } from "./node-views.js";
 import { virtualizationPlugin } from "./viewport-plugin.js";
 
 /** Paging only engages for notes larger than this many top-level blocks. */
@@ -28,11 +28,13 @@ const DEFAULT_THRESHOLD_BLOCKS = 300;
 export type VirtualizationOptions = {
   enabled: boolean;
   thresholdBlocks: number;
+  unit: VirtualizationUnit;
 };
 
 export type VirtualizationStorage = {
   enabled: boolean;
   thresholdBlocks: number;
+  unit: VirtualizationUnit;
   heightMap: HeightMap;
 };
 
@@ -49,13 +51,18 @@ export const Virtualization = Extension.create<VirtualizationOptions>({
   name: "virtualization",
 
   addOptions() {
-    return { enabled: false, thresholdBlocks: DEFAULT_THRESHOLD_BLOCKS };
+    return {
+      enabled: false,
+      thresholdBlocks: DEFAULT_THRESHOLD_BLOCKS,
+      unit: "blocks"
+    };
   },
 
   addStorage(): VirtualizationStorage {
     return {
       enabled: this.options.enabled,
       thresholdBlocks: this.options.thresholdBlocks,
+      unit: this.options.unit,
       heightMap: new HeightMap()
     };
   },
@@ -70,7 +77,7 @@ export const Virtualization = Extension.create<VirtualizationOptions>({
 
   addProseMirrorPlugins() {
     if (!this.options.enabled) return [];
-    return [virtualizationPlugin()];
+    return [virtualizationPlugin(this.options.unit)];
   }
 });
 
@@ -94,10 +101,7 @@ export function installVirtualization(editor: Editor): void {
     | undefined;
   if (!storage?.enabled) return;
 
-  const manager = editor.extensionManager as unknown as Record<
-    string,
-    unknown
-  >;
+  const manager = editor.extensionManager as unknown as Record<string, unknown>;
   if (Object.prototype.hasOwnProperty.call(manager, "nodeViews")) return;
 
   const proto = Object.getPrototypeOf(editor.extensionManager);
@@ -111,7 +115,8 @@ export function installVirtualization(editor: Editor): void {
       return withVirtualization(
         originalGetter.call(this),
         storage.heightMap,
-        storage.thresholdBlocks
+        storage.thresholdBlocks,
+        storage.unit
       );
     }
   });

packages/editor/src/extensions/virtualization/node-views.ts

@@ -28,6 +28,15 @@ import {
 import { profiler } from "../../utils/profiler.js";
 import { HeightMap } from "./height-map.js";
 
+export type VirtualizationUnit = "blocks" | "pages";
+
+/** A page is virtualized exactly like a block; it is just a much taller one. */
+export const PAGE_TYPES = ["page"];
+
+export function unitTypes(unit: VirtualizationUnit): string[] {
+  return unit === "pages" ? PAGE_TYPES : TOP_LEVEL_BLOCK_TYPES;
+}
+
 export const TOP_LEVEL_BLOCK_TYPES = [
   "paragraph",
   "heading",
@@ -212,17 +221,21 @@ function wrapCustom(
 export function withVirtualization(
   nodeViews: Record<string, NodeViewConstructor>,
   heightMap: HeightMap,
-  thresholdBlocks: number
+  thresholdBlocks: number,
+  unit: VirtualizationUnit = "blocks"
 ): Record<string, NodeViewConstructor> {
   const wrapped: Record<string, NodeViewConstructor> = { ...nodeViews };
 
-  for (const type of TOP_LEVEL_BLOCK_TYPES) {
+  for (const type of unitTypes(unit)) {
     const inner = nodeViews[type];
     wrapped[type] = (node, view, getPos, decorations, innerDecorations) => {
       const topLevel = isTopLevel(view, getPos as () => number | undefined);
       const materialize = isMaterialized(decorations);
 
-      const belowThreshold = view.state.doc.childCount <= thresholdBlocks;
+      // Pages only exist once a note is past the threshold, so their presence
+      // is the gate; blocks are counted directly.
+      const belowThreshold =
+        unit === "pages" ? false : view.state.doc.childCount <= thresholdBlocks;
 
       if (!topLevel || belowThreshold) {
         profiler.count("virtualization.nodeView.unvirtualized");

packages/editor/src/extensions/virtualization/viewport-plugin.ts

@@ -21,7 +21,7 @@ import { Node as ProsemirrorNode } from "@tiptap/pm/model";
 import { EditorState, Plugin, PluginKey } from "@tiptap/pm/state";
 import { Decoration, DecorationSet } from "@tiptap/pm/view";
 import { profiler } from "../../utils/profiler.js";
-import { TOP_LEVEL_BLOCK_TYPES } from "./node-views.js";
+import { VirtualizationUnit, unitTypes } from "./node-views.js";
 
 export const virtualizationKey = new PluginKey<VirtualizationState>(
   "notesnook-virtualization"
@@ -42,9 +42,7 @@ const KEEP_OVERSCAN = 1.5;
 const MATERIALIZE_ATTRS = {};
 const MATERIALIZE_SPEC = { materialize: true };
 
-function isPageable(typeName: string): boolean {
-  return TOP_LEVEL_BLOCK_TYPES.includes(typeName);
-}
+type IsPageable = (typeName: string) => boolean;
 
 function findScrollParent(node: HTMLElement): HTMLElement | null {
   let current: HTMLElement | null = node.parentElement;
@@ -107,7 +105,8 @@ function materializeDecoration(from: number, to: number): Decoration {
 function buildDecorations(
   doc: ProsemirrorNode,
   visible: Set<string>,
-  selectionIndex: number
+  selectionIndex: number,
+  isPageable: IsPageable
 ): DecorationSet {
   const end = profiler.start("virtualization.decorations");
   const decorations: Decoration[] = [];
@@ -185,7 +184,8 @@ function hasMaterializeDecoration(
  */
 function repairSelection(
   set: DecorationSet,
-  state: EditorState
+  state: EditorState,
+  isPageable: IsPageable
 ): DecorationSet {
   const missing: Decoration[] = [];
   for (const range of selectionRanges(state)) {
@@ -199,7 +199,12 @@ function repairSelection(
   return set.add(state.doc, missing);
 }
 
-export function virtualizationPlugin(): Plugin<VirtualizationState> {
+export function virtualizationPlugin(
+  unit: VirtualizationUnit = "blocks"
+): Plugin<VirtualizationState> {
+  const types = unitTypes(unit);
+  const isPageable: IsPageable = (typeName) => types.includes(typeName);
+
   return new Plugin<VirtualizationState>({
     key: virtualizationKey,
     state: {
@@ -212,7 +217,8 @@ export function virtualizationPlugin(): Plugin<VirtualizationState> {
           decorations: buildDecorations(
             state.doc,
             EMPTY_VISIBLE,
-            selectionIndex
+            selectionIndex,
+            isPageable
           )
         };
       },
@@ -237,7 +243,12 @@ export function virtualizationPlugin(): Plugin<VirtualizationState> {
             visible,
             selectionIndex,
             blockCount,
-            decorations: buildDecorations(tr.doc, visible, selectionIndex)
+            decorations: buildDecorations(
+              tr.doc,
+              visible,
+              selectionIndex,
+              isPageable
+            )
           };
         }
 
@@ -247,7 +258,8 @@ export function virtualizationPlugin(): Plugin<VirtualizationState> {
         const end = profiler.start("virtualization.decorationMap");
         const mapped = repairSelection(
           value.decorations.map(tr.mapping, tr.doc),
-          newState
+          newState,
+          isPageable
         );
         end();
         profiler.count("virtualization.decorationMaps");

packages/editor/src/index.ts

@@ -282,7 +282,10 @@ const useTiptap = (
         BlockId,
         PagedDocument,
         Page,
-        Virtualization.configure({ enabled: mode === "blocks" }),
+        Virtualization.configure({
+          enabled: mode !== "off",
+          unit: mode === "pages" ? "pages" : "blocks"
+        }),
         Paging.configure({ enabled: mode === "pages" }),
         EditorProfiler,
         Blockquote,