common: add export notes utility

this utility resolves attachments & internal links

packages/common/__tests__/path-tree.test.ts

@@ -0,0 +1,97 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { describe, test } from "vitest";
+import { PathTree } from "../src/utils/path-tree";
+
+test("adding duplicate path should make it unique", (t) => {
+  const tree = new PathTree();
+  t.expect(tree.add("/home/world/test.txt")).toBe("/home/world/test.txt");
+  t.expect(tree.add("/home/world/test.txt")).toBe("/home/world/test_1.txt");
+});
+
+test("adding path with same filename as directory shouldn't make it unique", (t) => {
+  const tree = new PathTree();
+  t.expect(tree.add("/home/world")).toBe("/home/world");
+  t.expect(tree.add("/home/world.txt")).toBe("/home/world.txt");
+});
+
+test("adding directory with same name as filename shouldn't make it unique", (t) => {
+  const tree = new PathTree();
+  t.expect(tree.add("/home/world.txt")).toBe("/home/world.txt");
+  t.expect(tree.add("/home/world.txt/world.hello")).toBe(
+    "/home/world.txt/world.hello"
+  );
+});
+
+test("different casing in filename", (t) => {
+  const tree = new PathTree();
+  t.expect(tree.add("/home/woRlD.txt")).toBe("/home/woRlD.txt");
+  t.expect(tree.add("/home/world.txt")).toBe("/home/world_1.txt");
+});
+
+test("different casing in directory name", (t) => {
+  const tree = new PathTree();
+  t.expect(tree.add("/home/woRlD/one.txt")).toBe("/home/woRlD/one.txt");
+  t.expect(tree.add("/home/world/one.txt")).toBe("/home/world/one_1.txt");
+});
+
+test("files with different extensions but same name are unique", (t) => {
+  const tree = new PathTree();
+  t.expect(tree.add("/home/world.txt")).toBe("/home/world.txt");
+  t.expect(tree.add("/home/world.hello")).toBe("/home/world.hello");
+});
+
+test("uniquify roots", (t) => {
+  const tree = new PathTree();
+  t.expect(tree.add("root")).toBe("root");
+  t.expect(tree.add("root")).toBe("root_1");
+});
+
+test("uniquify directories", (t) => {
+  const tree = new PathTree();
+  t.expect(tree.add("root/root/hello.txt")).toBe("root/root/hello.txt");
+  t.expect(tree.add("root")).toBe("root_1");
+  t.expect(tree.add("root/root")).toBe("root/root_1");
+});
+
+test("uniquify directories (underscore strategy)", (t) => {
+  const tree = new PathTree();
+  t.expect(tree.add("root/root/hello.txt")).toBe("root/root/hello.txt");
+  t.expect(tree.add("root", "underscore")).toBe("_root");
+  t.expect(tree.add("root/root", "underscore")).toBe("root/_root");
+});
+
+describe("exists", () => {
+  test("roots", (t) => {
+    const tree = new PathTree();
+    t.expect(tree.add("root")).toBe("root");
+    t.expect(tree.add("root")).toBe("root_1");
+    t.expect(tree.exists("root")).toBe(true);
+    t.expect(tree.exists("root_1")).toBe(true);
+  });
+
+  test("files with different extensions but same name", (t) => {
+    const tree = new PathTree();
+    t.expect(tree.add("/home/world.txt")).toBe("/home/world.txt");
+    t.expect(tree.add("/home/world.hello")).toBe("/home/world.hello");
+    t.expect(tree.exists("/home/world.txt")).toBe(true);
+    t.expect(tree.exists("/home/world.hello")).toBe(true);
+  });
+});

packages/common/package.json

@@ -30,15 +30,17 @@
   },
   "devDependencies": {
     "@notesnook/core": "file:../core",
+    "@types/react": "^18.2.39",
     "react": "18.2.0",
-    "@types/react": "^18.2.39"
+    "vitest": "^1.4.0"
   },
   "peerDependencies": {
-    "timeago.js": "4.0.2",
-    "react": ">=18"
+    "react": ">=18",
+    "timeago.js": "4.0.2"
   },
   "dependencies": {
-    "timeago.js": "4.0.2",
-    "@notesnook/core": "file:../core"
+    "@notesnook/core": "file:../core",
+    "pathe": "^1.1.2",
+    "timeago.js": "4.0.2"
   }
 }

packages/common/src/utils/export-notes.ts

@@ -0,0 +1,320 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Attachment, Note, parseInternalLink } from "@notesnook/core";
+import { FilteredSelector } from "@notesnook/core/dist/database/sql-collection";
+import { CHECK_IDS, checkIsUserPremium } from "@notesnook/core/dist/common";
+import { isImage, isWebClip } from "@notesnook/core/dist/utils/filename";
+import { sanitizeFilename } from "./file";
+import { database } from "../database";
+import { join, relative } from "pathe";
+import { EMPTY_CONTENT } from "@notesnook/core/dist/collections/content";
+import { getContentFromData } from "@notesnook/core/dist/content-types";
+import { PathTree } from "./path-tree";
+import { ResolveInternalLink } from "@notesnook/core/dist/content-types/tiptap";
+
+const FORMAT_TO_EXT = {
+  pdf: "pdf",
+  md: "md",
+  txt: "txt",
+  html: "html",
+  "md-frontmatter": "md"
+} as const;
+
+type BaseExportableItem = {
+  path: string;
+  mtime: Date;
+  ctime: Date;
+};
+export type ExportableItem = ExportableNote | ExportableAttachment;
+export type ExportableNote = BaseExportableItem & {
+  type: "note";
+  data: string;
+};
+export type ExportableAttachment = BaseExportableItem & {
+  type: "attachment";
+  data: Attachment;
+};
+
+export async function* exportNotes(
+  notes: FilteredSelector<Note>,
+  options: {
+    format: keyof typeof FORMAT_TO_EXT;
+    unlockVault?: () => Promise<boolean>;
+  }
+) {
+  const { format } = options;
+  if (format !== "txt" && !(await checkIsUserPremium(CHECK_IDS.noteExport)))
+    return;
+
+  const pathTree = new PathTree();
+  const notePathMap: Map<string, string[]> = new Map();
+
+  for await (const note of notes.fields(["notes.id", "notes.title"])) {
+    const filename = `${sanitizeFilename(note.title, { replacement: "-" })}.${
+      FORMAT_TO_EXT[format]
+    }`;
+    const notebooks = await database.relations
+      .to({ id: note.id, type: "note" }, "notebook")
+      .get();
+
+    const filePaths: string[] = [];
+    for (const { fromId: notebookId } of notebooks) {
+      const crumbs = (await database.notebooks.breadcrumbs(notebookId)).map(
+        (n) => n.title
+      );
+      filePaths.push([...crumbs, filename].join("/"));
+    }
+    if (filePaths.length === 0) filePaths.push(filename);
+
+    notePathMap.set(
+      note.id,
+      filePaths.map((p) => pathTree.add(p))
+    );
+  }
+
+  // case where the user has a notebook named attachments
+  const attachmentsRoot = pathTree.add("attachments", "underscore");
+  const pendingAttachments: Map<string, Attachment> = new Map();
+
+  for (const [id] of notePathMap) {
+    const note = await database.notes.note(id);
+    if (!note) continue;
+
+    const notePaths = notePathMap.get(note.id);
+
+    if (!notePaths) {
+      yield new Error("Cannot export note because it has unresolved paths.");
+      continue;
+    }
+
+    const content = await exportContent(note, {
+      unlockVault: options.unlockVault,
+      format,
+      attachmentsRoot,
+      pendingAttachments,
+      resolveInternalLink: (link) => {
+        const internalLink = parseInternalLink(link);
+        if (!internalLink) return link;
+        const paths = notePathMap.get(internalLink.id);
+        if (!paths) return link;
+        // if the internal link is linking within the same note
+        if (paths === notePaths) return `{{NOTE_PATH:}}`;
+        return `{{NOTE_PATH:${paths[0]}}}`;
+      }
+    });
+    if (!content) continue;
+
+    for (const path of notePaths) {
+      yield <ExportableNote>{
+        type: "note",
+        path,
+        data: resolvePaths(content, path),
+        mtime: new Date(note.dateEdited),
+        ctime: new Date(note.dateCreated)
+      };
+    }
+  }
+
+  for (const [path, attachment] of pendingAttachments) {
+    yield <ExportableAttachment>{
+      type: "attachment",
+      path,
+      data: attachment,
+      mtime: new Date(attachment.dateModified),
+      ctime: new Date(attachment.dateCreated)
+    };
+  }
+}
+
+export async function* exportNote(
+  note: Note,
+  options: {
+    format: keyof typeof FORMAT_TO_EXT;
+    unlockVault?: () => Promise<boolean>;
+  }
+) {
+  const { format } = options;
+  if (format !== "txt" && !(await checkIsUserPremium(CHECK_IDS.noteExport)))
+    return;
+
+  const attachmentsRoot = "attachments";
+  const filename = sanitizeFilename(note.title, { replacement: "-" });
+  const ext = FORMAT_TO_EXT[options.format];
+  const path = [filename, ext].join(".");
+  const pendingAttachments: Map<string, Attachment> = new Map();
+
+  try {
+    const content = await exportContent(note, {
+      format,
+      attachmentsRoot,
+      pendingAttachments
+    });
+    if (!content) return false;
+
+    yield <ExportableNote>{
+      type: "note",
+      path,
+      data: content,
+      mtime: new Date(note.dateEdited),
+      ctime: new Date(note.dateCreated)
+    };
+
+    for (const [path, attachment] of pendingAttachments) {
+      yield <ExportableAttachment>{
+        type: "attachment",
+        path,
+        data: attachment,
+        mtime: new Date(attachment.dateModified),
+        ctime: new Date(attachment.dateCreated)
+      };
+    }
+  } catch (e) {
+    yield new Error(
+      `Failed to export note "${note.title}": ${(e as Error).message}`
+    );
+  }
+}
+
+export async function exportContent(
+  note: Note,
+  options: {
+    format: keyof typeof FORMAT_TO_EXT;
+    unlockVault?: () => Promise<boolean>;
+    disableTemplate?: boolean;
+
+    // TODO: remove these
+    attachmentsRoot?: string;
+    pendingAttachments?: Map<string, Attachment>;
+    resolveInternalLink?: ResolveInternalLink;
+  }
+) {
+  const {
+    format,
+    unlockVault,
+    resolveInternalLink,
+    attachmentsRoot,
+    pendingAttachments,
+    disableTemplate
+  } = options;
+  const rawContent = await database.content.findByNoteId(note.id);
+
+  if (
+    rawContent?.locked &&
+    !database.vault.unlocked &&
+    !(await unlockVault?.())
+  ) {
+    throw new Error(`Could not export locked note: "${note.title}".`);
+  }
+
+  const contentItem = rawContent?.locked
+    ? await database.vault
+        .decryptContent(rawContent, note.id)
+        .catch(() => undefined)
+    : rawContent;
+
+  const { data, type } =
+    format === "pdf"
+      ? await database.content.downloadMedia(
+          `export-${note.id}`,
+          contentItem || EMPTY_CONTENT(note.id),
+          false
+        )
+      : contentItem || EMPTY_CONTENT(note.id);
+
+  const content = getContentFromData(type, data);
+  if (resolveInternalLink) content.resolveInternalLinks(resolveInternalLink);
+
+  if (
+    attachmentsRoot &&
+    pendingAttachments &&
+    format !== "txt" &&
+    format !== "pdf"
+  ) {
+    await content.resolveAttachments(async (elements) => {
+      const hashes = Object.keys(elements);
+      const attachments = await database.attachments.all
+        .where((eb) => eb("attachments.hash", "in", hashes))
+        .items();
+
+      const sources: Record<string, string> = {};
+      for (const attachment of attachments) {
+        const attachmentPath = join(attachmentsRoot, attachment.filename);
+        sources[attachment.hash] = resolveAttachment(
+          elements,
+          attachment,
+          attachmentPath,
+          format
+        );
+        pendingAttachments.set(attachmentPath, attachment);
+      }
+      return sources;
+    });
+  }
+
+  const exported = await database.notes.export(note, {
+    disableTemplate,
+    format: format === "pdf" ? "html" : format,
+    rawContent:
+      format === "html" || format === "pdf"
+        ? content.toHTML()
+        : format === "md" || format === "md-frontmatter"
+        ? content.toMD()
+        : content.toTXT()
+  });
+  if (typeof exported === "boolean" && !exported) return;
+  return exported;
+}
+
+function resolveAttachment(
+  elements: Record<string, Record<string, string>>,
+  attachment: Attachment,
+  attachmentPath: string,
+  format: keyof typeof FORMAT_TO_EXT
+) {
+  const relativePath = `{{NOTE_PATH:${attachmentPath}}}`;
+  const attributes = elements[attachment.hash];
+
+  if (isImage(attachment.mimeType)) {
+    const classes: string[] = [];
+    if (attributes["data-float"] === "true") classes.push("float");
+    if (attributes["data-align"] === "right") classes.push("align-right");
+    if (attributes["data-align"] === "center") classes.push("align-center");
+
+    return `<img class="${classes.join(" ")}" src="${relativePath}" alt="${
+      attachment.filename
+    }" width="${attributes.width}" height="${attributes.height}" />`;
+  }
+  // markdown doesn't allow arbitrary iframes in its html so no need
+  // to support that
+  else if (isWebClip(attachment.mimeType) && format === "html") {
+    return `<iframe src="${relativePath} "width="${attributes.width}" height="${attributes.height}" />`;
+  }
+
+  return `<a href="${relativePath}" title="${attachment.filename}">${attachment.filename}</a>`;
+}
+
+function resolvePaths(content: string, path: string) {
+  return content.replace(/\{\{NOTE_PATH:(.+?)\}\}/gm, (str, ...args) => {
+    console.log(str, args);
+    const [to] = args;
+    if (!to) return path;
+    return relative(path, to).replace(/^..\//, "./");
+  });
+}

packages/common/src/utils/index.ts

@@ -25,3 +25,4 @@ export * from "./random";
 export * from "./string";
 export * from "./resolve-items";
 export * from "./migrate-toolbar";
+export * from "./export-notes";

packages/common/src/utils/path-tree.ts

@@ -0,0 +1,103 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { join, parse } from "pathe";
+
+interface TreeNode extends Record<string, TreeNode> {}
+type UniquifyStrategy = "number" | "underscore";
+
+export class PathTree {
+  private root: TreeNode;
+
+  constructor() {
+    this.root = {};
+  }
+
+  add(path: string, strategy: UniquifyStrategy = "number"): string {
+    const { parts, ext, dir, name } = this.parse(path);
+    let node = this.root;
+
+    for (const part of parts) {
+      if (node[part]) node = node[part];
+      else node = node[part] = {};
+    }
+
+    const newName = this.uniquify(strategy, node, name, ext);
+    node[newName.toLowerCase()] = {};
+    return join(dir, newName);
+  }
+
+  exists(path: string): boolean {
+    const { parts, name, ext } = this.parse(path);
+    let node = this.root;
+
+    for (const part of parts) {
+      if (!node[part]) return false;
+      node = node[part];
+    }
+
+    return !!node[`${name}${ext}`.toLowerCase()];
+  }
+
+  private uniquify(
+    strategy: UniquifyStrategy,
+    node: TreeNode,
+    name: string,
+    ext: string
+  ) {
+    switch (strategy) {
+      case "number": {
+        let index = 1;
+        let newName = name + ext;
+        while (node[newName.toLowerCase()])
+          newName = `${name}_${index++}${ext}`;
+        return newName;
+      }
+      case "underscore": {
+        let newName = name + ext;
+        while (node[newName.toLowerCase()]) newName = `_${newName}`;
+        return newName;
+      }
+    }
+  }
+
+  private parse(path: string): {
+    parts: string[];
+    dir: string;
+    ext: string;
+    name: string;
+  } {
+    const { dir, name, ext } = parse(path);
+
+    return {
+      parts:
+        dir === "."
+          ? []
+          : dir
+              // case insensitive
+              .toLowerCase()
+              // Handle both Unix-like and Windows paths
+              .split(/[\\/]/)
+              .filter((part) => !!part.trim()),
+      dir,
+      ext,
+      name
+    };
+  }
+}