web: add support for streaming backup creation & restore

apps/desktop/src/api/os-integration.ts

@@ -117,6 +117,14 @@ export const osIntegrationRouter = t.router({
       writeFileSync(resolvedPath, data);
     }),
 
+  resolvePath: t.procedure
+    .input(z.object({ filePath: z.string() }))
+    .query(({ input }) => {
+      const { filePath } = input;
+      if (!filePath) return;
+      return resolvePath(filePath);
+    }),
+
   showNotification: t.procedure
     .input(NotificationOptions)
     .query(({ input }) => {

apps/desktop/src/preload.ts

@@ -31,7 +31,6 @@ declare global {
 }
 
 process.once("loaded", async () => {
-  console.log("HELLO!");
   const electronTRPC: RendererGlobalElectronTRPC = {
     sendMessage: (operation) =>
       ipcRenderer.send(ELECTRON_TRPC_CHANNEL, operation),

apps/web/src/common/desktop-bridge/index.desktop.ts

@@ -63,3 +63,17 @@ function attachListener(event: string) {
     }
   };
 }
+
+export async function createWritableStream(path: string) {
+  const resolvedPath = await desktop.integration.resolvePath.query({
+    filePath: path
+  });
+  if (!resolvedPath) throw new Error("invalid path.");
+  const fs = require("fs");
+  const { Writable } = require("stream");
+  return new WritableStream(
+    Writable.toWeb(
+      fs.createWriteStream(resolvedPath, { encoding: "utf-8" })
+    ).getWriter()
+  );
+}

apps/web/src/common/desktop-bridge/index.ts

@@ -17,6 +17,10 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+import { createWriteStream } from "../../utils/stream-saver";
 import { type desktop as bridge } from "./index.desktop";
 
 export const desktop: typeof bridge | undefined = undefined;
+export async function createWritableStream(filename: string) {
+  return createWriteStream(filename);
+}

apps/web/src/common/index.js

@@ -1,217 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import {
-  showFeatureDialog,
-  showLoadingDialog,
-  showPasswordDialog,
-  showReminderDialog
-} from "../common/dialog-controller";
-import Config from "../utils/config";
-import { hashNavigate, getCurrentHash } from "../navigation";
-import { db } from "./db";
-import { sanitizeFilename } from "@notesnook/common";
-import { store as userstore } from "../stores/user-store";
-import FileSaver from "file-saver";
-import { showToast } from "../utils/toast";
-import { SUBSCRIPTION_STATUS } from "./constants";
-import { showFilePicker } from "../utils/file-picker";
-import { logger } from "../utils/logger";
-import { PATHS } from "@notesnook/desktop";
-import { TaskManager } from "./task-manager";
-import { EVENTS } from "@notesnook/core/dist/common";
-import { getFormattedDate } from "@notesnook/common";
-import { desktop } from "./desktop-bridge";
-
-export const CREATE_BUTTON_MAP = {
-  notes: {
-    title: "Add a note",
-    onClick: () =>
-      hashNavigate("/notes/create", { addNonce: true, replace: true })
-  },
-  notebooks: {
-    title: "Create a notebook",
-    onClick: () => hashNavigate("/notebooks/create", { replace: true })
-  },
-  topics: {
-    title: "Create a topic",
-    onClick: () => hashNavigate(`/topics/create`, { replace: true })
-  },
-  tags: {
-    title: "Create a tag",
-    onClick: () => hashNavigate(`/tags/create`, { replace: true })
-  },
-  reminders: {
-    title: "Add a reminder",
-    onClick: () => hashNavigate(`/reminders/create`, { replace: true })
-  }
-};
-
-export async function introduceFeatures() {
-  const hash = getCurrentHash().replace("#", "");
-  if (!!hash || IS_TESTING) return;
-  const features = [];
-  for (let feature of features) {
-    if (!Config.get(`feature:${feature}`)) {
-      await showFeatureDialog(feature);
-    }
-  }
-}
-
-export const DEFAULT_CONTEXT = { colors: [], tags: [], notebook: {} };
-
-export async function createBackup() {
-  const encryptBackups =
-    userstore.get().isLoggedIn && Config.get("encryptBackups", false);
-  const data = await showLoadingDialog({
-    title: "Creating backup",
-    subtitle: "We are creating a backup of your data. Please wait...",
-    action: async () => {
-      return await db.backup.export("web", encryptBackups);
-    }
-  });
-  if (!data) {
-    showToast("error", "Could not create a backup of your data.");
-    return;
-  }
-
-  const filename = sanitizeFilename(`notesnook-backup-${getFormattedDate()}`);
-
-  const ext = "nnbackup";
-  if (IS_DESKTOP_APP) {
-    const directory = Config.get(
-      "backupStorageLocation",
-      PATHS.backupsDirectory
-    );
-    const filePath = `${directory}/${filename}.${ext}`;
-    await desktop?.integration.saveFile.query({ filePath, data });
-    showToast("success", `Backup saved at ${filePath}.`);
-  } else {
-    FileSaver.saveAs(new Blob([Buffer.from(data)]), `${filename}.${ext}`);
-  }
-}
-
-export async function selectBackupFile() {
-  const file = await showFilePicker({
-    acceptedFileTypes: ".nnbackup,application/json"
-  });
-  if (!file) return;
-
-  const reader = new FileReader();
-  const backup = await new Promise((resolve) => {
-    reader.addEventListener("load", (event) => {
-      const text = event.target.result;
-      try {
-        resolve(JSON.parse(text));
-      } catch (e) {
-        alert(
-          "Error: Could not read the backup file provided. Either it's corrupted or invalid."
-        );
-        resolve();
-      }
-    });
-    reader.readAsText(file);
-  });
-
-  return { file, backup };
-}
-
-export async function importBackup() {
-  const { backup } = await selectBackupFile();
-  await restoreBackupFile(backup);
-}
-
-export async function restoreBackupFile(backup) {
-  if (backup.data.iv && backup.data.salt) {
-    await showPasswordDialog("ask_backup_password", async ({ password }) => {
-      const error = await restoreWithProgress(backup, password);
-      return !error;
-    });
-  } else {
-    await restoreWithProgress(backup);
-  }
-}
-
-async function restoreWithProgress(backup, password) {
-  await TaskManager.startTask({
-    title: "Restoring backup",
-    subtitle: "This might take a while",
-    type: "modal",
-    action: (report) => {
-      db.eventManager.subscribe(
-        EVENTS.migrationProgress,
-        ({ collection, total, current }) => {
-          report({
-            text: `Restoring ${collection}...`,
-            current,
-            total
-          });
-        }
-      );
-
-      report({ text: `Restoring...` });
-      return restore(backup, password);
-    }
-  });
-}
-
-export async function verifyAccount() {
-  if (!(await db.user.getUser())) return true;
-  return showPasswordDialog("verify_account", ({ password }) => {
-    return db.user.verifyPassword(password);
-  });
-}
-
-export function totalSubscriptionConsumed(user) {
-  if (!user) return 0;
-  const start = user.subscription?.start;
-  const end = user.subscription?.expiry;
-  if (!start || !end) return 0;
-
-  const total = end - start;
-  const consumed = Date.now() - start;
-
-  return Math.round((consumed / total) * 100);
-}
-
-export async function showUpgradeReminderDialogs() {
-  if (IS_TESTING) return;
-
-  const user = userstore.get().user;
-  if (!user || !user.subscription || user.subscription?.expiry === 0) return;
-
-  const consumed = totalSubscriptionConsumed(user);
-  const isTrial = user?.subscription?.type === SUBSCRIPTION_STATUS.TRIAL;
-  const isBasic = user?.subscription?.type === SUBSCRIPTION_STATUS.BASIC;
-  if (isBasic && consumed >= 100) {
-    await showReminderDialog("trialexpired");
-  } else if (isTrial && consumed >= 75) {
-    await showReminderDialog("trialexpiring");
-  }
-}
-
-async function restore(backup, password) {
-  try {
-    await db.backup.import(backup, password);
-    showToast("success", "Backup restored!");
-  } catch (e) {
-    logger.error(e, "Could not restore the backup");
-    showToast("error", `Could not restore the backup: ${e.message || e}`);
-  }
-}

apps/web/src/common/index.ts

@@ -0,0 +1,290 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+  showFeatureDialog,
+  showPasswordDialog,
+  showReminderDialog
+} from "./dialog-controller";
+import Config from "../utils/config";
+import { hashNavigate, getCurrentHash } from "../navigation";
+import { db } from "./db";
+import { sanitizeFilename } from "@notesnook/common";
+import { store as userstore } from "../stores/user-store";
+import { showToast } from "../utils/toast";
+import { SUBSCRIPTION_STATUS } from "./constants";
+import { readFile, showFilePicker } from "../utils/file-picker";
+import { logger } from "../utils/logger";
+import { PATHS } from "@notesnook/desktop";
+import { TaskManager } from "./task-manager";
+import { EVENTS } from "@notesnook/core/dist/common";
+import { getFormattedDate } from "@notesnook/common";
+import { createWritableStream } from "./desktop-bridge";
+import { ZipStream } from "../utils/streams/zip-stream";
+import { register } from "../utils/stream-saver/mitm";
+import { FeatureKeys } from "../dialogs/feature-dialog";
+import { Reader } from "../utils/zip-reader";
+
+export const CREATE_BUTTON_MAP = {
+  notes: {
+    title: "Add a note",
+    onClick: () =>
+      hashNavigate("/notes/create", { addNonce: true, replace: true })
+  },
+  notebooks: {
+    title: "Create a notebook",
+    onClick: () => hashNavigate("/notebooks/create", { replace: true })
+  },
+  topics: {
+    title: "Create a topic",
+    onClick: () => hashNavigate(`/topics/create`, { replace: true })
+  },
+  tags: {
+    title: "Create a tag",
+    onClick: () => hashNavigate(`/tags/create`, { replace: true })
+  },
+  reminders: {
+    title: "Add a reminder",
+    onClick: () => hashNavigate(`/reminders/create`, { replace: true })
+  }
+};
+
+export async function introduceFeatures() {
+  const hash = getCurrentHash().replace("#", "");
+  if (!!hash || IS_TESTING) return;
+  const features: FeatureKeys[] = [];
+  for (const feature of features) {
+    if (!Config.get(`feature:${feature}`)) {
+      await showFeatureDialog(feature);
+    }
+  }
+}
+
+export const DEFAULT_CONTEXT = { colors: [], tags: [], notebook: {} };
+
+export async function createBackup() {
+  await register();
+  const encryptBackups =
+    userstore.get().isLoggedIn && Config.get("encryptBackups", false);
+
+  const filename = sanitizeFilename(
+    `notesnook-backup-${getFormattedDate(Date.now())}`
+  );
+  const directory = Config.get("backupStorageLocation", PATHS.backupsDirectory);
+  const ext = "nnbackupz";
+  const filePath = IS_DESKTOP_APP
+    ? `${directory}/${filename}.${ext}`
+    : `${filename}.${ext}`;
+
+  const encoder = new TextEncoder();
+  const error = await TaskManager.startTask<Error | void>({
+    type: "modal",
+    title: "Creating backup",
+    subtitle: "We are creating a backup of your data. Please wait...",
+    action: async (report) => {
+      const writeStream = await createWritableStream(filePath);
+
+      await new ReadableStream({
+        start() {},
+        async pull(controller) {
+          for await (const file of db.backup!.export("web", encryptBackups)) {
+            report({
+              text: `Saving chunk ${file.path}`
+            });
+            controller.enqueue({
+              path: file.path,
+              data: encoder.encode(file.data)
+            });
+          }
+          controller.close();
+        }
+      })
+        .pipeThrough(new ZipStream())
+        .pipeTo(writeStream);
+    }
+  });
+  if (error) {
+    showToast(
+      "error",
+      `Could not create a backup of your data: ${(error as Error).message}`
+    );
+    console.error(error);
+  } else {
+    showToast("success", `Backup saved at ${filePath}.`);
+  }
+}
+
+export async function selectBackupFile() {
+  const file = await showFilePicker({
+    acceptedFileTypes: ".nnbackup,.nnbackupz"
+  });
+  if (!file) return;
+  return file;
+}
+
+export async function importBackup() {
+  const backupFile = await selectBackupFile();
+  if (!backupFile) return;
+  await restoreBackupFile(backupFile);
+}
+
+export async function restoreBackupFile(backupFile: File) {
+  const isLegacy = !backupFile.name.endsWith(".nnbackupz");
+
+  if (isLegacy) {
+    const backup = JSON.parse(await readFile(backupFile));
+
+    if (backup.data.iv && backup.data.salt) {
+      await showPasswordDialog("ask_backup_password", async ({ password }) => {
+        if (!password) return false;
+        const error = await restoreWithProgress(backup, password);
+        return !error;
+      });
+    } else {
+      await restoreWithProgress(backup);
+    }
+  } else {
+    const error = await TaskManager.startTask<Error | void>({
+      title: "Restoring backup",
+      subtitle: "Please wait while we restore your backup...",
+      type: "modal",
+      action: async (report) => {
+        let cachedPassword: string | undefined = undefined;
+        const { read, totalFiles } = await Reader(backupFile);
+        let filesProcessed = 0;
+        for await (const entry of read()) {
+          if (filesProcessed++ === 0 && entry.name !== ".nnbackup")
+            throw new Error("Invalid backup.");
+          else if (entry.name === ".nnbackup") continue;
+
+          const backup = JSON.parse(await entry.text());
+          if (backup.encrypted) {
+            if (!cachedPassword) {
+              const result = await showPasswordDialog(
+                "ask_backup_password",
+                async ({ password }) => {
+                  if (!password) return false;
+                  await db.backup?.import(backup, password);
+                  cachedPassword = password;
+                  return true;
+                }
+              );
+              if (!result) break;
+            } else await db.backup?.import(backup, cachedPassword);
+          } else {
+            await db.backup?.import(backup, null);
+          }
+
+          report({
+            total: totalFiles,
+            text: `Processed ${entry.name}`,
+            current: filesProcessed
+          });
+        }
+        await db.initCollections();
+      }
+    });
+    if (error) {
+      console.error(error);
+      showToast("error", `Failed to restore backup: ${error.message}`);
+    }
+  }
+}
+
+async function restoreWithProgress(
+  backup: Record<string, unknown>,
+  password?: string
+) {
+  return await TaskManager.startTask<Error | void>({
+    title: "Restoring backup",
+    subtitle: "This might take a while",
+    type: "modal",
+    action: (report) => {
+      db.eventManager.subscribe(
+        EVENTS.migrationProgress,
+        ({
+          collection,
+          total,
+          current
+        }: {
+          collection: string;
+          total: number;
+          current: number;
+        }) => {
+          report({
+            text: `Restoring ${collection}...`,
+            current,
+            total
+          });
+        }
+      );
+
+      report({ text: `Restoring...` });
+      return restore(backup, password);
+    }
+  });
+}
+
+export async function verifyAccount() {
+  if (!(await db.user?.getUser())) return true;
+  return showPasswordDialog("verify_account", ({ password }) => {
+    return db.user?.verifyPassword(password) || false;
+  });
+}
+
+export function totalSubscriptionConsumed(user: User) {
+  if (!user) return 0;
+  const start = user.subscription?.start;
+  const end = user.subscription?.expiry;
+  if (!start || !end) return 0;
+
+  const total = end - start;
+  const consumed = Date.now() - start;
+
+  return Math.round((consumed / total) * 100);
+}
+
+export async function showUpgradeReminderDialogs() {
+  if (IS_TESTING) return;
+
+  const user = userstore.get().user;
+  if (!user || !user.subscription || user.subscription?.expiry === 0) return;
+
+  const consumed = totalSubscriptionConsumed(user);
+  const isTrial = user?.subscription?.type === SUBSCRIPTION_STATUS.TRIAL;
+  const isBasic = user?.subscription?.type === SUBSCRIPTION_STATUS.BASIC;
+  if (isBasic && consumed >= 100) {
+    await showReminderDialog("trialexpired");
+  } else if (isTrial && consumed >= 75) {
+    await showReminderDialog("trialexpiring");
+  }
+}
+
+async function restore(backup: Record<string, unknown>, password?: string) {
+  try {
+    await db.backup?.import(backup, password);
+    showToast("success", "Backup restored!");
+  } catch (e) {
+    logger.error(e as Error, "Could not restore the backup");
+    showToast(
+      "error",
+      `Could not restore the backup: ${(e as Error).message || e}`
+    );
+  }
+}

apps/web/src/utils/stream-saver/mitm.ts

@@ -16,11 +16,26 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-export {};
 
 let sw: ServiceWorker | null = null;
 let scope = "";
 
+let keepAlive = () => {
+  keepAlive = () => {};
+  const interval = setInterval(() => {
+    if (sw) {
+      sw.postMessage("ping");
+    } else {
+      const ping =
+        location.href.substr(0, location.href.lastIndexOf("/")) + "/ping";
+      fetch(ping).then((res) => {
+        !res.ok && clearInterval(interval);
+        return res.text();
+      });
+    }
+  }, 10000);
+};
+
 function registerWorker() {
   return navigator.serviceWorker
     .getRegistration("./")
@@ -104,9 +119,12 @@ export function postMessage(
   // messageChannel.port2 to the service worker. The service worker can
   // then use the transferred port to reply via postMessage(), which
   // will in turn trigger the onmessage handler on messageChannel.port1.
-
   const transferable = [ports[0]];
 
+  if (!data.transferringReadable) {
+    keepAlive();
+  }
+
   return sw?.postMessage(data, transferable);
 }
 
@@ -114,4 +132,8 @@ export async function register() {
   if (navigator.serviceWorker) {
     await registerWorker();
   }
+  // FF v102 just started to supports transferable streams, but still needs to ping sw.js
+  // even tough the service worker dose not have to do any kind of work and listen to any
+  // messages... #305
+  keepAlive();
 }

apps/web/src/views/recovery.tsx

@@ -40,10 +40,7 @@ type RecoveryKeyFormData = {
 };
 
 type BackupFileFormData = {
-  backupFile: {
-    file: File;
-    backup: Record<string, unknown>;
-  };
+  backupFile: File;
 };
 
 type NewPasswordFormData = BackupFileFormData & {
@@ -406,7 +403,7 @@ function BackupFileMethod(props: BaseRecoveryComponentProps<"method:backup">) {
     if (!backupFile) return;
     const backupFileInput = document.getElementById("backupFile");
     if (!(backupFileInput instanceof HTMLInputElement)) return;
-    backupFileInput.value = backupFile?.file?.name;
+    backupFileInput.value = backupFile?.name;
   }, [backupFile]);
 
   return (
@@ -519,7 +516,7 @@ function NewPassword(props: BaseRecoveryComponentProps<"new">) {
           throw new Error("Could not reset account password.");
 
         if (formData?.backupFile) {
-          await restoreBackupFile(formData?.backupFile.backup);
+          await restoreBackupFile(formData?.backupFile);
           await db.sync(true, true);
         }