web: add support for downloading all attachments as zip

apps/web/src/common/attachments/attachment-stream.ts

@@ -0,0 +1,108 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { StreamableFS } from "@notesnook/streamable-fs";
+import FS from "../../interfaces/fs";
+import { getNNCrypto } from "../../interfaces/nncrypto.stub";
+import { db } from "../db";
+import { ZipFile } from "./zip-stream";
+
+export type PackageMetadata = {
+  version: string;
+  attachments: string[];
+};
+
+export const METADATA_FILENAME = "metadata.json";
+
+export class AttachmentStream extends ReadableStream<ZipFile> {
+  constructor(attachments: Array<any>) {
+    super({
+      async pull(controller) {
+        const token = await db.fs.tokenManager.getAccessToken();
+        let index = 0;
+        for (const attachment of attachments) {
+          if (index > 2) {
+            return controller.close();
+          }
+          const url = `${hosts.API_HOST}/s3?name=${attachment.metadata.hash}`;
+
+          const { execute } = FS.downloadAttachment(attachment.metadata.hash, {
+            metadata: attachment.metadata,
+            url,
+            chunkSize: attachment.chunkSize,
+            headers: { Authorization: `Bearer ${token}` }
+          });
+          await execute();
+          const key = await db.attachments?.decryptKey(attachment.key);
+          const file = await saveFile(attachment.metadata.hash, {
+            key,
+            iv: attachment.iv,
+            name: attachment.metadata.filename,
+            type: attachment.metadata.type,
+            isUploaded: !!attachment.dateUploaded
+          });
+          const filePath = `/${attachment.metadata.hash}`;
+          if (file)
+            controller.enqueue({
+              path: filePath,
+              data: file
+            });
+          index++;
+        }
+      }
+    });
+  }
+}
+
+async function saveFile(filename: string, fileMetadata: any) {
+  if (!fileMetadata) return false;
+  const streamablefs = new StreamableFS("streamable-fs");
+
+  const fileHandle = await streamablefs.readFile(filename);
+  if (!fileHandle) return false;
+  const { key, iv, name, type, isUploaded } = fileMetadata;
+
+  const blobParts: Array<any> = [];
+  const reader = fileHandle.getReader();
+
+  const crypto = await getNNCrypto();
+  await crypto.decryptStream(
+    key,
+    iv,
+    {
+      read: async () => {
+        const { value } = await reader.read();
+        return value;
+      },
+      write: async (chunk: any) => {
+        blobParts.push(chunk.data);
+      }
+    },
+    filename
+  );
+
+  if (isUploaded) await streamablefs.deleteFile(filename);
+  return new Blob(blobParts, { type }).arrayBuffer().then((buffer) => {
+    return new Uint8Array(buffer);
+  });
+}
+
+const hosts = {
+  API_HOST: "https://api.notesnook.com"
+};

apps/web/src/common/attachments/mitm.ts

@@ -0,0 +1,116 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+export {};
+
+let sw: ServiceWorker | null = null;
+let scope = "";
+
+function registerWorker() {
+  return navigator.serviceWorker
+    .getRegistration("./")
+    .then((swReg) => {
+      return (
+        swReg || navigator.serviceWorker.register("sw.js", { scope: "./" })
+      );
+    })
+    .then((swReg) => {
+      const swRegTmp = swReg.installing || swReg.waiting;
+
+      scope = swReg.scope;
+      let fn: () => void;
+      return (
+        (sw = swReg.active) ||
+        new Promise((resolve) => {
+          swRegTmp?.addEventListener(
+            "statechange",
+            (fn = () => {
+              if (swRegTmp.state === "activated") {
+                swRegTmp.removeEventListener("statechange", fn);
+                sw = swReg.active;
+                resolve(undefined);
+              }
+            })
+          );
+        })
+      );
+    });
+}
+
+// Now that we have the Service Worker registered we can process messages
+export function postMessage(
+  data: {
+    origin?: string;
+    referrer?: string;
+    headers: Record<string, string>;
+    pathname: string;
+    url?: string;
+    transferringReadable: boolean;
+  },
+  ports: MessagePort[]
+) {
+  // It's important to have a messageChannel, don't want to interfere
+  // with other simultaneous downloads
+  if (!ports || !ports.length) {
+    throw new TypeError("[StreamSaver] You didn't send a messageChannel");
+  }
+
+  if (typeof data !== "object") {
+    throw new TypeError("[StreamSaver] You didn't send a object");
+  }
+
+  // the default public service worker for StreamSaver is shared among others.
+  // so all download links needs to be prefixed to avoid any other conflict
+  data.origin = window.location.origin;
+
+  // if we ever (in some feature versoin of streamsaver) would like to
+  // redirect back to the page of who initiated a http request
+  data.referrer = data.referrer || document.referrer || origin;
+
+  // test if it's correct
+  // should thorw a typeError if not
+  new Headers(data.headers);
+
+  // remove all leading slashes
+  data.pathname = data.pathname.replace(/^\/+/g, "");
+
+  // remove protocol
+  let org = origin.replace(/(^\w+:|^)\/\//, "");
+
+  // set the absolute pathname to the download url.
+  data.url = new URL(`${scope + org}/${data.pathname}`).toString();
+
+  if (!data.url.startsWith(`${scope + org}/`)) {
+    throw new TypeError("[StreamSaver] bad `data.pathname`");
+  }
+
+  // This sends the message data as well as transferring
+  // messageChannel.port2 to the service worker. The service worker can
+  // then use the transferred port to reply via postMessage(), which
+  // will in turn trigger the onmessage handler on messageChannel.port1.
+
+  const transferable = [ports[0]];
+
+  return sw?.postMessage(data, transferable);
+}
+
+export async function register() {
+  if (navigator.serviceWorker) {
+    await registerWorker();
+  }
+}

apps/web/src/common/attachments/stream-saver.ts

@@ -0,0 +1,227 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+import { postMessage } from "./mitm";
+
+let supportsTransferable = false;
+
+const isSecureContext = global.isSecureContext;
+// TODO: Must come up with a real detection test (#69)
+let useBlobFallback =
+  /constructor/i.test(global.HTMLElement.toString()) ||
+  "safari" in globalThis ||
+  "WebKitPoint" in globalThis;
+
+type IFrameContainer = HTMLIFrameElement & {
+  loaded: boolean;
+  isIFrame: boolean;
+  remove: () => void;
+  addEventListener: HTMLIFrameElement["addEventListener"];
+  dispatchEvent: HTMLIFrameElement["dispatchEvent"];
+  removeEventListener: HTMLIFrameElement["removeEventListener"];
+  postMessage(
+    message: any,
+    targetOrigin: string,
+    transfer?: Transferable[] | undefined
+  ): void;
+};
+
+/**
+ * create a hidden iframe and append it to the DOM (body)
+ *
+ * @param  {string} src page to load
+ * @return {HTMLIFrameElement} page to load
+ */
+function makeIframe(src: string, doc = true): IFrameContainer {
+  if (!src) throw new Error("meh");
+
+  const iframe = document.createElement("iframe") as IFrameContainer;
+  iframe.hidden = true;
+  if (doc) iframe.srcdoc = src;
+  else iframe.src = src;
+  iframe.name = "iframe";
+  iframe.loaded = false;
+  iframe.isIFrame = true;
+  iframe.postMessage = (message, targetOrigin, transfer) =>
+    iframe.contentWindow?.postMessage(message, targetOrigin, transfer);
+
+  iframe.addEventListener(
+    "load",
+    () => {
+      iframe.loaded = true;
+    },
+    { once: true }
+  );
+  document.body.appendChild(iframe);
+  return iframe;
+}
+
+try {
+  // We can't look for service worker since it may still work on http
+  new Response(new ReadableStream());
+  if (isSecureContext && !("serviceWorker" in navigator)) {
+    useBlobFallback = true;
+  }
+} catch (err) {
+  useBlobFallback = true;
+}
+
+function checkSupportsTransferable() {
+  // Transferable stream was first enabled in chrome v73 behind a flag
+  const { readable } = new TransformStream();
+  const mc = new MessageChannel();
+  // @ts-ignore
+  mc.port1.postMessage(readable, [readable]);
+  mc.port1.close();
+  mc.port2.close();
+  supportsTransferable = true;
+}
+checkSupportsTransferable();
+
+/**
+ * @param  {string} filename filename that should be used
+ * @param  {object} options  [description]
+ * @param  {number} size     deprecated
+ * @return {WritableStream<Uint8Array>}
+ */
+export function createWriteStream(
+  filename: string,
+  opts: {
+    size?: number;
+    pathname?: string;
+  } = {}
+): WritableStream<Uint8Array> {
+  // let bytesWritten = 0; // by StreamSaver.js (not the service worker)
+  let downloadUrl: string | null = null;
+  let channel: MessageChannel | null = null;
+  let ts = null;
+  if (!useBlobFallback) {
+    channel = new MessageChannel();
+
+    // Make filename RFC5987 compatible
+    filename = encodeURIComponent(filename.replace(/\//g, ":"))
+      .replace(/['()]/g, escape)
+      .replace(/\*/g, "%2A");
+    const response: {
+      transferringReadable: boolean;
+      pathname: string;
+      headers: Record<string, string>;
+    } = {
+      transferringReadable: supportsTransferable,
+      pathname:
+        opts.pathname || Math.random().toString().slice(-6) + "/" + filename,
+      headers: {
+        "Content-Type": "application/octet-stream; charset=utf-8",
+        "Content-Disposition": "attachment; filename*=UTF-8''" + filename
+      }
+    };
+
+    if (opts.size) {
+      response.headers["Content-Length"] = `${opts.size}`;
+    }
+
+    if (supportsTransferable) {
+      ts = new TransformStream();
+      const readableStream = ts.readable;
+
+      // @ts-ignore
+      channel.port1.postMessage({ readableStream }, [readableStream]);
+    }
+    channel.port1.onmessage = async (evt) => {
+      // Service worker sent us a link that we should open.
+      if (evt.data.download) {
+        // We never remove this iframes because it can interrupt saving
+        makeIframe(evt.data.download, false);
+      } else if (evt.data.abort) {
+        chunks = [];
+        if (channel) {
+          channel.port1.postMessage("abort"); //send back so controller is aborted
+          channel.port1.onmessage = null;
+          channel.port1.close();
+          channel.port2.close();
+          channel = null;
+        }
+      }
+    };
+
+    postMessage(response, [channel.port2]);
+  }
+
+  let chunks: Uint8Array[] = [];
+
+  return (
+    (!useBlobFallback && ts && ts.writable) ||
+    new WritableStream({
+      write(chunk) {
+        if (!(chunk instanceof Uint8Array)) {
+          throw new TypeError("Can only write Uint8Arrays");
+        }
+        if (useBlobFallback) {
+          // Safari... The new IE6
+          // https://github.com/jimmywarting/StreamSaver.js/issues/69
+          //
+          // even though it has everything it fails to download anything
+          // that comes from the service worker..!
+          chunks.push(chunk);
+          return;
+        }
+
+        // is called when a new chunk of data is ready to be written
+        // to the underlying sink. It can return a promise to signal
+        // success or failure of the write operation. The stream
+        // implementation guarantees that this method will be called
+        // only after previous writes have succeeded, and never after
+        // close or abort is called.
+
+        // TODO: Kind of important that service worker respond back when
+        // it has been written. Otherwise we can't handle backpressure
+        // EDIT: Transferable streams solves this...
+        channel?.port1.postMessage(chunk);
+        // bytesWritten += chunk.length;
+
+        if (downloadUrl) {
+          window.location.href = downloadUrl;
+          downloadUrl = null;
+        }
+      },
+      close() {
+        if (useBlobFallback) {
+          const blob = new Blob(chunks, {
+            type: "application/octet-stream; charset=utf-8"
+          });
+          const link = document.createElement("a");
+          link.href = URL.createObjectURL(blob);
+          link.download = filename;
+          link.click();
+        } else {
+          channel?.port1.postMessage("end");
+        }
+      },
+      abort() {
+        chunks = [];
+        if (channel) {
+          channel.port1.postMessage("abort");
+          channel.port1.onmessage = null;
+          channel.port1.close();
+          channel.port2.close();
+          channel = null;
+        }
+      }
+    })
+  );
+}

apps/web/src/common/attachments/zip-stream.ts

@@ -0,0 +1,46 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Zip, ZipDeflate } from "fflate";
+
+export type ZipFile = { path: string; data: Uint8Array };
+export class ZipStream extends TransformStream<ZipFile, Uint8Array> {
+  constructor() {
+    const zipper = new Zip();
+    super({
+      start(controller) {
+        zipper.ondata = (err, data) => {
+          if (err) controller.error(err);
+          else controller.enqueue(data);
+        };
+      },
+      transform(chunk) {
+        const fileStream = new ZipDeflate(chunk.path, {
+          level: 5,
+          mem: 8
+        });
+        zipper.add(fileStream);
+        fileStream.push(chunk.data, true);
+      },
+      flush() {
+        zipper.end();
+      }
+    });
+  }
+}

apps/web/src/components/dialogs/attachments-dialog.js

@@ -26,6 +26,9 @@ import Field from "../field";
 import ListContainer from "../list-container";
 import Dialog from "./dialog";
 import Placeholder from "../placeholders";
+import { AttachmentStream } from "../../common/attachments/attachment-stream";
+import { ZipStream } from "../../common/attachments/zip-stream";
+import { createWriteStream } from "../../common/attachments/stream-saver";
 
 function AttachmentsDialog({ onClose }) {
   const attachments = useStore((store) => store.attachments);
@@ -34,7 +37,6 @@ function AttachmentsDialog({ onClose }) {
   useEffect(() => {
     refresh();
   }, [refresh]);
-
   return (
     <Dialog
       isOpen={true}
@@ -46,6 +48,20 @@ function AttachmentsDialog({ onClose }) {
       onClose={onClose}
       noScroll
       negativeButton={{ text: "Close", onClick: onClose }}
+      positiveButton={{
+        text: "Download All Attachments",
+        onClick: async () => {
+          if (navigator.serviceWorker) {
+            console.log(
+              "attachment-dialog",
+              await new AttachmentStream(attachments)
+                .pipeThrough(new ZipStream())
+                .pipeTo(createWriteStream("notesnook-importer.zip"))
+            );
+          }
+        }
+      }}
+      show
     >
       <Flex px={2} sx={{ flexDirection: "column", height: 500 }}>
         <Field

apps/web/src/interfaces/fs.js

@@ -394,6 +394,59 @@ async function downloadFile(filename, requestOptions) {
   }
 }
 
+async function downloadAttachment(filename, requestOptions) {
+  const { url, headers, chunkSize, cancellationToken } = requestOptions;
+  if (await streamablefs.exists(filename)) return true;
+
+  try {
+    const signedUrlResponse = await axios.get(url, {
+      headers,
+      responseType: "text"
+    });
+
+    const signedUrl = signedUrlResponse.data;
+    const response = await axios.get(signedUrl, {
+      responseType: "arraybuffer",
+      cancelToken: cancellationToken
+    });
+
+    const contentType = response.headers["content-type"];
+    if (contentType === "application/xml") {
+      const error = parseS3Error(response.data);
+      if (error.Code !== "Unknown") {
+        throw new Error(`[${error.Code}] ${error.Message}`);
+      }
+    }
+
+    const contentLength = response.headers["content-length"];
+    if (contentLength === "0") {
+      const error = `File length is 0. Please upload this file again from the attachment manager. (File hash: ${filename})`;
+      await db.attachments.markAsFailed(filename, error);
+      throw new Error(error);
+    }
+
+    const distributor = new ChunkDistributor(chunkSize + ABYTES);
+    distributor.fill(new Uint8Array(response.data));
+    distributor.close();
+
+    const fileHandle = await streamablefs.createFile(
+      filename,
+      response.data.byteLength,
+      "application/octet-stream"
+    );
+
+    for (let chunk of distributor.chunks) {
+      await fileHandle.write(chunk.data);
+    }
+
+    return true;
+  } catch (e) {
+    handleS3Error(e, "Could not download file");
+    reportProgress(undefined, { type: "download", hash: filename });
+    return false;
+  }
+}
+
 function exists(filename) {
   return streamablefs.exists(filename);
 }
@@ -475,6 +528,7 @@ const FS = {
   readEncrypted,
   uploadFile: cancellable(uploadFile),
   downloadFile: cancellable(downloadFile),
+  downloadAttachment: cancellable(downloadAttachment),
   deleteFile,
   saveFile,
   exists,

packages/streamable-fs/src/filehandle.ts

@@ -17,10 +17,11 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+import { Chunk } from "@notesnook/crypto/dist/src/types";
 import FileStreamSource from "./filestreamsource";
 import { File } from "./types";
 
-export default class FileHandle extends ReadableStream {
+export default class FileHandle extends ReadableStream<Chunk> {
   private storage: LocalForage;
   private file: File;
 

packages/streamable-fs/src/filestreamsource.ts

@@ -20,7 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { File } from "./types";
 import { Chunk } from "@notesnook/crypto/dist/src/types";
 
-export default class FileStreamSource implements UnderlyingSource<Chunk> {
+export default class FileStreamSource {
   private storage: LocalForage;
   private file: File;
   private offset = 0;
@@ -32,7 +32,7 @@ export default class FileStreamSource implements UnderlyingSource<Chunk> {
 
   start() {}
 
-  async pull(controller: ReadableStreamController<Chunk>) {
+  async pull(controller: ReadableStreamDefaultController<Chunk>) {
     const data = await this.readChunk(this.offset++);
     const isFinalChunk = this.offset === this.file.chunks;