From 6f88b7093793cc7729b4669b8f3872573153c2f7 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Thu, 30 Mar 2023 16:42:40 +0500 Subject: [PATCH] web: add support for downloading all attachments as zip --- .../common/attachments/attachment-stream.ts | 108 +++++++++ apps/web/src/common/attachments/mitm.ts | 116 +++++++++ .../src/common/attachments/stream-saver.ts | 227 ++++++++++++++++++ apps/web/src/common/attachments/zip-stream.ts | 46 ++++ .../components/dialogs/attachments-dialog.js | 18 +- apps/web/src/interfaces/fs.js | 54 +++++ packages/streamable-fs/src/filehandle.ts | 3 +- .../streamable-fs/src/filestreamsource.ts | 4 +- 8 files changed, 572 insertions(+), 4 deletions(-) create mode 100644 apps/web/src/common/attachments/attachment-stream.ts create mode 100644 apps/web/src/common/attachments/mitm.ts create mode 100644 apps/web/src/common/attachments/stream-saver.ts create mode 100644 apps/web/src/common/attachments/zip-stream.ts diff --git a/apps/web/src/common/attachments/attachment-stream.ts b/apps/web/src/common/attachments/attachment-stream.ts new file mode 100644 index 000000000..3be132652 --- /dev/null +++ b/apps/web/src/common/attachments/attachment-stream.ts @@ -0,0 +1,108 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import { StreamableFS } from "@notesnook/streamable-fs"; +import FS from "../../interfaces/fs"; +import { getNNCrypto } from "../../interfaces/nncrypto.stub"; +import { db } from "../db"; +import { ZipFile } from "./zip-stream"; + +export type PackageMetadata = { + version: string; + attachments: string[]; +}; + +export const METADATA_FILENAME = "metadata.json"; + +export class AttachmentStream extends ReadableStream { + constructor(attachments: Array) { + super({ + async pull(controller) { + const token = await db.fs.tokenManager.getAccessToken(); + let index = 0; + for (const attachment of attachments) { + if (index > 2) { + return controller.close(); + } + const url = `${hosts.API_HOST}/s3?name=${attachment.metadata.hash}`; + + const { execute } = FS.downloadAttachment(attachment.metadata.hash, { + metadata: attachment.metadata, + url, + chunkSize: attachment.chunkSize, + headers: { Authorization: `Bearer ${token}` } + }); + await execute(); + const key = await db.attachments?.decryptKey(attachment.key); + const file = await saveFile(attachment.metadata.hash, { + key, + iv: attachment.iv, + name: attachment.metadata.filename, + type: attachment.metadata.type, + isUploaded: !!attachment.dateUploaded + }); + const filePath = `/${attachment.metadata.hash}`; + if (file) + controller.enqueue({ + path: filePath, + data: file + }); + index++; + } + } + }); + } +} + +async function saveFile(filename: string, fileMetadata: any) { + if (!fileMetadata) return false; + const streamablefs = new StreamableFS("streamable-fs"); + + const fileHandle = await streamablefs.readFile(filename); + if (!fileHandle) return false; + const { key, iv, name, type, isUploaded } = fileMetadata; + + const blobParts: Array = []; + const reader = fileHandle.getReader(); + + const crypto = await getNNCrypto(); + await crypto.decryptStream( + key, + iv, + { + read: async () => { + const { value } = await reader.read(); + return value; + }, + write: async (chunk: any) => { + blobParts.push(chunk.data); + } + }, + filename + ); + + if (isUploaded) await streamablefs.deleteFile(filename); + return new Blob(blobParts, { type }).arrayBuffer().then((buffer) => { + return new Uint8Array(buffer); + }); +} + +const hosts = { + API_HOST: "https://api.notesnook.com" +}; diff --git a/apps/web/src/common/attachments/mitm.ts b/apps/web/src/common/attachments/mitm.ts new file mode 100644 index 000000000..4de9a3169 --- /dev/null +++ b/apps/web/src/common/attachments/mitm.ts @@ -0,0 +1,116 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +export {}; + +let sw: ServiceWorker | null = null; +let scope = ""; + +function registerWorker() { + return navigator.serviceWorker + .getRegistration("./") + .then((swReg) => { + return ( + swReg || navigator.serviceWorker.register("sw.js", { scope: "./" }) + ); + }) + .then((swReg) => { + const swRegTmp = swReg.installing || swReg.waiting; + + scope = swReg.scope; + let fn: () => void; + return ( + (sw = swReg.active) || + new Promise((resolve) => { + swRegTmp?.addEventListener( + "statechange", + (fn = () => { + if (swRegTmp.state === "activated") { + swRegTmp.removeEventListener("statechange", fn); + sw = swReg.active; + resolve(undefined); + } + }) + ); + }) + ); + }); +} + +// Now that we have the Service Worker registered we can process messages +export function postMessage( + data: { + origin?: string; + referrer?: string; + headers: Record; + pathname: string; + url?: string; + transferringReadable: boolean; + }, + ports: MessagePort[] +) { + // It's important to have a messageChannel, don't want to interfere + // with other simultaneous downloads + if (!ports || !ports.length) { + throw new TypeError("[StreamSaver] You didn't send a messageChannel"); + } + + if (typeof data !== "object") { + throw new TypeError("[StreamSaver] You didn't send a object"); + } + + // the default public service worker for StreamSaver is shared among others. + // so all download links needs to be prefixed to avoid any other conflict + data.origin = window.location.origin; + + // if we ever (in some feature versoin of streamsaver) would like to + // redirect back to the page of who initiated a http request + data.referrer = data.referrer || document.referrer || origin; + + // test if it's correct + // should thorw a typeError if not + new Headers(data.headers); + + // remove all leading slashes + data.pathname = data.pathname.replace(/^\/+/g, ""); + + // remove protocol + let org = origin.replace(/(^\w+:|^)\/\//, ""); + + // set the absolute pathname to the download url. + data.url = new URL(`${scope + org}/${data.pathname}`).toString(); + + if (!data.url.startsWith(`${scope + org}/`)) { + throw new TypeError("[StreamSaver] bad `data.pathname`"); + } + + // This sends the message data as well as transferring + // messageChannel.port2 to the service worker. The service worker can + // then use the transferred port to reply via postMessage(), which + // will in turn trigger the onmessage handler on messageChannel.port1. + + const transferable = [ports[0]]; + + return sw?.postMessage(data, transferable); +} + +export async function register() { + if (navigator.serviceWorker) { + await registerWorker(); + } +} diff --git a/apps/web/src/common/attachments/stream-saver.ts b/apps/web/src/common/attachments/stream-saver.ts new file mode 100644 index 000000000..1abdda2bc --- /dev/null +++ b/apps/web/src/common/attachments/stream-saver.ts @@ -0,0 +1,227 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +import { postMessage } from "./mitm"; + +let supportsTransferable = false; + +const isSecureContext = global.isSecureContext; +// TODO: Must come up with a real detection test (#69) +let useBlobFallback = + /constructor/i.test(global.HTMLElement.toString()) || + "safari" in globalThis || + "WebKitPoint" in globalThis; + +type IFrameContainer = HTMLIFrameElement & { + loaded: boolean; + isIFrame: boolean; + remove: () => void; + addEventListener: HTMLIFrameElement["addEventListener"]; + dispatchEvent: HTMLIFrameElement["dispatchEvent"]; + removeEventListener: HTMLIFrameElement["removeEventListener"]; + postMessage( + message: any, + targetOrigin: string, + transfer?: Transferable[] | undefined + ): void; +}; + +/** + * create a hidden iframe and append it to the DOM (body) + * + * @param {string} src page to load + * @return {HTMLIFrameElement} page to load + */ +function makeIframe(src: string, doc = true): IFrameContainer { + if (!src) throw new Error("meh"); + + const iframe = document.createElement("iframe") as IFrameContainer; + iframe.hidden = true; + if (doc) iframe.srcdoc = src; + else iframe.src = src; + iframe.name = "iframe"; + iframe.loaded = false; + iframe.isIFrame = true; + iframe.postMessage = (message, targetOrigin, transfer) => + iframe.contentWindow?.postMessage(message, targetOrigin, transfer); + + iframe.addEventListener( + "load", + () => { + iframe.loaded = true; + }, + { once: true } + ); + document.body.appendChild(iframe); + return iframe; +} + +try { + // We can't look for service worker since it may still work on http + new Response(new ReadableStream()); + if (isSecureContext && !("serviceWorker" in navigator)) { + useBlobFallback = true; + } +} catch (err) { + useBlobFallback = true; +} + +function checkSupportsTransferable() { + // Transferable stream was first enabled in chrome v73 behind a flag + const { readable } = new TransformStream(); + const mc = new MessageChannel(); + // @ts-ignore + mc.port1.postMessage(readable, [readable]); + mc.port1.close(); + mc.port2.close(); + supportsTransferable = true; +} +checkSupportsTransferable(); + +/** + * @param {string} filename filename that should be used + * @param {object} options [description] + * @param {number} size deprecated + * @return {WritableStream} + */ +export function createWriteStream( + filename: string, + opts: { + size?: number; + pathname?: string; + } = {} +): WritableStream { + // let bytesWritten = 0; // by StreamSaver.js (not the service worker) + let downloadUrl: string | null = null; + let channel: MessageChannel | null = null; + let ts = null; + if (!useBlobFallback) { + channel = new MessageChannel(); + + // Make filename RFC5987 compatible + filename = encodeURIComponent(filename.replace(/\//g, ":")) + .replace(/['()]/g, escape) + .replace(/\*/g, "%2A"); + const response: { + transferringReadable: boolean; + pathname: string; + headers: Record; + } = { + transferringReadable: supportsTransferable, + pathname: + opts.pathname || Math.random().toString().slice(-6) + "/" + filename, + headers: { + "Content-Type": "application/octet-stream; charset=utf-8", + "Content-Disposition": "attachment; filename*=UTF-8''" + filename + } + }; + + if (opts.size) { + response.headers["Content-Length"] = `${opts.size}`; + } + + if (supportsTransferable) { + ts = new TransformStream(); + const readableStream = ts.readable; + + // @ts-ignore + channel.port1.postMessage({ readableStream }, [readableStream]); + } + channel.port1.onmessage = async (evt) => { + // Service worker sent us a link that we should open. + if (evt.data.download) { + // We never remove this iframes because it can interrupt saving + makeIframe(evt.data.download, false); + } else if (evt.data.abort) { + chunks = []; + if (channel) { + channel.port1.postMessage("abort"); //send back so controller is aborted + channel.port1.onmessage = null; + channel.port1.close(); + channel.port2.close(); + channel = null; + } + } + }; + + postMessage(response, [channel.port2]); + } + + let chunks: Uint8Array[] = []; + + return ( + (!useBlobFallback && ts && ts.writable) || + new WritableStream({ + write(chunk) { + if (!(chunk instanceof Uint8Array)) { + throw new TypeError("Can only write Uint8Arrays"); + } + if (useBlobFallback) { + // Safari... The new IE6 + // https://github.com/jimmywarting/StreamSaver.js/issues/69 + // + // even though it has everything it fails to download anything + // that comes from the service worker..! + chunks.push(chunk); + return; + } + + // is called when a new chunk of data is ready to be written + // to the underlying sink. It can return a promise to signal + // success or failure of the write operation. The stream + // implementation guarantees that this method will be called + // only after previous writes have succeeded, and never after + // close or abort is called. + + // TODO: Kind of important that service worker respond back when + // it has been written. Otherwise we can't handle backpressure + // EDIT: Transferable streams solves this... + channel?.port1.postMessage(chunk); + // bytesWritten += chunk.length; + + if (downloadUrl) { + window.location.href = downloadUrl; + downloadUrl = null; + } + }, + close() { + if (useBlobFallback) { + const blob = new Blob(chunks, { + type: "application/octet-stream; charset=utf-8" + }); + const link = document.createElement("a"); + link.href = URL.createObjectURL(blob); + link.download = filename; + link.click(); + } else { + channel?.port1.postMessage("end"); + } + }, + abort() { + chunks = []; + if (channel) { + channel.port1.postMessage("abort"); + channel.port1.onmessage = null; + channel.port1.close(); + channel.port2.close(); + channel = null; + } + } + }) + ); +} diff --git a/apps/web/src/common/attachments/zip-stream.ts b/apps/web/src/common/attachments/zip-stream.ts new file mode 100644 index 000000000..42ab8d953 --- /dev/null +++ b/apps/web/src/common/attachments/zip-stream.ts @@ -0,0 +1,46 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import { Zip, ZipDeflate } from "fflate"; + +export type ZipFile = { path: string; data: Uint8Array }; +export class ZipStream extends TransformStream { + constructor() { + const zipper = new Zip(); + super({ + start(controller) { + zipper.ondata = (err, data) => { + if (err) controller.error(err); + else controller.enqueue(data); + }; + }, + transform(chunk) { + const fileStream = new ZipDeflate(chunk.path, { + level: 5, + mem: 8 + }); + zipper.add(fileStream); + fileStream.push(chunk.data, true); + }, + flush() { + zipper.end(); + } + }); + } +} diff --git a/apps/web/src/components/dialogs/attachments-dialog.js b/apps/web/src/components/dialogs/attachments-dialog.js index 53c698a03..e2731ef51 100644 --- a/apps/web/src/components/dialogs/attachments-dialog.js +++ b/apps/web/src/components/dialogs/attachments-dialog.js @@ -26,6 +26,9 @@ import Field from "../field"; import ListContainer from "../list-container"; import Dialog from "./dialog"; import Placeholder from "../placeholders"; +import { AttachmentStream } from "../../common/attachments/attachment-stream"; +import { ZipStream } from "../../common/attachments/zip-stream"; +import { createWriteStream } from "../../common/attachments/stream-saver"; function AttachmentsDialog({ onClose }) { const attachments = useStore((store) => store.attachments); @@ -34,7 +37,6 @@ function AttachmentsDialog({ onClose }) { useEffect(() => { refresh(); }, [refresh]); - return ( { + if (navigator.serviceWorker) { + console.log( + "attachment-dialog", + await new AttachmentStream(attachments) + .pipeThrough(new ZipStream()) + .pipeTo(createWriteStream("notesnook-importer.zip")) + ); + } + } + }} + show > . */ +import { Chunk } from "@notesnook/crypto/dist/src/types"; import FileStreamSource from "./filestreamsource"; import { File } from "./types"; -export default class FileHandle extends ReadableStream { +export default class FileHandle extends ReadableStream { private storage: LocalForage; private file: File; diff --git a/packages/streamable-fs/src/filestreamsource.ts b/packages/streamable-fs/src/filestreamsource.ts index 53815d6dd..2b1e08b3d 100644 --- a/packages/streamable-fs/src/filestreamsource.ts +++ b/packages/streamable-fs/src/filestreamsource.ts @@ -20,7 +20,7 @@ along with this program. If not, see . import { File } from "./types"; import { Chunk } from "@notesnook/crypto/dist/src/types"; -export default class FileStreamSource implements UnderlyingSource { +export default class FileStreamSource { private storage: LocalForage; private file: File; private offset = 0; @@ -32,7 +32,7 @@ export default class FileStreamSource implements UnderlyingSource { start() {} - async pull(controller: ReadableStreamController) { + async pull(controller: ReadableStreamDefaultController) { const data = await this.readChunk(this.offset++); const isFinalChunk = this.offset === this.file.chunks;