feat: add upload/download cancellation support

packages/core/api/index.js

@@ -53,6 +53,9 @@ class Database {
       [EVENTS.userLoggedIn, EVENTS.userFetched, EVENTS.tokenRefreshed],
       this.connectSSE.bind(this)
     );
+    EV.subscribe(EVENTS.attachmentDeleted, async (attachment) => {
+      await this.fs.cancel(attachment.metadata.hash);
+    });
     EV.subscribe(EVENTS.userLoggedOut, async () => {
       await this.monographs.deinit();
       clearTimeout(this._syncTimeout);

packages/core/api/sync/collector.js

@@ -16,14 +16,10 @@ class Collector {
     this._lastSyncedTimestamp = lastSyncedTimestamp;
     this.key = await this._db.user.getEncryptionKey();
 
-    const contents = await this._db.content.extractAttachments(
-      this._collect(await this._db.content.all())
-    );
-
     return {
       notes: await this._encrypt(this._collect(this._db.notes.raw)),
       notebooks: await this._encrypt(this._collect(this._db.notebooks.raw)),
-      content: await this._encrypt(contents),
+      content: await this._encrypt(this._collect(await this._db.content.all())),
       attachments: await this._encrypt(this._collect(this._db.attachments.all)),
       settings: await this._encrypt(this._collect([this._db.settings.raw])),
       vaultKey: await this._serialize(await this._db.vault._getKey()),

packages/core/api/sync/index.js

@@ -73,7 +73,7 @@ export default class Sync {
       const now = Date.now();
       this._isSyncing = true;
 
-      await this._uploadAttachments(token);
+      await this._uploadAttachments();
 
       // we prepare local data before merging so we always have correct data
       const data = await this._collector.collect(lastSynced);
@@ -151,7 +151,7 @@ export default class Sync {
     return response.lastSynced;
   }
 
-  async _uploadAttachments(token) {
+  async _uploadAttachments() {
     const attachments = this._db.attachments.pending;
     console.log("Uploading attachments", this._db.attachments.pending);
     for (var i = 0; i < attachments.length; ++i) {
@@ -163,7 +163,7 @@ export default class Sync {
       });
       const { hash } = attachment.metadata;
 
-      const isUploaded = await this._db.fs.uploadFile(hash);
+      const isUploaded = await this._db.fs.uploadFile(hash, hash);
       if (!isUploaded) throw new Error("Failed to upload file.");
 
       await this._db.attachments.markAsUploaded(attachment.id);

packages/core/api/sync/merger.js

@@ -120,7 +120,7 @@ class Merger {
 
     await this._mergeArrayWithConflicts(
       content,
-      (id) => this._db.content.raw(id),
+      (id) => this._db.content.raw(id, false),
       (item) => this._db.content.add(item),
       async (local, remote) => {
         let note = this._db.notes.note(local.noteId);

packages/core/api/vault.js

@@ -189,7 +189,7 @@ export default class Vault {
 
   /** @private */
   async _decryptContent(contentId, password) {
-    let encryptedContent = await this._db.content.raw(contentId);
+    let encryptedContent = await this._db.content.raw(contentId, false);
 
     let decryptedContent = await this._storage.decrypt(
       { password },
@@ -215,12 +215,15 @@ export default class Vault {
       data = content.data;
       type = content.type;
     } else {
-      const [content] = await this._db.content.extractAttachments([
+      const content = await this._db.content.extractAttachments({
-        { data, type, noteId: id },
+        data,
-      ]);
+        type,
+        noteId: id,
+      });
       data = content.data;
       type = content.type;
     }
+    console.log("encrypting data:", data);
 
     await this._encryptContent(contentId, data, type, password);
 

packages/core/collections/attachments.js

@@ -4,6 +4,7 @@ import { deleteItem, hasItem } from "../utils/array";
 import hosts from "../utils/constants";
 import { EV, EVENTS } from "../common";
 import dataurl from "../utils/dataurl";
+import dayjs from "dayjs";
 
 export default class Attachments extends Collection {
   constructor(db, name, cached) {
@@ -50,6 +51,7 @@ export default class Attachments extends Collection {
       if (oldAttachment.noteIds.includes(noteId)) return;
 
       oldAttachment.noteIds.push(noteId);
+      oldAttachment.dateDeleted = undefined;
       return this._collection.updateItem(oldAttachment);
     }
 
@@ -81,11 +83,10 @@ export default class Attachments extends Collection {
     const attachment = this.all.find((a) => a.metadata.hash === hash);
     if (!attachment || !deleteItem(attachment.noteIds, noteId)) return;
     if (!attachment.noteIds.length) {
-      const isDeleted = await this._db.fs.deleteFile(attachment.metadata.hash);
-      if (!isDeleted) return;
       attachment.dateDeleted = Date.now();
+      EV.publish(EVENTS.attachmentDeleted, attachment);
     }
-    return this._collection.updateItem(attachment);
+    return await this._collection.updateItem(attachment);
   }
 
   async get(hash) {
@@ -141,10 +142,7 @@ export default class Attachments extends Collection {
 
       const { hash } = attachments[i].metadata;
 
-      const attachmentExists = await this._db.fs.exists(hash);
+      const isDownloaded = await this._db.fs.downloadFile(noteId, hash);
-      if (attachmentExists) continue;
-
-      const isDownloaded = await this._db.fs.downloadFile(hash);
       if (!isDownloaded) continue;
 
       const attachment = await this.get(hash);
@@ -165,6 +163,22 @@ export default class Attachments extends Collection {
     });
   }
 
+  get deleted() {
+    return this.all.filter((attachment) => attachment.dateDeleted > 0);
+  }
+
+  async cleanup() {
+    const now = dayjs().unix();
+    for (const attachment of this.deleted) {
+      if (dayjs(attachment.dateDeleted).add(7, "days").unix() < now) continue;
+
+      const isDeleted = await this._db.fs.deleteFile(attachment.metadata.hash);
+      if (!isDeleted) continue;
+
+      await this._collection.removeItem(attachment.id);
+    }
+  }
+
   get pending() {
     return this.all.filter(
       (attachment) => attachment.dateUploaded <= 0 || !attachment.dateUploaded

packages/core/collections/content.js

@@ -1,7 +1,8 @@
 import Collection from "./collection";
 import getId from "../utils/id";
 import { getContentFromData } from "../content-types";
-import { diff, hasItem } from "../utils/array";
+import { hasItem } from "../utils/array";
+import { diffArrays } from "diff";
 
 export default class Content extends Collection {
   async add(content) {
@@ -18,18 +19,20 @@ export default class Content extends Collection {
     }
 
     const id = content.id || getId();
-    await this._collection.addItem({
+    await this._collection.addItem(
-      noteId: content.noteId,
+      await this.extractAttachments({
-      id,
+        noteId: content.noteId,
-      type: content.type,
+        id,
-      data: content.data || content,
+        type: content.type,
-      dateEdited: content.dateEdited,
+        data: content.data || content,
-      dateCreated: content.dateCreated,
+        dateEdited: content.dateEdited,
-      remote: content.remote,
+        dateCreated: content.dateCreated,
-      localOnly: !!content.localOnly,
+        remote: content.remote,
-      conflicted: content.conflicted,
+        localOnly: !!content.localOnly,
-      dateResolved: content.dateResolved,
+        conflicted: content.conflicted,
-    });
+        dateResolved: content.dateResolved,
+      })
+    );
     return id;
   }
 
@@ -68,32 +71,38 @@ export default class Content extends Collection {
 
   /**
    *
-   * @param {Array} contents
+   * @param {any} contentItem
-   * @returns {Promise<Array>}
+   * @returns {Promise<any>}
    */
-  async extractAttachments(contents) {
+  async extractAttachments(contentItem) {
+    if (contentItem.localOnly || typeof contentItem.data !== "string")
+      return contentItem;
+
     const allAttachments = this._db.attachments.all;
-    for (let contentItem of contents) {
+    const content = getContentFromData(contentItem.type, contentItem.data);
-      const content = getContentFromData(contentItem.type, contentItem.data);
+    const { data, attachments } = await content.extractAttachments(
-      const { data, attachments } = await content.extractAttachments(
+      (data, type) => this._db.attachments.save(data, type)
-        (data, type) => this._db.attachments.save(data, type)
+    );
-      );
 
-      await diff(allAttachments, attachments, async (attachment, action) => {
+    const diff = diffArrays(allAttachments, attachments, {
-        if (hasItem(attachment.noteIds, contentItem.noteId)) return;
+      comparator: (left, right) => left.hash === right.metadata.hash,
+    });
 
-        if (action === "delete") {
+    for (const change of diff) {
+      for (let attachment of change.value) {
+        const exists = hasItem(attachment.noteIds, contentItem.noteId);
+        if (change.removed && exists) {
           await this._db.attachments.delete(
-            attachment.hash,
+            attachment.metadata.hash,
             contentItem.noteId
           );
-        } else if (action === "insert") {
+        } else if ((!change.removed || change.added) && !exists) {
           await this._db.attachments.add(attachment, contentItem.noteId);
         }
-      });
+      }
-
-      contentItem.data = data;
     }
-    return contents;
+
+    contentItem.data = data;
+    return contentItem;
   }
 }

packages/core/collections/trash.js

@@ -19,11 +19,11 @@ export default class Trash {
   }
 
   async cleanup() {
-    this.all.forEach(async (item) => {
+    const now = dayjs().unix();
-      if (dayjs(item.dateDeleted).add(7, "days").isBefore(dayjs())) {
+    for (const item of this.all) {
-        await this.delete(item.id);
+      if (dayjs(item.dateDeleted).add(7, "days").unix() < now) continue;
-      }
+      await this.delete(item.id);
-    });
+    }
   }
 
   get all() {

packages/core/common.js

@@ -38,6 +38,7 @@ export const EVENTS = {
   noteRemoved: "note:removed",
   tokenRefreshed: "token:refreshed",
   attachmentsLoading: "attachments:loading",
+  attachmentDeleted: "attachment:deleted",
   mediaAttachmentDownloaded: "attachments:mediaDownloaded",
 };
 

packages/core/database/fs.js

@@ -5,21 +5,36 @@ export default class FileStorage {
   constructor(fs, storage) {
     this.fs = fs;
     this.tokenManager = new TokenManager(storage);
+    this._queue = [];
   }
 
-  async downloadFile(hash) {
+  async downloadFile(groupId, hash) {
     const url = `${hosts.API_HOST}/s3?name=${hash}`;
     const token = await this.tokenManager.getAccessToken();
-    return await this.fs.downloadFile(hash, {
+    const { execute, cancel } = this.fs.downloadFile(hash, {
       url,
       headers: { Authorization: `Bearer ${token}` },
     });
+    this._queue.push({ groupId, hash, cancel, type: "download" });
+    return await execute();
   }
 
-  async uploadFile(hash) {
+  async uploadFile(groupId, hash) {
     const token = await this.tokenManager.getAccessToken();
     const url = await this._getPresignedURL(hash, token, "PUT");
-    return await this.fs.uploadFile(hash, { url });
+    const { execute, cancel } = this.fs.uploadFile(hash, { url });
+    this._queue.push({ groupId, hash, cancel, type: "upload" });
+    return await execute();
+  }
+
+  async cancel(groupId, type = undefined) {
+    await Promise.all(
+      this._queue
+        .filter(
+          (item) => item.groupId === groupId && (!type || item.type === type)
+        )
+        .map(async (op) => await op.cancel())
+    );
   }
 
   readEncrypted(filename, encryptionKey, cipherData) {
@@ -27,7 +42,7 @@ export default class FileStorage {
   }
 
   writeEncrypted(filename, data, type, encryptionKey) {
-    return this._db.fs.writeEncrypted(filename, {
+    return this.fs.writeEncrypted(filename, {
       data,
       type,
       key: encryptionKey,

packages/core/package-lock.json

@@ -10,8 +10,8 @@
       "dependencies": {
         "@stablelib/blake2s": "^1.0.1",
         "dayjs": "^1.10.6",
+        "diff": "^5.0.0",
         "fast-sort": "^2.0.1",
-        "lean-he": "^2.1.2",
         "no-internet": "^1.5.2",
         "qclone": "^1.0.4",
         "quill-delta-to-html": "^0.12.0",
@@ -3575,6 +3575,14 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/diff": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz",
+      "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==",
+      "engines": {
+        "node": ">=0.3.1"
+      }
+    },
     "node_modules/diff-sequences": {
       "version": "24.9.0",
       "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-24.9.0.tgz",
@@ -5600,11 +5608,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/lean-he": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/lean-he/-/lean-he-2.1.2.tgz",
-      "integrity": "sha512-g/cq01j/rnv7JWoxFmeLgJdd/CucksyDtS+pyepO89EdT0O4KfHJokOVz/xQ4mvjKJzcrj87Q3/s2ESou90WCQ=="
-    },
     "node_modules/left-pad": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz",
@@ -11204,6 +11207,11 @@
       "integrity": "sha1-9B8cEL5LAOh7XxPaaAdZ8sW/0+I=",
       "dev": true
     },
+    "diff": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz",
+      "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w=="
+    },
     "diff-sequences": {
       "version": "24.9.0",
       "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-24.9.0.tgz",
@@ -12815,11 +12823,6 @@
       "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==",
       "dev": true
     },
-    "lean-he": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/lean-he/-/lean-he-2.1.2.tgz",
-      "integrity": "sha512-g/cq01j/rnv7JWoxFmeLgJdd/CucksyDtS+pyepO89EdT0O4KfHJokOVz/xQ4mvjKJzcrj87Q3/s2ESou90WCQ=="
-    },
     "left-pad": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz",

packages/core/package.json

@@ -25,6 +25,7 @@
   "dependencies": {
     "@stablelib/blake2s": "^1.0.1",
     "dayjs": "^1.10.6",
+    "diff": "^5.0.0",
     "fast-sort": "^2.0.1",
     "no-internet": "^1.5.2",
     "qclone": "^1.0.4",

packages/core/utils/array.js

@@ -29,21 +29,6 @@ export function hasItem(array, item) {
   return array.indexOf(item) > -1;
 }
 
-export async function diff(arr1, arr2, action) {
-  let length = arr1.length + arr2.length;
-  for (var i = 0; i < length; ++i) {
-    var actionKey = "delete";
-    var item = arr1[i];
-
-    if (i >= arr1.length) {
-      var actionKey = "insert";
-      var item = arr2[i - arr1.length];
-    }
-
-    await action(item, actionKey);
-  }
-}
-
 function deleteAtIndex(array, index) {
   if (index === -1) return false;
   array.splice(index, 1);

packages/core/yarn.lock

@@ -2330,6 +2330,11 @@
   "resolved" "https://registry.npmjs.org/diff-sequences/-/diff-sequences-25.2.6.tgz"
   "version" "25.2.6"
 
+"diff@^5.0.0":
+  "integrity" "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w=="
+  "resolved" "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz"
+  "version" "5.0.0"
+
 "domexception@^1.0.1":
   "integrity" "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug=="
   "resolved" "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz"
@@ -3646,11 +3651,6 @@
   "resolved" "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz"
   "version" "3.0.3"
 
-"lean-he@^2.1.2":
-  "integrity" "sha512-g/cq01j/rnv7JWoxFmeLgJdd/CucksyDtS+pyepO89EdT0O4KfHJokOVz/xQ4mvjKJzcrj87Q3/s2ESou90WCQ=="
-  "resolved" "https://registry.npmjs.org/lean-he/-/lean-he-2.1.2.tgz"
-  "version" "2.1.2"
-
 "left-pad@^1.3.0":
   "integrity" "sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA=="
   "resolved" "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"