diff --git a/packages/core/api/index.js b/packages/core/api/index.js index 55bed9c92..91dcef219 100644 --- a/packages/core/api/index.js +++ b/packages/core/api/index.js @@ -53,6 +53,9 @@ class Database { [EVENTS.userLoggedIn, EVENTS.userFetched, EVENTS.tokenRefreshed], this.connectSSE.bind(this) ); + EV.subscribe(EVENTS.attachmentDeleted, async (attachment) => { + await this.fs.cancel(attachment.metadata.hash); + }); EV.subscribe(EVENTS.userLoggedOut, async () => { await this.monographs.deinit(); clearTimeout(this._syncTimeout); diff --git a/packages/core/api/sync/collector.js b/packages/core/api/sync/collector.js index fae23c244..409f80714 100644 --- a/packages/core/api/sync/collector.js +++ b/packages/core/api/sync/collector.js @@ -16,14 +16,10 @@ class Collector { this._lastSyncedTimestamp = lastSyncedTimestamp; this.key = await this._db.user.getEncryptionKey(); - const contents = await this._db.content.extractAttachments( - this._collect(await this._db.content.all()) - ); - return { notes: await this._encrypt(this._collect(this._db.notes.raw)), notebooks: await this._encrypt(this._collect(this._db.notebooks.raw)), - content: await this._encrypt(contents), + content: await this._encrypt(this._collect(await this._db.content.all())), attachments: await this._encrypt(this._collect(this._db.attachments.all)), settings: await this._encrypt(this._collect([this._db.settings.raw])), vaultKey: await this._serialize(await this._db.vault._getKey()), diff --git a/packages/core/api/sync/index.js b/packages/core/api/sync/index.js index 38a2a170d..90b4d3671 100644 --- a/packages/core/api/sync/index.js +++ b/packages/core/api/sync/index.js @@ -73,7 +73,7 @@ export default class Sync { const now = Date.now(); this._isSyncing = true; - await this._uploadAttachments(token); + await this._uploadAttachments(); // we prepare local data before merging so we always have correct data const data = await this._collector.collect(lastSynced); @@ -151,7 +151,7 @@ export default class Sync { return response.lastSynced; } - async _uploadAttachments(token) { + async _uploadAttachments() { const attachments = this._db.attachments.pending; console.log("Uploading attachments", this._db.attachments.pending); for (var i = 0; i < attachments.length; ++i) { @@ -163,7 +163,7 @@ export default class Sync { }); const { hash } = attachment.metadata; - const isUploaded = await this._db.fs.uploadFile(hash); + const isUploaded = await this._db.fs.uploadFile(hash, hash); if (!isUploaded) throw new Error("Failed to upload file."); await this._db.attachments.markAsUploaded(attachment.id); diff --git a/packages/core/api/sync/merger.js b/packages/core/api/sync/merger.js index ee2ed207e..18c3e9fb7 100644 --- a/packages/core/api/sync/merger.js +++ b/packages/core/api/sync/merger.js @@ -120,7 +120,7 @@ class Merger { await this._mergeArrayWithConflicts( content, - (id) => this._db.content.raw(id), + (id) => this._db.content.raw(id, false), (item) => this._db.content.add(item), async (local, remote) => { let note = this._db.notes.note(local.noteId); diff --git a/packages/core/api/vault.js b/packages/core/api/vault.js index 56f08adfd..6e4da674b 100644 --- a/packages/core/api/vault.js +++ b/packages/core/api/vault.js @@ -189,7 +189,7 @@ export default class Vault { /** @private */ async _decryptContent(contentId, password) { - let encryptedContent = await this._db.content.raw(contentId); + let encryptedContent = await this._db.content.raw(contentId, false); let decryptedContent = await this._storage.decrypt( { password }, @@ -215,12 +215,15 @@ export default class Vault { data = content.data; type = content.type; } else { - const [content] = await this._db.content.extractAttachments([ - { data, type, noteId: id }, - ]); + const content = await this._db.content.extractAttachments({ + data, + type, + noteId: id, + }); data = content.data; type = content.type; } + console.log("encrypting data:", data); await this._encryptContent(contentId, data, type, password); diff --git a/packages/core/collections/attachments.js b/packages/core/collections/attachments.js index 7d73b7c7b..e1e491257 100644 --- a/packages/core/collections/attachments.js +++ b/packages/core/collections/attachments.js @@ -4,6 +4,7 @@ import { deleteItem, hasItem } from "../utils/array"; import hosts from "../utils/constants"; import { EV, EVENTS } from "../common"; import dataurl from "../utils/dataurl"; +import dayjs from "dayjs"; export default class Attachments extends Collection { constructor(db, name, cached) { @@ -50,6 +51,7 @@ export default class Attachments extends Collection { if (oldAttachment.noteIds.includes(noteId)) return; oldAttachment.noteIds.push(noteId); + oldAttachment.dateDeleted = undefined; return this._collection.updateItem(oldAttachment); } @@ -81,11 +83,10 @@ export default class Attachments extends Collection { const attachment = this.all.find((a) => a.metadata.hash === hash); if (!attachment || !deleteItem(attachment.noteIds, noteId)) return; if (!attachment.noteIds.length) { - const isDeleted = await this._db.fs.deleteFile(attachment.metadata.hash); - if (!isDeleted) return; attachment.dateDeleted = Date.now(); + EV.publish(EVENTS.attachmentDeleted, attachment); } - return this._collection.updateItem(attachment); + return await this._collection.updateItem(attachment); } async get(hash) { @@ -141,10 +142,7 @@ export default class Attachments extends Collection { const { hash } = attachments[i].metadata; - const attachmentExists = await this._db.fs.exists(hash); - if (attachmentExists) continue; - - const isDownloaded = await this._db.fs.downloadFile(hash); + const isDownloaded = await this._db.fs.downloadFile(noteId, hash); if (!isDownloaded) continue; const attachment = await this.get(hash); @@ -165,6 +163,22 @@ export default class Attachments extends Collection { }); } + get deleted() { + return this.all.filter((attachment) => attachment.dateDeleted > 0); + } + + async cleanup() { + const now = dayjs().unix(); + for (const attachment of this.deleted) { + if (dayjs(attachment.dateDeleted).add(7, "days").unix() < now) continue; + + const isDeleted = await this._db.fs.deleteFile(attachment.metadata.hash); + if (!isDeleted) continue; + + await this._collection.removeItem(attachment.id); + } + } + get pending() { return this.all.filter( (attachment) => attachment.dateUploaded <= 0 || !attachment.dateUploaded diff --git a/packages/core/collections/content.js b/packages/core/collections/content.js index 715f08535..8532501ca 100644 --- a/packages/core/collections/content.js +++ b/packages/core/collections/content.js @@ -1,7 +1,8 @@ import Collection from "./collection"; import getId from "../utils/id"; import { getContentFromData } from "../content-types"; -import { diff, hasItem } from "../utils/array"; +import { hasItem } from "../utils/array"; +import { diffArrays } from "diff"; export default class Content extends Collection { async add(content) { @@ -18,18 +19,20 @@ export default class Content extends Collection { } const id = content.id || getId(); - await this._collection.addItem({ - noteId: content.noteId, - id, - type: content.type, - data: content.data || content, - dateEdited: content.dateEdited, - dateCreated: content.dateCreated, - remote: content.remote, - localOnly: !!content.localOnly, - conflicted: content.conflicted, - dateResolved: content.dateResolved, - }); + await this._collection.addItem( + await this.extractAttachments({ + noteId: content.noteId, + id, + type: content.type, + data: content.data || content, + dateEdited: content.dateEdited, + dateCreated: content.dateCreated, + remote: content.remote, + localOnly: !!content.localOnly, + conflicted: content.conflicted, + dateResolved: content.dateResolved, + }) + ); return id; } @@ -68,32 +71,38 @@ export default class Content extends Collection { /** * - * @param {Array} contents - * @returns {Promise} + * @param {any} contentItem + * @returns {Promise} */ - async extractAttachments(contents) { + async extractAttachments(contentItem) { + if (contentItem.localOnly || typeof contentItem.data !== "string") + return contentItem; + const allAttachments = this._db.attachments.all; - for (let contentItem of contents) { - const content = getContentFromData(contentItem.type, contentItem.data); - const { data, attachments } = await content.extractAttachments( - (data, type) => this._db.attachments.save(data, type) - ); + const content = getContentFromData(contentItem.type, contentItem.data); + const { data, attachments } = await content.extractAttachments( + (data, type) => this._db.attachments.save(data, type) + ); - await diff(allAttachments, attachments, async (attachment, action) => { - if (hasItem(attachment.noteIds, contentItem.noteId)) return; + const diff = diffArrays(allAttachments, attachments, { + comparator: (left, right) => left.hash === right.metadata.hash, + }); - if (action === "delete") { + for (const change of diff) { + for (let attachment of change.value) { + const exists = hasItem(attachment.noteIds, contentItem.noteId); + if (change.removed && exists) { await this._db.attachments.delete( - attachment.hash, + attachment.metadata.hash, contentItem.noteId ); - } else if (action === "insert") { + } else if ((!change.removed || change.added) && !exists) { await this._db.attachments.add(attachment, contentItem.noteId); } - }); - - contentItem.data = data; + } } - return contents; + + contentItem.data = data; + return contentItem; } } diff --git a/packages/core/collections/trash.js b/packages/core/collections/trash.js index 9f859448c..3f8b660fd 100644 --- a/packages/core/collections/trash.js +++ b/packages/core/collections/trash.js @@ -19,11 +19,11 @@ export default class Trash { } async cleanup() { - this.all.forEach(async (item) => { - if (dayjs(item.dateDeleted).add(7, "days").isBefore(dayjs())) { - await this.delete(item.id); - } - }); + const now = dayjs().unix(); + for (const item of this.all) { + if (dayjs(item.dateDeleted).add(7, "days").unix() < now) continue; + await this.delete(item.id); + } } get all() { diff --git a/packages/core/common.js b/packages/core/common.js index 18d7ae2d0..b65d18030 100644 --- a/packages/core/common.js +++ b/packages/core/common.js @@ -38,6 +38,7 @@ export const EVENTS = { noteRemoved: "note:removed", tokenRefreshed: "token:refreshed", attachmentsLoading: "attachments:loading", + attachmentDeleted: "attachment:deleted", mediaAttachmentDownloaded: "attachments:mediaDownloaded", }; diff --git a/packages/core/database/fs.js b/packages/core/database/fs.js index e5dadea83..b2a50c8de 100644 --- a/packages/core/database/fs.js +++ b/packages/core/database/fs.js @@ -5,21 +5,36 @@ export default class FileStorage { constructor(fs, storage) { this.fs = fs; this.tokenManager = new TokenManager(storage); + this._queue = []; } - async downloadFile(hash) { + async downloadFile(groupId, hash) { const url = `${hosts.API_HOST}/s3?name=${hash}`; const token = await this.tokenManager.getAccessToken(); - return await this.fs.downloadFile(hash, { + const { execute, cancel } = this.fs.downloadFile(hash, { url, headers: { Authorization: `Bearer ${token}` }, }); + this._queue.push({ groupId, hash, cancel, type: "download" }); + return await execute(); } - async uploadFile(hash) { + async uploadFile(groupId, hash) { const token = await this.tokenManager.getAccessToken(); const url = await this._getPresignedURL(hash, token, "PUT"); - return await this.fs.uploadFile(hash, { url }); + const { execute, cancel } = this.fs.uploadFile(hash, { url }); + this._queue.push({ groupId, hash, cancel, type: "upload" }); + return await execute(); + } + + async cancel(groupId, type = undefined) { + await Promise.all( + this._queue + .filter( + (item) => item.groupId === groupId && (!type || item.type === type) + ) + .map(async (op) => await op.cancel()) + ); } readEncrypted(filename, encryptionKey, cipherData) { @@ -27,7 +42,7 @@ export default class FileStorage { } writeEncrypted(filename, data, type, encryptionKey) { - return this._db.fs.writeEncrypted(filename, { + return this.fs.writeEncrypted(filename, { data, type, key: encryptionKey, diff --git a/packages/core/package-lock.json b/packages/core/package-lock.json index 648351b2e..2fdc8cdee 100644 --- a/packages/core/package-lock.json +++ b/packages/core/package-lock.json @@ -10,8 +10,8 @@ "dependencies": { "@stablelib/blake2s": "^1.0.1", "dayjs": "^1.10.6", + "diff": "^5.0.0", "fast-sort": "^2.0.1", - "lean-he": "^2.1.2", "no-internet": "^1.5.2", "qclone": "^1.0.4", "quill-delta-to-html": "^0.12.0", @@ -3575,6 +3575,14 @@ "node": ">=0.10.0" } }, + "node_modules/diff": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/diff-sequences": { "version": "24.9.0", "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-24.9.0.tgz", @@ -5600,11 +5608,6 @@ "node": ">=6" } }, - "node_modules/lean-he": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/lean-he/-/lean-he-2.1.2.tgz", - "integrity": "sha512-g/cq01j/rnv7JWoxFmeLgJdd/CucksyDtS+pyepO89EdT0O4KfHJokOVz/xQ4mvjKJzcrj87Q3/s2ESou90WCQ==" - }, "node_modules/left-pad": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", @@ -11204,6 +11207,11 @@ "integrity": "sha1-9B8cEL5LAOh7XxPaaAdZ8sW/0+I=", "dev": true }, + "diff": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==" + }, "diff-sequences": { "version": "24.9.0", "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-24.9.0.tgz", @@ -12815,11 +12823,6 @@ "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", "dev": true }, - "lean-he": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/lean-he/-/lean-he-2.1.2.tgz", - "integrity": "sha512-g/cq01j/rnv7JWoxFmeLgJdd/CucksyDtS+pyepO89EdT0O4KfHJokOVz/xQ4mvjKJzcrj87Q3/s2ESou90WCQ==" - }, "left-pad": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", diff --git a/packages/core/package.json b/packages/core/package.json index 473e70ed8..dd20f9764 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -25,6 +25,7 @@ "dependencies": { "@stablelib/blake2s": "^1.0.1", "dayjs": "^1.10.6", + "diff": "^5.0.0", "fast-sort": "^2.0.1", "no-internet": "^1.5.2", "qclone": "^1.0.4", diff --git a/packages/core/utils/array.js b/packages/core/utils/array.js index b398ae11d..66b32d43f 100644 --- a/packages/core/utils/array.js +++ b/packages/core/utils/array.js @@ -29,21 +29,6 @@ export function hasItem(array, item) { return array.indexOf(item) > -1; } -export async function diff(arr1, arr2, action) { - let length = arr1.length + arr2.length; - for (var i = 0; i < length; ++i) { - var actionKey = "delete"; - var item = arr1[i]; - - if (i >= arr1.length) { - var actionKey = "insert"; - var item = arr2[i - arr1.length]; - } - - await action(item, actionKey); - } -} - function deleteAtIndex(array, index) { if (index === -1) return false; array.splice(index, 1); diff --git a/packages/core/yarn.lock b/packages/core/yarn.lock index 60b696847..0f3c4e203 100644 --- a/packages/core/yarn.lock +++ b/packages/core/yarn.lock @@ -2330,6 +2330,11 @@ "resolved" "https://registry.npmjs.org/diff-sequences/-/diff-sequences-25.2.6.tgz" "version" "25.2.6" +"diff@^5.0.0": + "integrity" "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==" + "resolved" "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz" + "version" "5.0.0" + "domexception@^1.0.1": "integrity" "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==" "resolved" "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz" @@ -3646,11 +3651,6 @@ "resolved" "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz" "version" "3.0.3" -"lean-he@^2.1.2": - "integrity" "sha512-g/cq01j/rnv7JWoxFmeLgJdd/CucksyDtS+pyepO89EdT0O4KfHJokOVz/xQ4mvjKJzcrj87Q3/s2ESou90WCQ==" - "resolved" "https://registry.npmjs.org/lean-he/-/lean-he-2.1.2.tgz" - "version" "2.1.2" - "left-pad@^1.3.0": "integrity" "sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA==" "resolved" "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"