mobile: check if file exists based on file size

apps/mobile/app/common/filesystem/download.js

@@ -24,7 +24,7 @@ import { ToastEvent } from "../../services/event-manager";
 import { useAttachmentStore } from "../../stores/use-attachment-store";
 import { db } from "../database";
 import { cacheDir, fileCheck } from "./utils";
-import { createCacheDir } from "./io";
+import { createCacheDir, exists } from "./io";
 
 export async function downloadFile(filename, data, cancelToken) {
   if (!data) return false;
@@ -32,11 +32,10 @@ export async function downloadFile(filename, data, cancelToken) {
   await createCacheDir();
 
   let { url, headers } = data;
-
   let path = `${cacheDir}/${filename}`;
+
   try {
-    let exists = await RNFetchBlob.fs.exists(path);
-    if (exists) {
+    if (await exists(filename)) {
       return true;
     }
 
@@ -74,22 +73,24 @@ export async function downloadFile(filename, data, cancelToken) {
     useAttachmentStore.getState().remove(filename);
     return status >= 200 && status < 300;
   } catch (e) {
-    ToastEvent.show({
-      heading: "Error downloading file",
-      message: e.message,
-      type: "error",
-      context: "global"
-    });
-    ToastEvent.show({
-      heading: "Error downloading file",
-      message: e.message,
-      type: "error",
-      context: "local"
-    });
+    if (e.message !== "canceled") {
+      ToastEvent.show({
+        heading: "Error downloading file",
+        message: e.message,
+        type: "error",
+        context: "global"
+      });
+      ToastEvent.show({
+        heading: "Error downloading file",
+        message: e.message,
+        type: "error",
+        context: "local"
+      });
+    }
 
     useAttachmentStore.getState().remove(filename);
     RNFetchBlob.fs.unlink(path).catch(console.log);
-    console.log("download file error: ", e, url, headers);
+    console.log("Download file error:", e, url, headers);
     return false;
   }
 }

apps/mobile/app/common/filesystem/io.js

@@ -29,18 +29,9 @@ export async function readEncrypted(filename, key, cipherData) {
   await migrateFilesFromCache();
   console.log("read encrypted file...");
   let path = `${cacheDir}/${filename}`;
+
   try {
-    const iosAppGroup =
-      Platform.OS === "ios"
-        ? await RNFetchBlob.fs.pathForAppGroup(IOS_APPGROUPID)
-        : null;
-
-    const appGroupPath = `${iosAppGroup}/${filename}`;
-    let exists =
-      (await RNFetchBlob.fs.exists(path)) ||
-      (Platform.OS === "ios" && (await RNFetchBlob.fs.exists(appGroupPath)));
-
-    if (!exists) {
+    if (!(await exists(filename))) {
       return false;
     } else {
       RNFetchBlob.fs.stat(path).then((r) => {
@@ -181,7 +172,42 @@ export async function migrateFilesFromCache() {
   }
 }
 
+const ABYTES = 17;
 export async function exists(filename) {
-  let exists = await RNFetchBlob.fs.exists(`${cacheDir}/${filename}`);
+  let path = `${cacheDir}/${filename}`;
+
+  const iosAppGroup =
+    Platform.OS === "ios"
+      ? await RNFetchBlob.fs.pathForAppGroup(IOS_APPGROUPID)
+      : null;
+  const appGroupPath = `${iosAppGroup}/${filename}`;
+
+  let exists = await RNFetchBlob.fs.exists(path);
+
+  // Check if file is present in app group path.
+  let existsInAppGroup = false;
+  if (!exists && Platform.OS === "ios") {
+    existsInAppGroup = await RNFetchBlob.fs.exists(appGroupPath);
+  }
+
+  if (exists || existsInAppGroup) {
+    const attachment = db.attachments.attachment(filename);
+    const totalChunks = Math.ceil(attachment.length / attachment.chunkSize);
+    const totalAbytes = totalChunks * ABYTES;
+    const expectedFileSize = attachment.length + totalAbytes;
+
+    const stat = await RNFetchBlob.fs.stat(
+      existsInAppGroup ? appGroupPath : path
+    );
+
+    if (stat.size !== expectedFileSize) {
+      RNFetchBlob.fs
+        .unlink(existsInAppGroup ? appGroupPath : path)
+        .catch(console.log);
+      return false;
+    }
+
+    exists = true;
+  }
   return exists;
 }

apps/mobile/app/components/attachments/attachment-item.js

@@ -132,7 +132,7 @@ export const AttachmentItem = ({
           activeOpacity={0.9}
           onPress={() => {
             if (encryption || !pressable) return;
-            db.fs.cancel(attachment.metadata.hash);
+            db.fs.cancel(attachment.metadata.hash, currentProgress.type);
             setCurrentProgress(null);
           }}
           style={{

apps/mobile/app/components/sheets/publish-note/index.js

@@ -40,7 +40,9 @@ import { requestInAppReview } from "../../../services/app-review";
 const PublishNoteSheet = ({ note: item, update }) => {
   const { colors } = useThemeColors();
   const actionSheetRef = useRef();
-  const loading = useAttachmentStore((state) => state.loading);
+
+  const attachmentDownloads = useAttachmentStore((state) => state.downloading);
+  const downloading = attachmentDownloads[`monograph-${item.id}`];
   const [selfDestruct, setSelfDestruct] = useState(false);
   const [isLocked, setIsLocked] = useState(false);
   const [note, setNote] = useState(item);
@@ -137,9 +139,9 @@ const PublishNoteSheet = ({ note: item, update }) => {
             }}
           >
             Please wait...
-            {loading && loading.current && loading.total
+            {downloading && downloading.current && downloading.total
               ? `\nDownloading attachments (${
-                  loading?.current / loading?.total
+                  downloading?.current / downloading?.total
                 })`
               : ""}
           </Paragraph>

apps/mobile/app/hooks/use-app-events.js

@@ -83,7 +83,6 @@ import {
 import { getGithubVersion } from "../utils/github-version";
 import { tabBarRef } from "../utils/global-refs";
 import { sleep } from "../utils/time";
-import { useThemeColors } from "@notesnook/theme";
 
 const onCheckSyncStatus = async (type) => {
   const { disableSync, disableAutoSync } = SettingsService.get();
@@ -107,10 +106,20 @@ const onFileEncryptionProgress = ({ total, progress }) => {
     .setEncryptionProgress((progress / total).toFixed(2));
 };
 
-const onLoadingAttachmentProgress = (data) => {
-  useAttachmentStore
-    .getState()
-    .setLoading(data.total === data.current ? null : data);
+const onDownloadingAttachmentProgress = (data) => {
+  useAttachmentStore.getState().setDownloading(data);
+};
+
+const onUploadingAttachmentProgress = (data) => {
+  useAttachmentStore.getState().setUploading(data);
+};
+
+const onDownloadedAttachmentProgress = (data) => {
+  useAttachmentStore.getState().setDownloading(data);
+};
+
+const onUploadedAttachmentProgress = (data) => {
+  useAttachmentStore.getState().setUploading(data);
 };
 
 const onUserSessionExpired = async () => {
@@ -301,7 +310,16 @@ export const useAppEvents = () => {
         EVENTS.userSubscriptionUpdated,
         onUserSubscriptionStatusChanged
       ),
-      EV.subscribe(EVENTS.attachmentsLoading, onLoadingAttachmentProgress),
+      EV.subscribe(EVENTS.fileDownload, onDownloadingAttachmentProgress),
+      EV.subscribe(EVENTS.fileUpload, onUploadingAttachmentProgress),
+      EV.subscribe(EVENTS.fileDownloaded, onDownloadedAttachmentProgress),
+      EV.subscribe(EVENTS.fileUploaded, onUploadedAttachmentProgress),
+      EV.subscribe(EVENTS.downloadCanceled, (data) => {
+        useAttachmentStore.getState().setDownloading(data);
+      }),
+      EV.subscribe(EVENTS.uploadCanceled, (data) => {
+        useAttachmentStore.getState().setUploading(data);
+      }),
       eSubscribeEvent(eUserLoggedIn, onUserUpdated)
     ];
 

apps/mobile/app/screens/editor/progress.js

@@ -1,81 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import React, { useEffect, useRef, useState } from "react";
-import { View } from "react-native";
-import { ProgressBarComponent } from "../../components/ui/svg/lazy";
-import useGlobalSafeAreaInsets from "../../hooks/use-global-safe-area-insets";
-import { useAttachmentStore } from "../../stores/use-attachment-store";
-import { useThemeColors } from "@notesnook/theme";
-import { SIZE } from "../../utils/size";
-export const ProgressBar = () => {
-  const { colors } = useThemeColors();
-  const loading = useAttachmentStore((state) => state.loading);
-  const [prog, setProg] = useState(0);
-  const [visible, setVisible] = useState(false);
-  const timer = useRef();
-  const insets = useGlobalSafeAreaInsets();
-  const [width, setWidth] = useState(false);
-
-  useEffect(() => {
-    if (loading) {
-      if (loading.current !== loading.total) {
-        setVisible(true);
-        setProg(loading.current / loading.total);
-      } else {
-        clear();
-      }
-    } else {
-      clear();
-    }
-  }, [loading]);
-
-  const clear = () => {
-    clearTimeout(timer.current);
-    timer.current = null;
-    timer.current = setTimeout(() => {
-      setProg(1);
-      setTimeout(() => {
-        setVisible(false);
-      }, 1000);
-    }, 100);
-  };
-
-  return visible ? (
-    <View
-      style={{
-        justifyContent: "center",
-        position: "absolute",
-        zIndex: 1,
-        marginTop: insets.top + 45,
-        width: "100%"
-      }}
-      onLayout={(event) => setWidth(event.nativeEvent.layout.width)}
-    >
-      <ProgressBarComponent
-        size={SIZE.xxl}
-        progress={prog}
-        color={colors.primary.accent}
-        borderWidth={0}
-        height={1}
-        width={width || 400}
-      />
-    </View>
-  ) : null;
-};

apps/mobile/app/screens/editor/progress.tsx

@@ -0,0 +1,136 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { useThemeColors } from "@notesnook/theme";
+import React, { useEffect, useRef, useState } from "react";
+import { View } from "react-native";
+import { ProgressBarComponent } from "../../components/ui/svg/lazy";
+import useGlobalSafeAreaInsets from "../../hooks/use-global-safe-area-insets";
+import { useAttachmentStore } from "../../stores/use-attachment-store";
+import { useEditorStore } from "../../stores/use-editor-store";
+
+export const ProgressBar = () => {
+  const { colors } = useThemeColors();
+  const currentlyEditingNote = useEditorStore(
+    (state) => state.currentEditingNote
+  );
+  const downloading = useAttachmentStore((state) => state.downloading);
+
+  const loading = currentlyEditingNote
+    ? downloading?.[currentlyEditingNote]
+    : undefined;
+
+  const attachmentProgress = useAttachmentStore((state) => state.progress);
+  const [progress, setProgress] = useState(0);
+  const [visible, setVisible] = useState(false);
+  const timer = useRef<NodeJS.Timeout>();
+  const insets = useGlobalSafeAreaInsets();
+  const [width, setWidth] = useState(400);
+
+  const groupProgressInfo = useRef<{
+    [name: string]: {
+      total: number;
+      current: number;
+    };
+  }>({});
+
+  const currentItemProgress = loading?.filename
+    ? attachmentProgress?.[loading?.filename]
+    : undefined;
+
+  useEffect(() => {
+    if (loading) {
+      console.log(loading);
+      if (
+        loading.current === loading.total &&
+        typeof loading.success === "boolean"
+      ) {
+        clear();
+        return;
+      }
+      setVisible(true);
+      if (!loading.filename) return;
+
+      if (!groupProgressInfo.current[loading.filename]) {
+        groupProgressInfo.current[loading.filename] = {
+          total: 1,
+          current: 0
+        };
+      }
+
+      const itemProgressCurrent = groupProgressInfo.current[loading.filename];
+
+      const itemTotalSize =
+        currentItemProgress?.total || itemProgressCurrent.total;
+
+      const itemCurrentProgress =
+        currentItemProgress?.recieved ||
+        currentItemProgress?.sent ||
+        itemProgressCurrent.current;
+
+      groupProgressInfo.current[loading.filename] = {
+        current: itemCurrentProgress,
+        total: itemTotalSize
+      };
+
+      const itemProgressPercent = itemCurrentProgress / itemTotalSize;
+
+      setProgress(
+        ((loading.current || 0) + itemProgressPercent) / (loading.total || 1)
+      );
+    } else {
+      clear();
+    }
+  }, [currentItemProgress, loading]);
+
+  const clear = () => {
+    clearTimeout(timer.current);
+    timer.current = undefined;
+    timer.current = setTimeout(() => {
+      setProgress(1);
+      setTimeout(() => {
+        setVisible(false);
+        setProgress(0);
+        groupProgressInfo.current = {};
+      }, 1000);
+    }, 0);
+  };
+
+  return (
+    <View
+      style={{
+        justifyContent: "center",
+        position: "absolute",
+        zIndex: visible ? 1 : -1,
+        marginTop: insets.top + 45,
+        width: "100%"
+      }}
+      onLayout={(event) => setWidth(event.nativeEvent.layout.width)}
+    >
+      <ProgressBarComponent
+        progress={progress}
+        color={colors.primary.accent}
+        borderWidth={0}
+        borderRadius={0}
+        height={2}
+        width={width || 400}
+      />
+    </View>
+  );
+};

apps/mobile/app/screens/editor/tiptap/use-editor.ts

@@ -134,7 +134,8 @@ export const useEditor = (
 
   const reset = useCallback(
     async (resetState = true, resetContent = true) => {
-      currentNote.current?.id && db.fs?.cancel(currentNote.current.id, null);
+      currentNote.current?.id &&
+        db.fs?.cancel(currentNote.current.id, "download");
       currentNote.current = null;
       loadedImages.current = {};
       currentContent.current = null;

apps/mobile/app/stores/use-attachment-store.ts

@@ -20,6 +20,16 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import create from "zustand";
 import { editorController } from "../screens/editor/tiptap/utils";
 
+export type AttachmentGroupProgress = {
+  total: number;
+  current: number;
+  groupId: string;
+  filename: string;
+  canceled?: boolean;
+  success?: boolean;
+  error?: any;
+};
+
 interface AttachmentStore {
   progress?: {
     [name: string]: {
@@ -30,6 +40,7 @@ interface AttachmentStore {
       type: "upload" | "download";
     } | null;
   };
+
   encryptionProgress: number;
   setEncryptionProgress: (encryptionProgress: number) => void;
   remove: (hash: string) => void;
@@ -40,8 +51,14 @@ interface AttachmentStore {
     recieved: number,
     type: "upload" | "download"
   ) => void;
-  loading: { total: number; current: number };
-  setLoading: (data: { total: number; current: number }) => void;
+  downloading?: {
+    [groupId: string]: AttachmentGroupProgress | undefined;
+  };
+  setDownloading: (data: AttachmentGroupProgress) => void;
+  uploading?: {
+    [groupId: string]: AttachmentGroupProgress | undefined;
+  };
+  setUploading: (data: AttachmentGroupProgress) => void;
 }
 
 export const useAttachmentStore = create<AttachmentStore>((set, get) => ({
@@ -73,6 +90,21 @@ export const useAttachmentStore = create<AttachmentStore>((set, get) => ({
   encryptionProgress: 0,
   setEncryptionProgress: (encryptionProgress) =>
     set({ encryptionProgress: encryptionProgress }),
-  loading: { total: 0, current: 0 },
-  setLoading: (data) => set({ loading: data ? { ...data } : data })
+
+  downloading: {},
+  setDownloading: (data) =>
+    set({
+      downloading: {
+        ...get().downloading,
+        [data.groupId]: data?.canceled ? undefined : data
+      }
+    }),
+  uploading: {},
+  setUploading: (data) =>
+    set({
+      uploading: {
+        ...get().uploading,
+        [data.groupId]: data?.canceled ? undefined : data
+      }
+    })
 }));

apps/web/src/app-effects.tsx

@@ -122,28 +122,47 @@ export default function AppEffects({ setShow }: AppEffectsProps) {
     //   }
     // );
 
-    const attachmentsLoadingEvent = EV.subscribe(
-      EVENTS.attachmentsLoading,
-      ({
-        type,
-        total,
-        current
-      }: {
-        type: ProcessingType;
-        total: number;
-        current: number;
-      }) => {
-        const [key, status] = getProcessingStatusFromType(type);
+    function handleDownloadUploadProgress(
+      type: ProcessingType,
+      total: number,
+      current: number
+    ) {
+      const [key, status] = getProcessingStatusFromType(type);
 
-        if (current === total) {
-          removeStatus(key);
-        } else {
-          updateStatus({
-            key,
-            status: `${status} attachments (${current}/${total})`,
-            progress: 0
-          });
-        }
+      console.log("handleDownloadUploadProgresss", key, status, current, total);
+      if (current === total) {
+        removeStatus(key);
+      } else {
+        updateStatus({
+          key,
+          status: `${status} attachments`,
+          current,
+          total,
+          progress: 0
+        });
+      }
+    }
+
+    const fileDownloadEvents = EV.subscribeMulti(
+      [EVENTS.fileDownloaded, EVENTS.fileDownload],
+      ({ total, current }: { total: number; current: number }) => {
+        handleDownloadUploadProgress("download", total, current);
+      },
+      null
+    );
+
+    const fileUploadEvents = EV.subscribeMulti(
+      [EVENTS.fileUploaded, EVENTS.fileUpload],
+      ({ total, current }: { total: number; current: number }) => {
+        handleDownloadUploadProgress("upload", total, current);
+      },
+      null
+    );
+
+    const fileEncrypted = AppEventManager.subscribe(
+      AppEvents.fileEncrypted,
+      ({ total, current }: { total: number; current: number }) => {
+        handleDownloadUploadProgress("encrypt", total, current);
       }
     );
 
@@ -162,20 +181,36 @@ export default function AppEffects({ setShow }: AppEffectsProps) {
         if (!key) return;
 
         const percent = Math.round((loaded / total) * 100);
-        const text = getStatus(key)?.status || `${status} attachment`;
+        const oldStatus = getStatus(key);
+        const text = oldStatus?.status || `${status} attachment`;
 
-        updateStatus({
-          key,
-          status: text,
-          progress: loaded === total ? 100 : percent
-        });
+        if (
+          (!oldStatus ||
+            (oldStatus.total === undefined &&
+              oldStatus.current === undefined) ||
+            oldStatus.total === oldStatus.current) &&
+          loaded === total
+        ) {
+          removeStatus(key);
+        } else {
+          updateStatus({
+            ...oldStatus,
+            key,
+            status: text,
+            progress: loaded === total ? 100 : percent
+          });
+        }
       }
     );
 
     registerKeyMap();
     return () => {
-      attachmentsLoadingEvent.unsubscribe();
-      progressEvent.unsubscribe();
+      [
+        ...fileDownloadEvents,
+        ...fileUploadEvents,
+        progressEvent,
+        fileEncrypted
+      ].forEach((e) => e.unsubscribe());
       //  systemTimeInvalidEvent.unsubscribe();
     };
   }, []);

apps/web/src/common/app-events.ts

@@ -24,6 +24,7 @@ export const AppEvents = {
   UPDATE_ATTACHMENT_PROGRESS: "updateAttachmentProgress",
   UPDATE_STATUS: "updateStatus",
   REMOVE_STATUS: "removeStatus",
+  fileEncrypted: "file:encrypted",
 
   checkingForUpdate: "checkingForUpdate",
   updateAvailable: "updateAvailable",

apps/web/src/components/publish-view/index.jsx

@@ -49,17 +49,17 @@ function PublishView(props) {
   }, [noteId]);
 
   useEffect(() => {
-    const attachmentsLoadingEvent = EV.subscribe(
-      EVENTS.attachmentsLoading,
-      ({ type, groupId, total, current }) => {
-        if (!groupId || !groupId.includes(noteId) || type !== "download")
-          return;
+    const fileDownloadedEvent = EV.subscribe(
+      EVENTS.fileDownloaded,
+      ({ total, current, groupId }) => {
+        if (!groupId || !groupId.includes(noteId)) return;
         if (current === total) setProcessingStatus();
         else setProcessingStatus({ total, current });
       }
     );
+
     return () => {
-      attachmentsLoadingEvent.unsubscribe();
+      fileDownloadedEvent.unsubscribe();
     };
   }, [noteId]);
 

apps/web/src/components/status-bar/index.tsx

@@ -40,7 +40,7 @@ import {
   showIssueDialog,
   showUpdateAvailableNotice
 } from "../../common/dialog-controller";
-import useStatus from "../../hooks/use-status";
+import useStatus, { statusToString } from "../../hooks/use-status";
 import { ScopedThemeProvider } from "../theme-provider";
 import { checkForUpdate, installUpdate } from "../../utils/updater";
 import { toTitleCase } from "@notesnook/common";
@@ -132,7 +132,8 @@ function StatusBar() {
             Report an issue
           </Text>
         </Button>
-        {statuses?.map(({ key, status, progress, icon: Icon }) => {
+        {statuses?.map((status) => {
+          const { key, icon: Icon } = status;
           return (
             <Flex
               key={key}
@@ -141,7 +142,7 @@ function StatusBar() {
             >
               {Icon ? <Icon size={12} /> : <Loading size={12} />}
               <Text variant="subBody" ml={1} sx={{ color: "paragraph" }}>
-                {progress ? `${progress}% ${status}` : status}
+                {statusToString(status)}
               </Text>
             </Flex>
           );

apps/web/src/hooks/use-status.ts

@@ -24,6 +24,8 @@ import { Icon } from "../components/icons";
 type Status = {
   key: string;
   status: string;
+  total?: number;
+  current?: number;
   progress?: number;
   icon?: Icon | null;
 };
@@ -36,13 +38,20 @@ interface IStatusStore {
 const useStatusStore = create<IStatusStore>((set, get) => ({
   statuses: {},
   getStatus: (key: string) => get().statuses[key],
-  updateStatus: ({ key, status, progress, icon }: Status) =>
+  updateStatus: ({ key, status, progress, icon, current, total }: Status) =>
     set(
       produce((state) => {
         if (!key) return;
         const { statuses } = state;
         const statusText = status || statuses[key]?.status;
-        statuses[key] = { key, status: statusText, progress, icon };
+        statuses[key] = {
+          current,
+          total,
+          key,
+          status: statusText,
+          progress,
+          icon
+        };
       })
     ),
   removeStatus: (key) =>
@@ -63,3 +72,12 @@ export default function useStatus() {
 export const updateStatus = useStatusStore.getState().updateStatus;
 export const removeStatus = useStatusStore.getState().removeStatus;
 export const getStatus = useStatusStore.getState().getStatus;
+
+export function statusToString(status: Status) {
+  const parts: string[] = [];
+  if (status.progress) parts.push(`${status.progress}%`);
+  parts.push(status.status);
+  if (status.total !== undefined && status.current !== undefined)
+    parts.push(`(${status.current}/${status.total})`);
+  return parts.join(" ");
+}

apps/web/src/interfaces/fs.ts

@@ -24,7 +24,6 @@ import { AppEventManager, AppEvents } from "../common/app-events";
 import { StreamableFS } from "@notesnook/streamable-fs";
 import { NNCrypto } from "./nncrypto";
 import hosts from "@notesnook/core/dist/utils/constants";
-import { sendAttachmentsProgressEvent } from "@notesnook/core/dist/common";
 import { saveAs } from "file-saver";
 import { showToast } from "../utils/toast";
 import { db } from "../common/db";
@@ -61,7 +60,11 @@ async function writeEncryptedFile(
   // let offset = 0;
   // let encrypted = 0;
   const fileHandle = await streamablefs.createFile(hash, file.size, file.type);
-  sendAttachmentsProgressEvent("encrypt", hash, 1, 0);
+  AppEventManager.publish(AppEvents.fileEncrypted, {
+    hash,
+    total: 1,
+    current: 0
+  });
 
   const { iv, stream } = await NNCrypto.createEncryptionStream(key);
   await file
@@ -82,7 +85,11 @@ async function writeEncryptedFile(
     )
     .pipeTo(fileHandle.writeable);
 
-  sendAttachmentsProgressEvent("encrypt", hash, 1, 1);
+  AppEventManager.publish(AppEvents.fileEncrypted, {
+    hash,
+    total: 1,
+    current: 1
+  });
 
   return {
     chunkSize: CHUNK_SIZE,
@@ -504,8 +511,12 @@ async function downloadFile(filename: string, requestOptions: RequestOptions) {
   }
 }
 
-function exists(filename: string) {
-  return streamablefs.exists(filename);
+async function exists(filename: string) {
+  const handle = await streamablefs.readFile(filename);
+  return (
+    handle &&
+    handle.file.size === (await handle.size()) - handle.file.chunks * ABYTES
+  );
 }
 
 type FileMetadata = {

packages/core/src/api/sync/index.js

@@ -21,7 +21,6 @@ import {
   checkSyncStatus,
   EV,
   EVENTS,
-  sendAttachmentsProgressEvent,
   sendSyncProgressEvent,
   SYNC_CHECK_IDS
 } from "../../common";
@@ -339,23 +338,10 @@ class Sync {
     const attachments = this.db.attachments.pending;
     this.logger.info("Uploading attachments...", { total: attachments.length });
 
-    for (var i = 0; i < attachments.length; ++i) {
-      const attachment = attachments[i];
-      const { hash } = attachment.metadata;
-      sendAttachmentsProgressEvent("upload", hash, attachments.length, i);
-
-      try {
-        const isUploaded = await this.db.fs.uploadFile(hash, hash);
-        if (!isUploaded) throw new Error("Failed to upload file.");
-
-        await this.db.attachments.markAsUploaded(attachment.id);
-      } catch (e) {
-        logger.error(e, { attachment });
-        const error = e.message;
-        await this.db.attachments.markAsFailed(attachment.id, error);
-      }
-    }
-    sendAttachmentsProgressEvent("upload", null, attachments.length);
+    await this.db.fs.queueUploads(
+      attachments.map((a) => ({ filename: a.metadata.hash })),
+      "sync-uploads"
+    );
   }
 
   /**

packages/core/src/collections/attachments.js

@@ -20,7 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import Collection from "./collection";
 import { getId } from "../utils/id";
 import { deleteItem, hasItem } from "../utils/array";
-import { EV, EVENTS, sendAttachmentsProgressEvent } from "../common";
+import { EV, EVENTS } from "../common";
 import dataurl from "../utils/dataurl";
 import dayjs from "dayjs";
 import setManipulator from "../utils/set";
@@ -34,6 +34,36 @@ export default class Attachments extends Collection {
   constructor(db, name, cached) {
     super(db, name, cached);
     this.key = null;
+
+    EV.subscribe(
+      EVENTS.fileDownloaded,
+      async ({ success, filename, groupId, eventData }) => {
+        if (!success || !eventData || !eventData.readOnDownload) return;
+        const attachment = this.attachment(filename);
+        if (!attachment) return;
+
+        const src = await this.read(filename, getOutputType(attachment));
+        if (!src) return;
+
+        EV.publish(EVENTS.mediaAttachmentDownloaded, {
+          groupId,
+          hash: attachment.metadata.hash,
+          attachmentType: getAttachmentType(attachment),
+          src
+        });
+      }
+    );
+
+    EV.subscribe(EVENTS.fileUploaded, async ({ success, error, filename }) => {
+      const attachment = this.attachment(filename);
+      if (!attachment) return;
+      if (success) await this.markAsUploaded(attachment.id);
+      else
+        await this.markAsFailed(
+          attachment.id,
+          error || "Failed to upload attachment."
+        );
+    });
   }
 
   merge(localAttachment, remoteAttachment) {
@@ -317,47 +347,15 @@ export default class Attachments extends Collection {
         (!hashesToLoad || hasItem(hashesToLoad, attachment.metadata.hash))
     );
 
-    try {
-      for (let i = 0; i < attachments.length; i++) {
-        const attachment = attachments[i];
-        await this._download(attachment, {
-          total: attachments.length,
-          current: i,
-          groupId: noteId
-        });
-      }
-    } finally {
-      sendAttachmentsProgressEvent("download", noteId, attachments.length);
-    }
-  }
-
-  async _download(attachment, { total, current, groupId }, notify = true) {
-    const { metadata, chunkSize } = attachment;
-    const filename = metadata.hash;
-
-    if (notify)
-      sendAttachmentsProgressEvent("download", groupId, total, current);
-
-    const isDownloaded = await this._db.fs.downloadFile(
-      groupId,
-      filename,
-      chunkSize,
-      metadata
+    await this._db.fs.queueDownloads(
+      attachments.map((a) => ({
+        filename: a.metadata.hash,
+        metadata: a.metadata,
+        chunkSize: a.chunkSize
+      })),
+      noteId,
+      { readOnDownload: true }
     );
-    if (!isDownloaded) return;
-
-    const src = await this.read(metadata.hash, getOutputType(attachment));
-    if (!src) return;
-
-    if (notify)
-      EV.publish(EVENTS.mediaAttachmentDownloaded, {
-        groupId,
-        hash: metadata.hash,
-        attachmentType: getAttachmentType(attachment),
-        src
-      });
-
-    return src;
   }
 
   async cleanup() {
@@ -451,7 +449,7 @@ export default class Attachments extends Collection {
   }
 }
 
-function getOutputType(attachment) {
+export function getOutputType(attachment) {
   if (attachment.metadata.type === "application/vnd.notesnook.web-clip")
     return "text";
   else if (attachment.metadata.type.startsWith("image/")) return "base64";

packages/core/src/collections/content.js

@@ -21,6 +21,7 @@ import Collection from "./collection";
 import { getId } from "../utils/id";
 import { getContentFromData } from "../content-types";
 import { hasItem } from "../utils/array";
+import { getOutputType } from "./attachments";
 
 export default class Content extends Collection {
   async add(content) {
@@ -116,17 +117,27 @@ export default class Content extends Collection {
 
   async downloadMedia(groupId, contentItem, notify = true) {
     const content = getContentFromData(contentItem.type, contentItem.data);
-    contentItem.data = await content.insertMedia((hash, { total, current }) => {
-      const attachment = this._db.attachments.attachment(hash);
-      if (!attachment) return;
-
-      const progressData = {
-        total,
-        current,
-        groupId
-      };
-
-      return this._db.attachments._download(attachment, progressData, notify);
+    contentItem.data = await content.insertMedia(async (hashes) => {
+      const attachments = hashes.map((h) => this._db.attachments.attachment(h));
+      await this._db.fs.queueDownloads(
+        attachments.map((a) => ({
+          filename: a.metadata.hash,
+          metadata: a.metadata,
+          chunkSize: a.chunkSize
+        })),
+        groupId,
+        notify ? { readOnDownload: false } : undefined
+      );
+      const sources = {};
+      for (const attachment of attachments) {
+        const src = await this._db.attachments.read(
+          attachment.metadata.hash,
+          getOutputType(attachment)
+        );
+        if (!src) continue;
+        sources[attachment.metadata.hash] = src;
+      }
+      return sources;
     });
     return contentItem;
   }

packages/core/src/common.js

@@ -36,15 +36,6 @@ export async function checkSyncStatus(type) {
   return results.some((r) => r.type === type && r.result === true);
 }
 
-export function sendAttachmentsProgressEvent(type, groupId, total, current) {
-  EV.publish(EVENTS.attachmentsLoading, {
-    type,
-    groupId,
-    total,
-    current: current === undefined ? total : current
-  });
-}
-
 export function sendSyncProgressEvent(EV, type, current) {
   EV.publish(EVENTS.syncProgress, {
     type,
@@ -98,7 +89,12 @@ export const EVENTS = {
   noteRemoved: "note:removed",
   tokenRefreshed: "token:refreshed",
   userUnauthorized: "user:unauthorized",
-  attachmentsLoading: "attachments:loading",
+  downloadCanceled: "file:downloadCanceled",
+  uploadCanceled: "file:uploadCanceled",
+  fileDownload: "file:download",
+  fileUpload: "file:upload",
+  fileDownloaded: "file:downloaded",
+  fileUploaded: "file:uploaded",
   attachmentDeleted: "attachment:deleted",
   mediaAttachmentDownloaded: "attachments:mediaDownloaded",
   vaultLocked: "vault:locked",

packages/core/src/content-types/__tests__/tiptap.test.js

@@ -41,7 +41,13 @@ test("img src is present after insert attachments", async () => {
     return { key: "hello", metadata: {} };
   });
   const tiptap2 = new Tiptap(result.data);
-  const result2 = await tiptap2.insertMedia(() => "i am a data");
+  const result2 = await tiptap2.insertMedia((hashes) => {
+    const images = {};
+    for (const hash of hashes) {
+      images[hash] = "i am a data";
+    }
+    return images;
+  });
   expect(result2).toContain(`src="i am a data"`);
 });
 

packages/core/src/content-types/tiptap.js

@@ -100,7 +100,7 @@ export class Tiptap {
     return tokens.some((token) => lowercase.indexOf(token) > -1);
   }
 
-  async insertMedia(getData) {
+  async insertMedia(resolve) {
     let hashes = [];
     new HTMLParser({
       ontag: (name, attr) => {
@@ -108,21 +108,9 @@ export class Tiptap {
         if (name === "img" && hash) hashes.push(hash);
       }
     }).parse(this.data);
+    if (!hashes.length) return this.data;
 
-    const images = {};
-    let hasImages = false;
-    for (let i = 0; i < hashes.length; ++i) {
-      const hash = hashes[i];
-      const src = await getData(hash, {
-        total: hashes.length,
-        current: i
-      });
-      if (!src) continue;
-      images[hash] = src;
-      hasImages = true;
-    }
-
-    if (!hasImages) return this.data;
+    const images = await resolve(hashes);
     return new HTMLRewriter({
       ontag: (name, attr) => {
         const hash = attr[ATTRIBUTES.hash];

packages/core/src/database/fs.js

@@ -19,12 +19,103 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import hosts from "../utils/constants";
 import TokenManager from "../api/token-manager";
+import { EV, EVENTS } from "../common";
 
 export default class FileStorage {
   constructor(fs, storage) {
     this.fs = fs;
     this.tokenManager = new TokenManager(storage);
-    this._queue = [];
+    this.downloads = new Map();
+    this.uploads = new Map();
+  }
+
+  async queueDownloads(files, groupId, eventData) {
+    const token = await this.tokenManager.getAccessToken();
+    const total = files.length;
+    let current = 0;
+    this.downloads.set(groupId, files);
+    for (const file of files) {
+      const { filename, metadata, chunkSize } = file;
+      if (await this.exists(filename)) {
+        current++;
+        EV.publish(EVENTS.fileDownloaded, {
+          success: true,
+          groupId,
+          filename,
+          eventData
+        });
+        continue;
+      }
+
+      const url = `${hosts.API_HOST}/s3?name=${filename}`;
+      const { execute, cancel } = this.fs.downloadFile(filename, {
+        metadata,
+        url,
+        chunkSize,
+        headers: { Authorization: `Bearer ${token}` }
+      });
+      file.cancel = cancel;
+
+      EV.publish(EVENTS.fileDownload, {
+        total,
+        current,
+        groupId,
+        filename
+      });
+
+      const result = await execute().catch(() => false);
+      if (eventData)
+        EV.publish(EVENTS.fileDownloaded, {
+          success: result,
+          total,
+          current: ++current,
+          groupId,
+          filename,
+          eventData
+        });
+    }
+    this.downloads.delete(groupId);
+  }
+
+  async queueUploads(files, groupId) {
+    const token = await this.tokenManager.getAccessToken();
+    const total = files.length;
+    let current = 0;
+    this.uploads.set(groupId, files);
+
+    for (const file of files) {
+      const { filename } = file;
+      const url = `${hosts.API_HOST}/s3?name=${filename}`;
+      const { execute, cancel } = this.fs.uploadFile(filename, {
+        url,
+        headers: { Authorization: `Bearer ${token}` }
+      });
+      file.cancel = cancel;
+
+      EV.publish(EVENTS.fileUpload, {
+        total,
+        current,
+        groupId,
+        filename
+      });
+
+      let error = null;
+      const result = await execute().catch((e) => {
+        console.error("Failed to upload attachment:", e);
+        error = e;
+        return false;
+      });
+
+      EV.publish(EVENTS.fileUploaded, {
+        error,
+        success: result,
+        total,
+        current: ++current,
+        groupId,
+        filename
+      });
+    }
+    this.uploads.delete(groupId);
   }
 
   async downloadFile(groupId, filename, chunkSize, metadata) {
@@ -36,37 +127,30 @@ export default class FileStorage {
       chunkSize,
       headers: { Authorization: `Bearer ${token}` }
     });
-    this._queue.push({ groupId, filename, cancel, type: "download" });
+    this.downloads.set(groupId, [{ cancel }]);
     const result = await execute();
-    this._deleteOp(groupId, "download");
-    return result;
-  }
-
-  async uploadFile(groupId, filename) {
-    const token = await this.tokenManager.getAccessToken();
-    const url = `${hosts.API_HOST}/s3?name=${filename}`;
-    const { execute, cancel } = this.fs.uploadFile(filename, {
-      url,
-      headers: { Authorization: `Bearer ${token}` }
-    });
-    this._queue.push({ groupId, filename, cancel, type: "upload" });
-    const result = await execute();
-    this._deleteOp(groupId, "upload");
+    this.downloads.delete(groupId);
     return result;
   }
 
   async cancel(groupId, type) {
-    const [op] = this._deleteOp(groupId, type);
-    if (!op) return;
-    await op.cancel("Operation canceled.");
-  }
-
-  _deleteOp(groupId, type) {
-    const opIndex = this._queue.findIndex(
-      (item) => item.groupId === groupId && (!type || item.type === type)
-    );
-    if (opIndex < 0) return [];
-    return this._queue.splice(opIndex, 1);
+    const queue =
+      type === "download"
+        ? this.downloads.get(groupId)
+        : this.uploads.get(groupId);
+    if (!queue) return;
+    for (let i = 0; i < queue.length; ++i) {
+      const file = queue[i];
+      if (file.cancel) await file.cancel("Operation canceled.");
+      queue.splice(i, 1);
+    }
+    if (type === "download") {
+      this.downloads.delete(groupId);
+      EV.publish(EVENTS.downloadCanceled, { groupId, canceled: true });
+    } else if (type === "upload") {
+      this.uploads.delete(groupId);
+      EV.publish(EVENTS.uploadCanceled, { groupId, canceled: true });
+    }
   }
 
   readEncrypted(filename, encryptionKey, cipherData) {

packages/core/src/utils/event-manager.js

@@ -27,9 +27,7 @@ class EventManager {
   }
 
   subscribeMulti(names, handler, thisArg) {
-    names.forEach((name) => {
-      this.subscribe(name, handler.bind(thisArg));
-    });
+    return names.map((name) => this.subscribe(name, handler.bind(thisArg)));
   }
 
   subscribe(name, handler, once = false) {