mobile: check if file exists based on file size

* mobile: fix sorting notes in topics

* mobile: fix list is not updated

.github/workflows/desktop.publish.yml

@@ -97,6 +97,7 @@ jobs:
       - name: Get App Store Version
         id: appstore
         uses: streetwriters/appstore-connect-app-version@develop
+        if: inputs.publish-apple
         with:
           app-id: ${{ steps.app_metadata.outputs.apple_app_id }}
           key-id: ${{ secrets.api_key_id }}

apps/desktop/app-update.yml

@@ -0,0 +1,3 @@
+owner: streetwriters
+repo: notesnook
+provider: github

apps/desktop/package-lock.json

@@ -1,18 +1,18 @@
 {
   "name": "@notesnook/desktop",
-  "version": "2.6.3",
+  "version": "2.6.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@notesnook/desktop",
-      "version": "2.6.3",
+      "version": "2.6.5",
       "hasInstallScript": true,
       "license": "GPL-3.0-or-later",
       "dependencies": {
         "@notesnook/crypto": "file:../../packages/crypto",
-        "@trpc/client": "10.31.0",
-        "@trpc/server": "10.31.0",
+        "@trpc/client": "10.38.3",
+        "@trpc/server": "10.38.3",
         "electron-trpc": "0.5.2",
         "electron-updater": "6.1.4",
         "icojs": "^0.17.1",
@@ -24,7 +24,7 @@
         "@types/node": "18.16.1",
         "@types/yargs": "^17.0.24",
         "chokidar": "^3.5.3",
-        "electron": "25.8.0",
+        "electron": "25.8.1",
         "electron-builder": "^24.6.4",
         "electron-builder-notarize": "^1.5.1",
         "esbuild": "^0.17.19",
@@ -831,20 +831,20 @@
       }
     },
     "node_modules/@trpc/client": {
-      "version": "10.31.0",
-      "resolved": "https://registry.npmjs.org/@trpc/client/-/client-10.31.0.tgz",
-      "integrity": "sha512-VCqbJEvFJb8C4hQFw7AD+dkQTjgEdV/QAzO4D+/cX5e93u5NpfNXI+PKS0QFXwG/zqgwQwVV6OkYc/D/MFwA6g==",
+      "version": "10.38.3",
+      "resolved": "https://registry.npmjs.org/@trpc/client/-/client-10.38.3.tgz",
+      "integrity": "sha512-hHPsElTL4sB+UGzuAZ4iwRVHjELrk/Nsl2owsYw9ITJz3EY7VWRsQ6vK48kJvlHPYPhP+efID5UOgyZMs4bFXA==",
       "funding": [
         "https://trpc.io/sponsor"
       ],
       "peerDependencies": {
-        "@trpc/server": "10.31.0"
+        "@trpc/server": "10.38.3"
       }
     },
     "node_modules/@trpc/server": {
-      "version": "10.31.0",
-      "resolved": "https://registry.npmjs.org/@trpc/server/-/server-10.31.0.tgz",
-      "integrity": "sha512-9EnRTSDE9nF11LZsvSOqNKqkRYzHqFX4ch5AJ6VIu8uta2vxVTN4FxxsNRSOluTzVYZDeaCISbwmOJ5iihCCIg==",
+      "version": "10.38.3",
+      "resolved": "https://registry.npmjs.org/@trpc/server/-/server-10.38.3.tgz",
+      "integrity": "sha512-9s8/kwo2IDB5hwB2SKZZrfevRhdb1f9fdXtIYd3lbQuf2jQaC/LyQuHaIQjDQoUx9updBfsHXcFFPiCP1DL6pg==",
       "funding": [
         "https://trpc.io/sponsor"
       ]
@@ -1751,9 +1751,9 @@
       }
     },
     "node_modules/electron": {
-      "version": "25.8.0",
-      "resolved": "https://registry.npmjs.org/electron/-/electron-25.8.0.tgz",
-      "integrity": "sha512-T3kC1a/3ntSaYMCVVfUUc9v7myPzi6J2GP0Ad/CyfWKDPp054dGyKxb2EEjKnxQQ7wfjsT1JTEdBG04x6ekVBw==",
+      "version": "25.8.1",
+      "resolved": "https://registry.npmjs.org/electron/-/electron-25.8.1.tgz",
+      "integrity": "sha512-GtcP1nMrROZfFg0+mhyj1hamrHvukfF6of2B/pcWxmWkd5FVY1NJib0tlhiorFZRzQN5Z+APLPr7aMolt7i2AQ==",
       "hasInstallScript": true,
       "dependencies": {
         "@electron/get": "^2.0.0",

apps/desktop/package.json

@@ -2,7 +2,7 @@
   "name": "@notesnook/desktop",
   "productName": "Notesnook",
   "description": "Your private note taking space",
-  "version": "2.6.3",
+  "version": "2.6.5",
   "appAppleId": "1544027013",
   "private": true,
   "main": "./dist/index.js",
@@ -12,8 +12,8 @@
   "license": "GPL-3.0-or-later",
   "dependencies": {
     "@notesnook/crypto": "file:../../packages/crypto",
-    "@trpc/client": "10.31.0",
-    "@trpc/server": "10.31.0",
+    "@trpc/client": "10.38.3",
+    "@trpc/server": "10.38.3",
     "electron-trpc": "0.5.2",
     "electron-updater": "6.1.4",
     "icojs": "^0.17.1",
@@ -25,7 +25,7 @@
     "@types/node": "18.16.1",
     "@types/yargs": "^17.0.24",
     "chokidar": "^3.5.3",
-    "electron": "25.8.0",
+    "electron": "25.8.1",
     "electron-builder": "^24.6.4",
     "electron-builder-notarize": "^1.5.1",
     "esbuild": "^0.17.19",
@@ -191,6 +191,7 @@
       "allowNativeWayland": true
     },
     "extraResources": [
+      "app-update.yml",
       "./assets/**"
     ],
     "extraMetadata": {

apps/desktop/src/api/os-integration.ts

@@ -117,6 +117,14 @@ export const osIntegrationRouter = t.router({
       writeFileSync(resolvedPath, data);
     }),
 
+  resolvePath: t.procedure
+    .input(z.object({ filePath: z.string() }))
+    .query(({ input }) => {
+      const { filePath } = input;
+      if (!filePath) return;
+      return resolvePath(filePath);
+    }),
+
   showNotification: t.procedure
     .input(NotificationOptions)
     .query(({ input }) => {

apps/desktop/src/cli.ts

@@ -27,19 +27,23 @@ export type CLIOptions = {
   hidden: boolean;
 };
 
-export async function parseArguments(): Promise<CLIOptions> {
+export async function parseArguments(argv: string[]): Promise<CLIOptions> {
   const result: CLIOptions = {
     note: false,
     notebook: false,
     reminder: false,
     hidden: false
   };
-  const { hidden } = await yargs(hideBin(process.argv))
+  const { hidden } = await yargs(hideBin(argv))
     .boolean("hidden")
+    // have to account for this flag added on Windows when launching
+    // via Jumplist
+    .boolean("allow-file-access-from-files")
     .command("new", "Create a new item", (yargs) => {
       return yargs
         .command("note", "Create a new note", {}, () => {
           result.note = true;
+          console.log("HERE!");
         })
         .command("notebook", "Create a new notebook", {}, () => {
           result.notebook = true;

apps/desktop/src/main.ts

@@ -34,6 +34,7 @@ import { router, api } from "./api";
 import { config } from "./utils/config";
 import path from "path";
 import { bringToFront } from "./utils/bring-to-front";
+import { bridge } from "./api/bridge";
 
 // only run a single instance
 if (!MAC_APP_STORE && !app.requestSingleInstanceLock()) {
@@ -63,7 +64,7 @@ process.on("unhandledRejection", (reason) => {
 app.commandLine.appendSwitch("lang", "en-US");
 
 async function createWindow() {
-  const cliOptions = await parseArguments();
+  const cliOptions = await parseArguments(process.argv);
   setTheme(getTheme());
 
   const mainWindowState = new WindowState({});
@@ -90,6 +91,7 @@ async function createWindow() {
     }
   });
 
+  createIPCHandler({ router, windows: [mainWindow] });
   globalThis.window = mainWindow;
   mainWindowState.manage(mainWindow);
 
@@ -105,7 +107,6 @@ async function createWindow() {
 
   await AssetManager.loadIcons();
   setupDesktopIntegration();
-  createIPCHandler({ router, windows: [mainWindow] });
 
   mainWindow.webContents.session.setSpellCheckerDictionaryDownloadURL(
     "http://dictionaries.notesnook.com/"
@@ -146,8 +147,12 @@ app.once("window-all-closed", () => {
   }
 });
 
-app.on("second-instance", () => {
+app.on("second-instance", async (_ev, argv) => {
   if (!globalThis.window) return;
+  const cliOptions = await parseArguments(argv);
+  if (cliOptions.note) bridge.onCreateItem("note");
+  if (cliOptions.notebook) bridge.onCreateItem("notebook");
+  if (cliOptions.reminder) bridge.onCreateItem("reminder");
   bringToFront();
 });
 

apps/desktop/src/preload.ts

@@ -31,7 +31,6 @@ declare global {
 }
 
 process.once("loaded", async () => {
-  console.log("HELLO!");
   const electronTRPC: RendererGlobalElectronTRPC = {
     sendMessage: (operation) =>
       ipcRenderer.send(ELECTRON_TRPC_CHANNEL, operation),

apps/mobile/app/common/filesystem/download-attachment.js

@@ -62,6 +62,7 @@ export async function downloadAttachments(
   canceled,
   groupId
 ) {
+  await createCacheDir();
   if (!attachments || !attachments.length) return;
   const result = new Map();
 
@@ -182,6 +183,7 @@ export default async function downloadAttachment(
     groupId: undefined
   }
 ) {
+  await createCacheDir();
   let attachment = db.attachments.attachment(hash);
   if (!attachment) {
     console.log("attachment not found");

apps/mobile/app/common/filesystem/download.js

@@ -24,15 +24,18 @@ import { ToastEvent } from "../../services/event-manager";
 import { useAttachmentStore } from "../../stores/use-attachment-store";
 import { db } from "../database";
 import { cacheDir, fileCheck } from "./utils";
+import { createCacheDir, exists } from "./io";
 
 export async function downloadFile(filename, data, cancelToken) {
   if (!data) return false;
-  let { url, headers } = data;
 
+  await createCacheDir();
+
+  let { url, headers } = data;
   let path = `${cacheDir}/${filename}`;
+
   try {
-    let exists = await RNFetchBlob.fs.exists(path);
-    if (exists) {
+    if (await exists(filename)) {
       return true;
     }
 
@@ -60,29 +63,34 @@ export async function downloadFile(filename, data, cancelToken) {
         console.log("downloading: ", recieved, total);
       });
 
-    cancelToken.cancel = request.cancel;
+    cancelToken.cancel = () => {
+      useAttachmentStore.getState().remove(filename);
+      request.cancel();
+    };
     let response = await request;
     await fileCheck(response, totalSize);
     let status = response.info().status;
     useAttachmentStore.getState().remove(filename);
     return status >= 200 && status < 300;
   } catch (e) {
-    ToastEvent.show({
-      heading: "Error downloading file",
-      message: e.message,
-      type: "error",
-      context: "global"
-    });
-    ToastEvent.show({
-      heading: "Error downloading file",
-      message: e.message,
-      type: "error",
-      context: "local"
-    });
+    if (e.message !== "canceled") {
+      ToastEvent.show({
+        heading: "Error downloading file",
+        message: e.message,
+        type: "error",
+        context: "global"
+      });
+      ToastEvent.show({
+        heading: "Error downloading file",
+        message: e.message,
+        type: "error",
+        context: "local"
+      });
+    }
 
     useAttachmentStore.getState().remove(filename);
     RNFetchBlob.fs.unlink(path).catch(console.log);
-    console.log("download file error: ", e, url, headers);
+    console.log("Download file error:", e, url, headers);
     return false;
   }
 }

apps/mobile/app/common/filesystem/io.js

@@ -20,32 +20,29 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { Platform } from "react-native";
 import Sodium from "@ammarahmed/react-native-sodium";
 import RNFetchBlob from "react-native-blob-util";
-import { cacheDir, getRandomId } from "./utils";
+import { cacheDir, cacheDirOld, getRandomId } from "./utils";
 import { db } from "../database";
 import { compressToBase64 } from "./compress";
 import { IOS_APPGROUPID } from "../../utils/constants";
 
 export async function readEncrypted(filename, key, cipherData) {
+  await migrateFilesFromCache();
+  console.log("read encrypted file...");
   let path = `${cacheDir}/${filename}`;
+
   try {
-    const iosAppGroup =
-      Platform.OS === "ios"
-        ? await RNFetchBlob.fs.pathForAppGroup(IOS_APPGROUPID)
-        : null;
-
-    const appGroupPath = `${iosAppGroup}/${filename}`;
-    let exists =
-      (await RNFetchBlob.fs.exists(path)) ||
-      (Platform.OS === "ios" && (await RNFetchBlob.fs.exists(appGroupPath)));
-
-    if (!exists) {
+    if (!(await exists(filename))) {
       return false;
+    } else {
+      RNFetchBlob.fs.stat(path).then((r) => {
+        console.log(r);
+      });
     }
 
     const attachment = db.attachments.attachment(filename);
     const isPng = /(png)/g.test(attachment?.metadata.type);
     const isJpeg = /(jpeg|jpg)/g.test(attachment?.metadata.type);
-
+    console.log("decrypting....");
     let output = await Sodium.decryptFile(
       key,
       {
@@ -59,6 +56,7 @@ export async function readEncrypted(filename, key, cipherData) {
           : "base64"
         : "text"
     );
+    console.log("file decrypted...");
     if (cipherData.outputType === "base64" && (isPng || isJpeg)) {
       const dCachePath = `${cacheDir}/${output}`;
       output = await compressToBase64(
@@ -70,7 +68,7 @@ export async function readEncrypted(filename, key, cipherData) {
     return output;
   } catch (e) {
     RNFetchBlob.fs.unlink(path).catch(console.log);
-    console.log(e);
+    console.log("readEncrypted", e);
     return false;
   }
 }
@@ -88,6 +86,7 @@ export async function hashBase64(data) {
 }
 
 export async function writeEncryptedBase64({ data, key }) {
+  await createCacheDir();
   let filepath = cacheDir + `/${getRandomId("imagecache_")}`;
   await RNFetchBlob.fs.writeFile(filepath, data, "base64");
   let output = await Sodium.encryptFile(key, {
@@ -103,6 +102,7 @@ export async function writeEncryptedBase64({ data, key }) {
 }
 
 export async function deleteFile(filename, data) {
+  await createCacheDir();
   let delFilePath = cacheDir + `/${filename}`;
   if (!data) {
     if (!filename) return;
@@ -128,19 +128,86 @@ export async function deleteFile(filename, data) {
 export async function clearFileStorage() {
   try {
     let files = await RNFetchBlob.fs.ls(cacheDir);
+    let oldCache = await RNFetchBlob.fs.ls(cacheDirOld);
+
     for (let file of files) {
-      try {
-        await RNFetchBlob.fs.unlink(cacheDir + `/${file}`);
-      } catch (e) {
-        console.log(e);
-      }
+      await RNFetchBlob.fs.unlink(cacheDir + `/${file}`).catch(console.log);
+    }
+    for (let file of oldCache) {
+      await RNFetchBlob.fs.unlink(cacheDirOld + `/${file}`).catch(console.log);
     }
   } catch (e) {
-    console.log(e);
+    console.log("clearFileStorage", e);
   }
 }
 
+export async function createCacheDir() {
+  if (!(await RNFetchBlob.fs.exists(cacheDir))) {
+    await RNFetchBlob.fs.mkdir(cacheDir);
+    console.log("Cache directory created");
+  }
+}
+
+export async function migrateFilesFromCache() {
+  try {
+    await createCacheDir();
+    const migratedFilesPath = cacheDir + "/.migrated_1";
+    const migrated = await RNFetchBlob.fs.exists(migratedFilesPath);
+    if (migrated) return;
+
+    let files = await RNFetchBlob.fs.ls(cacheDir);
+    console.log("Files to migrate:", files.join(","));
+
+    let oldCache = await RNFetchBlob.fs.ls(cacheDirOld);
+    for (let file of oldCache) {
+      if (file.startsWith("org.") || file.startsWith("com.")) continue;
+      RNFetchBlob.fs
+        .mv(cacheDirOld + `/${file}`, cacheDir + `/${file}`)
+        .catch(console.log);
+      console.log("Moved", file);
+    }
+    await RNFetchBlob.fs.createFile(migratedFilesPath, "1", "utf8");
+  } catch (e) {
+    console.log("migrateFilesFromCache", e);
+  }
+}
+
+const ABYTES = 17;
 export async function exists(filename) {
-  let exists = await RNFetchBlob.fs.exists(`${cacheDir}/${filename}`);
+  let path = `${cacheDir}/${filename}`;
+
+  const iosAppGroup =
+    Platform.OS === "ios"
+      ? await RNFetchBlob.fs.pathForAppGroup(IOS_APPGROUPID)
+      : null;
+  const appGroupPath = `${iosAppGroup}/${filename}`;
+
+  let exists = await RNFetchBlob.fs.exists(path);
+
+  // Check if file is present in app group path.
+  let existsInAppGroup = false;
+  if (!exists && Platform.OS === "ios") {
+    existsInAppGroup = await RNFetchBlob.fs.exists(appGroupPath);
+  }
+
+  if (exists || existsInAppGroup) {
+    const attachment = db.attachments.attachment(filename);
+    const totalChunks = Math.ceil(attachment.length / attachment.chunkSize);
+    const totalAbytes = totalChunks * ABYTES;
+    const expectedFileSize = attachment.length + totalAbytes;
+
+    const stat = await RNFetchBlob.fs.stat(
+      existsInAppGroup ? appGroupPath : path
+    );
+
+    if (stat.size !== expectedFileSize) {
+      RNFetchBlob.fs
+        .unlink(existsInAppGroup ? appGroupPath : path)
+        .catch(console.log);
+      return false;
+    }
+
+    exists = true;
+  }
   return exists;
 }

apps/mobile/app/common/filesystem/upload.js

@@ -24,11 +24,12 @@ import { cacheDir } from "./utils";
 import { isImage, isDocument } from "@notesnook/core/dist/utils/filename";
 import { Platform } from "react-native";
 import { IOS_APPGROUPID } from "../../utils/constants";
+import { createCacheDir } from "./io";
 
 export async function uploadFile(filename, data, cancelToken) {
   if (!data) return false;
   let { url, headers } = data;
-
+  await createCacheDir();
   DatabaseLogger.info(`Preparing to upload file: ${filename}`);
 
   try {
@@ -72,7 +73,10 @@ export async function uploadFile(filename, data, cancelToken) {
         );
       });
 
-    cancelToken.cancel = request.cancel;
+    cancelToken.cancel = () => {
+      useAttachmentStore.getState().remove(filename);
+      request.cancel();
+    };
     let response = await request;
 
     let status = response.info().status;

apps/mobile/app/common/filesystem/utils.js

@@ -21,7 +21,12 @@ import * as ScopedStorage from "react-native-scoped-storage";
 import { Platform } from "react-native";
 import RNFetchBlob from "react-native-blob-util";
 
-export const cacheDir = RNFetchBlob.fs.dirs.CacheDir;
+export const cacheDirOld = RNFetchBlob.fs.dirs.CacheDir;
+
+export const cacheDir =
+  Platform.OS == "ios"
+    ? RNFetchBlob.fs.dirs.LibraryDir + "/.cache"
+    : RNFetchBlob.fs.dirs.DocumentDir + "/.cache";
 
 export function getRandomId(prefix) {
   return Math.random()
@@ -66,9 +71,12 @@ export function cancelable(operation) {
 }
 
 export function copyFileAsync(source, dest) {
-  return new Promise((resolve) => {
+  return new Promise((resolve, reject) => {
     ScopedStorage.copyFile(source, dest, (e, r) => {
-      console.log(e, r);
+      if (e) {
+        reject(e);
+        return;
+      }
       resolve();
     });
   });

apps/mobile/app/components/attachments/attachment-item.js

@@ -132,7 +132,7 @@ export const AttachmentItem = ({
           activeOpacity={0.9}
           onPress={() => {
             if (encryption || !pressable) return;
-            db.fs.cancel(attachment.metadata.hash);
+            db.fs.cancel(attachment.metadata.hash, currentProgress.type);
             setCurrentProgress(null);
           }}
           style={{

apps/mobile/app/components/list-items/headers/section-header.js

@@ -17,35 +17,25 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import React, { useEffect, useRef, useState } from "react";
-import { TouchableOpacity, useWindowDimensions, View } from "react-native";
 import { useThemeColors } from "@notesnook/theme";
-import { useSettingStore } from "../../../stores/use-setting-store";
-import {
-  eSendEvent,
-  eSubscribeEvent,
-  eUnSubscribeEvent,
-  presentSheet
-} from "../../../services/event-manager";
+import React, { useRef } from "react";
+import { TouchableOpacity, View, useWindowDimensions } from "react-native";
+import { eSendEvent, presentSheet } from "../../../services/event-manager";
 import SettingsService from "../../../services/settings";
-import { GROUP } from "../../../utils/constants";
+import { useSettingStore } from "../../../stores/use-setting-store";
 import { ColorValues } from "../../../utils/colors";
-import { db } from "../../../common/database";
+import { GROUP } from "../../../utils/constants";
 import { eOpenJumpToDialog } from "../../../utils/events";
 import { SIZE } from "../../../utils/size";
-import { IconButton } from "../../ui/icon-button";
-import { Button } from "../../ui/button";
 import Sort from "../../sheets/sort";
+import { Button } from "../../ui/button";
+import { IconButton } from "../../ui/icon-button";
 import Heading from "../../ui/typography/heading";
-import { useCallback } from "react";
 
 export const SectionHeader = React.memo(
-  function SectionHeader({ item, index, type, color, screen }) {
+  function SectionHeader({ item, index, type, color, screen, groupOptions }) {
     const { colors } = useThemeColors();
     const { fontScale } = useWindowDimensions();
-    const [groupOptions, setGroupOptions] = useState(
-      db.settings?.getGroupOptions(type)
-    );
     let groupBy = Object.keys(GROUP).find(
       (key) => GROUP[key] === groupOptions.groupBy
     );
@@ -65,17 +55,6 @@ export const SectionHeader = React.memo(
       ? "Default"
       : groupBy.slice(0, 1).toUpperCase() + groupBy.slice(1, groupBy.length);
 
-    const onUpdate = useCallback(() => {
-      setGroupOptions({ ...db.settings?.getGroupOptions(type) });
-    }, [type]);
-
-    useEffect(() => {
-      eSubscribeEvent("groupOptionsUpdate", onUpdate);
-      return () => {
-        eUnSubscribeEvent("groupOptionsUpdate", onUpdate);
-      };
-    }, [onUpdate]);
-
     return (
       <View
         style={{

apps/mobile/app/components/list/card.js

@@ -18,7 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import React from "react";
-import { View } from "react-native";
+import { Dimensions, View } from "react-native";
 import Icon from "react-native-vector-icons/MaterialCommunityIcons";
 import { useMessageStore } from "../../stores/use-message-store";
 import { useThemeColors } from "@notesnook/theme";
@@ -32,6 +32,7 @@ export const Card = ({ color, warning }) => {
   color = color ? color : colors.primary.accent;
   const messageBoardState = useMessageStore((state) => state.message);
   const announcement = useMessageStore((state) => state.announcement);
+  const fontScale = Dimensions.get("window").fontScale;
 
   return !messageBoardState.visible || announcement || warning ? null : (
     <View
@@ -44,7 +45,6 @@ export const Card = ({ color, warning }) => {
         type="gray"
         customStyle={{
           paddingVertical: 12,
-          width: "95%",
           flexDirection: "row",
           alignItems: "center",
           justifyContent: "space-between",
@@ -54,17 +54,18 @@ export const Card = ({ color, warning }) => {
         <View
           style={{
             flexDirection: "row",
-            alignItems: "center"
+            alignItems: "center",
+            flexShrink: 1
           }}
         >
           <View
             style={{
-              width: 40,
+              width: 40 * fontScale,
               backgroundColor:
                 messageBoardState.type === "error"
                   ? hexToRGBA(colors.static.red, 0.15)
                   : hexToRGBA(color, 0.15),
-              height: 40,
+              height: 40 * fontScale,
               borderRadius: 100,
               alignItems: "center",
               justifyContent: "center"
@@ -75,6 +76,7 @@ export const Card = ({ color, warning }) => {
               color={
                 messageBoardState.type === "error" ? colors.error.icon : color
               }
+              allowFontScaling
               name={messageBoardState.icon}
             />
           </View>
@@ -91,7 +93,7 @@ export const Card = ({ color, warning }) => {
             </Paragraph>
             <Paragraph
               style={{
-                flexWrap: "wrap",
+                flexWrap: "no-wrap",
                 flexShrink: 1
               }}
               color={colors.primary.heading}
@@ -101,22 +103,24 @@ export const Card = ({ color, warning }) => {
           </View>
         </View>
 
-        <View
-          style={{
-            width: 40,
-            height: 40,
-            justifyContent: "center",
-            alignItems: "center"
-          }}
-        >
-          <Icon
-            name="chevron-right"
-            color={
-              messageBoardState.type === "error" ? colors.error.icon : color
-            }
-            size={SIZE.lg}
-          />
-        </View>
+        {fontScale > 1 ? null : (
+          <View
+            style={{
+              width: 40,
+              height: 40,
+              justifyContent: "center",
+              alignItems: "center"
+            }}
+          >
+            <Icon
+              name="chevron-right"
+              color={
+                messageBoardState.type === "error" ? colors.error.icon : color
+              }
+              size={SIZE.lg}
+            />
+          </View>
+        )}
       </PressableButton>
     </View>
   );

apps/mobile/app/components/list/index.js

@@ -39,6 +39,7 @@ import { Empty } from "./empty";
 import { getTotalNotes } from "@notesnook/common";
 import { useSettingStore } from "../../stores/use-setting-store";
 import ReminderItem from "../list-items/reminder";
+import { useGroupOptions } from "../../hooks/use-group-options";
 
 const renderItems = {
   note: NoteWrapper,
@@ -99,9 +100,10 @@ const List = ({
     type === "notebooks" ||
     notebooksListMode === "compact";
   const groupType =
-    screen === "Notes" ? "home" : screen === "Favorites" ? "favorites" : type;
+    screen === "Home" ? "home" : screen === "Favorites" ? "favorites" : type;
+
+  const groupOptions = useGroupOptions(groupType);
 
-  const groupOptions = db.settings?.getGroupOptions(groupType);
   const dateBy =
     groupOptions.sortBy !== "title" ? groupOptions.sortBy : "dateEdited";
 
@@ -113,18 +115,21 @@ const List = ({
         color={headerProps?.color}
         title={headerProps?.heading}
         dateBy={dateBy}
-        type={
-          screen === "Notes"
-            ? "home"
-            : screen === "Favorites"
-            ? "favorites"
-            : type
-        }
+        type={groupType}
         screen={screen}
         isSheet={isSheet}
+        groupOptions={groupOptions}
       />
     ),
-    [headerProps?.color, headerProps?.heading, screen, type, isSheet, dateBy]
+    [
+      headerProps?.color,
+      headerProps?.heading,
+      screen,
+      isSheet,
+      dateBy,
+      groupType,
+      groupOptions
+    ]
   );
 
   const _onRefresh = async () => {
@@ -230,7 +235,7 @@ const List = ({
         <JumpToSectionDialog
           screen={screen}
           data={listData}
-          type={screen === "Notes" ? "home" : type}
+          type={screen === "Home" ? "home" : type}
           scrollRef={scrollRef}
         />
       ) : null}

apps/mobile/app/components/properties/index.js

@@ -16,8 +16,8 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-import React from "react";
-import { Platform, View } from "react-native";
+import React, { useRef } from "react";
+import { Dimensions, Platform, View, useWindowDimensions } from "react-native";
 import { FlatList } from "react-native-actions-sheet";
 import { db } from "../../common/database";
 import { DDS } from "../../services/device-detection";
@@ -54,7 +54,6 @@ export const Properties = ({ close = () => {}, item, buttons = [] }) => {
   const { colors } = useThemeColors();
   const alias = item.alias || item.title;
   const isColor = !!ColorValues[item.title];
-
   if (!item || !item.id) {
     return (
       <Paragraph style={{ marginVertical: 10, alignSelf: "center" }}>
@@ -62,6 +61,7 @@ export const Properties = ({ close = () => {}, item, buttons = [] }) => {
       </Paragraph>
     );
   }
+
   return (
     <FlatList
       keyboardShouldPersistTaps="always"
@@ -135,6 +135,7 @@ export const Properties = ({ close = () => {}, item, buttons = [] }) => {
               }, 1000);
             }}
           />
+
           <Synced item={item} close={close} />
 
           {DDS.isTab ? (

apps/mobile/app/components/properties/items.js

@@ -18,7 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import React from "react";
-import { FlatList, ScrollView, View } from "react-native";
+import { Dimensions, FlatList, ScrollView, View } from "react-native";
 import Icon from "react-native-vector-icons/MaterialCommunityIcons";
 import { useActions } from "../../hooks/use-actions";
 import { DDS } from "../../services/device-detection";
@@ -33,9 +33,11 @@ export const Items = ({ item, buttons, close }) => {
   const dimensions = useSettingStore((state) => state.dimensions);
   const actions = useActions({ item, close });
   const data = actions.filter((i) => buttons.indexOf(i.id) > -1 && !i.hidden);
-
   let width = dimensions.width > 600 ? 600 : dimensions.width;
-  let columnItemsCount = DDS.isLargeTablet() ? 7 : 5;
+  const shouldShrink =
+    Dimensions.get("window").fontScale > 1 &&
+    Dimensions.get("window").width < 450;
+  let columnItemsCount = DDS.isLargeTablet() ? 7 : shouldShrink ? 4 : 5;
   let columnItemWidth = DDS.isTab
     ? (width - 12) / columnItemsCount
     : (width - 12) / columnItemsCount;
@@ -66,6 +68,7 @@ export const Items = ({ item, buttons, close }) => {
         }}
       >
         <Icon
+          allowFontScaling
           name={item.icon}
           size={DDS.isTab ? SIZE.xxl : SIZE.lg}
           color={
@@ -78,7 +81,11 @@ export const Items = ({ item, buttons, close }) => {
         />
       </PressableButton>
 
-      <Paragraph size={SIZE.xs} style={{ textAlign: "center" }}>
+      <Paragraph
+        size={SIZE.xs}
+        textBreakStrategy="simple"
+        style={{ textAlign: "center" }}
+      >
         {item.title}
       </Paragraph>
     </View>
@@ -139,6 +146,7 @@ export const Items = ({ item, buttons, close }) => {
         >
           <Icon
             name={item.icon}
+            allowFontScaling
             size={DDS.isTab ? SIZE.xxl : SIZE.md + 4}
             color={
               item.on
@@ -150,7 +158,11 @@ export const Items = ({ item, buttons, close }) => {
           />
         </PressableButton>
 
-        <Paragraph size={SIZE.xxs + 1} style={{ textAlign: "center" }}>
+        <Paragraph
+          textBreakStrategy="simple"
+          size={SIZE.xxs + 1}
+          style={{ textAlign: "center" }}
+        >
           {item.title}
         </Paragraph>
       </PressableButton>
@@ -163,9 +175,12 @@ export const Items = ({ item, buttons, close }) => {
     "copy",
     "share",
     "export",
-    "lock-unlock",
-    "publish"
+    "lock-unlock"
   ];
+  if (!shouldShrink) {
+    topBarItemsList.push("publish");
+  }
+
   const topBarItems = data.filter(
     (item) => topBarItemsList.indexOf(item.id) > -1
   );

apps/mobile/app/components/properties/synced.js

@@ -17,10 +17,10 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import React from "react";
-import { View } from "react-native";
-import Icon from "react-native-vector-icons/MaterialCommunityIcons";
 import { useThemeColors } from "@notesnook/theme";
+import React from "react";
+import { View, useWindowDimensions } from "react-native";
+import Icon from "react-native-vector-icons/MaterialCommunityIcons";
 import { useUserStore } from "../../stores/use-user-store";
 import { openLinkInBrowser } from "../../utils/functions";
 import { SIZE } from "../../utils/size";
@@ -33,6 +33,8 @@ export const Synced = ({ item, close }) => {
   const user = useUserStore((state) => state.user);
   const lastSynced = useUserStore((state) => state.lastSynced);
 
+  const dimensions = useWindowDimensions();
+  const shouldShrink = dimensions.fontScale > 1 && dimensions.width < 450;
   return user && lastSynced >= item.dateModified ? (
     <View
       style={{
@@ -49,7 +51,11 @@ export const Synced = ({ item, close }) => {
         borderTopColor: colors.secondary.background
       }}
     >
-      <Icon name="shield-key-outline" color={colors.primary.accent} size={SIZE.xxxl} />
+      <Icon
+        name="shield-key-outline"
+        color={colors.primary.accent}
+        size={SIZE.xxxl}
+      />
 
       <View
         style={{
@@ -67,15 +73,17 @@ export const Synced = ({ item, close }) => {
         >
           Encrypted and synced
         </Heading>
-        <Paragraph
-          style={{
-            flexWrap: "wrap"
-          }}
-          size={SIZE.xs}
-          color={colors.primary.paragraph}
-        >
-          No one can view this {item.itemType || item.type} except you.
-        </Paragraph>
+        {shouldShrink ? null : (
+          <Paragraph
+            style={{
+              flexWrap: "wrap"
+            }}
+            size={SIZE.xs}
+            color={colors.primary.paragraph}
+          >
+            No one can view this {item.itemType || item.type} except you.
+          </Paragraph>
+        )}
       </View>
 
       <Button
@@ -92,7 +100,6 @@ export const Synced = ({ item, close }) => {
           }
         }}
         fontSize={SIZE.xs}
-        title="Learn more"
         height={30}
         type="grayAccent"
       />

apps/mobile/app/components/selection-header/index.js

@@ -207,7 +207,6 @@ export const SelectionHeader = React.memo(() => {
 
         {screen === "Trash" ||
         screen === "Notebooks" ||
-        screen === "Notebook" ||
         screen === "Reminders" ? null : (
           <>
             <IconButton
@@ -224,6 +223,21 @@ export const SelectionHeader = React.memo(() => {
               name="pound"
               size={SIZE.xl}
             />
+
+            <IconButton
+              onPress={async () => {
+                ExportNotesSheet.present(selectedItemsList);
+              }}
+              tooltipText="Export"
+              tooltipPosition={4}
+              customStyle={{
+                marginLeft: 10
+              }}
+              color={colors.primary.paragraph}
+              name="export"
+              size={SIZE.xl}
+            />
+
             <IconButton
               onPress={async () => {
                 //setSelectionMode(false);
@@ -239,20 +253,6 @@ export const SelectionHeader = React.memo(() => {
               name="plus"
               size={SIZE.xl}
             />
-
-            <IconButton
-              onPress={async () => {
-                ExportNotesSheet.present(selectedItemsList);
-              }}
-              tooltipText="Export"
-              tooltipPosition={4}
-              customStyle={{
-                marginLeft: 10
-              }}
-              color={colors.primary.paragraph}
-              name="export"
-              size={SIZE.xl}
-            />
           </>
         )}
 
@@ -317,23 +317,8 @@ export const SelectionHeader = React.memo(() => {
             customStyle={{
               marginLeft: 10
             }}
-            onPress={async () => {
-              presentDialog({
-                title: `Delete ${
-                  selectedItemsList.length > 1 ? "items" : "item"
-                }`,
-                paragraph: `Are you sure you want to delete ${
-                  selectedItemsList.length > 1 ? "these items?" : "this item?"
-                }`,
-                positiveText: "Delete",
-                negativeText: "Cancel",
-                positivePress: () => {
-                  deleteItems();
-                },
-                positiveType: "errorShade"
-              });
-
-              return;
+            onPress={() => {
+              deleteItems();
             }}
             tooltipText="Move to trash"
             tooltipPosition={1}

apps/mobile/app/components/sheet-provider/index.js

@@ -198,7 +198,6 @@ const SheetProvider = ({ context = "global" }) => {
             key={data.actionText}
             title={data.actionText}
             accentColor={data.iconColor}
-            accentText="light"
             type="accent"
             height={40}
             width={250}

apps/mobile/app/components/sheets/publish-note/index.js

@@ -40,7 +40,9 @@ import { requestInAppReview } from "../../../services/app-review";
 const PublishNoteSheet = ({ note: item, update }) => {
   const { colors } = useThemeColors();
   const actionSheetRef = useRef();
-  const loading = useAttachmentStore((state) => state.loading);
+
+  const attachmentDownloads = useAttachmentStore((state) => state.downloading);
+  const downloading = attachmentDownloads[`monograph-${item.id}`];
   const [selfDestruct, setSelfDestruct] = useState(false);
   const [isLocked, setIsLocked] = useState(false);
   const [note, setNote] = useState(item);
@@ -137,9 +139,9 @@ const PublishNoteSheet = ({ note: item, update }) => {
             }}
           >
             Please wait...
-            {loading && loading.current && loading.total
+            {downloading && downloading.current && downloading.total
               ? `\nDownloading attachments (${
-                  loading?.current / loading?.total
+                  downloading?.current / downloading?.total
                 })`
               : ""}
           </Paragraph>

apps/mobile/app/components/sheets/reminder/index.tsx

@@ -253,6 +253,57 @@ export default function ReminderSheet({
           marginBottom: DDS.isTab ? 25 : undefined
         }}
       >
+        {reminderMode === ReminderModes.Permanent ? null : (
+          <ScrollView
+            style={{
+              flexDirection: "row",
+              borderWidth: 1,
+              marginTop: 12,
+              borderRadius: 5,
+              borderColor: colors.primary.border,
+              paddingLeft: 12,
+              height: 50
+            }}
+            horizontal
+          >
+            {Object.keys(ReminderNotificationModes).map((mode) => (
+              <Button
+                key={mode}
+                title={mode}
+                style={{
+                  marginRight: 12,
+                  borderRadius: 100
+                }}
+                icon={
+                  mode === "Silent"
+                    ? "minus-circle"
+                    : mode === "Vibrate"
+                    ? "vibrate"
+                    : "volume-high"
+                }
+                height={35}
+                type={
+                  reminderNotificationMode ===
+                  ReminderNotificationModes[
+                    mode as keyof typeof ReminderNotificationModes
+                  ]
+                    ? "grayAccent"
+                    : "gray"
+                }
+                onPress={() => {
+                  const _mode = ReminderNotificationModes[
+                    mode as keyof typeof ReminderNotificationModes
+                  ] as Reminder["priority"];
+                  SettingsService.set({
+                    reminderNotificationMode: _mode
+                  });
+                  setReminderNotificatioMode(_mode);
+                }}
+              />
+            ))}
+          </ScrollView>
+        )}
+
         <Input
           fwdRef={titleRef}
           defaultValue={reminder?.title || referencedItem?.title}
@@ -280,15 +331,21 @@ export default function ReminderSheet({
           }}
           height={80}
           wrapperStyle={{
-            marginBottom: 20
+            marginBottom: 12
           }}
         />
 
-        <View
+        <ScrollView
           style={{
             flexDirection: "row",
-            marginBottom: 12
+            marginBottom: 12,
+            height: 50,
+            borderWidth: 1,
+            borderRadius: 5,
+            borderColor: colors.primary.border,
+            paddingLeft: 12
           }}
+          horizontal
         >
           {Object.keys(ReminderModes).map((mode) => (
             <Button
@@ -327,7 +384,7 @@ export default function ReminderSheet({
               }}
             />
           ))}
-        </View>
+        </ScrollView>
 
         {reminderMode === ReminderModes.Repeat ? (
           <View
@@ -498,53 +555,6 @@ export default function ReminderSheet({
           </View>
         )}
 
-        {reminderMode === ReminderModes.Permanent ? null : (
-          <View
-            style={{
-              flexDirection: "row",
-              marginBottom: 12,
-              paddingTop: 12,
-              borderTopWidth: 1,
-              borderTopColor: colors.secondary.background
-            }}
-          >
-            {Object.keys(ReminderNotificationModes).map((mode) => (
-              <Button
-                key={mode}
-                title={mode}
-                style={{
-                  marginRight: 12,
-                  borderRadius: 100
-                }}
-                icon={
-                  mode === "Silent"
-                    ? "minus-circle"
-                    : mode === "Vibrate"
-                    ? "vibrate"
-                    : "volume-high"
-                }
-                height={35}
-                type={
-                  reminderNotificationMode ===
-                  ReminderNotificationModes[
-                    mode as keyof typeof ReminderNotificationModes
-                  ]
-                    ? "grayAccent"
-                    : "gray"
-                }
-                onPress={() => {
-                  const _mode = ReminderNotificationModes[
-                    mode as keyof typeof ReminderNotificationModes
-                  ] as Reminder["priority"];
-                  SettingsService.set({
-                    reminderNotificationMode: _mode
-                  });
-                  setReminderNotificatioMode(_mode);
-                }}
-              />
-            ))}
-          </View>
-        )}
         {reminderMode === ReminderModes.Once ||
         reminderMode === ReminderModes.Permanent ? null : (
           <View
@@ -595,6 +605,11 @@ export default function ReminderSheet({
           }}
         />
       </ScrollView>
+      <View
+        style={{
+          height: 10
+        }}
+      />
     </View>
   );
 }

apps/mobile/app/components/sheets/restore-data/index.js

@@ -22,7 +22,9 @@ import React, { useCallback, useEffect, useRef, useState } from "react";
 import { ActivityIndicator, Platform, View } from "react-native";
 import { FlatList } from "react-native-actions-sheet";
 import RNFetchBlob from "react-native-blob-util";
-import DocumentPicker from "react-native-document-picker";
+import DocumentPicker, {
+  DocumentPickerResponse
+} from "react-native-document-picker";
 import * as ScopedStorage from "react-native-scoped-storage";
 import { db } from "../../../common/database";
 import storage from "../../../common/database/storage";
@@ -45,6 +47,8 @@ import Seperator from "../../ui/seperator";
 import SheetWrapper from "../../ui/sheet";
 import Paragraph from "../../ui/typography/paragraph";
 import { getFormattedDate } from "@notesnook/common";
+import { unzip } from "react-native-zip-archive";
+import { cacheDir, copyFileAsync } from "../../../common/filesystem/utils";
 
 const RestoreDataSheet = () => {
   const [visible, setVisible] = useState(false);
@@ -129,75 +133,71 @@ const RestoreDataComponent = ({ close, setRestoring, restoring }) => {
   useEffect(() => {
     setTimeout(() => {
       checkBackups();
-    }, 300);
+    }, 1000);
   }, []);
 
   const restore = async (item) => {
     if (restoring) {
       return;
     }
-    try {
-      setRestoring(true);
-      let prefix = Platform.OS === "ios" ? "" : "file:/";
-      let backup;
-      if (Platform.OS === "android") {
-        backup = await ScopedStorage.readFile(item.uri, "utf8");
-      } else {
-        backup = await RNFetchBlob.fs.readFile(prefix + item.path, "utf8");
-      }
-      backup = JSON.parse(backup);
 
-      if (backup.data.iv && backup.data.salt) {
-        withPassword(
-          async (value) => {
-            try {
-              await restoreBackup(backup, value);
-              close();
-              setRestoring(false);
-              return true;
-            } catch (e) {
-              backupError(e);
-              return false;
-            }
-          },
-          () => {
-            setRestoring(false);
+    try {
+      const file = Platform.OS === "ios" ? item.path : item.uri;
+      console.log(file);
+      if (file.endsWith(".nnbackupz")) {
+        setRestoring(true);
+
+        if (Platform.OS === "android") {
+          const cacheFile = `file://${RNFetchBlob.fs.dirs.CacheDir}/backup.zip`;
+          if (await RNFetchBlob.fs.exists(cacheFile)) {
+            await RNFetchBlob.fs.unlink(cacheFile);
           }
-        );
-      } else {
-        await restoreBackup(backup);
-        close();
+          await RNFetchBlob.fs.createFile(cacheFile, "", "utf8");
+          console.log("copying");
+          await copyFileAsync(file, cacheFile);
+          console.log("copied");
+          await restoreFromZip(cacheFile);
+        } else {
+          await restoreFromZip(file, false);
+        }
+      } else if (file.endsWith(".nnbackup")) {
+        let backup;
+        if (Platform.OS === "android") {
+          backup = await ScopedStorage.readFile(file, "utf8");
+        } else {
+          backup = await RNFetchBlob.fs.readFile(file, "utf8");
+        }
+        await restoreFromNNBackup(JSON.parse(backup));
       }
     } catch (e) {
+      console.log("error", e);
       setRestoring(false);
       backupError(e);
     }
   };
 
-  const withPassword = (onsubmit, onclose = () => {}) => {
-    presentDialog({
-      context: "local",
-      title: "Encrypted backup",
-      input: true,
-      inputPlaceholder: "Password",
-      paragraph: "Please enter password of this backup file to restore it",
-      positiveText: "Restore",
-      secureTextEntry: true,
-      onClose: onclose,
-      negativeText: "Cancel",
-      positivePress: async (password) => {
-        try {
-          return await onsubmit(password);
-        } catch (e) {
-          ToastEvent.show({
-            heading: "Failed to backup data",
-            message: e.message,
-            type: "error",
-            context: "global"
-          });
-          return false;
+  const withPassword = () => {
+    return new Promise((resolve) => {
+      let resolved = false;
+      presentDialog({
+        context: "local",
+        title: "Encrypted backup",
+        input: true,
+        inputPlaceholder: "Password",
+        paragraph: "Please enter password of this backup file to restore it",
+        positiveText: "Restore",
+        secureTextEntry: true,
+        onClose: () => {
+          if (resolved) return;
+          resolve(undefined);
+        },
+        negativeText: "Cancel",
+        positivePress: async (password) => {
+          resolve(password);
+          resolved = true;
+          return true;
         }
-      }
+      });
     });
   };
 
@@ -217,21 +217,164 @@ const RestoreDataComponent = ({ close, setRestoring, restoring }) => {
         let path = await storage.checkAndCreateDir("/backups/");
         files = await RNFetchBlob.fs.lstat(path);
       }
-      files = files.sort(function (a, b) {
-        let timeA = a.lastModified;
-        let timeB = b.lastModified;
-        return timeB - timeA;
-      });
+      files = files
+        .filter((file) => {
+          const name = Platform.OS === "android" ? file.name : file.filename;
+          return name.endsWith(".nnbackup") || name.endsWith(".nnbackupz");
+        })
+        .sort(function (a, b) {
+          let timeA = a.lastModified;
+          let timeB = b.lastModified;
+          return timeB - timeA;
+        });
       setFiles(files);
-      setTimeout(() => {
-        setLoading(false);
-      }, 1000);
+      setLoading(false);
     } catch (e) {
       console.log(e);
       setLoading(false);
     }
   };
 
+  const restoreBackup = async (backup, password) => {
+    await db.backup.import(backup, password);
+    await db.initCollections();
+    initialize();
+    ToastEvent.show({
+      heading: "Backup restored successfully.",
+      type: "success",
+      context: "global"
+    });
+    return true;
+  };
+
+  const backupError = (e) => {
+    ToastEvent.show({
+      heading: "Restore failed",
+      message:
+        e.message ||
+        "The selected backup data file is invalid. You must select a *.nnbackup file to restore.",
+      type: "error",
+      context: "local"
+    });
+  };
+
+  /**
+   *
+   * @param {string} file
+   */
+  async function restoreFromZip(file, remove) {
+    try {
+      const zipOutputFolder = `${cacheDir}/backup_extracted`;
+      if (await RNFetchBlob.fs.exists(zipOutputFolder)) {
+        await RNFetchBlob.fs.unlink(zipOutputFolder);
+        await RNFetchBlob.fs.mkdir(zipOutputFolder);
+      }
+      await unzip(file, zipOutputFolder);
+      console.log("Unzipped files successfully to", zipOutputFolder);
+
+      const backupFiles = await RNFetchBlob.fs.ls(zipOutputFolder);
+
+      if (backupFiles.findIndex((file) => file === ".nnbackup") === -1) {
+        throw new Error("Backup file is invalid");
+      }
+
+      let password;
+
+      console.log(`Found ${backupFiles?.length} files to restore from backup`);
+      for (const path of backupFiles) {
+        if (path === ".nnbackup") continue;
+        const filePath = `${zipOutputFolder}/${path}`;
+        const data = await RNFetchBlob.fs.readFile(filePath, "utf8");
+        const parsed = JSON.parse(data);
+
+        if (parsed.encrypted && !password) {
+          console.log("Backup is encrypted...", "requesting password");
+          password = await withPassword();
+          if (!password) throw new Error("Failed to decrypt backup");
+        }
+        await db.backup.import(parsed, password);
+        console.log("Imported", path);
+      }
+      // Remove files from cache
+      RNFetchBlob.fs.unlink(zipOutputFolder).catch(console.log);
+      if (remove) {
+        RNFetchBlob.fs.unlink(file).catch(console.log);
+      }
+
+      await db.initCollections();
+      initialize();
+      setRestoring(false);
+      close();
+      ToastEvent.show({
+        heading: "Backup restored successfully.",
+        type: "success",
+        context: "global"
+      });
+    } catch (e) {
+      backupError(e);
+      setRestoring(false);
+    }
+  }
+
+  /**
+   *
+   * @param {string} file
+   */
+  async function restoreFromNNBackup(backup) {
+    try {
+      if (backup.data.iv && backup.data.salt) {
+        const password = await withPassword();
+        if (password) {
+          try {
+            await restoreBackup(backup, password);
+            close();
+            setRestoring(false);
+          } catch (e) {
+            setRestoring(false);
+            backupError(e);
+          }
+        } else {
+          setRestoring(false);
+        }
+      } else {
+        await restoreBackup(backup);
+        setRestoring(false);
+        close();
+      }
+    } catch (e) {
+      setRestoring(false);
+      backupError(e);
+    }
+  }
+
+  const button = {
+    title: "Restore from files",
+    onPress: async () => {
+      if (restoring) {
+        return;
+      }
+      try {
+        const file = await DocumentPicker.pickSingle({
+          copyTo: "cachesDirectory"
+        });
+
+        if (file.name.endsWith(".nnbackupz")) {
+          setRestoring(true);
+          await restoreFromZip(file.fileCopyUri, true);
+        } else if (file.name.endsWith(".nnbackup")) {
+          RNFetchBlob.fs.unlink(file.fileCopyUri).catch(console.log);
+          setRestoring(true);
+          const data = await fetch(file.uri);
+          await restoreFromNNBackup(await data.json());
+        }
+      } catch (e) {
+        console.log("error", e.stack);
+        setRestoring(false);
+        backupError(e);
+      }
+    }
+  };
+
   const renderItem = ({ item, index }) => (
     <View
       style={{
@@ -271,76 +414,6 @@ const RestoreDataComponent = ({ close, setRestoring, restoring }) => {
     </View>
   );
 
-  const restoreBackup = async (backup, password) => {
-    await db.backup.import(backup, password);
-    setRestoring(false);
-    initialize();
-    ToastEvent.show({
-      heading: "Backup restored successfully.",
-      type: "success",
-      context: "global"
-    });
-  };
-
-  const backupError = (e) => {
-    ToastEvent.show({
-      heading: "Restore failed",
-      message:
-        e.message ||
-        "The selected backup data file is invalid. You must select a *.nnbackup file to restore.",
-      type: "error",
-      context: "local"
-    });
-  };
-
-  const button = {
-    title: "Restore from files",
-    onPress: () => {
-      if (restoring) {
-        return;
-      }
-
-      DocumentPicker.pickSingle()
-        .then((r) => {
-          setRestoring(true);
-          fetch(r.uri)
-            .then(async (r) => {
-              try {
-                let backup = await r.json();
-                if (backup.data.iv && backup.data.salt) {
-                  withPassword(
-                    async (value) => {
-                      try {
-                        restoreBackup(backup, value).then(() => {
-                          close();
-                          setRestoring(false);
-                        });
-                        return true;
-                      } catch (e) {
-                        backupError(e);
-                        setRestoring(false);
-                        return false;
-                      }
-                    },
-                    () => {
-                      setRestoring(false);
-                    }
-                  );
-                } else {
-                  await restoreBackup(backup);
-                  close();
-                }
-              } catch (e) {
-                setRestoring(false);
-                backupError(e);
-              }
-            })
-            .catch(console.log);
-        })
-        .catch(console.log);
-    }
-  };
-
   return (
     <>
       <View>

apps/mobile/app/components/sheets/topic-sheet/index.tsx

@@ -336,23 +336,10 @@ export const TopicsSheet = () => {
                   useSelectionStore.setState({
                     selectedItemsList: selection
                   });
-                  presentDialog({
-                    title: `Delete ${
-                      selection.length > 1 ? "topics" : "topics"
-                    }`,
-                    paragraph: `Are you sure you want to delete ${
-                      selection.length > 1 ? "these topics?" : "this topic?"
-                    }`,
-                    positiveText: "Delete",
-                    negativeText: "Cancel",
-                    positivePress: async () => {
-                      await deleteItems();
-                      useSelectionStore.getState().clearSelection();
-                      setEnabled(false);
-                      setSelection([]);
-                    },
-                    positiveType: "errorShade"
-                  });
+                  await deleteItems();
+                  useSelectionStore.getState().clearSelection();
+                  setEnabled(false);
+                  setSelection([]);
                   return;
                 }}
                 color={colors.primary.icon}

apps/mobile/app/components/side-menu/menu-item.js

@@ -19,7 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { useThemeColors } from "@notesnook/theme";
 import React, { useCallback, useEffect, useState } from "react";
-import { View } from "react-native";
+import { Dimensions, View } from "react-native";
 import Icon from "react-native-vector-icons/MaterialCommunityIcons";
 import ToggleSwitch from "toggle-switch-react-native";
 import Navigation from "../../services/navigation";
@@ -109,6 +109,7 @@ export const MenuItem = React.memo(
               textAlignVertical: "center",
               textAlign: "left"
             }}
+            allowFontScaling
             name={item.icon}
             color={
               item.icon === "crown"

apps/mobile/app/components/side-menu/pinned-section.js

@@ -209,13 +209,19 @@ export const PinItem = React.memo(
                 justifyContent: "center"
               }}
             >
-              <Icon color={color} size={SIZE.lg - 2} name={icons[item.type]} />
+              <Icon
+                allowFontScaling
+                color={color}
+                size={SIZE.lg - 2}
+                name={icons[item.type]}
+              />
               <Icon
                 style={{
                   position: "absolute",
                   bottom: -6,
                   left: -6
                 }}
+                allowFontScaling
                 color={color}
                 size={SIZE.xs}
                 name="arrow-top-right-thick"

apps/mobile/app/components/side-menu/user-status.js

@@ -113,6 +113,7 @@ export const UserStatus = () => {
               <Icon
                 name="checkbox-blank-circle"
                 size={11}
+                allowFontScaling
                 color={
                   !user || lastSyncStatus === SyncStatus.Failed
                     ? colors.error.icon
@@ -149,9 +150,15 @@ export const UserStatus = () => {
                 color={colors.error.icon}
                 name="sync-alert"
                 size={SIZE.lg}
+                allowFontScaling
               />
             ) : (
-              <Icon color={colors.primary.accent} name="sync" size={SIZE.lg} />
+              <Icon
+                allowFontScaling
+                color={colors.primary.accent}
+                name="sync"
+                size={SIZE.lg}
+              />
             )
           ) : null}
         </PressableButton>

apps/mobile/app/components/ui/button/index.tsx

@@ -134,6 +134,7 @@ export const Button = ({
       {icon && !loading && iconPosition === "left" ? (
         <Icon
           name={icon}
+          allowFontScaling
           style={[{ marginRight: 0 }, iconStyle as any]}
           color={iconColor || buttonType?.text || textColor}
           size={iconSize}
@@ -170,6 +171,7 @@ export const Button = ({
       {icon && !loading && iconPosition === "right" ? (
         <Icon
           name={icon}
+          allowFontScaling
           style={[{ marginLeft: 0 }, iconStyle as any]}
           color={iconColor || buttonType?.text || textColor}
           size={iconSize}

apps/mobile/app/components/ui/icon-button/index.tsx

@@ -92,6 +92,7 @@ export const IconButton = ({
         layout={Layout}
         name={name}
         style={iconStyle as any}
+        allowFontScaling
         color={
           restProps.disabled
             ? RGB_Linear_Shade(-0.05, hexToRGBA(colors.secondary.background))

apps/mobile/app/components/ui/typography/heading.tsx

@@ -59,7 +59,6 @@ const Heading = ({
     <Component
       layout={restProps.layout || Layout}
       allowFontScaling={true}
-      maxFontSizeMultiplier={1}
       {...restProps}
       style={[
         {

apps/mobile/app/components/ui/typography/paragraph.tsx

@@ -45,8 +45,6 @@ const Paragraph = ({
   return (
     <Component
       layout={restProps.layout || Layout}
-      allowFontScaling
-      maxFontSizeMultiplier={1}
       {...restProps}
       style={[
         {

apps/mobile/app/hooks/use-actions.js

@@ -649,7 +649,7 @@ export const useActions = ({ close = () => null, item }) => {
     },
     {
       id: "favorite",
-      title: item.favorite ? "Unfavorite" : "Favorite",
+      title: item.favorite ? "Unfav" : "Fav",
       icon: item.favorite ? "star-off" : "star-outline",
       func: addToFavorites,
       close: false,

apps/mobile/app/hooks/use-app-events.js

@@ -83,7 +83,6 @@ import {
 import { getGithubVersion } from "../utils/github-version";
 import { tabBarRef } from "../utils/global-refs";
 import { sleep } from "../utils/time";
-import { useThemeColors } from "@notesnook/theme";
 
 const onCheckSyncStatus = async (type) => {
   const { disableSync, disableAutoSync } = SettingsService.get();
@@ -107,10 +106,20 @@ const onFileEncryptionProgress = ({ total, progress }) => {
     .setEncryptionProgress((progress / total).toFixed(2));
 };
 
-const onLoadingAttachmentProgress = (data) => {
-  useAttachmentStore
-    .getState()
-    .setLoading(data.total === data.current ? null : data);
+const onDownloadingAttachmentProgress = (data) => {
+  useAttachmentStore.getState().setDownloading(data);
+};
+
+const onUploadingAttachmentProgress = (data) => {
+  useAttachmentStore.getState().setUploading(data);
+};
+
+const onDownloadedAttachmentProgress = (data) => {
+  useAttachmentStore.getState().setDownloading(data);
+};
+
+const onUploadedAttachmentProgress = (data) => {
+  useAttachmentStore.getState().setUploading(data);
 };
 
 const onUserSessionExpired = async () => {
@@ -301,7 +310,16 @@ export const useAppEvents = () => {
         EVENTS.userSubscriptionUpdated,
         onUserSubscriptionStatusChanged
       ),
-      EV.subscribe(EVENTS.attachmentsLoading, onLoadingAttachmentProgress),
+      EV.subscribe(EVENTS.fileDownload, onDownloadingAttachmentProgress),
+      EV.subscribe(EVENTS.fileUpload, onUploadingAttachmentProgress),
+      EV.subscribe(EVENTS.fileDownloaded, onDownloadedAttachmentProgress),
+      EV.subscribe(EVENTS.fileUploaded, onUploadedAttachmentProgress),
+      EV.subscribe(EVENTS.downloadCanceled, (data) => {
+        useAttachmentStore.getState().setDownloading(data);
+      }),
+      EV.subscribe(EVENTS.uploadCanceled, (data) => {
+        useAttachmentStore.getState().setUploading(data);
+      }),
       eSubscribeEvent(eUserLoggedIn, onUserUpdated)
     ];
 

apps/mobile/app/hooks/use-group-options.ts

@@ -0,0 +1,49 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+import { useEffect, useState } from "react";
+import { db } from "../common/database";
+import { eSubscribeEvent, eUnSubscribeEvent } from "../services/event-manager";
+import Navigation from "../services/navigation";
+
+export function useGroupOptions(type: any) {
+  const [groupOptions, setGroupOptions] = useState(
+    db.settings?.getGroupOptions(type)
+  );
+
+  useEffect(() => {
+    const onUpdate = () => {
+      const options = db.settings?.getGroupOptions(type) as any;
+      if (
+        groupOptions?.groupBy !== options.groupBy ||
+        groupOptions?.sortBy !== options.sortBy ||
+        groupOptions?.sortDirection !== groupOptions?.sortDirection
+      ) {
+        setGroupOptions({ ...options });
+        Navigation.queueRoutesForUpdate();
+      }
+    };
+
+    eSubscribeEvent("groupOptionsUpdate", onUpdate);
+    return () => {
+      eUnSubscribeEvent("groupOptionsUpdate", onUpdate);
+    };
+  }, [type, groupOptions]);
+
+  return groupOptions;
+}

apps/mobile/app/screens/editor/progress.js

@@ -1,81 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import React, { useEffect, useRef, useState } from "react";
-import { View } from "react-native";
-import { ProgressBarComponent } from "../../components/ui/svg/lazy";
-import useGlobalSafeAreaInsets from "../../hooks/use-global-safe-area-insets";
-import { useAttachmentStore } from "../../stores/use-attachment-store";
-import { useThemeColors } from "@notesnook/theme";
-import { SIZE } from "../../utils/size";
-export const ProgressBar = () => {
-  const { colors } = useThemeColors();
-  const loading = useAttachmentStore((state) => state.loading);
-  const [prog, setProg] = useState(0);
-  const [visible, setVisible] = useState(false);
-  const timer = useRef();
-  const insets = useGlobalSafeAreaInsets();
-  const [width, setWidth] = useState(false);
-
-  useEffect(() => {
-    if (loading) {
-      if (loading.current !== loading.total) {
-        setVisible(true);
-        setProg(loading.current / loading.total);
-      } else {
-        clear();
-      }
-    } else {
-      clear();
-    }
-  }, [loading]);
-
-  const clear = () => {
-    clearTimeout(timer.current);
-    timer.current = null;
-    timer.current = setTimeout(() => {
-      setProg(1);
-      setTimeout(() => {
-        setVisible(false);
-      }, 1000);
-    }, 100);
-  };
-
-  return visible ? (
-    <View
-      style={{
-        justifyContent: "center",
-        position: "absolute",
-        zIndex: 1,
-        marginTop: insets.top + 45,
-        width: "100%"
-      }}
-      onLayout={(event) => setWidth(event.nativeEvent.layout.width)}
-    >
-      <ProgressBarComponent
-        size={SIZE.xxl}
-        progress={prog}
-        color={colors.primary.accent}
-        borderWidth={0}
-        height={1}
-        width={width || 400}
-      />
-    </View>
-  ) : null;
-};

apps/mobile/app/screens/editor/progress.tsx

@@ -0,0 +1,136 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { useThemeColors } from "@notesnook/theme";
+import React, { useEffect, useRef, useState } from "react";
+import { View } from "react-native";
+import { ProgressBarComponent } from "../../components/ui/svg/lazy";
+import useGlobalSafeAreaInsets from "../../hooks/use-global-safe-area-insets";
+import { useAttachmentStore } from "../../stores/use-attachment-store";
+import { useEditorStore } from "../../stores/use-editor-store";
+
+export const ProgressBar = () => {
+  const { colors } = useThemeColors();
+  const currentlyEditingNote = useEditorStore(
+    (state) => state.currentEditingNote
+  );
+  const downloading = useAttachmentStore((state) => state.downloading);
+
+  const loading = currentlyEditingNote
+    ? downloading?.[currentlyEditingNote]
+    : undefined;
+
+  const attachmentProgress = useAttachmentStore((state) => state.progress);
+  const [progress, setProgress] = useState(0);
+  const [visible, setVisible] = useState(false);
+  const timer = useRef<NodeJS.Timeout>();
+  const insets = useGlobalSafeAreaInsets();
+  const [width, setWidth] = useState(400);
+
+  const groupProgressInfo = useRef<{
+    [name: string]: {
+      total: number;
+      current: number;
+    };
+  }>({});
+
+  const currentItemProgress = loading?.filename
+    ? attachmentProgress?.[loading?.filename]
+    : undefined;
+
+  useEffect(() => {
+    if (loading) {
+      console.log(loading);
+      if (
+        loading.current === loading.total &&
+        typeof loading.success === "boolean"
+      ) {
+        clear();
+        return;
+      }
+      setVisible(true);
+      if (!loading.filename) return;
+
+      if (!groupProgressInfo.current[loading.filename]) {
+        groupProgressInfo.current[loading.filename] = {
+          total: 1,
+          current: 0
+        };
+      }
+
+      const itemProgressCurrent = groupProgressInfo.current[loading.filename];
+
+      const itemTotalSize =
+        currentItemProgress?.total || itemProgressCurrent.total;
+
+      const itemCurrentProgress =
+        currentItemProgress?.recieved ||
+        currentItemProgress?.sent ||
+        itemProgressCurrent.current;
+
+      groupProgressInfo.current[loading.filename] = {
+        current: itemCurrentProgress,
+        total: itemTotalSize
+      };
+
+      const itemProgressPercent = itemCurrentProgress / itemTotalSize;
+
+      setProgress(
+        ((loading.current || 0) + itemProgressPercent) / (loading.total || 1)
+      );
+    } else {
+      clear();
+    }
+  }, [currentItemProgress, loading]);
+
+  const clear = () => {
+    clearTimeout(timer.current);
+    timer.current = undefined;
+    timer.current = setTimeout(() => {
+      setProgress(1);
+      setTimeout(() => {
+        setVisible(false);
+        setProgress(0);
+        groupProgressInfo.current = {};
+      }, 1000);
+    }, 0);
+  };
+
+  return (
+    <View
+      style={{
+        justifyContent: "center",
+        position: "absolute",
+        zIndex: visible ? 1 : -1,
+        marginTop: insets.top + 45,
+        width: "100%"
+      }}
+      onLayout={(event) => setWidth(event.nativeEvent.layout.width)}
+    >
+      <ProgressBarComponent
+        progress={progress}
+        color={colors.primary.accent}
+        borderWidth={0}
+        borderRadius={0}
+        height={2}
+        width={width || 400}
+      />
+    </View>
+  );
+};

apps/mobile/app/screens/editor/tiptap/types.ts

@@ -52,6 +52,7 @@ export type Settings = {
   fontFamily: string;
   dateFormat: string;
   timeFormat: string;
+  fontScale: number;
 };
 
 export type EditorProps = {

apps/mobile/app/screens/editor/tiptap/use-editor-events.ts

@@ -28,7 +28,8 @@ import {
   InteractionManager,
   Keyboard,
   KeyboardEventListener,
-  NativeEventSubscription
+  NativeEventSubscription,
+  useWindowDimensions
 } from "react-native";
 import { WebViewMessageEvent } from "react-native-webview";
 import { db } from "../../../common/database";
@@ -142,6 +143,8 @@ export const useEditorEvents = (
   const handleBack = useRef<NativeEventSubscription>();
   const readonly = useEditorStore((state) => state.readonly);
   const isPremium = useUserStore((state) => state.premium);
+  const { fontScale } = useWindowDimensions();
+
   const doubleSpacedLines = useSettingStore(
     (state) => state.settings?.doubleSpacedLines
   );
@@ -195,7 +198,8 @@ export const useEditorEvents = (
           : defaultFontSize,
       fontFamily: SettingsService.get().defaultFontFamily,
       dateFormat: db.settings?.getDateFormat(),
-      timeFormat: db.settings?.getTimeFormat()
+      timeFormat: db.settings?.getTimeFormat(),
+      fontScale
     });
   }, [
     fullscreen,
@@ -215,7 +219,8 @@ export const useEditorEvents = (
     defaultFontFamily,
     dateFormat,
     timeFormat,
-    loading
+    loading,
+    fontScale
   ]);
 
   const onBackPress = useCallback(async () => {

apps/mobile/app/screens/editor/tiptap/use-editor.ts

@@ -22,7 +22,7 @@ import { EVENTS } from "@notesnook/core/dist/common";
 import { useThemeEngineStore } from "@notesnook/theme";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import WebView from "react-native-webview";
-import { db } from "../../../common/database";
+import { DatabaseLogger, db } from "../../../common/database";
 import useGlobalSafeAreaInsets from "../../../hooks/use-global-safe-area-insets";
 import { DDS } from "../../../services/device-detection";
 import {
@@ -134,7 +134,8 @@ export const useEditor = (
 
   const reset = useCallback(
     async (resetState = true, resetContent = true) => {
-      currentNote.current?.id && db.fs?.cancel(currentNote.current.id, null);
+      currentNote.current?.id &&
+        db.fs?.cancel(currentNote.current.id, "download");
       currentNote.current = null;
       loadedImages.current = {};
       currentContent.current = null;
@@ -147,7 +148,12 @@ export const useEditor = (
       lastContentChangeTime.current = 0;
       resetContent && (await commands.clearContent());
       resetContent && (await commands.clearTags());
+
       if (resetState) {
+        const newSessionId = makeSessionId();
+        await commands.setSessionId(newSessionId);
+        setSessionId(newSessionId);
+
         isDefaultEditor &&
           useEditorStore.getState().setCurrentlyEditingNote(null);
         placeholderTip.current = TipManager.placeholderTip();
@@ -264,7 +270,8 @@ export const useEditor = (
         saveCount.current++;
         return id;
       } catch (e) {
-        console.log("Error saving note: ", e);
+        console.error(e);
+        DatabaseLogger.error(e as Error);
       }
     },
     [commands, isDefaultEditor, postMessage, readonly, reset]
@@ -279,6 +286,7 @@ export const useEditor = (
         noteId: currentNote.current?.id as string
       };
     } else {
+      if (!note.contentId) return;
       currentContent.current = await db.content?.raw(note.contentId);
     }
   }, []);
@@ -366,12 +374,23 @@ export const useEditor = (
         if (!item.forced && currentNote.current?.id === item.id) return;
         state.current.movedAway = false;
         state.current.currentlyEditing = true;
-        isDefaultEditor && editorState.setCurrentlyEditingNote(item.id);
-        currentNote.current && (await reset(false, false));
+
+        if (currentNote.current?.id !== item.id) {
+          currentNote.current && (await reset(false, false));
+          isDefaultEditor && editorState.setCurrentlyEditingNote(item.id);
+        }
+
         await loadContent(item as NoteType);
-        if (loadingState.current === currentContent.current?.data) {
+
+        if (
+          currentNote.current?.id === item.id &&
+          loadingState.current &&
+          currentContent.current?.data &&
+          loadingState.current === currentContent.current?.data
+        ) {
           return;
         }
+
         if (
           !currentContent.current?.data ||
           currentContent.current?.data.length < 50000
@@ -389,17 +408,20 @@ export const useEditor = (
         lockedSessionId.current = nextSessionId;
         sessionHistoryId.current = Date.now();
         setSessionId(nextSessionId);
+
         commands.setSessionId(nextSessionId);
         sessionIdRef.current = nextSessionId;
         currentNote.current = item as NoteType;
         await commands.setStatus(getFormattedDate(item.dateEdited), "Saved");
         await postMessage(EditorEvents.title, item.title);
         loadingState.current = currentContent.current?.data;
-        await postMessage(
-          EditorEvents.html,
-          currentContent.current?.data,
-          10000
-        );
+        if (currentContent.current?.data) {
+          await postMessage(
+            EditorEvents.html,
+            currentContent.current?.data,
+            10000
+          );
+        }
         loadingState.current = undefined;
         useEditorStore.getState().setReadonly(item.readonly);
         await commands.setTags(currentNote.current);
@@ -525,6 +547,14 @@ export const useEditor = (
     }) => {
       if (lock.current || lockedSessionId.current === forSessionId) return;
       lastContentChangeTime.current = Date.now();
+
+      if (
+        sessionHistoryId.current &&
+        Date.now() - sessionHistoryId.current > 5 * 60 * 1000
+      ) {
+        sessionHistoryId.current = Date.now();
+      }
+
       if (type === EditorEvents.content) {
         currentContent.current = {
           data: content,

apps/mobile/app/screens/home/index.tsx

@@ -71,7 +71,7 @@ export const Home = ({ navigation, route }: NavigationProps<"Notes">) => {
       <List
         listData={notes}
         type="notes"
-        screen="Notes"
+        screen="Home"
         loading={loading || !isFocused}
         headerProps={{
           heading: "Notes"

apps/mobile/app/screens/settings/settings-data.tsx

@@ -628,7 +628,9 @@ export const settingsGroups: SettingSection[] = [
         name: "CORS bypass proxy",
         description: "You can set a custom proxy URL to increase your privacy.",
         inputProperties: {
-          defaultValue: "https://cors.notesnook.com"
+          defaultValue: "https://cors.notesnook.com",
+          autoCorrect: false,
+          keyboardType: "url"
         },
         property: "corsProxy",
         icon: "arrow-decision-outline"

apps/mobile/app/services/backup.js

@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { sanitizeFilename } from "@notesnook/common";
 import { Platform } from "react-native";
 import RNFetchBlob from "react-native-blob-util";
 import FileViewer from "react-native-file-viewer";
@@ -30,6 +29,8 @@ import { eCloseSheet } from "../utils/events";
 import { sleep } from "../utils/time";
 import { ToastEvent, eSendEvent, presentSheet } from "./event-manager";
 import SettingsService from "./settings";
+import { cacheDir, copyFileAsync } from "../common/filesystem/utils";
+import { zip } from "react-native-zip-archive";
 
 const MS_DAY = 86400000;
 const MS_WEEK = MS_DAY * 7;
@@ -152,8 +153,6 @@ async function run(progress, context) {
   let androidBackupDirectory = await checkBackupDirExists(false, context);
   if (!androidBackupDirectory) return;
 
-  let backup;
-
   if (progress) {
     presentSheet({
       title: "Backing up your data",
@@ -163,43 +162,67 @@ async function run(progress, context) {
     });
   }
 
-  try {
-    backup = await db.backup.export(
-      "mobile",
-      SettingsService.get().encryptedBackup
-    );
-    if (!backup) throw new Error("Backup returned empty.");
-  } catch (e) {
-    await sleep(300);
-    eSendEvent(eCloseSheet);
-    ToastEvent.error(e, "Backup failed!");
-    return null;
+  let path;
+  let backupFilePath;
+  let backupFileName = "notesnook_backup_" + Date.now();
+
+  if (Platform.OS === "ios") {
+    path = await storage.checkAndCreateDir("/backups/");
   }
 
-  try {
-    let backupName = "notesnook_backup_" + Date.now();
-    backupName =
-      sanitizeFilename(backupName, { replacement: "_" }) + ".nnbackup";
-    let path;
-    let backupFilePath;
+  const zipSourceFolder = `${cacheDir}/${backupFileName}`;
+  const zipOutputFile =
+    Platform.OS === "ios"
+      ? `${path}/${backupFileName}.nnbackupz`
+      : `${cacheDir}/${backupFileName}.nnbackupz`;
 
-    if (Platform.OS === "ios") {
-      path = await storage.checkAndCreateDir("/backups/");
-      await RNFetchBlob.fs.writeFile(path + backupName, backup, "utf8");
-      backupFilePath = path + backupName;
-    } else {
-      backupFilePath = await ScopedStorage.writeFile(
-        androidBackupDirectory.uri,
-        backup,
-        backupName,
-        "nnbackup/json",
-        "utf8",
-        false
+  if (await RNFetchBlob.fs.exists(zipSourceFolder))
+    await RNFetchBlob.fs.unlink(zipSourceFolder);
+
+  await RNFetchBlob.fs.mkdir(zipSourceFolder);
+
+  try {
+    for await (const file of db.backup.export(
+      "mobile",
+      SettingsService.get().encryptedBackup
+    )) {
+      console.log("Writing backup chunk of size...", file?.data?.length);
+      await RNFetchBlob.fs.writeFile(
+        `${zipSourceFolder}/${file.path}`,
+        file.data,
+        "utf8"
       );
     }
 
+    await zip(zipSourceFolder, zipOutputFile);
+
+    console.log("Final zip:", await RNFetchBlob.fs.stat(zipOutputFile));
+
+    if (Platform.OS === "android") {
+      // Move the zip to user selected directory.
+      const file = await ScopedStorage.createFile(
+        androidBackupDirectory.uri,
+        `${backupFileName}.nnbackupz`,
+        "application/nnbackupz"
+      );
+      await copyFileAsync(`file://${zipOutputFile}`, file.uri);
+      path = file.uri;
+    } else {
+      path = zipOutputFile;
+    }
+    RNFetchBlob.fs.unlink(zipSourceFolder).catch(console.log);
     updateNextBackupTime();
 
+    let showBackupCompleteSheet = SettingsService.get().showBackupCompleteSheet;
+
+    if (context) return path;
+    await sleep(300);
+    if (showBackupCompleteSheet) {
+      presentBackupCompleteSheet(backupFilePath);
+    } else {
+      progress && eSendEvent(eCloseSheet);
+    }
+
     ToastEvent.show({
       heading: "Backup successful",
       message: "Your backup is stored in Notesnook folder on your phone.",
@@ -207,18 +230,10 @@ async function run(progress, context) {
       context: "global"
     });
 
-    let showBackupCompleteSheet = SettingsService.get().showBackupCompleteSheet;
-
-    if (context) return backupFilePath;
-    await sleep(300);
-    if (showBackupCompleteSheet) {
-      presentBackupCompleteSheet(backupFilePath);
-    } else {
-      progress && eSendEvent(eCloseSheet);
-    }
-    return backupFilePath;
+    return path;
   } catch (e) {
-    progress && eSendEvent(eCloseSheet);
+    await sleep(300);
+    eSendEvent(eCloseSheet);
     ToastEvent.error(e, "Backup failed!");
     return null;
   }

apps/mobile/app/services/message.js

@@ -50,7 +50,7 @@ export function setRateAppMessage() {
 
 const recoveryKeyMessage = {
   visible: true,
-  message: "Keep your data safe if you lose password",
+  message: "Keep your data safe",
   actionText: "Save your account recovery key",
   onPress: () => {
     verifyUser(

apps/mobile/app/services/sync.js

@@ -17,15 +17,14 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { EVENTS } from "@notesnook/core/dist/common";
 import NetInfo from "@react-native-community/netinfo";
 import { db } from "../common/database";
 import { DatabaseLogger } from "../common/database/index";
 import { initAfterSync } from "../stores/index";
 import { SyncStatus, useUserStore } from "../stores/use-user-store";
+import BackgroundSync from "./background-sync";
 import { ToastEvent } from "./event-manager";
 import SettingsService from "./settings";
-import BackgroundSync from "./background-sync";
 
 export const ignoredMessages = [
   "Sync already running",
@@ -33,7 +32,7 @@ export const ignoredMessages = [
   "WebSocket failed to connect",
   "Failed to start the HttpConnection before"
 ];
-
+let pendingSync = undefined;
 let syncTimer = 0;
 const run = async (
   context = "global",
@@ -43,10 +42,18 @@ const run = async (
   lastSyncTime
 ) => {
   if (useUserStore.getState().syncing) {
-    DatabaseLogger.log("Sync in progress");
-    console.log("Sync in progress");
+    DatabaseLogger.info("Sync in progress");
+    pendingSync = {
+      full: full
+    };
     return;
   }
+
+  if (pendingSync) {
+    pendingSync = undefined;
+    DatabaseLogger.info("Running pending sync...");
+  }
+
   clearTimeout(syncTimer);
   syncTimer = setTimeout(async () => {
     const status = await NetInfo.fetch();
@@ -55,56 +62,54 @@ const run = async (
     if (!status.isInternetReachable) {
       DatabaseLogger.warn("Internet not reachable");
     }
+
     if (
       !user ||
       !status.isInternetReachable ||
       SettingsService.get().disableSync
     ) {
       initAfterSync();
+      pendingSync = undefined;
       return onCompleted?.(false);
     }
     userstore.setSyncing(true);
+
     let error = null;
 
     try {
-      let res = await BackgroundSync.doInBackground(async () => {
+      await BackgroundSync.doInBackground(async () => {
         try {
           await db.sync(full, forced, lastSyncTime);
-          return true;
         } catch (e) {
           error = e;
-          return e.message;
         }
       });
-      if (!res) {
-        initAfterSync();
-        userstore.setSyncing(false, SyncStatus.Failed);
-        return onCompleted?.(false);
+
+      if (error) {
+        throw error;
       }
-      if (typeof res === "string") throw error;
-      userstore.setSyncing(false);
-      return onCompleted?.(true);
     } catch (e) {
       error = e;
       if (
-        !ignoredMessages.find((im) => e.message?.includes(im)) &&
-        userstore.user
+        !ignoredMessages.find((message) => e.message?.includes(message)) &&
+        userstore.user &&
+        status.isConnected &&
+        status.isInternetReachable
       ) {
-        userstore.setSyncing(false, SyncStatus.Failed);
-        if (status.isConnected && status.isInternetReachable) {
-          ToastEvent.error(e, "Sync failed", context);
-        }
+        ToastEvent.error(e, "Sync failed", context);
       }
+
       DatabaseLogger.error(e, "[Client] Failed to sync");
-      onCompleted?.(false);
     } finally {
+      initAfterSync();
       userstore.setSyncing(
         false,
         error ? SyncStatus.Failed : SyncStatus.Passed
       );
-      if (full || forced) {
-        db.eventManager.publish(EVENTS.syncCompleted);
-      }
+      onCompleted?.(error ? SyncStatus.Failed : SyncStatus.Passed);
+      setImmediate(() => {
+        if (pendingSync) Sync.run("global", false, pendingSync.full);
+      });
     }
   }, 300);
 };

apps/mobile/app/stores/use-attachment-store.ts

@@ -20,6 +20,16 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import create from "zustand";
 import { editorController } from "../screens/editor/tiptap/utils";
 
+export type AttachmentGroupProgress = {
+  total: number;
+  current: number;
+  groupId: string;
+  filename: string;
+  canceled?: boolean;
+  success?: boolean;
+  error?: any;
+};
+
 interface AttachmentStore {
   progress?: {
     [name: string]: {
@@ -30,6 +40,7 @@ interface AttachmentStore {
       type: "upload" | "download";
     } | null;
   };
+
   encryptionProgress: number;
   setEncryptionProgress: (encryptionProgress: number) => void;
   remove: (hash: string) => void;
@@ -40,8 +51,14 @@ interface AttachmentStore {
     recieved: number,
     type: "upload" | "download"
   ) => void;
-  loading: { total: number; current: number };
-  setLoading: (data: { total: number; current: number }) => void;
+  downloading?: {
+    [groupId: string]: AttachmentGroupProgress | undefined;
+  };
+  setDownloading: (data: AttachmentGroupProgress) => void;
+  uploading?: {
+    [groupId: string]: AttachmentGroupProgress | undefined;
+  };
+  setUploading: (data: AttachmentGroupProgress) => void;
 }
 
 export const useAttachmentStore = create<AttachmentStore>((set, get) => ({
@@ -73,6 +90,21 @@ export const useAttachmentStore = create<AttachmentStore>((set, get) => ({
   encryptionProgress: 0,
   setEncryptionProgress: (encryptionProgress) =>
     set({ encryptionProgress: encryptionProgress }),
-  loading: { total: 0, current: 0 },
-  setLoading: (data) => set({ loading: data ? { ...data } : data })
+
+  downloading: {},
+  setDownloading: (data) =>
+    set({
+      downloading: {
+        ...get().downloading,
+        [data.groupId]: data?.canceled ? undefined : data
+      }
+    }),
+  uploading: {},
+  setUploading: (data) =>
+    set({
+      uploading: {
+        ...get().uploading,
+        [data.groupId]: data?.canceled ? undefined : data
+      }
+    })
 }));

apps/mobile/app/utils/functions.js

@@ -107,46 +107,43 @@ export const deleteItems = async (item, context) => {
 
   if (topics?.length > 0) {
     const result = await confirmDeleteAllNotes(topics, "topic", context);
-    if (result.delete) {
-      for (const topic of topics) {
-        if (result.deleteNotes) {
+    if (!result.delete) return;
+    for (const topic of topics) {
+      if (result.deleteNotes) {
+        const notes = db.notebooks
+          .notebook(topic.notebookId)
+          .topics.topic(topic.id).all;
+        await db.notes.delete(...notes.map((note) => note.id));
+      }
+      await db.notebooks.notebook(topic.notebookId).topics.delete(topic.id);
+    }
+    useMenuStore.getState().setMenuPins();
+    ToastEvent.show({
+      heading: `${topics.length > 1 ? "Topics" : "Topic"} deleted`,
+      type: "success"
+    });
+  }
+
+  if (notebooks?.length > 0) {
+    const result = await confirmDeleteAllNotes(notebooks, "notebook", context);
+    if (!result.delete) return;
+    let ids = notebooks.map((i) => i.id);
+    if (result.deleteNotes) {
+      for (let id of ids) {
+        const notebook = db.notebooks.notebook(id);
+        const topics = notebook.topics.all;
+        for (let topic of topics) {
           const notes = db.notebooks
             .notebook(topic.notebookId)
             .topics.topic(topic.id).all;
           await db.notes.delete(...notes.map((note) => note.id));
         }
-        await db.notebooks.notebook(topic.notebookId).topics.delete(topic.id);
+        const notes = db.relations.from(notebook.data, "note");
+        await db.notes.delete(...notes.map((note) => note.id));
       }
-      useMenuStore.getState().setMenuPins();
-      ToastEvent.show({
-        heading: `${topics.length > 1 ? "Topics" : "Topic"} deleted`,
-        type: "success"
-      });
-    }
-  }
-
-  if (notebooks?.length > 0) {
-    const result = await confirmDeleteAllNotes(notebooks, "notebook", context);
-
-    if (result.delete) {
-      let ids = notebooks.map((i) => i.id);
-      if (result.deleteNotes) {
-        for (let id of ids) {
-          const notebook = db.notebooks.notebook(id);
-          const topics = notebook.topics.all;
-          for (let topic of topics) {
-            const notes = db.notebooks
-              .notebook(topic.notebookId)
-              .topics.topic(topic.id).all;
-            await db.notes.delete(...notes.map((note) => note.id));
-          }
-          const notes = db.relations.from(notebook.data, "note");
-          await db.notes.delete(...notes.map((note) => note.id));
-        }
-      }
-      await db.notebooks.delete(...ids);
-      useMenuStore.getState().setMenuPins();
     }
+    await db.notebooks.delete(...ids);
+    useMenuStore.getState().setMenuPins();
   }
 
   Navigation.queueRoutesForUpdate();

apps/mobile/app/utils/note-bundle.js

@@ -24,6 +24,12 @@ import RNFetchBlob from "react-native-blob-util";
 import { db } from "../common/database";
 import { IOS_APPGROUPID } from "./constants";
 
+const santizeUri = (uri) => {
+  uri = decodeURI(uri);
+  uri = Platform.OS === "ios" ? uri.replace("file:///", "/") : uri;
+  return uri;
+};
+
 export async function attachFile(uri, hash, type, filename, options) {
   try {
     let exists = db.attachments.exists(hash);
@@ -76,9 +82,7 @@ async function createNotes(bundle) {
 
   for (const file of bundle.files) {
     const uri =
-      Platform.OS === "ios"
-        ? `${file.value.replace("file://", "")}`
-        : `${file.value}`;
+      Platform.OS === "ios" ? santizeUri(file.value) : `${file.value}`;
     const hash = await Sodium.hashFile({
       uri: uri,
       type: "cache"

apps/mobile/e2e/tests/notebook.e2e.js

@@ -33,7 +33,7 @@ import {
   visibleById
 } from "./utils";
 
-async function createNotebook(
+export async function createNotebook(
   title = "Notebook 1",
   description = true,
   topic = true,

apps/mobile/e2e/tests/sort.e2e.js

@@ -18,6 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import { notesnook } from "../test.ids";
+import { createNotebook } from "./notebook.e2e";
 import {
   tapById,
   visibleByText,
@@ -25,7 +26,8 @@ import {
   prepare,
   tapByText,
   notVisibleByText,
-  sleep
+  sleep,
+  navigate
 } from "./utils";
 
 async function sortBy(sorting, elementText = "Default") {
@@ -104,6 +106,23 @@ describe("Sort & filter", () => {
     await visibleByText("Month");
   });
 
+  it("Sort notes in topic", async () => {
+    await prepare();
+    await navigate("Notebooks");
+    await sleep(500);
+    await createNotebook("Notebook 1", true, true);
+    await sleep(500);
+    await device.pressBack();
+    await sleep(500);
+    await tapByText("Topic");
+    await createNote("A", "A letter");
+    await sleep(500);
+    await createNote("B", "B letter");
+    await sortBy("Abc");
+    await sleep(300);
+    await visibleByText("N");
+  });
+
   it("Compact mode", async () => {
     await prepare();
     await createNote("Note 1", "Note 1");

apps/mobile/native/android/app/build.gradle

@@ -111,7 +111,7 @@ android {
         minSdkVersion rootProject.ext.minSdkVersion
         targetSdkVersion rootProject.ext.targetSdkVersion
         multiDexEnabled true
-        versionCode 2059
+        versionCode 2063
         versionName getNpmVersion()
         testBuildType System.getProperty('testBuildType', 'debug')
         testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner'

apps/mobile/native/android/releasenotes/whatsnew-en-US

@@ -1,4 +1,3 @@
-- Much improved and faster sync
-- Note pinned in notifications will update automatically when content changes
-- Quick notes from notifications now respect paragraph spacing settings.
-- Many bug fixes and small improvements
+- New backup format nnbackupz with upto 90% smaller backups
+- Improve UI of code blocks & language selection popup
+- Fixed some bugs in sync v2

apps/mobile/native/ios/Notesnook.xcodeproj/project.pbxproj

@@ -997,7 +997,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2056;
 				DEVELOPMENT_TEAM = 53CWBG3QUC;
 				ENABLE_BITCODE = NO;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1071,7 +1071,7 @@
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.5;
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",
@@ -1102,7 +1102,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution";
 				CODE_SIGN_STYLE = Manual;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2056;
 				DEVELOPMENT_TEAM = "";
 				"DEVELOPMENT_TEAM[sdk=iphoneos*]" = 53CWBG3QUC;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1176,7 +1176,7 @@
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.5;
 				ONLY_ACTIVE_ARCH = NO;
 				OTHER_LDFLAGS = (
 					"$(inherited)",
@@ -1335,7 +1335,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2056;
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				DEVELOPMENT_TEAM = 53CWBG3QUC;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1347,7 +1347,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.5;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.notewidget;
@@ -1378,7 +1378,7 @@
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution";
 				CODE_SIGN_STYLE = Manual;
 				COPY_PHASE_STRIP = NO;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2056;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				DEVELOPMENT_TEAM = "";
 				"DEVELOPMENT_TEAM[sdk=iphoneos*]" = 53CWBG3QUC;
@@ -1391,7 +1391,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.5;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.notewidget;
 				PRODUCT_NAME = "$(TARGET_NAME)";
@@ -1421,7 +1421,7 @@
 				CODE_SIGN_ENTITLEMENTS = "Make Note/Make Note.entitlements";
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2056;
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				DEVELOPMENT_TEAM = 53CWBG3QUC;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1495,7 +1495,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.5;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.share;
@@ -1526,7 +1526,7 @@
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution";
 				CODE_SIGN_STYLE = Manual;
 				COPY_PHASE_STRIP = NO;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2056;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				DEVELOPMENT_TEAM = "";
 				"DEVELOPMENT_TEAM[sdk=iphoneos*]" = 53CWBG3QUC;
@@ -1601,7 +1601,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.5;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.share;
 				PRODUCT_NAME = "$(TARGET_NAME)";

apps/mobile/native/ios/Podfile.lock

@@ -327,7 +327,7 @@ PODS:
     - React-Core
   - react-native-keep-awake (1.2.0):
     - React-Core
-  - react-native-mmkv-storage (0.10.0-alpha.7):
+  - react-native-mmkv-storage (0.10.0-alpha.9):
     - MMKV (~> 1.3.1)
     - React
     - React-Core
@@ -875,7 +875,7 @@ SPEC CHECKSUMS:
   react-native-image-resizer: 00ceb0e05586c7aadf061eea676957a6c2ec60fa
   react-native-in-app-review: db8bb167a5f238e7ceca5c242d6b36ce8c4404a4
   react-native-keep-awake: caee3ff89eaa21dfe29010f0d143566874a04441
-  react-native-mmkv-storage: ec32fd68168716755e1b30f09b476c236c6c68a8
+  react-native-mmkv-storage: d4ad55ab411b7f0d4e6269d801b31821ef48970b
   react-native-netinfo: ccbe1085dffd16592791d550189772e13bf479e2
   react-native-notification-sounds: da78c828fe1bcbb92d8b505d5261890ed315ff39
   react-native-orientation: f1caf84d65f1a4fd4511a18f2b924e634ad7a628

apps/mobile/native/package.json

@@ -7,7 +7,7 @@
   "dependencies": {
     "@ammarahmed/notifee-react-native": "7.4.4",
     "@ammarahmed/react-native-share-extension": "^2.5.5",
-    "@ammarahmed/react-native-sodium": "1.5.2",
+    "@ammarahmed/react-native-sodium": "1.5.4",
     "@bam.tech/react-native-image-resizer": "3.0.5",
     "@callstack/repack": "^3.2.0",
     "@react-native-clipboard/clipboard": "^1.9.0",

apps/mobile/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@notesnook/mobile",
-  "version": "2.6.2",
+  "version": "2.6.5",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@notesnook/mobile",
-      "version": "2.6.2",
+      "version": "2.6.5",
       "hasInstallScript": true,
       "license": "GPL-3.0-or-later",
       "workspaces": [
@@ -22,9 +22,9 @@
         "@notesnook/logger": "file:../../packages/logger",
         "@notesnook/themes-server": "file:../../servers/themes",
         "@tanstack/react-query": "^4.29.19",
-        "@trpc/client": "10.31.0",
-        "@trpc/react-query": "10.31.0",
-        "@trpc/server": "^10.31.0",
+        "@trpc/client": "10.38.3",
+        "@trpc/react-query": "10.38.3",
+        "@trpc/server": "10.38.3",
         "react": "18.2.0",
         "react-native": "0.72.0"
       },
@@ -85,7 +85,9 @@
         "abortcontroller-polyfill": "^1.7.3",
         "cross-env": "^7.0.3",
         "dotenv": "^16.0.1",
+        "event-source-polyfill": "^1.0.31",
         "eventsource": "^2.0.2",
+        "hash-wasm": "^4.9.0",
         "isomorphic-fetch": "^3.0.0",
         "jsdom": "^22.1.0",
         "mockdate": "^3.0.5",
@@ -222,7 +224,7 @@
       "license": "ISC",
       "dependencies": {
         "@orama/orama": "^1.0.8",
-        "@trpc/server": "10.31.0",
+        "@trpc/server": "10.38.3",
         "async-mutex": "^0.4.0",
         "cors": "^2.8.5",
         "util": "^0.12.5",
@@ -279,7 +281,7 @@
         "@ammarahmed/notifee-react-native": "7.4.4",
         "@ammarahmed/react-native-eventsource": "1.1.0",
         "@ammarahmed/react-native-share-extension": "^2.5.5",
-        "@ammarahmed/react-native-sodium": "1.5.2",
+        "@ammarahmed/react-native-sodium": "1.5.4",
         "@bam.tech/react-native-image-resizer": "3.0.5",
         "@callstack/repack": "^3.2.0",
         "@react-native-clipboard/clipboard": "^1.9.0",
@@ -2417,9 +2419,9 @@
       }
     },
     "node_modules/@ammarahmed/react-native-sodium": {
-      "version": "1.5.2",
-      "resolved": "https://registry.npmjs.org/@ammarahmed/react-native-sodium/-/react-native-sodium-1.5.2.tgz",
-      "integrity": "sha512-M309uV2DCK3EPNfsgWYthOApJhQgQKOOxYF19VerUeX7UYnm9VggW073oO0NNqqZqg5GS6bTlrtYMAz7MAKvUw=="
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/@ammarahmed/react-native-sodium/-/react-native-sodium-1.5.4.tgz",
+      "integrity": "sha512-+JwYnaSJxnwpI1947sUacFXxRxK1f9ykuxDJaGkCUKvh5CkDYC5VyZFnq2BbcuIcx4OTMOG8a/a46hWJbFKQPQ=="
     },
     "node_modules/@ampproject/remapping": {
       "version": "2.2.1",
@@ -7012,35 +7014,35 @@
       "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="
     },
     "node_modules/@trpc/client": {
-      "version": "10.31.0",
-      "resolved": "https://registry.npmjs.org/@trpc/client/-/client-10.31.0.tgz",
-      "integrity": "sha512-VCqbJEvFJb8C4hQFw7AD+dkQTjgEdV/QAzO4D+/cX5e93u5NpfNXI+PKS0QFXwG/zqgwQwVV6OkYc/D/MFwA6g==",
+      "version": "10.38.3",
+      "resolved": "https://registry.npmjs.org/@trpc/client/-/client-10.38.3.tgz",
+      "integrity": "sha512-hHPsElTL4sB+UGzuAZ4iwRVHjELrk/Nsl2owsYw9ITJz3EY7VWRsQ6vK48kJvlHPYPhP+efID5UOgyZMs4bFXA==",
       "funding": [
         "https://trpc.io/sponsor"
       ],
       "peerDependencies": {
-        "@trpc/server": "10.31.0"
+        "@trpc/server": "10.38.3"
       }
     },
     "node_modules/@trpc/react-query": {
-      "version": "10.31.0",
-      "resolved": "https://registry.npmjs.org/@trpc/react-query/-/react-query-10.31.0.tgz",
-      "integrity": "sha512-+M8sIsbf6e4H5XYvHlzDqhaf+ybfUigA/9OL3wXRp2vXhCedEiIERCnwNuHWFDRASl9vjOcM33AuJ4sbOOINEA==",
+      "version": "10.38.3",
+      "resolved": "https://registry.npmjs.org/@trpc/react-query/-/react-query-10.38.3.tgz",
+      "integrity": "sha512-j4xxg/y+vssGinZCrNQy0SMJNL2Nv8k3HcqMKa2mYt3jxOpaHuwxgarWvLWVqrUJqvp1JpGJ9sIjlOAZam5FeQ==",
       "funding": [
         "https://trpc.io/sponsor"
       ],
       "peerDependencies": {
         "@tanstack/react-query": "^4.18.0",
-        "@trpc/client": "10.31.0",
-        "@trpc/server": "10.31.0",
+        "@trpc/client": "10.38.3",
+        "@trpc/server": "10.38.3",
         "react": ">=16.8.0",
         "react-dom": ">=16.8.0"
       }
     },
     "node_modules/@trpc/server": {
-      "version": "10.31.0",
-      "resolved": "https://registry.npmjs.org/@trpc/server/-/server-10.31.0.tgz",
-      "integrity": "sha512-9EnRTSDE9nF11LZsvSOqNKqkRYzHqFX4ch5AJ6VIu8uta2vxVTN4FxxsNRSOluTzVYZDeaCISbwmOJ5iihCCIg==",
+      "version": "10.38.3",
+      "resolved": "https://registry.npmjs.org/@trpc/server/-/server-10.38.3.tgz",
+      "integrity": "sha512-9s8/kwo2IDB5hwB2SKZZrfevRhdb1f9fdXtIYd3lbQuf2jQaC/LyQuHaIQjDQoUx9updBfsHXcFFPiCP1DL6pg==",
       "funding": [
         "https://trpc.io/sponsor"
       ]
@@ -23701,9 +23703,9 @@
       }
     },
     "@ammarahmed/react-native-sodium": {
-      "version": "1.5.2",
-      "resolved": "https://registry.npmjs.org/@ammarahmed/react-native-sodium/-/react-native-sodium-1.5.2.tgz",
-      "integrity": "sha512-M309uV2DCK3EPNfsgWYthOApJhQgQKOOxYF19VerUeX7UYnm9VggW073oO0NNqqZqg5GS6bTlrtYMAz7MAKvUw=="
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/@ammarahmed/react-native-sodium/-/react-native-sodium-1.5.4.tgz",
+      "integrity": "sha512-+JwYnaSJxnwpI1947sUacFXxRxK1f9ykuxDJaGkCUKvh5CkDYC5VyZFnq2BbcuIcx4OTMOG8a/a46hWJbFKQPQ=="
     },
     "@ampproject/remapping": {
       "version": "2.2.1",
@@ -26438,7 +26440,9 @@
         "dayjs": "1.11.9",
         "dotenv": "^16.0.1",
         "entities": "^4.3.1",
+        "event-source-polyfill": "^1.0.31",
         "eventsource": "^2.0.2",
+        "hash-wasm": "^4.9.0",
         "html-to-text": "^9.0.5",
         "htmlparser2": "^8.0.1",
         "isomorphic-fetch": "^3.0.0",
@@ -26596,7 +26600,7 @@
         "@ammarahmed/notifee-react-native": "7.4.4",
         "@ammarahmed/react-native-eventsource": "1.1.0",
         "@ammarahmed/react-native-share-extension": "^2.5.5",
-        "@ammarahmed/react-native-sodium": "1.5.2",
+        "@ammarahmed/react-native-sodium": "1.5.4",
         "@babel/core": "^7.20.0",
         "@babel/eslint-parser": "^7.16.5",
         "@babel/plugin-transform-named-capturing-groups-regex": "^7.16.5",
@@ -26716,7 +26720,7 @@
       "requires": {
         "@notesnook/theme": "file:../../packages/theme",
         "@orama/orama": "^1.0.8",
-        "@trpc/server": "10.31.0",
+        "@trpc/server": "10.38.3",
         "@types/cors": "^2.8.13",
         "@vitejs/plugin-react": "^4.0.3",
         "async-mutex": "^0.4.0",
@@ -27325,21 +27329,21 @@
       "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="
     },
     "@trpc/client": {
-      "version": "10.31.0",
-      "resolved": "https://registry.npmjs.org/@trpc/client/-/client-10.31.0.tgz",
-      "integrity": "sha512-VCqbJEvFJb8C4hQFw7AD+dkQTjgEdV/QAzO4D+/cX5e93u5NpfNXI+PKS0QFXwG/zqgwQwVV6OkYc/D/MFwA6g==",
+      "version": "10.38.3",
+      "resolved": "https://registry.npmjs.org/@trpc/client/-/client-10.38.3.tgz",
+      "integrity": "sha512-hHPsElTL4sB+UGzuAZ4iwRVHjELrk/Nsl2owsYw9ITJz3EY7VWRsQ6vK48kJvlHPYPhP+efID5UOgyZMs4bFXA==",
       "requires": {}
     },
     "@trpc/react-query": {
-      "version": "10.31.0",
-      "resolved": "https://registry.npmjs.org/@trpc/react-query/-/react-query-10.31.0.tgz",
-      "integrity": "sha512-+M8sIsbf6e4H5XYvHlzDqhaf+ybfUigA/9OL3wXRp2vXhCedEiIERCnwNuHWFDRASl9vjOcM33AuJ4sbOOINEA==",
+      "version": "10.38.3",
+      "resolved": "https://registry.npmjs.org/@trpc/react-query/-/react-query-10.38.3.tgz",
+      "integrity": "sha512-j4xxg/y+vssGinZCrNQy0SMJNL2Nv8k3HcqMKa2mYt3jxOpaHuwxgarWvLWVqrUJqvp1JpGJ9sIjlOAZam5FeQ==",
       "requires": {}
     },
     "@trpc/server": {
-      "version": "10.31.0",
-      "resolved": "https://registry.npmjs.org/@trpc/server/-/server-10.31.0.tgz",
-      "integrity": "sha512-9EnRTSDE9nF11LZsvSOqNKqkRYzHqFX4ch5AJ6VIu8uta2vxVTN4FxxsNRSOluTzVYZDeaCISbwmOJ5iihCCIg=="
+      "version": "10.38.3",
+      "resolved": "https://registry.npmjs.org/@trpc/server/-/server-10.38.3.tgz",
+      "integrity": "sha512-9s8/kwo2IDB5hwB2SKZZrfevRhdb1f9fdXtIYd3lbQuf2jQaC/LyQuHaIQjDQoUx9updBfsHXcFFPiCP1DL6pg=="
     },
     "@tsconfig/react-native": {
       "version": "3.0.2",

apps/mobile/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@notesnook/mobile",
-  "version": "2.6.3",
+  "version": "2.6.5",
   "private": true,
   "license": "GPL-3.0-or-later",
   "workspaces": [
@@ -37,10 +37,10 @@
     "@notesnook/logger": "file:../../packages/logger",
     "@notesnook/themes-server": "file:../../servers/themes",
     "@tanstack/react-query": "^4.29.19",
-    "@trpc/client": "10.31.0",
-    "@trpc/react-query": "10.31.0",
-    "@trpc/server": "^10.31.0",
+    "@trpc/client": "10.38.3",
+    "@trpc/react-query": "10.38.3",
+    "@trpc/server": "10.38.3",
     "react": "18.2.0",
     "react-native": "0.72.0"
   }
-}
+}

apps/theme-builder/package-lock.json

@@ -17,6 +17,7 @@
         "@notesnook/web": "file:../web",
         "@theme-ui/components": "^0.14.7",
         "@theme-ui/core": "^0.14.7",
+        "@trpc/client": "^10.38.3",
         "clipboard-polyfill": "^4.0.0",
         "file-saver": "^2.0.5",
         "katex": "^0.16.2",
@@ -1376,7 +1377,7 @@
     },
     "../web": {
       "name": "@notesnook/web",
-      "version": "2.6.1",
+      "version": "2.6.5",
       "license": "GPL-3.0-or-later",
       "dependencies": {
         "@aws-sdk/util-base64-browser": "^3.208.0",
@@ -1405,8 +1406,8 @@
         "@theme-ui/color": "^0.14.7",
         "@theme-ui/components": "^0.14.7",
         "@theme-ui/core": "^0.14.7",
-        "@trpc/client": "10.31.0",
-        "@trpc/react-query": "10.31.0",
+        "@trpc/client": "10.38.3",
+        "@trpc/react-query": "10.38.3",
         "allotment": "^1.19.0",
         "axios": "^1.3.4",
         "clipboard-polyfill": "4.0.0",
@@ -1445,7 +1446,7 @@
       "devDependencies": {
         "@babel/core": "^7.22.5",
         "@playwright/test": "^1.36.2",
-        "@trpc/server": "10.31.0",
+        "@trpc/server": "10.38.3",
         "@types/babel__core": "^7.20.1",
         "@types/file-saver": "^2.0.5",
         "@types/marked": "^4.0.7",
@@ -2394,6 +2395,26 @@
         "react": ">16"
       }
     },
+    "node_modules/@trpc/client": {
+      "version": "10.38.3",
+      "resolved": "https://registry.npmjs.org/@trpc/client/-/client-10.38.3.tgz",
+      "integrity": "sha512-hHPsElTL4sB+UGzuAZ4iwRVHjELrk/Nsl2owsYw9ITJz3EY7VWRsQ6vK48kJvlHPYPhP+efID5UOgyZMs4bFXA==",
+      "funding": [
+        "https://trpc.io/sponsor"
+      ],
+      "peerDependencies": {
+        "@trpc/server": "10.38.3"
+      }
+    },
+    "node_modules/@trpc/server": {
+      "version": "10.38.3",
+      "resolved": "https://registry.npmjs.org/@trpc/server/-/server-10.38.3.tgz",
+      "integrity": "sha512-9s8/kwo2IDB5hwB2SKZZrfevRhdb1f9fdXtIYd3lbQuf2jQaC/LyQuHaIQjDQoUx9updBfsHXcFFPiCP1DL6pg==",
+      "funding": [
+        "https://trpc.io/sponsor"
+      ],
+      "peer": true
+    },
     "node_modules/@types/babel__core": {
       "version": "7.20.1",
       "dev": true,

apps/theme-builder/package.json

@@ -16,6 +16,7 @@
     "@notesnook/web": "file:../web",
     "@theme-ui/components": "^0.14.7",
     "@theme-ui/core": "^0.14.7",
+    "@trpc/client": "^10.38.3",
     "clipboard-polyfill": "^4.0.0",
     "file-saver": "^2.0.5",
     "katex": "^0.16.2",

apps/theme-builder/src/components/theme-builder/index.tsx

@@ -643,13 +643,25 @@ function exportTheme(theme: ThemeDefinition) {
     }),
     `${theme.id}.json`
   );
+  const confirmed = window.confirm(
+    "Do you also want to download code-block.css for this theme?"
+  );
+  if (confirmed) {
+    FileSaver.saveAs(
+      new Blob([theme.codeBlockCSS], {
+        type: "text/plain"
+      }),
+      `code-block.css`
+    );
+  }
 }
 
 function themeToJSON(theme: ThemeDefinition) {
   return JSON.stringify(
     {
       $schema: JSON_SCHEMA_URL,
-      ...theme
+      ...theme,
+      codeBlockCSS: undefined
     },
     undefined,
     2

apps/web/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@notesnook/web",
   "description": "Your private note taking space",
-  "version": "2.6.3",
+  "version": "2.6.5",
   "private": true,
   "main": "./src/app.js",
   "homepage": "https://notesnook.com/",
@@ -34,8 +34,8 @@
     "@theme-ui/color": "^0.14.7",
     "@theme-ui/components": "^0.14.7",
     "@theme-ui/core": "^0.14.7",
-    "@trpc/client": "10.31.0",
-    "@trpc/react-query": "10.31.0",
+    "@trpc/client": "10.38.3",
+    "@trpc/react-query": "10.38.3",
     "allotment": "^1.19.0",
     "axios": "^1.3.4",
     "clipboard-polyfill": "4.0.0",
@@ -74,7 +74,7 @@
   "devDependencies": {
     "@babel/core": "^7.22.5",
     "@playwright/test": "^1.36.2",
-    "@trpc/server": "10.31.0",
+    "@trpc/server": "10.38.3",
     "@types/babel__core": "^7.20.1",
     "@types/file-saver": "^2.0.5",
     "@types/marked": "^4.0.7",

apps/web/public/stream-saver-sw.js

@@ -1,154 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-/* eslint-disable no-restricted-globals */
-
-self.addEventListener("install", () => {
-  self.skipWaiting();
-});
-
-self.addEventListener("activate", (event) => {
-  event.waitUntil(self.clients.claim());
-});
-
-const map = new Map();
-
-// This should be called once per download
-// Each event has a dataChannel that the data will be piped through
-self.onmessage = (event) => {
-  // We send a heartbeat every x second to keep the
-  // service worker alive if a transferable stream is not sent
-  if (event.data === "ping") {
-    return;
-  }
-
-  const data = event.data;
-  const downloadUrl =
-    data.url ||
-    self.registration.scope +
-      Math.random() +
-      "/" +
-      (typeof data === "string" ? data : data.filename);
-  const port = event.ports[0];
-  const metadata = new Array(3); // [stream, data, port]
-
-  metadata[1] = data;
-  metadata[2] = port;
-
-  if (event.data.transferringReadable) {
-    port.onmessage = (evt) => {
-      port.onmessage = null;
-      metadata[0] = evt.data.readableStream;
-    };
-  } else {
-    metadata[0] = createStream(port);
-  }
-
-  map.set(downloadUrl, metadata);
-  port.postMessage({ download: downloadUrl });
-};
-
-function createStream(port) {
-  // ReadableStream is only supported by chrome 52
-  return new ReadableStream({
-    start(controller) {
-      // When we receive data on the messageChannel, we write
-      port.onmessage = ({ data }) => {
-        if (data === "end") {
-          return controller.close();
-        }
-
-        if (data === "abort") {
-          controller.error("Aborted the download");
-          return;
-        }
-
-        controller.enqueue(data);
-      };
-    },
-    cancel(reason) {
-      console.log("user aborted", reason);
-      port.postMessage({ abort: true });
-    }
-  });
-}
-
-self.onfetch = (event) => {
-  const url = event.request.url;
-
-  // this only works for Firefox
-  if (url.endsWith("/ping")) {
-    return event.respondWith(new Response("pong"));
-  }
-
-  const metadata = map.get(url);
-  if (!metadata) return null;
-
-  const [stream, data, port] = metadata;
-
-  map.delete(url);
-
-  // Not comfortable letting any user control all headers
-  // so we only copy over the length & disposition
-  const responseHeaders = new Headers({
-    "Content-Type": "application/octet-stream; charset=utf-8",
-
-    // To be on the safe side, The link can be opened in a iframe.
-    // but octet-stream should stop it.
-    "Content-Security-Policy": "default-src 'none'",
-    "X-Content-Security-Policy": "default-src 'none'",
-    "X-WebKit-CSP": "default-src 'none'",
-    "X-XSS-Protection": "1; mode=block"
-  });
-
-  let headers = new Headers(data.headers || {});
-
-  if (headers.has("Content-Length")) {
-    responseHeaders.set("Content-Length", headers.get("Content-Length"));
-  }
-
-  if (headers.has("Content-Disposition")) {
-    responseHeaders.set(
-      "Content-Disposition",
-      headers.get("Content-Disposition")
-    );
-  }
-
-  // data, data.filename and size should not be used anymore
-  if (data.size) {
-    console.warn("Depricated");
-    responseHeaders.set("Content-Length", data.size);
-  }
-
-  let fileName = typeof data === "string" ? data : data.filename;
-  if (fileName) {
-    console.warn("Depricated");
-    // Make filename RFC5987 compatible
-    fileName = encodeURIComponent(fileName)
-      .replace(/['()]/g, escape)
-      .replace(/\*/g, "%2A");
-    responseHeaders.set(
-      "Content-Disposition",
-      "attachment; filename*=UTF-8''" + fileName
-    );
-  }
-
-  event.respondWith(new Response(stream, { headers: responseHeaders }));
-
-  port.postMessage({ debug: "Download started" });
-};

apps/web/src/app-effects.tsx

@@ -122,28 +122,47 @@ export default function AppEffects({ setShow }: AppEffectsProps) {
     //   }
     // );
 
-    const attachmentsLoadingEvent = EV.subscribe(
-      EVENTS.attachmentsLoading,
-      ({
-        type,
-        total,
-        current
-      }: {
-        type: ProcessingType;
-        total: number;
-        current: number;
-      }) => {
-        const [key, status] = getProcessingStatusFromType(type);
+    function handleDownloadUploadProgress(
+      type: ProcessingType,
+      total: number,
+      current: number
+    ) {
+      const [key, status] = getProcessingStatusFromType(type);
 
-        if (current === total) {
-          removeStatus(key);
-        } else {
-          updateStatus({
-            key,
-            status: `${status} attachments (${current}/${total})`,
-            progress: 0
-          });
-        }
+      console.log("handleDownloadUploadProgresss", key, status, current, total);
+      if (current === total) {
+        removeStatus(key);
+      } else {
+        updateStatus({
+          key,
+          status: `${status} attachments`,
+          current,
+          total,
+          progress: 0
+        });
+      }
+    }
+
+    const fileDownloadEvents = EV.subscribeMulti(
+      [EVENTS.fileDownloaded, EVENTS.fileDownload],
+      ({ total, current }: { total: number; current: number }) => {
+        handleDownloadUploadProgress("download", total, current);
+      },
+      null
+    );
+
+    const fileUploadEvents = EV.subscribeMulti(
+      [EVENTS.fileUploaded, EVENTS.fileUpload],
+      ({ total, current }: { total: number; current: number }) => {
+        handleDownloadUploadProgress("upload", total, current);
+      },
+      null
+    );
+
+    const fileEncrypted = AppEventManager.subscribe(
+      AppEvents.fileEncrypted,
+      ({ total, current }: { total: number; current: number }) => {
+        handleDownloadUploadProgress("encrypt", total, current);
       }
     );
 
@@ -162,20 +181,36 @@ export default function AppEffects({ setShow }: AppEffectsProps) {
         if (!key) return;
 
         const percent = Math.round((loaded / total) * 100);
-        const text = getStatus(key)?.status || `${status} attachment`;
+        const oldStatus = getStatus(key);
+        const text = oldStatus?.status || `${status} attachment`;
 
-        updateStatus({
-          key,
-          status: text,
-          progress: loaded === total ? 100 : percent
-        });
+        if (
+          (!oldStatus ||
+            (oldStatus.total === undefined &&
+              oldStatus.current === undefined) ||
+            oldStatus.total === oldStatus.current) &&
+          loaded === total
+        ) {
+          removeStatus(key);
+        } else {
+          updateStatus({
+            ...oldStatus,
+            key,
+            status: text,
+            progress: loaded === total ? 100 : percent
+          });
+        }
       }
     );
 
     registerKeyMap();
     return () => {
-      attachmentsLoadingEvent.unsubscribe();
-      progressEvent.unsubscribe();
+      [
+        ...fileDownloadEvents,
+        ...fileUploadEvents,
+        progressEvent,
+        fileEncrypted
+      ].forEach((e) => e.unsubscribe());
       //  systemTimeInvalidEvent.unsubscribe();
     };
   }, []);

apps/web/src/common/app-events.ts

@@ -24,6 +24,7 @@ export const AppEvents = {
   UPDATE_ATTACHMENT_PROGRESS: "updateAttachmentProgress",
   UPDATE_STATUS: "updateStatus",
   REMOVE_STATUS: "removeStatus",
+  fileEncrypted: "file:encrypted",
 
   checkingForUpdate: "checkingForUpdate",
   updateAvailable: "updateAvailable",

apps/web/src/common/desktop-bridge/index.desktop.ts

@@ -21,40 +21,48 @@ import { createTRPCProxyClient } from "@trpc/client";
 import { ipcLink } from "electron-trpc/renderer";
 import type { AppRouter } from "@notesnook/desktop";
 import { AppEventManager, AppEvents } from "../app-events";
+import { TaskScheduler } from "../../utils/task-scheduler";
+import { checkForUpdate } from "../../utils/updater";
 
 export const desktop = createTRPCProxyClient<AppRouter>({
   links: [ipcLink()]
 });
 
-desktop.updater.onChecking.subscribe(
-  undefined,
-  attachListener(AppEvents.checkingForUpdate)
-);
+document.addEventListener("readystatechange", async () => {
+  desktop.updater.onChecking.subscribe(
+    undefined,
+    attachListener(AppEvents.checkingForUpdate)
+  );
 
-desktop.updater.onAvailable.subscribe(
-  undefined,
-  attachListener(AppEvents.updateAvailable)
-);
+  desktop.updater.onAvailable.subscribe(
+    undefined,
+    attachListener(AppEvents.updateAvailable)
+  );
 
-desktop.updater.onDownloaded.subscribe(
-  undefined,
-  attachListener(AppEvents.updateDownloadCompleted)
-);
+  desktop.updater.onDownloaded.subscribe(
+    undefined,
+    attachListener(AppEvents.updateDownloadCompleted)
+  );
 
-desktop.updater.onDownloadProgress.subscribe(
-  undefined,
-  attachListener(AppEvents.updateDownloadProgress)
-);
+  desktop.updater.onDownloadProgress.subscribe(
+    undefined,
+    attachListener(AppEvents.updateDownloadProgress)
+  );
 
-desktop.updater.onNotAvailable.subscribe(
-  undefined,
-  attachListener(AppEvents.updateNotAvailable)
-);
+  desktop.updater.onNotAvailable.subscribe(
+    undefined,
+    attachListener(AppEvents.updateNotAvailable)
+  );
 
-desktop.updater.onError.subscribe(
-  undefined,
-  attachListener(AppEvents.updateError)
-);
+  desktop.updater.onError.subscribe(
+    undefined,
+    attachListener(AppEvents.updateError)
+  );
+
+  TaskScheduler.register("updateCheck", "0 0 */12 * * * *", () => {
+    checkForUpdate();
+  });
+});
 
 function attachListener(event: string) {
   return {
@@ -63,3 +71,17 @@ function attachListener(event: string) {
     }
   };
 }
+
+export async function createWritableStream(path: string) {
+  const resolvedPath = await desktop.integration.resolvePath.query({
+    filePath: path
+  });
+  if (!resolvedPath) throw new Error("invalid path.");
+  const fs = require("fs");
+  const { Writable } = require("stream");
+  return new WritableStream(
+    Writable.toWeb(
+      fs.createWriteStream(resolvedPath, { encoding: "utf-8" })
+    ).getWriter()
+  );
+}

apps/web/src/common/desktop-bridge/index.ts

@@ -17,6 +17,10 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+import { createWriteStream } from "../../utils/stream-saver";
 import { type desktop as bridge } from "./index.desktop";
 
 export const desktop: typeof bridge | undefined = undefined;
+export function createWritableStream(filename: string) {
+  return createWriteStream(filename);
+}

apps/web/src/common/index.js

@@ -1,217 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import {
-  showFeatureDialog,
-  showLoadingDialog,
-  showPasswordDialog,
-  showReminderDialog
-} from "../common/dialog-controller";
-import Config from "../utils/config";
-import { hashNavigate, getCurrentHash } from "../navigation";
-import { db } from "./db";
-import { sanitizeFilename } from "@notesnook/common";
-import { store as userstore } from "../stores/user-store";
-import FileSaver from "file-saver";
-import { showToast } from "../utils/toast";
-import { SUBSCRIPTION_STATUS } from "./constants";
-import { showFilePicker } from "../utils/file-picker";
-import { logger } from "../utils/logger";
-import { PATHS } from "@notesnook/desktop";
-import { TaskManager } from "./task-manager";
-import { EVENTS } from "@notesnook/core/dist/common";
-import { getFormattedDate } from "@notesnook/common";
-import { desktop } from "./desktop-bridge";
-
-export const CREATE_BUTTON_MAP = {
-  notes: {
-    title: "Add a note",
-    onClick: () =>
-      hashNavigate("/notes/create", { addNonce: true, replace: true })
-  },
-  notebooks: {
-    title: "Create a notebook",
-    onClick: () => hashNavigate("/notebooks/create", { replace: true })
-  },
-  topics: {
-    title: "Create a topic",
-    onClick: () => hashNavigate(`/topics/create`, { replace: true })
-  },
-  tags: {
-    title: "Create a tag",
-    onClick: () => hashNavigate(`/tags/create`, { replace: true })
-  },
-  reminders: {
-    title: "Add a reminder",
-    onClick: () => hashNavigate(`/reminders/create`, { replace: true })
-  }
-};
-
-export async function introduceFeatures() {
-  const hash = getCurrentHash().replace("#", "");
-  if (!!hash || IS_TESTING) return;
-  const features = [];
-  for (let feature of features) {
-    if (!Config.get(`feature:${feature}`)) {
-      await showFeatureDialog(feature);
-    }
-  }
-}
-
-export const DEFAULT_CONTEXT = { colors: [], tags: [], notebook: {} };
-
-export async function createBackup() {
-  const encryptBackups =
-    userstore.get().isLoggedIn && Config.get("encryptBackups", false);
-  const data = await showLoadingDialog({
-    title: "Creating backup",
-    subtitle: "We are creating a backup of your data. Please wait...",
-    action: async () => {
-      return await db.backup.export("web", encryptBackups);
-    }
-  });
-  if (!data) {
-    showToast("error", "Could not create a backup of your data.");
-    return;
-  }
-
-  const filename = sanitizeFilename(`notesnook-backup-${getFormattedDate()}`);
-
-  const ext = "nnbackup";
-  if (IS_DESKTOP_APP) {
-    const directory = Config.get(
-      "backupStorageLocation",
-      PATHS.backupsDirectory
-    );
-    const filePath = `${directory}/${filename}.${ext}`;
-    await desktop?.integration.saveFile.query({ filePath, data });
-    showToast("success", `Backup saved at ${filePath}.`);
-  } else {
-    FileSaver.saveAs(new Blob([Buffer.from(data)]), `${filename}.${ext}`);
-  }
-}
-
-export async function selectBackupFile() {
-  const file = await showFilePicker({
-    acceptedFileTypes: ".nnbackup,application/json"
-  });
-  if (!file) return;
-
-  const reader = new FileReader();
-  const backup = await new Promise((resolve) => {
-    reader.addEventListener("load", (event) => {
-      const text = event.target.result;
-      try {
-        resolve(JSON.parse(text));
-      } catch (e) {
-        alert(
-          "Error: Could not read the backup file provided. Either it's corrupted or invalid."
-        );
-        resolve();
-      }
-    });
-    reader.readAsText(file);
-  });
-
-  return { file, backup };
-}
-
-export async function importBackup() {
-  const { backup } = await selectBackupFile();
-  await restoreBackupFile(backup);
-}
-
-export async function restoreBackupFile(backup) {
-  if (backup.data.iv && backup.data.salt) {
-    await showPasswordDialog("ask_backup_password", async ({ password }) => {
-      const error = await restoreWithProgress(backup, password);
-      return !error;
-    });
-  } else {
-    await restoreWithProgress(backup);
-  }
-}
-
-async function restoreWithProgress(backup, password) {
-  await TaskManager.startTask({
-    title: "Restoring backup",
-    subtitle: "This might take a while",
-    type: "modal",
-    action: (report) => {
-      db.eventManager.subscribe(
-        EVENTS.migrationProgress,
-        ({ collection, total, current }) => {
-          report({
-            text: `Restoring ${collection}...`,
-            current,
-            total
-          });
-        }
-      );
-
-      report({ text: `Restoring...` });
-      return restore(backup, password);
-    }
-  });
-}
-
-export async function verifyAccount() {
-  if (!(await db.user.getUser())) return true;
-  return showPasswordDialog("verify_account", ({ password }) => {
-    return db.user.verifyPassword(password);
-  });
-}
-
-export function totalSubscriptionConsumed(user) {
-  if (!user) return 0;
-  const start = user.subscription?.start;
-  const end = user.subscription?.expiry;
-  if (!start || !end) return 0;
-
-  const total = end - start;
-  const consumed = Date.now() - start;
-
-  return Math.round((consumed / total) * 100);
-}
-
-export async function showUpgradeReminderDialogs() {
-  if (IS_TESTING) return;
-
-  const user = userstore.get().user;
-  if (!user || !user.subscription || user.subscription?.expiry === 0) return;
-
-  const consumed = totalSubscriptionConsumed(user);
-  const isTrial = user?.subscription?.type === SUBSCRIPTION_STATUS.TRIAL;
-  const isBasic = user?.subscription?.type === SUBSCRIPTION_STATUS.BASIC;
-  if (isBasic && consumed >= 100) {
-    await showReminderDialog("trialexpired");
-  } else if (isTrial && consumed >= 75) {
-    await showReminderDialog("trialexpiring");
-  }
-}
-
-async function restore(backup, password) {
-  try {
-    await db.backup.import(backup, password);
-    showToast("success", "Backup restored!");
-  } catch (e) {
-    logger.error(e, "Could not restore the backup");
-    showToast("error", `Could not restore the backup: ${e.message || e}`);
-  }
-}

apps/web/src/common/index.ts

@@ -0,0 +1,296 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+  showFeatureDialog,
+  showPasswordDialog,
+  showReminderDialog
+} from "./dialog-controller";
+import Config from "../utils/config";
+import { hashNavigate, getCurrentHash } from "../navigation";
+import { db } from "./db";
+import { sanitizeFilename } from "@notesnook/common";
+import { store as userstore } from "../stores/user-store";
+import { showToast } from "../utils/toast";
+import { SUBSCRIPTION_STATUS } from "./constants";
+import { readFile, showFilePicker } from "../utils/file-picker";
+import { logger } from "../utils/logger";
+import { PATHS } from "@notesnook/desktop";
+import { TaskManager } from "./task-manager";
+import { EVENTS } from "@notesnook/core/dist/common";
+import { getFormattedDate } from "@notesnook/common";
+import { createWritableStream } from "./desktop-bridge";
+import { ZipStream } from "../utils/streams/zip-stream";
+import { FeatureKeys } from "../dialogs/feature-dialog";
+import { Entry, Reader } from "../utils/zip-reader";
+
+export const CREATE_BUTTON_MAP = {
+  notes: {
+    title: "Add a note",
+    onClick: () =>
+      hashNavigate("/notes/create", { addNonce: true, replace: true })
+  },
+  notebooks: {
+    title: "Create a notebook",
+    onClick: () => hashNavigate("/notebooks/create", { replace: true })
+  },
+  topics: {
+    title: "Create a topic",
+    onClick: () => hashNavigate(`/topics/create`, { replace: true })
+  },
+  tags: {
+    title: "Create a tag",
+    onClick: () => hashNavigate(`/tags/create`, { replace: true })
+  },
+  reminders: {
+    title: "Add a reminder",
+    onClick: () => hashNavigate(`/reminders/create`, { replace: true })
+  }
+};
+
+export async function introduceFeatures() {
+  const hash = getCurrentHash().replace("#", "");
+  if (!!hash || IS_TESTING) return;
+  const features: FeatureKeys[] = [];
+  for (const feature of features) {
+    if (!Config.get(`feature:${feature}`)) {
+      await showFeatureDialog(feature);
+    }
+  }
+}
+
+export const DEFAULT_CONTEXT = { colors: [], tags: [], notebook: {} };
+
+export async function createBackup() {
+  const encryptBackups =
+    userstore.get().isLoggedIn && Config.get("encryptBackups", false);
+
+  const filename = sanitizeFilename(
+    `notesnook-backup-${getFormattedDate(Date.now())}`
+  );
+  const directory = Config.get("backupStorageLocation", PATHS.backupsDirectory);
+  const ext = "nnbackupz";
+  const filePath = IS_DESKTOP_APP
+    ? `${directory}/${filename}.${ext}`
+    : `${filename}.${ext}`;
+
+  const encoder = new TextEncoder();
+  const error = await TaskManager.startTask<Error | void>({
+    type: "modal",
+    title: "Creating backup",
+    subtitle: "We are creating a backup of your data. Please wait...",
+    action: async (report) => {
+      const writeStream = await createWritableStream(filePath);
+
+      await new ReadableStream({
+        start() {},
+        async pull(controller) {
+          for await (const file of db.backup!.export("web", encryptBackups)) {
+            report({
+              text: `Saving chunk ${file.path}`
+            });
+            controller.enqueue({
+              path: file.path,
+              data: encoder.encode(file.data)
+            });
+          }
+          controller.close();
+        }
+      })
+        .pipeThrough(new ZipStream())
+        .pipeTo(writeStream);
+    }
+  });
+  if (error) {
+    showToast(
+      "error",
+      `Could not create a backup of your data: ${(error as Error).message}`
+    );
+    console.error(error);
+  } else {
+    showToast("success", `Backup saved at ${filePath}.`);
+  }
+}
+
+export async function selectBackupFile() {
+  const file = await showFilePicker({
+    acceptedFileTypes: ".nnbackup,.nnbackupz"
+  });
+  if (!file) return;
+  return file;
+}
+
+export async function importBackup() {
+  const backupFile = await selectBackupFile();
+  if (!backupFile) return;
+  await restoreBackupFile(backupFile);
+}
+
+export async function restoreBackupFile(backupFile: File) {
+  const isLegacy = !backupFile.name.endsWith(".nnbackupz");
+
+  if (isLegacy) {
+    const backup = JSON.parse(await readFile(backupFile));
+
+    if (backup.data.iv && backup.data.salt) {
+      await showPasswordDialog("ask_backup_password", async ({ password }) => {
+        if (!password) return false;
+        const error = await restoreWithProgress(backup, password);
+        return !error;
+      });
+    } else {
+      await restoreWithProgress(backup);
+    }
+  } else {
+    const error = await TaskManager.startTask<Error | void>({
+      title: "Restoring backup",
+      subtitle: "Please wait while we restore your backup...",
+      type: "modal",
+      action: async (report) => {
+        let cachedPassword: string | undefined = undefined;
+        const { read, totalFiles } = await Reader(backupFile);
+        const entries: Entry[] = [];
+        let filesProcessed = 0;
+
+        let isValid = false;
+        for await (const entry of read()) {
+          if (entry.name === ".nnbackup") {
+            isValid = true;
+            continue;
+          }
+          entries.push(entry);
+        }
+        if (!isValid) throw new Error("Invalid backup.");
+
+        for (const entry of entries) {
+          const backup = JSON.parse(await entry.text());
+          if (backup.encrypted) {
+            if (!cachedPassword) {
+              const result = await showPasswordDialog(
+                "ask_backup_password",
+                async ({ password }) => {
+                  if (!password) return false;
+                  await db.backup?.import(backup, password);
+                  cachedPassword = password;
+                  return true;
+                }
+              );
+              if (!result) break;
+            } else await db.backup?.import(backup, cachedPassword);
+          } else {
+            await db.backup?.import(backup, null);
+          }
+
+          report({
+            total: totalFiles,
+            text: `Processed ${entry.name}`,
+            current: filesProcessed++
+          });
+        }
+        await db.initCollections();
+      }
+    });
+    if (error) {
+      console.error(error);
+      showToast("error", `Failed to restore backup: ${error.message}`);
+    }
+  }
+}
+
+async function restoreWithProgress(
+  backup: Record<string, unknown>,
+  password?: string
+) {
+  return await TaskManager.startTask<Error | void>({
+    title: "Restoring backup",
+    subtitle: "This might take a while",
+    type: "modal",
+    action: (report) => {
+      db.eventManager.subscribe(
+        EVENTS.migrationProgress,
+        ({
+          collection,
+          total,
+          current
+        }: {
+          collection: string;
+          total: number;
+          current: number;
+        }) => {
+          report({
+            text: `Restoring ${collection}...`,
+            current,
+            total
+          });
+        }
+      );
+
+      report({ text: `Restoring...` });
+      return restore(backup, password);
+    }
+  });
+}
+
+export async function verifyAccount() {
+  if (!(await db.user?.getUser())) return true;
+  return showPasswordDialog("verify_account", ({ password }) => {
+    return db.user?.verifyPassword(password) || false;
+  });
+}
+
+export function totalSubscriptionConsumed(user: User) {
+  if (!user) return 0;
+  const start = user.subscription?.start;
+  const end = user.subscription?.expiry;
+  if (!start || !end) return 0;
+
+  const total = end - start;
+  const consumed = Date.now() - start;
+
+  return Math.round((consumed / total) * 100);
+}
+
+export async function showUpgradeReminderDialogs() {
+  if (IS_TESTING) return;
+
+  const user = userstore.get().user;
+  if (!user || !user.subscription || user.subscription?.expiry === 0) return;
+
+  const consumed = totalSubscriptionConsumed(user);
+  const isTrial = user?.subscription?.type === SUBSCRIPTION_STATUS.TRIAL;
+  const isBasic = user?.subscription?.type === SUBSCRIPTION_STATUS.BASIC;
+  if (isBasic && consumed >= 100) {
+    await showReminderDialog("trialexpired");
+  } else if (isTrial && consumed >= 75) {
+    await showReminderDialog("trialexpiring");
+  }
+}
+
+async function restore(backup: Record<string, unknown>, password?: string) {
+  try {
+    await db.backup?.import(backup, password);
+    showToast("success", "Backup restored!");
+  } catch (e) {
+    logger.error(e as Error, "Could not restore the backup");
+    showToast(
+      "error",
+      `Could not restore the backup: ${(e as Error).message || e}`
+    );
+  }
+}

apps/web/src/components/publish-view/index.jsx

@@ -49,17 +49,17 @@ function PublishView(props) {
   }, [noteId]);
 
   useEffect(() => {
-    const attachmentsLoadingEvent = EV.subscribe(
-      EVENTS.attachmentsLoading,
-      ({ type, groupId, total, current }) => {
-        if (!groupId || !groupId.includes(noteId) || type !== "download")
-          return;
+    const fileDownloadedEvent = EV.subscribe(
+      EVENTS.fileDownloaded,
+      ({ total, current, groupId }) => {
+        if (!groupId || !groupId.includes(noteId)) return;
         if (current === total) setProcessingStatus();
         else setProcessingStatus({ total, current });
       }
     );
+
     return () => {
-      attachmentsLoadingEvent.unsubscribe();
+      fileDownloadedEvent.unsubscribe();
     };
   }, [noteId]);
 

apps/web/src/components/status-bar/index.tsx

@@ -40,7 +40,7 @@ import {
   showIssueDialog,
   showUpdateAvailableNotice
 } from "../../common/dialog-controller";
-import useStatus from "../../hooks/use-status";
+import useStatus, { statusToString } from "../../hooks/use-status";
 import { ScopedThemeProvider } from "../theme-provider";
 import { checkForUpdate, installUpdate } from "../../utils/updater";
 import { toTitleCase } from "@notesnook/common";
@@ -132,7 +132,8 @@ function StatusBar() {
             Report an issue
           </Text>
         </Button>
-        {statuses?.map(({ key, status, progress, icon: Icon }) => {
+        {statuses?.map((status) => {
+          const { key, icon: Icon } = status;
           return (
             <Flex
               key={key}
@@ -141,13 +142,13 @@ function StatusBar() {
             >
               {Icon ? <Icon size={12} /> : <Loading size={12} />}
               <Text variant="subBody" ml={1} sx={{ color: "paragraph" }}>
-                {progress ? `${progress}% ${status}` : status}
+                {statusToString(status)}
               </Text>
             </Flex>
           );
         })}
 
-        {updateStatus && (
+        {updateStatus && updateStatus.type !== "updated" && (
           <Button
             variant="statusitem"
             onClick={async () => {
@@ -168,7 +169,6 @@ function StatusBar() {
           >
             <Update
               rotate={
-                updateStatus.type !== "updated" &&
                 updateStatus.type !== "completed" &&
                 updateStatus.type !== "available"
               }
@@ -192,8 +192,6 @@ function statusToInfoText(status: UpdateStatus) {
   const { type } = status;
   return type === "checking"
     ? "Checking for updates..."
-    : type === "updated"
-    ? "You are on latest version"
     : type === "downloading"
     ? `${Math.round(status.progress)}% updating...`
     : type === "completed"

apps/web/src/dialogs/settings/backup-export-settings.ts

@@ -50,7 +50,7 @@ export const BackupExportSettings: SettingsGroup[] = [
               )
                 useSettingStore.getState().toggleEncryptBackups();
               const verified =
-                !useSettingStore.getState().encryptBackups ||
+                useSettingStore.getState().encryptBackups ||
                 (await verifyAccount());
               if (verified) await createBackup();
             },

apps/web/src/dialogs/settings/index.tsx

@@ -379,7 +379,7 @@ function SettingsGroupComponent(props: { item: SettingsGroup }) {
 function SettingItem(props: { item: Setting }) {
   const { item } = props;
   const [state, setState] = useState<unknown>();
-  const [isWorking, setIsWorking] = useState(false);
+  const [workIndex, setWorkIndex] = useState<number>();
   const isUserPremium = useIsUserPremium();
 
   useEffect(() => {
@@ -388,16 +388,16 @@ function SettingItem(props: { item: Setting }) {
   }, [item]);
 
   const workWithLoading = useCallback(
-    async (action) => {
-      if (isWorking) return;
+    async (index: number, action: () => Promise<unknown> | void) => {
+      if (workIndex) return;
       try {
-        setIsWorking(true);
+        setWorkIndex(index);
         await action();
       } finally {
-        setIsWorking(false);
+        setWorkIndex(undefined);
       }
     },
-    [isWorking]
+    [workIndex]
   );
 
   if (item.isHidden && item.isHidden(state)) return null;
@@ -447,17 +447,17 @@ function SettingItem(props: { item: Setting }) {
             "& > label": { width: "auto" }
           }}
         >
-          {components.map((component) => {
+          {components.map((component, index) => {
             switch (component.type) {
               case "button":
                 return (
                   <Button
-                    disabled={isWorking}
+                    disabled={workIndex === index}
                     title={component.title}
                     variant={component.variant}
-                    onClick={() => workWithLoading(component.action)}
+                    onClick={() => workWithLoading(index, component.action)}
                   >
-                    {isWorking ? (
+                    {workIndex === index ? (
                       <Loading size={18} sx={{ mr: 2 }} />
                     ) : (
                       component.title
@@ -473,8 +473,8 @@ function SettingItem(props: { item: Setting }) {
                         ? "accent"
                         : "icon-secondary"
                     }}
-                    disabled={isWorking}
-                    onChange={() => workWithLoading(component.toggle)}
+                    disabled={workIndex === index}
+                    onChange={() => workWithLoading(index, component.toggle)}
                     checked={component.isToggled()}
                     data-checked={component.isToggled()}
                   />

apps/web/src/dialogs/settings/other-settings.ts

@@ -21,10 +21,11 @@ import { SettingsGroup } from "./types";
 import { appVersion } from "../../utils/version";
 import { writeText } from "clipboard-polyfill";
 import { showToast } from "../../utils/toast";
-import { checkForUpdate } from "../../utils/updater";
+import { checkForUpdate, downloadUpdate } from "../../utils/updater";
 import { isMacStoreApp } from "../../utils/platform";
 import { showIssueDialog } from "../../common/dialog-controller";
 import { clearLogs, downloadLogs } from "../../utils/logger";
+import { useAutoUpdateStore } from "../../hooks/use-auto-updater";
 
 export const AboutSettings: SettingsGroup[] = [
   {
@@ -35,24 +36,41 @@ export const AboutSettings: SettingsGroup[] = [
       {
         key: "version",
         title: "Version",
-        description: appVersion.formatted,
-        components: [
-          {
-            type: "button",
-            action: checkForUpdate,
-            title: "Check for updates",
-            variant: "secondary"
-          },
-          {
-            type: "button",
-            action: async () => {
-              await writeText(appVersion.formatted);
-              showToast("info", "Copied to clipboard!");
-            },
-            title: "Copy",
-            variant: "secondary"
-          }
-        ]
+        description: () => {
+          const status = useAutoUpdateStore.getState().status;
+          if (status?.type === "available")
+            return `New version (v${status.version}) is available for download.`;
+          return appVersion.formatted;
+        },
+        onStateChange: (listener) =>
+          useAutoUpdateStore.subscribe((s) => s.status, listener),
+        components: () => {
+          const status = useAutoUpdateStore.getState().status;
+          return [
+            status?.type === "available"
+              ? {
+                  type: "button",
+                  action: downloadUpdate,
+                  title: `Install update`,
+                  variant: "secondary"
+                }
+              : {
+                  type: "button",
+                  action: checkForUpdate,
+                  title: "Check for updates",
+                  variant: "secondary"
+                },
+            {
+              type: "button",
+              action: async () => {
+                await writeText(appVersion.formatted);
+                showToast("info", "Copied to clipboard!");
+              },
+              title: "Copy",
+              variant: "secondary"
+            }
+          ];
+        }
       },
       {
         key: "roadmap",
@@ -119,7 +137,8 @@ export const AboutSettings: SettingsGroup[] = [
         components: [
           {
             type: "button",
-            action: () => void window.open("https://fosstodon.org/@notesnook", "_blank"),
+            action: () =>
+              void window.open("https://fosstodon.org/@notesnook", "_blank"),
             title: "Follow",
             variant: "secondary"
           }

apps/web/src/hooks/use-auto-updater.ts

@@ -17,10 +17,12 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { useEffect, useState } from "react";
+import { useEffect } from "react";
 
 import { checkForUpdate } from "../utils/updater";
 import { AppEventManager, AppEvents } from "../common/app-events";
+import BaseStore from "../stores";
+import createStore from "../common/store";
 
 type CompletedUpdateStatus = { type: "completed"; version: string };
 type DownloadingUpdateStatus = { type: "downloading"; progress: number };
@@ -32,9 +34,18 @@ export type UpdateStatus =
   | DownloadingUpdateStatus
   | GenericUpdateStatus;
 
+class AutoUpdateStore extends BaseStore<AutoUpdateStore> {
+  status?: UpdateStatus;
+  setStatus = (status?: UpdateStatus) => {
+    this.set({ status });
+  };
+}
+
+const [useAutoUpdateStore] = createStore(AutoUpdateStore);
+
 let checkingForUpdateTimeout = 0;
 export function useAutoUpdater() {
-  const [status, setStatus] = useState<UpdateStatus>();
+  const { status, setStatus } = useAutoUpdateStore();
 
   useEffect(() => {
     function changeStatus(status?: UpdateStatus) {
@@ -104,3 +115,5 @@ export function useAutoUpdater() {
 
   return status;
 }
+
+export { useAutoUpdateStore };

apps/web/src/hooks/use-status.ts

@@ -24,6 +24,8 @@ import { Icon } from "../components/icons";
 type Status = {
   key: string;
   status: string;
+  total?: number;
+  current?: number;
   progress?: number;
   icon?: Icon | null;
 };
@@ -36,13 +38,20 @@ interface IStatusStore {
 const useStatusStore = create<IStatusStore>((set, get) => ({
   statuses: {},
   getStatus: (key: string) => get().statuses[key],
-  updateStatus: ({ key, status, progress, icon }: Status) =>
+  updateStatus: ({ key, status, progress, icon, current, total }: Status) =>
     set(
       produce((state) => {
         if (!key) return;
         const { statuses } = state;
         const statusText = status || statuses[key]?.status;
-        statuses[key] = { key, status: statusText, progress, icon };
+        statuses[key] = {
+          current,
+          total,
+          key,
+          status: statusText,
+          progress,
+          icon
+        };
       })
     ),
   removeStatus: (key) =>
@@ -63,3 +72,12 @@ export default function useStatus() {
 export const updateStatus = useStatusStore.getState().updateStatus;
 export const removeStatus = useStatusStore.getState().removeStatus;
 export const getStatus = useStatusStore.getState().getStatus;
+
+export function statusToString(status: Status) {
+  const parts: string[] = [];
+  if (status.progress) parts.push(`${status.progress}%`);
+  parts.push(status.status);
+  if (status.total !== undefined && status.current !== undefined)
+    parts.push(`(${status.current}/${status.total})`);
+  return parts.join(" ");
+}

apps/web/src/index.tsx

@@ -23,6 +23,7 @@ import { logger } from "./utils/logger";
 import { loadDatabase } from "./hooks/use-database";
 import { AppEventManager, AppEvents } from "./common/app-events";
 import { BaseThemeProvider } from "./components/theme-provider";
+import { register } from "./utils/stream-saver/mitm";
 
 renderApp();
 
@@ -66,6 +67,9 @@ async function initializeServiceWorker() {
         AppEventManager.publish(AppEvents.updateDownloadCompleted, {
           version: formatted
         });
+      },
+      onSuccess() {
+        register();
       }
     });
     // window.addEventListener("beforeinstallprompt", () => showInstallNotice());

apps/web/src/interfaces/fs.ts

@@ -24,7 +24,6 @@ import { AppEventManager, AppEvents } from "../common/app-events";
 import { StreamableFS } from "@notesnook/streamable-fs";
 import { NNCrypto } from "./nncrypto";
 import hosts from "@notesnook/core/dist/utils/constants";
-import { sendAttachmentsProgressEvent } from "@notesnook/core/dist/common";
 import { saveAs } from "file-saver";
 import { showToast } from "../utils/toast";
 import { db } from "../common/db";
@@ -61,7 +60,11 @@ async function writeEncryptedFile(
   // let offset = 0;
   // let encrypted = 0;
   const fileHandle = await streamablefs.createFile(hash, file.size, file.type);
-  sendAttachmentsProgressEvent("encrypt", hash, 1, 0);
+  AppEventManager.publish(AppEvents.fileEncrypted, {
+    hash,
+    total: 1,
+    current: 0
+  });
 
   const { iv, stream } = await NNCrypto.createEncryptionStream(key);
   await file
@@ -82,7 +85,11 @@ async function writeEncryptedFile(
     )
     .pipeTo(fileHandle.writeable);
 
-  sendAttachmentsProgressEvent("encrypt", hash, 1, 1);
+  AppEventManager.publish(AppEvents.fileEncrypted, {
+    hash,
+    total: 1,
+    current: 1
+  });
 
   return {
     chunkSize: CHUNK_SIZE,
@@ -504,8 +511,12 @@ async function downloadFile(filename: string, requestOptions: RequestOptions) {
   }
 }
 
-function exists(filename: string) {
-  return streamablefs.exists(filename);
+async function exists(filename: string) {
+  const handle = await streamablefs.readFile(filename);
+  return (
+    handle &&
+    handle.file.size === (await handle.size()) - handle.file.chunks * ABYTES
+  );
 }
 
 type FileMetadata = {

apps/web/src/service-worker-registration.ts

@@ -86,9 +86,11 @@ function registerValidSW(
   navigator.serviceWorker
     .register(swUrl)
     .then((registration) => {
+      if (config.onSuccess) config.onSuccess(registration);
       registration.onupdatefound = () => {
         const installingWorker = registration.installing;
         if (installingWorker == null) {
+          if (config.onSuccess) config.onSuccess(registration);
           return;
         }
         installingWorker.onstatechange = () => {

apps/web/src/service-worker.ts

@@ -80,6 +80,8 @@ registerRoute(
   })
 );
 
+const downloads = new Map<string, any[]>();
+
 // This allows the web app to trigger skipWaiting via
 // registration.waiting.postMessage({type: 'SKIP_WAITING'})
 self.addEventListener("message", (event) => {
@@ -87,6 +89,10 @@ self.addEventListener("message", (event) => {
   if (!data) return;
 
   switch (data.type) {
+    // We send a heartbeat every x second to keep the
+    // service worker alive if a transferable stream is not sent
+    case "PING":
+      break;
     case "SKIP_WAITING":
       self.skipWaiting();
       break;
@@ -100,7 +106,125 @@ self.addEventListener("message", (event) => {
         });
       }
       break;
+    case "REGISTER_DOWNLOAD":
+      {
+        console.log("register download", data);
+        const downloadUrl =
+          data.url ||
+          self.registration.scope +
+            Math.random() +
+            "/" +
+            (typeof data === "string" ? data : data.filename);
+        const port = event.ports[0];
+        const metadata = new Array(3); // [stream, data, port]
+
+        metadata[1] = data;
+        metadata[2] = port;
+
+        if (event.data.transferringReadable) {
+          port.onmessage = (evt) => {
+            port.onmessage = null;
+            metadata[0] = evt.data.readableStream;
+          };
+        } else {
+          metadata[0] = createStream(port);
+        }
+
+        downloads.set(downloadUrl, metadata);
+        port.postMessage({ download: downloadUrl });
+      }
+      break;
     default:
       break;
   }
 });
+
+self.addEventListener("fetch", (event) => {
+  const url = event.request.url;
+
+  // this only works for Firefox
+  if (url.endsWith("/ping")) {
+    return event.respondWith(new Response("pong"));
+  }
+
+  const metadata = downloads.get(url);
+  if (!metadata) return null;
+
+  const [stream, data, port] = metadata;
+
+  downloads.delete(url);
+
+  // Not comfortable letting any user control all headers
+  // so we only copy over the length & disposition
+  const responseHeaders = new Headers({
+    "Content-Type": "application/octet-stream; charset=utf-8",
+
+    // To be on the safe side, The link can be opened in a iframe.
+    // but octet-stream should stop it.
+    "Content-Security-Policy": "default-src 'none'",
+    "X-Content-Security-Policy": "default-src 'none'",
+    "X-WebKit-CSP": "default-src 'none'",
+    "X-XSS-Protection": "1; mode=block"
+  });
+
+  const headers = new Headers(data.headers || {});
+
+  if (headers.has("Content-Length")) {
+    responseHeaders.set("Content-Length", headers.get("Content-Length")!);
+  }
+
+  if (headers.has("Content-Disposition")) {
+    responseHeaders.set(
+      "Content-Disposition",
+      headers.get("Content-Disposition")!
+    );
+  }
+
+  // data, data.filename and size should not be used anymore
+  if (data.size) {
+    console.warn("Depricated");
+    responseHeaders.set("Content-Length", data.size);
+  }
+
+  let fileName = typeof data === "string" ? data : data.filename;
+  if (fileName) {
+    console.warn("Depricated");
+    // Make filename RFC5987 compatible
+    fileName = encodeURIComponent(fileName)
+      .replace(/['()]/g, escape)
+      .replace(/\*/g, "%2A");
+    responseHeaders.set(
+      "Content-Disposition",
+      "attachment; filename*=UTF-8''" + fileName
+    );
+  }
+
+  event.respondWith(new Response(stream, { headers: responseHeaders }));
+
+  port.postMessage({ debug: "Download started" });
+});
+
+function createStream(port: MessagePort) {
+  // ReadableStream is only supported by chrome 52
+  return new ReadableStream({
+    start(controller) {
+      // When we receive data on the messageChannel, we write
+      port.onmessage = ({ data }) => {
+        if (data === "end") {
+          return controller.close();
+        }
+
+        if (data === "abort") {
+          controller.error("Aborted the download");
+          return;
+        }
+
+        controller.enqueue(data);
+      };
+    },
+    cancel(reason) {
+      console.log("user aborted", reason);
+      port.postMessage({ abort: true });
+    }
+  });
+}

apps/web/src/stores/app-store.js

@@ -40,7 +40,7 @@ import { NetworkCheck } from "../utils/network-check";
 
 const networkCheck = new NetworkCheck();
 var syncStatusTimeout = 0;
-const BATCH_SIZE = 50;
+let pendingSync = false;
 
 /**
  * @extends {BaseStore<AppStore>}
@@ -288,8 +288,10 @@ class AppStore extends BaseStore {
         syncDisabled: !this.get().isSyncEnabled,
         offline: !navigator.onLine
       });
+      if (this.isSyncing()) pendingSync = { full };
       return;
     }
+    pendingSync = false;
 
     clearTimeout(syncStatusTimeout);
     this.updateLastSynced();
@@ -302,6 +304,12 @@ class AppStore extends BaseStore {
       this.updateSyncStatus("completed", true);
 
       await this.updateLastSynced();
+
+      if (pendingSync) {
+        logger.info("Running pending sync", pendingSync);
+        pendingSync = false;
+        await this.get().sync(pendingSync.full, false);
+      }
     } catch (err) {
       logger.error(err);
       if (err.cause === "MERGE_CONFLICT") {

apps/web/src/stores/attachment-store.js

@@ -23,7 +23,6 @@ import BaseStore from "./index";
 import { store as editorStore } from "./editor-store";
 import { checkAttachment } from "../common/attachments";
 import { showToast } from "../utils/toast";
-import { register } from "../utils/stream-saver/mitm";
 import { AttachmentStream } from "../utils/streams/attachment-stream";
 import { ZipStream } from "../utils/streams/zip-stream";
 import { createWriteStream } from "../utils/stream-saver";
@@ -57,7 +56,6 @@ class AttachmentStore extends BaseStore {
       (state) => (state.status = { current: 0, total: attachments.length })
     );
 
-    await register();
     abortController = new AbortController();
     const attachmentStream = new AttachmentStream(
       attachments,
@@ -71,7 +69,9 @@ class AttachmentStore extends BaseStore {
     await attachmentStream
       .pipeThrough(new ZipStream())
       .pipeTo(
-        createWriteStream("attachments.zip", { signal: abortController.signal })
+        await createWriteStream("attachments.zip", {
+          signal: abortController.signal
+        })
       );
 
     this.set((state) => (state.status = undefined));

apps/web/src/utils/stream-saver/index.ts

@@ -16,7 +16,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-import { postMessage } from "./mitm";
+import { findServiceWorker, postMessage } from "./mitm";
 
 let supportsTransferable = false;
 
@@ -66,26 +66,21 @@ function checkSupportsTransferable() {
 }
 checkSupportsTransferable();
 
-/**
- * @param  {string} filename filename that should be used
- * @param  {object} options  [description]
- * @param  {number} size     deprecated
- * @return {WritableStream<Uint8Array>}
- */
-export function createWriteStream(
+export async function createWriteStream(
   filename: string,
   opts: {
     size?: number;
     pathname?: string;
     signal?: AbortSignal;
   } = {}
-): WritableStream<Uint8Array> {
+): Promise<WritableStream<Uint8Array>> {
+  const { sw } = await findServiceWorker();
   // let bytesWritten = 0; // by StreamSaver.js (not the service worker)
   let downloadUrl: string | null = null;
   let channel: MessageChannel | null = null;
   let ts: TransformStream | null = null;
   let frame: HTMLIFrameElement | null = null;
-  if (!useBlobFallback) {
+  if (sw && !useBlobFallback) {
     channel = new MessageChannel();
 
     // Make filename RFC5987 compatible
@@ -137,19 +132,20 @@ export function createWriteStream(
       }
     };
 
-    postMessage(response, [channel.port2]);
+    await postMessage(response, [channel.port2]);
   }
 
   let chunks: Uint8Array[] = [];
-
   return (
-    (!useBlobFallback && ts && ts.writable) ||
+    (sw && !useBlobFallback && ts && ts.writable) ||
     new WritableStream({
       write(chunk) {
+        if (opts.signal?.aborted) return;
+
         if (!(chunk instanceof Uint8Array)) {
           throw new TypeError("Can only write Uint8Arrays");
         }
-        if (useBlobFallback) {
+        if (!sw || useBlobFallback) {
           // Safari... The new IE6
           // https://github.com/jimmywarting/StreamSaver.js/issues/69
           //
@@ -178,14 +174,21 @@ export function createWriteStream(
         }
       },
       close() {
-        if (useBlobFallback) {
+        if (opts.signal?.aborted) return;
+        if (!sw || useBlobFallback) {
           const blob = new Blob(chunks, {
             type: "application/octet-stream; charset=utf-8"
           });
           const link = document.createElement("a");
           link.href = URL.createObjectURL(blob);
           link.download = filename;
+          link.addEventListener("click", () => {
+            // `setTimeout()` due to
+            // https://github.com/LLK/scratch-gui/issues/1783#issuecomment-426286393
+            setTimeout(() => URL.revokeObjectURL(link.href), 30 * 1000);
+          });
           link.click();
+          chunks = [];
         } else {
           channel?.port1.postMessage("end");
         }

apps/web/src/utils/stream-saver/mitm.ts

@@ -16,45 +16,26 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-export {};
 
-let sw: ServiceWorker | null = null;
-let scope = "";
-
-function registerWorker() {
-  return navigator.serviceWorker
-    .getRegistration("./")
-    .then((swReg) => {
-      return (
-        swReg ||
-        navigator.serviceWorker.register("stream-saver-sw.js", { scope: "./" })
-      );
-    })
-    .then((swReg) => {
-      const swRegTmp = swReg.installing || swReg.waiting;
-
-      scope = swReg.scope;
-      let fn: () => void;
-      return (
-        (sw = swReg.active) ||
-        new Promise((resolve) => {
-          swRegTmp?.addEventListener(
-            "statechange",
-            (fn = () => {
-              if (swRegTmp.state === "activated") {
-                swRegTmp.removeEventListener("statechange", fn);
-                sw = swReg.active;
-                resolve(undefined);
-              }
-            })
-          );
-        })
-      );
-    });
-}
+let keepAlive = () => {
+  keepAlive = () => {};
+  const interval = setInterval(async () => {
+    const { sw } = await findServiceWorker();
+    if (sw) {
+      sw.postMessage({ type: "PING" });
+    } else {
+      const ping =
+        location.href.substr(0, location.href.lastIndexOf("/")) + "/ping";
+      fetch(ping).then((res) => {
+        !res.ok && clearInterval(interval);
+        return res.text();
+      });
+    }
+  }, 10000);
+};
 
 // Now that we have the Service Worker registered we can process messages
-export function postMessage(
+export async function postMessage(
   data: {
     origin?: string;
     referrer?: string;
@@ -65,6 +46,9 @@ export function postMessage(
   },
   ports: MessagePort[]
 ) {
+  const { scope, sw } = await findServiceWorker();
+  if (!sw) throw new Error("No service worker registered.");
+
   // It's important to have a messageChannel, don't want to interfere
   // with other simultaneous downloads
   if (!ports || !ports.length) {
@@ -107,11 +91,32 @@ export function postMessage(
 
   const transferable = [ports[0]];
 
-  return sw?.postMessage(data, transferable);
+  if (!data.transferringReadable) {
+    keepAlive();
+  }
+
+  return sw.postMessage({ type: "REGISTER_DOWNLOAD", ...data }, transferable);
 }
 
-export async function register() {
-  if (navigator.serviceWorker) {
-    await registerWorker();
-  }
+export function register() {
+  // FF v102 just started to supports transferable streams, but still needs to ping sw.js
+  // even tough the service worker dose not have to do any kind of work and listen to any
+  // messages... #305
+  keepAlive();
+}
+
+export async function findServiceWorker(): Promise<{
+  sw?: ServiceWorker;
+  scope?: string;
+}> {
+  if (!("serviceWorker" in navigator)) return {};
+
+  const registrations =
+    (await navigator.serviceWorker?.getRegistrations()) || [];
+  for (const registration of registrations) {
+    if (registration.active)
+      return { sw: registration.active, scope: registration.scope };
+  }
+
+  return {};
 }

apps/web/src/views/auth.tsx

@@ -272,6 +272,7 @@ function LoginPassword(props: BaseAuthComponentProps<"login:password">) {
       type="login:password"
       title="Your account password"
       subtitle={"Your password is always hashed before leaving this device."}
+      loadForever
       loading={{
         title: "Logging you in",
         subtitle: "Please wait while you are authenticated."
@@ -806,6 +807,7 @@ type AuthFormProps<TType extends AuthRoutes> = {
   loading: { title: string; subtitle: string };
   type: TType;
   onSubmit: (form: AuthFormData[TType]) => Promise<void>;
+  loadForever?: boolean;
   canSkip?: boolean;
   children?:
     | React.ReactNode
@@ -813,7 +815,7 @@ type AuthFormProps<TType extends AuthRoutes> = {
 };
 
 export function AuthForm<T extends AuthRoutes>(props: AuthFormProps<T>) {
-  const { title, subtitle, children, canSkip } = props;
+  const { title, subtitle, children, canSkip, loadForever } = props;
   const [isSubmitting, setIsSubmitting] = useState(false);
   const [error, setError] = useState<string>();
   const formRef = useRef<HTMLFormElement>(null);
@@ -838,7 +840,9 @@ export function AuthForm<T extends AuthRoutes>(props: AuthFormProps<T>) {
         try {
           setForm(form);
           await props.onSubmit(form);
+          if (!loadForever) setIsSubmitting(false);
         } catch (e) {
+          setIsSubmitting(false);
           const error = e as Error;
           if (error.message === "invalid_grant") {
             setError(
@@ -847,8 +851,6 @@ export function AuthForm<T extends AuthRoutes>(props: AuthFormProps<T>) {
             return;
           }
           setError(error.message);
-        } finally {
-          setIsSubmitting(false);
         }
       }}
       sx={{

apps/web/src/views/recovery.tsx

@@ -40,10 +40,7 @@ type RecoveryKeyFormData = {
 };
 
 type BackupFileFormData = {
-  backupFile: {
-    file: File;
-    backup: Record<string, unknown>;
-  };
+  backupFile: File;
 };
 
 type NewPasswordFormData = BackupFileFormData & {
@@ -406,7 +403,7 @@ function BackupFileMethod(props: BaseRecoveryComponentProps<"method:backup">) {
     if (!backupFile) return;
     const backupFileInput = document.getElementById("backupFile");
     if (!(backupFileInput instanceof HTMLInputElement)) return;
-    backupFileInput.value = backupFile?.file?.name;
+    backupFileInput.value = backupFile?.name;
   }, [backupFile]);
 
   return (
@@ -519,7 +516,7 @@ function NewPassword(props: BaseRecoveryComponentProps<"new">) {
           throw new Error("Could not reset account password.");
 
         if (formData?.backupFile) {
-          await restoreBackupFile(formData?.backupFile.backup);
+          await restoreBackupFile(formData?.backupFile);
           await db.sync(true, true);
         }
 

fastlane/metadata/android/en-US/changelogs/10309.txt

@@ -0,0 +1,3 @@
+- New backup format nnbackupz with upto 90% smaller backups
+- Improve UI of code blocks & language selection popup
+- Fixed some bugs in sync v2

fastlane/metadata/android/en-US/changelogs/10319.txt

@@ -0,0 +1,3 @@
+- New backup format nnbackupz with upto 90% smaller backups
+- Improve UI of code blocks & language selection popup
+- Fixed some bugs in sync v2

packages/common/package-lock.json

@@ -55,9 +55,11 @@
         "abortcontroller-polyfill": "^1.7.3",
         "cross-env": "^7.0.3",
         "dotenv": "^16.0.1",
+        "event-source-polyfill": "^1.0.31",
         "eventsource": "^2.0.2",
-        "happy-dom": "^10.9.0",
+        "hash-wasm": "^4.9.0",
         "isomorphic-fetch": "^3.0.0",
+        "jsdom": "^22.1.0",
         "mockdate": "^3.0.5",
         "otplib": "^12.0.1",
         "refractor": "^4.8.1",
@@ -157,11 +159,13 @@
         "dayjs": "1.11.9",
         "dotenv": "^16.0.1",
         "entities": "^4.3.1",
+        "event-source-polyfill": "^1.0.31",
         "eventsource": "^2.0.2",
-        "happy-dom": "^10.9.0",
+        "hash-wasm": "^4.9.0",
         "html-to-text": "^9.0.5",
         "htmlparser2": "^8.0.1",
         "isomorphic-fetch": "^3.0.0",
+        "jsdom": "^22.1.0",
         "katex": "0.16.2",
         "linkedom": "^0.14.17",
         "liqe": "^1.13.0",

packages/core/__e2e__/sync.test.js

@@ -0,0 +1,466 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import Database from "../src/api/index";
+import { NodeStorageInterface } from "../__mocks__/node-storage.mock";
+import { FS } from "../__mocks__/fs.mock";
+import Compressor from "../__mocks__/compressor.mock";
+import { CHECK_IDS, EV, EVENTS } from "../src/common";
+import { EventSource } from "event-source-polyfill";
+import { delay } from "../__tests__/utils";
+import { test, expect, vitest } from "vitest";
+import { login } from "./utils";
+
+const TEST_TIMEOUT = 60 * 1000;
+
+test(
+  "case 1: device A & B should only download the changes from device C (no uploading)",
+  async () => {
+    const types = [];
+    function onSyncProgress({ type }) {
+      types.push(type);
+    }
+
+    const [deviceA, deviceB, deviceC] = await Promise.all([
+      initializeDevice("deviceA"),
+      initializeDevice("deviceB"),
+      initializeDevice("deviceC")
+    ]);
+
+    deviceA.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
+    deviceB.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
+
+    await deviceC.notes.add({ title: "new note 1" });
+    await syncAndWait(deviceC, deviceC);
+
+    expect(types.every((t) => t === "download")).toBe(true);
+
+    await cleanup(deviceA, deviceB, deviceC);
+  },
+  TEST_TIMEOUT
+);
+
+test(
+  "case 3: Device A & B have unsynced changes but server has nothing",
+  async () => {
+    const [deviceA, deviceB] = await Promise.all([
+      initializeDevice("deviceA"),
+      initializeDevice("deviceB")
+    ]);
+
+    const note1Id = await deviceA.notes.add({
+      title: "Test note from device A"
+    });
+    const note2Id = await deviceB.notes.add({
+      title: "Test note from device B"
+    });
+
+    await syncAndWait(deviceA, deviceB);
+
+    expect(deviceA.notes.note(note2Id)).toBeTruthy();
+    expect(deviceB.notes.note(note1Id)).toBeTruthy();
+    expect(deviceA.notes.note(note1Id)).toBeTruthy();
+    expect(deviceB.notes.note(note2Id)).toBeTruthy();
+
+    await cleanup(deviceA, deviceA);
+  },
+  TEST_TIMEOUT
+);
+
+// test(
+//   "case 4: Device A's sync is interrupted halfway and Device B makes some changes afterwards and syncs.",
+//   async () => {
+//     const deviceA = await initializeDevice("deviceA");
+//     const deviceB = await initializeDevice("deviceB");
+
+//     const unsyncedNoteIds = [];
+//     for (let i = 0; i < 10; ++i) {
+//       const id = await deviceA.notes.add({
+//         title: `Test note ${i} from device A`,
+//       });
+//       unsyncedNoteIds.push(id);
+//     }
+
+//     const half = unsyncedNoteIds.length / 2 + 1;
+//     deviceA.eventManager.subscribe(
+//       EVENTS.syncProgress,
+//       async ({ type, current }) => {
+//         if (type === "upload" && current === half) {
+//           await deviceA.syncer.stop();
+//         }
+//       }
+//     );
+
+//     await expect(deviceA.sync(true)).rejects.toThrow();
+
+//     let syncedNoteIds = [];
+//     for (let i = 0; i < unsyncedNoteIds.length; ++i) {
+//       const expectedNoteId = unsyncedNoteIds[i];
+//       if (deviceB.notes.note(expectedNoteId))
+//         syncedNoteIds.push(expectedNoteId);
+//     }
+//     expect(
+//       syncedNoteIds.length === half - 1 || syncedNoteIds.length === half
+//     ).toBe(true);
+
+//     const deviceBNoteId = await deviceB.notes.add({
+//       title: "Test note of case 4 from device B",
+//     });
+
+//     await deviceB.sync(true);
+
+//     await syncAndWait(deviceA, deviceB);
+
+//     expect(deviceA.notes.note(deviceBNoteId)).toBeTruthy();
+//     expect(
+//       unsyncedNoteIds
+//         .map((id) => !!deviceB.notes.note(id))
+//         .every((res) => res === true)
+//     ).toBe(true);
+
+//     await cleanup(deviceA, deviceB);
+//   },
+//
+// );
+
+// test.only(
+//   "case 5: Device A's sync is interrupted halfway and Device B makes changes on the same note's content that didn't get synced on Device A due to interruption.",
+//   async () => {
+//     const deviceA = await initializeDevice("deviceA");
+//     const deviceB = await initializeDevice("deviceB");
+
+//     const noteIds = [];
+//     for (let i = 0; i < 10; ++i) {
+//       const id = await deviceA.notes.add({
+//         content: {
+//           type: "tiptap",
+//           data: `<p>deviceA=true</p>`,
+//         },
+//       });
+//       noteIds.push(id);
+//     }
+
+//     await deviceA.sync(true);
+//     await deviceB.sync(true);
+
+//     const unsyncedNoteIds = [];
+//     for (let id of noteIds) {
+//       const noteId = await deviceA.notes.add({
+//         id,
+//         content: {
+//           type: "tiptap",
+//           data: `<p>deviceA=true+changed=true</p>`,
+//         },
+//       });
+//       unsyncedNoteIds.push(noteId);
+//     }
+
+//     deviceA.eventManager.subscribe(
+//       EVENTS.syncProgress,
+//       async ({ type, total, current }) => {
+//         const half = total / 2 + 1;
+//         if (type === "upload" && current === half) {
+//           await deviceA.syncer.stop();
+//         }
+//       }
+//     );
+
+//     await expect(deviceA.sync(true)).rejects.toThrow();
+
+//     await delay(10 * 1000);
+
+//     for (let id of unsyncedNoteIds) {
+//       await deviceB.notes.add({
+//         id,
+//         content: {
+//           type: "tiptap",
+//           data: "<p>changes from device B</p>",
+//         },
+//       });
+//     }
+
+//     const error = await withError(async () => {
+//       await deviceB.sync(true);
+//       await deviceA.sync(true);
+//     });
+
+//     expect(error).not.toBeInstanceOf(NoErrorThrownError);
+//     expect(error.message.includes("Merge")).toBeTruthy();
+
+//     await cleanup(deviceA, deviceB);
+//   },
+//
+// );
+
+test(
+  "issue: running force sync from device A makes device B always download everything",
+  async () => {
+    const [deviceA, deviceB] = await Promise.all([
+      initializeDevice("deviceA"),
+      initializeDevice("deviceB")
+    ]);
+
+    await syncAndWait(deviceA, deviceB, true);
+
+    const handler = vitest.fn();
+    deviceB.eventManager.subscribe(EVENTS.syncProgress, handler);
+
+    await deviceB.sync(true);
+
+    expect(handler).not.toHaveBeenCalled();
+
+    await cleanup(deviceB);
+  },
+  TEST_TIMEOUT
+);
+
+test(
+  "issue: colors are not properly created if multiple notes are synced together",
+  async () => {
+    const [deviceA, deviceB] = await Promise.all([
+      initializeDevice("deviceA", [CHECK_IDS.noteColor]),
+      initializeDevice("deviceB", [CHECK_IDS.noteColor])
+    ]);
+
+    const noteIds = [];
+    for (let i = 0; i < 3; ++i) {
+      const id = await deviceA.notes.add({
+        content: {
+          type: "tiptap",
+          data: `<p>deviceA=true</p>`
+        }
+      });
+      noteIds.push(id);
+    }
+
+    await syncAndWait(deviceA, deviceB);
+
+    for (let noteId of noteIds) {
+      await deviceA.notes.note(noteId).color("purple");
+      expect(deviceB.notes.note(noteId)).toBeTruthy();
+      expect(deviceB.notes.note(noteId).data.color).toBeUndefined();
+    }
+
+    await syncAndWait(deviceA, deviceB);
+
+    const purpleColor = deviceB.colors.tag("purple");
+    expect(noteIds.every((id) => purpleColor.noteIds.indexOf(id) > -1)).toBe(
+      true
+    );
+
+    await cleanup(deviceA, deviceB);
+  },
+  TEST_TIMEOUT
+);
+
+test(
+  "issue: new topic on device A gets replaced by the new topic on device B",
+  async () => {
+    const [deviceA, deviceB] = await Promise.all([
+      initializeDevice("deviceA"),
+      initializeDevice("deviceB")
+    ]);
+    // const deviceA = await initializeDevice("deviceA");
+    // const deviceB = await initializeDevice("deviceB");
+
+    const id = await deviceA.notebooks.add({ title: "Notebook 1" });
+
+    await syncAndWait(deviceA, deviceB, false);
+
+    expect(deviceB.notebooks.notebook(id)).toBeDefined();
+
+    await deviceA.notebooks.notebook(id).topics.add("Topic 1");
+    // to create a conflict
+    await delay(1500);
+    await deviceB.notebooks.notebook(id).topics.add("Topic 2");
+
+    expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
+    expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
+
+    expect(
+      deviceB.notebooks.notebook(id).topics.topic("Topic 2")._topic.dateEdited
+    ).toBeGreaterThan(
+      deviceA.notebooks.notebook(id).topics.topic("Topic 1")._topic.dateEdited
+    );
+    expect(deviceB.notebooks.notebook(id).dateModified).toBeGreaterThan(
+      deviceA.notebooks.notebook(id).dateModified
+    );
+
+    await syncAndWait(deviceB, deviceA, false);
+
+    expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
+    expect(deviceB.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
+
+    expect(deviceA.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
+    expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
+
+    await cleanup(deviceA, deviceB);
+  },
+  TEST_TIMEOUT
+);
+
+test(
+  "issue: assigning 2 notes to the same topic should keep references of both notes in the topic",
+  async (ctx) => {
+    const [deviceA, deviceB] = await Promise.all([
+      initializeDevice("deviceA"),
+      initializeDevice("deviceB")
+    ]);
+
+    const id = await deviceA.notebooks.add({
+      title: "Notebook 1",
+      topics: ["Topic 1"]
+    });
+
+    const topic = deviceA.notebooks.notebook(id).topics.topic("Topic 1");
+
+    await syncAndWait(deviceA, deviceB, false);
+
+    expect(deviceB.notebooks.notebook(id)).toBeDefined();
+
+    const noteA = await deviceA.notes.add({ title: "Note 1" });
+    await deviceA.notes.addToNotebook({ id, topic: topic.id }, noteA);
+
+    expect(
+      deviceA.notebooks.notebook(id).topics.topic(topic.id).totalNotes
+    ).toBe(1);
+
+    await delay(2000);
+
+    const noteB = await deviceB.notes.add({ title: "Note 2" });
+    await deviceB.notes.addToNotebook({ id, topic: topic.id }, noteB);
+
+    expect(
+      deviceB.notebooks.notebook(id).topics.topic(topic.id).totalNotes
+    ).toBe(1);
+
+    ctx.onTestFailed(() => {
+      console.log(deviceA.notes.topicReferences.get(topic.id), noteA);
+      console.log(deviceB.notes.topicReferences.get(topic.id), noteB);
+
+      deviceB.notes.topicReferences.rebuild();
+      deviceA.notes.topicReferences.rebuild();
+
+      console.log(deviceA.notes.topicReferences.get(topic.id), noteA);
+      console.log(deviceB.notes.topicReferences.get(topic.id), noteB);
+    });
+    await syncAndWait(deviceB, deviceA, false);
+
+    expect(deviceA.notes.note(noteB)).toBeDefined();
+    expect(deviceB.notes.note(noteA)).toBeDefined();
+
+    expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(1);
+    expect(deviceA.notes.note(noteB).data.notebooks).toHaveLength(1);
+
+    expect(
+      deviceA.notebooks.notebook(id).topics.topic(topic.id).totalNotes
+    ).toBe(2);
+    expect(
+      deviceB.notebooks.notebook(id).topics.topic(topic.id).totalNotes
+    ).toBe(2);
+
+    await cleanup(deviceA, deviceB);
+  },
+  TEST_TIMEOUT
+);
+
+/**
+ *
+ * @param {string} id
+ * @returns {Promise<Database>}
+ */
+async function initializeDevice(id, capabilities = []) {
+  console.time(`Init ${id}`);
+  EV.subscribe(EVENTS.userCheckStatus, async (type) => {
+    return {
+      type,
+      result: capabilities.indexOf(type) > -1
+    };
+  });
+  EV.subscribe(EVENTS.syncCheckStatus, async (type) => {
+    return {
+      type,
+      result: true
+    };
+  });
+
+  const device = new Database();
+  device.setup(new NodeStorageInterface(), EventSource, FS, Compressor);
+
+  await device.init();
+
+  await login(device);
+
+  await device.user.resetUser(false);
+
+  await device.sync(true, false);
+
+  console.timeEnd(`Init ${id}`);
+  return device;
+}
+
+/**
+ *
+ * @param  {...Database} devices
+ */
+async function cleanup(...devices) {
+  await Promise.all([
+    devices.map(async (device) => {
+      await device.syncer.stop();
+      await device.user.logout();
+      device.eventManager.unsubscribeAll();
+    })
+  ]);
+  EV.unsubscribeAll();
+}
+
+/**
+ *
+ * @param {Database} deviceA
+ * @param {Database} deviceB
+ * @returns
+ */
+function syncAndWait(deviceA, deviceB, force = false) {
+  return new Promise((resolve, reject) => {
+    const ref2 = deviceB.eventManager.subscribe(
+      EVENTS.databaseSyncRequested,
+      (full, force, lastSynced) => {
+        console.log("sync requested by device A", full, force, lastSynced);
+        ref2.unsubscribe();
+        deviceB.sync(full, force, lastSynced).catch(reject);
+      }
+    );
+
+    const ref = deviceB.eventManager.subscribe(EVENTS.syncCompleted, () => {
+      ref.unsubscribe();
+      console.log("sync completed.");
+      resolve();
+    });
+
+    console.log(
+      "waiting for sync...",
+      "Device A:",
+      deviceA.syncer.sync.syncing,
+      "Device B:",
+      deviceB.syncer.sync.syncing
+    );
+
+    deviceA.sync(true, force).catch(reject);
+  });
+}

packages/core/__mocks__/fs.mock.ts

@@ -17,44 +17,55 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-var fs = {};
+import { DataFormat, SerializedKey } from "@notesnook/crypto";
+import { xxhash64 } from "hash-wasm";
+import { IDataType } from "hash-wasm/dist/lib/util";
+
+let fs = {};
 
 function hasItem(key) {
   return !!fs[key];
 }
 
-/**
- * We perform 4 steps here:
- * 1. We convert base64 to Uint8Array (if we get base64, that is)
- * 2. We hash the Uint8Array.
- * 3. We encrypt the Uint8Array
- * 4. We save the encrypted Uint8Array
- */
-async function writeEncrypted(filename, { data }) {
-  const { hash, type: hashType } = hashBuffer(data);
-  if (!filename) filename = hash;
-  if (hasItem(filename)) return { hash, hashType };
-  fs[filename] = data;
+async function writeEncryptedBase64(
+  data: string,
+  key: SerializedKey,
+  _mimeType: string
+) {
+  const bytes = new Uint8Array(Buffer.from(data, "base64"));
+
+  const { hash, type: hashType } = await hashBuffer(bytes);
+
+  if (hasItem(hash)) delete fs[hash];
+
+  fs[hash] = data;
   return {
     chunkSize: 512,
     alg: "xcha-stream",
     hash,
     hashType,
     iv: "some iv",
-    cipher: data,
-    salt: "i am some salt",
+    salt: key.salt!,
     length: data.length
   };
 }
 
-function hashBuffer(data) {
+function hashBase64(data: string) {
+  return hashBuffer(Buffer.from(data, "base64"));
+}
+
+export async function hashBuffer(data: IDataType) {
   return {
-    hash: hashCode(data).toString(16),
-    type: "xxh3"
+    hash: await xxhash64(data),
+    type: "xxh64"
   };
 }
 
-async function readEncrypted(filename) {
+async function readEncrypted<TOutputFormat extends DataFormat>(
+  filename: string,
+  _key: SerializedKey,
+  _cipherData: any
+) {
   const cipher = fs[filename];
   if (!cipher) {
     console.error(`File not found. Filename: ${filename}`);
@@ -63,23 +74,23 @@ async function readEncrypted(filename) {
   return cipher.data;
 }
 
-async function uploadFile(filename) {
-  let cipher = fs[filename];
+async function uploadFile(filename: string, _requestOptions: any) {
+  const cipher = fs[filename];
   if (!cipher) throw new Error(`File not found. Filename: ${filename}`);
   return true;
 }
 
-async function downloadFile(filename) {
+async function downloadFile(filename: string, _requestOptions: any) {
   return hasItem(filename);
 }
 
-async function deleteFile(filename) {
+async function deleteFile(filename: string, _requestOptions: any) {
   if (!hasItem(filename)) return true;
   delete fs[filename];
   return true;
 }
 
-function exists(filename) {
+async function exists(filename) {
   return hasItem(filename);
 }
 
@@ -87,34 +98,27 @@ async function clearFileStorage() {
   fs = {};
 }
 
-module.exports = {
-  writeEncrypted,
+export const FS = {
+  writeEncryptedBase64,
   readEncrypted,
   uploadFile: cancellable(uploadFile),
   downloadFile: cancellable(downloadFile),
   deleteFile,
   exists,
-  clearFileStorage
+  clearFileStorage,
+  hashBase64
 };
 
-function cancellable(operation) {
-  return function (filename, requestOptions) {
+function cancellable<T>(
+  operation: (filename: string, requestOptions: any) => Promise<T>
+) {
+  return function (filename: string, requestOptions: any) {
+    const abortController = new AbortController();
     return {
       execute: () => operation(filename, requestOptions),
-      cancel: () => {}
+      cancel: async (message: string) => {
+        abortController.abort(message);
+      }
     };
   };
 }
-
-function hashCode(str) {
-  var hash = 0,
-    i,
-    chr;
-  if (str.length === 0) return hash;
-  for (i = 0; i < str.length; i++) {
-    chr = str.charCodeAt(i);
-    hash = (hash << 5) - hash + chr;
-    hash |= 0; // Convert to 32bit integer
-  }
-  return hash;
-}

packages/core/__mocks__/node-storage.mock.js

@@ -1,94 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import { NNCrypto } from "@notesnook/crypto";
-
-export class NodeStorageInterface {
-  constructor() {
-    this.storage = {};
-    this.crypto = new NNCrypto();
-  }
-
-  async read(key) {
-    return new Promise((resolve) => resolve(this.storage[key]));
-  }
-
-  async readMulti(keys) {
-    return new Promise((resolve) => {
-      const result = [];
-      keys.forEach((key) => {
-        result.push([key, this.storage[key]]);
-      });
-      resolve(result);
-    });
-  }
-
-  async write(key, data) {
-    this.storage[key] = data;
-  }
-
-  async writeMulti(entries) {
-    for (const [key, value] of entries) {
-      this.storage[key] = value;
-    }
-  }
-
-  remove(key) {
-    delete this.storage[key];
-  }
-  clear() {
-    this.storage = {};
-  }
-  getAllKeys() {
-    return Object.keys(this.storage);
-  }
-
-  async encrypt(password, data) {
-    return await this.crypto.encrypt(password, data, "text", "base64");
-  }
-
-  async encryptMulti(password, items) {
-    return await this.crypto.encryptMulti(password, items, "text", "base64");
-  }
-
-  async decrypt(key, cipherData) {
-    cipherData.format = "base64";
-    return await this.crypto.decrypt(key, cipherData, "text");
-  }
-
-  async deriveCryptoKey(name, { password, salt }) {
-    const keyData = await this.crypto.exportKey(password, salt);
-    await this.write(`${name}@_k`, keyData.key);
-  }
-
-  async getCryptoKey(name) {
-    const key = await this.read(`${name}@_k`);
-    if (!key) return;
-    return key;
-  }
-
-  async hash(password, email) {
-    const APP_SALT = "oVzKtazBo7d8sb7TBvY9jw";
-    return await this.crypto.hash(password, `${APP_SALT}${email}`);
-  }
-
-  async generateCryptoKey(password, salt) {
-    return { password, salt };
-  }
-}

packages/core/__mocks__/node-storage.mock.ts

@@ -0,0 +1,108 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Cipher, NNCrypto, SerializedKey } from "@notesnook/crypto";
+
+export class NodeStorageInterface {
+  storage = {};
+  crypto = new NNCrypto();
+
+  async write<T>(key: string, data: T): Promise<void> {
+    this.storage[key] = data;
+  }
+
+  async writeMulti<T>(entries: [key: string, data: T][]) {
+    for (const [key, value] of entries) {
+      this.storage[key] = value;
+    }
+  }
+
+  async readMulti<T>(keys: string[]): Promise<[string, T][]> {
+    const result: [string, T][] = [];
+    keys.forEach((key) => {
+      result.push([key, this.storage[key]]);
+    });
+    return result;
+  }
+
+  async read<T>(
+    key: string,
+    isArray?: boolean | undefined
+  ): Promise<T | undefined> {
+    return this.storage[key];
+  }
+
+  async remove(key: string): Promise<void> {
+    delete this.storage[key];
+  }
+
+  async clear(): Promise<void> {
+    this.storage = {};
+  }
+
+  async getAllKeys(): Promise<string[]> {
+    return Object.keys(this.storage);
+  }
+
+  async encrypt(key: SerializedKey, plainText: string) {
+    return await this.crypto.encrypt(key, plainText, "text", "base64");
+  }
+
+  async encryptMulti(key: SerializedKey, items: string[]) {
+    return await this.crypto.encryptMulti(key, items, "text", "base64");
+  }
+
+  decrypt(key: SerializedKey, cipherData: Cipher<"base64">): Promise<string> {
+    cipherData.format = "base64";
+    return this.crypto.decrypt(key, cipherData, "text");
+  }
+
+  decryptMulti(key: SerializedKey, items: Cipher<"base64">[]) {
+    items.forEach((c) => (c.format = "base64"));
+    return this.crypto.decryptMulti(key, items, "text");
+  }
+
+  async deriveCryptoKey(
+    name: string,
+    credentials: SerializedKey
+  ): Promise<void> {
+    const { password, salt } = credentials;
+    if (!password || !salt) return;
+    const keyData = await this.crypto.exportKey(password, salt);
+    await this.write(`${name}@_k`, keyData.key);
+  }
+
+  async hash(password: string, email: string): Promise<string> {
+    const APP_SALT = "oVzKtazBo7d8sb7TBvY9jw";
+    return await this.crypto.hash(password, `${APP_SALT}${email}`);
+  }
+
+  async getCryptoKey(name: string): Promise<string | undefined> {
+    const key = await this.read<string>(`${name}@_k`);
+    if (!key) return;
+    return key;
+  }
+
+  async generateCryptoKey(
+    password: string,
+    salt?: string | undefined
+  ): Promise<SerializedKey> {
+    return { password, salt };
+  }
+}

packages/core/__tests__/backup.test.js

@@ -33,10 +33,19 @@ import { test, expect, describe } from "vitest";
 test("export backup", () =>
   noteTest().then(() =>
     notebookTest().then(async ({ db }) => {
-      const exp = await db.backup.export("node");
-      let backup = JSON.parse(exp);
+      const exp = [];
+      for await (const file of db.backup.export("node", false)) {
+        exp.push(file);
+      }
+
+      let backup = JSON.parse(exp[1].data);
+      expect(exp.length).toBe(2);
+      expect(exp[0].path).toBe(".nnbackup");
       expect(backup.type).toBe("node");
       expect(backup.date).toBeGreaterThan(0);
+      expect(backup.data).toBeTypeOf("string");
+      expect(backup.compressed).toBe(true);
+      expect(backup.encrypted).toBe(false);
     })
   ));
 
@@ -44,19 +53,34 @@ test("export encrypted backup", () =>
   notebookTest().then(async ({ db }) => {
     await loginFakeUser(db);
     await db.notes.add(TEST_NOTE);
-    const exp = await db.backup.export("node", true);
-    let backup = JSON.parse(exp);
+
+    const exp = [];
+    for await (const file of db.backup.export("node", true)) {
+      exp.push(file);
+    }
+
+    const backup = JSON.parse(exp[1].data);
+    expect(exp.length).toBe(2);
+    expect(exp[0].path).toBe(".nnbackup");
     expect(backup.type).toBe("node");
     expect(backup.date).toBeGreaterThan(0);
     expect(backup.data.iv).not.toBeUndefined();
+    expect(backup.data).toBeTypeOf("object");
+    expect(backup.compressed).toBe(true);
+    expect(backup.encrypted).toBe(true);
   }));
 
 test("import backup", () =>
   notebookTest().then(async ({ db, id }) => {
     await db.notes.add(TEST_NOTE);
-    const exp = await db.backup.export("node");
+
+    const exp = [];
+    for await (const file of db.backup.export("node", false)) {
+      exp.push(file);
+    }
+
     await db.storage.clear();
-    await db.backup.import(JSON.parse(exp));
+    await db.backup.import(JSON.parse(exp[1].data));
     expect(db.notebooks.notebook(id).data.id).toBe(id);
   }));
 
@@ -64,19 +88,29 @@ test("import encrypted backup", () =>
   notebookTest().then(async ({ db, id }) => {
     await loginFakeUser(db);
     await db.notes.add(TEST_NOTE);
-    const exp = await db.backup.export("node", true);
+
+    const exp = [];
+    for await (const file of db.backup.export("node", true)) {
+      exp.push(file);
+    }
+
     await db.storage.clear();
-    await db.backup.import(JSON.parse(exp), "password");
+    await db.backup.import(JSON.parse(exp[1].data), "password");
     expect(db.notebooks.notebook(id).data.id).toBe(id);
   }));
 
 test("import tempered backup", () =>
   notebookTest().then(async ({ db }) => {
     await db.notes.add(TEST_NOTE);
-    const exp = await db.backup.export("node");
+
+    const exp = [];
+    for await (const file of db.backup.export("node", false)) {
+      exp.push(file);
+    }
+
     await db.storage.clear();
-    const backup = JSON.parse(exp);
-    backup.data.hello = "world";
+    const backup = JSON.parse(exp[1].data);
+    backup.data += "hello";
     await expect(db.backup.import(backup)).rejects.toThrow(/tempered/);
   }));
 
@@ -147,21 +181,20 @@ describe.each([
     return databaseTest().then(async (db) => {
       await db.backup.import(qclone(data));
 
-      verifyIndex(data, db, "notes", "notes");
-      verifyIndex(data, db, "notebooks", "notebooks");
-      verifyIndex(data, db, "content", "content");
-      verifyIndex(data, db, "attachments", "attachments");
-      // verifyIndex(data, db, "trash", "trash");
+      const keys = await db.storage.getAllKeys();
+      for (let key in data.data) {
+        const item = data.data[key];
+        if (item && !item.type && item.deleted) continue;
+        if (
+          key.startsWith("_uk_") ||
+          key === "hasConflicts" ||
+          key === "monographs" ||
+          key === "token"
+        )
+          continue;
+
+        expect(keys.some((k) => k.startsWith(key))).toBeTruthy();
+      }
     });
   });
 });
-
-function verifyIndex(backup, db, backupCollection, collection) {
-  if (!backup.data[backupCollection]) return;
-
-  expect(
-    backup.data[backupCollection].every(
-      (v) => db[collection]._collection.indexer.indices.indexOf(v) > -1
-    )
-  ).toBeTruthy();
-}

packages/core/__tests__/utils/index.js

@@ -21,7 +21,7 @@ import DB from "../../src/api";
 import { NodeStorageInterface } from "../../__mocks__/node-storage.mock";
 import dayjs from "dayjs";
 import { groupArray } from "../../src/utils/grouping";
-import FS from "../../__mocks__/fs.mock";
+import { FS } from "../../__mocks__/fs.mock";
 import Compressor from "../../__mocks__/compressor.mock";
 import { expect } from "vitest";
 import EventSource from "eventsource";

packages/core/package-lock.json

@@ -36,7 +36,9 @@
         "abortcontroller-polyfill": "^1.7.3",
         "cross-env": "^7.0.3",
         "dotenv": "^16.0.1",
+        "event-source-polyfill": "^1.0.31",
         "eventsource": "^2.0.2",
+        "hash-wasm": "^4.9.0",
         "isomorphic-fetch": "^3.0.0",
         "jsdom": "^22.1.0",
         "mockdate": "^3.0.5",
@@ -1157,14 +1159,6 @@
         "url": "https://github.com/sponsors/fb55"
       }
     },
-    "node_modules/css.escape": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz",
-      "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==",
-      "dev": true,
-      "optional": true,
-      "peer": true
-    },
     "node_modules/cssom": {
       "version": "0.5.0",
       "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
@@ -1391,6 +1385,12 @@
         "@esbuild/win32-x64": "0.18.20"
       }
     },
+    "node_modules/event-source-polyfill": {
+      "version": "1.0.31",
+      "resolved": "https://registry.npmjs.org/event-source-polyfill/-/event-source-polyfill-1.0.31.tgz",
+      "integrity": "sha512-4IJSItgS/41IxN5UVAVuAyczwZF7ZIEsM1XAoUzIHA6A+xzusEZUutdXz2Nr+MQPLxfTiCvqE79/C8HT8fKFvA==",
+      "dev": true
+    },
     "node_modules/event-target-shim": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
@@ -1479,22 +1479,6 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/happy-dom": {
-      "version": "10.11.2",
-      "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-10.11.2.tgz",
-      "integrity": "sha512-rzgmLjLkhyaOdFEyU8CWXzbgyCyM7wJHLqhaoeEVSTyur1fjcUaiNTHx+D4CPaLvx16tGy+SBPd9TVnP/kzL3w==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "css.escape": "^1.5.1",
-        "entities": "^4.5.0",
-        "iconv-lite": "^0.6.3",
-        "webidl-conversions": "^7.0.0",
-        "whatwg-encoding": "^2.0.0",
-        "whatwg-mimetype": "^3.0.0"
-      }
-    },
     "node_modules/has-flag": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
@@ -1504,6 +1488,12 @@
         "node": ">=8"
       }
     },
+    "node_modules/hash-wasm": {
+      "version": "4.9.0",
+      "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.9.0.tgz",
+      "integrity": "sha512-7SW7ejyfnRxuOc7ptQHSf4LDoZaWOivfzqw+5rpcQku0nHfmicPKE51ra9BiRLAmT8+gGLestr1XroUkqdjL6w==",
+      "dev": true
+    },
     "node_modules/hast-util-parse-selector": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.1.tgz",
@@ -3712,14 +3702,6 @@
       "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
       "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw=="
     },
-    "css.escape": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz",
-      "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==",
-      "dev": true,
-      "optional": true,
-      "peer": true
-    },
     "cssom": {
       "version": "0.5.0",
       "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
@@ -3882,6 +3864,12 @@
         "@esbuild/win32-x64": "0.18.20"
       }
     },
+    "event-source-polyfill": {
+      "version": "1.0.31",
+      "resolved": "https://registry.npmjs.org/event-source-polyfill/-/event-source-polyfill-1.0.31.tgz",
+      "integrity": "sha512-4IJSItgS/41IxN5UVAVuAyczwZF7ZIEsM1XAoUzIHA6A+xzusEZUutdXz2Nr+MQPLxfTiCvqE79/C8HT8fKFvA==",
+      "dev": true
+    },
     "event-target-shim": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
@@ -3945,28 +3933,18 @@
         "path-is-absolute": "^1.0.0"
       }
     },
-    "happy-dom": {
-      "version": "10.11.2",
-      "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-10.11.2.tgz",
-      "integrity": "sha512-rzgmLjLkhyaOdFEyU8CWXzbgyCyM7wJHLqhaoeEVSTyur1fjcUaiNTHx+D4CPaLvx16tGy+SBPd9TVnP/kzL3w==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "requires": {
-        "css.escape": "^1.5.1",
-        "entities": "^4.5.0",
-        "iconv-lite": "^0.6.3",
-        "webidl-conversions": "^7.0.0",
-        "whatwg-encoding": "^2.0.0",
-        "whatwg-mimetype": "^3.0.0"
-      }
-    },
     "has-flag": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
       "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true
     },
+    "hash-wasm": {
+      "version": "4.9.0",
+      "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.9.0.tgz",
+      "integrity": "sha512-7SW7ejyfnRxuOc7ptQHSf4LDoZaWOivfzqw+5rpcQku0nHfmicPKE51ra9BiRLAmT8+gGLestr1XroUkqdjL6w==",
+      "dev": true
+    },
     "hast-util-parse-selector": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.1.tgz",
@@ -4196,8 +4174,7 @@
           "version": "8.13.0",
           "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
           "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
-          "dev": true,
-          "requires": {}
+          "dev": true
         }
       }
     },
@@ -4982,8 +4959,7 @@
     "ws": {
       "version": "7.5.9",
       "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz",
-      "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==",
-      "requires": {}
+      "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q=="
     },
     "xml-name-validator": {
       "version": "4.0.0",

packages/core/package.json

@@ -17,7 +17,9 @@
     "abortcontroller-polyfill": "^1.7.3",
     "cross-env": "^7.0.3",
     "dotenv": "^16.0.1",
+    "event-source-polyfill": "^1.0.31",
     "eventsource": "^2.0.2",
+    "hash-wasm": "^4.9.0",
     "isomorphic-fetch": "^3.0.0",
     "jsdom": "^22.1.0",
     "mockdate": "^3.0.5",

packages/core/src/api/migrations.js

@@ -66,11 +66,7 @@ class Migrations {
           dbCollection: this._db.colors
         },
         {
-          index: () => this._db.trash.raw,
-          dbCollection: this._db.trash
-        },
-        {
-          index: () => this._db.content.all(),
+          iterate: true,
           dbCollection: this._db.content
         },
         {
@@ -91,11 +87,11 @@ class Migrations {
           dbCollection: this._db.relations
         },
         {
-          index: () => this._db.noteHistory.sessionContent.all(),
+          iterate: true,
           dbCollection: this._db.noteHistory
         },
         {
-          index: () => this._db.noteHistory.sessionContent.all(),
+          iterate: true,
           dbCollection: this._db.noteHistory.sessionContent
         },
         {

packages/core/src/api/sync/__tests__/sync.test.skip.js

@@ -1,399 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import Database from "../../index";
-import { NodeStorageInterface } from "../../../../__mocks__/node-storage.mock";
-import FS from "../../../../__mocks__/fs.mock";
-import Compressor from "../../../../__mocks__/compressor.mock";
-import { CHECK_IDS, EV, EVENTS } from "../../../common";
-import EventSource from "eventsource";
-import { delay } from "../../../../__tests__/utils";
-
-jest.setTimeout(100 * 1000);
-
-test("case 1: device A & B should only download the changes from device C (no uploading)", async () => {
-  const types = [];
-  function onSyncProgress({ type }) {
-    types.push(type);
-  }
-
-  const deviceA = await initializeDevice("deviceA");
-  const deviceB = await initializeDevice("deviceB");
-
-  deviceA.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
-  deviceB.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
-
-  const deviceC = await initializeDevice("deviceC");
-
-  await deviceC.notes.add({ title: "new note 1" });
-  await syncAndWait(deviceC, deviceC);
-
-  expect(types.every((t) => t === "download")).toBe(true);
-
-  await cleanup(deviceA, deviceB, deviceC);
-});
-
-test("case 3: Device A & B have unsynced changes but server has nothing", async () => {
-  const deviceA = await initializeDevice("deviceA");
-  const deviceB = await initializeDevice("deviceB");
-
-  const note1Id = await deviceA.notes.add({
-    title: "Test note from device A"
-  });
-  const note2Id = await deviceB.notes.add({
-    title: "Test note from device B"
-  });
-
-  await syncAndWait(deviceA, deviceB);
-
-  expect(deviceA.notes.note(note2Id)).toBeTruthy();
-  expect(deviceB.notes.note(note1Id)).toBeTruthy();
-  expect(deviceA.notes.note(note1Id)).toBeTruthy();
-  expect(deviceB.notes.note(note2Id)).toBeTruthy();
-
-  await cleanup(deviceA, deviceA);
-});
-
-// test(
-//   "case 4: Device A's sync is interrupted halfway and Device B makes some changes afterwards and syncs.",
-//   async () => {
-//     const deviceA = await initializeDevice("deviceA");
-//     const deviceB = await initializeDevice("deviceB");
-
-//     const unsyncedNoteIds = [];
-//     for (let i = 0; i < 10; ++i) {
-//       const id = await deviceA.notes.add({
-//         title: `Test note ${i} from device A`,
-//       });
-//       unsyncedNoteIds.push(id);
-//     }
-
-//     const half = unsyncedNoteIds.length / 2 + 1;
-//     deviceA.eventManager.subscribe(
-//       EVENTS.syncProgress,
-//       async ({ type, current }) => {
-//         if (type === "upload" && current === half) {
-//           await deviceA.syncer.stop();
-//         }
-//       }
-//     );
-
-//     await expect(deviceA.sync(true)).rejects.toThrow();
-
-//     let syncedNoteIds = [];
-//     for (let i = 0; i < unsyncedNoteIds.length; ++i) {
-//       const expectedNoteId = unsyncedNoteIds[i];
-//       if (deviceB.notes.note(expectedNoteId))
-//         syncedNoteIds.push(expectedNoteId);
-//     }
-//     expect(
-//       syncedNoteIds.length === half - 1 || syncedNoteIds.length === half
-//     ).toBe(true);
-
-//     const deviceBNoteId = await deviceB.notes.add({
-//       title: "Test note of case 4 from device B",
-//     });
-
-//     await deviceB.sync(true);
-
-//     await syncAndWait(deviceA, deviceB);
-
-//     expect(deviceA.notes.note(deviceBNoteId)).toBeTruthy();
-//     expect(
-//       unsyncedNoteIds
-//         .map((id) => !!deviceB.notes.note(id))
-//         .every((res) => res === true)
-//     ).toBe(true);
-
-//     await cleanup(deviceA, deviceB);
-//   },
-//
-// );
-
-// test.only(
-//   "case 5: Device A's sync is interrupted halfway and Device B makes changes on the same note's content that didn't get synced on Device A due to interruption.",
-//   async () => {
-//     const deviceA = await initializeDevice("deviceA");
-//     const deviceB = await initializeDevice("deviceB");
-
-//     const noteIds = [];
-//     for (let i = 0; i < 10; ++i) {
-//       const id = await deviceA.notes.add({
-//         content: {
-//           type: "tiptap",
-//           data: `<p>deviceA=true</p>`,
-//         },
-//       });
-//       noteIds.push(id);
-//     }
-
-//     await deviceA.sync(true);
-//     await deviceB.sync(true);
-
-//     const unsyncedNoteIds = [];
-//     for (let id of noteIds) {
-//       const noteId = await deviceA.notes.add({
-//         id,
-//         content: {
-//           type: "tiptap",
-//           data: `<p>deviceA=true+changed=true</p>`,
-//         },
-//       });
-//       unsyncedNoteIds.push(noteId);
-//     }
-
-//     deviceA.eventManager.subscribe(
-//       EVENTS.syncProgress,
-//       async ({ type, total, current }) => {
-//         const half = total / 2 + 1;
-//         if (type === "upload" && current === half) {
-//           await deviceA.syncer.stop();
-//         }
-//       }
-//     );
-
-//     await expect(deviceA.sync(true)).rejects.toThrow();
-
-//     await delay(10 * 1000);
-
-//     for (let id of unsyncedNoteIds) {
-//       await deviceB.notes.add({
-//         id,
-//         content: {
-//           type: "tiptap",
-//           data: "<p>changes from device B</p>",
-//         },
-//       });
-//     }
-
-//     const error = await withError(async () => {
-//       await deviceB.sync(true);
-//       await deviceA.sync(true);
-//     });
-
-//     expect(error).not.toBeInstanceOf(NoErrorThrownError);
-//     expect(error.message.includes("Merge")).toBeTruthy();
-
-//     await cleanup(deviceA, deviceB);
-//   },
-//
-// );
-
-test("issue: running force sync from device A makes device B always download everything", async () => {
-  const deviceA = await initializeDevice("deviceA");
-  const deviceB = await initializeDevice("deviceB");
-
-  await syncAndWait(deviceA, deviceB, true);
-
-  const handler = jest.fn();
-  deviceB.eventManager.subscribe(EVENTS.syncProgress, handler);
-
-  await deviceB.sync(true);
-
-  expect(handler).not.toHaveBeenCalled();
-
-  await cleanup(deviceB);
-});
-
-test("issue: colors are not properly created if multiple notes are synced together", async () => {
-  const deviceA = await initializeDevice("deviceA", [CHECK_IDS.noteColor]);
-  const deviceB = await initializeDevice("deviceB", [CHECK_IDS.noteColor]);
-
-  const noteIds = [];
-  for (let i = 0; i < 3; ++i) {
-    const id = await deviceA.notes.add({
-      content: {
-        type: "tiptap",
-        data: `<p>deviceA=true</p>`
-      }
-    });
-    noteIds.push(id);
-  }
-
-  await syncAndWait(deviceA, deviceB);
-
-  for (let noteId of noteIds) {
-    await deviceA.notes.note(noteId).color("purple");
-    expect(deviceB.notes.note(noteId)).toBeTruthy();
-    expect(deviceB.notes.note(noteId).data.color).toBeUndefined();
-  }
-
-  await syncAndWait(deviceA, deviceB);
-
-  await delay(2000);
-
-  const purpleColor = deviceB.colors.tag("purple");
-  expect(noteIds.every((id) => purpleColor.noteIds.indexOf(id) > -1)).toBe(
-    true
-  );
-
-  await cleanup(deviceA, deviceB);
-});
-
-test("issue: new topic on device A gets replaced by the new topic on device B", async () => {
-  const deviceA = await initializeDevice("deviceA");
-  const deviceB = await initializeDevice("deviceB");
-
-  const id = await deviceA.notebooks.add({ title: "Notebook 1" });
-
-  await syncAndWait(deviceA, deviceB, false);
-
-  expect(deviceB.notebooks.notebook(id)).toBeDefined();
-
-  await deviceA.notebooks.notebook(id).topics.add("Topic 1");
-
-  // to create a conflict
-  await delay(1500);
-
-  await deviceB.notebooks.notebook(id).topics.add("Topic 2");
-
-  expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
-
-  expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
-
-  await syncAndWait(deviceA, deviceB, false);
-
-  await delay(1000);
-
-  await syncAndWait(deviceB, deviceB, false);
-
-  expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
-  expect(deviceB.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
-
-  expect(deviceA.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
-  expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
-
-  await cleanup(deviceA, deviceB);
-});
-
-test("issue: remove notebook reference from notes that are removed from topic during merge", async () => {
-  const deviceA = await initializeDevice("deviceA");
-  const deviceB = await initializeDevice("deviceB");
-
-  const id = await deviceA.notebooks.add({
-    title: "Notebook 1",
-    topics: ["Topic 1"]
-  });
-
-  await syncAndWait(deviceA, deviceB, false);
-
-  expect(deviceB.notebooks.notebook(id)).toBeDefined();
-
-  const noteA = await deviceA.notes.add({ title: "Note 1" });
-  await deviceA.notes.addToNotebook({ id, topic: "Topic 1" }, noteA);
-
-  expect(
-    deviceA.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
-  ).toBe(1);
-
-  await delay(2000);
-
-  const noteB = await deviceB.notes.add({ title: "Note 2" });
-  await deviceB.notes.addToNotebook({ id, topic: "Topic 1" }, noteB);
-
-  expect(
-    deviceB.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
-  ).toBe(1);
-
-  await syncAndWait(deviceB, deviceA, false);
-
-  expect(
-    deviceA.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
-  ).toBe(1);
-  expect(
-    deviceB.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
-  ).toBe(1);
-
-  expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(0);
-
-  await cleanup(deviceA, deviceB);
-});
-
-/**
- *
- * @param {string} id
- * @returns {Promise<Database>}
- */
-async function initializeDevice(id, capabilities = []) {
-  console.time("Init device");
-  EV.subscribe(EVENTS.userCheckStatus, async (type) => {
-    return {
-      type,
-      result: capabilities.indexOf(type) > -1
-    };
-  });
-
-  const device = new Database(
-    new NodeStorageInterface(),
-    EventSource,
-    FS,
-    Compressor
-  );
-  // device.host({
-  //   API_HOST: "http://192.168.10.29:5264",
-  //   AUTH_HOST: "http://192.168.10.29:8264",
-  //   SSE_HOST: "http://192.168.10.29:7264",
-  //   ISSUES_HOST: "http://192.168.10.29:2624",
-  //   SUBSCRIPTIONS_HOST: "http://192.168.10.29:9264",
-  // });
-
-  await device.init(id);
-
-  await device.user.login(
-    process.env.EMAIL,
-    process.env.PASSWORD,
-    process.env.HASHED_PASSWORD
-  );
-
-  await device.user.resetUser(false);
-
-  device.eventManager.subscribe(
-    EVENTS.databaseSyncRequested,
-    async (full, force) => {
-      await device.sync(full, force);
-    }
-  );
-
-  console.timeEnd("Init device");
-  return device;
-}
-
-async function cleanup(...devices) {
-  for (let device of devices) {
-    await device.user.logout();
-    device.eventManager.unsubscribeAll();
-  }
-  EV.unsubscribeAll();
-}
-
-/**
- *
- * @param {Database} deviceA
- * @param {Database} deviceB
- * @returns
- */
-function syncAndWait(deviceA, deviceB, force = false) {
-  return new Promise((resolve) => {
-    const ref = deviceB.eventManager.subscribe(EVENTS.syncCompleted, () => {
-      ref.unsubscribe();
-      resolve();
-    });
-    deviceA.sync(true, force);
-  });
-}