editor: fix code-block ui

.github/workflows/desktop.publish.yml

@@ -97,7 +97,6 @@ jobs:
       - name: Get App Store Version
         id: appstore
         uses: streetwriters/appstore-connect-app-version@develop
-        if: inputs.publish-apple
         with:
           app-id: ${{ steps.app_metadata.outputs.apple_app_id }}
           key-id: ${{ secrets.api_key_id }}

apps/desktop/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@notesnook/desktop",
-  "version": "2.6.4",
+  "version": "2.6.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@notesnook/desktop",
-      "version": "2.6.4",
+      "version": "2.6.3",
       "hasInstallScript": true,
       "license": "GPL-3.0-or-later",
       "dependencies": {

apps/desktop/package.json

@@ -2,7 +2,7 @@
   "name": "@notesnook/desktop",
   "productName": "Notesnook",
   "description": "Your private note taking space",
-  "version": "2.6.4",
+  "version": "2.6.3",
   "appAppleId": "1544027013",
   "private": true,
   "main": "./dist/index.js",

apps/desktop/src/api/os-integration.ts

@@ -117,14 +117,6 @@ export const osIntegrationRouter = t.router({
       writeFileSync(resolvedPath, data);
     }),
 
-  resolvePath: t.procedure
-    .input(z.object({ filePath: z.string() }))
-    .query(({ input }) => {
-      const { filePath } = input;
-      if (!filePath) return;
-      return resolvePath(filePath);
-    }),
-
   showNotification: t.procedure
     .input(NotificationOptions)
     .query(({ input }) => {

apps/desktop/src/preload.ts

@@ -31,6 +31,7 @@ declare global {
 }
 
 process.once("loaded", async () => {
+  console.log("HELLO!");
   const electronTRPC: RendererGlobalElectronTRPC = {
     sendMessage: (operation) =>
       ipcRenderer.send(ELECTRON_TRPC_CHANNEL, operation),

apps/mobile/app/common/filesystem/utils.js

@@ -66,12 +66,9 @@ export function cancelable(operation) {
 }
 
 export function copyFileAsync(source, dest) {
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve) => {
     ScopedStorage.copyFile(source, dest, (e, r) => {
-      if (e) {
-        reject(e);
-        return;
-      }
+      console.log(e, r);
       resolve();
     });
   });

apps/mobile/app/components/list-items/headers/section-header.js

@@ -17,25 +17,35 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+import React, { useEffect, useRef, useState } from "react";
+import { TouchableOpacity, useWindowDimensions, View } from "react-native";
 import { useThemeColors } from "@notesnook/theme";
-import React, { useRef } from "react";
-import { TouchableOpacity, View, useWindowDimensions } from "react-native";
-import { eSendEvent, presentSheet } from "../../../services/event-manager";
-import SettingsService from "../../../services/settings";
 import { useSettingStore } from "../../../stores/use-setting-store";
-import { ColorValues } from "../../../utils/colors";
+import {
+  eSendEvent,
+  eSubscribeEvent,
+  eUnSubscribeEvent,
+  presentSheet
+} from "../../../services/event-manager";
+import SettingsService from "../../../services/settings";
 import { GROUP } from "../../../utils/constants";
+import { ColorValues } from "../../../utils/colors";
+import { db } from "../../../common/database";
 import { eOpenJumpToDialog } from "../../../utils/events";
 import { SIZE } from "../../../utils/size";
-import Sort from "../../sheets/sort";
-import { Button } from "../../ui/button";
 import { IconButton } from "../../ui/icon-button";
+import { Button } from "../../ui/button";
+import Sort from "../../sheets/sort";
 import Heading from "../../ui/typography/heading";
+import { useCallback } from "react";
 
 export const SectionHeader = React.memo(
-  function SectionHeader({ item, index, type, color, screen, groupOptions }) {
+  function SectionHeader({ item, index, type, color, screen }) {
     const { colors } = useThemeColors();
     const { fontScale } = useWindowDimensions();
+    const [groupOptions, setGroupOptions] = useState(
+      db.settings?.getGroupOptions(type)
+    );
     let groupBy = Object.keys(GROUP).find(
       (key) => GROUP[key] === groupOptions.groupBy
     );
@@ -55,6 +65,17 @@ export const SectionHeader = React.memo(
       ? "Default"
       : groupBy.slice(0, 1).toUpperCase() + groupBy.slice(1, groupBy.length);
 
+    const onUpdate = useCallback(() => {
+      setGroupOptions({ ...db.settings?.getGroupOptions(type) });
+    }, [type]);
+
+    useEffect(() => {
+      eSubscribeEvent("groupOptionsUpdate", onUpdate);
+      return () => {
+        eUnSubscribeEvent("groupOptionsUpdate", onUpdate);
+      };
+    }, [onUpdate]);
+
     return (
       <View
         style={{

apps/mobile/app/components/list/index.js

@@ -39,7 +39,6 @@ import { Empty } from "./empty";
 import { getTotalNotes } from "@notesnook/common";
 import { useSettingStore } from "../../stores/use-setting-store";
 import ReminderItem from "../list-items/reminder";
-import { useGroupOptions } from "../../hooks/use-group-options";
 
 const renderItems = {
   note: NoteWrapper,
@@ -100,10 +99,9 @@ const List = ({
     type === "notebooks" ||
     notebooksListMode === "compact";
   const groupType =
-    screen === "Home" ? "home" : screen === "Favorites" ? "favorites" : type;
-
-  const groupOptions = useGroupOptions(groupType);
+    screen === "Notes" ? "home" : screen === "Favorites" ? "favorites" : type;
 
+  const groupOptions = db.settings?.getGroupOptions(groupType);
   const dateBy =
     groupOptions.sortBy !== "title" ? groupOptions.sortBy : "dateEdited";
 
@@ -115,21 +113,18 @@ const List = ({
         color={headerProps?.color}
         title={headerProps?.heading}
         dateBy={dateBy}
-        type={groupType}
+        type={
+          screen === "Notes"
+            ? "home"
+            : screen === "Favorites"
+            ? "favorites"
+            : type
+        }
         screen={screen}
         isSheet={isSheet}
-        groupOptions={groupOptions}
       />
     ),
-    [
-      headerProps?.color,
-      headerProps?.heading,
-      screen,
-      isSheet,
-      dateBy,
-      groupType,
-      groupOptions
-    ]
+    [headerProps?.color, headerProps?.heading, screen, type, isSheet, dateBy]
   );
 
   const _onRefresh = async () => {
@@ -235,7 +230,7 @@ const List = ({
         <JumpToSectionDialog
           screen={screen}
           data={listData}
-          type={screen === "Home" ? "home" : type}
+          type={screen === "Notes" ? "home" : type}
           scrollRef={scrollRef}
         />
       ) : null}

apps/mobile/app/components/selection-header/index.js

@@ -317,8 +317,23 @@ export const SelectionHeader = React.memo(() => {
             customStyle={{
               marginLeft: 10
             }}
-            onPress={() => {
-              deleteItems();
+            onPress={async () => {
+              presentDialog({
+                title: `Delete ${
+                  selectedItemsList.length > 1 ? "items" : "item"
+                }`,
+                paragraph: `Are you sure you want to delete ${
+                  selectedItemsList.length > 1 ? "these items?" : "this item?"
+                }`,
+                positiveText: "Delete",
+                negativeText: "Cancel",
+                positivePress: () => {
+                  deleteItems();
+                },
+                positiveType: "errorShade"
+              });
+
+              return;
             }}
             tooltipText="Move to trash"
             tooltipPosition={1}

apps/mobile/app/components/sheet-provider/index.js

@@ -198,6 +198,7 @@ const SheetProvider = ({ context = "global" }) => {
             key={data.actionText}
             title={data.actionText}
             accentColor={data.iconColor}
+            accentText="light"
             type="accent"
             height={40}
             width={250}

apps/mobile/app/components/sheets/restore-data/index.js

@@ -22,9 +22,7 @@ import React, { useCallback, useEffect, useRef, useState } from "react";
 import { ActivityIndicator, Platform, View } from "react-native";
 import { FlatList } from "react-native-actions-sheet";
 import RNFetchBlob from "react-native-blob-util";
-import DocumentPicker, {
-  DocumentPickerResponse
-} from "react-native-document-picker";
+import DocumentPicker from "react-native-document-picker";
 import * as ScopedStorage from "react-native-scoped-storage";
 import { db } from "../../../common/database";
 import storage from "../../../common/database/storage";
@@ -47,8 +45,6 @@ import Seperator from "../../ui/seperator";
 import SheetWrapper from "../../ui/sheet";
 import Paragraph from "../../ui/typography/paragraph";
 import { getFormattedDate } from "@notesnook/common";
-import { unzip } from "react-native-zip-archive";
-import { cacheDir, copyFileAsync } from "../../../common/filesystem/utils";
 
 const RestoreDataSheet = () => {
   const [visible, setVisible] = useState(false);
@@ -133,71 +129,75 @@ const RestoreDataComponent = ({ close, setRestoring, restoring }) => {
   useEffect(() => {
     setTimeout(() => {
       checkBackups();
-    }, 1000);
+    }, 300);
   }, []);
 
   const restore = async (item) => {
     if (restoring) {
       return;
     }
-
     try {
-      const file = Platform.OS === "ios" ? item.path : item.uri;
-      console.log(file);
-      if (file.endsWith(".nnbackupz")) {
-        setRestoring(true);
+      setRestoring(true);
+      let prefix = Platform.OS === "ios" ? "" : "file:/";
+      let backup;
+      if (Platform.OS === "android") {
+        backup = await ScopedStorage.readFile(item.uri, "utf8");
+      } else {
+        backup = await RNFetchBlob.fs.readFile(prefix + item.path, "utf8");
+      }
+      backup = JSON.parse(backup);
 
-        if (Platform.OS === "android") {
-          const cacheFile = `file://${RNFetchBlob.fs.dirs.CacheDir}/backup.zip`;
-          if (await RNFetchBlob.fs.exists(cacheFile)) {
-            await RNFetchBlob.fs.unlink(cacheFile);
+      if (backup.data.iv && backup.data.salt) {
+        withPassword(
+          async (value) => {
+            try {
+              await restoreBackup(backup, value);
+              close();
+              setRestoring(false);
+              return true;
+            } catch (e) {
+              backupError(e);
+              return false;
+            }
+          },
+          () => {
+            setRestoring(false);
           }
-          await RNFetchBlob.fs.createFile(cacheFile, "", "utf8");
-          console.log("copying");
-          await copyFileAsync(file, cacheFile);
-          console.log("copied");
-          await restoreFromZip(cacheFile);
-        } else {
-          await restoreFromZip(file, false);
-        }
-      } else if (file.endsWith(".nnbackup")) {
-        let backup;
-        if (Platform.OS === "android") {
-          backup = await ScopedStorage.readFile(file, "utf8");
-        } else {
-          backup = await RNFetchBlob.fs.readFile(file, "utf8");
-        }
-        await restoreFromNNBackup(JSON.parse(backup));
+        );
+      } else {
+        await restoreBackup(backup);
+        close();
       }
     } catch (e) {
-      console.log("error", e);
       setRestoring(false);
       backupError(e);
     }
   };
 
-  const withPassword = () => {
-    return new Promise((resolve) => {
-      let resolved = false;
-      presentDialog({
-        context: "local",
-        title: "Encrypted backup",
-        input: true,
-        inputPlaceholder: "Password",
-        paragraph: "Please enter password of this backup file to restore it",
-        positiveText: "Restore",
-        secureTextEntry: true,
-        onClose: () => {
-          if (resolved) return;
-          resolve(undefined);
-        },
-        negativeText: "Cancel",
-        positivePress: async (password) => {
-          resolve(password);
-          resolved = true;
-          return true;
+  const withPassword = (onsubmit, onclose = () => {}) => {
+    presentDialog({
+      context: "local",
+      title: "Encrypted backup",
+      input: true,
+      inputPlaceholder: "Password",
+      paragraph: "Please enter password of this backup file to restore it",
+      positiveText: "Restore",
+      secureTextEntry: true,
+      onClose: onclose,
+      negativeText: "Cancel",
+      positivePress: async (password) => {
+        try {
+          return await onsubmit(password);
+        } catch (e) {
+          ToastEvent.show({
+            heading: "Failed to backup data",
+            message: e.message,
+            type: "error",
+            context: "global"
+          });
+          return false;
         }
-      });
+      }
     });
   };
 
@@ -217,164 +217,21 @@ const RestoreDataComponent = ({ close, setRestoring, restoring }) => {
         let path = await storage.checkAndCreateDir("/backups/");
         files = await RNFetchBlob.fs.lstat(path);
       }
-      files = files
-        .filter((file) => {
-          const name = Platform.OS === "android" ? file.name : file.filename;
-          return name.endsWith(".nnbackup") || name.endsWith(".nnbackupz");
-        })
-        .sort(function (a, b) {
-          let timeA = a.lastModified;
-          let timeB = b.lastModified;
-          return timeB - timeA;
-        });
+      files = files.sort(function (a, b) {
+        let timeA = a.lastModified;
+        let timeB = b.lastModified;
+        return timeB - timeA;
+      });
       setFiles(files);
-      setLoading(false);
+      setTimeout(() => {
+        setLoading(false);
+      }, 1000);
     } catch (e) {
       console.log(e);
       setLoading(false);
     }
   };
 
-  const restoreBackup = async (backup, password) => {
-    await db.backup.import(backup, password);
-    await db.initCollections();
-    initialize();
-    ToastEvent.show({
-      heading: "Backup restored successfully.",
-      type: "success",
-      context: "global"
-    });
-    return true;
-  };
-
-  const backupError = (e) => {
-    ToastEvent.show({
-      heading: "Restore failed",
-      message:
-        e.message ||
-        "The selected backup data file is invalid. You must select a *.nnbackup file to restore.",
-      type: "error",
-      context: "local"
-    });
-  };
-
-  /**
-   *
-   * @param {string} file
-   */
-  async function restoreFromZip(file, remove) {
-    try {
-      const zipOutputFolder = `${cacheDir}/backup_extracted`;
-      if (await RNFetchBlob.fs.exists(zipOutputFolder)) {
-        await RNFetchBlob.fs.unlink(zipOutputFolder);
-        await RNFetchBlob.fs.mkdir(zipOutputFolder);
-      }
-      await unzip(file, zipOutputFolder);
-      console.log("Unzipped files successfully to", zipOutputFolder);
-
-      const backupFiles = await RNFetchBlob.fs.ls(zipOutputFolder);
-
-      if (backupFiles.findIndex((file) => file === ".nnbackup") === -1) {
-        throw new Error("Backup file is invalid");
-      }
-
-      let password;
-
-      console.log(`Found ${backupFiles?.length} files to restore from backup`);
-      for (const path of backupFiles) {
-        if (path === ".nnbackup") continue;
-        const filePath = `${zipOutputFolder}/${path}`;
-        const data = await RNFetchBlob.fs.readFile(filePath, "utf8");
-        const parsed = JSON.parse(data);
-
-        if (parsed.encrypted && !password) {
-          console.log("Backup is encrypted...", "requesting password");
-          password = await withPassword();
-          if (!password) throw new Error("Failed to decrypt backup");
-        }
-        await db.backup.import(parsed, password);
-        console.log("Imported", path);
-      }
-      // Remove files from cache
-      RNFetchBlob.fs.unlink(zipOutputFolder).catch(console.log);
-      if (remove) {
-        RNFetchBlob.fs.unlink(file).catch(console.log);
-      }
-
-      await db.initCollections();
-      initialize();
-      setRestoring(false);
-      close();
-      ToastEvent.show({
-        heading: "Backup restored successfully.",
-        type: "success",
-        context: "global"
-      });
-    } catch (e) {
-      backupError(e);
-      setRestoring(false);
-    }
-  }
-
-  /**
-   *
-   * @param {string} file
-   */
-  async function restoreFromNNBackup(backup) {
-    try {
-      if (backup.data.iv && backup.data.salt) {
-        const password = await withPassword();
-        if (password) {
-          try {
-            await restoreBackup(backup, password);
-            close();
-            setRestoring(false);
-          } catch (e) {
-            setRestoring(false);
-            backupError(e);
-          }
-        } else {
-          setRestoring(false);
-        }
-      } else {
-        await restoreBackup(backup);
-        setRestoring(false);
-        close();
-      }
-    } catch (e) {
-      setRestoring(false);
-      backupError(e);
-    }
-  }
-
-  const button = {
-    title: "Restore from files",
-    onPress: async () => {
-      if (restoring) {
-        return;
-      }
-      try {
-        const file = await DocumentPicker.pickSingle({
-          copyTo: "cachesDirectory"
-        });
-
-        if (file.name.endsWith(".nnbackupz")) {
-          setRestoring(true);
-          await restoreFromZip(file.fileCopyUri, true);
-        } else if (file.name.endsWith(".nnbackup")) {
-          RNFetchBlob.fs.unlink(file.fileCopyUri).catch(console.log);
-          setRestoring(true);
-          const data = await fetch(file.uri);
-          await restoreFromNNBackup(await data.json());
-        }
-      } catch (e) {
-        console.log("error", e.stack);
-        setRestoring(false);
-        backupError(e);
-      }
-    }
-  };
-
   const renderItem = ({ item, index }) => (
     <View
       style={{
@@ -414,6 +271,76 @@ const RestoreDataComponent = ({ close, setRestoring, restoring }) => {
     </View>
   );
 
+  const restoreBackup = async (backup, password) => {
+    await db.backup.import(backup, password);
+    setRestoring(false);
+    initialize();
+    ToastEvent.show({
+      heading: "Backup restored successfully.",
+      type: "success",
+      context: "global"
+    });
+  };
+
+  const backupError = (e) => {
+    ToastEvent.show({
+      heading: "Restore failed",
+      message:
+        e.message ||
+        "The selected backup data file is invalid. You must select a *.nnbackup file to restore.",
+      type: "error",
+      context: "local"
+    });
+  };
+
+  const button = {
+    title: "Restore from files",
+    onPress: () => {
+      if (restoring) {
+        return;
+      }
+
+      DocumentPicker.pickSingle()
+        .then((r) => {
+          setRestoring(true);
+          fetch(r.uri)
+            .then(async (r) => {
+              try {
+                let backup = await r.json();
+                if (backup.data.iv && backup.data.salt) {
+                  withPassword(
+                    async (value) => {
+                      try {
+                        restoreBackup(backup, value).then(() => {
+                          close();
+                          setRestoring(false);
+                        });
+                        return true;
+                      } catch (e) {
+                        backupError(e);
+                        setRestoring(false);
+                        return false;
+                      }
+                    },
+                    () => {
+                      setRestoring(false);
+                    }
+                  );
+                } else {
+                  await restoreBackup(backup);
+                  close();
+                }
+              } catch (e) {
+                setRestoring(false);
+                backupError(e);
+              }
+            })
+            .catch(console.log);
+        })
+        .catch(console.log);
+    }
+  };
+
   return (
     <>
       <View>

apps/mobile/app/hooks/use-group-options.ts

@@ -1,49 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-import { useEffect, useState } from "react";
-import { db } from "../common/database";
-import { eSubscribeEvent, eUnSubscribeEvent } from "../services/event-manager";
-import Navigation from "../services/navigation";
-
-export function useGroupOptions(type: any) {
-  const [groupOptions, setGroupOptions] = useState(
-    db.settings?.getGroupOptions(type)
-  );
-
-  useEffect(() => {
-    const onUpdate = () => {
-      const options = db.settings?.getGroupOptions(type) as any;
-      if (
-        groupOptions?.groupBy !== options.groupBy ||
-        groupOptions?.sortBy !== options.sortBy ||
-        groupOptions?.sortDirection !== groupOptions?.sortDirection
-      ) {
-        setGroupOptions({ ...options });
-        Navigation.queueRoutesForUpdate();
-      }
-    };
-
-    eSubscribeEvent("groupOptionsUpdate", onUpdate);
-    return () => {
-      eUnSubscribeEvent("groupOptionsUpdate", onUpdate);
-    };
-  }, [type, groupOptions]);
-
-  return groupOptions;
-}

apps/mobile/app/screens/home/index.tsx

@@ -71,7 +71,7 @@ export const Home = ({ navigation, route }: NavigationProps<"Notes">) => {
       <List
         listData={notes}
         type="notes"
-        screen="Home"
+        screen="Notes"
         loading={loading || !isFocused}
         headerProps={{
           heading: "Notes"

apps/mobile/app/services/backup.js

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+import { sanitizeFilename } from "@notesnook/common";
 import { Platform } from "react-native";
 import RNFetchBlob from "react-native-blob-util";
 import FileViewer from "react-native-file-viewer";
@@ -29,8 +30,6 @@ import { eCloseSheet } from "../utils/events";
 import { sleep } from "../utils/time";
 import { ToastEvent, eSendEvent, presentSheet } from "./event-manager";
 import SettingsService from "./settings";
-import { cacheDir, copyFileAsync } from "../common/filesystem/utils";
-import { zip } from "react-native-zip-archive";
 
 const MS_DAY = 86400000;
 const MS_WEEK = MS_DAY * 7;
@@ -153,6 +152,8 @@ async function run(progress, context) {
   let androidBackupDirectory = await checkBackupDirExists(false, context);
   if (!androidBackupDirectory) return;
 
+  let backup;
+
   if (progress) {
     presentSheet({
       title: "Backing up your data",
@@ -162,67 +163,43 @@ async function run(progress, context) {
     });
   }
 
-  let path;
-  let backupFilePath;
-  let backupFileName = "notesnook_backup_" + Date.now();
-
-  if (Platform.OS === "ios") {
-    path = await storage.checkAndCreateDir("/backups/");
-  }
-
-  const zipSourceFolder = `${cacheDir}/${backupFileName}`;
-  const zipOutputFile =
-    Platform.OS === "ios"
-      ? `${path}/${backupFileName}.nnbackupz`
-      : `${cacheDir}/${backupFileName}.nnbackupz`;
-
-  if (await RNFetchBlob.fs.exists(zipSourceFolder))
-    await RNFetchBlob.fs.unlink(zipSourceFolder);
-
-  await RNFetchBlob.fs.mkdir(zipSourceFolder);
-
   try {
-    for await (const file of db.backup.export(
+    backup = await db.backup.export(
       "mobile",
       SettingsService.get().encryptedBackup
-    )) {
-      console.log("Writing backup chunk of size...", file?.data?.length);
-      await RNFetchBlob.fs.writeFile(
-        `${zipSourceFolder}/${file.path}`,
-        file.data,
-        "utf8"
-      );
-    }
-
-    await zip(zipSourceFolder, zipOutputFile);
-
-    console.log("Final zip:", await RNFetchBlob.fs.stat(zipOutputFile));
-
-    if (Platform.OS === "android") {
-      // Move the zip to user selected directory.
-      const file = await ScopedStorage.createFile(
-        androidBackupDirectory.uri,
-        `${backupFileName}.nnbackupz`,
-        "application/nnbackupz"
-      );
-      await copyFileAsync(`file://${zipOutputFile}`, file.uri);
-      path = file.uri;
-    } else {
-      path = zipOutputFile;
-    }
-    RNFetchBlob.fs.unlink(zipSourceFolder).catch(console.log);
-    updateNextBackupTime();
-
-    let showBackupCompleteSheet = SettingsService.get().showBackupCompleteSheet;
-
-    if (context) return path;
+    );
+    if (!backup) throw new Error("Backup returned empty.");
+  } catch (e) {
     await sleep(300);
-    if (showBackupCompleteSheet) {
-      presentBackupCompleteSheet(backupFilePath);
+    eSendEvent(eCloseSheet);
+    ToastEvent.error(e, "Backup failed!");
+    return null;
+  }
+
+  try {
+    let backupName = "notesnook_backup_" + Date.now();
+    backupName =
+      sanitizeFilename(backupName, { replacement: "_" }) + ".nnbackup";
+    let path;
+    let backupFilePath;
+
+    if (Platform.OS === "ios") {
+      path = await storage.checkAndCreateDir("/backups/");
+      await RNFetchBlob.fs.writeFile(path + backupName, backup, "utf8");
+      backupFilePath = path + backupName;
     } else {
-      progress && eSendEvent(eCloseSheet);
+      backupFilePath = await ScopedStorage.writeFile(
+        androidBackupDirectory.uri,
+        backup,
+        backupName,
+        "nnbackup/json",
+        "utf8",
+        false
+      );
     }
 
+    updateNextBackupTime();
+
     ToastEvent.show({
       heading: "Backup successful",
       message: "Your backup is stored in Notesnook folder on your phone.",
@@ -230,10 +207,18 @@ async function run(progress, context) {
       context: "global"
     });
 
-    return path;
-  } catch (e) {
+    let showBackupCompleteSheet = SettingsService.get().showBackupCompleteSheet;
+
+    if (context) return backupFilePath;
     await sleep(300);
-    eSendEvent(eCloseSheet);
+    if (showBackupCompleteSheet) {
+      presentBackupCompleteSheet(backupFilePath);
+    } else {
+      progress && eSendEvent(eCloseSheet);
+    }
+    return backupFilePath;
+  } catch (e) {
+    progress && eSendEvent(eCloseSheet);
     ToastEvent.error(e, "Backup failed!");
     return null;
   }

apps/mobile/app/utils/functions.js

@@ -107,43 +107,46 @@ export const deleteItems = async (item, context) => {
 
   if (topics?.length > 0) {
     const result = await confirmDeleteAllNotes(topics, "topic", context);
-    if (!result.delete) return;
-    for (const topic of topics) {
-      if (result.deleteNotes) {
-        const notes = db.notebooks
-          .notebook(topic.notebookId)
-          .topics.topic(topic.id).all;
-        await db.notes.delete(...notes.map((note) => note.id));
-      }
-      await db.notebooks.notebook(topic.notebookId).topics.delete(topic.id);
-    }
-    useMenuStore.getState().setMenuPins();
-    ToastEvent.show({
-      heading: `${topics.length > 1 ? "Topics" : "Topic"} deleted`,
-      type: "success"
-    });
-  }
-
-  if (notebooks?.length > 0) {
-    const result = await confirmDeleteAllNotes(notebooks, "notebook", context);
-    if (!result.delete) return;
-    let ids = notebooks.map((i) => i.id);
-    if (result.deleteNotes) {
-      for (let id of ids) {
-        const notebook = db.notebooks.notebook(id);
-        const topics = notebook.topics.all;
-        for (let topic of topics) {
+    if (result.delete) {
+      for (const topic of topics) {
+        if (result.deleteNotes) {
           const notes = db.notebooks
             .notebook(topic.notebookId)
             .topics.topic(topic.id).all;
           await db.notes.delete(...notes.map((note) => note.id));
         }
-        const notes = db.relations.from(notebook.data, "note");
-        await db.notes.delete(...notes.map((note) => note.id));
+        await db.notebooks.notebook(topic.notebookId).topics.delete(topic.id);
       }
+      useMenuStore.getState().setMenuPins();
+      ToastEvent.show({
+        heading: `${topics.length > 1 ? "Topics" : "Topic"} deleted`,
+        type: "success"
+      });
+    }
+  }
+
+  if (notebooks?.length > 0) {
+    const result = await confirmDeleteAllNotes(notebooks, "notebook", context);
+
+    if (result.delete) {
+      let ids = notebooks.map((i) => i.id);
+      if (result.deleteNotes) {
+        for (let id of ids) {
+          const notebook = db.notebooks.notebook(id);
+          const topics = notebook.topics.all;
+          for (let topic of topics) {
+            const notes = db.notebooks
+              .notebook(topic.notebookId)
+              .topics.topic(topic.id).all;
+            await db.notes.delete(...notes.map((note) => note.id));
+          }
+          const notes = db.relations.from(notebook.data, "note");
+          await db.notes.delete(...notes.map((note) => note.id));
+        }
+      }
+      await db.notebooks.delete(...ids);
+      useMenuStore.getState().setMenuPins();
     }
-    await db.notebooks.delete(...ids);
-    useMenuStore.getState().setMenuPins();
   }
 
   Navigation.queueRoutesForUpdate();

apps/mobile/e2e/tests/notebook.e2e.js

@@ -33,7 +33,7 @@ import {
   visibleById
 } from "./utils";
 
-export async function createNotebook(
+async function createNotebook(
   title = "Notebook 1",
   description = true,
   topic = true,

apps/mobile/e2e/tests/sort.e2e.js

@@ -18,7 +18,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import { notesnook } from "../test.ids";
-import { createNotebook } from "./notebook.e2e";
 import {
   tapById,
   visibleByText,
@@ -26,8 +25,7 @@ import {
   prepare,
   tapByText,
   notVisibleByText,
-  sleep,
-  navigate
+  sleep
 } from "./utils";
 
 async function sortBy(sorting, elementText = "Default") {
@@ -106,23 +104,6 @@ describe("Sort & filter", () => {
     await visibleByText("Month");
   });
 
-  it("Sort notes in topic", async () => {
-    await prepare();
-    await navigate("Notebooks");
-    await sleep(500);
-    await createNotebook("Notebook 1", true, true);
-    await sleep(500);
-    await device.pressBack();
-    await sleep(500);
-    await tapByText("Topic");
-    await createNote("A", "A letter");
-    await sleep(500);
-    await createNote("B", "B letter");
-    await sortBy("Abc");
-    await sleep(300);
-    await visibleByText("N");
-  });
-
   it("Compact mode", async () => {
     await prepare();
     await createNote("Note 1", "Note 1");

apps/mobile/native/android/app/build.gradle

@@ -111,7 +111,7 @@ android {
         minSdkVersion rootProject.ext.minSdkVersion
         targetSdkVersion rootProject.ext.targetSdkVersion
         multiDexEnabled true
-        versionCode 2063
+        versionCode 2059
         versionName getNpmVersion()
         testBuildType System.getProperty('testBuildType', 'debug')
         testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner'

apps/mobile/native/android/releasenotes/whatsnew-en-US

@@ -1,3 +1,4 @@
-- New backup format nnbackupz with upto 90% smaller backups
-- Improve UI of code blocks & language selection popup
-- Fixed some bugs in sync v2
+- Much improved and faster sync
+- Note pinned in notifications will update automatically when content changes
+- Quick notes from notifications now respect paragraph spacing settings.
+- Many bug fixes and small improvements

apps/mobile/native/ios/Notesnook.xcodeproj/project.pbxproj

@@ -997,7 +997,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 2056;
+				CURRENT_PROJECT_VERSION = 2053;
 				DEVELOPMENT_TEAM = 53CWBG3QUC;
 				ENABLE_BITCODE = NO;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1071,7 +1071,7 @@
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 2.6.5;
+				MARKETING_VERSION = 2.6.3;
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",
@@ -1102,7 +1102,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution";
 				CODE_SIGN_STYLE = Manual;
-				CURRENT_PROJECT_VERSION = 2056;
+				CURRENT_PROJECT_VERSION = 2053;
 				DEVELOPMENT_TEAM = "";
 				"DEVELOPMENT_TEAM[sdk=iphoneos*]" = 53CWBG3QUC;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1176,7 +1176,7 @@
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 2.6.5;
+				MARKETING_VERSION = 2.6.3;
 				ONLY_ACTIVE_ARCH = NO;
 				OTHER_LDFLAGS = (
 					"$(inherited)",
@@ -1335,7 +1335,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 2056;
+				CURRENT_PROJECT_VERSION = 2053;
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				DEVELOPMENT_TEAM = 53CWBG3QUC;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1347,7 +1347,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.5;
+				MARKETING_VERSION = 2.6.3;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.notewidget;
@@ -1378,7 +1378,7 @@
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution";
 				CODE_SIGN_STYLE = Manual;
 				COPY_PHASE_STRIP = NO;
-				CURRENT_PROJECT_VERSION = 2056;
+				CURRENT_PROJECT_VERSION = 2053;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				DEVELOPMENT_TEAM = "";
 				"DEVELOPMENT_TEAM[sdk=iphoneos*]" = 53CWBG3QUC;
@@ -1391,7 +1391,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.5;
+				MARKETING_VERSION = 2.6.3;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.notewidget;
 				PRODUCT_NAME = "$(TARGET_NAME)";
@@ -1421,7 +1421,7 @@
 				CODE_SIGN_ENTITLEMENTS = "Make Note/Make Note.entitlements";
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 2056;
+				CURRENT_PROJECT_VERSION = 2053;
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				DEVELOPMENT_TEAM = 53CWBG3QUC;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1495,7 +1495,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.5;
+				MARKETING_VERSION = 2.6.3;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.share;
@@ -1526,7 +1526,7 @@
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution";
 				CODE_SIGN_STYLE = Manual;
 				COPY_PHASE_STRIP = NO;
-				CURRENT_PROJECT_VERSION = 2056;
+				CURRENT_PROJECT_VERSION = 2053;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				DEVELOPMENT_TEAM = "";
 				"DEVELOPMENT_TEAM[sdk=iphoneos*]" = 53CWBG3QUC;
@@ -1601,7 +1601,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.5;
+				MARKETING_VERSION = 2.6.3;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.share;
 				PRODUCT_NAME = "$(TARGET_NAME)";

apps/mobile/native/ios/Podfile.lock

@@ -327,7 +327,7 @@ PODS:
     - React-Core
   - react-native-keep-awake (1.2.0):
     - React-Core
-  - react-native-mmkv-storage (0.10.0-alpha.9):
+  - react-native-mmkv-storage (0.10.0-alpha.7):
     - MMKV (~> 1.3.1)
     - React
     - React-Core
@@ -875,7 +875,7 @@ SPEC CHECKSUMS:
   react-native-image-resizer: 00ceb0e05586c7aadf061eea676957a6c2ec60fa
   react-native-in-app-review: db8bb167a5f238e7ceca5c242d6b36ce8c4404a4
   react-native-keep-awake: caee3ff89eaa21dfe29010f0d143566874a04441
-  react-native-mmkv-storage: d4ad55ab411b7f0d4e6269d801b31821ef48970b
+  react-native-mmkv-storage: ec32fd68168716755e1b30f09b476c236c6c68a8
   react-native-netinfo: ccbe1085dffd16592791d550189772e13bf479e2
   react-native-notification-sounds: da78c828fe1bcbb92d8b505d5261890ed315ff39
   react-native-orientation: f1caf84d65f1a4fd4511a18f2b924e634ad7a628

apps/mobile/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@notesnook/mobile",
-  "version": "2.6.5",
+  "version": "2.6.3",
   "private": true,
   "license": "GPL-3.0-or-later",
   "workspaces": [

apps/theme-builder/src/components/theme-builder/index.tsx

@@ -643,25 +643,13 @@ function exportTheme(theme: ThemeDefinition) {
     }),
     `${theme.id}.json`
   );
-  const confirmed = window.confirm(
-    "Do you also want to download code-block.css for this theme?"
-  );
-  if (confirmed) {
-    FileSaver.saveAs(
-      new Blob([theme.codeBlockCSS], {
-        type: "text/plain"
-      }),
-      `code-block.css`
-    );
-  }
 }
 
 function themeToJSON(theme: ThemeDefinition) {
   return JSON.stringify(
     {
       $schema: JSON_SCHEMA_URL,
-      ...theme,
-      codeBlockCSS: undefined
+      ...theme
     },
     undefined,
     2

apps/web/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@notesnook/web",
-  "version": "2.6.5",
+  "version": "2.6.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@notesnook/web",
-      "version": "2.6.5",
+      "version": "2.6.3",
       "license": "GPL-3.0-or-later",
       "dependencies": {
         "@aws-sdk/util-base64-browser": "^3.208.0",

apps/web/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@notesnook/web",
   "description": "Your private note taking space",
-  "version": "2.6.5",
+  "version": "2.6.3",
   "private": true,
   "main": "./src/app.js",
   "homepage": "https://notesnook.com/",

apps/web/public/stream-saver-sw.js

@@ -0,0 +1,154 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/* eslint-disable no-restricted-globals */
+
+self.addEventListener("install", () => {
+  self.skipWaiting();
+});
+
+self.addEventListener("activate", (event) => {
+  event.waitUntil(self.clients.claim());
+});
+
+const map = new Map();
+
+// This should be called once per download
+// Each event has a dataChannel that the data will be piped through
+self.onmessage = (event) => {
+  // We send a heartbeat every x second to keep the
+  // service worker alive if a transferable stream is not sent
+  if (event.data === "ping") {
+    return;
+  }
+
+  const data = event.data;
+  const downloadUrl =
+    data.url ||
+    self.registration.scope +
+      Math.random() +
+      "/" +
+      (typeof data === "string" ? data : data.filename);
+  const port = event.ports[0];
+  const metadata = new Array(3); // [stream, data, port]
+
+  metadata[1] = data;
+  metadata[2] = port;
+
+  if (event.data.transferringReadable) {
+    port.onmessage = (evt) => {
+      port.onmessage = null;
+      metadata[0] = evt.data.readableStream;
+    };
+  } else {
+    metadata[0] = createStream(port);
+  }
+
+  map.set(downloadUrl, metadata);
+  port.postMessage({ download: downloadUrl });
+};
+
+function createStream(port) {
+  // ReadableStream is only supported by chrome 52
+  return new ReadableStream({
+    start(controller) {
+      // When we receive data on the messageChannel, we write
+      port.onmessage = ({ data }) => {
+        if (data === "end") {
+          return controller.close();
+        }
+
+        if (data === "abort") {
+          controller.error("Aborted the download");
+          return;
+        }
+
+        controller.enqueue(data);
+      };
+    },
+    cancel(reason) {
+      console.log("user aborted", reason);
+      port.postMessage({ abort: true });
+    }
+  });
+}
+
+self.onfetch = (event) => {
+  const url = event.request.url;
+
+  // this only works for Firefox
+  if (url.endsWith("/ping")) {
+    return event.respondWith(new Response("pong"));
+  }
+
+  const metadata = map.get(url);
+  if (!metadata) return null;
+
+  const [stream, data, port] = metadata;
+
+  map.delete(url);
+
+  // Not comfortable letting any user control all headers
+  // so we only copy over the length & disposition
+  const responseHeaders = new Headers({
+    "Content-Type": "application/octet-stream; charset=utf-8",
+
+    // To be on the safe side, The link can be opened in a iframe.
+    // but octet-stream should stop it.
+    "Content-Security-Policy": "default-src 'none'",
+    "X-Content-Security-Policy": "default-src 'none'",
+    "X-WebKit-CSP": "default-src 'none'",
+    "X-XSS-Protection": "1; mode=block"
+  });
+
+  let headers = new Headers(data.headers || {});
+
+  if (headers.has("Content-Length")) {
+    responseHeaders.set("Content-Length", headers.get("Content-Length"));
+  }
+
+  if (headers.has("Content-Disposition")) {
+    responseHeaders.set(
+      "Content-Disposition",
+      headers.get("Content-Disposition")
+    );
+  }
+
+  // data, data.filename and size should not be used anymore
+  if (data.size) {
+    console.warn("Depricated");
+    responseHeaders.set("Content-Length", data.size);
+  }
+
+  let fileName = typeof data === "string" ? data : data.filename;
+  if (fileName) {
+    console.warn("Depricated");
+    // Make filename RFC5987 compatible
+    fileName = encodeURIComponent(fileName)
+      .replace(/['()]/g, escape)
+      .replace(/\*/g, "%2A");
+    responseHeaders.set(
+      "Content-Disposition",
+      "attachment; filename*=UTF-8''" + fileName
+    );
+  }
+
+  event.respondWith(new Response(stream, { headers: responseHeaders }));
+
+  port.postMessage({ debug: "Download started" });
+};

apps/web/src/common/desktop-bridge/index.desktop.ts

@@ -63,17 +63,3 @@ function attachListener(event: string) {
     }
   };
 }
-
-export async function createWritableStream(path: string) {
-  const resolvedPath = await desktop.integration.resolvePath.query({
-    filePath: path
-  });
-  if (!resolvedPath) throw new Error("invalid path.");
-  const fs = require("fs");
-  const { Writable } = require("stream");
-  return new WritableStream(
-    Writable.toWeb(
-      fs.createWriteStream(resolvedPath, { encoding: "utf-8" })
-    ).getWriter()
-  );
-}

apps/web/src/common/desktop-bridge/index.ts

@@ -17,10 +17,6 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { createWriteStream } from "../../utils/stream-saver";
 import { type desktop as bridge } from "./index.desktop";
 
 export const desktop: typeof bridge | undefined = undefined;
-export function createWritableStream(filename: string) {
-  return createWriteStream(filename);
-}

apps/web/src/common/index.js

@@ -0,0 +1,217 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+  showFeatureDialog,
+  showLoadingDialog,
+  showPasswordDialog,
+  showReminderDialog
+} from "../common/dialog-controller";
+import Config from "../utils/config";
+import { hashNavigate, getCurrentHash } from "../navigation";
+import { db } from "./db";
+import { sanitizeFilename } from "@notesnook/common";
+import { store as userstore } from "../stores/user-store";
+import FileSaver from "file-saver";
+import { showToast } from "../utils/toast";
+import { SUBSCRIPTION_STATUS } from "./constants";
+import { showFilePicker } from "../utils/file-picker";
+import { logger } from "../utils/logger";
+import { PATHS } from "@notesnook/desktop";
+import { TaskManager } from "./task-manager";
+import { EVENTS } from "@notesnook/core/dist/common";
+import { getFormattedDate } from "@notesnook/common";
+import { desktop } from "./desktop-bridge";
+
+export const CREATE_BUTTON_MAP = {
+  notes: {
+    title: "Add a note",
+    onClick: () =>
+      hashNavigate("/notes/create", { addNonce: true, replace: true })
+  },
+  notebooks: {
+    title: "Create a notebook",
+    onClick: () => hashNavigate("/notebooks/create", { replace: true })
+  },
+  topics: {
+    title: "Create a topic",
+    onClick: () => hashNavigate(`/topics/create`, { replace: true })
+  },
+  tags: {
+    title: "Create a tag",
+    onClick: () => hashNavigate(`/tags/create`, { replace: true })
+  },
+  reminders: {
+    title: "Add a reminder",
+    onClick: () => hashNavigate(`/reminders/create`, { replace: true })
+  }
+};
+
+export async function introduceFeatures() {
+  const hash = getCurrentHash().replace("#", "");
+  if (!!hash || IS_TESTING) return;
+  const features = [];
+  for (let feature of features) {
+    if (!Config.get(`feature:${feature}`)) {
+      await showFeatureDialog(feature);
+    }
+  }
+}
+
+export const DEFAULT_CONTEXT = { colors: [], tags: [], notebook: {} };
+
+export async function createBackup() {
+  const encryptBackups =
+    userstore.get().isLoggedIn && Config.get("encryptBackups", false);
+  const data = await showLoadingDialog({
+    title: "Creating backup",
+    subtitle: "We are creating a backup of your data. Please wait...",
+    action: async () => {
+      return await db.backup.export("web", encryptBackups);
+    }
+  });
+  if (!data) {
+    showToast("error", "Could not create a backup of your data.");
+    return;
+  }
+
+  const filename = sanitizeFilename(`notesnook-backup-${getFormattedDate()}`);
+
+  const ext = "nnbackup";
+  if (IS_DESKTOP_APP) {
+    const directory = Config.get(
+      "backupStorageLocation",
+      PATHS.backupsDirectory
+    );
+    const filePath = `${directory}/${filename}.${ext}`;
+    await desktop?.integration.saveFile.query({ filePath, data });
+    showToast("success", `Backup saved at ${filePath}.`);
+  } else {
+    FileSaver.saveAs(new Blob([Buffer.from(data)]), `${filename}.${ext}`);
+  }
+}
+
+export async function selectBackupFile() {
+  const file = await showFilePicker({
+    acceptedFileTypes: ".nnbackup,application/json"
+  });
+  if (!file) return;
+
+  const reader = new FileReader();
+  const backup = await new Promise((resolve) => {
+    reader.addEventListener("load", (event) => {
+      const text = event.target.result;
+      try {
+        resolve(JSON.parse(text));
+      } catch (e) {
+        alert(
+          "Error: Could not read the backup file provided. Either it's corrupted or invalid."
+        );
+        resolve();
+      }
+    });
+    reader.readAsText(file);
+  });
+
+  return { file, backup };
+}
+
+export async function importBackup() {
+  const { backup } = await selectBackupFile();
+  await restoreBackupFile(backup);
+}
+
+export async function restoreBackupFile(backup) {
+  if (backup.data.iv && backup.data.salt) {
+    await showPasswordDialog("ask_backup_password", async ({ password }) => {
+      const error = await restoreWithProgress(backup, password);
+      return !error;
+    });
+  } else {
+    await restoreWithProgress(backup);
+  }
+}
+
+async function restoreWithProgress(backup, password) {
+  await TaskManager.startTask({
+    title: "Restoring backup",
+    subtitle: "This might take a while",
+    type: "modal",
+    action: (report) => {
+      db.eventManager.subscribe(
+        EVENTS.migrationProgress,
+        ({ collection, total, current }) => {
+          report({
+            text: `Restoring ${collection}...`,
+            current,
+            total
+          });
+        }
+      );
+
+      report({ text: `Restoring...` });
+      return restore(backup, password);
+    }
+  });
+}
+
+export async function verifyAccount() {
+  if (!(await db.user.getUser())) return true;
+  return showPasswordDialog("verify_account", ({ password }) => {
+    return db.user.verifyPassword(password);
+  });
+}
+
+export function totalSubscriptionConsumed(user) {
+  if (!user) return 0;
+  const start = user.subscription?.start;
+  const end = user.subscription?.expiry;
+  if (!start || !end) return 0;
+
+  const total = end - start;
+  const consumed = Date.now() - start;
+
+  return Math.round((consumed / total) * 100);
+}
+
+export async function showUpgradeReminderDialogs() {
+  if (IS_TESTING) return;
+
+  const user = userstore.get().user;
+  if (!user || !user.subscription || user.subscription?.expiry === 0) return;
+
+  const consumed = totalSubscriptionConsumed(user);
+  const isTrial = user?.subscription?.type === SUBSCRIPTION_STATUS.TRIAL;
+  const isBasic = user?.subscription?.type === SUBSCRIPTION_STATUS.BASIC;
+  if (isBasic && consumed >= 100) {
+    await showReminderDialog("trialexpired");
+  } else if (isTrial && consumed >= 75) {
+    await showReminderDialog("trialexpiring");
+  }
+}
+
+async function restore(backup, password) {
+  try {
+    await db.backup.import(backup, password);
+    showToast("success", "Backup restored!");
+  } catch (e) {
+    logger.error(e, "Could not restore the backup");
+    showToast("error", `Could not restore the backup: ${e.message || e}`);
+  }
+}

apps/web/src/common/index.ts

@@ -1,288 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import {
-  showFeatureDialog,
-  showPasswordDialog,
-  showReminderDialog
-} from "./dialog-controller";
-import Config from "../utils/config";
-import { hashNavigate, getCurrentHash } from "../navigation";
-import { db } from "./db";
-import { sanitizeFilename } from "@notesnook/common";
-import { store as userstore } from "../stores/user-store";
-import { showToast } from "../utils/toast";
-import { SUBSCRIPTION_STATUS } from "./constants";
-import { readFile, showFilePicker } from "../utils/file-picker";
-import { logger } from "../utils/logger";
-import { PATHS } from "@notesnook/desktop";
-import { TaskManager } from "./task-manager";
-import { EVENTS } from "@notesnook/core/dist/common";
-import { getFormattedDate } from "@notesnook/common";
-import { createWritableStream } from "./desktop-bridge";
-import { ZipStream } from "../utils/streams/zip-stream";
-import { FeatureKeys } from "../dialogs/feature-dialog";
-import { Reader } from "../utils/zip-reader";
-
-export const CREATE_BUTTON_MAP = {
-  notes: {
-    title: "Add a note",
-    onClick: () =>
-      hashNavigate("/notes/create", { addNonce: true, replace: true })
-  },
-  notebooks: {
-    title: "Create a notebook",
-    onClick: () => hashNavigate("/notebooks/create", { replace: true })
-  },
-  topics: {
-    title: "Create a topic",
-    onClick: () => hashNavigate(`/topics/create`, { replace: true })
-  },
-  tags: {
-    title: "Create a tag",
-    onClick: () => hashNavigate(`/tags/create`, { replace: true })
-  },
-  reminders: {
-    title: "Add a reminder",
-    onClick: () => hashNavigate(`/reminders/create`, { replace: true })
-  }
-};
-
-export async function introduceFeatures() {
-  const hash = getCurrentHash().replace("#", "");
-  if (!!hash || IS_TESTING) return;
-  const features: FeatureKeys[] = [];
-  for (const feature of features) {
-    if (!Config.get(`feature:${feature}`)) {
-      await showFeatureDialog(feature);
-    }
-  }
-}
-
-export const DEFAULT_CONTEXT = { colors: [], tags: [], notebook: {} };
-
-export async function createBackup() {
-  const encryptBackups =
-    userstore.get().isLoggedIn && Config.get("encryptBackups", false);
-
-  const filename = sanitizeFilename(
-    `notesnook-backup-${getFormattedDate(Date.now())}`
-  );
-  const directory = Config.get("backupStorageLocation", PATHS.backupsDirectory);
-  const ext = "nnbackupz";
-  const filePath = IS_DESKTOP_APP
-    ? `${directory}/${filename}.${ext}`
-    : `${filename}.${ext}`;
-
-  const encoder = new TextEncoder();
-  const error = await TaskManager.startTask<Error | void>({
-    type: "modal",
-    title: "Creating backup",
-    subtitle: "We are creating a backup of your data. Please wait...",
-    action: async (report) => {
-      const writeStream = await createWritableStream(filePath);
-
-      await new ReadableStream({
-        start() {},
-        async pull(controller) {
-          for await (const file of db.backup!.export("web", encryptBackups)) {
-            report({
-              text: `Saving chunk ${file.path}`
-            });
-            controller.enqueue({
-              path: file.path,
-              data: encoder.encode(file.data)
-            });
-          }
-          controller.close();
-        }
-      })
-        .pipeThrough(new ZipStream())
-        .pipeTo(writeStream);
-    }
-  });
-  if (error) {
-    showToast(
-      "error",
-      `Could not create a backup of your data: ${(error as Error).message}`
-    );
-    console.error(error);
-  } else {
-    showToast("success", `Backup saved at ${filePath}.`);
-  }
-}
-
-export async function selectBackupFile() {
-  const file = await showFilePicker({
-    acceptedFileTypes: ".nnbackup,.nnbackupz"
-  });
-  if (!file) return;
-  return file;
-}
-
-export async function importBackup() {
-  const backupFile = await selectBackupFile();
-  if (!backupFile) return;
-  await restoreBackupFile(backupFile);
-}
-
-export async function restoreBackupFile(backupFile: File) {
-  const isLegacy = !backupFile.name.endsWith(".nnbackupz");
-
-  if (isLegacy) {
-    const backup = JSON.parse(await readFile(backupFile));
-
-    if (backup.data.iv && backup.data.salt) {
-      await showPasswordDialog("ask_backup_password", async ({ password }) => {
-        if (!password) return false;
-        const error = await restoreWithProgress(backup, password);
-        return !error;
-      });
-    } else {
-      await restoreWithProgress(backup);
-    }
-  } else {
-    const error = await TaskManager.startTask<Error | void>({
-      title: "Restoring backup",
-      subtitle: "Please wait while we restore your backup...",
-      type: "modal",
-      action: async (report) => {
-        let cachedPassword: string | undefined = undefined;
-        const { read, totalFiles } = await Reader(backupFile);
-        let filesProcessed = 0;
-        for await (const entry of read()) {
-          if (filesProcessed++ === 0 && entry.name !== ".nnbackup")
-            throw new Error("Invalid backup.");
-          else if (entry.name === ".nnbackup") continue;
-
-          const backup = JSON.parse(await entry.text());
-          if (backup.encrypted) {
-            if (!cachedPassword) {
-              const result = await showPasswordDialog(
-                "ask_backup_password",
-                async ({ password }) => {
-                  if (!password) return false;
-                  await db.backup?.import(backup, password);
-                  cachedPassword = password;
-                  return true;
-                }
-              );
-              if (!result) break;
-            } else await db.backup?.import(backup, cachedPassword);
-          } else {
-            await db.backup?.import(backup, null);
-          }
-
-          report({
-            total: totalFiles,
-            text: `Processed ${entry.name}`,
-            current: filesProcessed
-          });
-        }
-        await db.initCollections();
-      }
-    });
-    if (error) {
-      console.error(error);
-      showToast("error", `Failed to restore backup: ${error.message}`);
-    }
-  }
-}
-
-async function restoreWithProgress(
-  backup: Record<string, unknown>,
-  password?: string
-) {
-  return await TaskManager.startTask<Error | void>({
-    title: "Restoring backup",
-    subtitle: "This might take a while",
-    type: "modal",
-    action: (report) => {
-      db.eventManager.subscribe(
-        EVENTS.migrationProgress,
-        ({
-          collection,
-          total,
-          current
-        }: {
-          collection: string;
-          total: number;
-          current: number;
-        }) => {
-          report({
-            text: `Restoring ${collection}...`,
-            current,
-            total
-          });
-        }
-      );
-
-      report({ text: `Restoring...` });
-      return restore(backup, password);
-    }
-  });
-}
-
-export async function verifyAccount() {
-  if (!(await db.user?.getUser())) return true;
-  return showPasswordDialog("verify_account", ({ password }) => {
-    return db.user?.verifyPassword(password) || false;
-  });
-}
-
-export function totalSubscriptionConsumed(user: User) {
-  if (!user) return 0;
-  const start = user.subscription?.start;
-  const end = user.subscription?.expiry;
-  if (!start || !end) return 0;
-
-  const total = end - start;
-  const consumed = Date.now() - start;
-
-  return Math.round((consumed / total) * 100);
-}
-
-export async function showUpgradeReminderDialogs() {
-  if (IS_TESTING) return;
-
-  const user = userstore.get().user;
-  if (!user || !user.subscription || user.subscription?.expiry === 0) return;
-
-  const consumed = totalSubscriptionConsumed(user);
-  const isTrial = user?.subscription?.type === SUBSCRIPTION_STATUS.TRIAL;
-  const isBasic = user?.subscription?.type === SUBSCRIPTION_STATUS.BASIC;
-  if (isBasic && consumed >= 100) {
-    await showReminderDialog("trialexpired");
-  } else if (isTrial && consumed >= 75) {
-    await showReminderDialog("trialexpiring");
-  }
-}
-
-async function restore(backup: Record<string, unknown>, password?: string) {
-  try {
-    await db.backup?.import(backup, password);
-    showToast("success", "Backup restored!");
-  } catch (e) {
-    logger.error(e as Error, "Could not restore the backup");
-    showToast(
-      "error",
-      `Could not restore the backup: ${(e as Error).message || e}`
-    );
-  }
-}

apps/web/src/dialogs/settings/backup-export-settings.ts

@@ -50,7 +50,7 @@ export const BackupExportSettings: SettingsGroup[] = [
               )
                 useSettingStore.getState().toggleEncryptBackups();
               const verified =
-                useSettingStore.getState().encryptBackups ||
+                !useSettingStore.getState().encryptBackups ||
                 (await verifyAccount());
               if (verified) await createBackup();
             },

apps/web/src/index.tsx

@@ -23,7 +23,6 @@ import { logger } from "./utils/logger";
 import { loadDatabase } from "./hooks/use-database";
 import { AppEventManager, AppEvents } from "./common/app-events";
 import { BaseThemeProvider } from "./components/theme-provider";
-import { register } from "./utils/stream-saver/mitm";
 
 renderApp();
 
@@ -67,9 +66,6 @@ async function initializeServiceWorker() {
         AppEventManager.publish(AppEvents.updateDownloadCompleted, {
           version: formatted
         });
-      },
-      onSuccess() {
-        register();
       }
     });
     // window.addEventListener("beforeinstallprompt", () => showInstallNotice());

apps/web/src/service-worker-registration.ts

@@ -86,11 +86,9 @@ function registerValidSW(
   navigator.serviceWorker
     .register(swUrl)
     .then((registration) => {
-      if (config.onSuccess) config.onSuccess(registration);
       registration.onupdatefound = () => {
         const installingWorker = registration.installing;
         if (installingWorker == null) {
-          if (config.onSuccess) config.onSuccess(registration);
           return;
         }
         installingWorker.onstatechange = () => {

apps/web/src/service-worker.ts

@@ -80,8 +80,6 @@ registerRoute(
   })
 );
 
-const downloads = new Map<string, any[]>();
-
 // This allows the web app to trigger skipWaiting via
 // registration.waiting.postMessage({type: 'SKIP_WAITING'})
 self.addEventListener("message", (event) => {
@@ -89,10 +87,6 @@ self.addEventListener("message", (event) => {
   if (!data) return;
 
   switch (data.type) {
-    // We send a heartbeat every x second to keep the
-    // service worker alive if a transferable stream is not sent
-    case "PING":
-      break;
     case "SKIP_WAITING":
       self.skipWaiting();
       break;
@@ -106,125 +100,7 @@ self.addEventListener("message", (event) => {
         });
       }
       break;
-    case "REGISTER_DOWNLOAD":
-      {
-        console.log("register download", data);
-        const downloadUrl =
-          data.url ||
-          self.registration.scope +
-            Math.random() +
-            "/" +
-            (typeof data === "string" ? data : data.filename);
-        const port = event.ports[0];
-        const metadata = new Array(3); // [stream, data, port]
-
-        metadata[1] = data;
-        metadata[2] = port;
-
-        if (event.data.transferringReadable) {
-          port.onmessage = (evt) => {
-            port.onmessage = null;
-            metadata[0] = evt.data.readableStream;
-          };
-        } else {
-          metadata[0] = createStream(port);
-        }
-
-        downloads.set(downloadUrl, metadata);
-        port.postMessage({ download: downloadUrl });
-      }
-      break;
     default:
       break;
   }
 });
-
-self.addEventListener("fetch", (event) => {
-  const url = event.request.url;
-
-  // this only works for Firefox
-  if (url.endsWith("/ping")) {
-    return event.respondWith(new Response("pong"));
-  }
-
-  const metadata = downloads.get(url);
-  if (!metadata) return null;
-
-  const [stream, data, port] = metadata;
-
-  downloads.delete(url);
-
-  // Not comfortable letting any user control all headers
-  // so we only copy over the length & disposition
-  const responseHeaders = new Headers({
-    "Content-Type": "application/octet-stream; charset=utf-8",
-
-    // To be on the safe side, The link can be opened in a iframe.
-    // but octet-stream should stop it.
-    "Content-Security-Policy": "default-src 'none'",
-    "X-Content-Security-Policy": "default-src 'none'",
-    "X-WebKit-CSP": "default-src 'none'",
-    "X-XSS-Protection": "1; mode=block"
-  });
-
-  const headers = new Headers(data.headers || {});
-
-  if (headers.has("Content-Length")) {
-    responseHeaders.set("Content-Length", headers.get("Content-Length")!);
-  }
-
-  if (headers.has("Content-Disposition")) {
-    responseHeaders.set(
-      "Content-Disposition",
-      headers.get("Content-Disposition")!
-    );
-  }
-
-  // data, data.filename and size should not be used anymore
-  if (data.size) {
-    console.warn("Depricated");
-    responseHeaders.set("Content-Length", data.size);
-  }
-
-  let fileName = typeof data === "string" ? data : data.filename;
-  if (fileName) {
-    console.warn("Depricated");
-    // Make filename RFC5987 compatible
-    fileName = encodeURIComponent(fileName)
-      .replace(/['()]/g, escape)
-      .replace(/\*/g, "%2A");
-    responseHeaders.set(
-      "Content-Disposition",
-      "attachment; filename*=UTF-8''" + fileName
-    );
-  }
-
-  event.respondWith(new Response(stream, { headers: responseHeaders }));
-
-  port.postMessage({ debug: "Download started" });
-});
-
-function createStream(port: MessagePort) {
-  // ReadableStream is only supported by chrome 52
-  return new ReadableStream({
-    start(controller) {
-      // When we receive data on the messageChannel, we write
-      port.onmessage = ({ data }) => {
-        if (data === "end") {
-          return controller.close();
-        }
-
-        if (data === "abort") {
-          controller.error("Aborted the download");
-          return;
-        }
-
-        controller.enqueue(data);
-      };
-    },
-    cancel(reason) {
-      console.log("user aborted", reason);
-      port.postMessage({ abort: true });
-    }
-  });
-}

apps/web/src/stores/attachment-store.js

@@ -23,6 +23,7 @@ import BaseStore from "./index";
 import { store as editorStore } from "./editor-store";
 import { checkAttachment } from "../common/attachments";
 import { showToast } from "../utils/toast";
+import { register } from "../utils/stream-saver/mitm";
 import { AttachmentStream } from "../utils/streams/attachment-stream";
 import { ZipStream } from "../utils/streams/zip-stream";
 import { createWriteStream } from "../utils/stream-saver";
@@ -56,6 +57,7 @@ class AttachmentStore extends BaseStore {
       (state) => (state.status = { current: 0, total: attachments.length })
     );
 
+    await register();
     abortController = new AbortController();
     const attachmentStream = new AttachmentStream(
       attachments,
@@ -69,9 +71,7 @@ class AttachmentStore extends BaseStore {
     await attachmentStream
       .pipeThrough(new ZipStream())
       .pipeTo(
-        await createWriteStream("attachments.zip", {
-          signal: abortController.signal
-        })
+        createWriteStream("attachments.zip", { signal: abortController.signal })
       );
 
     this.set((state) => (state.status = undefined));

apps/web/src/utils/stream-saver/index.ts

@@ -16,7 +16,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-import { findServiceWorker, postMessage } from "./mitm";
+import { postMessage } from "./mitm";
 
 let supportsTransferable = false;
 
@@ -66,21 +66,26 @@ function checkSupportsTransferable() {
 }
 checkSupportsTransferable();
 
-export async function createWriteStream(
+/**
+ * @param  {string} filename filename that should be used
+ * @param  {object} options  [description]
+ * @param  {number} size     deprecated
+ * @return {WritableStream<Uint8Array>}
+ */
+export function createWriteStream(
   filename: string,
   opts: {
     size?: number;
     pathname?: string;
     signal?: AbortSignal;
   } = {}
-): Promise<WritableStream<Uint8Array>> {
-  const { sw } = await findServiceWorker();
+): WritableStream<Uint8Array> {
   // let bytesWritten = 0; // by StreamSaver.js (not the service worker)
   let downloadUrl: string | null = null;
   let channel: MessageChannel | null = null;
   let ts: TransformStream | null = null;
   let frame: HTMLIFrameElement | null = null;
-  if (sw && !useBlobFallback) {
+  if (!useBlobFallback) {
     channel = new MessageChannel();
 
     // Make filename RFC5987 compatible
@@ -132,20 +137,19 @@ export async function createWriteStream(
       }
     };
 
-    await postMessage(response, [channel.port2]);
+    postMessage(response, [channel.port2]);
   }
 
   let chunks: Uint8Array[] = [];
+
   return (
-    (sw && !useBlobFallback && ts && ts.writable) ||
+    (!useBlobFallback && ts && ts.writable) ||
     new WritableStream({
       write(chunk) {
-        if (opts.signal?.aborted) return;
-
         if (!(chunk instanceof Uint8Array)) {
           throw new TypeError("Can only write Uint8Arrays");
         }
-        if (!sw || useBlobFallback) {
+        if (useBlobFallback) {
           // Safari... The new IE6
           // https://github.com/jimmywarting/StreamSaver.js/issues/69
           //
@@ -174,21 +178,14 @@ export async function createWriteStream(
         }
       },
       close() {
-        if (opts.signal?.aborted) return;
-        if (!sw || useBlobFallback) {
+        if (useBlobFallback) {
           const blob = new Blob(chunks, {
             type: "application/octet-stream; charset=utf-8"
           });
           const link = document.createElement("a");
           link.href = URL.createObjectURL(blob);
           link.download = filename;
-          link.addEventListener("click", () => {
-            // `setTimeout()` due to
-            // https://github.com/LLK/scratch-gui/issues/1783#issuecomment-426286393
-            setTimeout(() => URL.revokeObjectURL(link.href), 30 * 1000);
-          });
           link.click();
-          chunks = [];
         } else {
           channel?.port1.postMessage("end");
         }

apps/web/src/utils/stream-saver/mitm.ts

@@ -16,26 +16,45 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
+export {};
 
-let keepAlive = () => {
-  keepAlive = () => {};
-  const interval = setInterval(async () => {
-    const { sw } = await findServiceWorker();
-    if (sw) {
-      sw.postMessage({ type: "PING" });
-    } else {
-      const ping =
-        location.href.substr(0, location.href.lastIndexOf("/")) + "/ping";
-      fetch(ping).then((res) => {
-        !res.ok && clearInterval(interval);
-        return res.text();
-      });
-    }
-  }, 10000);
-};
+let sw: ServiceWorker | null = null;
+let scope = "";
+
+function registerWorker() {
+  return navigator.serviceWorker
+    .getRegistration("./")
+    .then((swReg) => {
+      return (
+        swReg ||
+        navigator.serviceWorker.register("stream-saver-sw.js", { scope: "./" })
+      );
+    })
+    .then((swReg) => {
+      const swRegTmp = swReg.installing || swReg.waiting;
+
+      scope = swReg.scope;
+      let fn: () => void;
+      return (
+        (sw = swReg.active) ||
+        new Promise((resolve) => {
+          swRegTmp?.addEventListener(
+            "statechange",
+            (fn = () => {
+              if (swRegTmp.state === "activated") {
+                swRegTmp.removeEventListener("statechange", fn);
+                sw = swReg.active;
+                resolve(undefined);
+              }
+            })
+          );
+        })
+      );
+    });
+}
 
 // Now that we have the Service Worker registered we can process messages
-export async function postMessage(
+export function postMessage(
   data: {
     origin?: string;
     referrer?: string;
@@ -46,9 +65,6 @@ export async function postMessage(
   },
   ports: MessagePort[]
 ) {
-  const { scope, sw } = await findServiceWorker();
-  if (!sw) throw new Error("No service worker registered.");
-
   // It's important to have a messageChannel, don't want to interfere
   // with other simultaneous downloads
   if (!ports || !ports.length) {
@@ -91,32 +107,11 @@ export async function postMessage(
 
   const transferable = [ports[0]];
 
-  if (!data.transferringReadable) {
-    keepAlive();
+  return sw?.postMessage(data, transferable);
+}
+
+export async function register() {
+  if (navigator.serviceWorker) {
+    await registerWorker();
   }
-
-  return sw.postMessage({ type: "REGISTER_DOWNLOAD", ...data }, transferable);
-}
-
-export function register() {
-  // FF v102 just started to supports transferable streams, but still needs to ping sw.js
-  // even tough the service worker dose not have to do any kind of work and listen to any
-  // messages... #305
-  keepAlive();
-}
-
-export async function findServiceWorker(): Promise<{
-  sw?: ServiceWorker;
-  scope?: string;
-}> {
-  if (!("serviceWorker" in navigator)) return {};
-
-  const registrations =
-    (await navigator.serviceWorker?.getRegistrations()) || [];
-  for (const registration of registrations) {
-    if (registration.active)
-      return { sw: registration.active, scope: registration.scope };
-  }
-
-  return {};
 }

apps/web/src/views/auth.tsx

@@ -272,7 +272,6 @@ function LoginPassword(props: BaseAuthComponentProps<"login:password">) {
       type="login:password"
       title="Your account password"
       subtitle={"Your password is always hashed before leaving this device."}
-      loadForever
       loading={{
         title: "Logging you in",
         subtitle: "Please wait while you are authenticated."
@@ -807,7 +806,6 @@ type AuthFormProps<TType extends AuthRoutes> = {
   loading: { title: string; subtitle: string };
   type: TType;
   onSubmit: (form: AuthFormData[TType]) => Promise<void>;
-  loadForever?: boolean;
   canSkip?: boolean;
   children?:
     | React.ReactNode
@@ -815,7 +813,7 @@ type AuthFormProps<TType extends AuthRoutes> = {
 };
 
 export function AuthForm<T extends AuthRoutes>(props: AuthFormProps<T>) {
-  const { title, subtitle, children, canSkip, loadForever } = props;
+  const { title, subtitle, children, canSkip } = props;
   const [isSubmitting, setIsSubmitting] = useState(false);
   const [error, setError] = useState<string>();
   const formRef = useRef<HTMLFormElement>(null);
@@ -840,9 +838,7 @@ export function AuthForm<T extends AuthRoutes>(props: AuthFormProps<T>) {
         try {
           setForm(form);
           await props.onSubmit(form);
-          if (!loadForever) setIsSubmitting(false);
         } catch (e) {
-          setIsSubmitting(false);
           const error = e as Error;
           if (error.message === "invalid_grant") {
             setError(
@@ -851,6 +847,8 @@ export function AuthForm<T extends AuthRoutes>(props: AuthFormProps<T>) {
             return;
           }
           setError(error.message);
+        } finally {
+          setIsSubmitting(false);
         }
       }}
       sx={{

apps/web/src/views/recovery.tsx

@@ -40,7 +40,10 @@ type RecoveryKeyFormData = {
 };
 
 type BackupFileFormData = {
-  backupFile: File;
+  backupFile: {
+    file: File;
+    backup: Record<string, unknown>;
+  };
 };
 
 type NewPasswordFormData = BackupFileFormData & {
@@ -403,7 +406,7 @@ function BackupFileMethod(props: BaseRecoveryComponentProps<"method:backup">) {
     if (!backupFile) return;
     const backupFileInput = document.getElementById("backupFile");
     if (!(backupFileInput instanceof HTMLInputElement)) return;
-    backupFileInput.value = backupFile?.name;
+    backupFileInput.value = backupFile?.file?.name;
   }, [backupFile]);
 
   return (
@@ -516,7 +519,7 @@ function NewPassword(props: BaseRecoveryComponentProps<"new">) {
           throw new Error("Could not reset account password.");
 
         if (formData?.backupFile) {
-          await restoreBackupFile(formData?.backupFile);
+          await restoreBackupFile(formData?.backupFile.backup);
           await db.sync(true, true);
         }
 

fastlane/metadata/android/en-US/changelogs/10309.txt

@@ -1,3 +0,0 @@
-- New backup format nnbackupz with upto 90% smaller backups
-- Improve UI of code blocks & language selection popup
-- Fixed some bugs in sync v2

fastlane/metadata/android/en-US/changelogs/10319.txt

@@ -1,3 +0,0 @@
-- New backup format nnbackupz with upto 90% smaller backups
-- Improve UI of code blocks & language selection popup
-- Fixed some bugs in sync v2

packages/core/__e2e__/sync.test.js

@@ -1,466 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import Database from "../src/api/index";
-import { NodeStorageInterface } from "../__mocks__/node-storage.mock";
-import { FS } from "../__mocks__/fs.mock";
-import Compressor from "../__mocks__/compressor.mock";
-import { CHECK_IDS, EV, EVENTS } from "../src/common";
-import { EventSource } from "event-source-polyfill";
-import { delay } from "../__tests__/utils";
-import { test, expect, vitest } from "vitest";
-import { login } from "./utils";
-
-const TEST_TIMEOUT = 60 * 1000;
-
-test(
-  "case 1: device A & B should only download the changes from device C (no uploading)",
-  async () => {
-    const types = [];
-    function onSyncProgress({ type }) {
-      types.push(type);
-    }
-
-    const [deviceA, deviceB, deviceC] = await Promise.all([
-      initializeDevice("deviceA"),
-      initializeDevice("deviceB"),
-      initializeDevice("deviceC")
-    ]);
-
-    deviceA.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
-    deviceB.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
-
-    await deviceC.notes.add({ title: "new note 1" });
-    await syncAndWait(deviceC, deviceC);
-
-    expect(types.every((t) => t === "download")).toBe(true);
-
-    await cleanup(deviceA, deviceB, deviceC);
-  },
-  TEST_TIMEOUT
-);
-
-test(
-  "case 3: Device A & B have unsynced changes but server has nothing",
-  async () => {
-    const [deviceA, deviceB] = await Promise.all([
-      initializeDevice("deviceA"),
-      initializeDevice("deviceB")
-    ]);
-
-    const note1Id = await deviceA.notes.add({
-      title: "Test note from device A"
-    });
-    const note2Id = await deviceB.notes.add({
-      title: "Test note from device B"
-    });
-
-    await syncAndWait(deviceA, deviceB);
-
-    expect(deviceA.notes.note(note2Id)).toBeTruthy();
-    expect(deviceB.notes.note(note1Id)).toBeTruthy();
-    expect(deviceA.notes.note(note1Id)).toBeTruthy();
-    expect(deviceB.notes.note(note2Id)).toBeTruthy();
-
-    await cleanup(deviceA, deviceA);
-  },
-  TEST_TIMEOUT
-);
-
-// test(
-//   "case 4: Device A's sync is interrupted halfway and Device B makes some changes afterwards and syncs.",
-//   async () => {
-//     const deviceA = await initializeDevice("deviceA");
-//     const deviceB = await initializeDevice("deviceB");
-
-//     const unsyncedNoteIds = [];
-//     for (let i = 0; i < 10; ++i) {
-//       const id = await deviceA.notes.add({
-//         title: `Test note ${i} from device A`,
-//       });
-//       unsyncedNoteIds.push(id);
-//     }
-
-//     const half = unsyncedNoteIds.length / 2 + 1;
-//     deviceA.eventManager.subscribe(
-//       EVENTS.syncProgress,
-//       async ({ type, current }) => {
-//         if (type === "upload" && current === half) {
-//           await deviceA.syncer.stop();
-//         }
-//       }
-//     );
-
-//     await expect(deviceA.sync(true)).rejects.toThrow();
-
-//     let syncedNoteIds = [];
-//     for (let i = 0; i < unsyncedNoteIds.length; ++i) {
-//       const expectedNoteId = unsyncedNoteIds[i];
-//       if (deviceB.notes.note(expectedNoteId))
-//         syncedNoteIds.push(expectedNoteId);
-//     }
-//     expect(
-//       syncedNoteIds.length === half - 1 || syncedNoteIds.length === half
-//     ).toBe(true);
-
-//     const deviceBNoteId = await deviceB.notes.add({
-//       title: "Test note of case 4 from device B",
-//     });
-
-//     await deviceB.sync(true);
-
-//     await syncAndWait(deviceA, deviceB);
-
-//     expect(deviceA.notes.note(deviceBNoteId)).toBeTruthy();
-//     expect(
-//       unsyncedNoteIds
-//         .map((id) => !!deviceB.notes.note(id))
-//         .every((res) => res === true)
-//     ).toBe(true);
-
-//     await cleanup(deviceA, deviceB);
-//   },
-//
-// );
-
-// test.only(
-//   "case 5: Device A's sync is interrupted halfway and Device B makes changes on the same note's content that didn't get synced on Device A due to interruption.",
-//   async () => {
-//     const deviceA = await initializeDevice("deviceA");
-//     const deviceB = await initializeDevice("deviceB");
-
-//     const noteIds = [];
-//     for (let i = 0; i < 10; ++i) {
-//       const id = await deviceA.notes.add({
-//         content: {
-//           type: "tiptap",
-//           data: `<p>deviceA=true</p>`,
-//         },
-//       });
-//       noteIds.push(id);
-//     }
-
-//     await deviceA.sync(true);
-//     await deviceB.sync(true);
-
-//     const unsyncedNoteIds = [];
-//     for (let id of noteIds) {
-//       const noteId = await deviceA.notes.add({
-//         id,
-//         content: {
-//           type: "tiptap",
-//           data: `<p>deviceA=true+changed=true</p>`,
-//         },
-//       });
-//       unsyncedNoteIds.push(noteId);
-//     }
-
-//     deviceA.eventManager.subscribe(
-//       EVENTS.syncProgress,
-//       async ({ type, total, current }) => {
-//         const half = total / 2 + 1;
-//         if (type === "upload" && current === half) {
-//           await deviceA.syncer.stop();
-//         }
-//       }
-//     );
-
-//     await expect(deviceA.sync(true)).rejects.toThrow();
-
-//     await delay(10 * 1000);
-
-//     for (let id of unsyncedNoteIds) {
-//       await deviceB.notes.add({
-//         id,
-//         content: {
-//           type: "tiptap",
-//           data: "<p>changes from device B</p>",
-//         },
-//       });
-//     }
-
-//     const error = await withError(async () => {
-//       await deviceB.sync(true);
-//       await deviceA.sync(true);
-//     });
-
-//     expect(error).not.toBeInstanceOf(NoErrorThrownError);
-//     expect(error.message.includes("Merge")).toBeTruthy();
-
-//     await cleanup(deviceA, deviceB);
-//   },
-//
-// );
-
-test(
-  "issue: running force sync from device A makes device B always download everything",
-  async () => {
-    const [deviceA, deviceB] = await Promise.all([
-      initializeDevice("deviceA"),
-      initializeDevice("deviceB")
-    ]);
-
-    await syncAndWait(deviceA, deviceB, true);
-
-    const handler = vitest.fn();
-    deviceB.eventManager.subscribe(EVENTS.syncProgress, handler);
-
-    await deviceB.sync(true);
-
-    expect(handler).not.toHaveBeenCalled();
-
-    await cleanup(deviceB);
-  },
-  TEST_TIMEOUT
-);
-
-test(
-  "issue: colors are not properly created if multiple notes are synced together",
-  async () => {
-    const [deviceA, deviceB] = await Promise.all([
-      initializeDevice("deviceA", [CHECK_IDS.noteColor]),
-      initializeDevice("deviceB", [CHECK_IDS.noteColor])
-    ]);
-
-    const noteIds = [];
-    for (let i = 0; i < 3; ++i) {
-      const id = await deviceA.notes.add({
-        content: {
-          type: "tiptap",
-          data: `<p>deviceA=true</p>`
-        }
-      });
-      noteIds.push(id);
-    }
-
-    await syncAndWait(deviceA, deviceB);
-
-    for (let noteId of noteIds) {
-      await deviceA.notes.note(noteId).color("purple");
-      expect(deviceB.notes.note(noteId)).toBeTruthy();
-      expect(deviceB.notes.note(noteId).data.color).toBeUndefined();
-    }
-
-    await syncAndWait(deviceA, deviceB);
-
-    const purpleColor = deviceB.colors.tag("purple");
-    expect(noteIds.every((id) => purpleColor.noteIds.indexOf(id) > -1)).toBe(
-      true
-    );
-
-    await cleanup(deviceA, deviceB);
-  },
-  TEST_TIMEOUT
-);
-
-test(
-  "issue: new topic on device A gets replaced by the new topic on device B",
-  async () => {
-    const [deviceA, deviceB] = await Promise.all([
-      initializeDevice("deviceA"),
-      initializeDevice("deviceB")
-    ]);
-    // const deviceA = await initializeDevice("deviceA");
-    // const deviceB = await initializeDevice("deviceB");
-
-    const id = await deviceA.notebooks.add({ title: "Notebook 1" });
-
-    await syncAndWait(deviceA, deviceB, false);
-
-    expect(deviceB.notebooks.notebook(id)).toBeDefined();
-
-    await deviceA.notebooks.notebook(id).topics.add("Topic 1");
-    // to create a conflict
-    await delay(1500);
-    await deviceB.notebooks.notebook(id).topics.add("Topic 2");
-
-    expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
-    expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
-
-    expect(
-      deviceB.notebooks.notebook(id).topics.topic("Topic 2")._topic.dateEdited
-    ).toBeGreaterThan(
-      deviceA.notebooks.notebook(id).topics.topic("Topic 1")._topic.dateEdited
-    );
-    expect(deviceB.notebooks.notebook(id).dateModified).toBeGreaterThan(
-      deviceA.notebooks.notebook(id).dateModified
-    );
-
-    await syncAndWait(deviceB, deviceA, false);
-
-    expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
-    expect(deviceB.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
-
-    expect(deviceA.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
-    expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
-
-    await cleanup(deviceA, deviceB);
-  },
-  TEST_TIMEOUT
-);
-
-test(
-  "issue: assigning 2 notes to the same topic should keep references of both notes in the topic",
-  async (ctx) => {
-    const [deviceA, deviceB] = await Promise.all([
-      initializeDevice("deviceA"),
-      initializeDevice("deviceB")
-    ]);
-
-    const id = await deviceA.notebooks.add({
-      title: "Notebook 1",
-      topics: ["Topic 1"]
-    });
-
-    const topic = deviceA.notebooks.notebook(id).topics.topic("Topic 1");
-
-    await syncAndWait(deviceA, deviceB, false);
-
-    expect(deviceB.notebooks.notebook(id)).toBeDefined();
-
-    const noteA = await deviceA.notes.add({ title: "Note 1" });
-    await deviceA.notes.addToNotebook({ id, topic: topic.id }, noteA);
-
-    expect(
-      deviceA.notebooks.notebook(id).topics.topic(topic.id).totalNotes
-    ).toBe(1);
-
-    await delay(2000);
-
-    const noteB = await deviceB.notes.add({ title: "Note 2" });
-    await deviceB.notes.addToNotebook({ id, topic: topic.id }, noteB);
-
-    expect(
-      deviceB.notebooks.notebook(id).topics.topic(topic.id).totalNotes
-    ).toBe(1);
-
-    ctx.onTestFailed(() => {
-      console.log(deviceA.notes.topicReferences.get(topic.id), noteA);
-      console.log(deviceB.notes.topicReferences.get(topic.id), noteB);
-
-      deviceB.notes.topicReferences.rebuild();
-      deviceA.notes.topicReferences.rebuild();
-
-      console.log(deviceA.notes.topicReferences.get(topic.id), noteA);
-      console.log(deviceB.notes.topicReferences.get(topic.id), noteB);
-    });
-    await syncAndWait(deviceB, deviceA, false);
-
-    expect(deviceA.notes.note(noteB)).toBeDefined();
-    expect(deviceB.notes.note(noteA)).toBeDefined();
-
-    expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(1);
-    expect(deviceA.notes.note(noteB).data.notebooks).toHaveLength(1);
-
-    expect(
-      deviceA.notebooks.notebook(id).topics.topic(topic.id).totalNotes
-    ).toBe(2);
-    expect(
-      deviceB.notebooks.notebook(id).topics.topic(topic.id).totalNotes
-    ).toBe(2);
-
-    await cleanup(deviceA, deviceB);
-  },
-  TEST_TIMEOUT
-);
-
-/**
- *
- * @param {string} id
- * @returns {Promise<Database>}
- */
-async function initializeDevice(id, capabilities = []) {
-  console.time(`Init ${id}`);
-  EV.subscribe(EVENTS.userCheckStatus, async (type) => {
-    return {
-      type,
-      result: capabilities.indexOf(type) > -1
-    };
-  });
-  EV.subscribe(EVENTS.syncCheckStatus, async (type) => {
-    return {
-      type,
-      result: true
-    };
-  });
-
-  const device = new Database();
-  device.setup(new NodeStorageInterface(), EventSource, FS, Compressor);
-
-  await device.init();
-
-  await login(device);
-
-  await device.user.resetUser(false);
-
-  await device.sync(true, false);
-
-  console.timeEnd(`Init ${id}`);
-  return device;
-}
-
-/**
- *
- * @param  {...Database} devices
- */
-async function cleanup(...devices) {
-  await Promise.all([
-    devices.map(async (device) => {
-      await device.syncer.stop();
-      await device.user.logout();
-      device.eventManager.unsubscribeAll();
-    })
-  ]);
-  EV.unsubscribeAll();
-}
-
-/**
- *
- * @param {Database} deviceA
- * @param {Database} deviceB
- * @returns
- */
-function syncAndWait(deviceA, deviceB, force = false) {
-  return new Promise((resolve, reject) => {
-    const ref2 = deviceB.eventManager.subscribe(
-      EVENTS.databaseSyncRequested,
-      (full, force, lastSynced) => {
-        console.log("sync requested by device A", full, force, lastSynced);
-        ref2.unsubscribe();
-        deviceB.sync(full, force, lastSynced).catch(reject);
-      }
-    );
-
-    const ref = deviceB.eventManager.subscribe(EVENTS.syncCompleted, () => {
-      ref.unsubscribe();
-      console.log("sync completed.");
-      resolve();
-    });
-
-    console.log(
-      "waiting for sync...",
-      "Device A:",
-      deviceA.syncer.sync.syncing,
-      "Device B:",
-      deviceB.syncer.sync.syncing
-    );
-
-    deviceA.sync(true, force).catch(reject);
-  });
-}

packages/core/__mocks__/fs.mock.js

@@ -17,55 +17,44 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { DataFormat, SerializedKey } from "@notesnook/crypto";
-import { xxhash64 } from "hash-wasm";
-import { IDataType } from "hash-wasm/dist/lib/util";
-
-let fs = {};
+var fs = {};
 
 function hasItem(key) {
   return !!fs[key];
 }
 
-async function writeEncryptedBase64(
-  data: string,
-  key: SerializedKey,
-  _mimeType: string
-) {
-  const bytes = new Uint8Array(Buffer.from(data, "base64"));
-
-  const { hash, type: hashType } = await hashBuffer(bytes);
-
-  if (hasItem(hash)) delete fs[hash];
-
-  fs[hash] = data;
+/**
+ * We perform 4 steps here:
+ * 1. We convert base64 to Uint8Array (if we get base64, that is)
+ * 2. We hash the Uint8Array.
+ * 3. We encrypt the Uint8Array
+ * 4. We save the encrypted Uint8Array
+ */
+async function writeEncrypted(filename, { data }) {
+  const { hash, type: hashType } = hashBuffer(data);
+  if (!filename) filename = hash;
+  if (hasItem(filename)) return { hash, hashType };
+  fs[filename] = data;
   return {
     chunkSize: 512,
     alg: "xcha-stream",
     hash,
     hashType,
     iv: "some iv",
-    salt: key.salt!,
+    cipher: data,
+    salt: "i am some salt",
     length: data.length
   };
 }
 
-function hashBase64(data: string) {
-  return hashBuffer(Buffer.from(data, "base64"));
-}
-
-export async function hashBuffer(data: IDataType) {
+function hashBuffer(data) {
   return {
-    hash: await xxhash64(data),
-    type: "xxh64"
+    hash: hashCode(data).toString(16),
+    type: "xxh3"
   };
 }
 
-async function readEncrypted<TOutputFormat extends DataFormat>(
-  filename: string,
-  _key: SerializedKey,
-  _cipherData: any
-) {
+async function readEncrypted(filename) {
   const cipher = fs[filename];
   if (!cipher) {
     console.error(`File not found. Filename: ${filename}`);
@@ -74,23 +63,23 @@ async function readEncrypted<TOutputFormat extends DataFormat>(
   return cipher.data;
 }
 
-async function uploadFile(filename: string, _requestOptions: any) {
-  const cipher = fs[filename];
+async function uploadFile(filename) {
+  let cipher = fs[filename];
   if (!cipher) throw new Error(`File not found. Filename: ${filename}`);
   return true;
 }
 
-async function downloadFile(filename: string, _requestOptions: any) {
+async function downloadFile(filename) {
   return hasItem(filename);
 }
 
-async function deleteFile(filename: string, _requestOptions: any) {
+async function deleteFile(filename) {
   if (!hasItem(filename)) return true;
   delete fs[filename];
   return true;
 }
 
-async function exists(filename) {
+function exists(filename) {
   return hasItem(filename);
 }
 
@@ -98,27 +87,34 @@ async function clearFileStorage() {
   fs = {};
 }
 
-export const FS = {
-  writeEncryptedBase64,
+module.exports = {
+  writeEncrypted,
   readEncrypted,
   uploadFile: cancellable(uploadFile),
   downloadFile: cancellable(downloadFile),
   deleteFile,
   exists,
-  clearFileStorage,
-  hashBase64
+  clearFileStorage
 };
 
-function cancellable<T>(
-  operation: (filename: string, requestOptions: any) => Promise<T>
-) {
-  return function (filename: string, requestOptions: any) {
-    const abortController = new AbortController();
+function cancellable(operation) {
+  return function (filename, requestOptions) {
     return {
       execute: () => operation(filename, requestOptions),
-      cancel: async (message: string) => {
-        abortController.abort(message);
-      }
+      cancel: () => {}
     };
   };
 }
+
+function hashCode(str) {
+  var hash = 0,
+    i,
+    chr;
+  if (str.length === 0) return hash;
+  for (i = 0; i < str.length; i++) {
+    chr = str.charCodeAt(i);
+    hash = (hash << 5) - hash + chr;
+    hash |= 0; // Convert to 32bit integer
+  }
+  return hash;
+}

packages/core/__mocks__/node-storage.mock.js

@@ -0,0 +1,94 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { NNCrypto } from "@notesnook/crypto";
+
+export class NodeStorageInterface {
+  constructor() {
+    this.storage = {};
+    this.crypto = new NNCrypto();
+  }
+
+  async read(key) {
+    return new Promise((resolve) => resolve(this.storage[key]));
+  }
+
+  async readMulti(keys) {
+    return new Promise((resolve) => {
+      const result = [];
+      keys.forEach((key) => {
+        result.push([key, this.storage[key]]);
+      });
+      resolve(result);
+    });
+  }
+
+  async write(key, data) {
+    this.storage[key] = data;
+  }
+
+  async writeMulti(entries) {
+    for (const [key, value] of entries) {
+      this.storage[key] = value;
+    }
+  }
+
+  remove(key) {
+    delete this.storage[key];
+  }
+  clear() {
+    this.storage = {};
+  }
+  getAllKeys() {
+    return Object.keys(this.storage);
+  }
+
+  async encrypt(password, data) {
+    return await this.crypto.encrypt(password, data, "text", "base64");
+  }
+
+  async encryptMulti(password, items) {
+    return await this.crypto.encryptMulti(password, items, "text", "base64");
+  }
+
+  async decrypt(key, cipherData) {
+    cipherData.format = "base64";
+    return await this.crypto.decrypt(key, cipherData, "text");
+  }
+
+  async deriveCryptoKey(name, { password, salt }) {
+    const keyData = await this.crypto.exportKey(password, salt);
+    await this.write(`${name}@_k`, keyData.key);
+  }
+
+  async getCryptoKey(name) {
+    const key = await this.read(`${name}@_k`);
+    if (!key) return;
+    return key;
+  }
+
+  async hash(password, email) {
+    const APP_SALT = "oVzKtazBo7d8sb7TBvY9jw";
+    return await this.crypto.hash(password, `${APP_SALT}${email}`);
+  }
+
+  async generateCryptoKey(password, salt) {
+    return { password, salt };
+  }
+}

packages/core/__mocks__/node-storage.mock.ts

@@ -1,108 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import { Cipher, NNCrypto, SerializedKey } from "@notesnook/crypto";
-
-export class NodeStorageInterface {
-  storage = {};
-  crypto = new NNCrypto();
-
-  async write<T>(key: string, data: T): Promise<void> {
-    this.storage[key] = data;
-  }
-
-  async writeMulti<T>(entries: [key: string, data: T][]) {
-    for (const [key, value] of entries) {
-      this.storage[key] = value;
-    }
-  }
-
-  async readMulti<T>(keys: string[]): Promise<[string, T][]> {
-    const result: [string, T][] = [];
-    keys.forEach((key) => {
-      result.push([key, this.storage[key]]);
-    });
-    return result;
-  }
-
-  async read<T>(
-    key: string,
-    isArray?: boolean | undefined
-  ): Promise<T | undefined> {
-    return this.storage[key];
-  }
-
-  async remove(key: string): Promise<void> {
-    delete this.storage[key];
-  }
-
-  async clear(): Promise<void> {
-    this.storage = {};
-  }
-
-  async getAllKeys(): Promise<string[]> {
-    return Object.keys(this.storage);
-  }
-
-  async encrypt(key: SerializedKey, plainText: string) {
-    return await this.crypto.encrypt(key, plainText, "text", "base64");
-  }
-
-  async encryptMulti(key: SerializedKey, items: string[]) {
-    return await this.crypto.encryptMulti(key, items, "text", "base64");
-  }
-
-  decrypt(key: SerializedKey, cipherData: Cipher<"base64">): Promise<string> {
-    cipherData.format = "base64";
-    return this.crypto.decrypt(key, cipherData, "text");
-  }
-
-  decryptMulti(key: SerializedKey, items: Cipher<"base64">[]) {
-    items.forEach((c) => (c.format = "base64"));
-    return this.crypto.decryptMulti(key, items, "text");
-  }
-
-  async deriveCryptoKey(
-    name: string,
-    credentials: SerializedKey
-  ): Promise<void> {
-    const { password, salt } = credentials;
-    if (!password || !salt) return;
-    const keyData = await this.crypto.exportKey(password, salt);
-    await this.write(`${name}@_k`, keyData.key);
-  }
-
-  async hash(password: string, email: string): Promise<string> {
-    const APP_SALT = "oVzKtazBo7d8sb7TBvY9jw";
-    return await this.crypto.hash(password, `${APP_SALT}${email}`);
-  }
-
-  async getCryptoKey(name: string): Promise<string | undefined> {
-    const key = await this.read<string>(`${name}@_k`);
-    if (!key) return;
-    return key;
-  }
-
-  async generateCryptoKey(
-    password: string,
-    salt?: string | undefined
-  ): Promise<SerializedKey> {
-    return { password, salt };
-  }
-}

packages/core/__tests__/backup.test.js

@@ -33,19 +33,10 @@ import { test, expect, describe } from "vitest";
 test("export backup", () =>
   noteTest().then(() =>
     notebookTest().then(async ({ db }) => {
-      const exp = [];
-      for await (const file of db.backup.export("node", false)) {
-        exp.push(file);
-      }
-
-      let backup = JSON.parse(exp[1].data);
-      expect(exp.length).toBe(2);
-      expect(exp[0].path).toBe(".nnbackup");
+      const exp = await db.backup.export("node");
+      let backup = JSON.parse(exp);
       expect(backup.type).toBe("node");
       expect(backup.date).toBeGreaterThan(0);
-      expect(backup.data).toBeTypeOf("string");
-      expect(backup.compressed).toBe(true);
-      expect(backup.encrypted).toBe(false);
     })
   ));
 
@@ -53,34 +44,19 @@ test("export encrypted backup", () =>
   notebookTest().then(async ({ db }) => {
     await loginFakeUser(db);
     await db.notes.add(TEST_NOTE);
-
-    const exp = [];
-    for await (const file of db.backup.export("node", true)) {
-      exp.push(file);
-    }
-
-    const backup = JSON.parse(exp[1].data);
-    expect(exp.length).toBe(2);
-    expect(exp[0].path).toBe(".nnbackup");
+    const exp = await db.backup.export("node", true);
+    let backup = JSON.parse(exp);
     expect(backup.type).toBe("node");
     expect(backup.date).toBeGreaterThan(0);
     expect(backup.data.iv).not.toBeUndefined();
-    expect(backup.data).toBeTypeOf("object");
-    expect(backup.compressed).toBe(true);
-    expect(backup.encrypted).toBe(true);
   }));
 
 test("import backup", () =>
   notebookTest().then(async ({ db, id }) => {
     await db.notes.add(TEST_NOTE);
-
-    const exp = [];
-    for await (const file of db.backup.export("node", false)) {
-      exp.push(file);
-    }
-
+    const exp = await db.backup.export("node");
     await db.storage.clear();
-    await db.backup.import(JSON.parse(exp[1].data));
+    await db.backup.import(JSON.parse(exp));
     expect(db.notebooks.notebook(id).data.id).toBe(id);
   }));
 
@@ -88,29 +64,19 @@ test("import encrypted backup", () =>
   notebookTest().then(async ({ db, id }) => {
     await loginFakeUser(db);
     await db.notes.add(TEST_NOTE);
-
-    const exp = [];
-    for await (const file of db.backup.export("node", true)) {
-      exp.push(file);
-    }
-
+    const exp = await db.backup.export("node", true);
     await db.storage.clear();
-    await db.backup.import(JSON.parse(exp[1].data), "password");
+    await db.backup.import(JSON.parse(exp), "password");
     expect(db.notebooks.notebook(id).data.id).toBe(id);
   }));
 
 test("import tempered backup", () =>
   notebookTest().then(async ({ db }) => {
     await db.notes.add(TEST_NOTE);
-
-    const exp = [];
-    for await (const file of db.backup.export("node", false)) {
-      exp.push(file);
-    }
-
+    const exp = await db.backup.export("node");
     await db.storage.clear();
-    const backup = JSON.parse(exp[1].data);
-    backup.data += "hello";
+    const backup = JSON.parse(exp);
+    backup.data.hello = "world";
     await expect(db.backup.import(backup)).rejects.toThrow(/tempered/);
   }));
 
@@ -181,20 +147,21 @@ describe.each([
     return databaseTest().then(async (db) => {
       await db.backup.import(qclone(data));
 
-      const keys = await db.storage.getAllKeys();
-      for (let key in data.data) {
-        const item = data.data[key];
-        if (item && !item.type && item.deleted) continue;
-        if (
-          key.startsWith("_uk_") ||
-          key === "hasConflicts" ||
-          key === "monographs" ||
-          key === "token"
-        )
-          continue;
-
-        expect(keys.some((k) => k.startsWith(key))).toBeTruthy();
-      }
+      verifyIndex(data, db, "notes", "notes");
+      verifyIndex(data, db, "notebooks", "notebooks");
+      verifyIndex(data, db, "content", "content");
+      verifyIndex(data, db, "attachments", "attachments");
+      // verifyIndex(data, db, "trash", "trash");
     });
   });
 });
+
+function verifyIndex(backup, db, backupCollection, collection) {
+  if (!backup.data[backupCollection]) return;
+
+  expect(
+    backup.data[backupCollection].every(
+      (v) => db[collection]._collection.indexer.indices.indexOf(v) > -1
+    )
+  ).toBeTruthy();
+}

packages/core/__tests__/utils/index.js

@@ -21,7 +21,7 @@ import DB from "../../src/api";
 import { NodeStorageInterface } from "../../__mocks__/node-storage.mock";
 import dayjs from "dayjs";
 import { groupArray } from "../../src/utils/grouping";
-import { FS } from "../../__mocks__/fs.mock";
+import FS from "../../__mocks__/fs.mock";
 import Compressor from "../../__mocks__/compressor.mock";
 import { expect } from "vitest";
 import EventSource from "eventsource";

packages/core/package-lock.json

@@ -36,9 +36,7 @@
         "abortcontroller-polyfill": "^1.7.3",
         "cross-env": "^7.0.3",
         "dotenv": "^16.0.1",
-        "event-source-polyfill": "^1.0.31",
         "eventsource": "^2.0.2",
-        "hash-wasm": "^4.9.0",
         "isomorphic-fetch": "^3.0.0",
         "jsdom": "^22.1.0",
         "mockdate": "^3.0.5",
@@ -1393,12 +1391,6 @@
         "@esbuild/win32-x64": "0.18.20"
       }
     },
-    "node_modules/event-source-polyfill": {
-      "version": "1.0.31",
-      "resolved": "https://registry.npmjs.org/event-source-polyfill/-/event-source-polyfill-1.0.31.tgz",
-      "integrity": "sha512-4IJSItgS/41IxN5UVAVuAyczwZF7ZIEsM1XAoUzIHA6A+xzusEZUutdXz2Nr+MQPLxfTiCvqE79/C8HT8fKFvA==",
-      "dev": true
-    },
     "node_modules/event-target-shim": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
@@ -1512,12 +1504,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/hash-wasm": {
-      "version": "4.9.0",
-      "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.9.0.tgz",
-      "integrity": "sha512-7SW7ejyfnRxuOc7ptQHSf4LDoZaWOivfzqw+5rpcQku0nHfmicPKE51ra9BiRLAmT8+gGLestr1XroUkqdjL6w==",
-      "dev": true
-    },
     "node_modules/hast-util-parse-selector": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.1.tgz",
@@ -3896,12 +3882,6 @@
         "@esbuild/win32-x64": "0.18.20"
       }
     },
-    "event-source-polyfill": {
-      "version": "1.0.31",
-      "resolved": "https://registry.npmjs.org/event-source-polyfill/-/event-source-polyfill-1.0.31.tgz",
-      "integrity": "sha512-4IJSItgS/41IxN5UVAVuAyczwZF7ZIEsM1XAoUzIHA6A+xzusEZUutdXz2Nr+MQPLxfTiCvqE79/C8HT8fKFvA==",
-      "dev": true
-    },
     "event-target-shim": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
@@ -3987,12 +3967,6 @@
       "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true
     },
-    "hash-wasm": {
-      "version": "4.9.0",
-      "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.9.0.tgz",
-      "integrity": "sha512-7SW7ejyfnRxuOc7ptQHSf4LDoZaWOivfzqw+5rpcQku0nHfmicPKE51ra9BiRLAmT8+gGLestr1XroUkqdjL6w==",
-      "dev": true
-    },
     "hast-util-parse-selector": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.1.tgz",

packages/core/package.json

@@ -17,9 +17,7 @@
     "abortcontroller-polyfill": "^1.7.3",
     "cross-env": "^7.0.3",
     "dotenv": "^16.0.1",
-    "event-source-polyfill": "^1.0.31",
     "eventsource": "^2.0.2",
-    "hash-wasm": "^4.9.0",
     "isomorphic-fetch": "^3.0.0",
     "jsdom": "^22.1.0",
     "mockdate": "^3.0.5",

packages/core/src/api/migrations.js

@@ -66,7 +66,11 @@ class Migrations {
           dbCollection: this._db.colors
         },
         {
-          iterate: true,
+          index: () => this._db.trash.raw,
+          dbCollection: this._db.trash
+        },
+        {
+          index: () => this._db.content.all(),
           dbCollection: this._db.content
         },
         {
@@ -87,11 +91,11 @@ class Migrations {
           dbCollection: this._db.relations
         },
         {
-          iterate: true,
+          index: () => this._db.noteHistory.sessionContent.all(),
           dbCollection: this._db.noteHistory
         },
         {
-          iterate: true,
+          index: () => this._db.noteHistory.sessionContent.all(),
           dbCollection: this._db.noteHistory.sessionContent
         },
         {

packages/core/src/api/sync/__tests__/sync.test.skip.js

@@ -0,0 +1,399 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import Database from "../../index";
+import { NodeStorageInterface } from "../../../../__mocks__/node-storage.mock";
+import FS from "../../../../__mocks__/fs.mock";
+import Compressor from "../../../../__mocks__/compressor.mock";
+import { CHECK_IDS, EV, EVENTS } from "../../../common";
+import EventSource from "eventsource";
+import { delay } from "../../../../__tests__/utils";
+
+jest.setTimeout(100 * 1000);
+
+test("case 1: device A & B should only download the changes from device C (no uploading)", async () => {
+  const types = [];
+  function onSyncProgress({ type }) {
+    types.push(type);
+  }
+
+  const deviceA = await initializeDevice("deviceA");
+  const deviceB = await initializeDevice("deviceB");
+
+  deviceA.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
+  deviceB.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
+
+  const deviceC = await initializeDevice("deviceC");
+
+  await deviceC.notes.add({ title: "new note 1" });
+  await syncAndWait(deviceC, deviceC);
+
+  expect(types.every((t) => t === "download")).toBe(true);
+
+  await cleanup(deviceA, deviceB, deviceC);
+});
+
+test("case 3: Device A & B have unsynced changes but server has nothing", async () => {
+  const deviceA = await initializeDevice("deviceA");
+  const deviceB = await initializeDevice("deviceB");
+
+  const note1Id = await deviceA.notes.add({
+    title: "Test note from device A"
+  });
+  const note2Id = await deviceB.notes.add({
+    title: "Test note from device B"
+  });
+
+  await syncAndWait(deviceA, deviceB);
+
+  expect(deviceA.notes.note(note2Id)).toBeTruthy();
+  expect(deviceB.notes.note(note1Id)).toBeTruthy();
+  expect(deviceA.notes.note(note1Id)).toBeTruthy();
+  expect(deviceB.notes.note(note2Id)).toBeTruthy();
+
+  await cleanup(deviceA, deviceA);
+});
+
+// test(
+//   "case 4: Device A's sync is interrupted halfway and Device B makes some changes afterwards and syncs.",
+//   async () => {
+//     const deviceA = await initializeDevice("deviceA");
+//     const deviceB = await initializeDevice("deviceB");
+
+//     const unsyncedNoteIds = [];
+//     for (let i = 0; i < 10; ++i) {
+//       const id = await deviceA.notes.add({
+//         title: `Test note ${i} from device A`,
+//       });
+//       unsyncedNoteIds.push(id);
+//     }
+
+//     const half = unsyncedNoteIds.length / 2 + 1;
+//     deviceA.eventManager.subscribe(
+//       EVENTS.syncProgress,
+//       async ({ type, current }) => {
+//         if (type === "upload" && current === half) {
+//           await deviceA.syncer.stop();
+//         }
+//       }
+//     );
+
+//     await expect(deviceA.sync(true)).rejects.toThrow();
+
+//     let syncedNoteIds = [];
+//     for (let i = 0; i < unsyncedNoteIds.length; ++i) {
+//       const expectedNoteId = unsyncedNoteIds[i];
+//       if (deviceB.notes.note(expectedNoteId))
+//         syncedNoteIds.push(expectedNoteId);
+//     }
+//     expect(
+//       syncedNoteIds.length === half - 1 || syncedNoteIds.length === half
+//     ).toBe(true);
+
+//     const deviceBNoteId = await deviceB.notes.add({
+//       title: "Test note of case 4 from device B",
+//     });
+
+//     await deviceB.sync(true);
+
+//     await syncAndWait(deviceA, deviceB);
+
+//     expect(deviceA.notes.note(deviceBNoteId)).toBeTruthy();
+//     expect(
+//       unsyncedNoteIds
+//         .map((id) => !!deviceB.notes.note(id))
+//         .every((res) => res === true)
+//     ).toBe(true);
+
+//     await cleanup(deviceA, deviceB);
+//   },
+//
+// );
+
+// test.only(
+//   "case 5: Device A's sync is interrupted halfway and Device B makes changes on the same note's content that didn't get synced on Device A due to interruption.",
+//   async () => {
+//     const deviceA = await initializeDevice("deviceA");
+//     const deviceB = await initializeDevice("deviceB");
+
+//     const noteIds = [];
+//     for (let i = 0; i < 10; ++i) {
+//       const id = await deviceA.notes.add({
+//         content: {
+//           type: "tiptap",
+//           data: `<p>deviceA=true</p>`,
+//         },
+//       });
+//       noteIds.push(id);
+//     }
+
+//     await deviceA.sync(true);
+//     await deviceB.sync(true);
+
+//     const unsyncedNoteIds = [];
+//     for (let id of noteIds) {
+//       const noteId = await deviceA.notes.add({
+//         id,
+//         content: {
+//           type: "tiptap",
+//           data: `<p>deviceA=true+changed=true</p>`,
+//         },
+//       });
+//       unsyncedNoteIds.push(noteId);
+//     }
+
+//     deviceA.eventManager.subscribe(
+//       EVENTS.syncProgress,
+//       async ({ type, total, current }) => {
+//         const half = total / 2 + 1;
+//         if (type === "upload" && current === half) {
+//           await deviceA.syncer.stop();
+//         }
+//       }
+//     );
+
+//     await expect(deviceA.sync(true)).rejects.toThrow();
+
+//     await delay(10 * 1000);
+
+//     for (let id of unsyncedNoteIds) {
+//       await deviceB.notes.add({
+//         id,
+//         content: {
+//           type: "tiptap",
+//           data: "<p>changes from device B</p>",
+//         },
+//       });
+//     }
+
+//     const error = await withError(async () => {
+//       await deviceB.sync(true);
+//       await deviceA.sync(true);
+//     });
+
+//     expect(error).not.toBeInstanceOf(NoErrorThrownError);
+//     expect(error.message.includes("Merge")).toBeTruthy();
+
+//     await cleanup(deviceA, deviceB);
+//   },
+//
+// );
+
+test("issue: running force sync from device A makes device B always download everything", async () => {
+  const deviceA = await initializeDevice("deviceA");
+  const deviceB = await initializeDevice("deviceB");
+
+  await syncAndWait(deviceA, deviceB, true);
+
+  const handler = jest.fn();
+  deviceB.eventManager.subscribe(EVENTS.syncProgress, handler);
+
+  await deviceB.sync(true);
+
+  expect(handler).not.toHaveBeenCalled();
+
+  await cleanup(deviceB);
+});
+
+test("issue: colors are not properly created if multiple notes are synced together", async () => {
+  const deviceA = await initializeDevice("deviceA", [CHECK_IDS.noteColor]);
+  const deviceB = await initializeDevice("deviceB", [CHECK_IDS.noteColor]);
+
+  const noteIds = [];
+  for (let i = 0; i < 3; ++i) {
+    const id = await deviceA.notes.add({
+      content: {
+        type: "tiptap",
+        data: `<p>deviceA=true</p>`
+      }
+    });
+    noteIds.push(id);
+  }
+
+  await syncAndWait(deviceA, deviceB);
+
+  for (let noteId of noteIds) {
+    await deviceA.notes.note(noteId).color("purple");
+    expect(deviceB.notes.note(noteId)).toBeTruthy();
+    expect(deviceB.notes.note(noteId).data.color).toBeUndefined();
+  }
+
+  await syncAndWait(deviceA, deviceB);
+
+  await delay(2000);
+
+  const purpleColor = deviceB.colors.tag("purple");
+  expect(noteIds.every((id) => purpleColor.noteIds.indexOf(id) > -1)).toBe(
+    true
+  );
+
+  await cleanup(deviceA, deviceB);
+});
+
+test("issue: new topic on device A gets replaced by the new topic on device B", async () => {
+  const deviceA = await initializeDevice("deviceA");
+  const deviceB = await initializeDevice("deviceB");
+
+  const id = await deviceA.notebooks.add({ title: "Notebook 1" });
+
+  await syncAndWait(deviceA, deviceB, false);
+
+  expect(deviceB.notebooks.notebook(id)).toBeDefined();
+
+  await deviceA.notebooks.notebook(id).topics.add("Topic 1");
+
+  // to create a conflict
+  await delay(1500);
+
+  await deviceB.notebooks.notebook(id).topics.add("Topic 2");
+
+  expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
+
+  expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
+
+  await syncAndWait(deviceA, deviceB, false);
+
+  await delay(1000);
+
+  await syncAndWait(deviceB, deviceB, false);
+
+  expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
+  expect(deviceB.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
+
+  expect(deviceA.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
+  expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
+
+  await cleanup(deviceA, deviceB);
+});
+
+test("issue: remove notebook reference from notes that are removed from topic during merge", async () => {
+  const deviceA = await initializeDevice("deviceA");
+  const deviceB = await initializeDevice("deviceB");
+
+  const id = await deviceA.notebooks.add({
+    title: "Notebook 1",
+    topics: ["Topic 1"]
+  });
+
+  await syncAndWait(deviceA, deviceB, false);
+
+  expect(deviceB.notebooks.notebook(id)).toBeDefined();
+
+  const noteA = await deviceA.notes.add({ title: "Note 1" });
+  await deviceA.notes.addToNotebook({ id, topic: "Topic 1" }, noteA);
+
+  expect(
+    deviceA.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
+  ).toBe(1);
+
+  await delay(2000);
+
+  const noteB = await deviceB.notes.add({ title: "Note 2" });
+  await deviceB.notes.addToNotebook({ id, topic: "Topic 1" }, noteB);
+
+  expect(
+    deviceB.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
+  ).toBe(1);
+
+  await syncAndWait(deviceB, deviceA, false);
+
+  expect(
+    deviceA.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
+  ).toBe(1);
+  expect(
+    deviceB.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
+  ).toBe(1);
+
+  expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(0);
+
+  await cleanup(deviceA, deviceB);
+});
+
+/**
+ *
+ * @param {string} id
+ * @returns {Promise<Database>}
+ */
+async function initializeDevice(id, capabilities = []) {
+  console.time("Init device");
+  EV.subscribe(EVENTS.userCheckStatus, async (type) => {
+    return {
+      type,
+      result: capabilities.indexOf(type) > -1
+    };
+  });
+
+  const device = new Database(
+    new NodeStorageInterface(),
+    EventSource,
+    FS,
+    Compressor
+  );
+  // device.host({
+  //   API_HOST: "http://192.168.10.29:5264",
+  //   AUTH_HOST: "http://192.168.10.29:8264",
+  //   SSE_HOST: "http://192.168.10.29:7264",
+  //   ISSUES_HOST: "http://192.168.10.29:2624",
+  //   SUBSCRIPTIONS_HOST: "http://192.168.10.29:9264",
+  // });
+
+  await device.init(id);
+
+  await device.user.login(
+    process.env.EMAIL,
+    process.env.PASSWORD,
+    process.env.HASHED_PASSWORD
+  );
+
+  await device.user.resetUser(false);
+
+  device.eventManager.subscribe(
+    EVENTS.databaseSyncRequested,
+    async (full, force) => {
+      await device.sync(full, force);
+    }
+  );
+
+  console.timeEnd("Init device");
+  return device;
+}
+
+async function cleanup(...devices) {
+  for (let device of devices) {
+    await device.user.logout();
+    device.eventManager.unsubscribeAll();
+  }
+  EV.unsubscribeAll();
+}
+
+/**
+ *
+ * @param {Database} deviceA
+ * @param {Database} deviceB
+ * @returns
+ */
+function syncAndWait(deviceA, deviceB, force = false) {
+  return new Promise((resolve) => {
+    const ref = deviceB.eventManager.subscribe(EVENTS.syncCompleted, () => {
+      ref.unsubscribe();
+      resolve();
+    });
+    deviceA.sync(true, force);
+  });
+}

packages/core/src/api/sync/index.js

@@ -315,16 +315,16 @@ class Sync {
   }
 
   async stop(lastSynced) {
-    // refresh topic references
-    this.db.notes.topicReferences.rebuild();
-    // refresh monographs on sync completed
-    await this.db.monographs.init();
-
     this.logger.info("Stopping sync", { lastSynced });
     const storedLastSynced = await this.db.lastSynced();
     if (lastSynced > storedLastSynced)
       await this.db.storage.write("lastSynced", lastSynced);
     this.db.eventManager.publish(EVENTS.syncCompleted);
+
+    // refresh monographs on sync completed
+    await this.db.monographs.init();
+    // refresh topic references
+    this.db.notes.topicReferences.rebuild();
   }
 
   async cancel() {
@@ -361,19 +361,13 @@ class Sync {
   /**
    * @private
    */
-  async onPushCompleted(lastSynced) {
-    // refresh topic references
-    this.db.notes.topicReferences.rebuild();
-
+  onPushCompleted(lastSynced) {
     this.db.eventManager.publish(
       EVENTS.databaseSyncRequested,
       false,
       false,
       lastSynced
     );
-
-    // refresh monographs on sync completed
-    await this.db.monographs.init();
   }
 
   async processChunk(chunk, key, dbLastSynced, notify = false) {
@@ -418,9 +412,8 @@ class Sync {
     }
 
     const collectionType = this.itemTypeToCollection[chunk.type];
-    if (collectionType && this.db[collectionType]) {
+    if (collectionType && this.db[collectionType])
       await this.db[collectionType]._collection.setItems(items);
-    }
   }
 
   /**
@@ -476,13 +469,7 @@ async function deserializeItem(decryptedItem, version, database) {
   deserialized.synced = true;
 
   if (!deserialized.alg && !deserialized.cipher) {
-    await migrateItem(
-      deserialized,
-      version,
-      deserialized.type,
-      database,
-      "sync"
-    );
+    await migrateItem(deserialized, version, deserialized.type, database);
   }
   return deserialized;
 }

packages/core/src/api/sync/merger.js

@@ -182,7 +182,6 @@ class Merger {
             remoteItem.noteIds,
             noteIds
           );
-          remoteItem.remote = false;
         }
         return this._db.attachments.merge(localItem, remoteItem);
       }

packages/core/src/api/user-manager.js

@@ -152,7 +152,6 @@ class UserManager {
       password,
       salt: user.salt
     });
-    await this._storage.write("lastSynced", 0);
 
     EV.publish(EVENTS.userLoggedIn, user);
   }
@@ -184,7 +183,6 @@ class UserManager {
       password,
       salt: user.salt
     });
-    await this._storage.write("lastSynced", 0);
 
     EV.publish(EVENTS.userLoggedIn, user);
   }

packages/core/src/collections/attachments.js

@@ -50,7 +50,6 @@ export default class Attachments extends Collection {
         remoteAttachment.noteIds,
         localAttachment.noteIds
       );
-      remoteAttachment.remote = false;
     }
 
     return remoteAttachment;

packages/core/src/collections/notebooks.js

@@ -26,15 +26,6 @@ import qclone from "qclone";
 export default class Notebooks extends Collection {
   merge(localNotebook, remoteNotebook, lastSyncedTimestamp) {
     if (remoteNotebook.deleted) return remoteNotebook;
-
-    if (
-      localNotebook &&
-      (localNotebook.type === "trash" || localNotebook.deleted)
-    ) {
-      if (localNotebook.dateModified > remoteNotebook.dateModified) return;
-      return remoteNotebook;
-    }
-
     if (localNotebook && localNotebook.topics?.length) {
       let isChanged = false;
       // merge new and old topics
@@ -91,8 +82,7 @@ export default class Notebooks extends Collection {
 
     let notebook = {
       ...oldNotebook,
-      ...notebookArg,
-      topics: oldNotebook?.topics || []
+      ...notebookArg
     };
 
     if (!notebook.title) throw new Error("Notebook must contain a title.");
@@ -112,8 +102,8 @@ export default class Notebooks extends Collection {
 
     await this._collection.addItem(notebook);
 
-    if (!oldNotebook && notebookArg.topics) {
-      await this.notebook(id).topics.add(...notebookArg.topics);
+    if (!oldNotebook) {
+      await this.notebook(notebook).topics.add(...notebook.topics);
     }
     return id;
   }

packages/core/src/collections/topics.js

@@ -86,7 +86,7 @@ export default class Topics {
         notebook.topics.push(topic);
       }
     }
-    return this._db.notebooks._collection.updateItem(notebook);
+    return this._db.notebooks.add(notebook);
   }
 
   /**
@@ -110,8 +110,7 @@ export default class Topics {
   }
 
   async delete(...topicIds) {
-    const notebook = qclone(this._db.notebooks.notebook(this._notebookId).data);
-    let allTopics = notebook.topics;
+    let allTopics = qclone(this.all);
 
     for (let topicId of topicIds) {
       const topic = this.topic(topicId);
@@ -126,7 +125,7 @@ export default class Topics {
       allTopics.splice(topicIndex, 1);
     }
 
-    await this._db.notebooks._collection.updateItem(notebook);
+    await this._db.notebooks.add({ id: this._notebookId, topics: allTopics });
   }
 }
 

packages/core/src/common.js

@@ -125,4 +125,4 @@ export const DATE_FORMATS = [
 
 export const TIME_FORMATS = ["12-hour", "24-hour"];
 
-export const CURRENT_DATABASE_VERSION = 5.9;
+export const CURRENT_DATABASE_VERSION = 5.8;

packages/core/src/database/backup.js

@@ -20,63 +20,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import SparkMD5 from "spark-md5";
 import { CURRENT_DATABASE_VERSION } from "../common.js";
 import Migrator from "./migrator.js";
-import { toChunks } from "../utils/array.js";
-import { migrateItem } from "../migrations.js";
-import Indexer from "./indexer.js";
-
-const COLORS = [
-  "red",
-  "orange",
-  "yellow",
-  "green",
-  "blue",
-  "purple",
-  "gray",
-  "black",
-  "white"
-];
-
-const invalidKeys = [
-  "user",
-  "t",
-  "v",
-  "lastBackupTime",
-  "lastSynced",
-  // all indexes
-  "notes",
-  "notebooks",
-  "content",
-  "tags",
-  "colors",
-  "attachments",
-  "relations",
-  "reminders",
-  "sessioncontent",
-  "notehistory",
-  "shortcuts",
-  "vaultKey",
-  "hasConflict",
-  "token",
-  "monographs"
-];
-
-const itemTypeToCollectionKey = {
-  note: "notes",
-  notebook: "notebooks",
-  tiptap: "content",
-  tiny: "content",
-  tag: "tags",
-  color: "colors",
-  attachment: "attachments",
-  relation: "relations",
-  reminder: "reminders",
-  sessioncontent: "sessioncontent",
-  session: "notehistory",
-  notehistory: "notehistory",
-  content: "content",
-  shortcut: "shortcuts"
-};
 
+const invalidKeys = ["user", "t", "v", "lastBackupTime", "lastSynced"];
+const invalidIndices = ["tags", "colors"];
 const validTypes = ["mobile", "web", "node"];
 export default class Backup {
   /**
@@ -100,75 +46,35 @@ export default class Backup {
    * @param {"web"|"mobile"|"node"} type
    * @param {boolean} encrypt
    */
-  async *export(type, encrypt = false) {
+  async export(type, encrypt = false) {
     if (!validTypes.some((t) => t === type))
       throw new Error("Invalid type. It must be one of 'mobile' or 'web'.");
     if (encrypt && !(await this._db.user.getUser()))
       throw new Error("Please login to create encrypted backups.");
 
-    yield {
-      path: ".nnbackup",
-      data: ""
-    };
-
     let keys = await this._db.storage.getAllKeys();
-    const key = await this._db.user.getEncryptionKey();
-    const chunks = toChunks(keys, 20);
-    let buffer = [];
-    let bufferLength = 0;
-    const MAX_CHUNK_SIZE = 10 * 1024 * 1024;
-    let chunkIndex = 0;
+    let data = filterData(
+      Object.fromEntries(await this._db.storage.readMulti(keys))
+    );
 
-    while (chunks.length > 0) {
-      const chunk = chunks.pop();
+    let hash = {};
 
-      const items = await this._db.storage.readMulti(chunk);
-      items.forEach(([id, item]) => {
-        if (
-          !item ||
-          invalidKeys.includes(id) ||
-          (item.deleted && !item.type) ||
-          id.startsWith("_uk_")
-        )
-          return;
-
-        const data = JSON.stringify(item);
-        buffer.push(data);
-        bufferLength += data.length;
-      });
-
-      if (bufferLength >= MAX_CHUNK_SIZE || chunks.length === 0) {
-        let itemsJSON = `[${buffer.join(",")}]`;
-
-        buffer = [];
-        bufferLength = 0;
-
-        itemsJSON = await this._db.compressor.compress(itemsJSON);
-
-        const hash = SparkMD5.hash(itemsJSON);
-
-        if (encrypt) itemsJSON = await this._db.storage.encrypt(key, itemsJSON);
-
-        yield {
-          path: `${chunkIndex++}-${encrypt ? "encrypted" : "plain"}-${hash}`,
-          data: `{
-"version": ${CURRENT_DATABASE_VERSION},
-"type": "${type}",
-"date": ${Date.now()},
-"data": ${JSON.stringify(itemsJSON)},
-"hash": "${hash}",
-"hash_type": "md5",
-"compressed": true,
-"encrypted": ${encrypt ? "true" : "false"}
-}`
-        };
-      }
+    if (encrypt) {
+      const key = await this._db.user.getEncryptionKey();
+      data = await this._db.storage.encrypt(key, JSON.stringify(data));
+    } else {
+      hash = { hash: SparkMD5.hash(JSON.stringify(data)), hash_type: "md5" };
     }
 
-    if (bufferLength > 0 || buffer.length > 0)
-      throw new Error("Buffer not empty.");
-
+    // save backup time
     await this.updateBackupTime();
+    return JSON.stringify({
+      version: CURRENT_DATABASE_VERSION,
+      type,
+      date: Date.now(),
+      data,
+      ...hash
+    });
   }
 
   /**
@@ -184,7 +90,7 @@ export default class Backup {
 
     let db = backup.data;
     const isEncrypted = db.salt && db.iv && db.cipher;
-    if (backup.encrypted || isEncrypted) {
+    if (isEncrypted) {
       if (!password)
         throw new Error(
           "Please provide a password to decrypt this backup & restore it."
@@ -195,7 +101,8 @@ export default class Backup {
         throw new Error("Could not generate encryption key for backup.");
 
       try {
-        backup.data = await this._db.storage.decrypt(key, db);
+        const decrypted = await this._db.storage.decrypt(key, db);
+        backup.data = JSON.parse(decrypted);
       } catch (e) {
         if (
           e.message.includes("ciphertext cannot be decrypted") ||
@@ -205,16 +112,9 @@ export default class Backup {
 
         throw new Error(`Could not decrypt backup: ${e.message}`);
       }
-    }
-
-    if (backup.hash && !this._verify(backup))
+    } else if (!this._verify(backup))
       throw new Error("Backup file has been tempered, aborting...");
 
-    if (backup.compressed)
-      backup.data = await this._db.compressor.decompress(backup.data);
-    backup.data =
-      typeof backup.data === "string" ? JSON.parse(backup.data) : backup.data;
-
     await this._migrateData(backup);
   }
 
@@ -222,12 +122,11 @@ export default class Backup {
     const { version = 0 } = backup;
     if (version > CURRENT_DATABASE_VERSION)
       throw new Error(
-        "This backup was made from a newer version of Notesnook. Cannot restore."
+        "This backup was made from a newer version of Notesnook. Cannot migrate."
       );
 
     switch (version) {
       case CURRENT_DATABASE_VERSION:
-      case 5.8:
       case 5.7:
       case 5.6:
       case 5.5:
@@ -246,38 +145,66 @@ export default class Backup {
   async _migrateData(backup) {
     const { data, version = 0 } = backup;
 
-    const toAdd = {};
-    for (const item of Array.isArray(data) ? data : Object.values(data)) {
-      // we do not want to restore deleted items
-      if (!item || (!item.type && item.deleted)) continue;
-      // in v5.6 of the database, we did not set note history session's type
-      if (!item.type && item.sessionContentId) item.type = "notehistory";
+    if (version > CURRENT_DATABASE_VERSION)
+      throw new Error(
+        "This backup was made from a newer version of Notesnook. Cannot migrate."
+      );
 
-      await migrateItem(item, version, item.type, this._db, "backup");
-      // since items in trash can have their own set of migrations,
-      // we have to run the migration again to account for that.
-      if (item.type === "trash" && item.itemType)
-        await migrateItem(item, version, item.itemType, this._db, "backup");
+    const collections = [
+      {
+        index: () => data["attachments"],
+        dbCollection: this._db.attachments
+      },
+      {
+        index: () => data["notebooks"],
+        dbCollection: this._db.notebooks
+      },
+      {
+        index: () => data["content"],
+        dbCollection: this._db.content
+      },
+      {
+        index: () => data["shortcuts"],
+        dbCollection: this._db.shortcuts
+      },
+      {
+        index: () => data["reminders"],
+        dbCollection: this._db.reminders
+      },
+      {
+        index: () => data["relations"],
+        dbCollection: this._db.relations
+      },
+      {
+        index: () => data["notehistory"],
+        dbCollection: this._db.noteHistory,
+        type: "notehistory"
+      },
+      {
+        index: () => data["sessioncontent"],
+        dbCollection: this._db.noteHistory.sessionContent,
+        type: "sessioncontent"
+      },
+      {
+        index: () => data["notes"],
+        dbCollection: this._db.notes
+      },
+      {
+        index: () => ["settings"],
+        dbCollection: this._db.settings,
+        type: "settings"
+      }
+    ];
 
-      // colors are naively of type "tag" instead of "color" so we have to fix that.
-      const itemType =
-        item.type === "tag" && COLORS.includes(item.title.toLowerCase())
-          ? "color"
-          : item.itemType || item.type;
-
-      const collectionKey = itemTypeToCollectionKey[itemType];
-      if (collectionKey) {
-        toAdd[collectionKey] = toAdd[collectionKey] || [];
-        toAdd[collectionKey].push([item.id, item]);
-      } else if (item.type === "settings")
-        await this._db.storage.write("settings", item);
-    }
-
-    for (const collectionKey in toAdd) {
-      const indexer = new Indexer(this._db.storage, collectionKey);
-      await indexer.init();
-      await indexer.writeMulti(toAdd[collectionKey]);
-    }
+    await this._db.syncer.acquireLock(async () => {
+      await this._migrator.migrate(
+        this._db,
+        collections,
+        (id, type) => (version < 5.8 ? data[id] : data[`${id}_${type}`]),
+        version,
+        true
+      );
+    });
   }
 
   _validate(backup) {
@@ -290,10 +217,10 @@ export default class Backup {
   }
 
   _verify(backup) {
-    const { compressed, hash, hash_type, data: db } = backup;
+    const { hash, hash_type, data: db } = backup;
     switch (hash_type) {
       case "md5": {
-        return hash === SparkMD5.hash(compressed ? db : JSON.stringify(db));
+        return hash === SparkMD5.hash(JSON.stringify(db));
       }
       default: {
         return false;
@@ -301,3 +228,15 @@ export default class Backup {
     }
   }
 }
+
+function filterData(data) {
+  let skippedKeys = [...invalidKeys, ...invalidIndices];
+  invalidIndices.forEach((key) => {
+    const index = data[key];
+    if (!index) return;
+    skippedKeys.push(...index);
+  });
+
+  skippedKeys.forEach((key) => delete data[key]);
+  return data;
+}

packages/core/src/database/indexed-collection.js

@@ -109,19 +109,7 @@ export default class IndexedCollection {
   }
 
   setItems(items) {
-    const entries = items.reduce((array, item) => {
-      if (!item) return array;
-
-      if (!item.remote) {
-        item.dateModified = Date.now();
-        item.synced = false;
-      }
-      delete item.remote;
-
-      array.push([item.id, item]);
-      return array;
-    }, []);
-    return this.indexer.writeMulti(entries);
+    return this.indexer.writeMulti(items);
   }
 
   async getEncryptionKey() {

packages/core/src/database/indexer.js

@@ -56,6 +56,7 @@ export default class Indexer extends Storage {
   }
 
   read(key, isArray = false) {
+    if (!this.exists(key)) return;
     return super.read(this.makeId(key), isArray);
   }
 
@@ -68,7 +69,9 @@ export default class Indexer extends Storage {
   }
 
   async readMulti(keys) {
-    const entries = await super.readMulti(keys.map(this.makeId, this));
+    const entries = await super.readMulti(
+      keys.filter(this.exists, this).map(this.makeId, this)
+    );
     entries.forEach((entry) => {
       entry[0] = entry[0].replace(`_${this.type}`, "");
     });
@@ -81,12 +84,14 @@ export default class Indexer extends Storage {
    * @returns
    */
   async writeMulti(items) {
-    const entries = items.map(([id, item]) => {
-      if (!this.indices.includes(id)) this.indices.push(id);
-      return [this.makeId(id), item];
-    });
-    entries.push([this.type, this.indices]);
+    const entries = items.reduce((array, item) => {
+      if (!item) return array;
+      if (!this.indices.includes(item.id)) this.indices.push(item.id);
+      array.push([this.makeId(item.id), item]);
+      return array;
+    }, []);
     await super.writeMulti(entries);
+    await super.write(this.type, this.indices);
   }
 
   async migrateIndices() {

packages/core/src/database/migrator.js

@@ -21,13 +21,9 @@ import { sendMigrationProgressEvent } from "../common";
 import { migrateCollection, migrateItem } from "../migrations";
 
 class Migrator {
-  async migrate(db, collections, get, version) {
+  async migrate(db, collections, get, version, restore = false) {
     for (let collection of collections) {
-      if (
-        (!collection.iterate && !collection.index) ||
-        !collection.dbCollection
-      )
-        continue;
+      if (!collection.index || !collection.dbCollection) continue;
 
       if (collection.dbCollection.collectionName)
         sendMigrationProgressEvent(
@@ -39,85 +35,61 @@ class Migrator {
 
       await migrateCollection(collection.dbCollection, version);
 
-      if (collection.index) {
-        await this.migrateItems(
-          db,
-          collection,
-          collection.index(),
-          get,
-          version
-        );
-      } else if (collection.iterate) {
-        for await (const index of collection.dbCollection._collection.iterate(
-          100
-        )) {
-          await this.migrateItems(
-            db,
-            collection,
-            index.map((item) => item[1]),
-            get,
-            version
-          );
+      const index = (await collection.index()) || [];
+      const toAdd = [];
+      for (var i = 0; i < index.length; ++i) {
+        let id = index[i];
+        let item = get(id, collection.dbCollection.collectionName);
+        if (!item) {
+          continue;
         }
+
+        // check if item is permanently deleted or just a soft delete
+        if (item.deleted && !item.type) {
+          await collection.dbCollection?._collection?.addItem(item);
+          continue;
+        }
+
+        const itemId = item.id;
+        const migrated = await migrateItem(
+          item,
+          version,
+          item.type || collection.type || collection.dbCollection.type,
+          db
+        );
+
+        if (migrated || restore) {
+          if (collection.type === "settings") {
+            await collection.dbCollection.merge(item);
+          } else if (item.type === "note") {
+            toAdd.push(await db.notes.merge(null, item));
+          } else if (collection.dbCollection._collection) {
+            toAdd.push(item);
+          } else {
+            throw new Error(
+              `No idea how to handle this kind of item: ${item.type}.`
+            );
+          }
+
+          // if id changed after migration, we need to delete the old one.
+          if (item.id !== itemId) {
+            await collection.dbCollection._collection.deleteItem(itemId);
+          }
+        }
+      }
+
+      if (toAdd.length > 0) {
+        await collection.dbCollection._collection.setItems(toAdd);
+        if (collection.dbCollection.collectionName)
+          sendMigrationProgressEvent(
+            db.eventManager,
+            collection.dbCollection.collectionName,
+            toAdd.length,
+            toAdd.length
+          );
       }
     }
     return true;
   }
-
-  async migrateItems(db, collection, index, get, version) {
-    const toAdd = [];
-    for (var i = 0; i < index.length; ++i) {
-      let id = index[i];
-      let item = get(id, collection.dbCollection.collectionName);
-      if (!item) {
-        continue;
-      }
-
-      // check if item is permanently deleted or just a soft delete
-      if (item.deleted && !item.type) {
-        await collection.dbCollection?._collection?.addItem(item);
-        continue;
-      }
-
-      const itemId = item.id;
-      const migrated = await migrateItem(
-        item,
-        version,
-        item.type || collection.type || collection.dbCollection.type,
-        db,
-        "local"
-      );
-
-      if (migrated) {
-        if (collection.type === "settings") {
-          await collection.dbCollection.merge(item);
-        } else if (item.type === "note") {
-          toAdd.push(await db.notes.merge(null, item));
-        } else if (collection.dbCollection._collection) {
-          toAdd.push(item);
-        } else {
-          throw new Error(
-            `No idea how to handle this kind of item: ${item.type}.`
-          );
-        }
-
-        // if id changed after migration, we need to delete the old one.
-        if (item.id !== itemId) {
-          await collection.dbCollection._collection.deleteItem(itemId);
-        }
-      }
-    }
-
-    if (toAdd.length > 0) {
-      await collection.dbCollection._collection.setItems(toAdd);
-      if (collection.dbCollection.collectionName)
-        sendMigrationProgressEvent(
-          db.eventManager,
-          collection.dbCollection.collectionName,
-          toAdd.length,
-          toAdd.length
-        );
-    }
-  }
 }
 export default Migrator;

packages/core/src/migrations.js

@@ -133,27 +133,10 @@ const migrations = [
       }
     }
   },
-  {
-    version: 5.8,
-    items: {
-      all: (item, _db, migrationType) => {
-        if (migrationType === "local") {
-          delete item.remote;
-          return true;
-        }
-      }
-    }
-  },
-  { version: 5.9, items: {} }
+  { version: 5.8, items: {} }
 ];
 
-export async function migrateItem(
-  item,
-  version,
-  type,
-  database,
-  migrationType
-) {
+export async function migrateItem(item, version, type, database) {
   let migrationStartIndex = migrations.findIndex((m) => m.version === version);
   if (migrationStartIndex <= -1) {
     throw new Error(
@@ -168,15 +151,9 @@ export async function migrateItem(
     const migration = migrations[migrationStartIndex];
     if (migration.version === CURRENT_DATABASE_VERSION) break;
 
-    if (
-      migration.items.all &&
-      (await migration.items.all(item, database, migrationType))
-    )
-      count++;
-
-    const itemMigrator = migration.items[type];
+    const itemMigrator = migration.items && migration.items[type];
     if (!itemMigrator) continue;
-    if (await itemMigrator(item, database, migrationType)) count++;
+    if (await itemMigrator(item, database)) count++;
   }
 
   return count > 0;