mobile: bump version code

.github/workflows/desktop.publish.yml

@@ -97,6 +97,7 @@ jobs:
       - name: Get App Store Version
         id: appstore
         uses: streetwriters/appstore-connect-app-version@develop
+        if: inputs.publish-apple
         with:
           app-id: ${{ steps.app_metadata.outputs.apple_app_id }}
           key-id: ${{ secrets.api_key_id }}

apps/desktop/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@notesnook/desktop",
-  "version": "2.6.3",
+  "version": "2.6.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@notesnook/desktop",
-      "version": "2.6.3",
+      "version": "2.6.4",
       "hasInstallScript": true,
       "license": "GPL-3.0-or-later",
       "dependencies": {

apps/desktop/package.json

@@ -2,7 +2,7 @@
   "name": "@notesnook/desktop",
   "productName": "Notesnook",
   "description": "Your private note taking space",
-  "version": "2.6.3",
+  "version": "2.6.4",
   "appAppleId": "1544027013",
   "private": true,
   "main": "./dist/index.js",

apps/desktop/src/api/os-integration.ts

@@ -117,6 +117,14 @@ export const osIntegrationRouter = t.router({
       writeFileSync(resolvedPath, data);
     }),
 
+  resolvePath: t.procedure
+    .input(z.object({ filePath: z.string() }))
+    .query(({ input }) => {
+      const { filePath } = input;
+      if (!filePath) return;
+      return resolvePath(filePath);
+    }),
+
   showNotification: t.procedure
     .input(NotificationOptions)
     .query(({ input }) => {

apps/desktop/src/preload.ts

@@ -31,7 +31,6 @@ declare global {
 }
 
 process.once("loaded", async () => {
-  console.log("HELLO!");
   const electronTRPC: RendererGlobalElectronTRPC = {
     sendMessage: (operation) =>
       ipcRenderer.send(ELECTRON_TRPC_CHANNEL, operation),

apps/mobile/app/common/filesystem/utils.js

@@ -66,9 +66,12 @@ export function cancelable(operation) {
 }
 
 export function copyFileAsync(source, dest) {
-  return new Promise((resolve) => {
+  return new Promise((resolve, reject) => {
     ScopedStorage.copyFile(source, dest, (e, r) => {
-      console.log(e, r);
+      if (e) {
+        reject(e);
+        return;
+      }
       resolve();
     });
   });

apps/mobile/app/components/selection-header/index.js

@@ -317,23 +317,8 @@ export const SelectionHeader = React.memo(() => {
             customStyle={{
               marginLeft: 10
             }}
-            onPress={async () => {
-              presentDialog({
-                title: `Delete ${
-                  selectedItemsList.length > 1 ? "items" : "item"
-                }`,
-                paragraph: `Are you sure you want to delete ${
-                  selectedItemsList.length > 1 ? "these items?" : "this item?"
-                }`,
-                positiveText: "Delete",
-                negativeText: "Cancel",
-                positivePress: () => {
-                  deleteItems();
-                },
-                positiveType: "errorShade"
-              });
-
-              return;
+            onPress={() => {
+              deleteItems();
             }}
             tooltipText="Move to trash"
             tooltipPosition={1}

apps/mobile/app/components/sheet-provider/index.js

@@ -198,7 +198,6 @@ const SheetProvider = ({ context = "global" }) => {
             key={data.actionText}
             title={data.actionText}
             accentColor={data.iconColor}
-            accentText="light"
             type="accent"
             height={40}
             width={250}

apps/mobile/app/components/sheets/restore-data/index.js

@@ -22,7 +22,9 @@ import React, { useCallback, useEffect, useRef, useState } from "react";
 import { ActivityIndicator, Platform, View } from "react-native";
 import { FlatList } from "react-native-actions-sheet";
 import RNFetchBlob from "react-native-blob-util";
-import DocumentPicker from "react-native-document-picker";
+import DocumentPicker, {
+  DocumentPickerResponse
+} from "react-native-document-picker";
 import * as ScopedStorage from "react-native-scoped-storage";
 import { db } from "../../../common/database";
 import storage from "../../../common/database/storage";
@@ -45,6 +47,8 @@ import Seperator from "../../ui/seperator";
 import SheetWrapper from "../../ui/sheet";
 import Paragraph from "../../ui/typography/paragraph";
 import { getFormattedDate } from "@notesnook/common";
+import { unzip } from "react-native-zip-archive";
+import { cacheDir, copyFileAsync } from "../../../common/filesystem/utils";
 
 const RestoreDataSheet = () => {
   const [visible, setVisible] = useState(false);
@@ -129,75 +133,71 @@ const RestoreDataComponent = ({ close, setRestoring, restoring }) => {
   useEffect(() => {
     setTimeout(() => {
       checkBackups();
-    }, 300);
+    }, 1000);
   }, []);
 
   const restore = async (item) => {
     if (restoring) {
       return;
     }
-    try {
-      setRestoring(true);
-      let prefix = Platform.OS === "ios" ? "" : "file:/";
-      let backup;
-      if (Platform.OS === "android") {
-        backup = await ScopedStorage.readFile(item.uri, "utf8");
-      } else {
-        backup = await RNFetchBlob.fs.readFile(prefix + item.path, "utf8");
-      }
-      backup = JSON.parse(backup);
 
-      if (backup.data.iv && backup.data.salt) {
-        withPassword(
-          async (value) => {
-            try {
-              await restoreBackup(backup, value);
-              close();
-              setRestoring(false);
-              return true;
-            } catch (e) {
-              backupError(e);
-              return false;
-            }
-          },
-          () => {
-            setRestoring(false);
+    try {
+      const file = Platform.OS === "ios" ? item.path : item.uri;
+      console.log(file);
+      if (file.endsWith(".nnbackupz")) {
+        setRestoring(true);
+
+        if (Platform.OS === "android") {
+          const cacheFile = `file://${RNFetchBlob.fs.dirs.CacheDir}/backup.zip`;
+          if (await RNFetchBlob.fs.exists(cacheFile)) {
+            await RNFetchBlob.fs.unlink(cacheFile);
           }
-        );
-      } else {
-        await restoreBackup(backup);
-        close();
+          await RNFetchBlob.fs.createFile(cacheFile, "", "utf8");
+          console.log("copying");
+          await copyFileAsync(file, cacheFile);
+          console.log("copied");
+          await restoreFromZip(cacheFile);
+        } else {
+          await restoreFromZip(file, false);
+        }
+      } else if (file.endsWith(".nnbackup")) {
+        let backup;
+        if (Platform.OS === "android") {
+          backup = await ScopedStorage.readFile(file, "utf8");
+        } else {
+          backup = await RNFetchBlob.fs.readFile(file, "utf8");
+        }
+        await restoreFromNNBackup(JSON.parse(backup));
       }
     } catch (e) {
+      console.log("error", e);
       setRestoring(false);
       backupError(e);
     }
   };
 
-  const withPassword = (onsubmit, onclose = () => {}) => {
-    presentDialog({
-      context: "local",
-      title: "Encrypted backup",
-      input: true,
-      inputPlaceholder: "Password",
-      paragraph: "Please enter password of this backup file to restore it",
-      positiveText: "Restore",
-      secureTextEntry: true,
-      onClose: onclose,
-      negativeText: "Cancel",
-      positivePress: async (password) => {
-        try {
-          return await onsubmit(password);
-        } catch (e) {
-          ToastEvent.show({
-            heading: "Failed to backup data",
-            message: e.message,
-            type: "error",
-            context: "global"
-          });
-          return false;
+  const withPassword = () => {
+    return new Promise((resolve) => {
+      let resolved = false;
+      presentDialog({
+        context: "local",
+        title: "Encrypted backup",
+        input: true,
+        inputPlaceholder: "Password",
+        paragraph: "Please enter password of this backup file to restore it",
+        positiveText: "Restore",
+        secureTextEntry: true,
+        onClose: () => {
+          if (resolved) return;
+          resolve(undefined);
+        },
+        negativeText: "Cancel",
+        positivePress: async (password) => {
+          resolve(password);
+          resolved = true;
+          return true;
         }
-      }
+      });
     });
   };
 
@@ -217,21 +217,164 @@ const RestoreDataComponent = ({ close, setRestoring, restoring }) => {
         let path = await storage.checkAndCreateDir("/backups/");
         files = await RNFetchBlob.fs.lstat(path);
       }
-      files = files.sort(function (a, b) {
-        let timeA = a.lastModified;
-        let timeB = b.lastModified;
-        return timeB - timeA;
-      });
+      files = files
+        .filter((file) => {
+          const name = Platform.OS === "android" ? file.name : file.filename;
+          return name.endsWith(".nnbackup") || name.endsWith(".nnbackupz");
+        })
+        .sort(function (a, b) {
+          let timeA = a.lastModified;
+          let timeB = b.lastModified;
+          return timeB - timeA;
+        });
       setFiles(files);
-      setTimeout(() => {
-        setLoading(false);
-      }, 1000);
+      setLoading(false);
     } catch (e) {
       console.log(e);
       setLoading(false);
     }
   };
 
+  const restoreBackup = async (backup, password) => {
+    await db.backup.import(backup, password);
+    await db.initCollections();
+    initialize();
+    ToastEvent.show({
+      heading: "Backup restored successfully.",
+      type: "success",
+      context: "global"
+    });
+    return true;
+  };
+
+  const backupError = (e) => {
+    ToastEvent.show({
+      heading: "Restore failed",
+      message:
+        e.message ||
+        "The selected backup data file is invalid. You must select a *.nnbackup file to restore.",
+      type: "error",
+      context: "local"
+    });
+  };
+
+  /**
+   *
+   * @param {string} file
+   */
+  async function restoreFromZip(file, remove) {
+    try {
+      const zipOutputFolder = `${cacheDir}/backup_extracted`;
+      if (await RNFetchBlob.fs.exists(zipOutputFolder)) {
+        await RNFetchBlob.fs.unlink(zipOutputFolder);
+        await RNFetchBlob.fs.mkdir(zipOutputFolder);
+      }
+      await unzip(file, zipOutputFolder);
+      console.log("Unzipped files successfully to", zipOutputFolder);
+
+      const backupFiles = await RNFetchBlob.fs.ls(zipOutputFolder);
+
+      if (backupFiles.findIndex((file) => file === ".nnbackup") === -1) {
+        throw new Error("Backup file is invalid");
+      }
+
+      let password;
+
+      console.log(`Found ${backupFiles?.length} files to restore from backup`);
+      for (const path of backupFiles) {
+        if (path === ".nnbackup") continue;
+        const filePath = `${zipOutputFolder}/${path}`;
+        const data = await RNFetchBlob.fs.readFile(filePath, "utf8");
+        const parsed = JSON.parse(data);
+
+        if (parsed.encrypted && !password) {
+          console.log("Backup is encrypted...", "requesting password");
+          password = await withPassword();
+          if (!password) throw new Error("Failed to decrypt backup");
+        }
+        await db.backup.import(parsed, password);
+        console.log("Imported", path);
+      }
+      // Remove files from cache
+      RNFetchBlob.fs.unlink(zipOutputFolder).catch(console.log);
+      if (remove) {
+        RNFetchBlob.fs.unlink(file).catch(console.log);
+      }
+
+      await db.initCollections();
+      initialize();
+      setRestoring(false);
+      close();
+      ToastEvent.show({
+        heading: "Backup restored successfully.",
+        type: "success",
+        context: "global"
+      });
+    } catch (e) {
+      backupError(e);
+      setRestoring(false);
+    }
+  }
+
+  /**
+   *
+   * @param {string} file
+   */
+  async function restoreFromNNBackup(backup) {
+    try {
+      if (backup.data.iv && backup.data.salt) {
+        const password = await withPassword();
+        if (password) {
+          try {
+            await restoreBackup(backup, password);
+            close();
+            setRestoring(false);
+          } catch (e) {
+            setRestoring(false);
+            backupError(e);
+          }
+        } else {
+          setRestoring(false);
+        }
+      } else {
+        await restoreBackup(backup);
+        setRestoring(false);
+        close();
+      }
+    } catch (e) {
+      setRestoring(false);
+      backupError(e);
+    }
+  }
+
+  const button = {
+    title: "Restore from files",
+    onPress: async () => {
+      if (restoring) {
+        return;
+      }
+      try {
+        const file = await DocumentPicker.pickSingle({
+          copyTo: "cachesDirectory"
+        });
+
+        if (file.name.endsWith(".nnbackupz")) {
+          setRestoring(true);
+          await restoreFromZip(file.fileCopyUri, true);
+        } else if (file.name.endsWith(".nnbackup")) {
+          RNFetchBlob.fs.unlink(file.fileCopyUri).catch(console.log);
+          setRestoring(true);
+          const data = await fetch(file.uri);
+          await restoreFromNNBackup(await data.json());
+        }
+      } catch (e) {
+        console.log("error", e.stack);
+        setRestoring(false);
+        backupError(e);
+      }
+    }
+  };
+
   const renderItem = ({ item, index }) => (
     <View
       style={{
@@ -271,76 +414,6 @@ const RestoreDataComponent = ({ close, setRestoring, restoring }) => {
     </View>
   );
 
-  const restoreBackup = async (backup, password) => {
-    await db.backup.import(backup, password);
-    setRestoring(false);
-    initialize();
-    ToastEvent.show({
-      heading: "Backup restored successfully.",
-      type: "success",
-      context: "global"
-    });
-  };
-
-  const backupError = (e) => {
-    ToastEvent.show({
-      heading: "Restore failed",
-      message:
-        e.message ||
-        "The selected backup data file is invalid. You must select a *.nnbackup file to restore.",
-      type: "error",
-      context: "local"
-    });
-  };
-
-  const button = {
-    title: "Restore from files",
-    onPress: () => {
-      if (restoring) {
-        return;
-      }
-
-      DocumentPicker.pickSingle()
-        .then((r) => {
-          setRestoring(true);
-          fetch(r.uri)
-            .then(async (r) => {
-              try {
-                let backup = await r.json();
-                if (backup.data.iv && backup.data.salt) {
-                  withPassword(
-                    async (value) => {
-                      try {
-                        restoreBackup(backup, value).then(() => {
-                          close();
-                          setRestoring(false);
-                        });
-                        return true;
-                      } catch (e) {
-                        backupError(e);
-                        setRestoring(false);
-                        return false;
-                      }
-                    },
-                    () => {
-                      setRestoring(false);
-                    }
-                  );
-                } else {
-                  await restoreBackup(backup);
-                  close();
-                }
-              } catch (e) {
-                setRestoring(false);
-                backupError(e);
-              }
-            })
-            .catch(console.log);
-        })
-        .catch(console.log);
-    }
-  };
-
   return (
     <>
       <View>

apps/mobile/app/services/backup.js

@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { sanitizeFilename } from "@notesnook/common";
 import { Platform } from "react-native";
 import RNFetchBlob from "react-native-blob-util";
 import FileViewer from "react-native-file-viewer";
@@ -30,6 +29,8 @@ import { eCloseSheet } from "../utils/events";
 import { sleep } from "../utils/time";
 import { ToastEvent, eSendEvent, presentSheet } from "./event-manager";
 import SettingsService from "./settings";
+import { cacheDir, copyFileAsync } from "../common/filesystem/utils";
+import { zip } from "react-native-zip-archive";
 
 const MS_DAY = 86400000;
 const MS_WEEK = MS_DAY * 7;
@@ -152,8 +153,6 @@ async function run(progress, context) {
   let androidBackupDirectory = await checkBackupDirExists(false, context);
   if (!androidBackupDirectory) return;
 
-  let backup;
-
   if (progress) {
     presentSheet({
       title: "Backing up your data",
@@ -163,43 +162,67 @@ async function run(progress, context) {
     });
   }
 
-  try {
-    backup = await db.backup.export(
-      "mobile",
-      SettingsService.get().encryptedBackup
-    );
-    if (!backup) throw new Error("Backup returned empty.");
-  } catch (e) {
-    await sleep(300);
-    eSendEvent(eCloseSheet);
-    ToastEvent.error(e, "Backup failed!");
-    return null;
+  let path;
+  let backupFilePath;
+  let backupFileName = "notesnook_backup_" + Date.now();
+
+  if (Platform.OS === "ios") {
+    path = await storage.checkAndCreateDir("/backups/");
   }
 
-  try {
-    let backupName = "notesnook_backup_" + Date.now();
-    backupName =
-      sanitizeFilename(backupName, { replacement: "_" }) + ".nnbackup";
-    let path;
-    let backupFilePath;
+  const zipSourceFolder = `${cacheDir}/${backupFileName}`;
+  const zipOutputFile =
+    Platform.OS === "ios"
+      ? `${path}/${backupFileName}.nnbackupz`
+      : `${cacheDir}/${backupFileName}.nnbackupz`;
 
-    if (Platform.OS === "ios") {
-      path = await storage.checkAndCreateDir("/backups/");
-      await RNFetchBlob.fs.writeFile(path + backupName, backup, "utf8");
-      backupFilePath = path + backupName;
-    } else {
-      backupFilePath = await ScopedStorage.writeFile(
-        androidBackupDirectory.uri,
-        backup,
-        backupName,
-        "nnbackup/json",
-        "utf8",
-        false
+  if (await RNFetchBlob.fs.exists(zipSourceFolder))
+    await RNFetchBlob.fs.unlink(zipSourceFolder);
+
+  await RNFetchBlob.fs.mkdir(zipSourceFolder);
+
+  try {
+    for await (const file of db.backup.export(
+      "mobile",
+      SettingsService.get().encryptedBackup
+    )) {
+      console.log("Writing backup chunk of size...", file?.data?.length);
+      await RNFetchBlob.fs.writeFile(
+        `${zipSourceFolder}/${file.path}`,
+        file.data,
+        "utf8"
       );
     }
 
+    await zip(zipSourceFolder, zipOutputFile);
+
+    console.log("Final zip:", await RNFetchBlob.fs.stat(zipOutputFile));
+
+    if (Platform.OS === "android") {
+      // Move the zip to user selected directory.
+      const file = await ScopedStorage.createFile(
+        androidBackupDirectory.uri,
+        `${backupFileName}.nnbackupz`,
+        "application/nnbackupz"
+      );
+      await copyFileAsync(`file://${zipOutputFile}`, file.uri);
+      path = file.uri;
+    } else {
+      path = zipOutputFile;
+    }
+    RNFetchBlob.fs.unlink(zipSourceFolder).catch(console.log);
     updateNextBackupTime();
 
+    let showBackupCompleteSheet = SettingsService.get().showBackupCompleteSheet;
+
+    if (context) return path;
+    await sleep(300);
+    if (showBackupCompleteSheet) {
+      presentBackupCompleteSheet(backupFilePath);
+    } else {
+      progress && eSendEvent(eCloseSheet);
+    }
+
     ToastEvent.show({
       heading: "Backup successful",
       message: "Your backup is stored in Notesnook folder on your phone.",
@@ -207,18 +230,10 @@ async function run(progress, context) {
       context: "global"
     });
 
-    let showBackupCompleteSheet = SettingsService.get().showBackupCompleteSheet;
-
-    if (context) return backupFilePath;
-    await sleep(300);
-    if (showBackupCompleteSheet) {
-      presentBackupCompleteSheet(backupFilePath);
-    } else {
-      progress && eSendEvent(eCloseSheet);
-    }
-    return backupFilePath;
+    return path;
   } catch (e) {
-    progress && eSendEvent(eCloseSheet);
+    await sleep(300);
+    eSendEvent(eCloseSheet);
     ToastEvent.error(e, "Backup failed!");
     return null;
   }

apps/mobile/app/utils/functions.js

@@ -107,46 +107,43 @@ export const deleteItems = async (item, context) => {
 
   if (topics?.length > 0) {
     const result = await confirmDeleteAllNotes(topics, "topic", context);
-    if (result.delete) {
-      for (const topic of topics) {
-        if (result.deleteNotes) {
+    if (!result.delete) return;
+    for (const topic of topics) {
+      if (result.deleteNotes) {
+        const notes = db.notebooks
+          .notebook(topic.notebookId)
+          .topics.topic(topic.id).all;
+        await db.notes.delete(...notes.map((note) => note.id));
+      }
+      await db.notebooks.notebook(topic.notebookId).topics.delete(topic.id);
+    }
+    useMenuStore.getState().setMenuPins();
+    ToastEvent.show({
+      heading: `${topics.length > 1 ? "Topics" : "Topic"} deleted`,
+      type: "success"
+    });
+  }
+
+  if (notebooks?.length > 0) {
+    const result = await confirmDeleteAllNotes(notebooks, "notebook", context);
+    if (!result.delete) return;
+    let ids = notebooks.map((i) => i.id);
+    if (result.deleteNotes) {
+      for (let id of ids) {
+        const notebook = db.notebooks.notebook(id);
+        const topics = notebook.topics.all;
+        for (let topic of topics) {
           const notes = db.notebooks
             .notebook(topic.notebookId)
             .topics.topic(topic.id).all;
           await db.notes.delete(...notes.map((note) => note.id));
         }
-        await db.notebooks.notebook(topic.notebookId).topics.delete(topic.id);
+        const notes = db.relations.from(notebook.data, "note");
+        await db.notes.delete(...notes.map((note) => note.id));
       }
-      useMenuStore.getState().setMenuPins();
-      ToastEvent.show({
-        heading: `${topics.length > 1 ? "Topics" : "Topic"} deleted`,
-        type: "success"
-      });
-    }
-  }
-
-  if (notebooks?.length > 0) {
-    const result = await confirmDeleteAllNotes(notebooks, "notebook", context);
-
-    if (result.delete) {
-      let ids = notebooks.map((i) => i.id);
-      if (result.deleteNotes) {
-        for (let id of ids) {
-          const notebook = db.notebooks.notebook(id);
-          const topics = notebook.topics.all;
-          for (let topic of topics) {
-            const notes = db.notebooks
-              .notebook(topic.notebookId)
-              .topics.topic(topic.id).all;
-            await db.notes.delete(...notes.map((note) => note.id));
-          }
-          const notes = db.relations.from(notebook.data, "note");
-          await db.notes.delete(...notes.map((note) => note.id));
-        }
-      }
-      await db.notebooks.delete(...ids);
-      useMenuStore.getState().setMenuPins();
     }
+    await db.notebooks.delete(...ids);
+    useMenuStore.getState().setMenuPins();
   }
 
   Navigation.queueRoutesForUpdate();

apps/mobile/native/android/app/build.gradle

@@ -111,7 +111,7 @@ android {
         minSdkVersion rootProject.ext.minSdkVersion
         targetSdkVersion rootProject.ext.targetSdkVersion
         multiDexEnabled true
-        versionCode 2059
+        versionCode 2061
         versionName getNpmVersion()
         testBuildType System.getProperty('testBuildType', 'debug')
         testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner'

apps/mobile/native/android/releasenotes/whatsnew-en-US

@@ -1,4 +1,3 @@
-- Much improved and faster sync
-- Note pinned in notifications will update automatically when content changes
-- Quick notes from notifications now respect paragraph spacing settings.
-- Many bug fixes and small improvements
+- New backup format nnbackupz with upto 90% smaller backups
+- Improve UI of code blocks & language selection popup
+- Fixed some bugs in sync v2

apps/mobile/native/ios/Notesnook.xcodeproj/project.pbxproj

@@ -997,7 +997,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2055;
 				DEVELOPMENT_TEAM = 53CWBG3QUC;
 				ENABLE_BITCODE = NO;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1071,7 +1071,7 @@
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.4;
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",
@@ -1102,7 +1102,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution";
 				CODE_SIGN_STYLE = Manual;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2055;
 				DEVELOPMENT_TEAM = "";
 				"DEVELOPMENT_TEAM[sdk=iphoneos*]" = 53CWBG3QUC;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1176,7 +1176,7 @@
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.4;
 				ONLY_ACTIVE_ARCH = NO;
 				OTHER_LDFLAGS = (
 					"$(inherited)",
@@ -1335,7 +1335,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2055;
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				DEVELOPMENT_TEAM = 53CWBG3QUC;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1347,7 +1347,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.4;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.notewidget;
@@ -1378,7 +1378,7 @@
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution";
 				CODE_SIGN_STYLE = Manual;
 				COPY_PHASE_STRIP = NO;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2055;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				DEVELOPMENT_TEAM = "";
 				"DEVELOPMENT_TEAM[sdk=iphoneos*]" = 53CWBG3QUC;
@@ -1391,7 +1391,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.4;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.notewidget;
 				PRODUCT_NAME = "$(TARGET_NAME)";
@@ -1421,7 +1421,7 @@
 				CODE_SIGN_ENTITLEMENTS = "Make Note/Make Note.entitlements";
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2055;
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				DEVELOPMENT_TEAM = 53CWBG3QUC;
 				"EXCLUDED_ARCHS[sdk=iphonesimulator*]" = arm64;
@@ -1495,7 +1495,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.4;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.share;
@@ -1526,7 +1526,7 @@
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Distribution";
 				CODE_SIGN_STYLE = Manual;
 				COPY_PHASE_STRIP = NO;
-				CURRENT_PROJECT_VERSION = 2053;
+				CURRENT_PROJECT_VERSION = 2055;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				DEVELOPMENT_TEAM = "";
 				"DEVELOPMENT_TEAM[sdk=iphoneos*]" = 53CWBG3QUC;
@@ -1601,7 +1601,7 @@
 					"@executable_path/Frameworks",
 					"@executable_path/../../Frameworks",
 				);
-				MARKETING_VERSION = 2.6.3;
+				MARKETING_VERSION = 2.6.4;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = org.streetwriters.notesnook.share;
 				PRODUCT_NAME = "$(TARGET_NAME)";

apps/mobile/native/ios/Podfile.lock

@@ -327,7 +327,7 @@ PODS:
     - React-Core
   - react-native-keep-awake (1.2.0):
     - React-Core
-  - react-native-mmkv-storage (0.10.0-alpha.7):
+  - react-native-mmkv-storage (0.10.0-alpha.9):
     - MMKV (~> 1.3.1)
     - React
     - React-Core
@@ -875,7 +875,7 @@ SPEC CHECKSUMS:
   react-native-image-resizer: 00ceb0e05586c7aadf061eea676957a6c2ec60fa
   react-native-in-app-review: db8bb167a5f238e7ceca5c242d6b36ce8c4404a4
   react-native-keep-awake: caee3ff89eaa21dfe29010f0d143566874a04441
-  react-native-mmkv-storage: ec32fd68168716755e1b30f09b476c236c6c68a8
+  react-native-mmkv-storage: d4ad55ab411b7f0d4e6269d801b31821ef48970b
   react-native-netinfo: ccbe1085dffd16592791d550189772e13bf479e2
   react-native-notification-sounds: da78c828fe1bcbb92d8b505d5261890ed315ff39
   react-native-orientation: f1caf84d65f1a4fd4511a18f2b924e634ad7a628

apps/mobile/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@notesnook/mobile",
-  "version": "2.6.3",
+  "version": "2.6.4",
   "private": true,
   "license": "GPL-3.0-or-later",
   "workspaces": [
@@ -43,4 +43,4 @@
     "react": "18.2.0",
     "react-native": "0.72.0"
   }
-}
+}

apps/theme-builder/src/components/theme-builder/index.tsx

@@ -643,13 +643,25 @@ function exportTheme(theme: ThemeDefinition) {
     }),
     `${theme.id}.json`
   );
+  const confirmed = window.confirm(
+    "Do you also want to download code-block.css for this theme?"
+  );
+  if (confirmed) {
+    FileSaver.saveAs(
+      new Blob([theme.codeBlockCSS], {
+        type: "text/plain"
+      }),
+      `code-block.css`
+    );
+  }
 }
 
 function themeToJSON(theme: ThemeDefinition) {
   return JSON.stringify(
     {
       $schema: JSON_SCHEMA_URL,
-      ...theme
+      ...theme,
+      codeBlockCSS: undefined
     },
     undefined,
     2

apps/web/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@notesnook/web",
-  "version": "2.6.3",
+  "version": "2.6.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@notesnook/web",
-      "version": "2.6.3",
+      "version": "2.6.4",
       "license": "GPL-3.0-or-later",
       "dependencies": {
         "@aws-sdk/util-base64-browser": "^3.208.0",

apps/web/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@notesnook/web",
   "description": "Your private note taking space",
-  "version": "2.6.3",
+  "version": "2.6.4",
   "private": true,
   "main": "./src/app.js",
   "homepage": "https://notesnook.com/",

apps/web/src/common/desktop-bridge/index.desktop.ts

@@ -63,3 +63,17 @@ function attachListener(event: string) {
     }
   };
 }
+
+export async function createWritableStream(path: string) {
+  const resolvedPath = await desktop.integration.resolvePath.query({
+    filePath: path
+  });
+  if (!resolvedPath) throw new Error("invalid path.");
+  const fs = require("fs");
+  const { Writable } = require("stream");
+  return new WritableStream(
+    Writable.toWeb(
+      fs.createWriteStream(resolvedPath, { encoding: "utf-8" })
+    ).getWriter()
+  );
+}

apps/web/src/common/desktop-bridge/index.ts

@@ -17,6 +17,10 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+import { createWriteStream } from "../../utils/stream-saver";
 import { type desktop as bridge } from "./index.desktop";
 
 export const desktop: typeof bridge | undefined = undefined;
+export async function createWritableStream(filename: string) {
+  return createWriteStream(filename);
+}

apps/web/src/common/index.js

@@ -1,217 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import {
-  showFeatureDialog,
-  showLoadingDialog,
-  showPasswordDialog,
-  showReminderDialog
-} from "../common/dialog-controller";
-import Config from "../utils/config";
-import { hashNavigate, getCurrentHash } from "../navigation";
-import { db } from "./db";
-import { sanitizeFilename } from "@notesnook/common";
-import { store as userstore } from "../stores/user-store";
-import FileSaver from "file-saver";
-import { showToast } from "../utils/toast";
-import { SUBSCRIPTION_STATUS } from "./constants";
-import { showFilePicker } from "../utils/file-picker";
-import { logger } from "../utils/logger";
-import { PATHS } from "@notesnook/desktop";
-import { TaskManager } from "./task-manager";
-import { EVENTS } from "@notesnook/core/dist/common";
-import { getFormattedDate } from "@notesnook/common";
-import { desktop } from "./desktop-bridge";
-
-export const CREATE_BUTTON_MAP = {
-  notes: {
-    title: "Add a note",
-    onClick: () =>
-      hashNavigate("/notes/create", { addNonce: true, replace: true })
-  },
-  notebooks: {
-    title: "Create a notebook",
-    onClick: () => hashNavigate("/notebooks/create", { replace: true })
-  },
-  topics: {
-    title: "Create a topic",
-    onClick: () => hashNavigate(`/topics/create`, { replace: true })
-  },
-  tags: {
-    title: "Create a tag",
-    onClick: () => hashNavigate(`/tags/create`, { replace: true })
-  },
-  reminders: {
-    title: "Add a reminder",
-    onClick: () => hashNavigate(`/reminders/create`, { replace: true })
-  }
-};
-
-export async function introduceFeatures() {
-  const hash = getCurrentHash().replace("#", "");
-  if (!!hash || IS_TESTING) return;
-  const features = [];
-  for (let feature of features) {
-    if (!Config.get(`feature:${feature}`)) {
-      await showFeatureDialog(feature);
-    }
-  }
-}
-
-export const DEFAULT_CONTEXT = { colors: [], tags: [], notebook: {} };
-
-export async function createBackup() {
-  const encryptBackups =
-    userstore.get().isLoggedIn && Config.get("encryptBackups", false);
-  const data = await showLoadingDialog({
-    title: "Creating backup",
-    subtitle: "We are creating a backup of your data. Please wait...",
-    action: async () => {
-      return await db.backup.export("web", encryptBackups);
-    }
-  });
-  if (!data) {
-    showToast("error", "Could not create a backup of your data.");
-    return;
-  }
-
-  const filename = sanitizeFilename(`notesnook-backup-${getFormattedDate()}`);
-
-  const ext = "nnbackup";
-  if (IS_DESKTOP_APP) {
-    const directory = Config.get(
-      "backupStorageLocation",
-      PATHS.backupsDirectory
-    );
-    const filePath = `${directory}/${filename}.${ext}`;
-    await desktop?.integration.saveFile.query({ filePath, data });
-    showToast("success", `Backup saved at ${filePath}.`);
-  } else {
-    FileSaver.saveAs(new Blob([Buffer.from(data)]), `${filename}.${ext}`);
-  }
-}
-
-export async function selectBackupFile() {
-  const file = await showFilePicker({
-    acceptedFileTypes: ".nnbackup,application/json"
-  });
-  if (!file) return;
-
-  const reader = new FileReader();
-  const backup = await new Promise((resolve) => {
-    reader.addEventListener("load", (event) => {
-      const text = event.target.result;
-      try {
-        resolve(JSON.parse(text));
-      } catch (e) {
-        alert(
-          "Error: Could not read the backup file provided. Either it's corrupted or invalid."
-        );
-        resolve();
-      }
-    });
-    reader.readAsText(file);
-  });
-
-  return { file, backup };
-}
-
-export async function importBackup() {
-  const { backup } = await selectBackupFile();
-  await restoreBackupFile(backup);
-}
-
-export async function restoreBackupFile(backup) {
-  if (backup.data.iv && backup.data.salt) {
-    await showPasswordDialog("ask_backup_password", async ({ password }) => {
-      const error = await restoreWithProgress(backup, password);
-      return !error;
-    });
-  } else {
-    await restoreWithProgress(backup);
-  }
-}
-
-async function restoreWithProgress(backup, password) {
-  await TaskManager.startTask({
-    title: "Restoring backup",
-    subtitle: "This might take a while",
-    type: "modal",
-    action: (report) => {
-      db.eventManager.subscribe(
-        EVENTS.migrationProgress,
-        ({ collection, total, current }) => {
-          report({
-            text: `Restoring ${collection}...`,
-            current,
-            total
-          });
-        }
-      );
-
-      report({ text: `Restoring...` });
-      return restore(backup, password);
-    }
-  });
-}
-
-export async function verifyAccount() {
-  if (!(await db.user.getUser())) return true;
-  return showPasswordDialog("verify_account", ({ password }) => {
-    return db.user.verifyPassword(password);
-  });
-}
-
-export function totalSubscriptionConsumed(user) {
-  if (!user) return 0;
-  const start = user.subscription?.start;
-  const end = user.subscription?.expiry;
-  if (!start || !end) return 0;
-
-  const total = end - start;
-  const consumed = Date.now() - start;
-
-  return Math.round((consumed / total) * 100);
-}
-
-export async function showUpgradeReminderDialogs() {
-  if (IS_TESTING) return;
-
-  const user = userstore.get().user;
-  if (!user || !user.subscription || user.subscription?.expiry === 0) return;
-
-  const consumed = totalSubscriptionConsumed(user);
-  const isTrial = user?.subscription?.type === SUBSCRIPTION_STATUS.TRIAL;
-  const isBasic = user?.subscription?.type === SUBSCRIPTION_STATUS.BASIC;
-  if (isBasic && consumed >= 100) {
-    await showReminderDialog("trialexpired");
-  } else if (isTrial && consumed >= 75) {
-    await showReminderDialog("trialexpiring");
-  }
-}
-
-async function restore(backup, password) {
-  try {
-    await db.backup.import(backup, password);
-    showToast("success", "Backup restored!");
-  } catch (e) {
-    logger.error(e, "Could not restore the backup");
-    showToast("error", `Could not restore the backup: ${e.message || e}`);
-  }
-}

apps/web/src/common/index.ts

@@ -0,0 +1,290 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+  showFeatureDialog,
+  showPasswordDialog,
+  showReminderDialog
+} from "./dialog-controller";
+import Config from "../utils/config";
+import { hashNavigate, getCurrentHash } from "../navigation";
+import { db } from "./db";
+import { sanitizeFilename } from "@notesnook/common";
+import { store as userstore } from "../stores/user-store";
+import { showToast } from "../utils/toast";
+import { SUBSCRIPTION_STATUS } from "./constants";
+import { readFile, showFilePicker } from "../utils/file-picker";
+import { logger } from "../utils/logger";
+import { PATHS } from "@notesnook/desktop";
+import { TaskManager } from "./task-manager";
+import { EVENTS } from "@notesnook/core/dist/common";
+import { getFormattedDate } from "@notesnook/common";
+import { createWritableStream } from "./desktop-bridge";
+import { ZipStream } from "../utils/streams/zip-stream";
+import { register } from "../utils/stream-saver/mitm";
+import { FeatureKeys } from "../dialogs/feature-dialog";
+import { Reader } from "../utils/zip-reader";
+
+export const CREATE_BUTTON_MAP = {
+  notes: {
+    title: "Add a note",
+    onClick: () =>
+      hashNavigate("/notes/create", { addNonce: true, replace: true })
+  },
+  notebooks: {
+    title: "Create a notebook",
+    onClick: () => hashNavigate("/notebooks/create", { replace: true })
+  },
+  topics: {
+    title: "Create a topic",
+    onClick: () => hashNavigate(`/topics/create`, { replace: true })
+  },
+  tags: {
+    title: "Create a tag",
+    onClick: () => hashNavigate(`/tags/create`, { replace: true })
+  },
+  reminders: {
+    title: "Add a reminder",
+    onClick: () => hashNavigate(`/reminders/create`, { replace: true })
+  }
+};
+
+export async function introduceFeatures() {
+  const hash = getCurrentHash().replace("#", "");
+  if (!!hash || IS_TESTING) return;
+  const features: FeatureKeys[] = [];
+  for (const feature of features) {
+    if (!Config.get(`feature:${feature}`)) {
+      await showFeatureDialog(feature);
+    }
+  }
+}
+
+export const DEFAULT_CONTEXT = { colors: [], tags: [], notebook: {} };
+
+export async function createBackup() {
+  await register();
+  const encryptBackups =
+    userstore.get().isLoggedIn && Config.get("encryptBackups", false);
+
+  const filename = sanitizeFilename(
+    `notesnook-backup-${getFormattedDate(Date.now())}`
+  );
+  const directory = Config.get("backupStorageLocation", PATHS.backupsDirectory);
+  const ext = "nnbackupz";
+  const filePath = IS_DESKTOP_APP
+    ? `${directory}/${filename}.${ext}`
+    : `${filename}.${ext}`;
+
+  const encoder = new TextEncoder();
+  const error = await TaskManager.startTask<Error | void>({
+    type: "modal",
+    title: "Creating backup",
+    subtitle: "We are creating a backup of your data. Please wait...",
+    action: async (report) => {
+      const writeStream = await createWritableStream(filePath);
+
+      await new ReadableStream({
+        start() {},
+        async pull(controller) {
+          for await (const file of db.backup!.export("web", encryptBackups)) {
+            report({
+              text: `Saving chunk ${file.path}`
+            });
+            controller.enqueue({
+              path: file.path,
+              data: encoder.encode(file.data)
+            });
+          }
+          controller.close();
+        }
+      })
+        .pipeThrough(new ZipStream())
+        .pipeTo(writeStream);
+    }
+  });
+  if (error) {
+    showToast(
+      "error",
+      `Could not create a backup of your data: ${(error as Error).message}`
+    );
+    console.error(error);
+  } else {
+    showToast("success", `Backup saved at ${filePath}.`);
+  }
+}
+
+export async function selectBackupFile() {
+  const file = await showFilePicker({
+    acceptedFileTypes: ".nnbackup,.nnbackupz"
+  });
+  if (!file) return;
+  return file;
+}
+
+export async function importBackup() {
+  const backupFile = await selectBackupFile();
+  if (!backupFile) return;
+  await restoreBackupFile(backupFile);
+}
+
+export async function restoreBackupFile(backupFile: File) {
+  const isLegacy = !backupFile.name.endsWith(".nnbackupz");
+
+  if (isLegacy) {
+    const backup = JSON.parse(await readFile(backupFile));
+
+    if (backup.data.iv && backup.data.salt) {
+      await showPasswordDialog("ask_backup_password", async ({ password }) => {
+        if (!password) return false;
+        const error = await restoreWithProgress(backup, password);
+        return !error;
+      });
+    } else {
+      await restoreWithProgress(backup);
+    }
+  } else {
+    const error = await TaskManager.startTask<Error | void>({
+      title: "Restoring backup",
+      subtitle: "Please wait while we restore your backup...",
+      type: "modal",
+      action: async (report) => {
+        let cachedPassword: string | undefined = undefined;
+        const { read, totalFiles } = await Reader(backupFile);
+        let filesProcessed = 0;
+        for await (const entry of read()) {
+          if (filesProcessed++ === 0 && entry.name !== ".nnbackup")
+            throw new Error("Invalid backup.");
+          else if (entry.name === ".nnbackup") continue;
+
+          const backup = JSON.parse(await entry.text());
+          if (backup.encrypted) {
+            if (!cachedPassword) {
+              const result = await showPasswordDialog(
+                "ask_backup_password",
+                async ({ password }) => {
+                  if (!password) return false;
+                  await db.backup?.import(backup, password);
+                  cachedPassword = password;
+                  return true;
+                }
+              );
+              if (!result) break;
+            } else await db.backup?.import(backup, cachedPassword);
+          } else {
+            await db.backup?.import(backup, null);
+          }
+
+          report({
+            total: totalFiles,
+            text: `Processed ${entry.name}`,
+            current: filesProcessed
+          });
+        }
+        await db.initCollections();
+      }
+    });
+    if (error) {
+      console.error(error);
+      showToast("error", `Failed to restore backup: ${error.message}`);
+    }
+  }
+}
+
+async function restoreWithProgress(
+  backup: Record<string, unknown>,
+  password?: string
+) {
+  return await TaskManager.startTask<Error | void>({
+    title: "Restoring backup",
+    subtitle: "This might take a while",
+    type: "modal",
+    action: (report) => {
+      db.eventManager.subscribe(
+        EVENTS.migrationProgress,
+        ({
+          collection,
+          total,
+          current
+        }: {
+          collection: string;
+          total: number;
+          current: number;
+        }) => {
+          report({
+            text: `Restoring ${collection}...`,
+            current,
+            total
+          });
+        }
+      );
+
+      report({ text: `Restoring...` });
+      return restore(backup, password);
+    }
+  });
+}
+
+export async function verifyAccount() {
+  if (!(await db.user?.getUser())) return true;
+  return showPasswordDialog("verify_account", ({ password }) => {
+    return db.user?.verifyPassword(password) || false;
+  });
+}
+
+export function totalSubscriptionConsumed(user: User) {
+  if (!user) return 0;
+  const start = user.subscription?.start;
+  const end = user.subscription?.expiry;
+  if (!start || !end) return 0;
+
+  const total = end - start;
+  const consumed = Date.now() - start;
+
+  return Math.round((consumed / total) * 100);
+}
+
+export async function showUpgradeReminderDialogs() {
+  if (IS_TESTING) return;
+
+  const user = userstore.get().user;
+  if (!user || !user.subscription || user.subscription?.expiry === 0) return;
+
+  const consumed = totalSubscriptionConsumed(user);
+  const isTrial = user?.subscription?.type === SUBSCRIPTION_STATUS.TRIAL;
+  const isBasic = user?.subscription?.type === SUBSCRIPTION_STATUS.BASIC;
+  if (isBasic && consumed >= 100) {
+    await showReminderDialog("trialexpired");
+  } else if (isTrial && consumed >= 75) {
+    await showReminderDialog("trialexpiring");
+  }
+}
+
+async function restore(backup: Record<string, unknown>, password?: string) {
+  try {
+    await db.backup?.import(backup, password);
+    showToast("success", "Backup restored!");
+  } catch (e) {
+    logger.error(e as Error, "Could not restore the backup");
+    showToast(
+      "error",
+      `Could not restore the backup: ${(e as Error).message || e}`
+    );
+  }
+}

apps/web/src/dialogs/settings/backup-export-settings.ts

@@ -50,7 +50,7 @@ export const BackupExportSettings: SettingsGroup[] = [
               )
                 useSettingStore.getState().toggleEncryptBackups();
               const verified =
-                !useSettingStore.getState().encryptBackups ||
+                useSettingStore.getState().encryptBackups ||
                 (await verifyAccount());
               if (verified) await createBackup();
             },

apps/web/src/utils/stream-saver/mitm.ts

@@ -16,11 +16,26 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-export {};
 
 let sw: ServiceWorker | null = null;
 let scope = "";
 
+let keepAlive = () => {
+  keepAlive = () => {};
+  const interval = setInterval(() => {
+    if (sw) {
+      sw.postMessage("ping");
+    } else {
+      const ping =
+        location.href.substr(0, location.href.lastIndexOf("/")) + "/ping";
+      fetch(ping).then((res) => {
+        !res.ok && clearInterval(interval);
+        return res.text();
+      });
+    }
+  }, 10000);
+};
+
 function registerWorker() {
   return navigator.serviceWorker
     .getRegistration("./")
@@ -104,9 +119,12 @@ export function postMessage(
   // messageChannel.port2 to the service worker. The service worker can
   // then use the transferred port to reply via postMessage(), which
   // will in turn trigger the onmessage handler on messageChannel.port1.
-
   const transferable = [ports[0]];
 
+  if (!data.transferringReadable) {
+    keepAlive();
+  }
+
   return sw?.postMessage(data, transferable);
 }
 
@@ -114,4 +132,8 @@ export async function register() {
   if (navigator.serviceWorker) {
     await registerWorker();
   }
+  // FF v102 just started to supports transferable streams, but still needs to ping sw.js
+  // even tough the service worker dose not have to do any kind of work and listen to any
+  // messages... #305
+  keepAlive();
 }

apps/web/src/views/auth.tsx

@@ -272,6 +272,7 @@ function LoginPassword(props: BaseAuthComponentProps<"login:password">) {
       type="login:password"
       title="Your account password"
       subtitle={"Your password is always hashed before leaving this device."}
+      loadForever
       loading={{
         title: "Logging you in",
         subtitle: "Please wait while you are authenticated."
@@ -806,6 +807,7 @@ type AuthFormProps<TType extends AuthRoutes> = {
   loading: { title: string; subtitle: string };
   type: TType;
   onSubmit: (form: AuthFormData[TType]) => Promise<void>;
+  loadForever?: boolean;
   canSkip?: boolean;
   children?:
     | React.ReactNode
@@ -813,7 +815,7 @@ type AuthFormProps<TType extends AuthRoutes> = {
 };
 
 export function AuthForm<T extends AuthRoutes>(props: AuthFormProps<T>) {
-  const { title, subtitle, children, canSkip } = props;
+  const { title, subtitle, children, canSkip, loadForever } = props;
   const [isSubmitting, setIsSubmitting] = useState(false);
   const [error, setError] = useState<string>();
   const formRef = useRef<HTMLFormElement>(null);
@@ -838,7 +840,9 @@ export function AuthForm<T extends AuthRoutes>(props: AuthFormProps<T>) {
         try {
           setForm(form);
           await props.onSubmit(form);
+          if (!loadForever) setIsSubmitting(false);
         } catch (e) {
+          setIsSubmitting(false);
           const error = e as Error;
           if (error.message === "invalid_grant") {
             setError(
@@ -847,8 +851,6 @@ export function AuthForm<T extends AuthRoutes>(props: AuthFormProps<T>) {
             return;
           }
           setError(error.message);
-        } finally {
-          setIsSubmitting(false);
         }
       }}
       sx={{

apps/web/src/views/recovery.tsx

@@ -40,10 +40,7 @@ type RecoveryKeyFormData = {
 };
 
 type BackupFileFormData = {
-  backupFile: {
-    file: File;
-    backup: Record<string, unknown>;
-  };
+  backupFile: File;
 };
 
 type NewPasswordFormData = BackupFileFormData & {
@@ -406,7 +403,7 @@ function BackupFileMethod(props: BaseRecoveryComponentProps<"method:backup">) {
     if (!backupFile) return;
     const backupFileInput = document.getElementById("backupFile");
     if (!(backupFileInput instanceof HTMLInputElement)) return;
-    backupFileInput.value = backupFile?.file?.name;
+    backupFileInput.value = backupFile?.name;
   }, [backupFile]);
 
   return (
@@ -519,7 +516,7 @@ function NewPassword(props: BaseRecoveryComponentProps<"new">) {
           throw new Error("Could not reset account password.");
 
         if (formData?.backupFile) {
-          await restoreBackupFile(formData?.backupFile.backup);
+          await restoreBackupFile(formData?.backupFile);
           await db.sync(true, true);
         }
 

fastlane/metadata/android/en-US/changelogs/10309.txt

@@ -0,0 +1,3 @@
+- New backup format nnbackupz with upto 90% smaller backups
+- Improve UI of code blocks & language selection popup
+- Fixed some bugs in sync v2

packages/core/__e2e__/sync.test.js

@@ -0,0 +1,466 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import Database from "../src/api/index";
+import { NodeStorageInterface } from "../__mocks__/node-storage.mock";
+import { FS } from "../__mocks__/fs.mock";
+import Compressor from "../__mocks__/compressor.mock";
+import { CHECK_IDS, EV, EVENTS } from "../src/common";
+import { EventSource } from "event-source-polyfill";
+import { delay } from "../__tests__/utils";
+import { test, expect, vitest } from "vitest";
+import { login } from "./utils";
+
+const TEST_TIMEOUT = 60 * 1000;
+
+test(
+  "case 1: device A & B should only download the changes from device C (no uploading)",
+  async () => {
+    const types = [];
+    function onSyncProgress({ type }) {
+      types.push(type);
+    }
+
+    const [deviceA, deviceB, deviceC] = await Promise.all([
+      initializeDevice("deviceA"),
+      initializeDevice("deviceB"),
+      initializeDevice("deviceC")
+    ]);
+
+    deviceA.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
+    deviceB.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
+
+    await deviceC.notes.add({ title: "new note 1" });
+    await syncAndWait(deviceC, deviceC);
+
+    expect(types.every((t) => t === "download")).toBe(true);
+
+    await cleanup(deviceA, deviceB, deviceC);
+  },
+  TEST_TIMEOUT
+);
+
+test(
+  "case 3: Device A & B have unsynced changes but server has nothing",
+  async () => {
+    const [deviceA, deviceB] = await Promise.all([
+      initializeDevice("deviceA"),
+      initializeDevice("deviceB")
+    ]);
+
+    const note1Id = await deviceA.notes.add({
+      title: "Test note from device A"
+    });
+    const note2Id = await deviceB.notes.add({
+      title: "Test note from device B"
+    });
+
+    await syncAndWait(deviceA, deviceB);
+
+    expect(deviceA.notes.note(note2Id)).toBeTruthy();
+    expect(deviceB.notes.note(note1Id)).toBeTruthy();
+    expect(deviceA.notes.note(note1Id)).toBeTruthy();
+    expect(deviceB.notes.note(note2Id)).toBeTruthy();
+
+    await cleanup(deviceA, deviceA);
+  },
+  TEST_TIMEOUT
+);
+
+// test(
+//   "case 4: Device A's sync is interrupted halfway and Device B makes some changes afterwards and syncs.",
+//   async () => {
+//     const deviceA = await initializeDevice("deviceA");
+//     const deviceB = await initializeDevice("deviceB");
+
+//     const unsyncedNoteIds = [];
+//     for (let i = 0; i < 10; ++i) {
+//       const id = await deviceA.notes.add({
+//         title: `Test note ${i} from device A`,
+//       });
+//       unsyncedNoteIds.push(id);
+//     }
+
+//     const half = unsyncedNoteIds.length / 2 + 1;
+//     deviceA.eventManager.subscribe(
+//       EVENTS.syncProgress,
+//       async ({ type, current }) => {
+//         if (type === "upload" && current === half) {
+//           await deviceA.syncer.stop();
+//         }
+//       }
+//     );
+
+//     await expect(deviceA.sync(true)).rejects.toThrow();
+
+//     let syncedNoteIds = [];
+//     for (let i = 0; i < unsyncedNoteIds.length; ++i) {
+//       const expectedNoteId = unsyncedNoteIds[i];
+//       if (deviceB.notes.note(expectedNoteId))
+//         syncedNoteIds.push(expectedNoteId);
+//     }
+//     expect(
+//       syncedNoteIds.length === half - 1 || syncedNoteIds.length === half
+//     ).toBe(true);
+
+//     const deviceBNoteId = await deviceB.notes.add({
+//       title: "Test note of case 4 from device B",
+//     });
+
+//     await deviceB.sync(true);
+
+//     await syncAndWait(deviceA, deviceB);
+
+//     expect(deviceA.notes.note(deviceBNoteId)).toBeTruthy();
+//     expect(
+//       unsyncedNoteIds
+//         .map((id) => !!deviceB.notes.note(id))
+//         .every((res) => res === true)
+//     ).toBe(true);
+
+//     await cleanup(deviceA, deviceB);
+//   },
+//
+// );
+
+// test.only(
+//   "case 5: Device A's sync is interrupted halfway and Device B makes changes on the same note's content that didn't get synced on Device A due to interruption.",
+//   async () => {
+//     const deviceA = await initializeDevice("deviceA");
+//     const deviceB = await initializeDevice("deviceB");
+
+//     const noteIds = [];
+//     for (let i = 0; i < 10; ++i) {
+//       const id = await deviceA.notes.add({
+//         content: {
+//           type: "tiptap",
+//           data: `<p>deviceA=true</p>`,
+//         },
+//       });
+//       noteIds.push(id);
+//     }
+
+//     await deviceA.sync(true);
+//     await deviceB.sync(true);
+
+//     const unsyncedNoteIds = [];
+//     for (let id of noteIds) {
+//       const noteId = await deviceA.notes.add({
+//         id,
+//         content: {
+//           type: "tiptap",
+//           data: `<p>deviceA=true+changed=true</p>`,
+//         },
+//       });
+//       unsyncedNoteIds.push(noteId);
+//     }
+
+//     deviceA.eventManager.subscribe(
+//       EVENTS.syncProgress,
+//       async ({ type, total, current }) => {
+//         const half = total / 2 + 1;
+//         if (type === "upload" && current === half) {
+//           await deviceA.syncer.stop();
+//         }
+//       }
+//     );
+
+//     await expect(deviceA.sync(true)).rejects.toThrow();
+
+//     await delay(10 * 1000);
+
+//     for (let id of unsyncedNoteIds) {
+//       await deviceB.notes.add({
+//         id,
+//         content: {
+//           type: "tiptap",
+//           data: "<p>changes from device B</p>",
+//         },
+//       });
+//     }
+
+//     const error = await withError(async () => {
+//       await deviceB.sync(true);
+//       await deviceA.sync(true);
+//     });
+
+//     expect(error).not.toBeInstanceOf(NoErrorThrownError);
+//     expect(error.message.includes("Merge")).toBeTruthy();
+
+//     await cleanup(deviceA, deviceB);
+//   },
+//
+// );
+
+test(
+  "issue: running force sync from device A makes device B always download everything",
+  async () => {
+    const [deviceA, deviceB] = await Promise.all([
+      initializeDevice("deviceA"),
+      initializeDevice("deviceB")
+    ]);
+
+    await syncAndWait(deviceA, deviceB, true);
+
+    const handler = vitest.fn();
+    deviceB.eventManager.subscribe(EVENTS.syncProgress, handler);
+
+    await deviceB.sync(true);
+
+    expect(handler).not.toHaveBeenCalled();
+
+    await cleanup(deviceB);
+  },
+  TEST_TIMEOUT
+);
+
+test(
+  "issue: colors are not properly created if multiple notes are synced together",
+  async () => {
+    const [deviceA, deviceB] = await Promise.all([
+      initializeDevice("deviceA", [CHECK_IDS.noteColor]),
+      initializeDevice("deviceB", [CHECK_IDS.noteColor])
+    ]);
+
+    const noteIds = [];
+    for (let i = 0; i < 3; ++i) {
+      const id = await deviceA.notes.add({
+        content: {
+          type: "tiptap",
+          data: `<p>deviceA=true</p>`
+        }
+      });
+      noteIds.push(id);
+    }
+
+    await syncAndWait(deviceA, deviceB);
+
+    for (let noteId of noteIds) {
+      await deviceA.notes.note(noteId).color("purple");
+      expect(deviceB.notes.note(noteId)).toBeTruthy();
+      expect(deviceB.notes.note(noteId).data.color).toBeUndefined();
+    }
+
+    await syncAndWait(deviceA, deviceB);
+
+    const purpleColor = deviceB.colors.tag("purple");
+    expect(noteIds.every((id) => purpleColor.noteIds.indexOf(id) > -1)).toBe(
+      true
+    );
+
+    await cleanup(deviceA, deviceB);
+  },
+  TEST_TIMEOUT
+);
+
+test(
+  "issue: new topic on device A gets replaced by the new topic on device B",
+  async () => {
+    const [deviceA, deviceB] = await Promise.all([
+      initializeDevice("deviceA"),
+      initializeDevice("deviceB")
+    ]);
+    // const deviceA = await initializeDevice("deviceA");
+    // const deviceB = await initializeDevice("deviceB");
+
+    const id = await deviceA.notebooks.add({ title: "Notebook 1" });
+
+    await syncAndWait(deviceA, deviceB, false);
+
+    expect(deviceB.notebooks.notebook(id)).toBeDefined();
+
+    await deviceA.notebooks.notebook(id).topics.add("Topic 1");
+    // to create a conflict
+    await delay(1500);
+    await deviceB.notebooks.notebook(id).topics.add("Topic 2");
+
+    expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
+    expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
+
+    expect(
+      deviceB.notebooks.notebook(id).topics.topic("Topic 2")._topic.dateEdited
+    ).toBeGreaterThan(
+      deviceA.notebooks.notebook(id).topics.topic("Topic 1")._topic.dateEdited
+    );
+    expect(deviceB.notebooks.notebook(id).dateModified).toBeGreaterThan(
+      deviceA.notebooks.notebook(id).dateModified
+    );
+
+    await syncAndWait(deviceB, deviceA, false);
+
+    expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
+    expect(deviceB.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
+
+    expect(deviceA.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
+    expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
+
+    await cleanup(deviceA, deviceB);
+  },
+  TEST_TIMEOUT
+);
+
+test(
+  "issue: assigning 2 notes to the same topic should keep references of both notes in the topic",
+  async (ctx) => {
+    const [deviceA, deviceB] = await Promise.all([
+      initializeDevice("deviceA"),
+      initializeDevice("deviceB")
+    ]);
+
+    const id = await deviceA.notebooks.add({
+      title: "Notebook 1",
+      topics: ["Topic 1"]
+    });
+
+    const topic = deviceA.notebooks.notebook(id).topics.topic("Topic 1");
+
+    await syncAndWait(deviceA, deviceB, false);
+
+    expect(deviceB.notebooks.notebook(id)).toBeDefined();
+
+    const noteA = await deviceA.notes.add({ title: "Note 1" });
+    await deviceA.notes.addToNotebook({ id, topic: topic.id }, noteA);
+
+    expect(
+      deviceA.notebooks.notebook(id).topics.topic(topic.id).totalNotes
+    ).toBe(1);
+
+    await delay(2000);
+
+    const noteB = await deviceB.notes.add({ title: "Note 2" });
+    await deviceB.notes.addToNotebook({ id, topic: topic.id }, noteB);
+
+    expect(
+      deviceB.notebooks.notebook(id).topics.topic(topic.id).totalNotes
+    ).toBe(1);
+
+    ctx.onTestFailed(() => {
+      console.log(deviceA.notes.topicReferences.get(topic.id), noteA);
+      console.log(deviceB.notes.topicReferences.get(topic.id), noteB);
+
+      deviceB.notes.topicReferences.rebuild();
+      deviceA.notes.topicReferences.rebuild();
+
+      console.log(deviceA.notes.topicReferences.get(topic.id), noteA);
+      console.log(deviceB.notes.topicReferences.get(topic.id), noteB);
+    });
+    await syncAndWait(deviceB, deviceA, false);
+
+    expect(deviceA.notes.note(noteB)).toBeDefined();
+    expect(deviceB.notes.note(noteA)).toBeDefined();
+
+    expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(1);
+    expect(deviceA.notes.note(noteB).data.notebooks).toHaveLength(1);
+
+    expect(
+      deviceA.notebooks.notebook(id).topics.topic(topic.id).totalNotes
+    ).toBe(2);
+    expect(
+      deviceB.notebooks.notebook(id).topics.topic(topic.id).totalNotes
+    ).toBe(2);
+
+    await cleanup(deviceA, deviceB);
+  },
+  TEST_TIMEOUT
+);
+
+/**
+ *
+ * @param {string} id
+ * @returns {Promise<Database>}
+ */
+async function initializeDevice(id, capabilities = []) {
+  console.time(`Init ${id}`);
+  EV.subscribe(EVENTS.userCheckStatus, async (type) => {
+    return {
+      type,
+      result: capabilities.indexOf(type) > -1
+    };
+  });
+  EV.subscribe(EVENTS.syncCheckStatus, async (type) => {
+    return {
+      type,
+      result: true
+    };
+  });
+
+  const device = new Database();
+  device.setup(new NodeStorageInterface(), EventSource, FS, Compressor);
+
+  await device.init();
+
+  await login(device);
+
+  await device.user.resetUser(false);
+
+  await device.sync(true, false);
+
+  console.timeEnd(`Init ${id}`);
+  return device;
+}
+
+/**
+ *
+ * @param  {...Database} devices
+ */
+async function cleanup(...devices) {
+  await Promise.all([
+    devices.map(async (device) => {
+      await device.syncer.stop();
+      await device.user.logout();
+      device.eventManager.unsubscribeAll();
+    })
+  ]);
+  EV.unsubscribeAll();
+}
+
+/**
+ *
+ * @param {Database} deviceA
+ * @param {Database} deviceB
+ * @returns
+ */
+function syncAndWait(deviceA, deviceB, force = false) {
+  return new Promise((resolve, reject) => {
+    const ref2 = deviceB.eventManager.subscribe(
+      EVENTS.databaseSyncRequested,
+      (full, force, lastSynced) => {
+        console.log("sync requested by device A", full, force, lastSynced);
+        ref2.unsubscribe();
+        deviceB.sync(full, force, lastSynced).catch(reject);
+      }
+    );
+
+    const ref = deviceB.eventManager.subscribe(EVENTS.syncCompleted, () => {
+      ref.unsubscribe();
+      console.log("sync completed.");
+      resolve();
+    });
+
+    console.log(
+      "waiting for sync...",
+      "Device A:",
+      deviceA.syncer.sync.syncing,
+      "Device B:",
+      deviceB.syncer.sync.syncing
+    );
+
+    deviceA.sync(true, force).catch(reject);
+  });
+}

packages/core/__mocks__/fs.mock.ts

@@ -17,44 +17,55 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-var fs = {};
+import { DataFormat, SerializedKey } from "@notesnook/crypto";
+import { xxhash64 } from "hash-wasm";
+import { IDataType } from "hash-wasm/dist/lib/util";
+
+let fs = {};
 
 function hasItem(key) {
   return !!fs[key];
 }
 
-/**
- * We perform 4 steps here:
- * 1. We convert base64 to Uint8Array (if we get base64, that is)
- * 2. We hash the Uint8Array.
- * 3. We encrypt the Uint8Array
- * 4. We save the encrypted Uint8Array
- */
-async function writeEncrypted(filename, { data }) {
-  const { hash, type: hashType } = hashBuffer(data);
-  if (!filename) filename = hash;
-  if (hasItem(filename)) return { hash, hashType };
-  fs[filename] = data;
+async function writeEncryptedBase64(
+  data: string,
+  key: SerializedKey,
+  _mimeType: string
+) {
+  const bytes = new Uint8Array(Buffer.from(data, "base64"));
+
+  const { hash, type: hashType } = await hashBuffer(bytes);
+
+  if (hasItem(hash)) delete fs[hash];
+
+  fs[hash] = data;
   return {
     chunkSize: 512,
     alg: "xcha-stream",
     hash,
     hashType,
     iv: "some iv",
-    cipher: data,
-    salt: "i am some salt",
+    salt: key.salt!,
     length: data.length
   };
 }
 
-function hashBuffer(data) {
+function hashBase64(data: string) {
+  return hashBuffer(Buffer.from(data, "base64"));
+}
+
+export async function hashBuffer(data: IDataType) {
   return {
-    hash: hashCode(data).toString(16),
-    type: "xxh3"
+    hash: await xxhash64(data),
+    type: "xxh64"
   };
 }
 
-async function readEncrypted(filename) {
+async function readEncrypted<TOutputFormat extends DataFormat>(
+  filename: string,
+  _key: SerializedKey,
+  _cipherData: any
+) {
   const cipher = fs[filename];
   if (!cipher) {
     console.error(`File not found. Filename: ${filename}`);
@@ -63,23 +74,23 @@ async function readEncrypted(filename) {
   return cipher.data;
 }
 
-async function uploadFile(filename) {
-  let cipher = fs[filename];
+async function uploadFile(filename: string, _requestOptions: any) {
+  const cipher = fs[filename];
   if (!cipher) throw new Error(`File not found. Filename: ${filename}`);
   return true;
 }
 
-async function downloadFile(filename) {
+async function downloadFile(filename: string, _requestOptions: any) {
   return hasItem(filename);
 }
 
-async function deleteFile(filename) {
+async function deleteFile(filename: string, _requestOptions: any) {
   if (!hasItem(filename)) return true;
   delete fs[filename];
   return true;
 }
 
-function exists(filename) {
+async function exists(filename) {
   return hasItem(filename);
 }
 
@@ -87,34 +98,27 @@ async function clearFileStorage() {
   fs = {};
 }
 
-module.exports = {
-  writeEncrypted,
+export const FS = {
+  writeEncryptedBase64,
   readEncrypted,
   uploadFile: cancellable(uploadFile),
   downloadFile: cancellable(downloadFile),
   deleteFile,
   exists,
-  clearFileStorage
+  clearFileStorage,
+  hashBase64
 };
 
-function cancellable(operation) {
-  return function (filename, requestOptions) {
+function cancellable<T>(
+  operation: (filename: string, requestOptions: any) => Promise<T>
+) {
+  return function (filename: string, requestOptions: any) {
+    const abortController = new AbortController();
     return {
       execute: () => operation(filename, requestOptions),
-      cancel: () => {}
+      cancel: async (message: string) => {
+        abortController.abort(message);
+      }
     };
   };
 }
-
-function hashCode(str) {
-  var hash = 0,
-    i,
-    chr;
-  if (str.length === 0) return hash;
-  for (i = 0; i < str.length; i++) {
-    chr = str.charCodeAt(i);
-    hash = (hash << 5) - hash + chr;
-    hash |= 0; // Convert to 32bit integer
-  }
-  return hash;
-}

packages/core/__mocks__/node-storage.mock.js

@@ -1,94 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import { NNCrypto } from "@notesnook/crypto";
-
-export class NodeStorageInterface {
-  constructor() {
-    this.storage = {};
-    this.crypto = new NNCrypto();
-  }
-
-  async read(key) {
-    return new Promise((resolve) => resolve(this.storage[key]));
-  }
-
-  async readMulti(keys) {
-    return new Promise((resolve) => {
-      const result = [];
-      keys.forEach((key) => {
-        result.push([key, this.storage[key]]);
-      });
-      resolve(result);
-    });
-  }
-
-  async write(key, data) {
-    this.storage[key] = data;
-  }
-
-  async writeMulti(entries) {
-    for (const [key, value] of entries) {
-      this.storage[key] = value;
-    }
-  }
-
-  remove(key) {
-    delete this.storage[key];
-  }
-  clear() {
-    this.storage = {};
-  }
-  getAllKeys() {
-    return Object.keys(this.storage);
-  }
-
-  async encrypt(password, data) {
-    return await this.crypto.encrypt(password, data, "text", "base64");
-  }
-
-  async encryptMulti(password, items) {
-    return await this.crypto.encryptMulti(password, items, "text", "base64");
-  }
-
-  async decrypt(key, cipherData) {
-    cipherData.format = "base64";
-    return await this.crypto.decrypt(key, cipherData, "text");
-  }
-
-  async deriveCryptoKey(name, { password, salt }) {
-    const keyData = await this.crypto.exportKey(password, salt);
-    await this.write(`${name}@_k`, keyData.key);
-  }
-
-  async getCryptoKey(name) {
-    const key = await this.read(`${name}@_k`);
-    if (!key) return;
-    return key;
-  }
-
-  async hash(password, email) {
-    const APP_SALT = "oVzKtazBo7d8sb7TBvY9jw";
-    return await this.crypto.hash(password, `${APP_SALT}${email}`);
-  }
-
-  async generateCryptoKey(password, salt) {
-    return { password, salt };
-  }
-}

packages/core/__mocks__/node-storage.mock.ts

@@ -0,0 +1,108 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Cipher, NNCrypto, SerializedKey } from "@notesnook/crypto";
+
+export class NodeStorageInterface {
+  storage = {};
+  crypto = new NNCrypto();
+
+  async write<T>(key: string, data: T): Promise<void> {
+    this.storage[key] = data;
+  }
+
+  async writeMulti<T>(entries: [key: string, data: T][]) {
+    for (const [key, value] of entries) {
+      this.storage[key] = value;
+    }
+  }
+
+  async readMulti<T>(keys: string[]): Promise<[string, T][]> {
+    const result: [string, T][] = [];
+    keys.forEach((key) => {
+      result.push([key, this.storage[key]]);
+    });
+    return result;
+  }
+
+  async read<T>(
+    key: string,
+    isArray?: boolean | undefined
+  ): Promise<T | undefined> {
+    return this.storage[key];
+  }
+
+  async remove(key: string): Promise<void> {
+    delete this.storage[key];
+  }
+
+  async clear(): Promise<void> {
+    this.storage = {};
+  }
+
+  async getAllKeys(): Promise<string[]> {
+    return Object.keys(this.storage);
+  }
+
+  async encrypt(key: SerializedKey, plainText: string) {
+    return await this.crypto.encrypt(key, plainText, "text", "base64");
+  }
+
+  async encryptMulti(key: SerializedKey, items: string[]) {
+    return await this.crypto.encryptMulti(key, items, "text", "base64");
+  }
+
+  decrypt(key: SerializedKey, cipherData: Cipher<"base64">): Promise<string> {
+    cipherData.format = "base64";
+    return this.crypto.decrypt(key, cipherData, "text");
+  }
+
+  decryptMulti(key: SerializedKey, items: Cipher<"base64">[]) {
+    items.forEach((c) => (c.format = "base64"));
+    return this.crypto.decryptMulti(key, items, "text");
+  }
+
+  async deriveCryptoKey(
+    name: string,
+    credentials: SerializedKey
+  ): Promise<void> {
+    const { password, salt } = credentials;
+    if (!password || !salt) return;
+    const keyData = await this.crypto.exportKey(password, salt);
+    await this.write(`${name}@_k`, keyData.key);
+  }
+
+  async hash(password: string, email: string): Promise<string> {
+    const APP_SALT = "oVzKtazBo7d8sb7TBvY9jw";
+    return await this.crypto.hash(password, `${APP_SALT}${email}`);
+  }
+
+  async getCryptoKey(name: string): Promise<string | undefined> {
+    const key = await this.read<string>(`${name}@_k`);
+    if (!key) return;
+    return key;
+  }
+
+  async generateCryptoKey(
+    password: string,
+    salt?: string | undefined
+  ): Promise<SerializedKey> {
+    return { password, salt };
+  }
+}

packages/core/__tests__/backup.test.js

@@ -33,10 +33,19 @@ import { test, expect, describe } from "vitest";
 test("export backup", () =>
   noteTest().then(() =>
     notebookTest().then(async ({ db }) => {
-      const exp = await db.backup.export("node");
-      let backup = JSON.parse(exp);
+      const exp = [];
+      for await (const file of db.backup.export("node", false)) {
+        exp.push(file);
+      }
+
+      let backup = JSON.parse(exp[1].data);
+      expect(exp.length).toBe(2);
+      expect(exp[0].path).toBe(".nnbackup");
       expect(backup.type).toBe("node");
       expect(backup.date).toBeGreaterThan(0);
+      expect(backup.data).toBeTypeOf("string");
+      expect(backup.compressed).toBe(true);
+      expect(backup.encrypted).toBe(false);
     })
   ));
 
@@ -44,19 +53,34 @@ test("export encrypted backup", () =>
   notebookTest().then(async ({ db }) => {
     await loginFakeUser(db);
     await db.notes.add(TEST_NOTE);
-    const exp = await db.backup.export("node", true);
-    let backup = JSON.parse(exp);
+
+    const exp = [];
+    for await (const file of db.backup.export("node", true)) {
+      exp.push(file);
+    }
+
+    const backup = JSON.parse(exp[1].data);
+    expect(exp.length).toBe(2);
+    expect(exp[0].path).toBe(".nnbackup");
     expect(backup.type).toBe("node");
     expect(backup.date).toBeGreaterThan(0);
     expect(backup.data.iv).not.toBeUndefined();
+    expect(backup.data).toBeTypeOf("object");
+    expect(backup.compressed).toBe(true);
+    expect(backup.encrypted).toBe(true);
   }));
 
 test("import backup", () =>
   notebookTest().then(async ({ db, id }) => {
     await db.notes.add(TEST_NOTE);
-    const exp = await db.backup.export("node");
+
+    const exp = [];
+    for await (const file of db.backup.export("node", false)) {
+      exp.push(file);
+    }
+
     await db.storage.clear();
-    await db.backup.import(JSON.parse(exp));
+    await db.backup.import(JSON.parse(exp[1].data));
     expect(db.notebooks.notebook(id).data.id).toBe(id);
   }));
 
@@ -64,19 +88,29 @@ test("import encrypted backup", () =>
   notebookTest().then(async ({ db, id }) => {
     await loginFakeUser(db);
     await db.notes.add(TEST_NOTE);
-    const exp = await db.backup.export("node", true);
+
+    const exp = [];
+    for await (const file of db.backup.export("node", true)) {
+      exp.push(file);
+    }
+
     await db.storage.clear();
-    await db.backup.import(JSON.parse(exp), "password");
+    await db.backup.import(JSON.parse(exp[1].data), "password");
     expect(db.notebooks.notebook(id).data.id).toBe(id);
   }));
 
 test("import tempered backup", () =>
   notebookTest().then(async ({ db }) => {
     await db.notes.add(TEST_NOTE);
-    const exp = await db.backup.export("node");
+
+    const exp = [];
+    for await (const file of db.backup.export("node", false)) {
+      exp.push(file);
+    }
+
     await db.storage.clear();
-    const backup = JSON.parse(exp);
-    backup.data.hello = "world";
+    const backup = JSON.parse(exp[1].data);
+    backup.data += "hello";
     await expect(db.backup.import(backup)).rejects.toThrow(/tempered/);
   }));
 
@@ -147,21 +181,20 @@ describe.each([
     return databaseTest().then(async (db) => {
       await db.backup.import(qclone(data));
 
-      verifyIndex(data, db, "notes", "notes");
-      verifyIndex(data, db, "notebooks", "notebooks");
-      verifyIndex(data, db, "content", "content");
-      verifyIndex(data, db, "attachments", "attachments");
-      // verifyIndex(data, db, "trash", "trash");
+      const keys = await db.storage.getAllKeys();
+      for (let key in data.data) {
+        const item = data.data[key];
+        if (item && !item.type && item.deleted) continue;
+        if (
+          key.startsWith("_uk_") ||
+          key === "hasConflicts" ||
+          key === "monographs" ||
+          key === "token"
+        )
+          continue;
+
+        expect(keys.some((k) => k.startsWith(key))).toBeTruthy();
+      }
     });
   });
 });
-
-function verifyIndex(backup, db, backupCollection, collection) {
-  if (!backup.data[backupCollection]) return;
-
-  expect(
-    backup.data[backupCollection].every(
-      (v) => db[collection]._collection.indexer.indices.indexOf(v) > -1
-    )
-  ).toBeTruthy();
-}

packages/core/__tests__/utils/index.js

@@ -21,7 +21,7 @@ import DB from "../../src/api";
 import { NodeStorageInterface } from "../../__mocks__/node-storage.mock";
 import dayjs from "dayjs";
 import { groupArray } from "../../src/utils/grouping";
-import FS from "../../__mocks__/fs.mock";
+import { FS } from "../../__mocks__/fs.mock";
 import Compressor from "../../__mocks__/compressor.mock";
 import { expect } from "vitest";
 import EventSource from "eventsource";

packages/core/package-lock.json

@@ -36,7 +36,9 @@
         "abortcontroller-polyfill": "^1.7.3",
         "cross-env": "^7.0.3",
         "dotenv": "^16.0.1",
+        "event-source-polyfill": "^1.0.31",
         "eventsource": "^2.0.2",
+        "hash-wasm": "^4.9.0",
         "isomorphic-fetch": "^3.0.0",
         "jsdom": "^22.1.0",
         "mockdate": "^3.0.5",
@@ -1391,6 +1393,12 @@
         "@esbuild/win32-x64": "0.18.20"
       }
     },
+    "node_modules/event-source-polyfill": {
+      "version": "1.0.31",
+      "resolved": "https://registry.npmjs.org/event-source-polyfill/-/event-source-polyfill-1.0.31.tgz",
+      "integrity": "sha512-4IJSItgS/41IxN5UVAVuAyczwZF7ZIEsM1XAoUzIHA6A+xzusEZUutdXz2Nr+MQPLxfTiCvqE79/C8HT8fKFvA==",
+      "dev": true
+    },
     "node_modules/event-target-shim": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
@@ -1504,6 +1512,12 @@
         "node": ">=8"
       }
     },
+    "node_modules/hash-wasm": {
+      "version": "4.9.0",
+      "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.9.0.tgz",
+      "integrity": "sha512-7SW7ejyfnRxuOc7ptQHSf4LDoZaWOivfzqw+5rpcQku0nHfmicPKE51ra9BiRLAmT8+gGLestr1XroUkqdjL6w==",
+      "dev": true
+    },
     "node_modules/hast-util-parse-selector": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.1.tgz",
@@ -3882,6 +3896,12 @@
         "@esbuild/win32-x64": "0.18.20"
       }
     },
+    "event-source-polyfill": {
+      "version": "1.0.31",
+      "resolved": "https://registry.npmjs.org/event-source-polyfill/-/event-source-polyfill-1.0.31.tgz",
+      "integrity": "sha512-4IJSItgS/41IxN5UVAVuAyczwZF7ZIEsM1XAoUzIHA6A+xzusEZUutdXz2Nr+MQPLxfTiCvqE79/C8HT8fKFvA==",
+      "dev": true
+    },
     "event-target-shim": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
@@ -3967,6 +3987,12 @@
       "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true
     },
+    "hash-wasm": {
+      "version": "4.9.0",
+      "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.9.0.tgz",
+      "integrity": "sha512-7SW7ejyfnRxuOc7ptQHSf4LDoZaWOivfzqw+5rpcQku0nHfmicPKE51ra9BiRLAmT8+gGLestr1XroUkqdjL6w==",
+      "dev": true
+    },
     "hast-util-parse-selector": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.1.tgz",

packages/core/package.json

@@ -17,7 +17,9 @@
     "abortcontroller-polyfill": "^1.7.3",
     "cross-env": "^7.0.3",
     "dotenv": "^16.0.1",
+    "event-source-polyfill": "^1.0.31",
     "eventsource": "^2.0.2",
+    "hash-wasm": "^4.9.0",
     "isomorphic-fetch": "^3.0.0",
     "jsdom": "^22.1.0",
     "mockdate": "^3.0.5",

packages/core/src/api/migrations.js

@@ -66,11 +66,7 @@ class Migrations {
           dbCollection: this._db.colors
         },
         {
-          index: () => this._db.trash.raw,
-          dbCollection: this._db.trash
-        },
-        {
-          index: () => this._db.content.all(),
+          iterate: true,
           dbCollection: this._db.content
         },
         {
@@ -91,11 +87,11 @@ class Migrations {
           dbCollection: this._db.relations
         },
         {
-          index: () => this._db.noteHistory.sessionContent.all(),
+          iterate: true,
           dbCollection: this._db.noteHistory
         },
         {
-          index: () => this._db.noteHistory.sessionContent.all(),
+          iterate: true,
           dbCollection: this._db.noteHistory.sessionContent
         },
         {

packages/core/src/api/sync/__tests__/sync.test.skip.js

@@ -1,399 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import Database from "../../index";
-import { NodeStorageInterface } from "../../../../__mocks__/node-storage.mock";
-import FS from "../../../../__mocks__/fs.mock";
-import Compressor from "../../../../__mocks__/compressor.mock";
-import { CHECK_IDS, EV, EVENTS } from "../../../common";
-import EventSource from "eventsource";
-import { delay } from "../../../../__tests__/utils";
-
-jest.setTimeout(100 * 1000);
-
-test("case 1: device A & B should only download the changes from device C (no uploading)", async () => {
-  const types = [];
-  function onSyncProgress({ type }) {
-    types.push(type);
-  }
-
-  const deviceA = await initializeDevice("deviceA");
-  const deviceB = await initializeDevice("deviceB");
-
-  deviceA.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
-  deviceB.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
-
-  const deviceC = await initializeDevice("deviceC");
-
-  await deviceC.notes.add({ title: "new note 1" });
-  await syncAndWait(deviceC, deviceC);
-
-  expect(types.every((t) => t === "download")).toBe(true);
-
-  await cleanup(deviceA, deviceB, deviceC);
-});
-
-test("case 3: Device A & B have unsynced changes but server has nothing", async () => {
-  const deviceA = await initializeDevice("deviceA");
-  const deviceB = await initializeDevice("deviceB");
-
-  const note1Id = await deviceA.notes.add({
-    title: "Test note from device A"
-  });
-  const note2Id = await deviceB.notes.add({
-    title: "Test note from device B"
-  });
-
-  await syncAndWait(deviceA, deviceB);
-
-  expect(deviceA.notes.note(note2Id)).toBeTruthy();
-  expect(deviceB.notes.note(note1Id)).toBeTruthy();
-  expect(deviceA.notes.note(note1Id)).toBeTruthy();
-  expect(deviceB.notes.note(note2Id)).toBeTruthy();
-
-  await cleanup(deviceA, deviceA);
-});
-
-// test(
-//   "case 4: Device A's sync is interrupted halfway and Device B makes some changes afterwards and syncs.",
-//   async () => {
-//     const deviceA = await initializeDevice("deviceA");
-//     const deviceB = await initializeDevice("deviceB");
-
-//     const unsyncedNoteIds = [];
-//     for (let i = 0; i < 10; ++i) {
-//       const id = await deviceA.notes.add({
-//         title: `Test note ${i} from device A`,
-//       });
-//       unsyncedNoteIds.push(id);
-//     }
-
-//     const half = unsyncedNoteIds.length / 2 + 1;
-//     deviceA.eventManager.subscribe(
-//       EVENTS.syncProgress,
-//       async ({ type, current }) => {
-//         if (type === "upload" && current === half) {
-//           await deviceA.syncer.stop();
-//         }
-//       }
-//     );
-
-//     await expect(deviceA.sync(true)).rejects.toThrow();
-
-//     let syncedNoteIds = [];
-//     for (let i = 0; i < unsyncedNoteIds.length; ++i) {
-//       const expectedNoteId = unsyncedNoteIds[i];
-//       if (deviceB.notes.note(expectedNoteId))
-//         syncedNoteIds.push(expectedNoteId);
-//     }
-//     expect(
-//       syncedNoteIds.length === half - 1 || syncedNoteIds.length === half
-//     ).toBe(true);
-
-//     const deviceBNoteId = await deviceB.notes.add({
-//       title: "Test note of case 4 from device B",
-//     });
-
-//     await deviceB.sync(true);
-
-//     await syncAndWait(deviceA, deviceB);
-
-//     expect(deviceA.notes.note(deviceBNoteId)).toBeTruthy();
-//     expect(
-//       unsyncedNoteIds
-//         .map((id) => !!deviceB.notes.note(id))
-//         .every((res) => res === true)
-//     ).toBe(true);
-
-//     await cleanup(deviceA, deviceB);
-//   },
-//
-// );
-
-// test.only(
-//   "case 5: Device A's sync is interrupted halfway and Device B makes changes on the same note's content that didn't get synced on Device A due to interruption.",
-//   async () => {
-//     const deviceA = await initializeDevice("deviceA");
-//     const deviceB = await initializeDevice("deviceB");
-
-//     const noteIds = [];
-//     for (let i = 0; i < 10; ++i) {
-//       const id = await deviceA.notes.add({
-//         content: {
-//           type: "tiptap",
-//           data: `<p>deviceA=true</p>`,
-//         },
-//       });
-//       noteIds.push(id);
-//     }
-
-//     await deviceA.sync(true);
-//     await deviceB.sync(true);
-
-//     const unsyncedNoteIds = [];
-//     for (let id of noteIds) {
-//       const noteId = await deviceA.notes.add({
-//         id,
-//         content: {
-//           type: "tiptap",
-//           data: `<p>deviceA=true+changed=true</p>`,
-//         },
-//       });
-//       unsyncedNoteIds.push(noteId);
-//     }
-
-//     deviceA.eventManager.subscribe(
-//       EVENTS.syncProgress,
-//       async ({ type, total, current }) => {
-//         const half = total / 2 + 1;
-//         if (type === "upload" && current === half) {
-//           await deviceA.syncer.stop();
-//         }
-//       }
-//     );
-
-//     await expect(deviceA.sync(true)).rejects.toThrow();
-
-//     await delay(10 * 1000);
-
-//     for (let id of unsyncedNoteIds) {
-//       await deviceB.notes.add({
-//         id,
-//         content: {
-//           type: "tiptap",
-//           data: "<p>changes from device B</p>",
-//         },
-//       });
-//     }
-
-//     const error = await withError(async () => {
-//       await deviceB.sync(true);
-//       await deviceA.sync(true);
-//     });
-
-//     expect(error).not.toBeInstanceOf(NoErrorThrownError);
-//     expect(error.message.includes("Merge")).toBeTruthy();
-
-//     await cleanup(deviceA, deviceB);
-//   },
-//
-// );
-
-test("issue: running force sync from device A makes device B always download everything", async () => {
-  const deviceA = await initializeDevice("deviceA");
-  const deviceB = await initializeDevice("deviceB");
-
-  await syncAndWait(deviceA, deviceB, true);
-
-  const handler = jest.fn();
-  deviceB.eventManager.subscribe(EVENTS.syncProgress, handler);
-
-  await deviceB.sync(true);
-
-  expect(handler).not.toHaveBeenCalled();
-
-  await cleanup(deviceB);
-});
-
-test("issue: colors are not properly created if multiple notes are synced together", async () => {
-  const deviceA = await initializeDevice("deviceA", [CHECK_IDS.noteColor]);
-  const deviceB = await initializeDevice("deviceB", [CHECK_IDS.noteColor]);
-
-  const noteIds = [];
-  for (let i = 0; i < 3; ++i) {
-    const id = await deviceA.notes.add({
-      content: {
-        type: "tiptap",
-        data: `<p>deviceA=true</p>`
-      }
-    });
-    noteIds.push(id);
-  }
-
-  await syncAndWait(deviceA, deviceB);
-
-  for (let noteId of noteIds) {
-    await deviceA.notes.note(noteId).color("purple");
-    expect(deviceB.notes.note(noteId)).toBeTruthy();
-    expect(deviceB.notes.note(noteId).data.color).toBeUndefined();
-  }
-
-  await syncAndWait(deviceA, deviceB);
-
-  await delay(2000);
-
-  const purpleColor = deviceB.colors.tag("purple");
-  expect(noteIds.every((id) => purpleColor.noteIds.indexOf(id) > -1)).toBe(
-    true
-  );
-
-  await cleanup(deviceA, deviceB);
-});
-
-test("issue: new topic on device A gets replaced by the new topic on device B", async () => {
-  const deviceA = await initializeDevice("deviceA");
-  const deviceB = await initializeDevice("deviceB");
-
-  const id = await deviceA.notebooks.add({ title: "Notebook 1" });
-
-  await syncAndWait(deviceA, deviceB, false);
-
-  expect(deviceB.notebooks.notebook(id)).toBeDefined();
-
-  await deviceA.notebooks.notebook(id).topics.add("Topic 1");
-
-  // to create a conflict
-  await delay(1500);
-
-  await deviceB.notebooks.notebook(id).topics.add("Topic 2");
-
-  expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
-
-  expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
-
-  await syncAndWait(deviceA, deviceB, false);
-
-  await delay(1000);
-
-  await syncAndWait(deviceB, deviceB, false);
-
-  expect(deviceA.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
-  expect(deviceB.notebooks.notebook(id).topics.has("Topic 1")).toBeTruthy();
-
-  expect(deviceA.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
-  expect(deviceB.notebooks.notebook(id).topics.has("Topic 2")).toBeTruthy();
-
-  await cleanup(deviceA, deviceB);
-});
-
-test("issue: remove notebook reference from notes that are removed from topic during merge", async () => {
-  const deviceA = await initializeDevice("deviceA");
-  const deviceB = await initializeDevice("deviceB");
-
-  const id = await deviceA.notebooks.add({
-    title: "Notebook 1",
-    topics: ["Topic 1"]
-  });
-
-  await syncAndWait(deviceA, deviceB, false);
-
-  expect(deviceB.notebooks.notebook(id)).toBeDefined();
-
-  const noteA = await deviceA.notes.add({ title: "Note 1" });
-  await deviceA.notes.addToNotebook({ id, topic: "Topic 1" }, noteA);
-
-  expect(
-    deviceA.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
-  ).toBe(1);
-
-  await delay(2000);
-
-  const noteB = await deviceB.notes.add({ title: "Note 2" });
-  await deviceB.notes.addToNotebook({ id, topic: "Topic 1" }, noteB);
-
-  expect(
-    deviceB.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
-  ).toBe(1);
-
-  await syncAndWait(deviceB, deviceA, false);
-
-  expect(
-    deviceA.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
-  ).toBe(1);
-  expect(
-    deviceB.notebooks.notebook(id).topics.topic("Topic 1").totalNotes
-  ).toBe(1);
-
-  expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(0);
-
-  await cleanup(deviceA, deviceB);
-});
-
-/**
- *
- * @param {string} id
- * @returns {Promise<Database>}
- */
-async function initializeDevice(id, capabilities = []) {
-  console.time("Init device");
-  EV.subscribe(EVENTS.userCheckStatus, async (type) => {
-    return {
-      type,
-      result: capabilities.indexOf(type) > -1
-    };
-  });
-
-  const device = new Database(
-    new NodeStorageInterface(),
-    EventSource,
-    FS,
-    Compressor
-  );
-  // device.host({
-  //   API_HOST: "http://192.168.10.29:5264",
-  //   AUTH_HOST: "http://192.168.10.29:8264",
-  //   SSE_HOST: "http://192.168.10.29:7264",
-  //   ISSUES_HOST: "http://192.168.10.29:2624",
-  //   SUBSCRIPTIONS_HOST: "http://192.168.10.29:9264",
-  // });
-
-  await device.init(id);
-
-  await device.user.login(
-    process.env.EMAIL,
-    process.env.PASSWORD,
-    process.env.HASHED_PASSWORD
-  );
-
-  await device.user.resetUser(false);
-
-  device.eventManager.subscribe(
-    EVENTS.databaseSyncRequested,
-    async (full, force) => {
-      await device.sync(full, force);
-    }
-  );
-
-  console.timeEnd("Init device");
-  return device;
-}
-
-async function cleanup(...devices) {
-  for (let device of devices) {
-    await device.user.logout();
-    device.eventManager.unsubscribeAll();
-  }
-  EV.unsubscribeAll();
-}
-
-/**
- *
- * @param {Database} deviceA
- * @param {Database} deviceB
- * @returns
- */
-function syncAndWait(deviceA, deviceB, force = false) {
-  return new Promise((resolve) => {
-    const ref = deviceB.eventManager.subscribe(EVENTS.syncCompleted, () => {
-      ref.unsubscribe();
-      resolve();
-    });
-    deviceA.sync(true, force);
-  });
-}

packages/core/src/api/sync/index.js

@@ -315,16 +315,16 @@ class Sync {
   }
 
   async stop(lastSynced) {
+    // refresh topic references
+    this.db.notes.topicReferences.rebuild();
+    // refresh monographs on sync completed
+    await this.db.monographs.init();
+
     this.logger.info("Stopping sync", { lastSynced });
     const storedLastSynced = await this.db.lastSynced();
     if (lastSynced > storedLastSynced)
       await this.db.storage.write("lastSynced", lastSynced);
     this.db.eventManager.publish(EVENTS.syncCompleted);
-
-    // refresh monographs on sync completed
-    await this.db.monographs.init();
-    // refresh topic references
-    this.db.notes.topicReferences.rebuild();
   }
 
   async cancel() {
@@ -361,13 +361,19 @@ class Sync {
   /**
    * @private
    */
-  onPushCompleted(lastSynced) {
+  async onPushCompleted(lastSynced) {
+    // refresh topic references
+    this.db.notes.topicReferences.rebuild();
+
     this.db.eventManager.publish(
       EVENTS.databaseSyncRequested,
       false,
       false,
       lastSynced
     );
+
+    // refresh monographs on sync completed
+    await this.db.monographs.init();
   }
 
   async processChunk(chunk, key, dbLastSynced, notify = false) {
@@ -412,8 +418,9 @@ class Sync {
     }
 
     const collectionType = this.itemTypeToCollection[chunk.type];
-    if (collectionType && this.db[collectionType])
+    if (collectionType && this.db[collectionType]) {
       await this.db[collectionType]._collection.setItems(items);
+    }
   }
 
   /**
@@ -469,7 +476,13 @@ async function deserializeItem(decryptedItem, version, database) {
   deserialized.synced = true;
 
   if (!deserialized.alg && !deserialized.cipher) {
-    await migrateItem(deserialized, version, deserialized.type, database);
+    await migrateItem(
+      deserialized,
+      version,
+      deserialized.type,
+      database,
+      "sync"
+    );
   }
   return deserialized;
 }

packages/core/src/api/sync/merger.js

@@ -182,6 +182,7 @@ class Merger {
             remoteItem.noteIds,
             noteIds
           );
+          remoteItem.remote = false;
         }
         return this._db.attachments.merge(localItem, remoteItem);
       }

packages/core/src/api/user-manager.js

@@ -152,6 +152,7 @@ class UserManager {
       password,
       salt: user.salt
     });
+    await this._storage.write("lastSynced", 0);
 
     EV.publish(EVENTS.userLoggedIn, user);
   }
@@ -183,6 +184,7 @@ class UserManager {
       password,
       salt: user.salt
     });
+    await this._storage.write("lastSynced", 0);
 
     EV.publish(EVENTS.userLoggedIn, user);
   }

packages/core/src/collections/attachments.js

@@ -50,6 +50,7 @@ export default class Attachments extends Collection {
         remoteAttachment.noteIds,
         localAttachment.noteIds
       );
+      remoteAttachment.remote = false;
     }
 
     return remoteAttachment;

packages/core/src/collections/notebooks.js

@@ -26,6 +26,15 @@ import qclone from "qclone";
 export default class Notebooks extends Collection {
   merge(localNotebook, remoteNotebook, lastSyncedTimestamp) {
     if (remoteNotebook.deleted) return remoteNotebook;
+
+    if (
+      localNotebook &&
+      (localNotebook.type === "trash" || localNotebook.deleted)
+    ) {
+      if (localNotebook.dateModified > remoteNotebook.dateModified) return;
+      return remoteNotebook;
+    }
+
     if (localNotebook && localNotebook.topics?.length) {
       let isChanged = false;
       // merge new and old topics
@@ -82,7 +91,8 @@ export default class Notebooks extends Collection {
 
     let notebook = {
       ...oldNotebook,
-      ...notebookArg
+      ...notebookArg,
+      topics: oldNotebook?.topics || []
     };
 
     if (!notebook.title) throw new Error("Notebook must contain a title.");
@@ -102,8 +112,8 @@ export default class Notebooks extends Collection {
 
     await this._collection.addItem(notebook);
 
-    if (!oldNotebook) {
-      await this.notebook(notebook).topics.add(...notebook.topics);
+    if (!oldNotebook && notebookArg.topics) {
+      await this.notebook(id).topics.add(...notebookArg.topics);
     }
     return id;
   }

packages/core/src/collections/topics.js

@@ -86,7 +86,7 @@ export default class Topics {
         notebook.topics.push(topic);
       }
     }
-    return this._db.notebooks.add(notebook);
+    return this._db.notebooks._collection.updateItem(notebook);
   }
 
   /**
@@ -110,7 +110,8 @@ export default class Topics {
   }
 
   async delete(...topicIds) {
-    let allTopics = qclone(this.all);
+    const notebook = qclone(this._db.notebooks.notebook(this._notebookId).data);
+    let allTopics = notebook.topics;
 
     for (let topicId of topicIds) {
       const topic = this.topic(topicId);
@@ -125,7 +126,7 @@ export default class Topics {
       allTopics.splice(topicIndex, 1);
     }
 
-    await this._db.notebooks.add({ id: this._notebookId, topics: allTopics });
+    await this._db.notebooks._collection.updateItem(notebook);
   }
 }
 

packages/core/src/common.js

@@ -125,4 +125,4 @@ export const DATE_FORMATS = [
 
 export const TIME_FORMATS = ["12-hour", "24-hour"];
 
-export const CURRENT_DATABASE_VERSION = 5.8;
+export const CURRENT_DATABASE_VERSION = 5.9;

packages/core/src/database/backup.js

@@ -20,9 +20,63 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import SparkMD5 from "spark-md5";
 import { CURRENT_DATABASE_VERSION } from "../common.js";
 import Migrator from "./migrator.js";
+import { toChunks } from "../utils/array.js";
+import { migrateItem } from "../migrations.js";
+import Indexer from "./indexer.js";
+
+const COLORS = [
+  "red",
+  "orange",
+  "yellow",
+  "green",
+  "blue",
+  "purple",
+  "gray",
+  "black",
+  "white"
+];
+
+const invalidKeys = [
+  "user",
+  "t",
+  "v",
+  "lastBackupTime",
+  "lastSynced",
+  // all indexes
+  "notes",
+  "notebooks",
+  "content",
+  "tags",
+  "colors",
+  "attachments",
+  "relations",
+  "reminders",
+  "sessioncontent",
+  "notehistory",
+  "shortcuts",
+  "vaultKey",
+  "hasConflict",
+  "token",
+  "monographs"
+];
+
+const itemTypeToCollectionKey = {
+  note: "notes",
+  notebook: "notebooks",
+  tiptap: "content",
+  tiny: "content",
+  tag: "tags",
+  color: "colors",
+  attachment: "attachments",
+  relation: "relations",
+  reminder: "reminders",
+  sessioncontent: "sessioncontent",
+  session: "notehistory",
+  notehistory: "notehistory",
+  content: "content",
+  shortcut: "shortcuts"
+};
 
-const invalidKeys = ["user", "t", "v", "lastBackupTime", "lastSynced"];
-const invalidIndices = ["tags", "colors"];
 const validTypes = ["mobile", "web", "node"];
 export default class Backup {
   /**
@@ -46,35 +100,74 @@ export default class Backup {
    * @param {"web"|"mobile"|"node"} type
    * @param {boolean} encrypt
    */
-  async export(type, encrypt = false) {
+  async *export(type, encrypt = false) {
     if (!validTypes.some((t) => t === type))
       throw new Error("Invalid type. It must be one of 'mobile' or 'web'.");
     if (encrypt && !(await this._db.user.getUser()))
       throw new Error("Please login to create encrypted backups.");
 
+    yield {
+      path: ".nnbackup",
+      data: ""
+    };
+
     let keys = await this._db.storage.getAllKeys();
-    let data = filterData(
-      Object.fromEntries(await this._db.storage.readMulti(keys))
-    );
+    const key = await this._db.user.getEncryptionKey();
+    const chunks = toChunks(keys, 20);
+    let buffer = [];
+    let bufferLength = 0;
+    const MAX_CHUNK_SIZE = 10 * 1024 * 1024;
+    let chunkIndex = 0;
 
-    let hash = {};
+    while (chunks.length > 0) {
+      const chunk = chunks.pop();
 
-    if (encrypt) {
-      const key = await this._db.user.getEncryptionKey();
-      data = await this._db.storage.encrypt(key, JSON.stringify(data));
-    } else {
-      hash = { hash: SparkMD5.hash(JSON.stringify(data)), hash_type: "md5" };
+      const items = await this._db.storage.readMulti(chunk);
+      items.forEach(([id, item]) => {
+        if (
+          invalidKeys.includes(id) ||
+          (item.deleted && !item.type) ||
+          id.startsWith("_uk_")
+        )
+          return;
+
+        const data = JSON.stringify(item);
+        buffer.push(data);
+        bufferLength += data.length;
+      });
+
+      if (bufferLength >= MAX_CHUNK_SIZE || chunks.length === 0) {
+        let itemsJSON = `[${buffer.join(",")}]`;
+
+        buffer = [];
+        bufferLength = 0;
+
+        itemsJSON = await this._db.compressor.compress(itemsJSON);
+
+        const hash = SparkMD5.hash(itemsJSON);
+
+        if (encrypt) itemsJSON = await this._db.storage.encrypt(key, itemsJSON);
+
+        yield {
+          path: `${chunkIndex++}-${encrypt ? "encrypted" : "plain"}-${hash}`,
+          data: `{
+"version": ${CURRENT_DATABASE_VERSION},
+"type": "${type}",
+"date": ${Date.now()},
+"data": ${JSON.stringify(itemsJSON)},
+"hash": "${hash}",
+"hash_type": "md5",
+"compressed": true,
+"encrypted": ${encrypt ? "true" : "false"}
+}`
+        };
+      }
     }
 
-    // save backup time
+    if (bufferLength > 0 || buffer.length > 0)
+      throw new Error("Buffer not empty.");
+
     await this.updateBackupTime();
-    return JSON.stringify({
-      version: CURRENT_DATABASE_VERSION,
-      type,
-      date: Date.now(),
-      data,
-      ...hash
-    });
   }
 
   /**
@@ -90,7 +183,7 @@ export default class Backup {
 
     let db = backup.data;
     const isEncrypted = db.salt && db.iv && db.cipher;
-    if (isEncrypted) {
+    if (backup.encrypted || isEncrypted) {
       if (!password)
         throw new Error(
           "Please provide a password to decrypt this backup & restore it."
@@ -101,8 +194,7 @@ export default class Backup {
         throw new Error("Could not generate encryption key for backup.");
 
       try {
-        const decrypted = await this._db.storage.decrypt(key, db);
-        backup.data = JSON.parse(decrypted);
+        backup.data = await this._db.storage.decrypt(key, db);
       } catch (e) {
         if (
           e.message.includes("ciphertext cannot be decrypted") ||
@@ -112,9 +204,16 @@ export default class Backup {
 
         throw new Error(`Could not decrypt backup: ${e.message}`);
       }
-    } else if (!this._verify(backup))
+    }
+
+    if (backup.hash && !this._verify(backup))
       throw new Error("Backup file has been tempered, aborting...");
 
+    if (backup.compressed)
+      backup.data = await this._db.compressor.decompress(backup.data);
+    backup.data =
+      typeof backup.data === "string" ? JSON.parse(backup.data) : backup.data;
+
     await this._migrateData(backup);
   }
 
@@ -122,11 +221,12 @@ export default class Backup {
     const { version = 0 } = backup;
     if (version > CURRENT_DATABASE_VERSION)
       throw new Error(
-        "This backup was made from a newer version of Notesnook. Cannot migrate."
+        "This backup was made from a newer version of Notesnook. Cannot restore."
       );
 
     switch (version) {
       case CURRENT_DATABASE_VERSION:
+      case 5.8:
       case 5.7:
       case 5.6:
       case 5.5:
@@ -145,66 +245,38 @@ export default class Backup {
   async _migrateData(backup) {
     const { data, version = 0 } = backup;
 
-    if (version > CURRENT_DATABASE_VERSION)
-      throw new Error(
-        "This backup was made from a newer version of Notesnook. Cannot migrate."
-      );
+    const toAdd = {};
+    for (const item of Array.isArray(data) ? data : Object.values(data)) {
+      // we do not want to restore deleted items
+      if (!item || (!item.type && item.deleted)) continue;
+      // in v5.6 of the database, we did not set note history session's type
+      if (!item.type && item.sessionContentId) item.type = "notehistory";
 
-    const collections = [
-      {
-        index: () => data["attachments"],
-        dbCollection: this._db.attachments
-      },
-      {
-        index: () => data["notebooks"],
-        dbCollection: this._db.notebooks
-      },
-      {
-        index: () => data["content"],
-        dbCollection: this._db.content
-      },
-      {
-        index: () => data["shortcuts"],
-        dbCollection: this._db.shortcuts
-      },
-      {
-        index: () => data["reminders"],
-        dbCollection: this._db.reminders
-      },
-      {
-        index: () => data["relations"],
-        dbCollection: this._db.relations
-      },
-      {
-        index: () => data["notehistory"],
-        dbCollection: this._db.noteHistory,
-        type: "notehistory"
-      },
-      {
-        index: () => data["sessioncontent"],
-        dbCollection: this._db.noteHistory.sessionContent,
-        type: "sessioncontent"
-      },
-      {
-        index: () => data["notes"],
-        dbCollection: this._db.notes
-      },
-      {
-        index: () => ["settings"],
-        dbCollection: this._db.settings,
-        type: "settings"
-      }
-    ];
+      await migrateItem(item, version, item.type, this._db, "backup");
+      // since items in trash can have their own set of migrations,
+      // we have to run the migration again to account for that.
+      if (item.type === "trash" && item.itemType)
+        await migrateItem(item, version, item.itemType, this._db, "backup");
 
-    await this._db.syncer.acquireLock(async () => {
-      await this._migrator.migrate(
-        this._db,
-        collections,
-        (id, type) => (version < 5.8 ? data[id] : data[`${id}_${type}`]),
-        version,
-        true
-      );
-    });
+      // colors are naively of type "tag" instead of "color" so we have to fix that.
+      const itemType =
+        item.type === "tag" && COLORS.includes(item.title.toLowerCase())
+          ? "color"
+          : item.itemType || item.type;
+
+      const collectionKey = itemTypeToCollectionKey[itemType];
+      if (collectionKey) {
+        toAdd[collectionKey] = toAdd[collectionKey] || [];
+        toAdd[collectionKey].push([item.id, item]);
+      } else if (item.type === "settings")
+        await this._db.storage.write("settings", item);
+    }
+
+    for (const collectionKey in toAdd) {
+      const indexer = new Indexer(this._db.storage, collectionKey);
+      await indexer.init();
+      await indexer.writeMulti(toAdd[collectionKey]);
+    }
   }
 
   _validate(backup) {
@@ -217,10 +289,10 @@ export default class Backup {
   }
 
   _verify(backup) {
-    const { hash, hash_type, data: db } = backup;
+    const { compressed, hash, hash_type, data: db } = backup;
     switch (hash_type) {
       case "md5": {
-        return hash === SparkMD5.hash(JSON.stringify(db));
+        return hash === SparkMD5.hash(compressed ? db : JSON.stringify(db));
       }
       default: {
         return false;
@@ -228,15 +300,3 @@ export default class Backup {
     }
   }
 }
-
-function filterData(data) {
-  let skippedKeys = [...invalidKeys, ...invalidIndices];
-  invalidIndices.forEach((key) => {
-    const index = data[key];
-    if (!index) return;
-    skippedKeys.push(...index);
-  });
-
-  skippedKeys.forEach((key) => delete data[key]);
-  return data;
-}

packages/core/src/database/indexed-collection.js

@@ -109,7 +109,19 @@ export default class IndexedCollection {
   }
 
   setItems(items) {
-    return this.indexer.writeMulti(items);
+    const entries = items.reduce((array, item) => {
+      if (!item) return array;
+
+      if (!item.remote) {
+        item.dateModified = Date.now();
+        item.synced = false;
+      }
+      delete item.remote;
+
+      array.push([item.id, item]);
+      return array;
+    }, []);
+    return this.indexer.writeMulti(entries);
   }
 
   async getEncryptionKey() {

packages/core/src/database/indexer.js

@@ -56,7 +56,6 @@ export default class Indexer extends Storage {
   }
 
   read(key, isArray = false) {
-    if (!this.exists(key)) return;
     return super.read(this.makeId(key), isArray);
   }
 
@@ -69,9 +68,7 @@ export default class Indexer extends Storage {
   }
 
   async readMulti(keys) {
-    const entries = await super.readMulti(
-      keys.filter(this.exists, this).map(this.makeId, this)
-    );
+    const entries = await super.readMulti(keys.map(this.makeId, this));
     entries.forEach((entry) => {
       entry[0] = entry[0].replace(`_${this.type}`, "");
     });
@@ -84,14 +81,12 @@ export default class Indexer extends Storage {
    * @returns
    */
   async writeMulti(items) {
-    const entries = items.reduce((array, item) => {
-      if (!item) return array;
-      if (!this.indices.includes(item.id)) this.indices.push(item.id);
-      array.push([this.makeId(item.id), item]);
-      return array;
-    }, []);
+    const entries = items.map(([id, item]) => {
+      if (!this.indices.includes(id)) this.indices.push(id);
+      return [this.makeId(id), item];
+    });
+    entries.push([this.type, this.indices]);
     await super.writeMulti(entries);
-    await super.write(this.type, this.indices);
   }
 
   async migrateIndices() {

packages/core/src/database/migrator.js

@@ -21,9 +21,13 @@ import { sendMigrationProgressEvent } from "../common";
 import { migrateCollection, migrateItem } from "../migrations";
 
 class Migrator {
-  async migrate(db, collections, get, version, restore = false) {
+  async migrate(db, collections, get, version) {
     for (let collection of collections) {
-      if (!collection.index || !collection.dbCollection) continue;
+      if (
+        (!collection.iterate && !collection.index) ||
+        !collection.dbCollection
+      )
+        continue;
 
       if (collection.dbCollection.collectionName)
         sendMigrationProgressEvent(
@@ -35,61 +39,85 @@ class Migrator {
 
       await migrateCollection(collection.dbCollection, version);
 
-      const index = (await collection.index()) || [];
-      const toAdd = [];
-      for (var i = 0; i < index.length; ++i) {
-        let id = index[i];
-        let item = get(id, collection.dbCollection.collectionName);
-        if (!item) {
-          continue;
-        }
-
-        // check if item is permanently deleted or just a soft delete
-        if (item.deleted && !item.type) {
-          await collection.dbCollection?._collection?.addItem(item);
-          continue;
-        }
-
-        const itemId = item.id;
-        const migrated = await migrateItem(
-          item,
-          version,
-          item.type || collection.type || collection.dbCollection.type,
-          db
+      if (collection.index) {
+        await this.migrateItems(
+          db,
+          collection,
+          collection.index(),
+          get,
+          version
         );
-
-        if (migrated || restore) {
-          if (collection.type === "settings") {
-            await collection.dbCollection.merge(item);
-          } else if (item.type === "note") {
-            toAdd.push(await db.notes.merge(null, item));
-          } else if (collection.dbCollection._collection) {
-            toAdd.push(item);
-          } else {
-            throw new Error(
-              `No idea how to handle this kind of item: ${item.type}.`
-            );
-          }
-
-          // if id changed after migration, we need to delete the old one.
-          if (item.id !== itemId) {
-            await collection.dbCollection._collection.deleteItem(itemId);
-          }
-        }
-      }
-
-      if (toAdd.length > 0) {
-        await collection.dbCollection._collection.setItems(toAdd);
-        if (collection.dbCollection.collectionName)
-          sendMigrationProgressEvent(
-            db.eventManager,
-            collection.dbCollection.collectionName,
-            toAdd.length,
-            toAdd.length
+      } else if (collection.iterate) {
+        for await (const index of collection.dbCollection._collection.iterate(
+          100
+        )) {
+          await this.migrateItems(
+            db,
+            collection,
+            index.map((item) => item[1]),
+            get,
+            version
           );
+        }
       }
     }
     return true;
   }
+
+  async migrateItems(db, collection, index, get, version) {
+    const toAdd = [];
+    for (var i = 0; i < index.length; ++i) {
+      let id = index[i];
+      let item = get(id, collection.dbCollection.collectionName);
+      if (!item) {
+        continue;
+      }
+
+      // check if item is permanently deleted or just a soft delete
+      if (item.deleted && !item.type) {
+        await collection.dbCollection?._collection?.addItem(item);
+        continue;
+      }
+
+      const itemId = item.id;
+      const migrated = await migrateItem(
+        item,
+        version,
+        item.type || collection.type || collection.dbCollection.type,
+        db,
+        "local"
+      );
+
+      if (migrated) {
+        if (collection.type === "settings") {
+          await collection.dbCollection.merge(item);
+        } else if (item.type === "note") {
+          toAdd.push(await db.notes.merge(null, item));
+        } else if (collection.dbCollection._collection) {
+          toAdd.push(item);
+        } else {
+          throw new Error(
+            `No idea how to handle this kind of item: ${item.type}.`
+          );
+        }
+
+        // if id changed after migration, we need to delete the old one.
+        if (item.id !== itemId) {
+          await collection.dbCollection._collection.deleteItem(itemId);
+        }
+      }
+    }
+
+    if (toAdd.length > 0) {
+      await collection.dbCollection._collection.setItems(toAdd);
+      if (collection.dbCollection.collectionName)
+        sendMigrationProgressEvent(
+          db.eventManager,
+          collection.dbCollection.collectionName,
+          toAdd.length,
+          toAdd.length
+        );
+    }
+  }
 }
 export default Migrator;

packages/core/src/migrations.js

@@ -133,10 +133,27 @@ const migrations = [
       }
     }
   },
-  { version: 5.8, items: {} }
+  {
+    version: 5.8,
+    items: {
+      all: (item, _db, migrationType) => {
+        if (migrationType === "local") {
+          delete item.remote;
+          return true;
+        }
+      }
+    }
+  },
+  { version: 5.9, items: {} }
 ];
 
-export async function migrateItem(item, version, type, database) {
+export async function migrateItem(
+  item,
+  version,
+  type,
+  database,
+  migrationType
+) {
   let migrationStartIndex = migrations.findIndex((m) => m.version === version);
   if (migrationStartIndex <= -1) {
     throw new Error(
@@ -151,9 +168,15 @@ export async function migrateItem(item, version, type, database) {
     const migration = migrations[migrationStartIndex];
     if (migration.version === CURRENT_DATABASE_VERSION) break;
 
-    const itemMigrator = migration.items && migration.items[type];
+    if (
+      migration.items.all &&
+      (await migration.items.all(item, database, migrationType))
+    )
+      count++;
+
+    const itemMigrator = migration.items[type];
     if (!itemMigrator) continue;
-    if (await itemMigrator(item, database)) count++;
+    if (await itemMigrator(item, database, migrationType)) count++;
   }
 
   return count > 0;

packages/editor/src/extensions/code-block/component.tsx

@@ -73,11 +73,11 @@ export function CodeblockComponent(
 
               "::selection,*::selection": {
                 bg: "background-selected",
-                color: "paragraph-selected"
+                color: "inherit"
               },
               "::-moz-selection,*::-moz-selection": {
                 bg: "background-selected",
-                color: "paragraph-selected"
+                color: "inherit"
               }
             },
             fontFamily: "monospace",
@@ -202,7 +202,7 @@ export function CodeblockComponent(
           location: "top",
           yOffset: 5
         }}
-        title="Change code block language"
+        title="Select language"
       >
         <LanguageSelector
           selectedLanguage={languageDefinition?.filename || "Plaintext"}
@@ -230,13 +230,12 @@ function LanguageSelector(props: LanguageSelectorProps) {
   const [languages, setLanguages] = useState(Languages);
 
   return (
-    <Popup title="Select language" onClose={onClose}>
+    <Popup onClose={onClose}>
       <Flex
         sx={{
           flexDirection: "column",
           height: 200,
           width: ["auto", 300],
-          overflowY: "auto",
           bg: "background"
         }}
       >
@@ -245,12 +244,11 @@ function LanguageSelector(props: LanguageSelectorProps) {
           placeholder="Search languages"
           sx={{
             width: "auto",
-            position: "sticky",
-            top: 0,
             bg: "background",
             mx: 2,
             p: "7px",
-            zIndex: 999
+            zIndex: 999,
+            mt: 1
           }}
           onChange={(e) => {
             if (!e.target.value) return setLanguages(Languages);
@@ -271,7 +269,8 @@ function LanguageSelector(props: LanguageSelectorProps) {
           sx={{
             flexDirection: "column",
             pt: 1,
-            mt: 1
+            mt: 1,
+            overflowY: "auto"
           }}
         >
           {languages.map((lang) => (
@@ -292,7 +291,7 @@ function LanguageSelector(props: LanguageSelectorProps) {
                 <Icon path={Icons.check} size="small" />
               ) : lang.alias ? (
                 <Text variant={"subBody"} sx={{ fontSize: "10px" }}>
-                  {lang.alias.slice(0, 3).join(", ")}
+                  {lang.alias.slice(0, 3).join(", ").toUpperCase()}
                 </Text>
               ) : null}
             </Button>