Merge branch 'master' into mobile/fix-quick-notes

Signed-off-by: Ammar Ahmed <40239442+ammarahm-ed@users.noreply.github.com>

apps/desktop/global.d.ts

@@ -19,9 +19,40 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 /* eslint-disable no-var */
 
 import { BrowserWindow } from "electron";
+import {
+  type FormData as FormDataType,
+  type Headers as HeadersType,
+  type Request as RequestType,
+  type Response as ResponseType
+} from "undici";
 
 declare global {
   var window: BrowserWindow | null;
   var RELEASE: boolean;
   var MAC_APP_STORE: boolean;
+
+  // Re-export undici fetch function and various classes to global scope.
+  // These are classes and functions expected to be at global scope according to Node.js v18 API
+  // documentation.
+  // See: https://nodejs.org/dist/latest-v18.x/docs/api/globals.html
+  // eslint-disable-next-line no-var
+  export var {
+    FormData,
+    Headers,
+    Request,
+    Response,
+    fetch
+  }: typeof import("undici");
+
+  type FormData = FormDataType;
+  type Headers = HeadersType;
+  type Request = RequestType;
+  type Response = ResponseType;
 }
+
+// NOTE: the import in the global block above needs to be a var for this to work properly.
+globalThis.fetch = fetch;
+globalThis.FormData = FormData;
+globalThis.Headers = Headers;
+globalThis.Request = Request;
+globalThis.Response = Response;

apps/desktop/package.json

@@ -22,15 +22,15 @@
     "zod": "^3.21.4"
   },
   "devDependencies": {
-    "@types/node": "^18.15.0",
+    "@types/node": "18.16.1",
     "@types/yargs": "^17.0.24",
     "chokidar": "^3.5.3",
-    "electron": "24.5.1",
-    "electron-builder": "24.4.0",
+    "electron": "^26.1.0",
+    "electron-builder": "^24.6.3",
     "electron-builder-notarize": "^1.5.1",
     "esbuild": "^0.17.19",
-    "node-fetch": "^3.3.1",
-    "tree-kill": "^1.2.2"
+    "tree-kill": "^1.2.2",
+    "undici": "^5.23.0"
   },
   "optionalDependencies": {
     "dmg-license": "^1.0.11"

apps/desktop/src/utils/protocol.ts

@@ -17,16 +17,14 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { protocol, ProtocolRequest } from "electron";
+import { protocol, net } from "electron";
 import { isDevelopment } from "./index";
 import { createReadStream } from "fs";
 import { extname, normalize } from "path";
 import { URL } from "url";
-import fetch, { Response } from "node-fetch";
 
 const BASE_PATH = isDevelopment() ? "../public" : "";
 const HOSTNAME = `app.notesnook.com`;
-const FILE_NOT_FOUND = -6;
 const SCHEME = "https";
 const extensionToMimeType: Record<string, string> = {
   html: "text/html",
@@ -34,101 +32,47 @@ const extensionToMimeType: Record<string, string> = {
   js: "application/javascript",
   css: "text/css",
   svg: "image/svg+xml",
-  png: "image/png"
+  png: "image/png",
+  jpg: "image/jpg",
+  ttf: "font/ttf",
+  woff: "font/woff",
+  woff2: "font/woff2"
 };
 
 function registerProtocol() {
-  const protocolInterceptionResult = protocol.interceptStreamProtocol(
-    SCHEME,
-    async (request, callback) => {
-      const url = new URL(request.url);
-      if (shouldInterceptRequest(url)) {
-        console.info("Intercepting request:", request.url);
-
-        const loadIndex = !extname(url.pathname);
-        const filePath = normalize(
-          `${__dirname}${
-            loadIndex
-              ? `${BASE_PATH}/index.html`
-              : `${BASE_PATH}/${url.pathname}`
-          }`
-        );
-        if (!filePath) {
-          console.error("Local asset file not found at", filePath);
-          callback({ error: FILE_NOT_FOUND });
-          return;
-        }
-        const fileExtension = extname(filePath).replace(".", "");
-
-        const data = createReadStream(filePath);
-        callback({
-          data,
-          mimeType: extensionToMimeType[fileExtension]
-        });
-      } else {
-        let response: Response;
-        try {
-          const body = await getBody(request);
-          response = await fetch(request.url, {
-            ...request,
-            body,
-            headers: {
-              ...request.headers
-              // origin: `${PROTOCOL}://${HOSTNAME}/`
-            },
-            referrer: request.referrer,
-            redirect: "manual"
-          });
-        } catch (e) {
-          console.error(e);
-          console.error(`Error sending request to `, request.url, "Error: ", e);
-          callback({ statusCode: 400 });
-          return;
-        }
-        callback({
-          statusCode: response.status,
-          data: response.body || undefined,
-          headers: Object.fromEntries(response.headers.entries()),
-          mimeType: response.headers.get("Content-Type") || undefined
+  protocol.handle(SCHEME, async (request) => {
+    const url = new URL(request.url);
+    if (shouldInterceptRequest(url)) {
+      console.info("Intercepting request:", request.url);
+      const loadIndex = !extname(url.pathname);
+      const filePath = normalize(
+        `${__dirname}${
+          loadIndex ? `${BASE_PATH}/index.html` : `${BASE_PATH}/${url.pathname}`
+        }`
+      );
+      if (!filePath) {
+        console.error("Local asset file not found at", filePath);
+        return new Response(undefined, {
+          status: 404,
+          statusText: "FILE_NOT_FOUND"
         });
       }
+      const fileExtension = extname(filePath).replace(".", "");
+      return new Response(createReadStream(filePath), {
+        headers: { "Content-Type": extensionToMimeType[fileExtension] }
+      });
+    } else {
+      try {
+        return net.fetch(request, {
+          bypassCustomProtocolHandlers: true
+        });
+      } catch (e) {
+        console.error(e);
+      }
+      return Response.error();
     }
-  );
-
-  console.info(
-    `${SCHEME} protocol inteception ${
-      protocolInterceptionResult ? "successful" : "failed"
-    }.`
-  );
-
-  // protocol.handle(SCHEME, (request) => {
-  //   const url = new URL(request.url);
-  //   if (shouldInterceptRequest(url)) {
-  //     console.info("Intercepting request:", request.url);
-  //     const loadIndex = !extname(url.pathname);
-  //     const absoluteFilePath = normalize(
-  //       `${__dirname}${
-  //         loadIndex ? `${BASE_PATH}/index.html` : `${BASE_PATH}/${url.pathname}`
-  //       }`
-  //     );
-  //     const filePath = getPath(absoluteFilePath);
-  //     if (!filePath) {
-  //       console.error("Local asset file not found at", filePath);
-  //       return new Response(undefined, {
-  //         status: 404,
-  //         statusText: "FILE_NOT_FOUND"
-  //       });
-  //     }
-  //     const fileExtension = extname(filePath).replace(".", "");
-  //     const data = createReadStream(filePath);
-  //     return new Response(data, {
-  //       headers: { "Content-Type": extensionToMimeType[fileExtension] }
-  //     });
-  //   } else {
-  //     return net.fetch(request);
-  //   }
-  // });
-  // console.info(`${SCHEME} protocol inteception "successful"`);
+  });
+  console.info(`${SCHEME} protocol inteception "successful"`);
 }
 
 const bypassedRoutes: string[] = [];
@@ -139,22 +83,3 @@ function shouldInterceptRequest(url: URL) {
 
 const PROTOCOL_URL = `${SCHEME}://${HOSTNAME}/`;
 export { registerProtocol, PROTOCOL_URL };
-
-async function getBody(request: ProtocolRequest) {
-  const session = globalThis?.window?.webContents?.session;
-
-  const blobParts = [];
-  if (!request.uploadData || !request.uploadData.length) return null;
-  for (const data of request.uploadData) {
-    if (data.bytes) {
-      blobParts.push(new Uint8Array(data.bytes));
-    } else if (session && data.blobUUID) {
-      const buffer = await session.getBlobData(data.blobUUID);
-      if (!buffer) continue;
-      blobParts.push(new Uint8Array(buffer));
-    }
-  }
-  const blob = new Blob(blobParts);
-  return blob;
-}
-

apps/mobile/app/app.js

@@ -16,6 +16,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
+import "@azure/core-asynciterator-polyfill";
 import SettingsService from "./services/settings";
 import {
   THEME_COMPATIBILITY_VERSION,
@@ -45,6 +46,7 @@ const App = () => {
     if (appLockMode && appLockMode !== "none") {
       useUserStore.getState().lockApp(true);
     }
+    globalThis["IS_MAIN_APP_RUNNING"] = true;
     init();
     setTimeout(async () => {
       SettingsService.onFirstLaunch();

apps/mobile/app/common/database/encryption.js

@@ -126,6 +126,18 @@ export async function decrypt(password, data) {
   return await Sodium.decrypt(password, _data);
 }
 
+export async function decryptMulti(password, data) {
+  if (!password.password && !password.key) return undefined;
+  if (password.password && password.password === "" && !password.key)
+    return undefined;
+
+  data = data.map((d) => {
+    d.output = "plain";
+    return d;
+  });
+  return await Sodium.decryptMulti(password, data);
+}
+
 export function parseAlgorithm(alg) {
   if (!alg) return {};
   const [enc, kdf, compressed, compressionAlg, base64variant] = alg.split("-");
@@ -154,3 +166,24 @@ export async function encrypt(password, data) {
     alg: getAlgorithm(7)
   };
 }
+
+export async function encryptMulti(password, data) {
+  if (!password.password && !password.key) return undefined;
+  if (password.password && password.password === "" && !password.key)
+    return undefined;
+
+  let results = await Sodium.encryptMulti(
+    password,
+    data.map((item) => ({
+      type: "plain",
+      data: item
+    }))
+  );
+
+  return !results
+    ? []
+    : results.map((result) => ({
+        ...result,
+        alg: getAlgorithm(7)
+      }));
+}

apps/mobile/app/common/database/index.js

@@ -38,11 +38,11 @@ database.host(
         SSE_HOST: "https://events.streetwriters.co",
         SUBSCRIPTIONS_HOST: "https://subscriptions.streetwriters.co",
         ISSUES_HOST: "https://issues.streetwriters.co"
-        // API_HOST: "http://192.168.8.101:5264",
-        // AUTH_HOST: "http://192.168.8.101:8264",
-        // SSE_HOST: "http://192.168.8.101:7264",
-        // SUBSCRIPTIONS_HOST: "http://192.168.8.101:9264",
-        // ISSUES_HOST: "http://192.168.8.101:2624"
+        // API_HOST: "http://192.168.43.108:5264",
+        // AUTH_HOST: "http://192.168.43.108:8264",
+        // SSE_HOST: "http://192.168.43.108:7264",
+        // SUBSCRIPTIONS_HOST: "http://192.168.43.108:9264",
+        // ISSUES_HOST: "http://192.168.43.108:2624"
       }
     : {
         API_HOST: "https://api.notesnook.com",
@@ -63,7 +63,7 @@ database.setup(
   }
 );
 
-initalize(new KV(LoggerStorage), true);
+initalize(new KV(LoggerStorage));
 
 export const db = database;
 export const DatabaseLogger = dbLogger;

apps/mobile/app/common/database/mmkv.js

@@ -18,9 +18,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import { Platform } from "react-native";
-import MMKVStorage, { ProcessingModes } from "react-native-mmkv-storage";
+import { ProcessingModes, MMKVLoader } from "react-native-mmkv-storage";
 
-export const MMKV = new MMKVStorage.Loader()
+export const MMKV = new MMKVLoader()
   .setProcessingMode(
     Platform.OS === "ios"
       ? ProcessingModes.MULTI_PROCESS

apps/mobile/app/common/database/storage.js

@@ -27,11 +27,16 @@ import {
   getCryptoKey,
   getRandomBytes,
   hash,
-  removeCryptoKey
+  removeCryptoKey,
+  decryptMulti,
+  encryptMulti
 } from "./encryption";
 import { MMKV } from "./mmkv";
 
 export class KV {
+  /**
+   * @type {typeof MMKV}
+   */
   storage = null;
   constructor(storage) {
     this.storage = storage;
@@ -53,6 +58,7 @@ export class KV {
       key,
       typeof data === "string" ? data : JSON.stringify(data)
     );
+
     return true;
   }
 
@@ -60,27 +66,32 @@ export class KV {
     if (keys.length <= 0) {
       return [];
     } else {
-      let data = await this.storage.getMultipleItemsAsync(keys.slice());
-
-      return data.map(([key, value]) => {
-        let obj;
-        try {
-          obj = JSON.parse(value);
-        } catch (e) {
-          obj = value;
-        }
-
-        return [key, obj];
-      });
+      try {
+        let data = await this.storage.getMultipleItemsAsync(
+          keys.slice(),
+          "string"
+        );
+        return data.map(([key, value]) => {
+          let obj;
+          try {
+            obj = JSON.parse(value);
+          } catch (e) {
+            obj = value;
+          }
+          return [key, obj];
+        });
+      } catch (e) {
+        console.log(e);
+      }
     }
   }
 
   async remove(key) {
-    return await this.storage.removeItem(key);
+    return this.storage.removeItem(key);
   }
 
   async clear() {
-    return await this.storage.clearStore();
+    return this.storage.clearStore();
   }
 
   async getAllKeys() {
@@ -96,6 +107,10 @@ export class KV {
     );
     return keys;
   }
+
+  async writeMulti(items) {
+    return this.storage.setMultipleItemsAsync(items, "object");
+  }
 }
 
 const DefaultStorage = new KV(MMKV);
@@ -129,8 +144,10 @@ export default {
   remove: (key) => DefaultStorage.remove(key),
   clear: () => DefaultStorage.clear(),
   getAllKeys: () => DefaultStorage.getAllKeys(),
+  writeMulti: (items) => DefaultStorage.writeMulti(items),
   encrypt,
   decrypt,
+  decryptMulti,
   getRandomBytes,
   checkAndCreateDir,
   requestPermission,
@@ -138,5 +155,6 @@ export default {
   getCryptoKey,
   removeCryptoKey,
   hash,
-  generateCryptoKey
+  generateCryptoKey,
+  encryptMulti
 };

apps/mobile/app/components/auth/signup.js

@@ -67,6 +67,7 @@ export const Signup = ({ changeMode, trial }) => {
 
   const signup = async () => {
     if (!validateInfo() || error) return;
+    if (loading) return;
     setLoading(true);
     try {
       await db.user.signup(email.current.toLowerCase(), password.current);

apps/mobile/app/components/auth/use-login.js

@@ -64,6 +64,7 @@ export const useLogin = (onFinishLogin) => {
   const login = async () => {
     if (!validateInfo() || error) return;
     try {
+      if (loading) return;
       setLoading(true);
       switch (step) {
         case LoginSteps.emailAuth: {

apps/mobile/app/screens/editor/tiptap/use-editor.ts

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+import { getFormattedDate } from "@notesnook/common";
 import { EVENTS } from "@notesnook/core/dist/common";
 import { useThemeEngineStore } from "@notesnook/theme";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
@@ -31,6 +32,7 @@ import {
   openVault
 } from "../../../services/event-manager";
 import Navigation from "../../../services/navigation";
+import Notifications from "../../../services/notifications";
 import SettingsService from "../../../services/settings";
 import { TipManager } from "../../../services/tip-manager";
 import { useEditorStore } from "../../../stores/use-editor-store";
@@ -52,7 +54,6 @@ import {
   makeSessionId,
   post
 } from "./utils";
-import { getFormattedDate } from "@notesnook/common";
 
 export const useEditor = (
   editorId = "",
@@ -133,7 +134,7 @@ export const useEditor = (
 
   const reset = useCallback(
     async (resetState = true, resetContent = true) => {
-      currentNote.current?.id && db.fs?.cancel(currentNote.current.id);
+      currentNote.current?.id && db.fs?.cancel(currentNote.current.id, null);
       currentNote.current = null;
       loadedImages.current = {};
       currentContent.current = null;
@@ -234,6 +235,10 @@ export const useEditor = (
               id && useEditorStore.getState().setCurrentlyEditingNote(id);
             });
           }
+
+          if (Notifications.isNotePinned(id as string)) {
+            Notifications.pinNote(id as string);
+          }
         } else {
           noteData.contentId = note?.contentId;
           // eslint-disable-next-line @typescript-eslint/no-explicit-any

apps/mobile/app/screens/settings/home.tsx

@@ -87,12 +87,6 @@ const Home = ({
               alignItems: "center"
             }}
           >
-            <Heading color={colors.primary.heading} size={SIZE.lg}>
-              Logging out
-            </Heading>
-            <Paragraph color={colors.secondary.icon}>
-              Please wait while we log out and clear app data.
-            </Paragraph>
             <View
               style={{
                 width: "100%",

apps/mobile/app/screens/settings/settings-data.tsx

@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { ThemeDark, ThemePitchBlack } from "@notesnook/theme";
 import notifee from "@notifee/react-native";
 import dayjs from "dayjs";
 import React from "react";
@@ -288,9 +287,9 @@ export const settingsGroups: SettingSection[] = [
                 positiveText: "Logout",
                 positivePress: async () => {
                   try {
-                    eSendEvent("settings-loading", true);
-                    setImmediate(async () => {
-                      eSendEvent(eCloseSimpleDialog);
+                    eSendEvent(eCloseSimpleDialog);
+                    setTimeout(async () => {
+                      eSendEvent("settings-loading", true);
                       Navigation.popToTop();
                       await db.user?.logout();
                       setLoginMessage();
@@ -306,7 +305,7 @@ export const settingsGroups: SettingSection[] = [
                       setTimeout(() => {
                         eSendEvent("settings-loading", false);
                       }, 2000);
-                    });
+                    }, 300);
                   } catch (e) {
                     ToastEvent.error(e as Error, "Error logging out");
                     eSendEvent("settings-loading", false);

apps/mobile/app/services/notifications.ts

@@ -46,6 +46,9 @@ import { useRelationStore } from "../stores/use-relation-store";
 import { useReminderStore } from "../stores/use-reminder-store";
 import { presentDialog } from "../components/dialog/functions";
 import NetInfo from "@react-native-community/netinfo";
+import { encodeNonAsciiHTML } from "entities";
+import { convertNoteToText } from "../utils/note-to-text";
+
 
 export type Reminder = {
   id: string;
@@ -86,6 +89,28 @@ async function getNextMonthlyReminderDate(
   return await getNextMonthlyReminderDate(reminder, dayjs().year() + 1);
 }
 
+export function textToHTML(src: string) {
+  return src
+    .split(/[\r\n]/)
+    .map((line) =>
+      line
+        ? `<p data-spacing="single">${encodeLine(line)}</p>`
+        : `<p data-spacing="single"></p>`
+    )
+    .join("");
+}
+
+function encodeLine(line: string) {
+  line = encodeNonAsciiHTML(line);
+  line = line.replace(/(^ +)|( {2,})/g, (sub, ...args) => {
+    const [starting, inline] = args;
+    if (starting) return "&nbsp;".repeat(starting.length);
+    if (inline) return "&nbsp;".repeat(inline.length);
+    return sub;
+  });
+  return line;
+}
+
 async function initDatabase(notes = true) {
   if (!db.isInitialized) {
     await db.initCollections();
@@ -206,16 +231,41 @@ const onEvent = async ({ type, detail }: Event) => {
         });
         if (!db.isInitialized) await db.init();
         await db.notes?.init();
-        await db.notes?.add({
+
+        const id = await db.notes?.add({
           content: {
             type: "tiptap",
-            data: `<p>${input} </p>`
+            data: textToHTML(input as string)
           }
         });
+
+        const defaultNotebook = db.settings?.getDefaultNotebook();
+
+        if (defaultNotebook) {
+          if (!defaultNotebook.topic) {
+            await db.relations?.add(
+              { type: "notebook", id: defaultNotebook.id },
+              { type: "note", id: id }
+            );
+          } else {
+            await db.notes?.addToNotebook(
+              {
+                topic: defaultNotebook.topic,
+                id: defaultNotebook?.id
+              },
+              id
+            );
+          }
+        }
+
         const status = await NetInfo.fetch();
         if (status.isInternetReachable) {
           try {
-            await db.sync(false, false);
+            if (!globalThis["IS_MAIN_APP_RUNNING" as never]) {
+              await db.sync(false, false);
+            } else {
+              console.log("main app running, skipping sync");
+            }
           } catch (e) {
             console.log(e, (e as Error).stack);
           }
@@ -757,6 +807,11 @@ function getPinnedNotes(): DisplayedNotification[] {
   return pinned;
 }
 
+function isNotePinned(id: string) {
+  if (Platform.OS !== "android") return false;
+  return pinned.findIndex((notification) => notification.id === id) > -1;
+}
+
 function get(): Promise<DisplayedNotification[]> {
   return new Promise((resolve) => {
     if (Platform.OS === "ios") resolve([]);
@@ -844,6 +899,26 @@ async function setupReminders(checkNeedsScheduling = false) {
   );
 }
 
+async function pinNote(id: string) {
+  try {
+    const note = db.notes?.note(id as string) as any;
+    let text = await convertNoteToText(note as any, false);
+    if (!text) text = "";
+    let html = text.replace(/\n/g, "<br />");
+    Notifications.displayNotification({
+      title: note.title,
+      message: note.headline || text,
+      subtitle: note.headline || text,
+      bigText: html,
+      ongoing: true,
+      actions: ["UNPIN"],
+      id: note.id
+    });
+  } catch (e) {
+    console.log(e);
+  }
+}
+
 const Notifications = {
   init,
   displayNotification,
@@ -859,7 +934,9 @@ const Notifications = {
   checkAndRequestPermissions,
   clearAllTriggers,
   setupReminders,
-  getChannelId
+  getChannelId,
+  isNotePinned,
+  pinNote
 };
 
 export default Notifications;

apps/mobile/app/services/sync.js

@@ -41,6 +41,11 @@ const run = async (
   full = true,
   onCompleted
 ) => {
+  if (useUserStore.getState().syncing) {
+    DatabaseLogger.log("Sync in progress");
+    console.log("Sync in progress");
+    return;
+  }
   clearTimeout(syncTimer);
   syncTimer = setTimeout(async () => {
     const status = await NetInfo.fetch();

apps/mobile/native/ios/Notesnook/Info.plist

@@ -153,5 +153,7 @@
 	</array>
 	<key>appGroupId</key>
 	<string>group.org.streetwriters.notesnook</string>
+	<key>disableMMKVBackup</key>
+	<true/>
 </dict>
 </plist>

apps/mobile/native/ios/Podfile.lock

@@ -19,9 +19,9 @@ PODS:
   - glog (0.3.5)
   - GZIP (1.3.0)
   - JWTDecode (3.0.1)
-  - MMKV (1.2.13):
-    - MMKVCore (~> 1.2.13)
-  - MMKVCore (1.2.13)
+  - MMKV (1.3.1):
+    - MMKVCore (~> 1.3.1)
+  - MMKVCore (1.3.1)
   - pop (1.0.12)
   - RCT-Folly (2021.07.22.00):
     - boost
@@ -313,7 +313,7 @@ PODS:
     - React-Core
   - react-native-get-random-values (1.9.0):
     - React-Core
-  - react-native-gzip (1.0.0):
+  - react-native-gzip (1.1.0):
     - Base64
     - GZIP
     - React-Core
@@ -327,8 +327,9 @@ PODS:
     - React-Core
   - react-native-keep-awake (1.2.0):
     - React-Core
-  - react-native-mmkv-storage (0.9.1):
-    - MMKV (= 1.2.13)
+  - react-native-mmkv-storage (0.10.0-alpha.6):
+    - MMKV (~> 1.3.1)
+    - React
     - React-Core
   - react-native-netinfo (9.3.10):
     - React-Core
@@ -346,7 +347,7 @@ PODS:
     - ReactCommon/turbomodule/core
   - react-native-share-extension (2.5.6):
     - React
-  - react-native-sodium (1.4.1):
+  - react-native-sodium (1.5.1):
     - React
   - react-native-webview (11.26.1):
     - React-Core
@@ -841,8 +842,8 @@ SPEC CHECKSUMS:
   glog: 04b94705f318337d7ead9e6d17c019bd9b1f6b1b
   GZIP: 416858efbe66b41b206895ac6dfd5493200d95b3
   JWTDecode: 2eed97c2fa46ccaf3049a787004eedf0be474a87
-  MMKV: aac95d817a100479445633f2b3ed8961b4ac5043
-  MMKVCore: 3388952ded307e41b3ed8a05892736a236ed1b8e
+  MMKV: 5a07930c70c70b86cd87761a42c8f3836fb681d7
+  MMKVCore: e50135dbd33235b6ab390635991bab437ab873c0
   pop: d582054913807fd11fd50bfe6a539d91c7e1a55a
   RCT-Folly: 424b8c9a7a0b9ab2886ffe9c3b041ef628fd4fb1
   RCTRequired: 656ef0536dd60a9740961ade6a64ba0cb0572d2b
@@ -868,20 +869,20 @@ SPEC CHECKSUMS:
   react-native-document-picker: ec07866a30707f23660c0f3ae591d669d3e89096
   react-native-fingerprint-scanner: be63e626b31fb951780a5fac5328b065a61a3d6e
   react-native-get-random-values: dee677497c6a740b71e5612e8dbd83e7539ed5bb
-  react-native-gzip: 02f9968afa759e189f0414d41f8f4a951a86b4f1
+  react-native-gzip: c5e87ee9e359f02350e3a2ee52eb35eddc398868
   react-native-html-to-pdf-lite: 21bfb169bf4cbcd7bec9f736975ee1b3f5292d4a
   react-native-image-picker: 9c8a2687b69300ad9e95cec5d38f35ab9d32467d
   react-native-image-resizer: 00ceb0e05586c7aadf061eea676957a6c2ec60fa
   react-native-in-app-review: db8bb167a5f238e7ceca5c242d6b36ce8c4404a4
   react-native-keep-awake: caee3ff89eaa21dfe29010f0d143566874a04441
-  react-native-mmkv-storage: cfb6854594cfdc5f7383a9e464bb025417d1721c
+  react-native-mmkv-storage: a38beae34b1f7906a2f5e715f54216f9ddab14c0
   react-native-netinfo: ccbe1085dffd16592791d550189772e13bf479e2
   react-native-notification-sounds: da78c828fe1bcbb92d8b505d5261890ed315ff39
   react-native-orientation: f1caf84d65f1a4fd4511a18f2b924e634ad7a628
   react-native-pdf: 33c622cbdf776a649929e8b9d1ce2d313347c4fa
   react-native-safe-area-context: 36cc67648134e89465663b8172336a19eeda493d
   react-native-share-extension: df66a2ee48a62277d79898375e2142bde0782063
-  react-native-sodium: f4e3986ddcb73482f8679e534b448a0675d0cf13
+  react-native-sodium: 08d459dfcb5d246363a30712d816acea56ab8f1c
   react-native-webview: 9f111dfbcfc826084d6c507f569e5e03342ee1c1
   React-NativeModulesApple: 1d81d927ef1a67a3545a01e14c2e98500bf9b199
   React-perflogger: 684a11499a0589cc42135d6d5cc04d0e4e0e261a
@@ -930,4 +931,4 @@ SPEC CHECKSUMS:
 
 PODFILE CHECKSUM: b0cadd188aa428b98d1eaa0b84b3b6208e714119
 
-COCOAPODS: 1.11.3
+COCOAPODS: 1.12.1

apps/mobile/native/package.json

@@ -7,7 +7,7 @@
   "dependencies": {
     "@ammarahmed/notifee-react-native": "7.4.4",
     "@ammarahmed/react-native-share-extension": "^2.5.5",
-    "@ammarahmed/react-native-sodium": "1.4.1",
+    "@ammarahmed/react-native-sodium": "1.5.2",
     "@bam.tech/react-native-image-resizer": "3.0.5",
     "@callstack/repack": "^3.2.0",
     "@react-native-clipboard/clipboard": "^1.9.0",
@@ -38,13 +38,13 @@
     "react-native-fingerprint-scanner": "https://github.com/ammarahm-ed/react-native-fingerprint-scanner.git",
     "react-native-gesture-handler": "^2.12.0",
     "react-native-get-random-values": "^1.7.0",
-    "react-native-gzip": "1.0.0",
+    "react-native-gzip": "1.1.0",
     "react-native-html-to-pdf-lite": "^0.9.1",
     "react-native-iap": "7.5.6",
     "react-native-image-picker": "4.1.2",
     "react-native-in-app-review": "4.3.3",
     "react-native-keychain": "4.0.5",
-    "react-native-mmkv-storage": "^0.9.1",
+    "react-native-mmkv-storage": "^0.10.0-alpha.7",
     "react-native-modal-datetime-picker": "14.0.0",
     "react-native-navigation-bar-color": "2.0.2",
     "react-native-notification-sounds": "0.5.5",
@@ -53,7 +53,7 @@
     "react-native-privacy-snapshot": "https://github.com/standardnotes/react-native-privacy-snapshot.git",
     "react-native-reanimated": "3.3.0",
     "react-native-safe-area-context": "^4.3.1",
-    "react-native-scoped-storage": "^1.9.3",
+    "react-native-scoped-storage": "^1.9.5",
     "react-native-screens": "^3.13.1",
     "react-native-securerandom": "^1.0.1",
     "react-native-share": "^7.2.0",

apps/mobile/native/webpack.config.js

@@ -185,6 +185,8 @@ module.exports = (env) => {
             /node_modules(.*[/\\])+@tanstack[/\\]react-query/,
             /node_modules(.*[/\\])+@trpc[/\\]react-query/,
             /node_modules(.*[/\\])+katex/,
+            /node_modules(.*[/\\])+@notesnook[/\\]core/,
+
           ],
           use: {
             loader: "babel-loader",

apps/mobile/package.json

@@ -29,17 +29,18 @@
     "react-refresh": "0.14.0"
   },
   "dependencies": {
+    "@azure/core-asynciterator-polyfill": "^1.0.2",
     "@notesnook/common": "file:../../packages/common",
     "@notesnook/core": "file:../../packages/core",
     "@notesnook/editor": "file:../../packages/editor",
     "@notesnook/editor-mobile": "file:../../packages/editor-mobile",
     "@notesnook/logger": "file:../../packages/logger",
     "@notesnook/themes-server": "file:../../servers/themes",
-    "react": "18.2.0",
-    "react-native": "0.72.0",
+    "@tanstack/react-query": "^4.29.19",
     "@trpc/client": "10.31.0",
     "@trpc/react-query": "10.31.0",
     "@trpc/server": "^10.31.0",
-    "@tanstack/react-query": "^4.29.19"
+    "react": "18.2.0",
+    "react-native": "0.72.0"
   }
-}
+}

apps/mobile/patches/react-native-mmkv-storage+0.9.1.patch

@@ -1,42 +0,0 @@
-diff --git a/node_modules/react-native-mmkv-storage/ios/MMKVNative.mm b/node_modules/react-native-mmkv-storage/ios/MMKVNative.mm
-index 9af089d..f64a04a 100644
---- a/node_modules/react-native-mmkv-storage/ios/MMKVNative.mm
-+++ b/node_modules/react-native-mmkv-storage/ios/MMKVNative.mm
-@@ -30,13 +30,23 @@ - (instancetype)init
- {
-     self = [super init];
-     RCTExecuteOnMainQueue(^{
--        NSArray *paths = NSSearchPathForDirectoriesInDomains(NSLibraryDirectory,
--                                                             NSUserDomainMask, YES);
--        NSString *libraryPath = (NSString *)[paths firstObject];
--        NSString *rootDir = [libraryPath stringByAppendingPathComponent:@"mmkv"];
-+//        NSArray *paths = NSSearchPathForDirectoriesInDomains(NSLibraryDirectory,
-+//                                                             NSUserDomainMask, YES);
-+//        NSString *libraryPath = (NSString *)[paths firstObject];
-+        
-+        NSString *myGroupID = @"group.org.streetwriters.notesnook"; // the group dir that can be accessed by App & extensions
-+
-+        NSString *groupDir = [[NSFileManager defaultManager] containerURLForSecurityApplicationGroupIdentifier:myGroupID].path;
-+        NSString *rootDir = [groupDir stringByAppendingPathComponent:@"mmkv"];
-+              
-+        // Do not include this directory in cloud backups.
-+        NSError *error = nil;
-+        NSURL *url = [NSURL fileURLWithPath:rootDir isDirectory:YES];
-+        [url setResourceValue:[NSNumber numberWithBool:YES] forKey:NSURLIsExcludedFromBackupKey error:&error];
-         rPath = rootDir;
-+        // Set the groupDir
-         _secureStorage = [[SecureStorage alloc] init];
--        [MMKV initializeMMKV:rootDir];
-+        [MMKV initializeMMKV:nil groupDir:rootDir logLevel:MMKVLogInfo];
-     });
-     
-     
-@@ -706,7 +716,7 @@ static void install(jsi::Runtime &jsiRuntime) {
- }
- 
- - (void)migrate {
--    MMKV *kv = [MMKV mmkvWithID:@"mmkvIdStore"];
-+    MMKV *kv = [MMKV mmkvWithID:@"mmkvIdStore" mode:MMKVMultiProcess];
-     [mmkvInstances setObject:kv forKey:@"mmkvIdStore"];
-     if ([kv containsKey:@"mmkvIdData"]) {
-         NSMutableDictionary *oldStore =

apps/mobile/share/search.js

@@ -34,6 +34,7 @@ import { db } from "../app/common/database";
 import { getElevationStyle } from "../app/utils/elevation";
 import { initDatabase, useShareStore } from "./store";
 import { useThemeColors } from "@notesnook/theme";
+import { groupArray } from "@notesnook/core/dist/utils/grouping";
 
 const ListItem = ({ item, mode, close }) => {
   const { colors } = useThemeColors();
@@ -97,7 +98,7 @@ const ListItem = ({ item, mode, close }) => {
       style={{
         flexDirection: "column",
         borderBottomWidth: item.topics?.length > 0 ? 0 : 1,
-        borderBottomColor: colors.secondary.background,
+        borderBottomColor: colors.primary.border,
         justifyContent: "center",
         paddingVertical: 12
       }}
@@ -160,8 +161,15 @@ const ListItem = ({ item, mode, close }) => {
 };
 
 const SearchGetters = {
-  appendNote: () => db.notes.all,
-  selectNotebooks: () => db.notebooks.all,
+  appendNote: () =>
+    groupArray([...db.notes.all], db.settings.getGroupOptions("notes")).filter(
+      (item) => item.type === "note"
+    ),
+  selectNotebooks: () =>
+    groupArray(
+      [...db.notebooks.all],
+      db.settings.getGroupOptions("notebooks")
+    ).filter((item) => item.type === "notebook"),
   selectTags: () => {
     const selected = useShareStore.getState().selectedTags;
     const tags = [];
@@ -179,14 +187,30 @@ const SearchGetters = {
       if (index > -1) continue;
       tags.push(tag);
     }
-    return tags;
+    return groupArray([...tags], {
+      groupBy: "none",
+      sortDirection: "asc",
+      sortBy: "title"
+    }).filter((item) => item.type === "tag");
   }
 };
 
 const SearchLookup = {
-  appendNote: (items, kwd) => db.lookup.notes(items, kwd),
-  selectNotebooks: (items, kwd) => db.lookup.notebooks(items, kwd),
-  selectTags: (items, kwd) => db.lookup.tags(items, kwd)
+  appendNote: (items, kwd) => {
+    return db.lookup.notes(items, kwd);
+  },
+  selectNotebooks: (items, kwd) => {
+    return db.lookup.notebooks(items, kwd);
+  },
+  selectTags: (items, kwd) => {
+    return db.lookup.tags(items, kwd);
+  }
+};
+
+const SearchPlaceholder = {
+  appendNote: "Search for a note",
+  selectNotebooks: "Search for a notebook",
+  selectTags: "Search for a tag"
 };
 
 export const Search = ({ close, getKeyboardHeight, quicknote, mode }) => {
@@ -286,7 +310,7 @@ export const Search = ({ close, getKeyboardHeight, quicknote, mode }) => {
         />
         <TextInput
           ref={inputRef}
-          placeholder="Search for a note"
+          placeholder={SearchPlaceholder[mode]}
           placeholderTextColor={colors.primary.placeholder}
           style={{
             fontSize: 15,

apps/mobile/share/share.js

@@ -377,7 +377,11 @@ const ShareView = ({ quicknote = false }) => {
     });
 
     try {
-      await db.sync(false, false);
+      if (!globalThis["IS_MAIN_APP_RUNNING"]) {
+        await db.sync(false, false);
+      } else {
+        console.log("main app running, skipping sync");
+      }
     } catch (e) {
       console.log(e, e.stack);
     }

apps/web/package.json

@@ -20,7 +20,6 @@
     "@notesnook/common": "file:../../packages/common",
     "@notesnook/core": "file:../../packages/core",
     "@notesnook/crypto": "file:../../packages/crypto",
-    "@notesnook/crypto-worker": "file:../../packages/crypto-worker",
     "@notesnook/desktop": "file:../desktop",
     "@notesnook/editor": "file:../../packages/editor",
     "@notesnook/logger": "file:../../packages/logger",

apps/web/src/app-effects.tsx

@@ -164,15 +164,11 @@ export default function AppEffects({ setShow }: AppEffectsProps) {
         const percent = Math.round((loaded / total) * 100);
         const text = getStatus(key)?.status || `${status} attachment`;
 
-        if (loaded === total) {
-          removeStatus(key);
-        } else {
-          updateStatus({
-            key,
-            status: text,
-            progress: loaded === total ? 100 : percent
-          });
-        }
+        updateStatus({
+          key,
+          status: text,
+          progress: loaded === total ? 100 : percent
+        });
       }
     );
 

apps/web/src/components/list-item/index.tsx

@@ -96,11 +96,13 @@ function ListItem(props: ListItemProps) {
           selectedItems.push(item);
         }
 
+        let menuItems = props.menuItems?.(item, selectedItems);
+
         if (selectedItems.length > 1) {
           title = `${selectedItems.length} items selected`;
+          menuItems = menuItems?.filter((i) => i.multiSelect === true);
         }
 
-        const menuItems = props.menuItems?.(item, selectedItems);
         if (!menuItems) return;
 
         openMenu(menuItems, {

apps/web/src/components/status-bar/index.tsx

@@ -231,7 +231,7 @@ function SyncStatus() {
       data-test-id={`sync-status-${status.key}`}
     >
       {syncStatus.progress ? (
-        <Text variant={"subBody"}>{syncStatus.progress}%</Text>
+        <Text variant={"subBody"}>{syncStatus.progress}</Text>
       ) : (
         <status.icon
           size={12}

apps/web/src/dialogs/settings/backup-export-settings.ts

@@ -44,7 +44,10 @@ export const BackupExportSettings: SettingsGroup[] = [
             type: "button",
             title: "Create backup",
             action: async () => {
-              if (!isUserPremium() && useSettingStore.getState().encryptBackups)
+              if (
+                !useUserStore.getState().isLoggedIn &&
+                useSettingStore.getState().encryptBackups
+              )
                 useSettingStore.getState().toggleEncryptBackups();
               if (await verifyAccount()) await createBackup();
             },

apps/web/src/dialogs/settings/index.tsx

@@ -190,9 +190,14 @@ export default function SettingsDialog(props: SettingsDialogProps) {
         }}
       >
         <SettingsSideBar
-          onNavigate={(settings) => setActiveSettings(settings)}
+          onNavigate={(settings) => {
+            const scrollbar = document.getElementById("settings-scrollbar");
+            if (scrollbar !== null) scrollbar.scrollTop = 0;
+            setActiveSettings(settings);
+          }}
         />
         <FlexScrollContainer
+          id="settings-scrollbar"
           style={{
             display: "flex",
             backgroundColor: "var(--background)",

apps/web/src/interfaces/fs.ts

@@ -22,7 +22,7 @@ import { xxhash64, createXXHash64 } from "hash-wasm";
 import axios, { AxiosProgressEvent } from "axios";
 import { AppEventManager, AppEvents } from "../common/app-events";
 import { StreamableFS } from "@notesnook/streamable-fs";
-import { getNNCrypto } from "./nncrypto.stub";
+import { NNCrypto } from "./nncrypto";
 import hosts from "@notesnook/core/dist/utils/constants";
 import { sendAttachmentsProgressEvent } from "@notesnook/core/dist/common";
 import { saveAs } from "file-saver";
@@ -34,7 +34,7 @@ import { ProgressStream } from "../utils/streams/progress-stream";
 import { consumeReadableStream } from "../utils/stream";
 import { Base64DecoderStream } from "../utils/streams/base64-decoder-stream";
 import { toBlob } from "@notesnook-importer/core/dist/src/utils/stream";
-import { Cipher, OutputFormat, SerializedKey } from "@notesnook/crypto";
+import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";
 import { IDataType } from "hash-wasm/dist/lib/util";
 import { IndexedDBKVStore } from "./key-value";
 import FileHandle from "@notesnook/streamable-fs/dist/src/filehandle";
@@ -53,8 +53,6 @@ async function writeEncryptedFile(
   key: SerializedKey,
   hash: string
 ) {
-  const crypto = await getNNCrypto();
-
   if (!IndexedDBKVStore.isIndexedDBSupported())
     throw new Error("This browser does not support IndexedDB.");
 
@@ -65,7 +63,7 @@ async function writeEncryptedFile(
   const fileHandle = await streamablefs.createFile(hash, file.size, file.type);
   sendAttachmentsProgressEvent("encrypt", hash, 1, 0);
 
-  const { iv, stream } = await crypto.createEncryptionStream(key);
+  const { iv, stream } = await NNCrypto.createEncryptionStream(key);
   await file
     .stream()
     .pipeThrough(new ChunkedStream(CHUNK_SIZE))
@@ -84,7 +82,7 @@ async function writeEncryptedFile(
     )
     .pipeTo(fileHandle.writeable);
 
-  sendAttachmentsProgressEvent("encrypt", hash, 1);
+  sendAttachmentsProgressEvent("encrypt", hash, 1, 1);
 
   return {
     chunkSize: CHUNK_SIZE,
@@ -155,15 +153,14 @@ async function hashStream(reader: ReadableStreamDefaultReader<Uint8Array>) {
 async function readEncrypted(
   filename: string,
   key: SerializedKey,
-  cipherData: Cipher & { outputType: OutputFormat }
+  cipherData: Cipher<DataFormat> & { outputType: DataFormat }
 ) {
   const fileHandle = await streamablefs.readFile(filename);
   if (!fileHandle) {
     console.error(`File not found. (File hash: ${filename})`);
     return null;
   }
-  const crypto = await getNNCrypto();
-  const decryptionStream = await crypto.createDecryptionStream(
+  const decryptionStream = await NNCrypto.createDecryptionStream(
     key,
     cipherData.iv
   );
@@ -268,9 +265,11 @@ async function singlePartUploadFile(
     url: uploadUrl,
     method: "PUT",
     headers: {
-      "Content-Type": "application/octet-stream"
+      "Content-Type": ""
     },
-    data: await fileHandle.toBlob(),
+    data: IS_DESKTOP_APP
+      ? await (await fileHandle.toBlob()).arrayBuffer()
+      : await fileHandle.toBlob(),
     signal,
     onUploadProgress: (ev) =>
       reportProgress(
@@ -527,8 +526,7 @@ export async function decryptFile(
 
   const { key, iv } = fileMetadata;
 
-  const crypto = await getNNCrypto();
-  const decryptionStream = await crypto.createDecryptionStream(key, iv);
+  const decryptionStream = await NNCrypto.createDecryptionStream(key, iv);
   return await toBlob(fileHandle.readable.pipeThrough(decryptionStream));
 }
 

apps/web/src/interfaces/key-value.ts

@@ -33,7 +33,7 @@ export interface IKVStore {
    * @param key
    * @param value
    */
-  set(key: string, value: any): Promise<void>;
+  set<T>(key: string, value: T): Promise<void>;
 
   /**
    * Set multiple values at once. This is faster than calling set() multiple times.
@@ -41,7 +41,7 @@ export interface IKVStore {
    *
    * @param entries Array of entries, where each entry is an array of `[key, value]`.
    */
-  setMany(entries: [string, any][]): Promise<void>;
+  setMany<T>(entries: [string, T][]): Promise<void>;
 
   /**
    * Get multiple values by their keys

apps/web/src/interfaces/nncrypto.stub.ts

@@ -1,67 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import { INNCrypto } from "@notesnook/crypto/dist/src/interfaces";
-import CryptoWorker from "@notesnook/crypto-worker/dist/src/worker.js?worker";
-
-async function loadNNCrypto() {
-  const hasWorker = "Worker" in window || "Worker" in global;
-  // if (IS_DESKTOP_APP && window.NativeNNCrypto) {
-  //   return window.NativeNNCrypto;
-  // } else
-  if (hasWorker) {
-    const { NNCryptoWorker } = await import("@notesnook/crypto-worker");
-    return NNCryptoWorker;
-  } else {
-    const { NNCrypto } = await import("@notesnook/crypto");
-    return NNCrypto;
-  }
-}
-
-let instance: INNCrypto | null = null;
-
-export function getNNCrypto(): Promise<INNCrypto> {
-  if (instance) return Promise.resolve(instance);
-  return queueify<INNCrypto>(async () => {
-    const NNCrypto = await loadNNCrypto();
-    instance = new NNCrypto(new CryptoWorker());
-    return instance;
-  });
-}
-
-let processing = false;
-type PromiseResolve = <T>(value: Awaited<T>) => void;
-const queue: Array<PromiseResolve> = [];
-async function queueify<T>(action: () => Promise<T>): Promise<T> {
-  if (processing)
-    return new Promise((resolve) => {
-      queue.push(resolve as PromiseResolve);
-    });
-
-  processing = true;
-  const result = await action();
-  processing = false;
-
-  while (queue.length > 0) {
-    const resolve = queue.pop();
-    if (!resolve) continue;
-    resolve(result);
-  }
-  return result;
-}

apps/web/src/interfaces/nncrypto.ts

@@ -0,0 +1,24 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { INNCrypto } from "@notesnook/crypto";
+import CryptoWorker from "./nncrypto.worker?worker";
+import { wrap } from "comlink";
+
+export const NNCrypto = wrap<INNCrypto>(new CryptoWorker()) as INNCrypto;

apps/web/src/interfaces/nncrypto.worker.ts

@@ -0,0 +1,40 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { NNCrypto, Chunk, SerializedKey } from "@notesnook/crypto";
+import { expose, transfer } from "comlink";
+
+class NNCryptoWorker extends NNCrypto {
+  override async createDecryptionStream(
+    key: SerializedKey,
+    iv: string
+  ): Promise<TransformStream<Uint8Array, Uint8Array>> {
+    const stream = await super.createDecryptionStream(key, iv);
+    return transfer(stream, [stream]);
+  }
+
+  override async createEncryptionStream(
+    key: SerializedKey
+  ): Promise<{ iv: string; stream: TransformStream<Chunk, Uint8Array> }> {
+    const result = await super.createEncryptionStream(key);
+    return transfer(result, [result.stream]);
+  }
+}
+
+expose(new NNCryptoWorker());

apps/web/src/interfaces/storage.ts

@@ -23,7 +23,7 @@ import {
   MemoryKVStore,
   IKVStore
 } from "./key-value";
-import { getNNCrypto } from "./nncrypto.stub";
+import { NNCrypto } from "./nncrypto";
 import type { Cipher, SerializedKey } from "@notesnook/crypto/dist/src/types";
 
 type EncryptedKey = { iv: Uint8Array; cipher: BufferSource };
@@ -58,6 +58,10 @@ export class NNStorage {
     return this.database.getMany(keys.sort());
   }
 
+  writeMulti<T>(entries: [string, T][]) {
+    return this.database.setMany(entries);
+  }
+
   write<T>(key: string, data: T) {
     return this.database.set(key, data);
   }
@@ -78,8 +82,7 @@ export class NNStorage {
     const { password, salt } = credentials;
     if (!password) throw new Error("Invalid data provided to deriveCryptoKey.");
 
-    const crypto = await getNNCrypto();
-    const keyData = await crypto.exportKey(password, salt);
+    const keyData = await NNCrypto.exportKey(password, salt);
 
     if (
       (await IndexedDBKVStore.isIndexedDBSupported()) &&
@@ -120,32 +123,39 @@ export class NNStorage {
   ): Promise<SerializedKey> {
     if (!password)
       throw new Error("Invalid data provided to generateCryptoKey.");
-    const crypto = await getNNCrypto();
-    return await crypto.exportKey(password, salt);
+
+    return await NNCrypto.exportKey(password, salt);
   }
 
   async hash(password: string, email: string): Promise<string> {
-    const crypto = await getNNCrypto();
-    return await crypto.hash(password, `${APP_SALT}${email}`);
+    return await NNCrypto.hash(password, `${APP_SALT}${email}`);
   }
 
-  async encrypt(key: SerializedKey, plainText: string): Promise<Cipher> {
-    const crypto = await getNNCrypto();
-    return await crypto.encrypt(
-      key,
-      { format: "text", data: plainText },
-      "base64"
-    );
+  encrypt(key: SerializedKey, plainText: string): Promise<Cipher<"base64">> {
+    return NNCrypto.encrypt(key, plainText, "text", "base64");
   }
 
-  async decrypt(
+  encryptMulti(
     key: SerializedKey,
-    cipherData: Cipher
+    items: string[]
+  ): Promise<Cipher<"base64">[]> {
+    return NNCrypto.encryptMulti(key, items, "text", "base64");
+  }
+
+  decrypt(
+    key: SerializedKey,
+    cipherData: Cipher<"base64">
   ): Promise<string | undefined> {
-    const crypto = await getNNCrypto();
     cipherData.format = "base64";
-    const result = await crypto.decrypt(key, cipherData);
-    if (typeof result.data === "string") return result.data;
+    return NNCrypto.decrypt(key, cipherData, "text");
+  }
+
+  decryptMulti(
+    key: SerializedKey,
+    items: Cipher<"base64">[]
+  ): Promise<string[] | undefined> {
+    items.forEach((c) => (c.format = "base64"));
+    return NNCrypto.decryptMulti(key, items, "text");
   }
 }
 
@@ -213,23 +223,3 @@ async function aesDecrypt(
   );
   return dec.decode(plainText);
 }
-
-// async function main() {
-//   const nncrypto = await getNNCrypto();
-//   const electronNNCrypto = new NNCryptoElectron();
-
-//   console.time("nncrypto");
-//   for (let i = 0; i < 100; ++i) {
-//     await nncrypto.hash("mypassword", APP_SALT);
-//   }
-//   console.timeEnd("nncrypto");
-
-//   console.time("electron");
-//   for (let i = 0; i < 100; ++i) {
-//     await electronNNCrypto.hash("mypassword", APP_SALT);
-//   }
-//   console.timeEnd("electron");
-// }
-
-// main();
-// setTimeout(main, 10000);

apps/web/src/stores/app-store.js

@@ -78,27 +78,17 @@ class AppStore extends BaseStore {
     reminderStore.refresh();
     announcementStore.refresh();
 
-    let count = 0;
     EV.subscribe(EVENTS.appRefreshRequested, () => this.refresh());
 
-    db.eventManager.subscribe(
-      EVENTS.syncProgress,
-      ({ type, total, current }) => {
-        if (total === current) return;
-        this.set((state) => {
-          state.syncStatus = {
-            key: "syncing",
-            progress: ((current / total) * 100).toFixed(),
-            type
-          };
-        });
-
-        if (type === "download" && ++count >= BATCH_SIZE) {
-          count = 0;
-          this.refresh();
-        }
-      }
-    );
+    db.eventManager.subscribe(EVENTS.syncProgress, ({ type, current }) => {
+      this.set((state) => {
+        state.syncStatus = {
+          key: "syncing",
+          progress: current,
+          type
+        };
+      });
+    });
 
     EV.subscribe(EVENTS.syncCheckStatus, async (type) => {
       const { isAutoSyncEnabled, isSyncEnabled } = this.get();
@@ -128,7 +118,6 @@ class AppStore extends BaseStore {
     db.eventManager.subscribe(EVENTS.syncCompleted, async () => {
       await this.updateLastSynced();
       this.updateSyncStatus("completed", true);
-      count = 0;
       this.refresh();
     });
 

package-lock.json

@@ -2321,6 +2321,7 @@
       "version": "4.4.0",
       "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
       "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==",
+      "dev": true,
       "dependencies": {
         "eslint-visitor-keys": "^3.3.0"
       },
@@ -2335,6 +2336,7 @@
       "version": "4.5.1",
       "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.5.1.tgz",
       "integrity": "sha512-Z5ba73P98O1KUYCCJTUeVpja9RcGoMdncZ6T49FCUl2lN38JtCJ+3WgIDBv0AuY4WChU5PmtJmOCTlN6FZTFKQ==",
+      "dev": true,
       "engines": {
         "node": "^12.0.0 || ^14.0.0 || >=16.0.0"
       }
@@ -2343,6 +2345,7 @@
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.3.tgz",
       "integrity": "sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ==",
+      "dev": true,
       "dependencies": {
         "ajv": "^6.12.4",
         "debug": "^4.3.2",
@@ -2365,6 +2368,7 @@
       "version": "6.12.6",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
       "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "dev": true,
       "dependencies": {
         "fast-deep-equal": "^3.1.1",
         "fast-json-stable-stringify": "^2.0.0",
@@ -2379,12 +2383,14 @@
     "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+      "dev": true
     },
     "node_modules/@eslint/js": {
       "version": "8.42.0",
       "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.42.0.tgz",
       "integrity": "sha512-6SWlXpWU5AvId8Ac7zjzmIOqMOba/JWY8XZ4A7q7Gn1Vlfg/SFFIlrtHXt9nPn4op9ZPAkl91Jao+QQv3r/ukw==",
+      "dev": true,
       "engines": {
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       }
@@ -2393,6 +2399,7 @@
       "version": "0.11.10",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz",
       "integrity": "sha512-KVVjQmNUepDVGXNuoRRdmmEjruj0KfiGSbS8LVc12LMsWDQzRXJ0qdhN8L8uUigKpfEHRhlaQFY0ib1tnUbNeQ==",
+      "dev": true,
       "dependencies": {
         "@humanwhocodes/object-schema": "^1.2.1",
         "debug": "^4.1.1",
@@ -2406,6 +2413,7 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
       "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==",
+      "dev": true,
       "engines": {
         "node": ">=12.22"
       },
@@ -2417,7 +2425,8 @@
     "node_modules/@humanwhocodes/object-schema": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",
-      "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA=="
+      "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
+      "dev": true
     },
     "node_modules/@jridgewell/gen-mapping": {
       "version": "0.1.1",
@@ -3402,6 +3411,7 @@
       "version": "8.8.2",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz",
       "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==",
+      "dev": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -3413,6 +3423,7 @@
       "version": "5.3.2",
       "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
       "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
+      "dev": true,
       "peerDependencies": {
         "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
       }
@@ -4049,6 +4060,7 @@
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
       "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+      "dev": true,
       "engines": {
         "node": ">=6"
       }
@@ -4502,6 +4514,7 @@
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
       "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "dev": true,
       "dependencies": {
         "ms": "2.1.2"
       },
@@ -4588,7 +4601,8 @@
     "node_modules/deep-is": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
-      "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="
+      "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
+      "dev": true
     },
     "node_modules/define-lazy-prop": {
       "version": "2.0.0",
@@ -4647,6 +4661,7 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
       "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
+      "dev": true,
       "dependencies": {
         "esutils": "^2.0.2"
       },
@@ -4852,6 +4867,7 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
       "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
+      "dev": true,
       "engines": {
         "node": ">=10"
       },
@@ -4863,6 +4879,7 @@
       "version": "8.42.0",
       "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.42.0.tgz",
       "integrity": "sha512-ulg9Ms6E1WPf67PHaEY4/6E2tEn5/f7FXGzr3t9cBMugOmf1INYvuUwwh1aXQN4MfJ6a5K2iNwP3w4AColvI9A==",
+      "dev": true,
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.2.0",
         "@eslint-community/regexpp": "^4.4.0",
@@ -5393,6 +5410,7 @@
       "version": "3.4.1",
       "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz",
       "integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==",
+      "dev": true,
       "engines": {
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       },
@@ -5404,6 +5422,7 @@
       "version": "6.12.6",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
       "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "dev": true,
       "dependencies": {
         "fast-deep-equal": "^3.1.1",
         "fast-json-stable-stringify": "^2.0.0",
@@ -5419,6 +5438,7 @@
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.0.tgz",
       "integrity": "sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw==",
+      "dev": true,
       "dependencies": {
         "esrecurse": "^4.3.0",
         "estraverse": "^5.2.0"
@@ -5433,12 +5453,14 @@
     "node_modules/eslint/node_modules/json-schema-traverse": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+      "dev": true
     },
     "node_modules/espree": {
       "version": "9.5.2",
       "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.2.tgz",
       "integrity": "sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw==",
+      "dev": true,
       "dependencies": {
         "acorn": "^8.8.0",
         "acorn-jsx": "^5.3.2",
@@ -5467,6 +5489,7 @@
       "version": "1.4.2",
       "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.2.tgz",
       "integrity": "sha512-JVSoLdTlTDkmjFmab7H/9SL9qGSyjElT3myyKp7krqjVFQCDLmj1QFaCLRFBszBKI0XVZaiiXvuPIX3ZwHe1Ng==",
+      "dev": true,
       "dependencies": {
         "estraverse": "^5.1.0"
       },
@@ -5478,6 +5501,7 @@
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
       "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+      "dev": true,
       "dependencies": {
         "estraverse": "^5.2.0"
       },
@@ -5489,6 +5513,7 @@
       "version": "5.3.0",
       "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
       "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+      "dev": true,
       "engines": {
         "node": ">=4.0"
       }
@@ -5497,6 +5522,7 @@
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
       "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
+      "dev": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -5527,7 +5553,8 @@
     "node_modules/fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
-      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+      "dev": true
     },
     "node_modules/fast-glob": {
       "version": "3.3.0",
@@ -5560,12 +5587,14 @@
     "node_modules/fast-json-stable-stringify": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
-      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="
+      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+      "dev": true
     },
     "node_modules/fast-levenshtein": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
-      "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="
+      "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
+      "dev": true
     },
     "node_modules/fast-url-parser": {
       "version": "1.1.3",
@@ -5623,6 +5652,7 @@
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
       "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==",
+      "dev": true,
       "dependencies": {
         "flat-cache": "^3.0.4"
       },
@@ -5645,6 +5675,7 @@
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
       "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==",
+      "dev": true,
       "dependencies": {
         "locate-path": "^6.0.0",
         "path-exists": "^4.0.0"
@@ -5676,6 +5707,7 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz",
       "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==",
+      "dev": true,
       "dependencies": {
         "flatted": "^3.1.0",
         "rimraf": "^3.0.2"
@@ -5687,7 +5719,8 @@
     "node_modules/flatted": {
       "version": "3.2.7",
       "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz",
-      "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ=="
+      "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
+      "dev": true
     },
     "node_modules/follow-redirects": {
       "version": "1.15.2",
@@ -5898,6 +5931,7 @@
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
       "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+      "dev": true,
       "dependencies": {
         "is-glob": "^4.0.3"
       },
@@ -5921,6 +5955,7 @@
       "version": "13.20.0",
       "resolved": "https://registry.npmjs.org/globals/-/globals-13.20.0.tgz",
       "integrity": "sha512-Qg5QtVkCy/kv3FUSlu4ukeZDVf9ee0iXLAUYX13gbR17bnejFTzr4iS9bY7kwCf1NztRNm1t91fjOiyx4CSwPQ==",
+      "dev": true,
       "dependencies": {
         "type-fest": "^0.20.2"
       },
@@ -5992,7 +6027,8 @@
     "node_modules/graphemer": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz",
-      "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag=="
+      "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==",
+      "dev": true
     },
     "node_modules/hard-rejection": {
       "version": "2.1.0",
@@ -6187,6 +6223,7 @@
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
       "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
+      "dev": true,
       "dependencies": {
         "parent-module": "^1.0.0",
         "resolve-from": "^4.0.0"
@@ -6202,6 +6239,7 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
       "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
+      "dev": true,
       "engines": {
         "node": ">=4"
       }
@@ -6210,6 +6248,7 @@
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
       "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
+      "dev": true,
       "engines": {
         "node": ">=0.8.19"
       }
@@ -6469,6 +6508,7 @@
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz",
       "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==",
+      "dev": true,
       "engines": {
         "node": ">=8"
       }
@@ -6709,7 +6749,8 @@
     "node_modules/json-stable-stringify-without-jsonify": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
-      "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw=="
+      "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==",
+      "dev": true
     },
     "node_modules/json5": {
       "version": "2.2.3",
@@ -6812,6 +6853,7 @@
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
       "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
+      "dev": true,
       "dependencies": {
         "prelude-ls": "^1.2.1",
         "type-check": "~0.4.0"
@@ -7132,6 +7174,7 @@
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
       "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==",
+      "dev": true,
       "dependencies": {
         "p-locate": "^5.0.0"
       },
@@ -7181,7 +7224,8 @@
     "node_modules/lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
-      "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="
+      "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
+      "dev": true
     },
     "node_modules/lodash.mergewith": {
       "version": "4.6.2",
@@ -7683,12 +7727,14 @@
     "node_modules/ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+      "dev": true
     },
     "node_modules/natural-compare": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
-      "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="
+      "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
+      "dev": true
     },
     "node_modules/natural-compare-lite": {
       "version": "1.4.0",
@@ -8131,6 +8177,7 @@
       "version": "0.9.1",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
       "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==",
+      "dev": true,
       "dependencies": {
         "deep-is": "^0.1.3",
         "fast-levenshtein": "^2.0.6",
@@ -8155,6 +8202,7 @@
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
       "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
+      "dev": true,
       "dependencies": {
         "yocto-queue": "^0.1.0"
       },
@@ -8169,6 +8217,7 @@
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
       "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
+      "dev": true,
       "dependencies": {
         "p-limit": "^3.0.2"
       },
@@ -8192,6 +8241,7 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
       "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
+      "dev": true,
       "dependencies": {
         "callsites": "^3.0.0"
       },
@@ -8319,6 +8369,7 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
       "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+      "dev": true,
       "engines": {
         "node": ">=8"
       }
@@ -8387,6 +8438,7 @@
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
       "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
+      "dev": true,
       "engines": {
         "node": ">= 0.8.0"
       }
@@ -9262,7 +9314,8 @@
     "node_modules/text-table": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
-      "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw=="
+      "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
+      "dev": true
     },
     "node_modules/through": {
       "version": "2.3.8",
@@ -9430,6 +9483,7 @@
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
       "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
+      "dev": true,
       "dependencies": {
         "prelude-ls": "^1.2.1"
       },
@@ -9441,6 +9495,7 @@
       "version": "0.20.2",
       "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
       "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+      "dev": true,
       "engines": {
         "node": ">=10"
       },
@@ -9577,6 +9632,7 @@
       "version": "4.4.1",
       "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
       "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
+      "dev": true,
       "dependencies": {
         "punycode": "^2.1.0"
       }
@@ -9585,6 +9641,7 @@
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
       "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
+      "dev": true,
       "engines": {
         "node": ">=6"
       }
@@ -9752,6 +9809,7 @@
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
       "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+      "dev": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -9851,6 +9909,7 @@
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
       "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
+      "dev": true,
       "engines": {
         "node": ">=10"
       },
@@ -11483,6 +11542,7 @@
       "version": "4.4.0",
       "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
       "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==",
+      "dev": true,
       "requires": {
         "eslint-visitor-keys": "^3.3.0"
       }
@@ -11490,12 +11550,14 @@
     "@eslint-community/regexpp": {
       "version": "4.5.1",
       "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.5.1.tgz",
-      "integrity": "sha512-Z5ba73P98O1KUYCCJTUeVpja9RcGoMdncZ6T49FCUl2lN38JtCJ+3WgIDBv0AuY4WChU5PmtJmOCTlN6FZTFKQ=="
+      "integrity": "sha512-Z5ba73P98O1KUYCCJTUeVpja9RcGoMdncZ6T49FCUl2lN38JtCJ+3WgIDBv0AuY4WChU5PmtJmOCTlN6FZTFKQ==",
+      "dev": true
     },
     "@eslint/eslintrc": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.3.tgz",
       "integrity": "sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ==",
+      "dev": true,
       "requires": {
         "ajv": "^6.12.4",
         "debug": "^4.3.2",
@@ -11512,6 +11574,7 @@
           "version": "6.12.6",
           "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
           "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+          "dev": true,
           "requires": {
             "fast-deep-equal": "^3.1.1",
             "fast-json-stable-stringify": "^2.0.0",
@@ -11522,19 +11585,22 @@
         "json-schema-traverse": {
           "version": "0.4.1",
           "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-          "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
+          "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+          "dev": true
         }
       }
     },
     "@eslint/js": {
       "version": "8.42.0",
       "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.42.0.tgz",
-      "integrity": "sha512-6SWlXpWU5AvId8Ac7zjzmIOqMOba/JWY8XZ4A7q7Gn1Vlfg/SFFIlrtHXt9nPn4op9ZPAkl91Jao+QQv3r/ukw=="
+      "integrity": "sha512-6SWlXpWU5AvId8Ac7zjzmIOqMOba/JWY8XZ4A7q7Gn1Vlfg/SFFIlrtHXt9nPn4op9ZPAkl91Jao+QQv3r/ukw==",
+      "dev": true
     },
     "@humanwhocodes/config-array": {
       "version": "0.11.10",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz",
       "integrity": "sha512-KVVjQmNUepDVGXNuoRRdmmEjruj0KfiGSbS8LVc12LMsWDQzRXJ0qdhN8L8uUigKpfEHRhlaQFY0ib1tnUbNeQ==",
+      "dev": true,
       "requires": {
         "@humanwhocodes/object-schema": "^1.2.1",
         "debug": "^4.1.1",
@@ -11544,12 +11610,14 @@
     "@humanwhocodes/module-importer": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
-      "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA=="
+      "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==",
+      "dev": true
     },
     "@humanwhocodes/object-schema": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",
-      "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA=="
+      "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
+      "dev": true
     },
     "@jridgewell/gen-mapping": {
       "version": "0.1.1",
@@ -12170,13 +12238,14 @@
     "acorn": {
       "version": "8.8.2",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz",
-      "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw=="
+      "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==",
+      "dev": true
     },
     "acorn-jsx": {
       "version": "5.3.2",
       "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
       "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
-      "requires": {}
+      "dev": true
     },
     "acorn-walk": {
       "version": "8.2.0",
@@ -12617,7 +12686,8 @@
     "callsites": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-      "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ=="
+      "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+      "dev": true
     },
     "camelcase": {
       "version": "5.3.1",
@@ -12901,8 +12971,7 @@
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.3.0.tgz",
       "integrity": "sha512-NTxV1MFfZDLPiBMjxbHRwSh5LaLcPMwNdCutmnHJCKoVnlvldPWlllonKwrsRJ5pYZBIBGRWWU2tfvzxgeSW5Q==",
-      "dev": true,
-      "requires": {}
+      "dev": true
     },
     "create-require": {
       "version": "1.1.1",
@@ -12941,6 +13010,7 @@
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
       "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "dev": true,
       "requires": {
         "ms": "2.1.2"
       }
@@ -13003,7 +13073,8 @@
     "deep-is": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
-      "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="
+      "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
+      "dev": true
     },
     "define-lazy-prop": {
       "version": "2.0.0",
@@ -13044,6 +13115,7 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
       "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
+      "dev": true,
       "requires": {
         "esutils": "^2.0.2"
       }
@@ -13212,12 +13284,14 @@
     "escape-string-regexp": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
-      "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="
+      "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
+      "dev": true
     },
     "eslint": {
       "version": "8.42.0",
       "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.42.0.tgz",
       "integrity": "sha512-ulg9Ms6E1WPf67PHaEY4/6E2tEn5/f7FXGzr3t9cBMugOmf1INYvuUwwh1aXQN4MfJ6a5K2iNwP3w4AColvI9A==",
+      "dev": true,
       "requires": {
         "@eslint-community/eslint-utils": "^4.2.0",
         "@eslint-community/regexpp": "^4.4.0",
@@ -13264,6 +13338,7 @@
           "version": "6.12.6",
           "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
           "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+          "dev": true,
           "requires": {
             "fast-deep-equal": "^3.1.1",
             "fast-json-stable-stringify": "^2.0.0",
@@ -13275,6 +13350,7 @@
           "version": "7.2.0",
           "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.0.tgz",
           "integrity": "sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw==",
+          "dev": true,
           "requires": {
             "esrecurse": "^4.3.0",
             "estraverse": "^5.2.0"
@@ -13283,7 +13359,8 @@
         "json-schema-traverse": {
           "version": "0.4.1",
           "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-          "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
+          "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+          "dev": true
         }
       }
     },
@@ -13390,8 +13467,7 @@
     "eslint-plugin-header": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/eslint-plugin-header/-/eslint-plugin-header-3.1.1.tgz",
-      "integrity": "sha512-9vlKxuJ4qf793CmeeSrZUvVClw6amtpghq3CuWcB5cUNnWHQhgcqy5eF8oVKFk1G3Y/CbchGfEaw3wiIJaNmVg==",
-      "requires": {}
+      "integrity": "sha512-9vlKxuJ4qf793CmeeSrZUvVClw6amtpghq3CuWcB5cUNnWHQhgcqy5eF8oVKFk1G3Y/CbchGfEaw3wiIJaNmVg=="
     },
     "eslint-plugin-import": {
       "version": "2.27.5",
@@ -13555,8 +13631,7 @@
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz",
       "integrity": "sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g==",
-      "dev": true,
-      "requires": {}
+      "dev": true
     },
     "eslint-plugin-react-native": {
       "version": "4.0.0",
@@ -13636,12 +13711,14 @@
     "eslint-visitor-keys": {
       "version": "3.4.1",
       "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz",
-      "integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA=="
+      "integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==",
+      "dev": true
     },
     "espree": {
       "version": "9.5.2",
       "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.2.tgz",
       "integrity": "sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw==",
+      "dev": true,
       "requires": {
         "acorn": "^8.8.0",
         "acorn-jsx": "^5.3.2",
@@ -13657,6 +13734,7 @@
       "version": "1.4.2",
       "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.2.tgz",
       "integrity": "sha512-JVSoLdTlTDkmjFmab7H/9SL9qGSyjElT3myyKp7krqjVFQCDLmj1QFaCLRFBszBKI0XVZaiiXvuPIX3ZwHe1Ng==",
+      "dev": true,
       "requires": {
         "estraverse": "^5.1.0"
       }
@@ -13665,6 +13743,7 @@
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
       "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+      "dev": true,
       "requires": {
         "estraverse": "^5.2.0"
       }
@@ -13672,12 +13751,14 @@
     "estraverse": {
       "version": "5.3.0",
       "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
-      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA=="
+      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+      "dev": true
     },
     "esutils": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
-      "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g=="
+      "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
+      "dev": true
     },
     "execa": {
       "version": "5.1.1",
@@ -13699,7 +13780,8 @@
     "fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
-      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+      "dev": true
     },
     "fast-glob": {
       "version": "3.3.0",
@@ -13728,12 +13810,14 @@
     "fast-json-stable-stringify": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
-      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="
+      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+      "dev": true
     },
     "fast-levenshtein": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
-      "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="
+      "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
+      "dev": true
     },
     "fast-url-parser": {
       "version": "1.1.3",
@@ -13755,8 +13839,7 @@
     "fdir": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.0.2.tgz",
-      "integrity": "sha512-XJVxBciDoEpRipMYyrTCqVQA4jMTfHNiYNy8OvIGTaQzEFPuMJEvmps+Rouo6rsnivkQax9s5m5gy1lHmY2Hmg==",
-      "requires": {}
+      "integrity": "sha512-XJVxBciDoEpRipMYyrTCqVQA4jMTfHNiYNy8OvIGTaQzEFPuMJEvmps+Rouo6rsnivkQax9s5m5gy1lHmY2Hmg=="
     },
     "figures": {
       "version": "3.2.0",
@@ -13777,6 +13860,7 @@
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
       "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==",
+      "dev": true,
       "requires": {
         "flat-cache": "^3.0.4"
       }
@@ -13793,6 +13877,7 @@
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
       "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==",
+      "dev": true,
       "requires": {
         "locate-path": "^6.0.0",
         "path-exists": "^4.0.0"
@@ -13815,6 +13900,7 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz",
       "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==",
+      "dev": true,
       "requires": {
         "flatted": "^3.1.0",
         "rimraf": "^3.0.2"
@@ -13823,7 +13909,8 @@
     "flatted": {
       "version": "3.2.7",
       "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz",
-      "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ=="
+      "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
+      "dev": true
     },
     "follow-redirects": {
       "version": "1.15.2",
@@ -13969,6 +14056,7 @@
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
       "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+      "dev": true,
       "requires": {
         "is-glob": "^4.0.3"
       }
@@ -13986,6 +14074,7 @@
       "version": "13.20.0",
       "resolved": "https://registry.npmjs.org/globals/-/globals-13.20.0.tgz",
       "integrity": "sha512-Qg5QtVkCy/kv3FUSlu4ukeZDVf9ee0iXLAUYX13gbR17bnejFTzr4iS9bY7kwCf1NztRNm1t91fjOiyx4CSwPQ==",
+      "dev": true,
       "requires": {
         "type-fest": "^0.20.2"
       }
@@ -14036,7 +14125,8 @@
     "graphemer": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz",
-      "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag=="
+      "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==",
+      "dev": true
     },
     "hard-rejection": {
       "version": "2.1.0",
@@ -14161,6 +14251,7 @@
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
       "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
+      "dev": true,
       "requires": {
         "parent-module": "^1.0.0",
         "resolve-from": "^4.0.0"
@@ -14169,14 +14260,16 @@
         "resolve-from": {
           "version": "4.0.0",
           "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
-          "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g=="
+          "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
+          "dev": true
         }
       }
     },
     "imurmurhash": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
-      "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA=="
+      "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
+      "dev": true
     },
     "indent-string": {
       "version": "4.0.0",
@@ -14351,7 +14444,8 @@
     "is-path-inside": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz",
-      "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ=="
+      "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==",
+      "dev": true
     },
     "is-plain-obj": {
       "version": "1.1.0",
@@ -14520,7 +14614,8 @@
     "json-stable-stringify-without-jsonify": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
-      "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw=="
+      "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==",
+      "dev": true
     },
     "json5": {
       "version": "2.2.3",
@@ -14600,6 +14695,7 @@
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
       "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
+      "dev": true,
       "requires": {
         "prelude-ls": "^1.2.1",
         "type-check": "~0.4.0"
@@ -14839,6 +14935,7 @@
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
       "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==",
+      "dev": true,
       "requires": {
         "p-locate": "^5.0.0"
       }
@@ -14882,7 +14979,8 @@
     "lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
-      "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="
+      "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
+      "dev": true
     },
     "lodash.mergewith": {
       "version": "4.6.2",
@@ -15267,12 +15365,14 @@
     "ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+      "dev": true
     },
     "natural-compare": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
-      "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="
+      "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
+      "dev": true
     },
     "natural-compare-lite": {
       "version": "1.4.0",
@@ -15598,6 +15698,7 @@
       "version": "0.9.1",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
       "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==",
+      "dev": true,
       "requires": {
         "deep-is": "^0.1.3",
         "fast-levenshtein": "^2.0.6",
@@ -15616,6 +15717,7 @@
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
       "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
+      "dev": true,
       "requires": {
         "yocto-queue": "^0.1.0"
       }
@@ -15624,6 +15726,7 @@
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
       "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
+      "dev": true,
       "requires": {
         "p-limit": "^3.0.2"
       }
@@ -15638,6 +15741,7 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
       "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
+      "dev": true,
       "requires": {
         "callsites": "^3.0.0"
       }
@@ -15731,7 +15835,8 @@
     "path-exists": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
-      "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="
+      "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+      "dev": true
     },
     "path-is-absolute": {
       "version": "1.0.1",
@@ -15781,7 +15886,8 @@
     "prelude-ls": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
-      "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g=="
+      "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
+      "dev": true
     },
     "prettier": {
       "version": "2.8.8",
@@ -16436,7 +16542,8 @@
     "text-table": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
-      "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw=="
+      "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
+      "dev": true
     },
     "through": {
       "version": "2.3.8",
@@ -16564,6 +16671,7 @@
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
       "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
+      "dev": true,
       "requires": {
         "prelude-ls": "^1.2.1"
       }
@@ -16571,7 +16679,8 @@
     "type-fest": {
       "version": "0.20.2",
       "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
-      "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ=="
+      "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+      "dev": true
     },
     "typed-array-length": {
       "version": "1.0.4",
@@ -16658,6 +16767,7 @@
       "version": "4.4.1",
       "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
       "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
+      "dev": true,
       "requires": {
         "punycode": "^2.1.0"
       },
@@ -16665,7 +16775,8 @@
         "punycode": {
           "version": "2.3.0",
           "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
-          "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA=="
+          "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
+          "dev": true
         }
       }
     },
@@ -16788,7 +16899,8 @@
     "word-wrap": {
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
-      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ=="
+      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+      "dev": true
     },
     "wrap-ansi": {
       "version": "7.0.0",
@@ -16862,7 +16974,8 @@
     "yocto-queue": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
-      "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="
+      "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
+      "dev": true
     }
   }
 }

packages/common/package-lock.json

@@ -28,8 +28,8 @@
       "dev": true,
       "license": "GPL-3.0-or-later",
       "dependencies": {
-        "@microsoft/signalr": "^6.0.7",
-        "@microsoft/signalr-protocol-msgpack": "^6.0.7",
+        "@microsoft/signalr": "^7.0.10",
+        "@microsoft/signalr-protocol-msgpack": "^7.0.10",
         "@notesnook/logger": "file:../logger",
         "@streetwriters/showdown": "^3.0.1-alpha.2",
         "async-mutex": "^0.3.2",
@@ -51,6 +51,7 @@
         "@types/katex": "^0.16.1",
         "@types/prismjs": "^1.26.0",
         "@types/showdown": "^2.0.0",
+        "@vitest/coverage-v8": "^0.34.1",
         "abortcontroller-polyfill": "^1.7.3",
         "cross-env": "^7.0.3",
         "dotenv": "^16.0.1",
@@ -140,8 +141,8 @@
     "@notesnook/core": {
       "version": "file:../core",
       "requires": {
-        "@microsoft/signalr": "^6.0.7",
-        "@microsoft/signalr-protocol-msgpack": "^6.0.7",
+        "@microsoft/signalr": "^7.0.10",
+        "@microsoft/signalr-protocol-msgpack": "^7.0.10",
         "@notesnook/crypto": "file:../crypto",
         "@notesnook/logger": "file:../logger",
         "@streetwriters/showdown": "^3.0.1-alpha.2",
@@ -149,6 +150,7 @@
         "@types/katex": "^0.16.1",
         "@types/prismjs": "^1.26.0",
         "@types/showdown": "^2.0.0",
+        "@vitest/coverage-v8": "^0.34.1",
         "abortcontroller-polyfill": "^1.7.3",
         "async-mutex": "^0.3.2",
         "cross-env": "^7.0.3",

packages/core/__mocks__/node-storage.mock.js

@@ -53,17 +53,16 @@ export class NodeStorageInterface {
   }
 
   async encrypt(password, data) {
-    return await this.crypto.encrypt(
-      password,
-      { format: "text", data },
-      "base64"
-    );
+    return await this.crypto.encrypt(password, data, "text", "base64");
+  }
+
+  async encryptMulti(password, items) {
+    return await this.crypto.encryptMulti(password, items, "text", "base64");
   }
 
   async decrypt(key, cipherData) {
     cipherData.format = "base64";
-    const result = await this.crypto.decrypt(key, cipherData);
-    if (typeof result.data === "string") return result.data;
+    return await this.crypto.decrypt(key, cipherData, "text");
   }
 
   async deriveCryptoKey(name, { password, salt }) {

packages/core/__tests__/content.test.js

@@ -24,7 +24,6 @@ test("adding a deleted content should not throw", () =>
   databaseTest().then(async (db) => {
     await expect(
       db.content.add({
-        remote: true,
         deleted: true,
         dateEdited: new Date(),
         id: "hello",

packages/core/__tests__/notebooks.test.js

@@ -20,6 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { notebookTest, TEST_NOTEBOOK, TEST_NOTE, delay } from "./utils";
 import { makeTopic } from "../src/collections/topics";
 import { test, expect } from "vitest";
+import qclone from "qclone";
 
 test("add a notebook", () =>
   notebookTest().then(({ db, id }) => {
@@ -62,47 +63,61 @@ test("merge notebook with new topics", () =>
   notebookTest().then(async ({ db, id }) => {
     let notebook = db.notebooks.notebook(id);
 
-    const newNotebook = { ...notebook.data, remote: true };
-    newNotebook.topics.push(makeTopic("Home", id));
+    const newNotebook = db.notebooks.merge(notebook.data, {
+      ...notebook.data,
+      topics: [...notebook.data.topics, makeTopic("Home", id)],
+      remote: true
+    });
 
-    await expect(db.notebooks.merge(newNotebook)).resolves.not.toThrow();
-
-    expect(notebook.topics.has("Home")).toBe(true);
-    expect(notebook.topics.has("hello")).toBe(true);
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "Home")
+    ).toBeGreaterThanOrEqual(0);
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "hello")
+    ).toBeGreaterThanOrEqual(0);
   }));
 
 test("merge notebook with topics removed", () =>
   notebookTest().then(async ({ db, id }) => {
     let notebook = db.notebooks.notebook(id);
 
-    const newNotebook = { ...notebook.data, remote: true };
-    newNotebook.topics.splice(0, 1); // remove hello topic
-    newNotebook.topics.push(makeTopic("Home", id));
+    const newNotebook = db.notebooks.merge(notebook.data, {
+      ...notebook.data,
+      topics: [makeTopic("Home", id)],
+      remote: true
+    });
 
-    await expect(db.notebooks.merge(newNotebook)).resolves.not.toThrow();
-
-    expect(notebook.topics.has("Home")).toBe(true);
-    expect(notebook.topics.has("hello")).toBe(false);
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "Home")
+    ).toBeGreaterThanOrEqual(0);
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "hello")
+    ).toBeLessThan(0);
   }));
 
 test("merge notebook with topic edited", () =>
   notebookTest().then(async ({ db, id }) => {
     let notebook = db.notebooks.notebook(id);
 
-    const newNotebook = { ...notebook.data, remote: true };
-    newNotebook.topics[0].title = "hello (edited)";
+    const newNotebook = db.notebooks.merge(notebook.data, {
+      ...notebook.data,
+      topics: [{ ...notebook.data.topics[0], title: "hello (edited)" }],
+      remote: true
+    });
 
-    await expect(db.notebooks.merge(newNotebook)).resolves.not.toThrow();
-
-    expect(notebook.topics.has("hello (edited)")).toBe(true);
-    expect(notebook.topics.has("hello")).toBe(false);
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "hello (edited)")
+    ).toBeGreaterThanOrEqual(0);
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "hello")
+    ).toBeLessThan(0);
   }));
 
 test("merge notebook when local notebook is also edited", () =>
   notebookTest().then(async ({ db, id }) => {
     let notebook = db.notebooks.notebook(id);
 
-    const newNotebook = { ...notebook.data, remote: true };
+    let newNotebook = { ...qclone(notebook.data), remote: true };
     newNotebook.topics[0].title = "hello (edited)";
 
     await delay(500);
@@ -112,11 +127,20 @@ test("merge notebook when local notebook is also edited", () =>
       title: "hello (edited too)"
     });
 
-    await expect(db.notebooks.merge(newNotebook)).resolves.not.toThrow();
-
-    expect(notebook.topics.has("hello (edited too)")).toBe(true);
-    expect(notebook.topics.has("hello (edited)")).toBe(false);
-    expect(notebook.topics.has("hello")).toBe(false);
+    newNotebook = db.notebooks.merge(
+      db.notebooks.notebook(id).data,
+      newNotebook,
+      0
+    );
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "hello (edited too)")
+    ).toBeGreaterThanOrEqual(0);
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "hello (edited)")
+    ).toBeLessThan(0);
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "hello")
+    ).toBeLessThan(0);
   }));
 
 test("merging notebook when local notebook is not edited should not update remote notebook dateEdited", () =>
@@ -129,9 +153,10 @@ test("merging notebook when local notebook is not edited should not update remot
       note
     );
 
-    const newNotebook = { ...notebook.data, remote: true };
-
-    await expect(db.notebooks.merge(newNotebook)).resolves.not.toThrow();
+    const newNotebook = db.notebooks.merge(notebook.data, {
+      ...notebook.data,
+      remote: true
+    });
 
     expect(db.notebooks.notebook(id).dateEdited).toBe(newNotebook.dateEdited);
   }));
@@ -140,10 +165,10 @@ test("merge notebook with topic removed that is edited in the local notebook", (
   notebookTest().then(async ({ db, id }) => {
     let notebook = db.notebooks.notebook(id);
 
-    const newNotebook = { ...notebook.data, remote: true };
+    let newNotebook = { ...qclone(notebook.data), remote: true };
     newNotebook.topics.splice(0, 1); // remove hello topic
 
-    await db.storage.write("lastSynced", Date.now());
+    const lastSynced = Date.now();
 
     await delay(500);
 
@@ -152,8 +177,16 @@ test("merge notebook with topic removed that is edited in the local notebook", (
       title: "hello (i exist)"
     });
 
-    await expect(db.notebooks.merge(newNotebook)).resolves.not.toThrow();
+    newNotebook = db.notebooks.merge(
+      db.notebooks.notebook(id).data,
+      newNotebook,
+      lastSynced
+    );
 
-    expect(notebook.topics.has("hello (i exist)")).toBe(true);
-    expect(notebook.topics.has("hello")).toBe(false);
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "hello (i exist)")
+    ).toBeGreaterThanOrEqual(0);
+    expect(
+      newNotebook.topics.findIndex((t) => t.title === "hello")
+    ).toBeLessThan(0);
   }));

packages/core/__tests__/settings.test.js

@@ -30,10 +30,10 @@ test("settings' dateModified should not update on init", () =>
 
 test("settings' dateModified should update after merge conflict resolve", () =>
   databaseTest().then(async (db) => {
-    await db.storage.write("lastSynced", 0);
-    const beforeDateModified = (db.settings._settings.dateModified = 1);
-    await db.settings.merge({ groupOptions: {}, aliases: {} });
-    const afterDateModified = db.settings._settings.dateModified;
+    // await db.storage.write("lastSynced", 0);
+    const beforeDateModified = (db.settings.raw.dateModified = 1);
+    await db.settings.merge({ groupOptions: {}, aliases: {} }, 0);
+    const afterDateModified = db.settings.raw.dateModified;
     expect(afterDateModified).toBeGreaterThan(beforeDateModified);
   }));
 

packages/core/package-lock.json

@@ -9,8 +9,8 @@
       "version": "7.4.1",
       "license": "GPL-3.0-or-later",
       "dependencies": {
-        "@microsoft/signalr": "^6.0.7",
-        "@microsoft/signalr-protocol-msgpack": "^6.0.7",
+        "@microsoft/signalr": "^7.0.10",
+        "@microsoft/signalr-protocol-msgpack": "^7.0.10",
         "@notesnook/logger": "file:../logger",
         "@streetwriters/showdown": "^3.0.1-alpha.2",
         "async-mutex": "^0.3.2",
@@ -491,34 +491,26 @@
       }
     },
     "node_modules/@microsoft/signalr": {
-      "version": "6.0.11",
-      "resolved": "https://registry.npmjs.org/@microsoft/signalr/-/signalr-6.0.11.tgz",
-      "integrity": "sha512-5flnqEgl+7AK3+NCcavmzC4AKRt1bqa/T3zd8acyQRf/VyUe2nsFgSdaXQKZx4z3v8OIe1aQloRDvDEI4JjpAg==",
+      "version": "7.0.10",
+      "resolved": "https://registry.npmjs.org/@microsoft/signalr/-/signalr-7.0.10.tgz",
+      "integrity": "sha512-tOEn32i5EatAx4sZbzmLgcBc2VbKQmx+F4rI2/Ioq2MnBaYcFxbDzOoZgISIS4IR9H1ij/sKoU8zQOAFC8GJKg==",
       "dependencies": {
         "abort-controller": "^3.0.0",
-        "eventsource": "^1.0.7",
-        "fetch-cookie": "^0.11.0",
+        "eventsource": "^2.0.2",
+        "fetch-cookie": "^2.0.3",
         "node-fetch": "^2.6.7",
         "ws": "^7.4.5"
       }
     },
     "node_modules/@microsoft/signalr-protocol-msgpack": {
-      "version": "6.0.11",
-      "resolved": "https://registry.npmjs.org/@microsoft/signalr-protocol-msgpack/-/signalr-protocol-msgpack-6.0.11.tgz",
-      "integrity": "sha512-OgIxwTFZFfcM9wZSNcGMc2xWaC0G/sSZjXG3rScgqKnLV2QW88787Nw25py39o4UdeFB89btGXOlgF6zvyOmTg==",
+      "version": "7.0.10",
+      "resolved": "https://registry.npmjs.org/@microsoft/signalr-protocol-msgpack/-/signalr-protocol-msgpack-7.0.10.tgz",
+      "integrity": "sha512-iZacNFQ3+BT3wZjFN2qcuQQJWK0ZlyCek4plWw1QrFqqOMBYEwPY4BCbLcwNZcTiOpTK65es1CCf3Yxb6lwlVQ==",
       "dependencies": {
-        "@microsoft/signalr": ">=6.0.11",
+        "@microsoft/signalr": ">=7.0.10",
         "@msgpack/msgpack": "^2.7.0"
       }
     },
-    "node_modules/@microsoft/signalr/node_modules/eventsource": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-1.1.2.tgz",
-      "integrity": "sha512-xAH3zWhgO2/3KIniEKYPr8plNSzlGINOUqYj0m0u7AB81iRw8b/3E73W6AuU+6klLbaSFmZnaETQ2lXPfAydrA==",
-      "engines": {
-        "node": ">=0.12.0"
-      }
-    },
     "node_modules/@msgpack/msgpack": {
       "version": "2.8.0",
       "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-2.8.0.tgz",
@@ -1311,20 +1303,17 @@
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-2.0.2.tgz",
       "integrity": "sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==",
-      "dev": true,
       "engines": {
         "node": ">=12.0.0"
       }
     },
     "node_modules/fetch-cookie": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/fetch-cookie/-/fetch-cookie-0.11.0.tgz",
-      "integrity": "sha512-BQm7iZLFhMWFy5CZ/162sAGjBfdNWb7a8LEqqnzsHFhxT/X/SVj/z2t2nu3aJvjlbQkrAlTUApplPRjWyH4mhA==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/fetch-cookie/-/fetch-cookie-2.1.0.tgz",
+      "integrity": "sha512-39+cZRbWfbibmj22R2Jy6dmTbAWC+oqun1f1FzQaNurkPDUP4C38jpeZbiXCR88RKRVDp8UcDrbFXkNhN+NjYg==",
       "dependencies": {
-        "tough-cookie": "^2.3.3 || ^3.0.1 || ^4.0.0"
-      },
-      "engines": {
-        "node": ">=8"
+        "set-cookie-parser": "^2.4.8",
+        "tough-cookie": "^4.0.0"
       }
     },
     "node_modules/fs.realpath": {
@@ -2036,9 +2025,9 @@
       "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag=="
     },
     "node_modules/punycode": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
-      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
+      "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
       "engines": {
         "node": ">=6"
       }
@@ -2153,6 +2142,11 @@
         "node": ">=10"
       }
     },
+    "node_modules/set-cookie-parser": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.6.0.tgz",
+      "integrity": "sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ=="
+    },
     "node_modules/shebang-command": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -2297,9 +2291,9 @@
       }
     },
     "node_modules/tough-cookie": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz",
-      "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==",
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
+      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
       "dependencies": {
         "psl": "^1.1.33",
         "punycode": "^2.1.1",
@@ -2858,30 +2852,23 @@
       }
     },
     "@microsoft/signalr": {
-      "version": "6.0.11",
-      "resolved": "https://registry.npmjs.org/@microsoft/signalr/-/signalr-6.0.11.tgz",
-      "integrity": "sha512-5flnqEgl+7AK3+NCcavmzC4AKRt1bqa/T3zd8acyQRf/VyUe2nsFgSdaXQKZx4z3v8OIe1aQloRDvDEI4JjpAg==",
+      "version": "7.0.10",
+      "resolved": "https://registry.npmjs.org/@microsoft/signalr/-/signalr-7.0.10.tgz",
+      "integrity": "sha512-tOEn32i5EatAx4sZbzmLgcBc2VbKQmx+F4rI2/Ioq2MnBaYcFxbDzOoZgISIS4IR9H1ij/sKoU8zQOAFC8GJKg==",
       "requires": {
         "abort-controller": "^3.0.0",
-        "eventsource": "^1.0.7",
-        "fetch-cookie": "^0.11.0",
+        "eventsource": "^2.0.2",
+        "fetch-cookie": "^2.0.3",
         "node-fetch": "^2.6.7",
         "ws": "^7.4.5"
-      },
-      "dependencies": {
-        "eventsource": {
-          "version": "1.1.2",
-          "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-1.1.2.tgz",
-          "integrity": "sha512-xAH3zWhgO2/3KIniEKYPr8plNSzlGINOUqYj0m0u7AB81iRw8b/3E73W6AuU+6klLbaSFmZnaETQ2lXPfAydrA=="
-        }
       }
     },
     "@microsoft/signalr-protocol-msgpack": {
-      "version": "6.0.11",
-      "resolved": "https://registry.npmjs.org/@microsoft/signalr-protocol-msgpack/-/signalr-protocol-msgpack-6.0.11.tgz",
-      "integrity": "sha512-OgIxwTFZFfcM9wZSNcGMc2xWaC0G/sSZjXG3rScgqKnLV2QW88787Nw25py39o4UdeFB89btGXOlgF6zvyOmTg==",
+      "version": "7.0.10",
+      "resolved": "https://registry.npmjs.org/@microsoft/signalr-protocol-msgpack/-/signalr-protocol-msgpack-7.0.10.tgz",
+      "integrity": "sha512-iZacNFQ3+BT3wZjFN2qcuQQJWK0ZlyCek4plWw1QrFqqOMBYEwPY4BCbLcwNZcTiOpTK65es1CCf3Yxb6lwlVQ==",
       "requires": {
-        "@microsoft/signalr": ">=6.0.11",
+        "@microsoft/signalr": ">=7.0.10",
         "@msgpack/msgpack": "^2.7.0"
       }
     },
@@ -3494,15 +3481,15 @@
     "eventsource": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-2.0.2.tgz",
-      "integrity": "sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==",
-      "dev": true
+      "integrity": "sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA=="
     },
     "fetch-cookie": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/fetch-cookie/-/fetch-cookie-0.11.0.tgz",
-      "integrity": "sha512-BQm7iZLFhMWFy5CZ/162sAGjBfdNWb7a8LEqqnzsHFhxT/X/SVj/z2t2nu3aJvjlbQkrAlTUApplPRjWyH4mhA==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/fetch-cookie/-/fetch-cookie-2.1.0.tgz",
+      "integrity": "sha512-39+cZRbWfbibmj22R2Jy6dmTbAWC+oqun1f1FzQaNurkPDUP4C38jpeZbiXCR88RKRVDp8UcDrbFXkNhN+NjYg==",
       "requires": {
-        "tough-cookie": "^2.3.3 || ^3.0.1 || ^4.0.0"
+        "set-cookie-parser": "^2.4.8",
+        "tough-cookie": "^4.0.0"
       }
     },
     "fs.realpath": {
@@ -4035,9 +4022,9 @@
       "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag=="
     },
     "punycode": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
-      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
+      "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA=="
     },
     "qclone": {
       "version": "1.2.0",
@@ -4123,6 +4110,11 @@
         "lru-cache": "^6.0.0"
       }
     },
+    "set-cookie-parser": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.6.0.tgz",
+      "integrity": "sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ=="
+    },
     "shebang-command": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -4235,9 +4227,9 @@
       "dev": true
     },
     "tough-cookie": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz",
-      "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==",
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
+      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
       "requires": {
         "psl": "^1.1.33",
         "punycode": "^2.1.1",
@@ -4419,8 +4411,7 @@
     "ws": {
       "version": "7.5.9",
       "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz",
-      "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==",
-      "requires": {}
+      "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q=="
     },
     "yallist": {
       "version": "4.0.0",

packages/core/package.json

@@ -35,8 +35,8 @@
     "test": "vitest run"
   },
   "dependencies": {
-    "@microsoft/signalr": "^6.0.7",
-    "@microsoft/signalr-protocol-msgpack": "^6.0.7",
+    "@microsoft/signalr": "^7.0.10",
+    "@microsoft/signalr-protocol-msgpack": "^7.0.10",
     "@notesnook/logger": "file:../logger",
     "@streetwriters/showdown": "^3.0.1-alpha.2",
     "async-mutex": "^0.3.2",

packages/core/src/api/settings.js

@@ -45,24 +45,24 @@ class Settings {
     return this._settings;
   }
 
-  async merge(item) {
-    if (this._settings.dateModified > (await this._db.lastSynced())) {
-      this._settings.id = item.id;
+  async merge(remoteItem, lastSynced) {
+    if (this._settings.dateModified > lastSynced) {
+      this._settings.id = remoteItem.id;
       this._settings.groupOptions = {
         ...this._settings.groupOptions,
-        ...item.groupOptions
+        ...remoteItem.groupOptions
       };
       this._settings.toolbarConfig = {
         ...this._settings.toolbarConfig,
-        ...item.toolbarConfig
+        ...remoteItem.toolbarConfig
       };
       this._settings.aliases = {
         ...this._settings.aliases,
-        ...item.aliases
+        ...remoteItem.aliases
       };
       this._settings.dateModified = Date.now();
     } else {
-      this._initSettings(item);
+      this._initSettings(remoteItem);
     }
     await this._saveSettings(false);
   }
@@ -215,7 +215,11 @@ class Settings {
     }
 
     await this._db.storage.write("settings", this._settings);
-    this._db.eventManager.publish(EVENTS.databaseUpdated, this._settings);
+    this._db.eventManager.publish(
+      EVENTS.databaseUpdated,
+      "settings",
+      this._settings
+    );
   }
 }
 export default Settings;

packages/core/src/api/sync/__tests__/collector.test.js

@@ -17,30 +17,39 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { databaseTest, TEST_NOTE, delay } from "../../../../__tests__/utils";
+import {
+  databaseTest,
+  TEST_NOTE,
+  delay,
+  loginFakeUser
+} from "../../../../__tests__/utils";
 import Collector from "../collector";
 import { test, expect } from "vitest";
 
 test("newly created note should get included in collector", () =>
   databaseTest().then(async (db) => {
+    await loginFakeUser(db);
     const collector = new Collector(db);
 
     const lastSyncedTime = Date.now() - 10000;
 
     const noteId = await db.notes.add(TEST_NOTE);
 
-    const data = await collector.collect(lastSyncedTime);
+    const items = [];
+    for await (const item of collector.collect(100, lastSyncedTime, false)) {
+      items.push(item);
+    }
 
-    expect(data.items).toHaveLength(2);
-    expect(data.items[0].type).toBe("note");
-    expect(data.items[0].id).toBe(noteId);
-    expect(data.types[0]).toBe("note");
-    expect(data.types[1]).toBe("content");
-    expect(data.items[1].type).toBe("tiptap");
+    expect(items).toHaveLength(2);
+    expect(items[0].items[0].id).toBe(noteId);
+    expect(items[0].type).toBe("note");
+    expect(items[1].type).toBe("content");
+    expect(items[1].items[0].id).toBe(db.notes.note(noteId).data.contentId);
   }));
 
 test("edited note after last synced time should get included in collector", () =>
   databaseTest().then(async (db) => {
+    await loginFakeUser(db);
     const collector = new Collector(db);
     const noteId = await db.notes.add(TEST_NOTE);
 
@@ -50,14 +59,18 @@ test("edited note after last synced time should get included in collector", () =
 
     await db.notes.add({ id: noteId, pinned: true });
 
-    const data = await collector.collect(lastSyncedTime);
+    const items = [];
+    for await (const item of collector.collect(100, lastSyncedTime, false)) {
+      items.push(item);
+    }
 
-    expect(data.items).toHaveLength(1);
-    expect(data.items[0].id).toBe(noteId);
+    expect(items).toHaveLength(1);
+    expect(items[0].items[0].id).toBe(noteId);
   }));
 
 test("note edited before last synced time should not get included in collector", () =>
   databaseTest().then(async (db) => {
+    await loginFakeUser(db);
     const collector = new Collector(db);
     const noteId = await db.notes.add(TEST_NOTE);
 
@@ -67,21 +80,29 @@ test("note edited before last synced time should not get included in collector",
 
     const lastSyncedTime = Date.now();
 
-    const data = await collector.collect(lastSyncedTime);
-    const notes = data.items.filter((i) => i.collectionId === "note");
+    const items = [];
+    for await (const item of collector.collect(100, lastSyncedTime, false)) {
+      items.push(item);
+    }
 
-    expect(notes).toHaveLength(0);
+    expect(items).toHaveLength(0);
   }));
 
 test("localOnly note should get included as a deleted item in collector", () =>
   databaseTest().then(async (db) => {
+    await loginFakeUser(db);
     const collector = new Collector(db);
     await db.notes.add({ ...TEST_NOTE, localOnly: true });
 
-    const data = await collector.collect(0);
-    expect(data.items).toHaveLength(2);
-    expect(data.items[0].deleted).toBe(true);
-    expect(data.items[1].deleted).toBe(true);
-    expect(data.types[0]).toBe("note");
-    expect(data.types[1]).toBe("content");
+    const items = [];
+    for await (const item of collector.collect(100, 0, false)) {
+      items.push(item);
+    }
+
+    expect(items).toHaveLength(2);
+
+    expect(items[0].items[0].length).toBe(104);
+    expect(items[1].items[0].length).toBe(104);
+    expect(items[0].type).toBe("note");
+    expect(items[1].type).toBe("content");
   }));

packages/core/src/api/sync/auto-sync.js

@@ -59,7 +59,11 @@ export class AutoSync {
    * @private
    */
   schedule(id, item) {
-    if (item && (item.remote || item.localOnly || item.failed)) return;
+    if (
+      item &&
+      (item.remote || item.localOnly || item.failed || !!item.dateUploaded)
+    )
+      return;
 
     clearTimeout(this.timeout);
     // auto sync interval must not be 0 to avoid issues

packages/core/src/api/sync/collector.js

@@ -20,6 +20,18 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { CURRENT_DATABASE_VERSION } from "../../common";
 import { logger } from "../../logger";
 
+const SYNC_COLLECTIONS_MAP = {
+  attachment: "attachments",
+  note: "notes",
+  notebook: "notebooks",
+  shortcut: "shortcuts",
+  reminder: "reminders",
+  relation: "relations"
+};
+
+const ASYNC_COLLECTIONS_MAP = {
+  content: "content"
+};
 class Collector {
   /**
    *
@@ -30,95 +42,106 @@ class Collector {
     this.logger = logger.scope("SyncCollector");
   }
 
-  async collect(lastSyncedTimestamp, isForceSync) {
-    await this._db.notes.init();
-
-    this._lastSyncedTimestamp = lastSyncedTimestamp;
-    this.key = await this._db.user.getEncryptionKey();
-    const vaultKey = await this._db.vault._getKey();
-
-    const collections = {
-      note: this._db.notes.raw,
-      shortcut: this._db.shortcuts.raw,
-      notebook: this._db.notebooks.raw,
-      content: await this._db.content.all(),
-      attachment: this._db.attachments.syncable,
-      reminder: this._db.reminders.raw,
-      relation: this._db.relations.raw,
-      settings: [this._db.settings.raw]
-    };
-
-    const result = { items: [], types: [] };
-    for (const type in collections) {
-      this._collect(type, collections[type], result, isForceSync);
-    }
-
-    if (vaultKey) {
-      result.items.push(vaultKey);
-      result.types.push("vaultKey");
-    }
-
-    return result;
-  }
-
-  _serialize(item) {
-    if (!item) return null;
-    return this._db.storage.encrypt(this.key, JSON.stringify(item));
-  }
-
-  encrypt(array) {
-    if (!array.length) return [];
-    return Promise.all(array.map(this._map, this));
-  }
-
-  _collect(itemType, items, result, isForceSync) {
-    if (!items || !items.length) return;
-
-    for (const item of items) {
-      if (!item) continue;
-
-      const isSyncable = !item.synced || isForceSync;
-      const isUnsynced =
-        item.dateModified > this._lastSyncedTimestamp || isForceSync;
-
-      if (item.localOnly) {
-        result.items.push({
-          id: item.id,
-          deleted: true,
-          dateModified: item.dateModified,
-          deleteReason: "localOnly"
-        });
-        result.types.push(itemType);
-      } else if (isUnsynced && isSyncable) {
-        result.items.push(item);
-        result.types.push(itemType);
+  async *collect(chunkSize, lastSyncedTimestamp, isForceSync) {
+    const key = await this._db.user.getEncryptionKey();
+    for (const itemType in SYNC_COLLECTIONS_MAP) {
+      const collectionKey = SYNC_COLLECTIONS_MAP[itemType];
+      const collection = this._db[collectionKey]._collection;
+      for (const chunk of collection.iterateSync(chunkSize)) {
+        const items = await this.prepareChunk(
+          chunk,
+          lastSyncedTimestamp,
+          isForceSync,
+          key,
+          itemType
+        );
+        if (!items) continue;
+        yield items;
       }
     }
+
+    for (const itemType in ASYNC_COLLECTIONS_MAP) {
+      const collectionKey = ASYNC_COLLECTIONS_MAP[itemType];
+      const collection = this._db[collectionKey]._collection;
+      for await (const chunk of collection.iterate(chunkSize)) {
+        const items = await this.prepareChunk(
+          chunk.map((item) => item[1]),
+          lastSyncedTimestamp,
+          isForceSync,
+          key,
+          itemType
+        );
+        if (!items) continue;
+        yield items;
+      }
+    }
+
+    const items = await this.prepareChunk(
+      [this._db.settings.raw],
+      lastSyncedTimestamp,
+      isForceSync,
+      key,
+      "settings"
+    );
+    if (!items) return;
+    yield items;
   }
 
-  // _map(item) {
-  //   return {
-  //     id: item.id,
-  //     v: CURRENT_DATABASE_VERSION,
-  //     iv: item.iv,
-  //     cipher: item.cipher,
-  //     length: item.length,
-  //     alg: item.alg,
-  //     dateModified: item.dateModified,
-  //   };
-  // }
+  async prepareChunk(chunk, lastSyncedTimestamp, isForceSync, key, itemType) {
+    const { ids, items } = filterSyncableItems(
+      chunk,
+      lastSyncedTimestamp,
+      isForceSync
+    );
+    if (!ids.length) return;
+    const ciphers = await this._db.storage.encryptMulti(key, items);
+    return toPushItem(itemType, ids, ciphers);
+  }
+}
+export default Collector;
+
+function toPushItem(type, ids, ciphers) {
+  const items = ciphers.map((cipher, index) => {
+    cipher.v = CURRENT_DATABASE_VERSION;
+    cipher.id = ids[index];
+    return cipher;
+  });
+  return {
+    items,
+    type
+  };
+}
+
+function filterSyncableItems(items, lastSyncedTimestamp, isForceSync) {
+  if (!items || !items.length) return { items: [], ids: [] };
+
+  const ids = [];
+  const syncableItems = [];
+  for (const item of items) {
+    if (!item) continue;
+
+    const isSyncable = !item.synced || isForceSync;
+    const isUnsynced = item.dateModified > lastSyncedTimestamp || isForceSync;
 
-  async _map(item) {
     // in case of resolved content
     delete item.resolved;
     // synced is a local only property
     delete item.synced;
 
-    return {
-      id: item.id,
-      v: CURRENT_DATABASE_VERSION,
-      ...(await this._serialize(item))
-    };
+    if (item.localOnly) {
+      ids.push(item.id);
+      syncableItems.push(
+        JSON.stringify({
+          id: item.id,
+          deleted: true,
+          dateModified: item.dateModified,
+          deleteReason: "localOnly"
+        })
+      );
+    } else if (isUnsynced && isSyncable) {
+      ids.push(item.id);
+      syncableItems.push(JSON.stringify(item));
+    }
   }
+  return { items: syncableItems, ids };
 }
-export default Collector;

packages/core/src/api/sync/index.js

@@ -32,32 +32,18 @@ import * as signalr from "@microsoft/signalr";
 import Merger from "./merger";
 import Conflicts from "./conflicts";
 import { AutoSync } from "./auto-sync";
-import { toChunks } from "../../utils/array";
 import { MessagePackHubProtocol } from "@microsoft/signalr-protocol-msgpack";
 import { logger } from "../../logger";
 import { Mutex } from "async-mutex";
+import { migrateItem } from "../../migrations";
 
 /**
  * @typedef {{
- *  item: string,
- *  itemType: string,
- *  lastSynced: number,
- *  current: number,
- *  total: number,
- *  synced?: boolean
+ *  items: any[],
+ *  type: string,
  * }} SyncTransferItem
  */
 
-/**
- * @typedef {{
- *  items:  string[],
- *  types: string[],
- *  lastSynced: number,
- *  current: number,
- *  total: number,
- * }} BatchedSyncTransferItem
- */
-
 export default class SyncManager {
   /**
    *
@@ -123,6 +109,15 @@ class Sync {
     this.autoSync = new AutoSync(db, 1000);
     this.logger = logger.scope("Sync");
     this.syncConnectionMutex = new Mutex();
+    this.itemTypeToCollection = {
+      note: "notes",
+      notebook: "notebooks",
+      content: "content",
+      attachment: "attachments",
+      relation: "relations",
+      reminder: "reminders",
+      shortcut: "shortcuts"
+    };
     let remoteSyncTimeout = 0;
 
     const tokenManager = new TokenManager(db.storage);
@@ -149,30 +144,34 @@ class Sync {
       })
       .withHubProtocol(new MessagePackHubProtocol({ ignoreUndefined: true }))
       .build();
-
+    this.connection.serverTimeoutInMilliseconds = 60 * 1000 * 5;
     EV.subscribe(EVENTS.userLoggedOut, async () => {
       await this.connection.stop();
       this.autoSync.stop();
     });
 
-    this.connection.on("SyncItem", async (payload) => {
+    let count = 0;
+    this.connection.on("PushItems", async (chunk) => {
+      if (this.connection.state !== signalr.HubConnectionState.Connected)
+        return;
+
+      count += chunk.items.length;
+      sendSyncProgressEvent(this.db.eventManager, "download", count);
+
       clearTimeout(remoteSyncTimeout);
       remoteSyncTimeout = setTimeout(() => {
-        db.eventManager.publish(EVENTS.syncAborted);
+        this.db.eventManager.publish(EVENTS.syncAborted);
       }, 15000);
 
-      await this.onSyncItem(payload);
-      sendSyncProgressEvent(
-        this.db.eventManager,
-        "download",
-        payload.total,
-        payload.current
-      );
+      const key = await this.db.user.getEncryptionKey();
+      const dbLastSynced = await this.db.lastSynced();
+      await this.processChunk(chunk, key, dbLastSynced, true);
     });
 
-    this.connection.on("RemoteSyncCompleted", (lastSynced) => {
+    this.connection.on("PushCompleted", (lastSynced) => {
+      count = 0;
       clearTimeout(remoteSyncTimeout);
-      this.onRemoteSyncCompleted(lastSynced);
+      this.onPushCompleted(lastSynced);
     });
   }
 
@@ -192,6 +191,8 @@ class Sync {
     this.logger.info("Starting sync", { full, force, serverLastSynced });
 
     this.connection.onclose((error) => {
+      this.db.eventManager.publish(EVENTS.syncAborted);
+      console.error(error);
       this.logger.error(error || new Error("Connection closed."));
       throw new Error("Connection closed.");
     });
@@ -199,17 +200,12 @@ class Sync {
     const { lastSynced, oldLastSynced } = await this.init(force);
     this.logger.info("Initialized sync", { lastSynced, oldLastSynced });
 
-    const { newLastSynced, data } = await this.collect(lastSynced, force);
-    this.logger.info("Data collected for sync", {
-      newLastSynced,
-      length: data.items.length,
-      isEmpty: data.items.length <= 0
-    });
+    const newLastSynced = Date.now();
 
     const serverResponse = full ? await this.fetch(lastSynced) : null;
     this.logger.info("Data fetched", serverResponse);
 
-    if (await this.send(data, newLastSynced)) {
+    if (await this.send(lastSynced, force, newLastSynced)) {
       this.logger.info("New data sent");
       await this.stop(newLastSynced);
     } else if (serverResponse) {
@@ -243,95 +239,82 @@ class Sync {
   async fetch(lastSynced) {
     await this.checkConnection();
 
-    const serverResponse = await new Promise((resolve, reject) => {
-      let counter = { count: 0, queue: null };
-      this.connection.stream("FetchItems", lastSynced).subscribe({
-        next: (/** @type {SyncTransferItem} */ syncStatus) => {
-          const { total, item, synced, lastSynced } = syncStatus;
-          if (synced) {
-            resolve({ synced, lastSynced });
-            return;
-          }
-          if (!item) return;
-          if (counter.queue === null) counter.queue = total;
+    const key = await this.db.user.getEncryptionKey();
+    if (!key || !key.key || !key.salt) {
+      EV.publish(EVENTS.userSessionExpired);
+      throw new Error("User encryption key not generated. Please relogin.");
+    }
 
-          this.onSyncItem(syncStatus)
-            .then(() => {
-              sendSyncProgressEvent(
-                this.db.eventManager,
-                `download`,
-                total,
-                ++counter.count
-              );
-            })
-            .catch(reject)
-            .finally(() => {
-              if (--counter.queue <= 0) resolve({ synced, lastSynced });
-            });
-        },
-        complete: () => {},
-        error: reject
-      });
+    const dbLastSynced = await this.db.lastSynced();
+    let count = 0;
+    this.connection.off("SendItems");
+    this.connection.on("SendItems", async (chunk) => {
+      if (this.connection.state !== signalr.HubConnectionState.Connected)
+        return;
+
+      count += chunk.items.length;
+      sendSyncProgressEvent(this.db.eventManager, `download`, count);
+
+      await this.processChunk(chunk, key, dbLastSynced);
+
+      return true;
     });
+    const serverResponse = await this.connection.invoke(
+      "RequestFetch",
+      lastSynced
+    );
+
+    if (serverResponse.vaultKey) {
+      await this.merger.mergeItem(
+        "vaultKey",
+        serverResponse.vaultKey,
+        serverResponse.lastSynced
+      );
+    }
+
+    this.connection.off("SendItems");
 
     if (await this.conflicts.check()) {
       this.conflicts.throw();
     }
 
-    return serverResponse;
+    return { lastSynced: serverResponse.lastSynced };
   }
 
-  async collect(lastSynced, force) {
-    const newLastSynced = Date.now();
-    const data = await this.collector.collect(lastSynced, force);
-    return { newLastSynced, data };
-  }
-
-  /**
-   *
-   * @param {{ items: any[]; vaultKey: any; types: string[]; }} data
-   * @param {number} lastSynced
-   * @returns {Promise<boolean>}
-   */
-  async send(data, lastSynced) {
+  async send(oldLastSynced, isForceSync, newLastSynced) {
     await this.uploadAttachments();
 
-    if (data.types.length === 1 && data.types[0] === "vaultKey") return false;
-    if (data.items.length <= 0) return false;
-
-    let total = data.items.length;
-
-    const types = toChunks(data.types, 30);
-    const items = toChunks(data.items, 30);
-
+    let isSyncInitialized = false;
     let done = 0;
-    for (let i = 0; i < items.length; ++i) {
-      this.logger.info(`Sending batch ${done}/${total}`);
-
-      const encryptedItems = (await this.collector.encrypt(items[i])).map(
-        (item) => JSON.stringify(item)
-      );
-
-      const result = await this.sendBatchToServer({
-        lastSynced,
-        current: i,
-        total,
-        items: encryptedItems,
-        types: types[i]
-      });
+    for await (const item of this.collector.collect(
+      100,
+      oldLastSynced,
+      isForceSync
+    )) {
+      if (!isSyncInitialized) {
+        const vaultKey = await this.db.vault._getKey();
+        newLastSynced = await this.connection.invoke("InitializePush", {
+          vaultKey,
+          lastSynced: newLastSynced
+        });
+        isSyncInitialized = true;
+      }
 
+      const result = await this.pushItem(item, newLastSynced);
       if (result) {
-        done += encryptedItems.length;
-        sendSyncProgressEvent(this.db.eventManager, "upload", total, done);
+        done += item.items.length;
+        sendSyncProgressEvent(this.db.eventManager, "upload", done);
 
-        this.logger.info(`Batch sent (${done}/${total})`);
+        this.logger.info(`Batch sent (${done})`);
       } else {
         this.logger.error(
           new Error(`Failed to send batch. Server returned falsy response.`)
         );
       }
     }
-    return await this.connection.invoke("SyncCompleted", lastSynced);
+    if (!isSyncInitialized) return;
+    await this.connection.send("SyncCompleted", newLastSynced);
+    return true;
   }
 
   async stop(lastSynced) {
@@ -376,7 +359,7 @@ class Sync {
   /**
    * @private
    */
-  async onRemoteSyncCompleted(lastSynced) {
+  async onPushCompleted(lastSynced) {
     // refresh monographs on sync completed
     await this.db.monographs.init();
     // refresh topic references
@@ -385,31 +368,62 @@ class Sync {
     await this.start(false, false, lastSynced);
   }
 
-  /**
-   * @param {SyncTransferItem} syncStatus
-   * @private
-   */
-  async onSyncItem(syncStatus) {
-    const { item: itemJSON, itemType } = syncStatus;
-    const item = JSON.parse(itemJSON);
+  async processChunk(chunk, key, dbLastSynced, notify = false) {
+    const decrypted = await this.db.storage.decryptMulti(key, chunk.items);
 
-    const remoteItem = await this.merger.mergeItem(itemType, item);
-    if (remoteItem)
-      this.db.eventManager.publish(EVENTS.syncItemMerged, remoteItem);
+    const deserialized = await Promise.all(
+      decrypted.map((item, index) =>
+        deserializeItem(item, chunk.items[index].v, this.db)
+      )
+    );
+
+    let items = [];
+    if (this.merger.isSyncCollection(chunk.type)) {
+      items = deserialized.map((item) =>
+        this.merger.mergeItemSync(item, chunk.type, dbLastSynced)
+      );
+    } else if (chunk.type === "content") {
+      const localItems = await this.db.content.multi(
+        chunk.items.map((i) => i.id)
+      );
+      items = await Promise.all(
+        deserialized.map((item) =>
+          this.merger.mergeContent(item, localItems[item.id], dbLastSynced)
+        )
+      );
+    } else {
+      items = await Promise.all(
+        deserialized.map((item) =>
+          this.merger.mergeItem(item, chunk.type, dbLastSynced)
+        )
+      );
+    }
+
+    if (
+      notify &&
+      (chunk.type === "content" || chunk.type === "note") &&
+      items.length > 0
+    ) {
+      items.forEach((item) =>
+        this.db.eventManager.publish(EVENTS.syncItemMerged, item)
+      );
+    }
+
+    const collectionType = this.itemTypeToCollection[chunk.type];
+    if (collectionType && this.db[collectionType])
+      await this.db[collectionType]._collection.setItems(items);
   }
 
   /**
    *
-   * @param {BatchedSyncTransferItem} batch
+   * @param {SyncTransferItem} item
    * @returns {Promise<boolean>}
    * @private
    */
-  async sendBatchToServer(batch) {
-    if (!batch) return false;
+  async pushItem(item, newLastSynced) {
     await this.checkConnection();
-
-    const result = await this.connection.invoke("SyncItem", batch);
-    return result === 1;
+    await this.connection.send("PushItems", item, newLastSynced);
+    return true; // () === 1;
   }
 
   async checkConnection() {
@@ -445,3 +459,14 @@ function promiseTimeout(ms, promise) {
   // Returns a race between our timeout and the passed in promise
   return Promise.race([promise, timeout]);
 }
+
+async function deserializeItem(decryptedItem, version, database) {
+  const deserialized = JSON.parse(decryptedItem);
+  deserialized.remote = true;
+  deserialized.synced = true;
+
+  if (!deserialized.alg && !deserialized.cipher) {
+    await migrateItem(deserialized, version, deserialized.type, database);
+  }
+  return deserialized;
+}

packages/core/src/api/sync/merger.js

@@ -17,11 +17,9 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { migrateItem } from "../../migrations";
 import setManipulator from "../../utils/set";
 import { logger } from "../../logger";
 import { isHTMLEqual } from "../../utils/html-diff";
-import { EV, EVENTS } from "../../common";
 
 class Merger {
   /**
@@ -32,217 +30,161 @@ class Merger {
     this._db = db;
     this.logger = logger.scope("Merger");
 
-    this._mergeDefinition = {
-      settings: {
-        threshold: 1000,
-        get: () => this._db.settings.raw,
-        set: (item) => this._db.settings.merge(item),
-        conflict: (_local, remote) => this._db.settings.merge(remote)
-      },
-      note: {
-        get: (id) => this._db.notes.note(id),
-        set: (item) => this._db.notes.merge(item)
-      },
-      shortcut: {
-        get: (id) => this._db.shortcuts.shortcut(id),
-        set: (item) => this._db.shortcuts.merge(item)
-      },
-      reminder: {
-        get: (id) => this._db.reminders.reminder(id),
-        set: (item) => this._db.reminders.merge(item)
-      },
-      relation: {
-        get: (id) => this._db.relations.relation(id),
-        set: (item) => this._db.relations.merge(item)
-      },
-      notebook: {
-        threshold: 1000,
-        get: (id) => this._db.notebooks.notebook(id),
-        set: (item) => this._db.notebooks.merge(item),
-        conflict: (_local, remote) => this._db.notebooks.merge(remote)
-      },
-      content: {
-        threshold: process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000,
-        get: (id) => this._db.content.raw(id),
-        set: (item) => this._db.content.add(item),
-        conflict: async (local, remote) => {
-          let note = this._db.notes.note(local.noteId);
-          if (!note || !note.data) return;
-          note = note.data;
-
-          // if hashes are equal do nothing
-          if (
-            !note.locked &&
-            (!remote ||
-              !local ||
-              !local.data ||
-              !remote.data ||
-              remote.data === "undefined" || //TODO not sure about this
-              isHTMLEqual(local.data, remote.data))
-          )
-            return;
-
-          if (remote.deleted || local.deleted || note.locked) {
-            // if note is locked or content is deleted we keep the most recent version.
-            if (remote.dateModified > local.dateModified)
-              await this._db.content.add({ id: local.id, ...remote });
-          } else {
-            // otherwise we trigger the conflicts
-            await this._db.content.add({ ...local, conflicted: remote });
-            await this._db.notes.add({ id: local.noteId, conflicted: true });
-            await this._db.storage.write("hasConflicts", true);
-          }
-        }
-      },
-      attachment: {
-        set: async (item) => {
-          const remoteAttachment = await this._deserialize(item);
-          if (remoteAttachment.deleted) {
-            await this._db.attachments.merge(remoteAttachment);
-            return;
-          }
-
-          const localAttachment = this._db.attachments.attachment(
-            remoteAttachment.metadata.hash
-          );
-          if (
-            localAttachment &&
-            localAttachment.dateUploaded !== remoteAttachment.dateUploaded
-          ) {
-            const noteIds = localAttachment.noteIds.slice();
-            const isRemoved = await this._db.attachments.remove(
-              localAttachment.metadata.hash,
-              true
-            );
-            if (!isRemoved)
-              throw new Error(
-                "Conflict could not be resolved in one of the attachments."
-              );
-            remoteAttachment.noteIds = setManipulator.union(
-              remoteAttachment.noteIds,
-              noteIds
-            );
-          }
-          await this._db.attachments.merge(remoteAttachment);
-        }
-      },
-      vaultKey: {
-        set: async (vaultKey) =>
-          this._db.vault._setKey(await this._deserialize(vaultKey, false))
-      }
+    this.syncCollectionMap = {
+      shortcut: "shortcuts",
+      reminder: "reminders",
+      relation: "relations",
+      notebook: "notebooks"
     };
   }
 
-  async _migrate(deserialized, version) {
-    // it is a locked note, bail out.
-    if (deserialized.alg && deserialized.cipher) return deserialized;
-
-    return migrateItem(deserialized, version, deserialized.type, this._db);
+  isSyncCollection(type) {
+    return !!this.syncCollectionMap[type];
   }
 
-  async _deserialize(item, migrate = true) {
-    const decrypted = await this._db.storage.decrypt(this.key, item);
-    if (!decrypted) {
-      throw new Error("Decrypted item cannot be undefined.");
-    }
+  isConflicted(localItem, remoteItem, lastSynced, conflictThreshold) {
+    const isResolved = localItem.dateResolved === remoteItem.dateModified;
+    const isModified =
+      // the local item is modified if it was changed/modified after the last
+      // sync i.e. it wasn't synced yet.
+      // However, in case a sync is interrupted the local item's date modified
+      // will be ahead of last sync. In that case, we also have to check if the
+      // synced flag is false (it is only false if a user makes edits on the
+      // local device).
+      localItem.dateModified > lastSynced && !localItem.synced;
+    if (isModified && !isResolved) {
+      // If time difference between local item's edits & remote item's edits
+      // is less than threshold, we shouldn't trigger a merge conflict; instead
+      // we will keep the most recently changed item.
+      const timeDiff =
+        Math.max(remoteItem.dateModified, localItem.dateModified) -
+        Math.min(remoteItem.dateModified, localItem.dateModified);
 
-    const deserialized = JSON.parse(decrypted);
-    deserialized.remote = true;
-    deserialized.synced = true;
-    if (!migrate) return deserialized;
-    await this._migrate(deserialized, item.v);
-    return deserialized;
-  }
-
-  async _mergeItem(remoteItem, get, add) {
-    remoteItem = await this._deserialize(remoteItem);
-    let localItem = await get(remoteItem.id);
-    if (!localItem || remoteItem.dateModified > localItem.dateModified) {
-      await add(remoteItem);
-      return remoteItem;
-    }
-  }
-
-  async _mergeItemWithConflicts(
-    remoteItem,
-    get,
-    add,
-    markAsConflicted,
-    threshold
-  ) {
-    remoteItem = await this._deserialize(remoteItem);
-    let localItem = await get(remoteItem.id);
-
-    if (!localItem) {
-      await add(remoteItem);
-      return remoteItem;
-    } else {
-      const isResolved = localItem.dateResolved === remoteItem.dateModified;
-      const isModified =
-        // the local item is modified if it was changed/modified after the last sync
-        // i.e. it wasn't synced yet.
-        // However, in case a sync is interrupted the local item's date modified will
-        // be ahead of last sync. In that case, we also have to check if the synced flag
-        // is false (it is only false if a user makes edits on the local device).
-        localItem.dateModified > this._lastSynced && !localItem.synced;
-      if (isModified && !isResolved) {
-        // If time difference between local item's edits & remote item's edits
-        // is less than threshold, we shouldn't trigger a merge conflict; instead
-        // we will keep the most recently changed item.
-        const timeDiff =
-          Math.max(remoteItem.dateModified, localItem.dateModified) -
-          Math.min(remoteItem.dateModified, localItem.dateModified);
-
-        if (timeDiff < threshold) {
-          if (remoteItem.dateModified > localItem.dateModified) {
-            await add(remoteItem);
-            return remoteItem;
-          }
-          return;
+      if (timeDiff < conflictThreshold) {
+        if (remoteItem.dateModified > localItem.dateModified) {
+          return "merge";
         }
+        return;
+      }
 
-        this.logger.info("Conflict detected", {
-          itemId: remoteItem.id,
-          isResolved,
-          isModified,
-          timeDiff,
-          remote: remoteItem.dateModified,
-          local: localItem.dateModified,
-          lastSynced: this._lastSynced
-        });
+      return "conflict";
+    } else if (!isResolved) {
+      return "merge";
+    }
+  }
 
-        await markAsConflicted(localItem, remoteItem);
-      } else if (!isResolved) {
-        await add(remoteItem);
-        return remoteItem;
+  mergeItemSync(remoteItem, type, lastSynced) {
+    switch (type) {
+      case "shortcut":
+      case "reminder":
+      case "relation": {
+        const localItem = this._db[
+          this.syncCollectionMap[type]
+        ]._collection.getItem(remoteItem.id);
+        if (!localItem || remoteItem.dateModified > localItem.dateModified) {
+          return remoteItem;
+        }
+        break;
+      }
+      case "notebook": {
+        const THRESHOLD = 1000;
+        const localItem = this._db.notebooks._collection.getItem(remoteItem.id);
+        if (
+          !localItem ||
+          this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD)
+        ) {
+          return this._db.notebooks.merge(localItem, remoteItem, lastSynced);
+        }
+        break;
       }
     }
   }
 
-  async mergeItem(type, item) {
-    this._lastSynced = await this._db.lastSynced();
+  async mergeContent(remoteItem, localItem, lastSynced) {
+    if (localItem && localItem.localOnly) return;
 
-    const definition = this._mergeDefinition[type];
-    if (!type || !item || !definition) return;
+    const THRESHOLD = process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000;
+    const conflicted =
+      localItem &&
+      this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD);
+    if (!localItem || conflicted === "merge") {
+      return remoteItem;
+    } else if (conflicted === "conflict") {
+      const note = this._db.notes._collection.getItem(localItem.noteId);
+      if (!note || note.deleted) return;
 
-    if (!this.key) this.key = await this._db.user.getEncryptionKey();
-    if (!this.key || !this.key.key || !this.key.salt) {
-      EV.publish(EVENTS.userSessionExpired);
-      throw new Error("User encryption key not generated. Please relogin.");
+      // if hashes are equal do nothing
+      if (
+        !note.locked &&
+        (!remoteItem ||
+          !remoteItem ||
+          !localItem.data ||
+          !remoteItem.data ||
+          isHTMLEqual(localItem.data, remoteItem.data))
+      )
+        return;
+
+      if (remoteItem.deleted || localItem.deleted || note.locked) {
+        // if note is locked or content is deleted we keep the most recent version.
+        if (remoteItem.dateModified > localItem.dateModified) return remoteItem;
+      } else {
+        // otherwise we trigger the conflicts
+        await this._db.notes.add({
+          id: localItem.noteId,
+          conflicted: true
+        });
+        await this._db.storage.write("hasConflicts", true);
+        return {
+          ...localItem,
+          conflicted: remoteItem
+        };
+      }
     }
+  }
 
-    if (definition.conflict) {
-      return await this._mergeItemWithConflicts(
-        item,
-        definition.get,
-        definition.set,
-        definition.conflict,
-        definition.threshold
-      );
-    } else if (definition.get && definition.set) {
-      return await this._mergeItem(item, definition.get, definition.set);
-    } else if (!definition.get && definition.set) {
-      await definition.set(item);
+  async mergeItem(remoteItem, type, lastSynced) {
+    switch (type) {
+      case "note": {
+        const localItem = this._db.notes._collection.getItem(remoteItem.id);
+        if (!localItem || remoteItem.dateModified > localItem.dateModified) {
+          return await this._db.notes.merge(localItem, remoteItem);
+        }
+        break;
+      }
+      case "settings": {
+        const localItem = this._db.settings.raw;
+        if (
+          !localItem ||
+          this.isConflicted(localItem, remoteItem, lastSynced, 1000)
+        ) {
+          await this._db.settings.merge(remoteItem, lastSynced);
+        }
+        break;
+      }
+      case "attachment": {
+        if (remoteItem.deleted)
+          return this._db.attachments.merge(null, remoteItem);
+
+        const localItem = this._db.attachments.attachment(
+          remoteItem.metadata.hash
+        );
+        if (localItem && localItem.dateUploaded !== remoteItem.dateUploaded) {
+          const noteIds = localItem.noteIds.slice();
+          const isRemoved = await this._db.attachments.remove(
+            localItem.metadata.hash,
+            true
+          );
+          if (!isRemoved)
+            throw new Error(
+              "Conflict could not be resolved in one of the attachments."
+            );
+          remoteItem.noteIds = setManipulator.union(
+            remoteItem.noteIds,
+            noteIds
+          );
+        }
+        return this._db.attachments.merge(localItem, remoteItem);
+      }
     }
   }
 }

packages/core/src/collections/attachments.js

@@ -36,12 +36,8 @@ export default class Attachments extends Collection {
     this.key = null;
   }
 
-  merge(remoteAttachment) {
-    if (remoteAttachment.deleted)
-      return this._collection.addItem(remoteAttachment);
-
-    const id = remoteAttachment.id;
-    let localAttachment = this._collection.getItem(id);
+  merge(localAttachment, remoteAttachment) {
+    if (remoteAttachment.deleted) return remoteAttachment;
 
     if (localAttachment && localAttachment.noteIds) {
       remoteAttachment.noteIds = setManipulator.union(
@@ -50,7 +46,7 @@ export default class Attachments extends Collection {
       );
     }
 
-    return this._collection.addItem(remoteAttachment);
+    return remoteAttachment;
   }
 
   /**

packages/core/src/collections/content.js

@@ -26,6 +26,12 @@ export default class Content extends Collection {
   async add(content) {
     if (!content) return;
 
+    if (content.remote)
+      throw new Error(
+        "Please do not use this method for merging. Instead add the item directly to database."
+      );
+    if (content.deleted) return await this._collection.addItem(content);
+
     if (typeof content.data === "object") {
       if (typeof content.data.data === "string")
         content.data = content.data.data;
@@ -35,11 +41,6 @@ export default class Content extends Collection {
         )}</p>`;
     }
 
-    if (content.remote || content.deleted)
-      return await this._collection.addItem(
-        await this.extractAttachments(content)
-      );
-
     const oldContent = await this.raw(content.id);
     if (content.id && oldContent) {
       content = {

packages/core/src/collections/notebooks.js

@@ -24,15 +24,9 @@ import { CHECK_IDS, checkIsUserPremium } from "../common";
 import qclone from "qclone";
 
 export default class Notebooks extends Collection {
-  async merge(remoteNotebook) {
-    if (remoteNotebook.deleted)
-      return await this._collection.addItem(remoteNotebook);
-
-    const id = remoteNotebook.id || id();
-    let localNotebook = this._collection.getItem(id);
-
+  merge(localNotebook, remoteNotebook, lastSyncedTimestamp) {
+    if (remoteNotebook.deleted) return remoteNotebook;
     if (localNotebook && localNotebook.topics?.length) {
-      const lastSyncedTimestamp = await this._db.lastSynced();
       let isChanged = false;
       // merge new and old topics
       for (let oldTopic of localNotebook.topics) {
@@ -65,7 +59,7 @@ export default class Notebooks extends Collection {
       }
       remoteNotebook.remote = !isChanged;
     }
-    return await this._collection.addItem(remoteNotebook);
+    return remoteNotebook;
   }
 
   async add(notebookArg) {

packages/core/src/collections/notes.js

@@ -44,12 +44,12 @@ export default class Notes extends Collection {
     return this.raw.find((item) => item.dateDeleted > 0 && item.id === id);
   }
 
-  async merge(remoteNote) {
-    if (!remoteNote) return;
-
+  async merge(localNote, remoteNote) {
     const id = remoteNote.id;
-    const localNote = this._collection.getItem(id);
+
     if (localNote) {
+      if (localNote.localOnly) return;
+
       if (localNote.color) await this._db.colors.untag(localNote.color, id);
 
       for (let tag of localNote.tags || []) {
@@ -57,16 +57,9 @@ export default class Notes extends Collection {
       }
     }
 
-    if (
-      remoteNote.deleted &&
-      remoteNote.deleteReason !== "localOnly" &&
-      (!localNote || !localNote.localOnly)
-    )
-      return await this._collection.addItem(remoteNote);
-
     await this._resolveColorAndTags(remoteNote);
 
-    return await this._collection.addItem(remoteNote);
+    return remoteNote;
   }
 
   async add(noteArg) {

packages/core/src/collections/relations.js

@@ -39,7 +39,7 @@ import Collection from "./collection";
 export default class Relations extends Collection {
   async merge(relation) {
     if (!relation) return;
-    await this._collection.addItem(relation);
+    return relation;
   }
 
   /**

packages/core/src/collections/reminders.js

@@ -54,7 +54,7 @@ dayjs.extend(isToday);
 export default class Reminders extends Collection {
   async merge(reminder) {
     if (!reminder) return;
-    await this._collection.addItem(reminder);
+    return reminder;
   }
 
   /**

packages/core/src/collections/shortcuts.js

@@ -41,7 +41,7 @@ const ALLOWED_SHORTCUT_TYPES = ["notebook", "topic", "tag"];
 export default class Shortcuts extends Collection {
   async merge(shortcut) {
     if (!shortcut) return;
-    await this._collection.addItem(shortcut);
+    return shortcut;
   }
 
   /**

packages/core/src/common.js

@@ -45,11 +45,10 @@ export function sendAttachmentsProgressEvent(type, groupId, total, current) {
   });
 }
 
-export function sendSyncProgressEvent(EV, type, total, current) {
+export function sendSyncProgressEvent(EV, type, current) {
   EV.publish(EVENTS.syncProgress, {
     type,
-    total,
-    current: current === undefined ? total : current
+    current
   });
 }
 

packages/core/src/content-types/tiptap.js

@@ -148,6 +148,15 @@ export class Tiptap {
   }
 
   async extractAttachments(store) {
+    if (
+      !this.data.includes(ATTRIBUTES.src) &&
+      !this.data.includes(ATTRIBUTES.hash)
+    )
+      return {
+        data: this.data,
+        attachments: []
+      };
+
     let sources = [];
     new HTMLParser({
       ontag: (name, attr, pos) => {

packages/core/src/database/cached-collection.js

@@ -19,6 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import IndexedCollection from "./indexed-collection";
 import MapStub from "../utils/map";
+import { toChunks } from "../utils/array";
 
 export default class CachedCollection extends IndexedCollection {
   constructor(context, type, eventManager) {
@@ -102,6 +103,24 @@ export default class CachedCollection extends IndexedCollection {
     return this.items;
   }
 
+  async setItems(items) {
+    await super.setItems(items);
+    for (let item of items) {
+      if (item) {
+        this.map.set(item.id, item);
+      }
+    }
+
+    this.invalidateCache();
+  }
+
+  *iterateSync(chunkSize) {
+    const chunks = toChunks(Array.from(this.map.values()), chunkSize);
+    for (const chunk of chunks) {
+      yield chunk;
+    }
+  }
+
   invalidateCache() {
     this.items = undefined;
   }

packages/core/src/database/indexed-collection.js

@@ -18,6 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import { EVENTS } from "../common";
+import { toChunks } from "../utils/array";
 import Indexer from "./indexer";
 
 export default class IndexedCollection {
@@ -107,10 +108,21 @@ export default class IndexedCollection {
     return Object.fromEntries(data);
   }
 
+  setItems(items) {
+    return this.indexer.writeMulti(items);
+  }
+
   async getEncryptionKey() {
     if (!this.encryptionKeyFactory) return;
     if (this.encryptionKey) return this.encryptionKey;
     this.encryptionKey = await this.encryptionKeyFactory();
     return this.encryptionKey;
   }
+
+  async *iterate(chunkSize) {
+    const chunks = toChunks(this.indexer.indices, chunkSize);
+    for (const chunk of chunks) {
+      yield await this.indexer.readMulti(chunk);
+    }
+  }
 }

packages/core/src/database/indexer.js

@@ -56,6 +56,7 @@ export default class Indexer extends Storage {
   }
 
   read(key, isArray = false) {
+    if (!this.exists(key)) return;
     return super.read(this.makeId(key), isArray);
   }
 
@@ -68,13 +69,31 @@ export default class Indexer extends Storage {
   }
 
   async readMulti(keys) {
-    const entries = await super.readMulti(keys.map(this.makeId));
+    const entries = await super.readMulti(
+      keys.filter(this.exists, this).map(this.makeId, this)
+    );
     entries.forEach((entry) => {
       entry[0] = entry[0].replace(`_${this.type}`, "");
     });
     return entries;
   }
 
+  /**
+   *
+   * @param {any[]} items
+   * @returns
+   */
+  async writeMulti(items) {
+    const entries = items.reduce((array, item) => {
+      if (!item) return array;
+      if (!this.indices.includes(item.id)) this.indices.push(item.id);
+      array.push([this.makeId(item.id), item]);
+      return array;
+    }, []);
+    await super.writeMulti(entries);
+    await super.write(this.type, this.indices);
+  }
+
   async migrateIndices() {
     const keys = (await super.getAllKeys()).filter(
       (key) => !key.endsWith(`_${this.type}`) && this.exists(key)

packages/core/src/database/migrator.js

@@ -66,10 +66,14 @@ class Migrator {
         );
 
         if (migrated || restore) {
-          if (collection.dbCollection.merge) {
+          if (collection.type === "settings") {
             await collection.dbCollection.merge(item);
-          } else if (collection.dbCollection.add) {
-            await collection.dbCollection.add(item);
+          } else if (collection.dbCollection._collection) {
+            await collection.dbCollection._collection?.addItem(item);
+          } else {
+            throw new Error(
+              `No idea how to handle this kind of item: ${item.type}.`
+            );
           }
 
           // if id changed after migration, we need to delete the old one.

packages/core/src/database/storage.js

@@ -32,6 +32,10 @@ export default class Storage {
     return this.storage.readMulti(keys);
   }
 
+  writeMulti(entries) {
+    return this.storage.writeMulti(entries);
+  }
+
   read(key, isArray = false) {
     return this.storage.read(key, isArray);
   }
@@ -52,10 +56,18 @@ export default class Storage {
     return this.storage.encrypt(password, data);
   }
 
+  encryptMulti(password, data) {
+    return this.storage.encryptMulti(password, data);
+  }
+
   decrypt(password, cipher) {
     return this.storage.decrypt(password, cipher);
   }
 
+  decryptMulti(password, items) {
+    return this.storage.decryptMulti(password, items);
+  }
+
   deriveCryptoKey(name, data) {
     return this.storage.deriveCryptoKey(name, data);
   }

packages/crypto-worker/index.ts

@@ -1,177 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import {
-  SerializedKey,
-  Plaintext,
-  OutputFormat,
-  Cipher,
-  EncryptionKey,
-  INNCrypto
-} from "@notesnook/crypto";
-import { NNCryptoWorkerModule } from "./src/worker";
-import { wrap } from "comlink";
-
-export class NNCryptoWorker implements INNCrypto {
-  private workermodule?: NNCryptoWorkerModule;
-  private isReady = false;
-
-  constructor(private readonly worker?: Worker) {}
-
-  private async init() {
-    if (!this.worker) throw new Error("worker cannot be undefined.");
-    if (this.isReady) return;
-
-    this.workermodule = wrap<NNCryptoWorkerModule>(this.worker);
-    // this.workermodule = await spawn<NNCryptoWorkerModule>(this.worker);
-    this.isReady = true;
-  }
-
-  async encrypt(
-    key: SerializedKey,
-    plaintext: Plaintext,
-    outputFormat: OutputFormat = "uint8array"
-  ): Promise<Cipher> {
-    await this.init();
-    if (!this.workermodule) throw new Error("Worker module is not ready.");
-
-    return this.workermodule.encrypt(key, plaintext, outputFormat);
-  }
-
-  async decrypt(
-    key: SerializedKey,
-    cipherData: Cipher,
-    outputFormat: OutputFormat = "text"
-  ): Promise<Plaintext> {
-    await this.init();
-    if (!this.workermodule) throw new Error("Worker module is not ready.");
-
-    return this.workermodule.decrypt(key, cipherData, outputFormat);
-  }
-
-  async hash(password: string, salt: string): Promise<string> {
-    await this.init();
-    if (!this.workermodule) throw new Error("Worker module is not ready.");
-
-    return this.workermodule.hash(password, salt);
-  }
-
-  async deriveKey(password: string, salt?: string): Promise<EncryptionKey> {
-    await this.init();
-    if (!this.workermodule) throw new Error("Worker module is not ready.");
-
-    return this.workermodule.deriveKey(password, salt);
-  }
-
-  async exportKey(password: string, salt?: string): Promise<SerializedKey> {
-    await this.init();
-    if (!this.workermodule) throw new Error("Worker module is not ready.");
-
-    return this.workermodule.exportKey(password, salt);
-  }
-
-  async createEncryptionStream(key: SerializedKey) {
-    await this.init();
-    if (!this.workermodule) throw new Error("Worker module is not ready.");
-    return this.workermodule.createEncryptionStream(key);
-  }
-
-  async createDecryptionStream(key: SerializedKey, iv: string) {
-    await this.init();
-    if (!this.workermodule) throw new Error("Worker module is not ready.");
-    const { stream } = await this.workermodule.createDecryptionStream(key, iv);
-    return stream;
-  }
-  // async encryptStream(
-  //   key: SerializedKey,
-  //   stream: IStreamable,
-  //   streamId?: string
-  // ): Promise<string> {
-  //   if (!streamId) throw new Error("streamId is required.");
-  //   await this.init();
-  //   if (!this.workermodule) throw new Error("Worker module is not ready.");
-  //   if (!this.worker) throw new Error("Worker is not ready.");
-
-  //   const eventListener = await this.createWorkerStream(
-  //     streamId,
-  //     stream,
-  //     () => {
-  //       if (this.worker)
-  //         this.worker.removeEventListener("message", eventListener);
-  //     }
-  //   );
-  //   this.worker.addEventListener("message", eventListener);
-  //   const iv = await this.workermodule.createEncryptionStream(streamId, key);
-  //   this.worker.removeEventListener("message", eventListener);
-  //   return iv;
-  // }
-
-  // async decryptStream(
-  //   key: SerializedKey,
-  //   iv: string,
-  //   stream: IStreamable,
-  //   streamId?: string
-  // ): Promise<void> {
-  //   if (!streamId) throw new Error("streamId is required.");
-  //   await this.init();
-  //   if (!this.workermodule) throw new Error("Worker module is not ready.");
-  //   if (!this.worker) throw new Error("Worker is not ready.");
-
-  //   const eventListener = await this.createWorkerStream(
-  //     streamId,
-  //     stream,
-  //     () => {
-  //       if (this.worker)
-  //         this.worker.removeEventListener("message", eventListener);
-  //     }
-  //   );
-  //   this.worker.addEventListener("message", eventListener);
-  //   await this.workermodule.createDecryptionStream(streamId, iv, key);
-  //   this.worker.removeEventListener("message", eventListener);
-  // }
-
-  // private async createWorkerStream(
-  //   streamId: string,
-  //   stream: IStreamable,
-  //   done: () => void
-  // ): Promise<EventListenerObject> {
-  //   const readEventType = `${streamId}:read`;
-  //   const writeEventType = `${streamId}:write`;
-  //   let finished = false;
-  //   return {
-  //     handleEvent: async (ev: MessageEvent) => {
-  //       if (finished) return;
-
-  //       const { type } = ev.data;
-  //       if (type === readEventType) {
-  //         const chunk = await stream.read();
-  //         if (!chunk || !this.worker || !chunk.data) return;
-  //         this.worker.postMessage({ type, data: chunk }, [chunk.data.buffer]);
-  //       } else if (type === writeEventType) {
-  //         const chunk = ev.data.data as Chunk;
-  //         await stream.write(chunk);
-  //         if (chunk.final) {
-  //           finished = true;
-  //           done();
-  //         }
-  //       }
-  //     }
-  //   };
-  // }
-}

packages/crypto-worker/package-lock.json

@@ -1,49 +0,0 @@
-{
-  "name": "@notesnook/crypto-worker",
-  "version": "1.0.0",
-  "lockfileVersion": 2,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "@notesnook/crypto-worker",
-      "version": "1.0.0",
-      "license": "GPL-3.0-or-later",
-      "dependencies": {
-        "@notesnook/crypto": "file:../crypto",
-        "comlink": "^4.3.1"
-      },
-      "devDependencies": {}
-    },
-    "../crypto": {
-      "name": "@notesnook/crypto",
-      "version": "1.1.1",
-      "license": "GPL-3.0-or-later",
-      "dependencies": {
-        "@notesnook/sodium": "file:../sodium"
-      },
-      "devDependencies": {}
-    },
-    "node_modules/@notesnook/crypto": {
-      "resolved": "../crypto",
-      "link": true
-    },
-    "node_modules/comlink": {
-      "version": "4.3.1",
-      "resolved": "https://registry.npmjs.org/comlink/-/comlink-4.3.1.tgz",
-      "integrity": "sha512-+YbhUdNrpBZggBAHWcgQMLPLH1KDF3wJpeqrCKieWQ8RL7atmgsgTQko1XEBK6PsecfopWNntopJ+ByYG1lRaA=="
-    }
-  },
-  "dependencies": {
-    "@notesnook/crypto": {
-      "version": "file:../crypto",
-      "requires": {
-        "@notesnook/sodium": "file:../sodium"
-      }
-    },
-    "comlink": {
-      "version": "4.3.1",
-      "resolved": "https://registry.npmjs.org/comlink/-/comlink-4.3.1.tgz",
-      "integrity": "sha512-+YbhUdNrpBZggBAHWcgQMLPLH1KDF3wJpeqrCKieWQ8RL7atmgsgTQko1XEBK6PsecfopWNntopJ+ByYG1lRaA=="
-    }
-  }
-}

packages/crypto-worker/package.json

@@ -1,17 +0,0 @@
-{
-  "name": "@notesnook/crypto-worker",
-  "version": "1.0.0",
-  "description": "",
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
-  "scripts": {
-    "build": "tsc --declaration --outDir ./dist"
-  },
-  "author": "",
-  "license": "GPL-3.0-or-later",
-  "dependencies": {
-    "@notesnook/crypto": "file:../crypto",
-    "comlink": "^4.3.1"
-  },
-  "devDependencies": {}
-}

packages/crypto-worker/src/worker.ts

@@ -1,83 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import {
-  Cipher,
-  OutputFormat,
-  Plaintext,
-  SerializedKey
-} from "@notesnook/crypto/dist/src/types";
-import { expose, transfer } from "comlink";
-import { NNCrypto } from "@notesnook/crypto";
-
-let crypto: NNCrypto | null = null;
-async function loadNNCrypto(): Promise<NNCrypto> {
-  if (crypto) return crypto;
-  const { NNCrypto } = await import("@notesnook/crypto");
-  return (crypto = new NNCrypto());
-}
-
-const module = {
-  exportKey: async function (password: string, salt?: string) {
-    const crypto = await loadNNCrypto();
-    return crypto.exportKey(password, salt);
-  },
-  deriveKey: async function (password: string, salt?: string) {
-    const crypto = await loadNNCrypto();
-    return crypto.deriveKey(password, salt);
-  },
-  hash: async function (password: string, salt: string) {
-    const crypto = await loadNNCrypto();
-    return crypto.hash(password, salt);
-  },
-  encrypt: async function (
-    key: SerializedKey,
-    plaintext: Plaintext,
-    outputFormat?: OutputFormat
-  ) {
-    const crypto = await loadNNCrypto();
-    return crypto.encrypt(key, plaintext, outputFormat);
-  },
-  decrypt: async function (
-    key: SerializedKey,
-    cipherData: Cipher,
-    outputFormat?: OutputFormat
-  ) {
-    const crypto = await loadNNCrypto();
-    return crypto.decrypt(key, cipherData, outputFormat);
-  },
-  createEncryptionStream: async function (key: SerializedKey) {
-    const crypto = await loadNNCrypto();
-    const stream = await crypto.createEncryptionStream(key);
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    // @ts-ignore
-    return transfer(stream, [stream.stream]);
-  },
-  createDecryptionStream: async function (key: SerializedKey, iv: string) {
-    const crypto = await loadNNCrypto();
-    const obj = { stream: await crypto.createDecryptionStream(key, iv) };
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    // @ts-ignore
-    return transfer(obj, [obj.stream]);
-  }
-};
-
-export type NNCryptoWorkerModule = typeof module;
-
-expose(module);

packages/crypto-worker/tsconfig.json

@@ -1,8 +0,0 @@
-{
-  "extends": "../../tsconfig",
-  "compilerOptions": {
-    "lib": ["DOM", "WebWorker", "ES2015"],
-    "outDir": "./dist"
-  },
-  "include": ["./index.ts", "./src/"]
-}

packages/crypto/index.ts

@@ -26,8 +26,9 @@ import Password from "./src/password";
 import {
   Cipher,
   EncryptionKey,
-  OutputFormat,
-  Plaintext,
+  Input,
+  Output,
+  DataFormat,
   SerializedKey
 } from "./src/types";
 
@@ -40,24 +41,55 @@ export class NNCrypto implements INNCrypto {
     this.isReady = true;
   }
 
-  async encrypt(
+  async encrypt<TOutputFormat extends DataFormat>(
     key: SerializedKey,
-    plaintext: Plaintext,
-    outputFormat: OutputFormat = "uint8array"
-  ): Promise<Cipher> {
+    input: Input<DataFormat>,
+    format: DataFormat,
+    outputFormat: TOutputFormat = "uint8array" as TOutputFormat
+  ): Promise<Cipher<TOutputFormat>> {
     await this.init();
-    return Encryption.encrypt(key, plaintext, outputFormat);
+    return Encryption.encrypt(
+      key,
+      input,
+      format,
+      outputFormat
+    ) as Cipher<TOutputFormat>;
   }
 
-  async decrypt(
+  async encryptMulti<TOutputFormat extends DataFormat>(
     key: SerializedKey,
-    cipherData: Cipher,
-    outputFormat: OutputFormat = "text"
-  ): Promise<Plaintext> {
+    items: Input<DataFormat>[],
+    format: DataFormat,
+    outputFormat = "uint8array" as TOutputFormat
+  ): Promise<Cipher<TOutputFormat>[]> {
+    await this.init();
+    return items.map((data) =>
+      Encryption.encrypt(key, data, format, outputFormat)
+    );
+  }
+
+  async decrypt<TOutputFormat extends DataFormat>(
+    key: SerializedKey,
+    cipherData: Cipher<DataFormat>,
+    outputFormat: TOutputFormat = "text" as TOutputFormat
+  ): Promise<Output<TOutputFormat>> {
     await this.init();
     return Decryption.decrypt(key, cipherData, outputFormat);
   }
 
+  async decryptMulti<TOutputFormat extends DataFormat>(
+    key: SerializedKey,
+    items: Cipher<DataFormat>[],
+    outputFormat: TOutputFormat = "text" as TOutputFormat
+  ): Promise<Output<TOutputFormat>[]> {
+    await this.init();
+    const decryptedItems: Output<TOutputFormat>[] = [];
+    for (const cipherData of items) {
+      decryptedItems.push(Decryption.decrypt(key, cipherData, outputFormat));
+    }
+    return decryptedItems;
+  }
+
   async hash(password: string, salt: string): Promise<string> {
     await this.init();
     return Password.hash(password, salt);
@@ -139,3 +171,4 @@ export class NNCrypto implements INNCrypto {
 
 export * from "./src/types";
 export * from "./src/interfaces";
+export { Decryption };

packages/crypto/src/decryption.ts

@@ -29,10 +29,10 @@ import {
   from_hex
 } from "@notesnook/sodium";
 import KeyUtils from "./keyutils";
-import { Cipher, OutputFormat, Plaintext, SerializedKey } from "./types";
+import { Cipher, Output, DataFormat, SerializedKey } from "./types";
 
 export default class Decryption {
-  private static transformInput(cipherData: Cipher): Uint8Array {
+  private static transformInput(cipherData: Cipher<DataFormat>): Uint8Array {
     let input: Uint8Array | null = null;
     if (
       typeof cipherData.cipher === "string" &&
@@ -54,11 +54,11 @@ export default class Decryption {
     return input;
   }
 
-  static decrypt(
+  static decrypt<TOutputFormat extends DataFormat>(
     key: SerializedKey,
-    cipherData: Cipher,
-    outputFormat: OutputFormat = "text"
-  ): Plaintext {
+    cipherData: Cipher<DataFormat>,
+    outputFormat: TOutputFormat = "text" as TOutputFormat
+  ): Output<TOutputFormat> {
     if (!key.salt && cipherData.salt) key.salt = cipherData.salt;
     const encryptionKey = KeyUtils.transform(key);
 
@@ -71,15 +71,13 @@ export default class Decryption {
       encryptionKey.key
     );
 
-    return {
-      format: outputFormat,
-      data:
-        outputFormat === "base64"
-          ? to_base64(plaintext, base64_variants.ORIGINAL)
-          : outputFormat === "text"
-          ? to_string(plaintext)
-          : plaintext
-    };
+    return (
+      outputFormat === "base64"
+        ? to_base64(plaintext, base64_variants.ORIGINAL)
+        : outputFormat === "text"
+        ? to_string(plaintext)
+        : plaintext
+    ) as Output<TOutputFormat>;
   }
 
   static createStream(

packages/crypto/src/encryption.ts

@@ -30,30 +30,34 @@ import {
   base64_variants
 } from "@notesnook/sodium";
 import KeyUtils from "./keyutils";
-import { Chunk, Cipher, OutputFormat, Plaintext, SerializedKey } from "./types";
+import { Chunk, Cipher, Input, DataFormat, SerializedKey } from "./types";
 
 const encoder = new TextEncoder();
 export default class Encryption {
-  private static transformInput(plaintext: Plaintext): Uint8Array {
+  private static transformInput(
+    input: Input<DataFormat>,
+    format: DataFormat
+  ): Uint8Array {
     let data: Uint8Array | null = null;
-    if (typeof plaintext.data === "string" && plaintext.format === "base64") {
-      data = from_base64(plaintext.data, base64_variants.ORIGINAL);
-    } else if (typeof plaintext.data === "string") {
-      data = encoder.encode(plaintext.data);
-    } else if (plaintext.data instanceof Uint8Array) {
-      data = plaintext.data;
+    if (typeof input === "string" && format === "base64") {
+      data = from_base64(input, base64_variants.ORIGINAL);
+    } else if (typeof input === "string") {
+      data = encoder.encode(input);
+    } else if (input instanceof Uint8Array) {
+      data = input;
     }
     if (!data) throw new Error("Data cannot be null.");
     return data;
   }
 
-  static encrypt(
+  static encrypt<TOutputFormat extends DataFormat>(
     key: SerializedKey,
-    plaintext: Plaintext,
-    outputFormat: OutputFormat = "uint8array"
-  ): Cipher {
+    input: Input<DataFormat>,
+    format: DataFormat,
+    outputFormat: TOutputFormat = "uint8array" as TOutputFormat
+  ): Cipher<TOutputFormat> {
     const encryptionKey = KeyUtils.transform(key);
-    const data = this.transformInput(plaintext);
+    const data = this.transformInput(input, format);
 
     const nonce = randombytes_buf(crypto_aead_xchacha20poly1305_ietf_NPUBBYTES);
 
@@ -79,7 +83,7 @@ export default class Encryption {
       iv,
       salt: encryptionKey.salt,
       length: data.length
-    };
+    } as Cipher<TOutputFormat>;
   }
 
   static createStream(key: SerializedKey): {

packages/crypto/src/interfaces.ts

@@ -20,10 +20,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import {
   Cipher,
   EncryptionKey,
-  OutputFormat,
-  Plaintext,
+  DataFormat,
   SerializedKey,
-  Chunk
+  Chunk,
+  Output,
+  Input
 } from "./types";
 
 export interface IStreamable {
@@ -32,17 +33,31 @@ export interface IStreamable {
 }
 
 export interface INNCrypto {
-  encrypt(
+  encrypt<TOutputFormat extends DataFormat>(
     key: SerializedKey,
-    plaintext: Plaintext,
-    outputFormat?: OutputFormat
-  ): Promise<Cipher>;
+    data: Input<DataFormat>,
+    format: DataFormat,
+    outputFormat?: TOutputFormat
+  ): Promise<Cipher<TOutputFormat>>;
 
-  decrypt(
+  encryptMulti<TOutputFormat extends DataFormat>(
     key: SerializedKey,
-    cipherData: Cipher,
-    outputFormat?: OutputFormat
-  ): Promise<Plaintext>;
+    data: Input<DataFormat>[],
+    format: DataFormat,
+    outputFormat?: TOutputFormat
+  ): Promise<Cipher<TOutputFormat>[]>;
+
+  decrypt<TOutputFormat extends DataFormat>(
+    key: SerializedKey,
+    cipherData: Cipher<DataFormat>,
+    outputFormat?: TOutputFormat
+  ): Promise<Output<TOutputFormat>>;
+
+  decryptMulti<TOutputFormat extends DataFormat>(
+    key: SerializedKey,
+    cipherData: Cipher<DataFormat>[],
+    outputFormat?: TOutputFormat
+  ): Promise<Output<TOutputFormat>[]>;
 
   hash(password: string, salt: string): Promise<string>;
 

packages/crypto/src/types.ts

@@ -19,21 +19,20 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { StringOutputFormat, Uint8ArrayOutputFormat } from "@notesnook/sodium";
 
-export type OutputFormat = Uint8ArrayOutputFormat | StringOutputFormat;
+export type DataFormat = Uint8ArrayOutputFormat | StringOutputFormat;
 
-export type Cipher = {
-  format: OutputFormat;
+export type Cipher<TFormat extends DataFormat> = {
+  format: TFormat;
   alg: string;
-  cipher: string | Uint8Array;
+  cipher: Output<TFormat>;
   iv: string;
   salt: string;
   length: number;
 };
 
-export type Plaintext = {
-  format: OutputFormat;
-  data: string | Uint8Array;
-};
+export type Output<TFormat extends DataFormat> =
+  TFormat extends StringOutputFormat ? string : Uint8Array;
+export type Input<TFormat extends DataFormat> = Output<TFormat>;
 
 export type SerializedKey = {
   password?: string;

packages/editor/package-lock.json

@@ -38,8 +38,8 @@
         "@tiptap/extension-underline": "2.0.3",
         "@tiptap/pm": "2.0.3",
         "@tiptap/starter-kit": "2.0.3",
-        "clipboard-polyfill": "4.0.0",
         "detect-indent": "^7.0.0",
+        "entities": "^4.5.0",
         "katex": "0.16.4",
         "nanoid": "^4.0.1",
         "prism-themes": "^1.9.0",
@@ -88,8 +88,8 @@
       "version": "7.4.1",
       "license": "GPL-3.0-or-later",
       "dependencies": {
-        "@microsoft/signalr": "^6.0.7",
-        "@microsoft/signalr-protocol-msgpack": "^6.0.7",
+        "@microsoft/signalr": "^7.0.10",
+        "@microsoft/signalr-protocol-msgpack": "^7.0.10",
         "@notesnook/logger": "file:../logger",
         "@streetwriters/showdown": "^3.0.1-alpha.2",
         "async-mutex": "^0.3.2",
@@ -111,6 +111,7 @@
         "@types/katex": "^0.16.1",
         "@types/prismjs": "^1.26.0",
         "@types/showdown": "^2.0.0",
+        "@vitest/coverage-v8": "^0.34.1",
         "abortcontroller-polyfill": "^1.7.3",
         "cross-env": "^7.0.3",
         "dotenv": "^16.0.1",
@@ -1908,11 +1909,6 @@
         "url": "https://github.com/chalk/strip-ansi?sponsor=1"
       }
     },
-    "node_modules/clipboard-polyfill": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/clipboard-polyfill/-/clipboard-polyfill-4.0.0.tgz",
-      "integrity": "sha512-U4KPNJqAYuyOtixCZZUyWTcj+wlI66j07g5ggMRE2DR1VFu/3ZWXkjLAslmme8i065gBSCUblHET7DKQ2Xg3RA=="
-    },
     "node_modules/color-convert": {
       "version": "1.9.3",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
@@ -2093,9 +2089,15 @@
       "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
     },
     "node_modules/entities": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
-      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+      "engines": {
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
     },
     "node_modules/error-ex": {
       "version": "1.3.2",
@@ -2337,6 +2339,11 @@
         "readable-stream": "^3.1.1"
       }
     },
+    "node_modules/htmlparser2/node_modules/entities": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
+      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
+    },
     "node_modules/iconv-lite": {
       "version": "0.4.24",
       "license": "MIT",
@@ -3085,7 +3092,6 @@
       "version": "17.0.2",
       "resolved": "https://registry.npmjs.org/react/-/react-17.0.2.tgz",
       "integrity": "sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA==",
-      "dev": true,
       "dependencies": {
         "loose-envify": "^1.1.0",
         "object-assign": "^4.1.1"
@@ -3106,7 +3112,6 @@
       "version": "17.0.2",
       "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-17.0.2.tgz",
       "integrity": "sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA==",
-      "dev": true,
       "dependencies": {
         "loose-envify": "^1.1.0",
         "object-assign": "^4.1.1",
@@ -3239,7 +3244,6 @@
       "version": "0.20.2",
       "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz",
       "integrity": "sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ==",
-      "dev": true,
       "dependencies": {
         "loose-envify": "^1.1.0",
         "object-assign": "^4.1.1"
@@ -4057,7 +4061,8 @@
     "@emotion/use-insertion-effect-with-fallbacks": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.0.1.tgz",
-      "integrity": "sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw=="
+      "integrity": "sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw==",
+      "requires": {}
     },
     "@emotion/utils": {
       "version": "1.2.1",
@@ -4306,8 +4311,8 @@
     "@notesnook/core": {
       "version": "file:../core",
       "requires": {
-        "@microsoft/signalr": "^6.0.7",
-        "@microsoft/signalr-protocol-msgpack": "^6.0.7",
+        "@microsoft/signalr": "^7.0.10",
+        "@microsoft/signalr-protocol-msgpack": "^7.0.10",
         "@notesnook/crypto": "file:../crypto",
         "@notesnook/logger": "file:../logger",
         "@streetwriters/showdown": "^3.0.1-alpha.2",
@@ -4315,6 +4320,7 @@
         "@types/katex": "^0.16.1",
         "@types/prismjs": "^1.26.0",
         "@types/showdown": "^2.0.0",
+        "@vitest/coverage-v8": "^0.34.1",
         "abortcontroller-polyfill": "^1.7.3",
         "async-mutex": "^0.3.2",
         "cross-env": "^7.0.3",
@@ -4528,87 +4534,104 @@
     "@tiptap/core": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/core/-/core-2.0.3.tgz",
-      "integrity": "sha512-jLyVIWAdjjlNzrsRhSE2lVL/7N8228/1R1QtaVU85UlMIwHFAcdzhD8FeiKkqxpTnGpaDVaTy7VNEtEgaYdCyA=="
+      "integrity": "sha512-jLyVIWAdjjlNzrsRhSE2lVL/7N8228/1R1QtaVU85UlMIwHFAcdzhD8FeiKkqxpTnGpaDVaTy7VNEtEgaYdCyA==",
+      "requires": {}
     },
     "@tiptap/extension-blockquote": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-blockquote/-/extension-blockquote-2.0.3.tgz",
-      "integrity": "sha512-rkUcFv2iL6f86DBBHoa4XdKNG2StvkJ7tfY9GoMpT46k3nxOaMTqak9/qZOo79TWxMLYtXzoxtKIkmWsbbcj4A=="
+      "integrity": "sha512-rkUcFv2iL6f86DBBHoa4XdKNG2StvkJ7tfY9GoMpT46k3nxOaMTqak9/qZOo79TWxMLYtXzoxtKIkmWsbbcj4A==",
+      "requires": {}
     },
     "@tiptap/extension-bold": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-bold/-/extension-bold-2.0.3.tgz",
-      "integrity": "sha512-OGT62fMRovSSayjehumygFWTg2Qn0IDbqyMpigg/RUAsnoOI2yBZFVrdM2gk1StyoSay7gTn2MLw97IUfr7FXg=="
+      "integrity": "sha512-OGT62fMRovSSayjehumygFWTg2Qn0IDbqyMpigg/RUAsnoOI2yBZFVrdM2gk1StyoSay7gTn2MLw97IUfr7FXg==",
+      "requires": {}
     },
     "@tiptap/extension-bullet-list": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-bullet-list/-/extension-bullet-list-2.0.3.tgz",
-      "integrity": "sha512-RtaLiRvZbMTOje+FW5bn+mYogiIgNxOm065wmyLPypnTbLSeHeYkoqVSqzZeqUn+7GLnwgn1shirUe6csVE/BA=="
+      "integrity": "sha512-RtaLiRvZbMTOje+FW5bn+mYogiIgNxOm065wmyLPypnTbLSeHeYkoqVSqzZeqUn+7GLnwgn1shirUe6csVE/BA==",
+      "requires": {}
     },
     "@tiptap/extension-character-count": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-character-count/-/extension-character-count-2.0.3.tgz",
-      "integrity": "sha512-Ge4aUmgYOmQR/HLPkbQSFKEywyRu6IalHAQmH3laY6LB9qrmT90AoaiFnaVCDpphYFQ7RygnBXJMgjtJ3WpZmw=="
+      "integrity": "sha512-Ge4aUmgYOmQR/HLPkbQSFKEywyRu6IalHAQmH3laY6LB9qrmT90AoaiFnaVCDpphYFQ7RygnBXJMgjtJ3WpZmw==",
+      "requires": {}
     },
     "@tiptap/extension-code": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-code/-/extension-code-2.0.3.tgz",
-      "integrity": "sha512-LsVCKVxgBtkstAr1FjxN8T3OjlC76a2X8ouoZpELMp+aXbjqyanCKzt+sjjUhE4H0yLFd4v+5v6UFoCv4EILiw=="
+      "integrity": "sha512-LsVCKVxgBtkstAr1FjxN8T3OjlC76a2X8ouoZpELMp+aXbjqyanCKzt+sjjUhE4H0yLFd4v+5v6UFoCv4EILiw==",
+      "requires": {}
     },
     "@tiptap/extension-code-block": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-code-block/-/extension-code-block-2.0.3.tgz",
-      "integrity": "sha512-F4xMy18EwgpyY9f5Te7UuF7UwxRLptOtCq1p2c2DfxBvHDWhAjQqVqcW/sq/I/WuED7FwCnPLyyAasPiVPkLPw=="
+      "integrity": "sha512-F4xMy18EwgpyY9f5Te7UuF7UwxRLptOtCq1p2c2DfxBvHDWhAjQqVqcW/sq/I/WuED7FwCnPLyyAasPiVPkLPw==",
+      "requires": {}
     },
     "@tiptap/extension-color": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-color/-/extension-color-2.0.3.tgz",
-      "integrity": "sha512-LYj3CWahhuJOy4/bwOur+cob8eky7xx7wyyBFIYELuzLcZt9hBmZwXxinQzD7BaQv4YdT+3oqr8BhChuPNj52w=="
+      "integrity": "sha512-LYj3CWahhuJOy4/bwOur+cob8eky7xx7wyyBFIYELuzLcZt9hBmZwXxinQzD7BaQv4YdT+3oqr8BhChuPNj52w==",
+      "requires": {}
     },
     "@tiptap/extension-document": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-document/-/extension-document-2.0.3.tgz",
-      "integrity": "sha512-PsYeNQQBYIU9ayz1R11Kv/kKNPFNIV8tApJ9pxelXjzcAhkjncNUazPN/dyho60mzo+WpsmS3ceTj/gK3bCtWA=="
+      "integrity": "sha512-PsYeNQQBYIU9ayz1R11Kv/kKNPFNIV8tApJ9pxelXjzcAhkjncNUazPN/dyho60mzo+WpsmS3ceTj/gK3bCtWA==",
+      "requires": {}
     },
     "@tiptap/extension-dropcursor": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-dropcursor/-/extension-dropcursor-2.0.3.tgz",
-      "integrity": "sha512-McthMrfusn6PjcaynJLheZJcXto8TaIW5iVitYh8qQrDXr31MALC/5GvWuiswmQ8bAXiWPwlLDYE/OJfwtggaw=="
+      "integrity": "sha512-McthMrfusn6PjcaynJLheZJcXto8TaIW5iVitYh8qQrDXr31MALC/5GvWuiswmQ8bAXiWPwlLDYE/OJfwtggaw==",
+      "requires": {}
     },
     "@tiptap/extension-font-family": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-font-family/-/extension-font-family-2.0.3.tgz",
-      "integrity": "sha512-Fg7lqoaiKfBHFzJDLa2QE4QtF/dX2KG0kV4P+Kx2s0S0Z6vfKQ7KLgOg8QBkgNsI/b1KdXN7hAefHCC/L8l7bQ=="
+      "integrity": "sha512-Fg7lqoaiKfBHFzJDLa2QE4QtF/dX2KG0kV4P+Kx2s0S0Z6vfKQ7KLgOg8QBkgNsI/b1KdXN7hAefHCC/L8l7bQ==",
+      "requires": {}
     },
     "@tiptap/extension-gapcursor": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-gapcursor/-/extension-gapcursor-2.0.3.tgz",
-      "integrity": "sha512-6I9EzzsYOyyqDvDvxIK6Rv3EXB+fHKFj8ntHO8IXmeNJ6pkhOinuXVsW6Yo7TcDYoTj4D5I2MNFAW2rIkgassw=="
+      "integrity": "sha512-6I9EzzsYOyyqDvDvxIK6Rv3EXB+fHKFj8ntHO8IXmeNJ6pkhOinuXVsW6Yo7TcDYoTj4D5I2MNFAW2rIkgassw==",
+      "requires": {}
     },
     "@tiptap/extension-hard-break": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-hard-break/-/extension-hard-break-2.0.3.tgz",
-      "integrity": "sha512-RCln6ARn16jvKTjhkcAD5KzYXYS0xRMc0/LrHeV8TKdCd4Yd0YYHe0PU4F9gAgAfPQn7Dgt4uTVJLN11ICl8sQ=="
+      "integrity": "sha512-RCln6ARn16jvKTjhkcAD5KzYXYS0xRMc0/LrHeV8TKdCd4Yd0YYHe0PU4F9gAgAfPQn7Dgt4uTVJLN11ICl8sQ==",
+      "requires": {}
     },
     "@tiptap/extension-heading": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-heading/-/extension-heading-2.0.3.tgz",
-      "integrity": "sha512-f0IEv5ms6aCzL80WeZ1qLCXTkRVwbpRr1qAETjg3gG4eoJN18+lZNOJYpyZy3P92C5KwF2T3Av00eFyVLIbb8Q=="
+      "integrity": "sha512-f0IEv5ms6aCzL80WeZ1qLCXTkRVwbpRr1qAETjg3gG4eoJN18+lZNOJYpyZy3P92C5KwF2T3Av00eFyVLIbb8Q==",
+      "requires": {}
     },
     "@tiptap/extension-history": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-history/-/extension-history-2.0.3.tgz",
-      "integrity": "sha512-00KHIcJ8kivn2ARI6NQYphv2LfllVCXViHGm0EhzDW6NQxCrriJKE3tKDcTFCu7LlC5doMpq9Z6KXdljc4oVeQ=="
+      "integrity": "sha512-00KHIcJ8kivn2ARI6NQYphv2LfllVCXViHGm0EhzDW6NQxCrriJKE3tKDcTFCu7LlC5doMpq9Z6KXdljc4oVeQ==",
+      "requires": {}
     },
     "@tiptap/extension-horizontal-rule": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-horizontal-rule/-/extension-horizontal-rule-2.0.3.tgz",
-      "integrity": "sha512-SZRUSh07b/M0kJHNKnfBwBMWrZBEm/E2LrK1NbluwT3DBhE+gvwiEdBxgB32zKHNxaDEXUJwUIPNC3JSbKvPUA=="
+      "integrity": "sha512-SZRUSh07b/M0kJHNKnfBwBMWrZBEm/E2LrK1NbluwT3DBhE+gvwiEdBxgB32zKHNxaDEXUJwUIPNC3JSbKvPUA==",
+      "requires": {}
     },
     "@tiptap/extension-italic": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-italic/-/extension-italic-2.0.3.tgz",
-      "integrity": "sha512-cfS5sW0gu7qf4ihwnLtW/QMTBrBEXaT0sJl3RwkhjIBg/65ywJKE5Nz9ewnQHmDeT18hvMJJ1VIb4j4ze9jj9A=="
+      "integrity": "sha512-cfS5sW0gu7qf4ihwnLtW/QMTBrBEXaT0sJl3RwkhjIBg/65ywJKE5Nz9ewnQHmDeT18hvMJJ1VIb4j4ze9jj9A==",
+      "requires": {}
     },
     "@tiptap/extension-link": {
       "version": "2.0.3",
@@ -4621,87 +4644,104 @@
     "@tiptap/extension-list-item": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-list-item/-/extension-list-item-2.0.3.tgz",
-      "integrity": "sha512-p7cUsk0LpM1PfdAuFE8wYBNJ3gvA0UhNGR08Lo++rt9UaCeFLSN1SXRxg97c0oa5+Ski7SrCjIJ5Ynhz0viTjQ=="
+      "integrity": "sha512-p7cUsk0LpM1PfdAuFE8wYBNJ3gvA0UhNGR08Lo++rt9UaCeFLSN1SXRxg97c0oa5+Ski7SrCjIJ5Ynhz0viTjQ==",
+      "requires": {}
     },
     "@tiptap/extension-ordered-list": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-ordered-list/-/extension-ordered-list-2.0.3.tgz",
-      "integrity": "sha512-ZB3MpZh/GEy1zKgw7XDQF4FIwycZWNof1k9WbDZOI063Ch4qHZowhVttH2mTCELuyvTMM/o9a8CS7qMqQB48bw=="
+      "integrity": "sha512-ZB3MpZh/GEy1zKgw7XDQF4FIwycZWNof1k9WbDZOI063Ch4qHZowhVttH2mTCELuyvTMM/o9a8CS7qMqQB48bw==",
+      "requires": {}
     },
     "@tiptap/extension-paragraph": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-paragraph/-/extension-paragraph-2.0.3.tgz",
-      "integrity": "sha512-a+tKtmj4bU3GVCH1NE8VHWnhVexxX5boTVxsHIr4yGG3UoKo1c5AO7YMaeX2W5xB5iIA+BQqOPCDPEAx34dd2A=="
+      "integrity": "sha512-a+tKtmj4bU3GVCH1NE8VHWnhVexxX5boTVxsHIr4yGG3UoKo1c5AO7YMaeX2W5xB5iIA+BQqOPCDPEAx34dd2A==",
+      "requires": {}
     },
     "@tiptap/extension-placeholder": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-placeholder/-/extension-placeholder-2.0.3.tgz",
-      "integrity": "sha512-Z42jo0termRAf0S0L8oxrts94IWX5waU4isS2CUw8xCUigYyCFslkhQXkWATO1qRbjNFLKN2C9qvCgGf4UeBrw=="
+      "integrity": "sha512-Z42jo0termRAf0S0L8oxrts94IWX5waU4isS2CUw8xCUigYyCFslkhQXkWATO1qRbjNFLKN2C9qvCgGf4UeBrw==",
+      "requires": {}
     },
     "@tiptap/extension-strike": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-strike/-/extension-strike-2.0.3.tgz",
-      "integrity": "sha512-RO4/EYe2iPD6ifDHORT8fF6O9tfdtnzxLGwZIKZXnEgtweH+MgoqevEzXYdS+54Wraq4TUQGNcsYhe49pv7Rlw=="
+      "integrity": "sha512-RO4/EYe2iPD6ifDHORT8fF6O9tfdtnzxLGwZIKZXnEgtweH+MgoqevEzXYdS+54Wraq4TUQGNcsYhe49pv7Rlw==",
+      "requires": {}
     },
     "@tiptap/extension-subscript": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-subscript/-/extension-subscript-2.0.3.tgz",
-      "integrity": "sha512-XFAEUaKxWRmTq7ePEF4aj7knelJPr2fTz0y/iSXydtS094LKwBHBzxatIZY3phrgfpDc+f51ycwarsgz27UJfg=="
+      "integrity": "sha512-XFAEUaKxWRmTq7ePEF4aj7knelJPr2fTz0y/iSXydtS094LKwBHBzxatIZY3phrgfpDc+f51ycwarsgz27UJfg==",
+      "requires": {}
     },
     "@tiptap/extension-superscript": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-superscript/-/extension-superscript-2.0.3.tgz",
-      "integrity": "sha512-5EBjUvkw2SXL1e8C1i0UF26/GBNHxEbiNQKw7Shy88omVa4HTY+D8KWC/j29ZW/IomUbGPlbpXp1z+1TETzmyw=="
+      "integrity": "sha512-5EBjUvkw2SXL1e8C1i0UF26/GBNHxEbiNQKw7Shy88omVa4HTY+D8KWC/j29ZW/IomUbGPlbpXp1z+1TETzmyw==",
+      "requires": {}
     },
     "@tiptap/extension-table": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-table/-/extension-table-2.0.3.tgz",
-      "integrity": "sha512-8swHqm8vRM1w9WzaAhLmY24gGoTozctz4KHKBjvFY/Ka0yXabT0+hoCCdkZLnXWi15H3pbHs2HnDBaTGL9bZTw=="
+      "integrity": "sha512-8swHqm8vRM1w9WzaAhLmY24gGoTozctz4KHKBjvFY/Ka0yXabT0+hoCCdkZLnXWi15H3pbHs2HnDBaTGL9bZTw==",
+      "requires": {}
     },
     "@tiptap/extension-table-cell": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-table-cell/-/extension-table-cell-2.0.3.tgz",
-      "integrity": "sha512-d0vpwQfRIOhqKJdoiOJybwWhjnug3QA4Mkgccp378moDRyOer3hPKavG1Ljgz087qHrN4WfdUlMGEvasYsWE7w=="
+      "integrity": "sha512-d0vpwQfRIOhqKJdoiOJybwWhjnug3QA4Mkgccp378moDRyOer3hPKavG1Ljgz087qHrN4WfdUlMGEvasYsWE7w==",
+      "requires": {}
     },
     "@tiptap/extension-table-header": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-table-header/-/extension-table-header-2.0.3.tgz",
-      "integrity": "sha512-SnGl1U6usRRS6LyAjSdhaCYLF6NWbGhjVFSmiPrjb0pOzsiVeDOiUNCyUAIYaDNnjAF2pfK6+H+uHzYPqTi+/w=="
+      "integrity": "sha512-SnGl1U6usRRS6LyAjSdhaCYLF6NWbGhjVFSmiPrjb0pOzsiVeDOiUNCyUAIYaDNnjAF2pfK6+H+uHzYPqTi+/w==",
+      "requires": {}
     },
     "@tiptap/extension-table-row": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-table-row/-/extension-table-row-2.0.3.tgz",
-      "integrity": "sha512-tyqeXmQLNSBsYyiNsnQuJMxNbz6dYt+P5W58+h10mjbt+hERA5+alQQyP06O2DggsT3Z0LPt7QRAlNmOBe7cyQ=="
+      "integrity": "sha512-tyqeXmQLNSBsYyiNsnQuJMxNbz6dYt+P5W58+h10mjbt+hERA5+alQQyP06O2DggsT3Z0LPt7QRAlNmOBe7cyQ==",
+      "requires": {}
     },
     "@tiptap/extension-task-item": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-task-item/-/extension-task-item-2.0.3.tgz",
-      "integrity": "sha512-13u1Q769WiSNcjFieYAMuJyWXNaY9yOdw6WFg9tQg4EZ5h6+2DaxB0qmu6I3pH+wwSn2UkCkXIirAo/k7wnzbw=="
+      "integrity": "sha512-13u1Q769WiSNcjFieYAMuJyWXNaY9yOdw6WFg9tQg4EZ5h6+2DaxB0qmu6I3pH+wwSn2UkCkXIirAo/k7wnzbw==",
+      "requires": {}
     },
     "@tiptap/extension-task-list": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-task-list/-/extension-task-list-2.0.3.tgz",
-      "integrity": "sha512-NdW0RtMF2L96qy+j946mTB5Av6Qn5L3vGVWFmJA6/JPXr9Uj/grItCmqUQKHfPBSFow7UqBY82ODblP+GQFgew=="
+      "integrity": "sha512-NdW0RtMF2L96qy+j946mTB5Av6Qn5L3vGVWFmJA6/JPXr9Uj/grItCmqUQKHfPBSFow7UqBY82ODblP+GQFgew==",
+      "requires": {}
     },
     "@tiptap/extension-text": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-text/-/extension-text-2.0.3.tgz",
-      "integrity": "sha512-LvzChcTCcPSMNLUjZe/A9SHXWGDHtvk73fR7CBqAeNU0MxhBPEBI03GFQ6RzW3xX0CmDmjpZoDxFMB+hDEtW1A=="
+      "integrity": "sha512-LvzChcTCcPSMNLUjZe/A9SHXWGDHtvk73fR7CBqAeNU0MxhBPEBI03GFQ6RzW3xX0CmDmjpZoDxFMB+hDEtW1A==",
+      "requires": {}
     },
     "@tiptap/extension-text-align": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-text-align/-/extension-text-align-2.0.3.tgz",
-      "integrity": "sha512-VlLgqncKdjMjVjbU60/ALYhFs0wUdjAyvjDXnH1OoM/HuzbILvufPMYz4DUieJIWVJOYUKHQgg4XwBWceAM2Tw=="
+      "integrity": "sha512-VlLgqncKdjMjVjbU60/ALYhFs0wUdjAyvjDXnH1OoM/HuzbILvufPMYz4DUieJIWVJOYUKHQgg4XwBWceAM2Tw==",
+      "requires": {}
     },
     "@tiptap/extension-text-style": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-text-style/-/extension-text-style-2.0.3.tgz",
-      "integrity": "sha512-yHIYtZVewSwfBfI6TffnsDRiOuXzytppcCsaDlsZFm8OtLG8v9ioH0ItMoOstmZZBiWJOm8iOy2yWSc4rNQEJw=="
+      "integrity": "sha512-yHIYtZVewSwfBfI6TffnsDRiOuXzytppcCsaDlsZFm8OtLG8v9ioH0ItMoOstmZZBiWJOm8iOy2yWSc4rNQEJw==",
+      "requires": {}
     },
     "@tiptap/extension-underline": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@tiptap/extension-underline/-/extension-underline-2.0.3.tgz",
-      "integrity": "sha512-oMYa7qib/5wJjpUp79GZEe+E/iyf1oZBsgiG26IspEtVTHZmpn3+Ktud7l43y/hpTeEzFTKOF1/uVbayHtSERg=="
+      "integrity": "sha512-oMYa7qib/5wJjpUp79GZEe+E/iyf1oZBsgiG26IspEtVTHZmpn3+Ktud7l43y/hpTeEzFTKOF1/uVbayHtSERg==",
+      "requires": {}
     },
     "@tiptap/pm": {
       "version": "2.0.3",
@@ -5052,11 +5092,6 @@
         }
       }
     },
-    "clipboard-polyfill": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/clipboard-polyfill/-/clipboard-polyfill-4.0.0.tgz",
-      "integrity": "sha512-U4KPNJqAYuyOtixCZZUyWTcj+wlI66j07g5ggMRE2DR1VFu/3ZWXkjLAslmme8i065gBSCUblHET7DKQ2Xg3RA=="
-    },
     "color-convert": {
       "version": "1.9.3",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
@@ -5195,9 +5230,9 @@
       "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
     },
     "entities": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
-      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="
     },
     "error-ex": {
       "version": "1.3.2",
@@ -5380,6 +5415,13 @@
         "entities": "^1.1.1",
         "inherits": "^2.0.1",
         "readable-stream": "^3.1.1"
+      },
+      "dependencies": {
+        "entities": {
+          "version": "1.1.2",
+          "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
+          "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
+        }
       }
     },
     "iconv-lite": {
@@ -5747,7 +5789,8 @@
       }
     },
     "prosemirror-codemark": {
-      "version": "0.4.2"
+      "version": "0.4.2",
+      "requires": {}
     },
     "prosemirror-collab": {
       "version": "1.3.0",
@@ -5891,26 +5934,26 @@
       }
     },
     "re-resizable": {
-      "version": "6.9.9"
+      "version": "6.9.9",
+      "requires": {}
     },
     "react": {
       "version": "17.0.2",
       "resolved": "https://registry.npmjs.org/react/-/react-17.0.2.tgz",
       "integrity": "sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA==",
-      "dev": true,
       "requires": {
         "loose-envify": "^1.1.0",
         "object-assign": "^4.1.1"
       }
     },
     "react-colorful": {
-      "version": "5.6.1"
+      "version": "5.6.1",
+      "requires": {}
     },
     "react-dom": {
       "version": "17.0.2",
       "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-17.0.2.tgz",
       "integrity": "sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA==",
-      "dev": true,
       "requires": {
         "loose-envify": "^1.1.0",
         "object-assign": "^4.1.1",
@@ -5997,7 +6040,6 @@
       "version": "0.20.2",
       "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz",
       "integrity": "sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ==",
-      "dev": true,
       "requires": {
         "loose-envify": "^1.1.0",
         "object-assign": "^4.1.1"
@@ -6225,7 +6267,8 @@
       }
     },
     "use-sync-external-store": {
-      "version": "1.2.0"
+      "version": "1.2.0",
+      "requires": {}
     },
     "util-deprecate": {
       "version": "1.0.2",

packages/editor/package.json

@@ -33,8 +33,8 @@
     "@tiptap/extension-underline": "2.0.3",
     "@tiptap/pm": "2.0.3",
     "@tiptap/starter-kit": "2.0.3",
-    "clipboard-polyfill": "4.0.0",
     "detect-indent": "^7.0.0",
+    "entities": "^4.5.0",
     "katex": "0.16.4",
     "nanoid": "^4.0.1",
     "prism-themes": "^1.9.0",

packages/editor/src/extensions/clipboard-text-serializer/index.ts

@@ -1,139 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import { Extension, TextSerializer } from "@tiptap/core";
-import { Plugin, PluginKey } from "prosemirror-state";
-import { Fragment, Schema, Slice } from "prosemirror-model";
-import { ListItem } from "../list-item";
-import { LIST_NODE_TYPES } from "../../utils/node-types";
-import { DOMSerializer } from "@tiptap/pm/model";
-
-export class ClipboardDOMSerializer extends DOMSerializer {
-  static fromSchema(schema: Schema): ClipboardDOMSerializer {
-    return (
-      schema.cached.domSerializer2 ||
-      (schema.cached.domSerializer2 = new ClipboardDOMSerializer(
-        this.nodesFromSchema(schema),
-        this.marksFromSchema(schema)
-      ))
-    );
-  }
-
-  serializeFragment(
-    fragment: Fragment,
-    options?: { document?: Document | undefined } | undefined,
-    target?: HTMLElement | DocumentFragment | undefined
-  ): HTMLElement | DocumentFragment {
-    const dom = super.serializeFragment(fragment, options, target);
-    for (const p of dom.querySelectorAll("li > p")) {
-      if (p.parentElement && p.parentElement.childElementCount > 1) continue;
-      p.parentElement?.append(...p.childNodes);
-      p.remove();
-    }
-
-    for (const p of dom.querySelectorAll('p[data-spacing="single"]')) {
-      if (!p.previousElementSibling || p.previousElementSibling.tagName !== "P")
-        continue;
-      if (p.previousElementSibling.childNodes.length > 0)
-        p.previousElementSibling.appendChild(document.createElement("br"));
-      p.previousElementSibling.append(...p.childNodes);
-      p.remove();
-    }
-
-    return dom;
-  }
-}
-
-export const ClipboardTextSerializer = Extension.create({
-  name: "clipboardTextSerializer",
-
-  addProseMirrorPlugins() {
-    return [
-      new Plugin({
-        key: new PluginKey("clipboardTextSerializer"),
-        props: {
-          transformCopied,
-          clipboardSerializer: ClipboardDOMSerializer.fromSchema(
-            this.editor.view.state.schema
-          ),
-          clipboardTextSerializer: (content, view) => {
-            return getTextBetween(content, view.state.schema);
-          }
-        }
-      })
-    ];
-  }
-});
-
-export function transformCopied(slice: Slice) {
-  // when copying a single list item, we shouldn't retain the
-  // list formatting but copy it as a paragraph.
-  const maybeList = slice.content.firstChild;
-  if (
-    maybeList &&
-    LIST_NODE_TYPES.includes(maybeList.type.name) &&
-    maybeList.childCount === 1 &&
-    maybeList.firstChild
-  ) {
-    return transformCopied(new Slice(maybeList.firstChild.content, 0, 0));
-  }
-  return slice;
-}
-
-export function getTextBetween(slice: Slice, schema: Schema): string {
-  const range = { from: 0, to: slice.size };
-  const separator = "\n";
-  let text = "";
-  let separated = true;
-
-  slice.content.nodesBetween(0, slice.size, (node, pos, parent, index) => {
-    const textSerializer = schema.nodes[node.type.name]?.spec
-      .toText as TextSerializer;
-
-    if (textSerializer) {
-      if (node.isBlock && !separated) {
-        text += separator;
-        separated = true;
-      }
-
-      if (parent) {
-        text += textSerializer({
-          node,
-          pos,
-          parent,
-          index,
-          range
-        });
-      }
-    } else if (node.isText) {
-      text += node?.text;
-      separated = false;
-    } else if (node.isBlock && !!text) {
-      // we don't want double spaced list items when pasting
-      if (index === 0 && parent?.type.name === ListItem.name) return;
-
-      text += separator;
-      if (node.attrs.spacing === "double" && node.childCount > 0)
-        text += separator;
-      separated = true;
-    }
-  });
-
-  return text;
-}

packages/editor/src/extensions/clipboard/clipboard-dom-parser.ts

@@ -17,12 +17,51 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-export function convertBrToParagraph(html: string) {
-  const doc = new DOMParser().parseFromString(
-    convertNewlinesToBr(html),
-    "text/html"
-  );
-  for (const br of doc.querySelectorAll("br")) {
+import {
+  DOMParser as ProsemirrorDOMParser,
+  ParseOptions
+} from "@tiptap/pm/model";
+import { encodeNonAsciiHTML } from "entities";
+import { Schema, Slice } from "prosemirror-model";
+import { inferLanguage } from "../code-block";
+
+export class ClipboardDOMParser extends ProsemirrorDOMParser {
+  static fromSchema(schema: Schema): ClipboardDOMParser {
+    return (
+      (schema.cached.clipboardDomParser as ClipboardDOMParser) ||
+      (schema.cached.clipboardDomParser = new ClipboardDOMParser(
+        schema,
+        (ProsemirrorDOMParser as any).schemaRules(schema)
+      ))
+    );
+  }
+
+  parseSlice(dom: Node, options?: ParseOptions | undefined): Slice {
+    if (dom instanceof HTMLElement) {
+      formatCodeblocks(dom);
+      convertBrToSingleSpacedParagraphs(dom);
+    }
+    return super.parseSlice(dom, options);
+  }
+}
+
+export function formatCodeblocks(dom: HTMLElement | Document) {
+  for (const pre of dom.querySelectorAll("pre")) {
+    const codeAsText = pre.textContent;
+    const languageElement = pre.querySelector(
+      '[class*="language-"],[class*="lang-"]'
+    );
+    const language = inferLanguage(languageElement || pre);
+    if (language) pre.classList.add(`language-${language}`);
+
+    const code = document.createElement("code");
+    code.innerHTML = encodeNonAsciiHTML(codeAsText || "");
+    pre.replaceChildren(code);
+  }
+}
+
+export function convertBrToSingleSpacedParagraphs(dom: HTMLElement | Document) {
+  for (const br of dom.querySelectorAll("br")) {
     let paragraph = br.closest("p");
 
     // if no paragraph is found over the br, we add one.
@@ -38,14 +77,13 @@ export function convertBrToParagraph(html: string) {
     if (paragraph) {
       splitOn(paragraph, br);
       const children = Array.from(paragraph.childNodes.values());
-      const newParagraph = doc.createElement("p");
+      const newParagraph = document.createElement("p");
       newParagraph.dataset.spacing = "single";
       newParagraph.append(...children.slice(children.indexOf(br) + 1));
       paragraph.insertAdjacentElement("afterend", newParagraph);
       br.remove();
     }
   }
-  return doc;
 }
 
 function splitOn(bound: Element, cutElement: Element) {
@@ -64,12 +102,3 @@ function splitOn(bound: Element, cutElement: Element) {
     }
   }
 }
-
-function convertNewlinesToBr(html: string) {
-  const lines = html.split(/\n/gm);
-  for (let i = 0; i < lines.length; ++i) {
-    if (lines[i].trim().endsWith(">")) continue;
-    lines[i] += "<br>";
-  }
-  return lines.join("");
-}

packages/editor/src/extensions/clipboard/clipboard-dom-serializer.ts

@@ -0,0 +1,57 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Fragment, Schema } from "prosemirror-model";
+import { DOMSerializer } from "@tiptap/pm/model";
+
+export class ClipboardDOMSerializer extends DOMSerializer {
+  static fromSchema(schema: Schema): ClipboardDOMSerializer {
+    return (
+      schema.cached.clipboardDomSerializer ||
+      (schema.cached.clipboardDomSerializer = new ClipboardDOMSerializer(
+        this.nodesFromSchema(schema),
+        this.marksFromSchema(schema)
+      ))
+    );
+  }
+
+  serializeFragment(
+    fragment: Fragment,
+    options?: { document?: Document | undefined } | undefined,
+    target?: HTMLElement | DocumentFragment | undefined
+  ): HTMLElement | DocumentFragment {
+    const dom = super.serializeFragment(fragment, options, target);
+    for (const p of dom.querySelectorAll("li > p")) {
+      if (p.parentElement && p.parentElement.childElementCount > 1) continue;
+      p.parentElement?.append(...p.childNodes);
+      p.remove();
+    }
+
+    for (const p of dom.querySelectorAll('p[data-spacing="single"]')) {
+      if (!p.previousElementSibling || p.previousElementSibling.tagName !== "P")
+        continue;
+      if (p.previousElementSibling.childNodes.length > 0)
+        p.previousElementSibling.appendChild(document.createElement("br"));
+      p.previousElementSibling.append(...p.childNodes);
+      p.remove();
+    }
+
+    return dom;
+  }
+}

packages/editor/src/extensions/clipboard/clipboard-text-parser.ts

@@ -0,0 +1,60 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { ResolvedPos, Slice } from "@tiptap/pm/model";
+import { encodeNonAsciiHTML } from "entities";
+import { ClipboardDOMParser } from "./clipboard-dom-parser";
+import { EditorView } from "@tiptap/pm/view";
+
+export function clipboardTextParser(
+  text: string,
+  _$context: ResolvedPos,
+  _plain: boolean,
+  view: EditorView
+): Slice {
+  const doc = new DOMParser().parseFromString(
+    convertTextToHTML(text),
+    "text/html"
+  );
+  return ClipboardDOMParser.fromSchema(view.state.schema).parseSlice(doc, {
+    preserveWhitespace: "full"
+  });
+}
+
+export function convertTextToHTML(src: string) {
+  return src
+    .split(/[\r\n]/)
+    .map((line) =>
+      line
+        ? `<p data-spacing="single">${encodeLine(line)}</p>`
+        : `<p data-spacing="single"></p>`
+    )
+    .join("");
+}
+
+function encodeLine(line: string) {
+  line = encodeNonAsciiHTML(line);
+  line = line.replace(/(^ +)|( {2,})/g, (sub, ...args) => {
+    const [starting, inline] = args;
+    if (starting) return "&nbsp;".repeat(starting.length);
+    if (inline) return "&nbsp;".repeat(inline.length);
+    return sub;
+  });
+  return line;
+}

packages/editor/src/extensions/clipboard/clipboard-text-serializer.ts

@@ -0,0 +1,69 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { TextSerializer } from "@tiptap/core";
+import { Schema, Slice } from "prosemirror-model";
+import { ListItem } from "../list-item";
+import { EditorView } from "@tiptap/pm/view";
+
+export function clipboardTextSerializer(content: Slice, view: EditorView) {
+  return getTextBetween(content, view.state.schema);
+}
+
+export function getTextBetween(slice: Slice, schema: Schema): string {
+  const range = { from: 0, to: slice.size };
+  const separator = "\n";
+  let text = "";
+  let separated = true;
+
+  slice.content.nodesBetween(0, slice.size, (node, pos, parent, index) => {
+    const textSerializer = schema.nodes[node.type.name]?.spec
+      .toText as TextSerializer;
+
+    if (textSerializer) {
+      if (node.isBlock && !separated) {
+        text += separator;
+        separated = true;
+      }
+
+      if (parent) {
+        text += textSerializer({
+          node,
+          pos,
+          parent,
+          index,
+          range
+        });
+      }
+    } else if (node.isText) {
+      text += node?.text;
+      separated = false;
+    } else if (node.isBlock && !!text) {
+      // we don't want double spaced list items when pasting
+      if (index === 0 && parent?.type.name === ListItem.name) return;
+
+      text += separator;
+      if (node.attrs.spacing === "double" && node.childCount > 0)
+        text += separator;
+      separated = true;
+    }
+  });
+
+  return text;
+}

packages/editor/src/extensions/clipboard/clipboard.ts

@@ -16,8 +16,15 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
+
 import { Extension } from "@tiptap/core";
-import { writeText } from "clipboard-polyfill";
+import { Plugin, PluginKey } from "prosemirror-state";
+import { Slice } from "prosemirror-model";
+import { LIST_NODE_TYPES } from "../../utils/node-types";
+import { ClipboardDOMParser } from "./clipboard-dom-parser";
+import { ClipboardDOMSerializer } from "./clipboard-dom-serializer";
+import { clipboardTextParser } from "./clipboard-text-parser";
+import { clipboardTextSerializer } from "./clipboard-text-serializer";
 
 declare module "@tiptap/core" {
   interface Commands<ReturnType> {
@@ -31,14 +38,15 @@ export type ClipboardOptions = {
   copyToClipboard: (text: string) => void;
 };
 
-export const Clipboard = Extension.create<ClipboardOptions>({
+export const Clipboard = Extension.create({
+  name: "clipboard",
+
   addOptions() {
     return {
-      copyToClipboard: (text) => {
-        writeText(text);
-      }
+      copyToClipboard: () => {}
     };
   },
+
   addCommands() {
     return {
       copyToClipboard: (text: string) => (props) => {
@@ -46,5 +54,39 @@ export const Clipboard = Extension.create<ClipboardOptions>({
         return true;
       }
     };
+  },
+
+  addProseMirrorPlugins() {
+    return [
+      new Plugin({
+        key: new PluginKey("clipboard"),
+        props: {
+          clipboardParser: ClipboardDOMParser.fromSchema(
+            this.editor.view.state.schema
+          ),
+          clipboardSerializer: ClipboardDOMSerializer.fromSchema(
+            this.editor.view.state.schema
+          ),
+          transformCopied,
+          clipboardTextParser,
+          clipboardTextSerializer
+        }
+      })
+    ];
   }
 });
+
+export function transformCopied(slice: Slice) {
+  // when copying a single list item, we shouldn't retain the
+  // list formatting but copy it as a paragraph.
+  const maybeList = slice.content.firstChild;
+  if (
+    maybeList &&
+    LIST_NODE_TYPES.includes(maybeList.type.name) &&
+    maybeList.childCount === 1 &&
+    maybeList.firstChild
+  ) {
+    return transformCopied(new Slice(maybeList.firstChild.content, 0, 0));
+  }
+  return slice;
+}

packages/editor/src/extensions/clipboard/index.ts

@@ -16,8 +16,6 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-import { Clipboard } from "./clipboard";
 
 export * from "./clipboard";
-
-export default Clipboard;
+export { Clipboard as default } from "./clipboard";

packages/editor/src/extensions/clipboard/tests/__snapshots__/clipboard-dom-parser.test.ts.snap

@@ -0,0 +1,68 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`convert br tags to paragraphs 1`] = `"<p>line 1</p><p data-spacing=\\"single\\">line 2</p>"`;
+
+exports[`convert br tags to paragraphs 2`] = `"<p>line <em>1</em></p><p data-spacing=\\"single\\"><em>line</em> 2</p>"`;
+
+exports[`convert br tags to paragraphs 3`] = `"<p>line <span><em>1</em></span></p><p data-spacing=\\"single\\"><span><em>line</em></span> 2</p>"`;
+
+exports[`convert br tags to paragraphs 4`] = `"<p>line <span><em>1</em></span></p><p data-spacing=\\"single\\"><span><em>line</em></span> 2</p>"`;
+
+exports[`convert br tags to paragraphs 5`] = `"<p><br></p>"`;
+
+exports[`convert br tags to paragraphs 6`] = `
+"<p>
+<!--StartFragment-->A troll, they call me, but I have no wish</p><p data-spacing=\\"single\\">
+to be associated with those dolls</p><p data-spacing=\\"single\\">
+</p><p data-spacing=\\"single\\">
+We lack religion, purpose, politics,</p><p data-spacing=\\"single\\">
+and yet, we somehow manage to get by.</p><p data-spacing=\\"single\\">
+
+  </p>"
+`;
+
+exports[`convert br tags to paragraphs 7`] = `
+"<!--StartFragment--><p dir=\\"auto\\">When I try to paste something (e.g. email content) to a note, the styling is kept, which is good, but the newlines are removed.</p><p data-spacing=\\"single\\">
+    Also when I share the selection to Notesnook via the share functionality from Android, I have the same issue.</p>
+    <hr>
+    <p dir=\\"auto\\"><strong>Device information:</strong></p><p data-spacing=\\"single\\">
+    App version: 2.3.0</p><p data-spacing=\\"single\\">
+    Platform: android</p><p data-spacing=\\"single\\">
+    Model: OnePlus-CPH2409-31</p><p data-spacing=\\"single\\">
+    Pro: true</p><p data-spacing=\\"single\\">
+    Logged in: yes</p><!--EndFragment-->"
+`;
+
+exports[`convert br tags to paragraphs 8`] = `
+"<!--StartFragment--><span class=\\"css-901oao css-16my406 r-poiln3 r-bcqeeo r-qvutc0\\">Why switch from Gmail? 
+
+    Not sacrificing features for more privacy, prefer using one app, in many public groups and channels (Telegram)
+    
+    LibreOffice Slow &amp; buggy
+    
+    Switched to Brave for the better Android app, more private out of the box &amp; unsure if uBlock Origin closes gap</span><!--EndFragment-->"
+`;
+
+exports[`properly format codeblocks 1`] = `
+"<div>
+  <!--StartFragment--><p>Sure! Here's an implementation of a word counter for Thai that considers each syllable consisting of a consonant sound followed by a vowel sound as a word:</p><pre><code>javascript</code></pre><pre class=\\"language-javascript\\"><code>function countThaiWords(text) {
+    // define a regular expression to match Thai syllables
+    const thaiSyllableRegex = /[&#xe01;-&#xe3a;&#xe40;-&#xe5b;]+[&#xe00;-&#xe7f;]?/g;
+  
+    // count the number of matches of the Thai syllable regex in the text
+    const matches = text.match(thaiSyllableRegex) || [];
+    const wordCount = matches.length;
+  
+    // return the number of words
+    return wordCount;
+  }
+  
+  // example usage
+  const text = &quot;&#xe2a;&#xe27;&#xe31;&#xe2a;&#xe14;&#xe35;&#xe04;&#xe48;&#xe30;&#xe22;&#xe34;&#xe19;&#xe14;&#xe35;&#xe15;&#xe49;&#xe2d;&#xe19;&#xe23;&#xe31;&#xe1a;&#xe40;&#xe02;&#xe49;&#xe32;&#xe2a;&#xe39;&#xe48;&#xe42;&#xe25;&#xe01;&#xe02;&#xe2d;&#xe07;&#xe09;&#xe31;&#xe19;&quot;;
+  const wordCount = countThaiWords(text);
+  console.log(\`Word count: &dollar;{wordCount}\`);
+  // output: Word count: 9
+  </code></pre><p>This implementation defines a regular expression to match Thai syllables, which are composed of one or more Thai characters. It then uses the <code>match</code> function of the <code>String</code> object to count the number of matches of the Thai syllable regex in the input text.</p><p>Note that this implementation assumes that each syllable consisting of a consonant sound followed by a vowel sound is considered a word in Thai. This may not always be accurate, as some words in Thai may consist of multiple syllables, and some syllables may be used as prefixes or suffixes to modify other words. However, this approach should work well for many cases where each syllable is considered a separate word.</p><!--EndFragment-->
+  
+  </div>"
+`;

packages/editor/src/extensions/clipboard/tests/clipboard-dom-parser.test.ts

@@ -0,0 +1,109 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { test } from "vitest";
+import {
+  formatCodeblocks,
+  convertBrToSingleSpacedParagraphs
+} from "../clipboard-dom-parser";
+
+const cases = [
+  [`<p>line 1<br>line 2</p>`],
+  [`<p>line <em>1<br>line</em> 2</p>`],
+  [`<p>line <span><em>1<br>line</em></span> 2</p>`],
+  [`<p>line <span><em>1<br data-some="hello">line</em></span> 2</p>`],
+  [`<p><br/></p>`],
+  [
+    `
+  <html><body>
+<!--StartFragment-->A troll, they call me, but I have no wish<br>
+to be associated with those dolls<br>
+<br>
+We lack religion, purpose, politics,<br>
+and yet, we somehow manage to get by.<br>
+</body>
+</html>`
+  ],
+  [
+    `<html><body>
+    <!--StartFragment--><p dir="auto">When I try to paste something (e.g. email content) to a note, the styling is kept, which is good, but the newlines are removed.<br>
+    Also when I share the selection to Notesnook via the share functionality from Android, I have the same issue.</p>
+    <hr>
+    <p dir="auto"><strong>Device information:</strong><br>
+    App version: 2.3.0<br>
+    Platform: android<br>
+    Model: OnePlus-CPH2409-31<br>
+    Pro: true<br>
+    Logged in: yes</p><!--EndFragment-->
+    </body>
+    </html>`
+  ],
+  [
+    `<html><body>
+    <!--StartFragment--><span class="css-901oao css-16my406 r-poiln3 r-bcqeeo r-qvutc0">Why switch from Gmail? 
+
+    Not sacrificing features for more privacy, prefer using one app, in many public groups and channels (Telegram)
+    
+    LibreOffice Slow &amp; buggy
+    
+    Switched to Brave for the better Android app, more private out of the box &amp; unsure if uBlock Origin closes gap</span><!--EndFragment-->
+    </body>
+    </html>`
+  ]
+];
+
+for (const testCase of cases) {
+  const [html, expected] = testCase;
+  test(`convert br tags to paragraphs`, (t) => {
+    const element = new DOMParser().parseFromString(html, "text/html");
+    convertBrToSingleSpacedParagraphs(element);
+    t.expect(element.body.innerHTML.trim()).toMatchSnapshot();
+  });
+}
+
+const codeBlocks = [
+  `<div>
+  <!--StartFragment--><p>Sure! Here's an implementation of a word counter for Thai that considers each syllable consisting of a consonant sound followed by a vowel sound as a word:</p><pre><div class="bg-black rounded-md mb-4"><div class="flex items-center relative text-gray-200 bg-gray-800 px-4 py-2 text-xs font-sans justify-between rounded-t-md"><span>javascript</span></div></div></pre><pre><div class="bg-black rounded-md mb-4"><div class="p-4 overflow-y-auto"><code class="!whitespace-pre hljs language-javascript"><span class="hljs-keyword">function</span> <span class="hljs-title function_">countThaiWords</span>(<span class="hljs-params">text</span>) {
+    <span class="hljs-comment">// define a regular expression to match Thai syllables</span>
+    <span class="hljs-keyword">const</span> thaiSyllableRegex = <span class="hljs-regexp">/[\u0E01-\u0E3A\u0E40-\u0E5B]+[\u0E00-\u0E7F]?/g</span>;
+  
+    <span class="hljs-comment">// count the number of matches of the Thai syllable regex in the text</span>
+    <span class="hljs-keyword">const</span> matches = text.<span class="hljs-title function_">match</span>(thaiSyllableRegex) || [];
+    <span class="hljs-keyword">const</span> wordCount = matches.<span class="hljs-property">length</span>;
+  
+    <span class="hljs-comment">// return the number of words</span>
+    <span class="hljs-keyword">return</span> wordCount;
+  }
+  
+  <span class="hljs-comment">// example usage</span>
+  <span class="hljs-keyword">const</span> text = <span class="hljs-string">"สวัสดีค่ะยินดีต้อนรับเข้าสู่โลกของฉัน"</span>;
+  <span class="hljs-keyword">const</span> wordCount = <span class="hljs-title function_">countThaiWords</span>(text);
+  <span class="hljs-variable language_">console</span>.<span class="hljs-title function_">log</span>(<span class="hljs-string">\`Word count: <span class="hljs-subst">\${wordCount}</span>\`</span>);
+  <span class="hljs-comment">// output: Word count: 9</span>
+  </code></div></div></pre><p>This implementation defines a regular expression to match Thai syllables, which are composed of one or more Thai characters. It then uses the <code>match</code> function of the <code>String</code> object to count the number of matches of the Thai syllable regex in the input text.</p><p>Note that this implementation assumes that each syllable consisting of a consonant sound followed by a vowel sound is considered a word in Thai. This may not always be accurate, as some words in Thai may consist of multiple syllables, and some syllables may be used as prefixes or suffixes to modify other words. However, this approach should work well for many cases where each syllable is considered a separate word.</p><!--EndFragment-->
+  
+  </div>`
+];
+for (const codeBlock of codeBlocks) {
+  test(`properly format codeblocks`, (t) => {
+    const element = new DOMParser().parseFromString(codeBlock, "text/html");
+    formatCodeblocks(element);
+    t.expect(element.body.innerHTML.trim()).toMatchSnapshot();
+  });
+}

packages/editor/src/extensions/clipboard/tests/clipboard-text-serializer.test.ts

@@ -21,12 +21,10 @@ import { test } from "vitest";
 import { createEditor, h } from "../../../../test-utils";
 import OrderedList from "../../ordered-list";
 import { ListItem } from "../../list-item";
-import {
-  getTextBetween,
-  transformCopied,
-  ClipboardDOMSerializer
-} from "../index";
+import { transformCopied } from "../index";
 import { Paragraph } from "../../paragraph";
+import { ClipboardDOMSerializer } from "../clipboard-dom-serializer";
+import { clipboardTextSerializer } from "../clipboard-text-serializer";
 
 test("copied list items shouldn't contain extra newlines", (t) => {
   const { editor } = createEditor({
@@ -59,9 +57,9 @@ test("copied list items shouldn't contain extra newlines", (t) => {
   );
 
   t.expect(
-    getTextBetween(
+    clipboardTextSerializer(
       editor.state.doc.slice(0, editor.state.doc.nodeSize - 2),
-      editor.schema
+      editor.view
     )
   ).toBe(`This is line: number 1.
 And this is line number 2.
@@ -194,9 +192,9 @@ for (const testCase of paragraphTestCases) {
     ).toBe(testCase.expectedHtml);
 
     t.expect(
-      getTextBetween(
+      clipboardTextSerializer(
         editor.state.doc.slice(0, editor.state.doc.nodeSize - 2),
-        editor.schema
+        editor.view
       )
     ).toBe(testCase.expectedText);
   });

packages/editor/src/extensions/code-block/code-block.ts

@@ -26,7 +26,7 @@ import {
   Transaction,
   Selection
 } from "prosemirror-state";
-import { ResolvedPos, Node as ProsemirrorNode } from "prosemirror-model";
+import { ResolvedPos, Node as ProsemirrorNode, Slice } from "prosemirror-model";
 import { CodeblockComponent } from "./component";
 import { HighlighterPlugin } from "./highlighter";
 import { createNodeView } from "../react";
@@ -508,7 +508,6 @@ export const CodeBlock = Node.create<CodeBlockOptions>({
             if (!event.clipboardData) {
               return false;
             }
-            const text = event.clipboardData.getData("text/plain");
             const { isCode, language } = detectCodeBlock(event.clipboardData);
 
             const isInsideCodeBlock = this.editor.isActive(this.type.name);
@@ -516,6 +515,12 @@ export const CodeBlock = Node.create<CodeBlockOptions>({
               return false;
             }
 
+            const text = event.clipboardData
+              .getData("text/plain")
+              // strip carriage return chars from text pasted as code
+              // see: https://github.com/ProseMirror/prosemirror-view/commit/a50a6bcceb4ce52ac8fcc6162488d8875613aacd
+              .replace(/\r\n?/g, "\n");
+
             const indent = fixIndentation(
               text,
               parseIndentation(view.state.selection.$from.parent)
@@ -523,22 +528,36 @@ export const CodeBlock = Node.create<CodeBlockOptions>({
 
             const { tr } = view.state;
 
-            // create an empty code block if not already within one
-            if (isCode && !isInsideCodeBlock) {
-              tr.replaceSelectionWith(
-                this.type.create({
-                  id: createCodeblockId(),
-                  language,
-                  indentType: indent.type,
-                  indentLength: indent.amount
+            const isInlineCode =
+              indent.code.length < 80 &&
+              indent.code.split(/[\r\n]/).length === 1;
+            if (isInlineCode && !isInsideCodeBlock) {
+              tr.replaceSelection(
+                Slice.fromJSON(this.editor.view.state.schema, {
+                  content: [
+                    {
+                      type: "text",
+                      text: indent.code,
+                      marks: [{ type: "code" }]
+                    }
+                  ]
                 })
               );
-            }
+            } else {
+              // create an empty code block if not already within one
+              if (!isInsideCodeBlock)
+                tr.replaceSelectionWith(
+                  this.type.create({
+                    id: createCodeblockId(),
+                    language,
+                    indentType: indent.type,
+                    indentLength: indent.amount
+                  })
+                );
 
-            // add text to code block
-            // strip carriage return chars from text pasted as code
-            // see: https://github.com/ProseMirror/prosemirror-view/commit/a50a6bcceb4ce52ac8fcc6162488d8875613aacd
-            tr.insertText(indent.code.replace(/\r\n?/g, "\n"));
+              // add text to code block
+              tr.insertText(indent.code);
+            }
 
             // store meta information
             // this is useful for other plugins that depends on the paste event
@@ -787,8 +806,8 @@ function detectCodeBlock(dataTransfer: DataTransfer) {
   const isVSCode =
     vscode ||
     (document.body.firstElementChild instanceof HTMLDivElement &&
-      document.body.firstElementChild.style.fontFamily ===
-        `'Droid Sans Mono', 'monospace', monospace`);
+      document.body.firstElementChild.style.fontFamily.includes("monospace") &&
+      document.body.firstElementChild.style.whiteSpace.includes("pre"));
 
   const language =
     vscodeData?.mode ||

packages/editor/src/extensions/code-block/tests/code-block.test.ts

@@ -122,7 +122,10 @@ test("pasting code from vscode should automatically create a syntax highlighted
   (clipboardEvent as unknown as any)["clipboardData"] = {
     getData: (type: string) =>
       type === "text/plain"
-        ? "function hello() { }"
+        ? `function hello()
+{
+  const world = "hello";
+}`
         : type === "vscode-editor-data"
         ? JSON.stringify({ mode: "javascript" })
         : undefined

packages/editor/src/extensions/image/image.ts

@@ -178,6 +178,7 @@ export const ImageNode = Node.create<ImageOptions>({
 
   addNodeView() {
     return createSelectionBasedNodeView(ImageComponent, {
+      componentKey: (node) => node.attrs.hash,
       shouldUpdate: (prev, next) => !hasSameAttributes(prev.attrs, next.attrs),
       forceEnableSelection: true
     });

packages/editor/src/extensions/react/selection-based-react-node-view.tsx

@@ -95,7 +95,11 @@ export class SelectionBasedNodeView<
       this.isSelectedNode(this.editor.view.state.selection);
 
     return (
-      <EmotionThemeProvider scope="editor" injectCssVars={false}>
+      <EmotionThemeProvider
+        key={this.options.componentKey?.(this.node)}
+        scope="editor"
+        injectCssVars={false}
+      >
         <this.options.component
           {...props}
           editor={this.editor}

packages/editor/src/extensions/react/types.ts

@@ -59,6 +59,7 @@ export type SelectionBasedReactNodeViewProps<TAttributes = Attrs> =
 export type ReactNodeViewOptions<P> = {
   props?: P;
   component?: React.ComponentType<P>;
+  componentKey?: (node: PMNode) => string;
   shouldUpdate?: ShouldUpdate;
   contentDOMFactory?: (() => ContentDOM) | boolean;
   wrapperFactory?: () => HTMLElement;

packages/editor/src/index.ts

@@ -40,7 +40,6 @@ import { useEffect, useMemo } from "react";
 import "./extensions";
 import { AttachmentNode, AttachmentOptions } from "./extensions/attachment";
 import BulletList from "./extensions/bullet-list";
-import { ClipboardTextSerializer } from "./extensions/clipboard-text-serializer";
 import { CodeBlock } from "./extensions/code-block";
 import { Codemark } from "./extensions/code-mark";
 import { DateTime, DateTimeOptions } from "./extensions/date-time";
@@ -76,7 +75,6 @@ import { useToolbarStore } from "./toolbar/stores/toolbar-store";
 import { DownloadOptions } from "./utils/downloader";
 import { Heading } from "./extensions/heading";
 import Clipboard, { ClipboardOptions } from "./extensions/clipboard";
-import { convertBrToParagraph } from "./utils/html";
 import Blockquote from "./extensions/blockquote";
 
 declare global {
@@ -147,14 +145,10 @@ const useTiptap = (
     () => ({
       enableCoreExtensions: false,
       editorProps: {
-        ...editorProps,
-        transformPastedHTML(html) {
-          return convertBrToParagraph(html).documentElement.outerHTML;
-        }
+        ...editorProps
       },
       extensions: [
         ...CoreExtensions,
-        ClipboardTextSerializer,
         NodeViewSelectionNotifier,
         SearchReplace,
         TextStyle.extend({

packages/editor/src/utils/tests/__snapshots__/html.test.ts.snap

@@ -1,45 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`convert br tags to paragraphs (
-  <html><body>
-<!--StartFragment-->A troll, they call me, but I have no wish<br>
-to be associated with those dolls<br>
-<br>
-We lack religion, purpose, politics,<br>
-and yet, we somehow manage to get by.<br>
-</body>
-</html>) 1`] = `"<p><!--StartFragment-->A troll, they call me, but I have no wish</p><p data-spacing=\\"single\\">to be associated with those dolls</p><p data-spacing=\\"single\\"></p><p data-spacing=\\"single\\">We lack religion, purpose, politics,</p><p data-spacing=\\"single\\">and yet, we somehow manage to get by.</p><p data-spacing=\\"single\\"></p><p data-spacing=\\"single\\">  </p>"`;
-
-exports[`convert br tags to paragraphs (<html><body>
-    <!--StartFragment--><p dir="auto">When I try to paste something (e.g. email content) to a note, the styling is kept, which is good, but the newlines are removed.<br>
-    Also when I share the selection to Notesnook via the share functionality from Android, I have the same issue.</p>
-    <hr>
-    <p dir="auto"><strong>Device information:</strong><br>
-    App version: 2.3.0<br>
-    Platform: android<br>
-    Model: OnePlus-CPH2409-31<br>
-    Pro: true<br>
-    Logged in: yes</p><!--EndFragment-->
-    </body>
-    </html>) 1`] = `"<!--StartFragment--><p dir=\\"auto\\">When I try to paste something (e.g. email content) to a note, the styling is kept, which is good, but the newlines are removed.</p><p data-spacing=\\"single\\">    Also when I share the selection to Notesnook via the share functionality from Android, I have the same issue.</p>    <hr>    <p dir=\\"auto\\"><strong>Device information:</strong></p><p data-spacing=\\"single\\">    App version: 2.3.0</p><p data-spacing=\\"single\\">    Platform: android</p><p data-spacing=\\"single\\">    Model: OnePlus-CPH2409-31</p><p data-spacing=\\"single\\">    Pro: true</p><p data-spacing=\\"single\\">    Logged in: yes</p><!--EndFragment-->"`;
-
-exports[`convert br tags to paragraphs (<html><body>
-    <!--StartFragment--><span class="css-901oao css-16my406 r-poiln3 r-bcqeeo r-qvutc0">Why switch from Gmail? 
-
-    Not sacrificing features for more privacy, prefer using one app, in many public groups and channels (Telegram)
-    
-    LibreOffice Slow &amp; buggy
-    
-    Switched to Brave for the better Android app, more private out of the box &amp; unsure if uBlock Origin closes gap</span><!--EndFragment-->
-    </body>
-    </html>) 1`] = `"<!--StartFragment--><span class=\\"css-901oao css-16my406 r-poiln3 r-bcqeeo r-qvutc0\\"><p>Why switch from Gmail? </p><p data-spacing=\\"single\\"></p><p data-spacing=\\"single\\">    Not sacrificing features for more privacy, prefer using one app, in many public groups and channels (Telegram)</p><p data-spacing=\\"single\\">    </p><p data-spacing=\\"single\\">    LibreOffice Slow &amp; buggy</p><p data-spacing=\\"single\\">    </p><p data-spacing=\\"single\\">    Switched to Brave for the better Android app, more private out of the box &amp; unsure if uBlock Origin closes gap</p></span><!--EndFragment-->"`;
-
-exports[`convert br tags to paragraphs (<p><br/></p>) 1`] = `"<p><br></p>"`;
-
-exports[`convert br tags to paragraphs (<p>line <em>1<br>line</em> 2</p>) 1`] = `"<p>line <em>1</em></p><p data-spacing=\\"single\\"><em>line</em> 2</p>"`;
-
-exports[`convert br tags to paragraphs (<p>line <span><em>1<br data-some="hello">line</em></span> 2</p>) 1`] = `"<p>line <span><em>1</em></span></p><p data-spacing=\\"single\\"><span><em>line</em></span> 2</p>"`;
-
-exports[`convert br tags to paragraphs (<p>line <span><em>1<br>line</em></span> 2</p>) 1`] = `"<p>line <span><em>1</em></span></p><p data-spacing=\\"single\\"><span><em>line</em></span> 2</p>"`;
-
-exports[`convert br tags to paragraphs (<p>line 1<br>line 2</p>) 1`] = `"<p>line 1</p><p data-spacing=\\"single\\">line 2</p>"`;

packages/editor/src/utils/tests/html.test.ts

@@ -1,75 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import { test } from "vitest";
-import { convertBrToParagraph } from "../html";
-
-const cases = [
-  [`<p>line 1<br>line 2</p>`],
-  [`<p>line <em>1<br>line</em> 2</p>`],
-  [`<p>line <span><em>1<br>line</em></span> 2</p>`],
-  [`<p>line <span><em>1<br data-some="hello">line</em></span> 2</p>`],
-  [`<p><br/></p>`],
-  [
-    `
-  <html><body>
-<!--StartFragment-->A troll, they call me, but I have no wish<br>
-to be associated with those dolls<br>
-<br>
-We lack religion, purpose, politics,<br>
-and yet, we somehow manage to get by.<br>
-</body>
-</html>`
-  ],
-  [
-    `<html><body>
-    <!--StartFragment--><p dir="auto">When I try to paste something (e.g. email content) to a note, the styling is kept, which is good, but the newlines are removed.<br>
-    Also when I share the selection to Notesnook via the share functionality from Android, I have the same issue.</p>
-    <hr>
-    <p dir="auto"><strong>Device information:</strong><br>
-    App version: 2.3.0<br>
-    Platform: android<br>
-    Model: OnePlus-CPH2409-31<br>
-    Pro: true<br>
-    Logged in: yes</p><!--EndFragment-->
-    </body>
-    </html>`
-  ],
-  [
-    `<html><body>
-    <!--StartFragment--><span class="css-901oao css-16my406 r-poiln3 r-bcqeeo r-qvutc0">Why switch from Gmail? 
-
-    Not sacrificing features for more privacy, prefer using one app, in many public groups and channels (Telegram)
-    
-    LibreOffice Slow &amp; buggy
-    
-    Switched to Brave for the better Android app, more private out of the box &amp; unsure if uBlock Origin closes gap</span><!--EndFragment-->
-    </body>
-    </html>`
-  ]
-];
-
-for (const testCase of cases) {
-  const [html, expected] = testCase;
-  test(`convert br tags to paragraphs (${testCase})`, (t) => {
-    t.expect(
-      convertBrToParagraph(html).body.innerHTML.trim()
-    ).toMatchSnapshot();
-  });
-}