desktop: use better-sqlite3 on desktop for SQLite

apps/desktop/src/preload.ts

@@ -23,11 +23,13 @@ import { type RendererGlobalElectronTRPC } from "electron-trpc/src/types";
 import type { NNCrypto } from "@notesnook/crypto";
 import { ipcRenderer } from "electron";
 import { platform } from "os";
+import sqlite3, { Database } from "better-sqlite3-multiple-ciphers";
 
 declare global {
   var os: () => "mas" | ReturnType<typeof platform>;
   var electronTRPC: RendererGlobalElectronTRPC;
   var NativeNNCrypto: (new () => NNCrypto) | undefined;
+  var createSQLite3Database: (filename: string) => Database;
 }
 
 process.once("loaded", async () => {
@@ -41,4 +43,5 @@ process.once("loaded", async () => {
 });
 
 globalThis.NativeNNCrypto = require("@notesnook/crypto").NNCrypto;
+globalThis.createSQLite3Database = (filename) => sqlite3(filename);
 globalThis.os = () => (MAC_APP_STORE ? "mas" : platform());

apps/web/package-lock.json

@@ -59,6 +59,7 @@
         "immer": "^10.0.3",
         "katex": "0.16.2",
         "kysely": "^0.26.3",
+        "libsodium-wrappers": "^0.7.13",
         "mac-scrollbar": "^0.13.5",
         "marked": "^4.1.0",
         "pdfjs-dist": "3.6.172",
@@ -70,6 +71,7 @@
         "react-day-picker": "^8.9.1",
         "react-dom": "18.2.0",
         "react-dropzone": "^14.2.3",
+        "react-error-boundary": "^4.0.12",
         "react-hot-toast": "^2.4.1",
         "react-loading-skeleton": "^3.3.1",
         "react-modal": "3.16.1",
@@ -101,6 +103,7 @@
         "@types/wicg-file-system-access": "^2020.9.6",
         "@vitejs/plugin-react-swc": "3.3.2",
         "autoprefixer": "^10.4.14",
+        "better-sqlite3-multiple-ciphers": "^9.4.0",
         "buffer": "^6.0.3",
         "chalk": "^4.1.0",
         "cross-env": "^7.0.3",
@@ -31951,9 +31954,11 @@
         "@notesnook/crypto": "file:../../packages/crypto",
         "@trpc/client": "10.38.3",
         "@trpc/server": "10.38.3",
+        "better-sqlite3-multiple-ciphers": "^9.4.0",
         "electron-trpc": "0.5.2",
         "electron-updater": "6.1.4",
         "icojs": "^0.17.1",
+        "sodium-native": "^4.0.6",
         "typed-emitter": "^2.1.0",
         "yargs": "^17.6.2",
         "zod": "^3.21.4"
@@ -31962,11 +31967,11 @@
         "@types/node": "18.16.1",
         "@types/yargs": "^17.0.24",
         "chokidar": "^3.5.3",
-        "electron": "25.9.8",
+        "electron": "^28.2.1",
         "electron-builder": "^24.9.1",
-        "esbuild": "^0.17.19",
+        "esbuild": "^0.20.0",
         "tree-kill": "^1.2.2",
-        "undici": "^5.23.0"
+        "undici": "^6.6.1"
       },
       "optionalDependencies": {
         "dmg-license": "^1.0.11"
@@ -38713,6 +38718,17 @@
       ],
       "license": "MIT"
     },
+    "node_modules/better-sqlite3-multiple-ciphers": {
+      "version": "9.4.1",
+      "resolved": "https://registry.npmjs.org/better-sqlite3-multiple-ciphers/-/better-sqlite3-multiple-ciphers-9.4.1.tgz",
+      "integrity": "sha512-9WIeXiGodJ0bJLLMdxicmGpJHe0ahpiaNC3VLv3QQj8/h4RLOcs4yskecSkSF3Pj/u8f7juYADpdMBvx71HlLQ==",
+      "dev": true,
+      "hasInstallScript": true,
+      "dependencies": {
+        "bindings": "^1.5.0",
+        "prebuild-install": "^7.1.1"
+      }
+    },
     "node_modules/big.js": {
       "version": "5.2.2",
       "dev": true,
@@ -38721,10 +38737,19 @@
         "node": "*"
       }
     },
+    "node_modules/bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+      "dev": true,
+      "dependencies": {
+        "file-uri-to-path": "1.0.0"
+      }
+    },
     "node_modules/bl": {
       "version": "4.1.0",
+      "devOptional": true,
       "license": "MIT",
-      "optional": true,
       "dependencies": {
         "buffer": "^5.5.0",
         "inherits": "^2.0.4",
@@ -38733,6 +38758,7 @@
     },
     "node_modules/bl/node_modules/buffer": {
       "version": "5.7.1",
+      "devOptional": true,
       "funding": [
         {
           "type": "github",
@@ -38748,7 +38774,6 @@
         }
       ],
       "license": "MIT",
-      "optional": true,
       "dependencies": {
         "base64-js": "^1.3.1",
         "ieee754": "^1.1.13"
@@ -39343,8 +39368,8 @@
     },
     "node_modules/deep-extend": {
       "version": "0.6.0",
+      "devOptional": true,
       "license": "MIT",
-      "optional": true,
       "engines": {
         "node": ">=4.0.0"
       }
@@ -39414,8 +39439,8 @@
     },
     "node_modules/detect-libc": {
       "version": "2.0.2",
+      "devOptional": true,
       "license": "Apache-2.0",
-      "optional": true,
       "engines": {
         "node": ">=8"
       }
@@ -40033,8 +40058,8 @@
     },
     "node_modules/expand-template": {
       "version": "2.0.3",
+      "devOptional": true,
       "license": "(MIT OR WTFPL)",
-      "optional": true,
       "engines": {
         "node": ">=6"
       }
@@ -40150,6 +40175,12 @@
         "node": ">= 12"
       }
     },
+    "node_modules/file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
+      "dev": true
+    },
     "node_modules/filelist": {
       "version": "1.0.4",
       "dev": true,
@@ -40279,8 +40310,8 @@
     },
     "node_modules/fs-constants": {
       "version": "1.0.0",
-      "license": "MIT",
+      "devOptional": true,
-      "optional": true
+      "license": "MIT"
     },
     "node_modules/fs-minipass": {
       "version": "2.1.0",
@@ -40439,8 +40470,8 @@
     },
     "node_modules/github-from-package": {
       "version": "0.0.0",
-      "license": "MIT",
+      "devOptional": true,
-      "optional": true
+      "license": "MIT"
     },
     "node_modules/glob": {
       "version": "7.2.3",
@@ -40899,8 +40930,8 @@
     },
     "node_modules/ini": {
       "version": "1.3.8",
-      "license": "ISC",
+      "devOptional": true,
-      "optional": true
+      "license": "ISC"
     },
     "node_modules/internal-slot": {
       "version": "1.0.6",
@@ -41482,6 +41513,19 @@
         "node": ">=6"
       }
     },
+    "node_modules/libsodium": {
+      "version": "0.7.13",
+      "resolved": "https://registry.npmjs.org/libsodium/-/libsodium-0.7.13.tgz",
+      "integrity": "sha512-mK8ju0fnrKXXfleL53vtp9xiPq5hKM0zbDQtcxQIsSmxNgSxqCj6R7Hl9PkrNe2j29T4yoDaF7DJLK9/i5iWUw=="
+    },
+    "node_modules/libsodium-wrappers": {
+      "version": "0.7.13",
+      "resolved": "https://registry.npmjs.org/libsodium-wrappers/-/libsodium-wrappers-0.7.13.tgz",
+      "integrity": "sha512-kasvDsEi/r1fMzKouIDv7B8I6vNmknXwGiYodErGuESoFTohGSKZplFtVxZqHaoQ217AynyIFgnOVRitpHs0Qw==",
+      "dependencies": {
+        "libsodium": "^0.7.13"
+      }
+    },
     "node_modules/lines-and-columns": {
       "version": "1.2.4",
       "license": "MIT"
@@ -42540,8 +42584,8 @@
     },
     "node_modules/minimist": {
       "version": "1.2.8",
+      "devOptional": true,
       "license": "MIT",
-      "optional": true,
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -42595,8 +42639,8 @@
     },
     "node_modules/mkdirp-classic": {
       "version": "0.5.3",
-      "license": "MIT",
+      "devOptional": true,
-      "optional": true
+      "license": "MIT"
     },
     "node_modules/mlly": {
       "version": "1.4.2",
@@ -42644,8 +42688,8 @@
     },
     "node_modules/napi-build-utils": {
       "version": "1.0.2",
-      "license": "MIT",
+      "devOptional": true,
-      "optional": true
+      "license": "MIT"
     },
     "node_modules/neo-async": {
       "version": "2.6.2",
@@ -42664,8 +42708,8 @@
     },
     "node_modules/node-abi": {
       "version": "3.54.0",
+      "devOptional": true,
       "license": "MIT",
-      "optional": true,
       "dependencies": {
         "semver": "^7.3.5"
       },
@@ -42675,8 +42719,8 @@
     },
     "node_modules/node-abi/node_modules/lru-cache": {
       "version": "6.0.0",
+      "devOptional": true,
       "license": "ISC",
-      "optional": true,
       "dependencies": {
         "yallist": "^4.0.0"
       },
@@ -42686,8 +42730,8 @@
     },
     "node_modules/node-abi/node_modules/semver": {
       "version": "7.5.4",
+      "devOptional": true,
       "license": "ISC",
-      "optional": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
       },
@@ -42700,8 +42744,8 @@
     },
     "node_modules/node-abi/node_modules/yallist": {
       "version": "4.0.0",
-      "license": "ISC",
+      "devOptional": true,
-      "optional": true
+      "license": "ISC"
     },
     "node_modules/node-addon-api": {
       "version": "4.3.0",
@@ -43085,8 +43129,8 @@
     },
     "node_modules/prebuild-install": {
       "version": "7.1.1",
+      "devOptional": true,
       "license": "MIT",
-      "optional": true,
       "dependencies": {
         "detect-libc": "^2.0.0",
         "expand-template": "^2.0.3",
@@ -43110,6 +43154,7 @@
     },
     "node_modules/prebuild-install/node_modules/simple-get": {
       "version": "4.0.1",
+      "devOptional": true,
       "funding": [
         {
           "type": "github",
@@ -43125,7 +43170,6 @@
         }
       ],
       "license": "MIT",
-      "optional": true,
       "dependencies": {
         "decompress-response": "^6.0.0",
         "once": "^1.3.1",
@@ -43265,8 +43309,8 @@
     },
     "node_modules/rc": {
       "version": "1.2.8",
+      "devOptional": true,
       "license": "(BSD-2-Clause OR MIT OR Apache-2.0)",
-      "optional": true,
       "dependencies": {
         "deep-extend": "^0.6.0",
         "ini": "~1.3.0",
@@ -43333,6 +43377,17 @@
         "react": ">= 16.8 || 18.0.0"
       }
     },
+    "node_modules/react-error-boundary": {
+      "version": "4.0.13",
+      "resolved": "https://registry.npmjs.org/react-error-boundary/-/react-error-boundary-4.0.13.tgz",
+      "integrity": "sha512-b6PwbdSv8XeOSYvjt8LpgpKrZ0yGdtZokYwkwV2wlcZbxgopHX/hgPl5VgpnoVOWd868n1hktM8Qm4b+02MiLQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.12.5"
+      },
+      "peerDependencies": {
+        "react": ">=16.13.1"
+      }
+    },
     "node_modules/react-hot-toast": {
       "version": "2.4.1",
       "license": "MIT",
@@ -43415,8 +43470,8 @@
     },
     "node_modules/readable-stream": {
       "version": "3.6.2",
+      "devOptional": true,
       "license": "MIT",
-      "optional": true,
       "dependencies": {
         "inherits": "^2.0.3",
         "string_decoder": "^1.1.1",
@@ -43985,6 +44040,7 @@
     },
     "node_modules/simple-concat": {
       "version": "1.0.1",
+      "devOptional": true,
       "funding": [
         {
           "type": "github",
@@ -43999,8 +44055,7 @@
           "url": "https://feross.org/support"
         }
       ],
-      "license": "MIT",
+      "license": "MIT"
-      "optional": true
     },
     "node_modules/simple-get": {
       "version": "3.1.1",
@@ -44108,8 +44163,8 @@
     },
     "node_modules/string_decoder": {
       "version": "1.3.0",
+      "devOptional": true,
       "license": "MIT",
-      "optional": true,
       "dependencies": {
         "safe-buffer": "~5.2.0"
       }
@@ -44241,8 +44296,8 @@
     },
     "node_modules/strip-json-comments": {
       "version": "2.0.1",
+      "devOptional": true,
       "license": "MIT",
-      "optional": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -44339,8 +44394,8 @@
     },
     "node_modules/tar-fs": {
       "version": "2.1.1",
+      "devOptional": true,
       "license": "MIT",
-      "optional": true,
       "dependencies": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",
@@ -44350,13 +44405,13 @@
     },
     "node_modules/tar-fs/node_modules/chownr": {
       "version": "1.1.4",
-      "license": "ISC",
+      "devOptional": true,
-      "optional": true
+      "license": "ISC"
     },
     "node_modules/tar-stream": {
       "version": "2.2.0",
+      "devOptional": true,
       "license": "MIT",
-      "optional": true,
       "dependencies": {
         "bl": "^4.0.3",
         "end-of-stream": "^1.4.1",
@@ -44574,8 +44629,8 @@
     },
     "node_modules/tunnel-agent": {
       "version": "0.6.0",
+      "devOptional": true,
       "license": "Apache-2.0",
-      "optional": true,
       "dependencies": {
         "safe-buffer": "^5.0.1"
       },
@@ -44918,8 +44973,8 @@
     },
     "node_modules/util-deprecate": {
       "version": "1.0.2",
-      "license": "MIT",
+      "devOptional": true,
-      "optional": true
+      "license": "MIT"
     },
     "node_modules/uuid": {
       "version": "8.3.2",

apps/web/package.json

@@ -57,6 +57,7 @@
     "immer": "^10.0.3",
     "katex": "0.16.2",
     "kysely": "^0.26.3",
+    "libsodium-wrappers": "^0.7.13",
     "mac-scrollbar": "^0.13.5",
     "marked": "^4.1.0",
     "pdfjs-dist": "3.6.172",
@@ -68,6 +69,7 @@
     "react-day-picker": "^8.9.1",
     "react-dom": "18.2.0",
     "react-dropzone": "^14.2.3",
+    "react-error-boundary": "^4.0.12",
     "react-hot-toast": "^2.4.1",
     "react-loading-skeleton": "^3.3.1",
     "react-modal": "3.16.1",
@@ -99,6 +101,7 @@
     "@types/wicg-file-system-access": "^2020.9.6",
     "@vitejs/plugin-react-swc": "3.3.2",
     "autoprefixer": "^10.4.14",
+    "better-sqlite3-multiple-ciphers": "^9.4.0",
     "buffer": "^6.0.3",
     "chalk": "^4.1.0",
     "cross-env": "^7.0.3",

apps/web/src/common/db.ts

@@ -21,17 +21,12 @@ import { EventSourcePolyfill as EventSource } from "event-source-polyfill";
 import { DatabasePersistence, NNStorage } from "../interfaces/storage";
 import { logger } from "../utils/logger";
 import { showMigrationDialog } from "./dialog-controller";
-// import { SQLocalKysely } from "sqlocal/kysely";
-import { WaSqliteWorkerDriver } from "./sqlite/sqlite.kysely";
-import { SqliteAdapter, SqliteQueryCompiler, SqliteIntrospector } from "kysely";
 import { database } from "@notesnook/common";
-// import SQLiteESMFactory from "./sqlite/wa-sqlite-async";
+import { createDialect } from "./sqlite";
-// import * as SQLite from "./sqlite/sqlite-api";
+import { isFeatureSupported } from "../utils/feature-check";
-// import { IDBBatchAtomicVFS } from "./sqlite/IDBBatchAtomicVFS";
 
 const db = database;
 async function initializeDatabase(persistence: DatabasePersistence) {
-  console.log("initi");
   logger.measure("Database initialization");
 
   const { FileStorage } = await import("../interfaces/fs");
@@ -49,21 +44,21 @@ async function initializeDatabase(persistence: DatabasePersistence) {
   });
 
   const storage = new NNStorage("Notesnook", KeyChain, persistence);
+  await storage.migrate();
+
   database.setup({
     sqliteOptions: {
-      dialect: (name) => ({
+      dialect: createDialect,
-        createDriver: () =>
+      ...(isFeatureSupported("opfs")
-          new WaSqliteWorkerDriver({ async: true, dbName: name }),
+        ? { journalMode: "WAL" }
-        createAdapter: () => new SqliteAdapter(),
+        : {
-        createIntrospector: (db) => new SqliteIntrospector(db),
+            journalMode: "MEMORY",
-        createQueryCompiler: () => new SqliteQueryCompiler()
+            lockingMode: "exclusive"
-      }),
+          }),
-      journalMode: "MEMORY",
       tempStore: "memory",
       synchronous: "normal",
       pageSize: 8192,
       cacheSize: -16000,
-      lockingMode: "exclusive",
       password: databaseKey
     },
     storage: storage,

apps/web/src/common/sqlite/AccessHandlePoolVFS.js

@@ -61,6 +61,9 @@ export class AccessHandlePoolVFS extends VFS.Base {
   // The OPFS files all have randomly-generated names that do not match
   // the SQLite files whose data they contain. This map links those names
   // with their respective OPFS access handles.
+  /**
+   * @type {Map<FileSystemSyncAccessHandle, string>}
+   */
   #mapAccessHandleToName = new Map();
 
   // When a SQLite file is associated with an OPFS file, that association
@@ -209,6 +212,16 @@ export class AccessHandlePoolVFS extends VFS.Base {
     await this.#releaseAccessHandles();
   }
 
+  async delete() {
+    console.log("CLSOGING");
+    await this.close();
+    console.log("CLSOGING", this.#directoryHandle);
+    for await (const [name] of this.#directoryHandle) {
+      console.log("DELETING", name);
+      await this.#directoryHandle.removeEntry(name, { recursive: true });
+    }
+  }
+
   /**
    * Release and reacquire all OPFS access handles. This must be called
    * and awaited before any SQLite call that uses the VFS and also before

apps/web/src/common/sqlite/IDBBatchAtomicVFS.js

@@ -1,22 +1,4 @@
-/*
+/* eslint-disable header/header */
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
 // Copyright 2022 Roy T. Hashimoto. All Rights Reserved.
 import * as VFS from "./VFS.js";
 import { WebLocksExclusive as WebLocks } from "./WebLocks.js";
@@ -77,6 +59,12 @@ export class IDBBatchAtomicVFS extends VFS.Base {
   #taskTimestamp = performance.now();
   #pendingAsync = new Set();
 
+  // Asyncify can grow WebAssembly memory during an asynchronous call.
+  // If this happens, then any array buffer arguments will be detached.
+  // The workaround is when finding a detached buffer, set this handler
+  // function to process the new buffer outside handlerAsync().
+  #growthHandler = null;
+
   constructor(idbDatabaseName = "wa-sqlite", options = DEFAULT_OPTIONS) {
     super();
     this.name = idbDatabaseName;
@@ -86,6 +74,11 @@ export class IDBBatchAtomicVFS extends VFS.Base {
     });
   }
 
+  async delete() {
+    await this.close();
+    await deleteDatabase(this.name);
+  }
+
   async close() {
     for (const fileId of this.#mapIdToFile.keys()) {
       await this.xClose(fileId);
@@ -103,7 +96,7 @@ export class IDBBatchAtomicVFS extends VFS.Base {
    * @returns {number}
    */
   xOpen(name, fileId, flags, pOutFlags) {
-    return this.handleAsync(async () => {
+    const result = this.handleAsync(async () => {
       if (name === null) name = `null_${fileId}`;
       log(`xOpen ${name} 0x${fileId.toString(16)} 0x${flags.toString(16)}`);
 
@@ -137,6 +130,14 @@ export class IDBBatchAtomicVFS extends VFS.Base {
             }
           }
         });
+
+        // @ts-ignore
+        if (pOutFlags.buffer.detached || !pOutFlags.buffer.byteLength) {
+          pOutFlags = new DataView(new ArrayBuffer(4));
+          this.#growthHandler = (pOutFlagsNew) => {
+            pOutFlagsNew.setInt32(0, pOutFlags.getInt32(0, true), true);
+          };
+        }
         pOutFlags.setInt32(0, flags & VFS.SQLITE_OPEN_READONLY, true);
         return VFS.SQLITE_OK;
       } catch (e) {
@@ -144,6 +145,10 @@ export class IDBBatchAtomicVFS extends VFS.Base {
         return VFS.SQLITE_CANTOPEN;
       }
     });
+
+    this.#growthHandler?.(pOutFlags);
+    this.#growthHandler = null;
+    return result;
   }
 
   /**
@@ -179,7 +184,8 @@ export class IDBBatchAtomicVFS extends VFS.Base {
    * @returns {number}
    */
   xRead(fileId, pData, iOffset) {
-    return this.handleAsync(async () => {
+    const byteLength = pData.byteLength;
+    const result = this.handleAsync(async () => {
       const file = this.#mapIdToFile.get(fileId);
       log(`xRead ${file.path} ${pData.byteLength} ${iOffset}`);
 
@@ -189,6 +195,15 @@ export class IDBBatchAtomicVFS extends VFS.Base {
         // one case - rollback after journal spill - where reads cross
         // write boundaries so we have to allow for that.
         const result = await this.#idb.run("readonly", async ({ blocks }) => {
+          // @ts-ignore
+          if (pData.buffer.detached || !pData.buffer.byteLength) {
+            // WebAssembly memory has grown, invalidating our buffer. Use
+            // a temporary buffer and copy after this asynchronous call
+            // completes.
+            pData = new Uint8Array(byteLength);
+            this.#growthHandler = (pDataNew) => pDataNew.set(pData);
+          }
+
           let pDataOffset = 0;
           while (pDataOffset < pData.byteLength) {
             // Fetch the IndexedDB block for this file location.
@@ -223,6 +238,10 @@ export class IDBBatchAtomicVFS extends VFS.Base {
         return VFS.SQLITE_IOERR;
       }
     });
+
+    this.#growthHandler?.(pData);
+    this.#growthHandler = null;
+    return result;
   }
 
   /**
@@ -244,7 +263,7 @@ export class IDBBatchAtomicVFS extends VFS.Base {
         }
         await new Promise((resolve) => setTimeout(resolve));
 
-        const result = this.#xWriteHelper(fileId, pData, iOffset);
+        const result = this.#xWriteHelper(fileId, pData.slice(), iOffset);
         this.#taskTimestamp = performance.now();
         return result;
       });
@@ -452,6 +471,7 @@ export class IDBBatchAtomicVFS extends VFS.Base {
     return this.handleAsync(async () => {
       const file = this.#mapIdToFile.get(fileId);
       log(`xUnlock ${file.path} ${flags}`);
+
       try {
         return file.locks.unlock(flags);
       } catch (e) {
@@ -467,14 +487,26 @@ export class IDBBatchAtomicVFS extends VFS.Base {
    * @returns {number}
    */
   xCheckReservedLock(fileId, pResOut) {
-    return this.handleAsync(async () => {
+    const result = this.handleAsync(async () => {
       const file = this.#mapIdToFile.get(fileId);
       log(`xCheckReservedLock ${file.path}`);
 
       const isReserved = await file.locks.isSomewhereReserved();
+
+      // @ts-ignore
+      if (pResOut.buffer.detached || !pResOut.buffer.byteLength) {
+        pResOut = new DataView(new ArrayBuffer(4));
+        this.#growthHandler = (pResOutNew) => {
+          pResOutNew.setInt32(0, pResOut.getInt32(0, true), true);
+        };
+      }
       pResOut.setInt32(0, isReserved ? 1 : 0, true);
       return VFS.SQLITE_OK;
     });
+
+    this.#growthHandler?.(pResOut);
+    this.#growthHandler = null;
+    return result;
   }
 
   /**
@@ -649,7 +681,7 @@ export class IDBBatchAtomicVFS extends VFS.Base {
    * @returns {number}
    */
   xAccess(name, flags, pResOut) {
-    return this.handleAsync(async () => {
+    const result = this.handleAsync(async () => {
       try {
         if (name.includes("-journal") || name.includes("-wal")) {
           pResOut.setInt32(0, 0, true);
@@ -663,6 +695,14 @@ export class IDBBatchAtomicVFS extends VFS.Base {
         const key = await this.#idb.run("readonly", ({ blocks }) => {
           return blocks.getKey(this.#bound({ path }, 0));
         });
+
+        // @ts-ignore
+        if (pResOut.buffer.detached || !pResOut.buffer.byteLength) {
+          pResOut = new DataView(new ArrayBuffer(4));
+          this.#growthHandler = (pResOutNew) => {
+            pResOutNew.setInt32(0, pResOut.getInt32(0, true), true);
+          };
+        }
         pResOut.setInt32(0, key ? 1 : 0, true);
         return VFS.SQLITE_OK;
       } catch (e) {
@@ -670,6 +710,10 @@ export class IDBBatchAtomicVFS extends VFS.Base {
         return VFS.SQLITE_IOERR;
       }
     });
+
+    this.#growthHandler?.(pResOut);
+    this.#growthHandler = null;
+    return result;
   }
 
   /**
@@ -867,6 +911,18 @@ export class IDBBatchAtomicVFS extends VFS.Base {
   }
 }
 
+function deleteDatabase(idbDatabaseName) {
+  return new Promise((resolve, reject) => {
+    const request = globalThis.indexedDB.deleteDatabase(idbDatabaseName);
+    request.addEventListener("success", () => {
+      resolve();
+    });
+    request.addEventListener("error", () => {
+      reject(request.error);
+    });
+  });
+}
+
 function openDatabase(idbDatabaseName) {
   return new Promise((resolve, reject) => {
     const request = globalThis.indexedDB.open(idbDatabaseName, 5);

apps/web/src/common/sqlite/index.desktop.ts

@@ -0,0 +1,58 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+  SqliteDriver as KSqliteDriver,
+  SqliteDialectConfig,
+  Dialect,
+  SqliteAdapter,
+  SqliteIntrospector,
+  SqliteQueryCompiler
+} from "kysely";
+import { desktop } from "../desktop-bridge";
+
+class SqliteDriver extends KSqliteDriver {
+  constructor(private readonly config: SqliteDialectConfig & { name: string }) {
+    super(config);
+  }
+  async delete() {
+    const path = await desktop!.integration.resolvePath.query({
+      filePath: `userData/${this.config.name}.sql`
+    });
+    await desktop?.integration.deleteFile.query(path);
+  }
+}
+
+export const createDialect = (name: string): Dialect => {
+  return {
+    createDriver: () =>
+      new SqliteDriver({
+        name,
+        database: async () => {
+          const path = await desktop!.integration.resolvePath.query({
+            filePath: `userData/${name}.sql`
+          });
+          return window.createSQLite3Database(path).unsafeMode(true);
+        }
+      }),
+    createAdapter: () => new SqliteAdapter(),
+    createIntrospector: (db) => new SqliteIntrospector(db),
+    createQueryCompiler: () => new SqliteQueryCompiler()
+  };
+};

apps/web/src/common/sqlite/index.ts

@@ -0,0 +1,46 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+  SqliteAdapter,
+  SqliteQueryCompiler,
+  SqliteIntrospector,
+  Dialect
+} from "kysely";
+import { WaSqliteWorkerDriver } from "./wa-sqlite-kysely-driver";
+import { isFeatureSupported } from "../../utils/feature-check";
+
+declare module "kysely" {
+  interface Driver {
+    delete(): Promise<void>;
+  }
+}
+
+export const createDialect = (name: string): Dialect => {
+  return {
+    createDriver: () =>
+      new WaSqliteWorkerDriver({
+        async: isFeatureSupported("opfs") ? false : true,
+        dbName: name
+      }),
+    createAdapter: () => new SqliteAdapter(),
+    createIntrospector: (db) => new SqliteIntrospector(db),
+    createQueryCompiler: () => new SqliteQueryCompiler()
+  };
+};

apps/web/src/common/sqlite/sqlite-types.ts

@@ -34,7 +34,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
  * each element converted to a byte); SQLite always returns blob data as
  * `Uint8Array`
  */
-type SQLiteCompatibleType =
+export type SQLiteCompatibleType =
   | number
   | string
   | Uint8Array
@@ -58,7 +58,7 @@ type SQLiteCompatibleType =
  * @see https://sqlite.org/vfs.html
  * @see https://sqlite.org/c3ref/io_methods.html
  */
-declare interface SQLiteVFS {
+export interface SQLiteVFS {
   /** Maximum length of a file path in UTF-8 bytes (default 64) */
   mxPathName?: number;
 
@@ -115,7 +115,7 @@ declare interface SQLiteVFS {
  * {@link SQLiteModule.xBestIndex}
  * @see https://sqlite.org/c3ref/index_info.html
  */
-declare interface SQLiteModuleIndexInfo {
+export interface SQLiteModuleIndexInfo {
   nConstraint: number;
   aConstraint: Array<{
     iColumn: number;
@@ -152,13 +152,13 @@ declare interface SQLiteModuleIndexInfo {
  *
  * @see https://sqlite.org/vtab.html
  */
-declare interface SQLiteModule {
+export interface SQLiteModule {
   /**
    * @see https://sqlite.org/vtab.html#the_xcreate_method
    */
   xCreate?(
     db: number,
-    appData,
+    appData: any,
     argv: string[],
     pVTab: number,
     pzErr: DataView
@@ -169,7 +169,7 @@ declare interface SQLiteModule {
    */
   xConnect(
     db: number,
-    appData,
+    appData: any,
     argv: string[],
     pVTab: number,
     pzErr: DataView
@@ -304,7 +304,7 @@ declare interface SQLiteModule {
  *
  * @see https://sqlite.org/c3ref/funclist.html
  */
-declare interface SQLiteAPI {
+export interface SQLiteAPI {
   /**
    * Bind a collection of values to a statement
    *
@@ -466,7 +466,7 @@ declare interface SQLiteAPI {
    * @param db database pointer
    * @returns number of rows modified
    */
-  changes(db): number;
+  changes(db: number): number;
 
   /**
    * Close database connection
@@ -474,7 +474,7 @@ declare interface SQLiteAPI {
    * @param db database pointer
    * @returns `SQLITE_OK` (throws exception on error)
    */
-  close(db): Promise<number>;
+  close(db: number): Promise<number>;
 
   /**
    * Call the appropriate `column_*` function based on the column type
@@ -625,7 +625,7 @@ declare interface SQLiteAPI {
     db: number,
     zName: string,
     module: SQLiteModule,
-    appData?
+    appData?: any
   ): number;
 
   /**
@@ -783,8 +783,8 @@ declare interface SQLiteAPI {
     db: number,
     nProgressOps: number,
     handler: (userData: any) => number,
-    userData
+    userData: any
-  );
+  ): any;
 
   /**
    * Reset a prepared statement object
@@ -1082,645 +1082,3 @@ declare interface SQLiteAPI {
    */
   vfs_register(vfs: SQLiteVFS, makeDefault?: boolean): number;
 }
-
-/** @ignore */
-declare module "wa-sqlite/src/sqlite-constants.js" {
-  export const SQLITE_OK: 0;
-  export const SQLITE_ERROR: 1;
-  export const SQLITE_INTERNAL: 2;
-  export const SQLITE_PERM: 3;
-  export const SQLITE_ABORT: 4;
-  export const SQLITE_BUSY: 5;
-  export const SQLITE_LOCKED: 6;
-  export const SQLITE_NOMEM: 7;
-  export const SQLITE_READONLY: 8;
-  export const SQLITE_INTERRUPT: 9;
-  export const SQLITE_IOERR: 10;
-  export const SQLITE_CORRUPT: 11;
-  export const SQLITE_NOTFOUND: 12;
-  export const SQLITE_FULL: 13;
-  export const SQLITE_CANTOPEN: 14;
-  export const SQLITE_PROTOCOL: 15;
-  export const SQLITE_EMPTY: 16;
-  export const SQLITE_SCHEMA: 17;
-  export const SQLITE_TOOBIG: 18;
-  export const SQLITE_CONSTRAINT: 19;
-  export const SQLITE_MISMATCH: 20;
-  export const SQLITE_MISUSE: 21;
-  export const SQLITE_NOLFS: 22;
-  export const SQLITE_AUTH: 23;
-  export const SQLITE_FORMAT: 24;
-  export const SQLITE_RANGE: 25;
-  export const SQLITE_NOTADB: 26;
-  export const SQLITE_NOTICE: 27;
-  export const SQLITE_WARNING: 28;
-  export const SQLITE_ROW: 100;
-  export const SQLITE_DONE: 101;
-  export const SQLITE_IOERR_ACCESS: 3338;
-  export const SQLITE_IOERR_CHECKRESERVEDLOCK: 3594;
-  export const SQLITE_IOERR_CLOSE: 4106;
-  export const SQLITE_IOERR_DATA: 8202;
-  export const SQLITE_IOERR_DELETE: 2570;
-  export const SQLITE_IOERR_DELETE_NOENT: 5898;
-  export const SQLITE_IOERR_DIR_FSYNC: 1290;
-  export const SQLITE_IOERR_FSTAT: 1802;
-  export const SQLITE_IOERR_FSYNC: 1034;
-  export const SQLITE_IOERR_GETTEMPPATH: 6410;
-  export const SQLITE_IOERR_LOCK: 3850;
-  export const SQLITE_IOERR_NOMEM: 3082;
-  export const SQLITE_IOERR_READ: 266;
-  export const SQLITE_IOERR_RDLOCK: 2314;
-  export const SQLITE_IOERR_SEEK: 5642;
-  export const SQLITE_IOERR_SHORT_READ: 522;
-  export const SQLITE_IOERR_TRUNCATE: 1546;
-  export const SQLITE_IOERR_UNLOCK: 2058;
-  export const SQLITE_IOERR_VNODE: 6922;
-  export const SQLITE_IOERR_WRITE: 778;
-  export const SQLITE_IOERR_BEGIN_ATOMIC: 7434;
-  export const SQLITE_IOERR_COMMIT_ATOMIC: 7690;
-  export const SQLITE_IOERR_ROLLBACK_ATOMIC: 7946;
-  export const SQLITE_CONSTRAINT_CHECK: 275;
-  export const SQLITE_CONSTRAINT_COMMITHOOK: 531;
-  export const SQLITE_CONSTRAINT_FOREIGNKEY: 787;
-  export const SQLITE_CONSTRAINT_FUNCTION: 1043;
-  export const SQLITE_CONSTRAINT_NOTNULL: 1299;
-  export const SQLITE_CONSTRAINT_PINNED: 2835;
-  export const SQLITE_CONSTRAINT_PRIMARYKEY: 1555;
-  export const SQLITE_CONSTRAINT_ROWID: 2579;
-  export const SQLITE_CONSTRAINT_TRIGGER: 1811;
-  export const SQLITE_CONSTRAINT_UNIQUE: 2067;
-  export const SQLITE_CONSTRAINT_VTAB: 2323;
-  export const SQLITE_OPEN_READONLY: 1;
-  export const SQLITE_OPEN_READWRITE: 2;
-  export const SQLITE_OPEN_CREATE: 4;
-  export const SQLITE_OPEN_DELETEONCLOSE: 8;
-  export const SQLITE_OPEN_EXCLUSIVE: 16;
-  export const SQLITE_OPEN_AUTOPROXY: 32;
-  export const SQLITE_OPEN_URI: 64;
-  export const SQLITE_OPEN_MEMORY: 128;
-  export const SQLITE_OPEN_MAIN_DB: 256;
-  export const SQLITE_OPEN_TEMP_DB: 512;
-  export const SQLITE_OPEN_TRANSIENT_DB: 1024;
-  export const SQLITE_OPEN_MAIN_JOURNAL: 2048;
-  export const SQLITE_OPEN_TEMP_JOURNAL: 4096;
-  export const SQLITE_OPEN_SUBJOURNAL: 8192;
-  export const SQLITE_OPEN_SUPER_JOURNAL: 16384;
-  export const SQLITE_OPEN_NOMUTEX: 32768;
-  export const SQLITE_OPEN_FULLMUTEX: 65536;
-  export const SQLITE_OPEN_SHAREDCACHE: 131072;
-  export const SQLITE_OPEN_PRIVATECACHE: 262144;
-  export const SQLITE_OPEN_WAL: 524288;
-  export const SQLITE_OPEN_NOFOLLOW: 16777216;
-  export const SQLITE_LOCK_NONE: 0;
-  export const SQLITE_LOCK_SHARED: 1;
-  export const SQLITE_LOCK_RESERVED: 2;
-  export const SQLITE_LOCK_PENDING: 3;
-  export const SQLITE_LOCK_EXCLUSIVE: 4;
-  export const SQLITE_IOCAP_ATOMIC: 1;
-  export const SQLITE_IOCAP_ATOMIC512: 2;
-  export const SQLITE_IOCAP_ATOMIC1K: 4;
-  export const SQLITE_IOCAP_ATOMIC2K: 8;
-  export const SQLITE_IOCAP_ATOMIC4K: 16;
-  export const SQLITE_IOCAP_ATOMIC8K: 32;
-  export const SQLITE_IOCAP_ATOMIC16K: 64;
-  export const SQLITE_IOCAP_ATOMIC32K: 128;
-  export const SQLITE_IOCAP_ATOMIC64K: 256;
-  export const SQLITE_IOCAP_SAFE_APPEND: 512;
-  export const SQLITE_IOCAP_SEQUENTIAL: 1024;
-  export const SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN: 2048;
-  export const SQLITE_IOCAP_POWERSAFE_OVERWRITE: 4096;
-  export const SQLITE_IOCAP_IMMUTABLE: 8192;
-  export const SQLITE_IOCAP_BATCH_ATOMIC: 16384;
-  export const SQLITE_ACCESS_EXISTS: 0;
-  export const SQLITE_ACCESS_READWRITE: 1;
-  export const SQLITE_ACCESS_READ: 2;
-  export const SQLITE_FCNTL_LOCKSTATE: 1;
-  export const SQLITE_FCNTL_GET_LOCKPROXYFILE: 2;
-  export const SQLITE_FCNTL_SET_LOCKPROXYFILE: 3;
-  export const SQLITE_FCNTL_LAST_ERRNO: 4;
-  export const SQLITE_FCNTL_SIZE_HINT: 5;
-  export const SQLITE_FCNTL_CHUNK_SIZE: 6;
-  export const SQLITE_FCNTL_FILE_POINTER: 7;
-  export const SQLITE_FCNTL_SYNC_OMITTED: 8;
-  export const SQLITE_FCNTL_WIN32_AV_RETRY: 9;
-  export const SQLITE_FCNTL_PERSIST_WAL: 10;
-  export const SQLITE_FCNTL_OVERWRITE: 11;
-  export const SQLITE_FCNTL_VFSNAME: 12;
-  export const SQLITE_FCNTL_POWERSAFE_OVERWRITE: 13;
-  export const SQLITE_FCNTL_PRAGMA: 14;
-  export const SQLITE_FCNTL_BUSYHANDLER: 15;
-  export const SQLITE_FCNTL_TEMPFILENAME: 16;
-  export const SQLITE_FCNTL_MMAP_SIZE: 18;
-  export const SQLITE_FCNTL_TRACE: 19;
-  export const SQLITE_FCNTL_HAS_MOVED: 20;
-  export const SQLITE_FCNTL_SYNC: 21;
-  export const SQLITE_FCNTL_COMMIT_PHASETWO: 22;
-  export const SQLITE_FCNTL_WIN32_SET_HANDLE: 23;
-  export const SQLITE_FCNTL_WAL_BLOCK: 24;
-  export const SQLITE_FCNTL_ZIPVFS: 25;
-  export const SQLITE_FCNTL_RBU: 26;
-  export const SQLITE_FCNTL_VFS_POINTER: 27;
-  export const SQLITE_FCNTL_JOURNAL_POINTER: 28;
-  export const SQLITE_FCNTL_WIN32_GET_HANDLE: 29;
-  export const SQLITE_FCNTL_PDB: 30;
-  export const SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: 31;
-  export const SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: 32;
-  export const SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: 33;
-  export const SQLITE_FCNTL_LOCK_TIMEOUT: 34;
-  export const SQLITE_FCNTL_DATA_VERSION: 35;
-  export const SQLITE_FCNTL_SIZE_LIMIT: 36;
-  export const SQLITE_FCNTL_CKPT_DONE: 37;
-  export const SQLITE_FCNTL_RESERVE_BYTES: 38;
-  export const SQLITE_FCNTL_CKPT_START: 39;
-  export const SQLITE_INTEGER: 1;
-  export const SQLITE_FLOAT: 2;
-  export const SQLITE_TEXT: 3;
-  export const SQLITE_BLOB: 4;
-  export const SQLITE_NULL: 5;
-  export const SQLITE_STATIC: 0;
-  export const SQLITE_TRANSIENT: -1;
-  export const SQLITE_UTF8: 1;
-  export const SQLITE_UTF16LE: 2;
-  export const SQLITE_UTF16BE: 3;
-  export const SQLITE_UTF16: 4;
-  export const SQLITE_INDEX_CONSTRAINT_EQ: 2;
-  export const SQLITE_INDEX_CONSTRAINT_GT: 4;
-  export const SQLITE_INDEX_CONSTRAINT_LE: 8;
-  export const SQLITE_INDEX_CONSTRAINT_LT: 16;
-  export const SQLITE_INDEX_CONSTRAINT_GE: 32;
-  export const SQLITE_INDEX_CONSTRAINT_MATCH: 64;
-  export const SQLITE_INDEX_CONSTRAINT_LIKE: 65;
-  export const SQLITE_INDEX_CONSTRAINT_GLOB: 66;
-  export const SQLITE_INDEX_CONSTRAINT_REGEXP: 67;
-  export const SQLITE_INDEX_CONSTRAINT_NE: 68;
-  export const SQLITE_INDEX_CONSTRAINT_ISNOT: 69;
-  export const SQLITE_INDEX_CONSTRAINT_ISNOTNULL: 70;
-  export const SQLITE_INDEX_CONSTRAINT_ISNULL: 71;
-  export const SQLITE_INDEX_CONSTRAINT_IS: 72;
-  export const SQLITE_INDEX_CONSTRAINT_FUNCTION: 150;
-  export const SQLITE_INDEX_SCAN_UNIQUE: 1;
-  export const SQLITE_DETERMINISTIC: 0x000000800;
-  export const SQLITE_DIRECTONLY: 0x000080000;
-  export const SQLITE_SUBTYPE: 0x000100000;
-  export const SQLITE_INNOCUOUS: 0x000200000;
-  export const SQLITE_SYNC_NORMAL: 0x00002;
-  export const SQLITE_SYNC_FULL: 0x00003;
-  export const SQLITE_SYNC_DATAONLY: 0x00010;
-  export const SQLITE_CREATE_INDEX: 1;
-  export const SQLITE_CREATE_TABLE: 2;
-  export const SQLITE_CREATE_TEMP_INDEX: 3;
-  export const SQLITE_CREATE_TEMP_TABLE: 4;
-  export const SQLITE_CREATE_TEMP_TRIGGER: 5;
-  export const SQLITE_CREATE_TEMP_VIEW: 6;
-  export const SQLITE_CREATE_TRIGGER: 7;
-  export const SQLITE_CREATE_VIEW: 8;
-  export const SQLITE_DELETE: 9;
-  export const SQLITE_DROP_INDEX: 10;
-  export const SQLITE_DROP_TABLE: 11;
-  export const SQLITE_DROP_TEMP_INDEX: 12;
-  export const SQLITE_DROP_TEMP_TABLE: 13;
-  export const SQLITE_DROP_TEMP_TRIGGER: 14;
-  export const SQLITE_DROP_TEMP_VIEW: 15;
-  export const SQLITE_DROP_TRIGGER: 16;
-  export const SQLITE_DROP_VIEW: 17;
-  export const SQLITE_INSERT: 18;
-  export const SQLITE_PRAGMA: 19;
-  export const SQLITE_READ: 20;
-  export const SQLITE_SELECT: 21;
-  export const SQLITE_TRANSACTION: 22;
-  export const SQLITE_UPDATE: 23;
-  export const SQLITE_ATTACH: 24;
-  export const SQLITE_DETACH: 25;
-  export const SQLITE_ALTER_TABLE: 26;
-  export const SQLITE_REINDEX: 27;
-  export const SQLITE_ANALYZE: 28;
-  export const SQLITE_CREATE_VTABLE: 29;
-  export const SQLITE_DROP_VTABLE: 30;
-  export const SQLITE_FUNCTION: 31;
-  export const SQLITE_SAVEPOINT: 32;
-  export const SQLITE_COPY: 0;
-  export const SQLITE_RECURSIVE: 33;
-  export const SQLITE_DENY: 1;
-  export const SQLITE_IGNORE: 2;
-  export const SQLITE_LIMIT_LENGTH: 0;
-  export const SQLITE_LIMIT_SQL_LENGTH: 1;
-  export const SQLITE_LIMIT_COLUMN: 2;
-  export const SQLITE_LIMIT_EXPR_DEPTH: 3;
-  export const SQLITE_LIMIT_COMPOUND_SELECT: 4;
-  export const SQLITE_LIMIT_VDBE_OP: 5;
-  export const SQLITE_LIMIT_FUNCTION_ARG: 6;
-  export const SQLITE_LIMIT_ATTACHED: 7;
-  export const SQLITE_LIMIT_LIKE_PATTERN_LENGTH: 8;
-  export const SQLITE_LIMIT_VARIABLE_NUMBER: 9;
-  export const SQLITE_LIMIT_TRIGGER_DEPTH: 10;
-  export const SQLITE_LIMIT_WORKER_THREADS: 11;
-}
-
-/** @ignore */
-declare module "wa-sqlite" {
-  export * from "wa-sqlite/src/sqlite-constants.js";
-
-  /**
-   * Builds a Javascript API from the Emscripten module. This API is still
-   * low-level and closely corresponds to the C API exported by the module,
-   * but differs in some specifics like throwing exceptions on errors.
-   * @param {*} Module SQLite module
-   * @returns {SQLiteAPI}
-   */
-  export function Factory(Module: any): SQLiteAPI;
-
-  export class SQLiteError extends Error {
-    constructor(message: any, code: any);
-    code: any;
-  }
-}
-
-/** @ignore */
-declare module "wa-sqlite/dist/wa-sqlite.mjs" {
-  function ModuleFactory(config?: object): Promise<any>;
-  export = ModuleFactory;
-}
-
-/** @ignore */
-declare module "wa-sqlite/dist/wa-sqlite-async.mjs" {
-  function ModuleFactory(config?: object): Promise<any>;
-  export = ModuleFactory;
-}
-
-/** @ignore */
-declare module "wa-sqlite/src/VFS.js" {
-  export * from "wa-sqlite/src/sqlite-constants.js";
-
-  export class Base {
-    mxPathName: number;
-    /**
-     * @param {number} fileId
-     * @returns {number|Promise<number>}
-     */
-    xClose(fileId: number): number;
-    /**
-     * @param {number} fileId
-     * @param {Uint8Array} pData
-     * @param {number} iOffset
-     * @returns {number}
-     */
-    xRead(
-      fileId: number,
-      pData: {
-        size: number;
-        value: Uint8Array;
-      },
-      iOffset: number
-    ): number;
-    /**
-     * @param {number} fileId
-     * @param {Uint8Array} pData
-     * @param {number} iOffset
-     * @returns {number}
-     */
-    xWrite(
-      fileId: number,
-      pData: {
-        size: number;
-        value: Uint8Array;
-      },
-      iOffset: number
-    ): number;
-    /**
-     * @param {number} fileId
-     * @param {number} iSize
-     * @returns {number}
-     */
-    xTruncate(fileId: number, iSize: number): number;
-    /**
-     * @param {number} fileId
-     * @param {*} flags
-     * @returns {number}
-     */
-    xSync(fileId: number, flags: any): number;
-    /**
-     * @param {number} fileId
-     * @param {DataView} pSize64
-     * @returns {number|Promise<number>}
-     */
-    xFileSize(fileId: number, pSize64: DataView): number;
-    /**
-     * @param {number} fileId
-     * @param {number} flags
-     * @returns {number}
-     */
-    xLock(fileId: number, flags: number): number;
-    /**
-     * @param {number} fileId
-     * @param {number} flags
-     * @returns {number}
-     */
-    xUnlock(fileId: number, flags: number): number;
-    /**
-     * @param {number} fileId
-     * @param {DataView} pResOut
-     * @returns {number}
-     */
-    xCheckReservedLock(fileId: number, pResOut: DataView): number;
-    /**
-     * @param {number} fileId
-     * @param {number} flags
-     * @param {DataView} pArg
-     * @returns {number}
-     */
-    xFileControl(fileId: number, flags: number, pArg: DataView): number;
-    /**
-     * @param {number} fileId
-     * @returns {number}
-     */
-    xSectorSize(fileId: number): number;
-    /**
-     * @param {number} fileId
-     * @returns {number}
-     */
-    xDeviceCharacteristics(fileId: number): number;
-    /**
-     * @param {string?} name
-     * @param {number} fileId
-     * @param {number} flags
-     * @param {DataView} pOutFlags
-     * @returns {number}
-     */
-    xOpen(
-      name: string | null,
-      fileId: number,
-      flags: number,
-      pOutFlags: DataView
-    ): number;
-    /**
-     *
-     * @param {string} name
-     * @param {number} syncDir
-     * @returns {number}
-     */
-    xDelete(name: string, syncDir: number): number;
-    /**
-     * @param {string} name
-     * @param {number} flags
-     * @param {DataView} pResOut
-     * @returns {number}
-     */
-    xAccess(name: string, flags: number, pResOut: DataView): number;
-    /**
-     * Handle asynchronous operation. This implementation will be overriden on
-     * registration by an Asyncify build.
-     * @param {function(): Promise<number>} f
-     * @returns {number}
-     */
-    handleAsync(f: () => Promise<number>): number;
-  }
-}
-
-/** @ignore */
-declare module "wa-sqlite/src/examples/ArrayModule.js" {
-  export class ArrayModule {
-    /**
-     * @param {SQLiteAPI} sqlite3
-     * @param {number} db
-     * @param {Array<Array>} rows Table data.
-     * @param {Array<string>} columns Column names.
-     */
-    constructor(
-      sqlite3: any,
-      db: number,
-      rows: Array<any[]>,
-      columns: Array<string>
-    );
-    mapCursorToState: Map<any, any>;
-    sqlite3: any;
-    db: number;
-    rows: any[][];
-    columns: string[];
-    /**
-     * @param {number} db
-     * @param {*} appData Application data passed to `SQLiteAPI.create_module`.
-     * @param {Array<string>} argv
-     * @param {number} pVTab
-     * @param {{ set: function(string): void}} pzErr
-     * @returns {number|Promise<number>}
-     */
-    xCreate(
-      db: number,
-      appData: any,
-      argv: Array<string>,
-      pVTab: number,
-      pzErr: {
-        set: (arg0: string) => void;
-      }
-    ): number | Promise<number>;
-    /**
-     * @param {number} db
-     * @param {*} appData Application data passed to `SQLiteAPI.create_module`.
-     * @param {Array<string>} argv
-     * @param {number} pVTab
-     * @param {{ set: function(string): void}} pzErr
-     * @returns {number|Promise<number>}
-     */
-    xConnect(
-      db: number,
-      appData: any,
-      argv: Array<string>,
-      pVTab: number,
-      pzErr: {
-        set: (arg0: string) => void;
-      }
-    ): number | Promise<number>;
-    /**
-     * @param {number} pVTab
-     * @param {SQLiteModuleIndexInfo} indexInfo
-     * @returns {number|Promise<number>}
-     */
-    xBestIndex(pVTab: number, indexInfo: any): number | Promise<number>;
-    /**
-     * @param {number} pVTab
-     * @returns {number|Promise<number>}
-     */
-    xDisconnect(pVTab: number): number | Promise<number>;
-    /**
-     * @param {number} pVTab
-     * @returns {number|Promise<number>}
-     */
-    xDestroy(pVTab: number): number | Promise<number>;
-    /**
-     * @param {number} pVTab
-     * @param {number} pCursor
-     * @returns {number|Promise<number>}
-     */
-    xOpen(pVTab: number, pCursor: number): number | Promise<number>;
-    /**
-     * @param {number} pCursor
-     * @returns {number|Promise<number>}
-     */
-    xClose(pCursor: number): number | Promise<number>;
-    /**
-     * @param {number} pCursor
-     * @param {number} idxNum
-     * @param {string?} idxStr
-     * @param {Array<number>} values
-     * @returns {number|Promise<number>}
-     */
-    xFilter(
-      pCursor: number,
-      idxNum: number,
-      idxStr: string | null,
-      values: Array<number>
-    ): number | Promise<number>;
-    /**
-     * @param {number} pCursor
-     * @returns {number|Promise<number>}
-     */
-    xNext(pCursor: number): number | Promise<number>;
-    /**
-     * @param {number} pCursor
-     * @returns {number|Promise<number>}
-     */
-    xEof(pCursor: number): number | Promise<number>;
-    /**
-     * @param {number} pCursor
-     * @param {number} pContext
-     * @param {number} iCol
-     * @returns {number|Promise<number>}
-     */
-    xColumn(
-      pCursor: number,
-      pContext: number,
-      iCol: number
-    ): number | Promise<number>;
-    /**
-     * @param {number} pCursor
-     * @param {{ set: function(number): void}} pRowid
-     * @returns {number|Promise<number>}
-     */
-    xRowid(
-      pCursor: number,
-      pRowid: {
-        set: (arg0: number) => void;
-      }
-    ): number | Promise<number>;
-    /**
-     * @param {number} pVTab
-     * @param {Array<number>} values sqlite3_value pointers
-     * @param {{ set: function(number): void}} pRowid
-     * @returns {number|Promise<number>}
-     */
-    xUpdate(
-      pVTab: number,
-      values: Array<number>,
-      pRowid: {
-        set: (arg0: number) => void;
-      }
-    ): number | Promise<number>;
-  }
-}
-
-/** @ignore */
-declare module "wa-sqlite/src/examples/ArrayAsyncModule.js" {
-  import { ArrayModule } from "wa-sqlite/src/examples/ArrayModule.js";
-  export class ArrayAsyncModule extends ArrayModule {
-    /**
-     * @param {function} f
-     * @returns {Promise<number>}
-     */
-    handleAsync(f: Function): Promise<number>;
-  }
-}
-
-/** @ignore */
-declare module "wa-sqlite/src/examples/IndexedDbVFS.js" {
-  import * as VFS from "wa-sqlite/src/VFS.js";
-  export class IndexedDbVFS extends VFS.Base {
-    /**
-     * @param {string} idbName Name of IndexedDB database.
-     */
-    constructor(idbName?: string);
-    name: string;
-    mapIdToFile: Map<any, any>;
-    cacheSize: number;
-    db: any;
-    close(): Promise<void>;
-    /**
-     * Delete a file from IndexedDB.
-     * @param {string} name
-     */
-    deleteFile(name: string): Promise<void>;
-    /**
-     * Forcibly clear an orphaned file lock.
-     * @param {string} name
-     */
-    forceClearLock(name: string): Promise<void>;
-    _getStore(mode?: string): any;
-    /**
-     * Returns the key for file metadata.
-     * @param {string} name
-     * @returns
-     */
-    _metaKey(name: string): string;
-    /**
-     * Returns the key for file block data.
-     * @param {string} name
-     * @param {number} index
-     * @returns
-     */
-    _blockKey(name: string, index: number): string;
-    _getBlock(store: any, file: any, index: any): Promise<any>;
-    _putBlock(store: any, file: any, index: any, blockData: any): void;
-    _purgeCache(store: any, file: any, size?: number): void;
-    _flushCache(store: any, file: any): Promise<void>;
-    _sync(file: any): Promise<void>;
-    /**
-     * Helper function that deletes all keys greater or equal to `key`
-     * provided they start with `prefix`.
-     * @param {string} key
-     * @param {string} [prefix]
-     * @returns
-     */
-    _delete(key: string, prefix?: string): Promise<any>;
-  }
-}
-
-/** @ignore */
-declare module "wa-sqlite/src/examples/MemoryVFS.js" {
-  import * as VFS from "wa-sqlite/src/VFS.js";
-  export class MemoryVFS extends VFS.Base {
-    name: string;
-    mapNameToFile: Map<any, any>;
-    mapIdToFile: Map<any, any>;
-  }
-}
-
-/** @ignore */
-declare module "wa-sqlite/src/examples/MemoryAsyncVFS.js" {
-  import { MemoryVFS } from "wa-sqlite/src/examples/MemoryVFS.js";
-  export class MemoryAsyncVFS extends MemoryVFS {}
-}
-
-/** @ignore */
-declare module "wa-sqlite/src/examples/tag.js" {
-  /**
-   * Template tag builder. This function creates a tag with an API and
-   * database from the same module, then the tag can be used like this:
-   * ```
-   * const sql = tag(sqlite3, db);
-   * const results = await sql`
-   *   SELECT 1 + 1;
-   *   SELECT 6 * 7;
-   * `;
-   * ```
-   * The returned Promise value contains an array of results for each
-   * SQL statement that produces output. Each result is an object with
-   * properties `columns` (array of names) and `rows` (array of array
-   * of values).
-   * @param {SQLiteAPI} sqlite3
-   * @param {number} db
-   * @returns {function(TemplateStringsArray, ...any): Promise<object[]>}
-   */
-  export function tag(
-    sqlite3: any,
-    db: number
-  ): (arg0: TemplateStringsArray, ...args: any[]) => Promise<object[]>;
-}

apps/web/src/common/sqlite/sqlite.worker.ts

@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-// import type { SQLiteAPI, SQLiteCompatibleType } from "./index.d.ts";
+import type { SQLiteAPI, SQLiteCompatibleType } from "./sqlite-types";
 import { Factory, SQLITE_ROW } from "./sqlite-api";
 import SQLiteAsyncESMFactory from "./wa-sqlite-async";
 import SQLiteSyncESMFactory from "./wa-sqlite";
@@ -48,7 +48,7 @@ async function init(dbName: string, async: boolean, url?: string) {
     ? new IDBBatchAtomicVFS(dbName, { durability: "strict" })
     : new AccessHandlePoolVFS(dbName);
   if ("isReady" in vfs) await vfs.isReady;
-
+  console.log(vfs, SQLiteAsyncModule);
   sqlite.vfs_register(vfs, false);
   db = await sqlite.open_v2(dbName, undefined, `multipleciphers-${vfs.name}`);
 }
@@ -138,11 +138,16 @@ async function exportDatabase(dbName: string, async: boolean) {
   return transfer(stream, [stream]);
 }
 
+async function deleteDatabase() {
+  await vfs?.delete();
+}
+
 const worker = {
   close,
   init,
   run: exec,
-  export: exportDatabase
+  export: exportDatabase,
+  delete: deleteDatabase
 };
 
 export type SQLiteWorker = typeof worker;

apps/web/src/common/sqlite/wa-sqlite-kysely-driver.ts

@@ -78,6 +78,11 @@ export class WaSqliteWorkerDriver implements Driver {
     return await this.worker.close();
   }
 
+  async delete() {
+    console.log("DELETING");
+    return this.worker.delete();
+  }
+
   async export() {
     return this.worker.export(this.config.dbName, this.config.async);
   }

apps/web/src/components/error-boundary/index.tsx

@@ -0,0 +1,164 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { PropsWithChildren } from "react";
+import { ErrorText } from "../error-text";
+import { BaseThemeProvider } from "../theme-provider";
+import { Button, Flex, Image, Text } from "@theme-ui/components";
+import {
+  ErrorBoundary as RErrorBoundary,
+  FallbackProps
+} from "react-error-boundary";
+import Logo from "../../assets/logo.svg";
+import LogoDark from "../../assets/logo-dark.svg";
+import { useStore as useThemeStore } from "../../stores/theme-store";
+import { createDialect } from "../../common/sqlite";
+import { getDeviceInfo } from "../../dialogs/issue-dialog";
+
+export function ErrorBoundary(props: PropsWithChildren) {
+  return (
+    <RErrorBoundary FallbackComponent={ErrorComponent}>
+      {props.children}
+    </RErrorBoundary>
+  );
+}
+
+export function ErrorComponent({ error, resetErrorBoundary }: FallbackProps) {
+  const help = getErrorHelp({ error, resetErrorBoundary });
+  const colorScheme = useThemeStore((store) => store.colorScheme);
+
+  return (
+    <BaseThemeProvider
+      onRender={() => document.getElementById("splash")?.remove()}
+      addGlobalStyles
+      sx={{
+        height: "100%",
+        bg: "background",
+        display: "flex",
+        flexDirection: "column",
+        justifyContent: "center",
+        alignItems: "center",
+        overflowY: "auto"
+      }}
+    >
+      <Flex
+        sx={{
+          width: ["95%", "50%"],
+          flexDirection: "column"
+        }}
+      >
+        <Image
+          src={colorScheme === "dark" ? LogoDark : Logo}
+          sx={{ borderRadius: "default", width: 60, alignSelf: "start" }}
+          mb={4}
+        />
+        <Text
+          variant="heading"
+          sx={{ borderBottom: "1px solid var(--border)", pb: 1 }}
+        >
+          Something went wrong
+        </Text>
+        <ErrorText error={error} />
+        {help ? (
+          <>
+            <Text variant="subtitle" sx={{ mt: 2 }}>
+              What went wrong?
+            </Text>
+            <Text variant="body">{help.explanation}</Text>
+            <Text variant="subtitle" sx={{ mt: 1 }}>
+              How to fix it?
+            </Text>
+            <Text variant="body">{help.action}</Text>
+            <Flex sx={{ gap: 1 }}>
+              <Button
+                variant="error"
+                sx={{ alignSelf: "start", px: 30, mt: 1 }}
+                onClick={() => help.fix().catch((e) => alert(e))}
+              >
+                Fix it
+              </Button>
+              <Button
+                variant="secondary"
+                sx={{ alignSelf: "start", px: 30, mt: 1 }}
+                onClick={() => {
+                  const mailto = new URL("mailto:support@streetwriters.co");
+                  mailto.searchParams.set(
+                    "body",
+                    `${
+                      error instanceof Error
+                        ? error.stack
+                        : -typeof error
+                        ? error
+                        : JSON.stringify(error)
+                    }
+
+---
+Device information:
+
+${getDeviceInfo()}`
+                  );
+                  window.open(mailto.toString(), "_blank");
+                }}
+              >
+                Contact support
+              </Button>
+            </Flex>
+          </>
+        ) : (
+          <>
+            <Button
+              variant="secondary"
+              sx={{ alignSelf: "start", px: 30, mt: 1 }}
+              onClick={() =>
+                window.open("mailto:support@streetwriters.co", "_blank")
+              }
+            >
+              Contact support
+            </Button>
+          </>
+        )}
+      </Flex>
+    </BaseThemeProvider>
+  );
+}
+
+function getErrorHelp(props: FallbackProps) {
+  const { error, resetErrorBoundary } = props;
+  const errorText =
+    typeof error === "string"
+      ? error
+      : error instanceof Error
+      ? error.toString()
+      : JSON.stringify(error);
+  if (errorText.includes("file is not a database")) {
+    return {
+      explanation: `This error usually means the database file is either corrupt or it could not be decrypted.`,
+      action:
+        "This error can only be fixed by wiping & reseting the database. Beware that this will wipe all your data inside the database with no way to recover it later on.",
+      fix: async () => {
+        const dialect = createDialect("notesnook");
+        const driver = dialect.createDriver();
+        if (!IS_DESKTOP_APP) await driver.init();
+        await driver.delete();
+        await driver.destroy();
+        resetErrorBoundary();
+      }
+    };
+  }
+}

apps/web/src/components/error-text/index.tsx

@@ -19,9 +19,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { Flex, FlexProps, Text } from "@theme-ui/components";
 
-import { Error } from "../icons";
+import { Error as ErrorIcon } from "../icons";
 
-type ErrorTextProps = { error?: string | null | false } & FlexProps;
+type ErrorTextProps = { error?: string | Error | null | false } & FlexProps;
 export function ErrorText(props: ErrorTextProps) {
   const { error, sx, ...restProps } = props;
 
@@ -34,9 +34,14 @@ export function ErrorText(props: ErrorTextProps) {
       sx={{ borderRadius: "default", ...sx }}
       {...restProps}
     >
-      <Error size={15} color="var(--icon-error)" />
+      <ErrorIcon size={15} color="var(--icon-error)" />
-      <Text variant={"error"} ml={1}>
+      <Text
-        {error}
+        className="selectable"
+        variant={"error"}
+        ml={1}
+        sx={{ whiteSpace: "pre-wrap" }}
+      >
+        {error instanceof Error ? <>{error.stack}</> : error}
       </Text>
     </Flex>
   );

apps/web/src/global.d.ts

@@ -20,6 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import "vite/client";
 import "vite-plugin-svgr/client";
+import "@notesnook/desktop/dist/preload";
 
 declare global {
   var PUBLIC_URL: string;
@@ -32,11 +33,6 @@ declare global {
   var APP_TITLE: string;
   var IS_THEME_BUILDER: boolean;
 
-  interface Window {
-    os?: () => NodeJS.Platform | "mas";
-    NativeNNCrypto?: new () => import("@notesnook/crypto").NNCrypto;
-  }
-
   interface AuthenticationExtensionsClientInputs {
     prf?: {
       eval: {

apps/web/src/index.tsx

@@ -24,35 +24,41 @@ import { AppEventManager, AppEvents } from "./common/app-events";
 import { BaseThemeProvider } from "./components/theme-provider";
 import { register } from "./utils/stream-saver/mitm";
 import { getServiceWorkerVersion } from "./utils/version";
+import { ErrorBoundary, ErrorComponent } from "./components/error-boundary";
 
 renderApp();
 
 async function renderApp() {
-  const { component, props, path } = await init();
-
-  if (serviceWorkerWhitelist.includes(path)) await initializeServiceWorker();
-  if (IS_DESKTOP_APP) {
-    const { loadDatabase } = await import("./hooks/use-database");
-    await loadDatabase("db");
-  }
-
-  const { default: Component } = await component();
   const rootElement = document.getElementById("root");
   if (!rootElement) return;
-
-  const { default: AppLock } = await import("./views/app-lock");
   const root = createRoot(rootElement);
-  root.render(
+
-    <BaseThemeProvider
+  try {
-      onRender={() => document.getElementById("splash")?.remove()}
+    const { component, props, path } = await init();
-      addGlobalStyles
+
-      sx={{ height: "100%" }}
+    if (serviceWorkerWhitelist.includes(path)) await initializeServiceWorker();
-    >
+
-      <AppLock>
+    const { default: Component } = await component();
-        <Component route={props?.route || "login:email"} />
+
-      </AppLock>
+    const { default: AppLock } = await import("./views/app-lock");
-    </BaseThemeProvider>
+    root.render(
-  );
+      <ErrorBoundary>
+        <BaseThemeProvider
+          onRender={() => document.getElementById("splash")?.remove()}
+          addGlobalStyles
+          sx={{ height: "100%", bg: "background" }}
+        >
+          <AppLock>
+            <Component route={props?.route || "login:email"} />
+          </AppLock>
+        </BaseThemeProvider>
+      </ErrorBoundary>
+    );
+  } catch (e) {
+    root.render(
+      <ErrorComponent error={e} resetErrorBoundary={() => renderApp()} />
+    );
+  }
 }
 
 const serviceWorkerWhitelist: Routes[] = ["default"];

apps/web/vite.config.ts

@@ -88,10 +88,14 @@ export default defineConfig({
 
     alias: [
       {
-        find: /desktop-bridge/gm,
+        find: /\/desktop-bridge$/gm,
         replacement: isDesktop
-          ? "desktop-bridge/index.desktop"
+          ? "/desktop-bridge/index.desktop"
-          : "desktop-bridge/index"
+          : "/desktop-bridge/index"
+      },
+      {
+        find: /\/sqlite$/gm,
+        replacement: isDesktop ? "/sqlite/index.desktop" : "/sqlite/index"
       }
     ]
   },