mobile: use sqlite based logger

apps/mobile/app/app.tsx

@@ -16,15 +16,16 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-import "react-native-gesture-handler";
 import {
   THEME_COMPATIBILITY_VERSION,
   useThemeEngineStore
 } from "@notesnook/theme";
 import React, { useEffect } from "react";
 import { I18nManager, View } from "react-native";
+import "react-native-gesture-handler";
 import { GestureHandlerRootView } from "react-native-gesture-handler";
 import { SafeAreaProvider } from "react-native-safe-area-context";
+import { initializeLogger } from "./common/database/logger";
 import AppLockedOverlay from "./components/app-lock-overlay";
 import { withErrorBoundry } from "./components/exception-handler";
 import GlobalSafeAreaProvider from "./components/globalsafearea";
@@ -44,23 +45,28 @@ I18nManager.swapLeftAndRightInRTL(false);
 const App = () => {
   const init = useAppEvents();
   useEffect(() => {
-    const { appLockEnabled, appLockMode } = SettingsService.get();
-    if (appLockEnabled || appLockMode !== "none") {
-      useUserStore.getState().lockApp(true);
-    }
-
-    //@ts-ignore
-    globalThis["IS_MAIN_APP_RUNNING"] = true;
-    init();
-    setTimeout(async () => {
-      SettingsService.onFirstLaunch();
-      await Notifications.get();
-      if (SettingsService.get().notifNotes) {
-        Notifications.pinQuickNote(true);
-      }
-      TipManager.init();
-    }, 100);
+    initializeLogger()
+      .catch((e) => {
+        console.log(e);
+      })
+      .finally(() => {
+        const { appLockEnabled, appLockMode } = SettingsService.get();
+        if (appLockEnabled || appLockMode !== "none") {
+          useUserStore.getState().lockApp(true);
+        }
 
+        //@ts-ignore
+        globalThis["IS_MAIN_APP_RUNNING"] = true;
+        init();
+        setTimeout(async () => {
+          SettingsService.onFirstLaunch();
+          await Notifications.get();
+          if (SettingsService.get().notifNotes) {
+            Notifications.pinQuickNote(true);
+          }
+          TipManager.init();
+        }, 100);
+      });
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 

apps/mobile/app/common/database/index.js

@@ -85,4 +85,10 @@ export async function setupDatabase(password) {
 }
 
 export const db = database;
-export const DatabaseLogger = dbLogger;
+let DatabaseLogger = dbLogger.scope(Platform.OS);
+
+const setLogger = () => {
+  DatabaseLogger = dbLogger.scope(Platform.OS);
+};
+
+export { DatabaseLogger, setLogger };

apps/mobile/app/common/database/logger.js

@@ -16,14 +16,27 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
-import { MMKVLoader } from "react-native-mmkv-storage";
+
 import { initialize } from "@notesnook/core/dist/logger";
-import { KV } from "./storage";
+import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
+import { Platform } from "react-native";
+import { setLogger } from ".";
+import { RNSqliteDriver } from "./sqlite.kysely";
 
-const LoggerStorage = new MMKVLoader()
-  .withInstanceID("notesnook_logs")
-  .initialize();
+const initializeLogger = async () => {
+  await initialize({
+    dialect: (name) => ({
+      createDriver: () => {
+        return new RNSqliteDriver({ async: true, dbName: name });
+      },
+      createAdapter: () => new SqliteAdapter(),
+      createIntrospector: (db) => new SqliteIntrospector(db),
+      createQueryCompiler: () => new SqliteQueryCompiler()
+    }),
+    tempStore: "memory",
+    journalMode: Platform.OS === "ios" ? "DELETE" : "WAL"
+  });
+  setLogger();
+};
 
-initialize(new KV(LoggerStorage));
-
-export { LoggerStorage };
+export { initializeLogger };

apps/mobile/app/screens/settings/debug.tsx

@@ -122,7 +122,8 @@ export default function DebugLogs() {
       colors.primary.paragraph,
       colors.error.paragraph,
       colors.static.black,
-      colors.static.orange
+      colors.static.orange,
+      colors.primary.border
     ]
   );
 

apps/web/src/common/db.ts

@@ -58,7 +58,7 @@ async function initializeDatabase(persistence: DatabasePersistence) {
 
   database.setup({
     sqliteOptions: {
-      dialect: createDialect,
+      dialect: (name, init) => createDialect(name, true, init),
       ...(IS_DESKTOP_APP || isFeatureSupported("opfs")
         ? { journalMode: "WAL", lockingMode: "exclusive" }
         : {

apps/web/src/common/sqlite/index.desktop.ts

@@ -40,7 +40,7 @@ class SqliteDriver extends KSqliteDriver {
   }
 }
 
-export const createDialect = (name: string): Dialect => {
+export const createDialect = (name: string, _encrypted: boolean): Dialect => {
   return {
     createDriver: () =>
       new SqliteDriver({

apps/web/src/common/sqlite/index.ts

@@ -34,6 +34,7 @@ declare module "kysely" {
 
 export const createDialect = (
   name: string,
+  encrypted: boolean,
   init?: () => Promise<void>
 ): Dialect => {
   return {
@@ -41,6 +42,7 @@ export const createDialect = (
       new WaSqliteWorkerDriver({
         async: !isFeatureSupported("opfs"),
         dbName: name,
+        encrypted,
         init
       }),
     createAdapter: () => new SqliteAdapter(),

apps/web/src/common/sqlite/sqlite.worker.ts

@@ -32,221 +32,230 @@ type PreparedStatement = {
   columns: string[];
 };
 
-let sqlite: SQLiteAPI;
-let db: number | undefined = undefined;
-let vfs: IDBBatchAtomicVFS | AccessHandlePoolVFS | null = null;
-let initialized = false;
-const preparedStatements: Map<string, PreparedStatement> = new Map();
-const retryCounter: Record<string, number> = {};
-console.log("new sqlite worker");
-
-async function open(dbName: string, async: boolean, url?: string) {
-  if (db) {
-    console.error("Database is already initialized", db);
-    return;
-  }
-
-  const option = url ? { locateFile: () => url } : {};
-  const sqliteModule = async
-    ? await import("./wa-sqlite-async").then(
-        ({ default: SQLiteAsyncESMFactory }) => SQLiteAsyncESMFactory(option)
-      )
-    : await import("./wa-sqlite").then(({ default: SQLiteSyncESMFactory }) =>
-        SQLiteSyncESMFactory(option)
-      );
-  sqlite = Factory(sqliteModule);
-  vfs = await getVFS(dbName, async);
-
-  sqlite.vfs_register(vfs, false);
-  db = await sqlite.open_v2(dbName, undefined, `multipleciphers-${vfs.name}`);
-}
-
-/**
- * Wrapper function for preparing SQL statements with caching
- * to avoid unnecessary computations.
- */
-async function prepare(sql: string) {
-  if (!db) throw new Error("Database is not initialized.");
-  try {
-    const cached = preparedStatements.get(sql);
-    if (cached !== undefined) return cached;
-
-    const str = sqlite.str_new(db, sql);
-    const prepared = await sqlite.prepare_v2(db, sqlite.str_value(str));
-    if (!prepared) return;
-
-    const statement: PreparedStatement = {
-      stmt: prepared.stmt,
-      columns: sqlite.column_names(prepared.stmt)
-    };
-    preparedStatements.set(sql, statement);
-
-    sqlite.str_finish(str);
-
-    // reset retry count on success
-    retryCounter[sql] = 0;
-    return statement;
-  } catch (ex) {
-    console.error(ex);
-
-    // statement prepare process can be flaky so retry at least 5 times
-    // before giving up.
-    if (retryCounter[sql] < 5) {
-      retryCounter[sql] = (retryCounter[sql] || 0) + 1;
-      console.warn("Failed to prepare statement. Retrying:", sql);
-      return prepare(sql);
-    } else retryCounter[sql] = 0;
-
-    if (ex instanceof Error || ex instanceof SQLiteError)
-      ex.message += ` (query: ${sql})`;
-    throw ex;
-  }
-}
-
-async function run(
-  sql: string,
-  mode: RunMode,
-  parameters?: SQLiteCompatibleType[]
-) {
-  const prepared = await prepare(sql);
-  if (!prepared) return [];
-  try {
-    if (parameters) sqlite.bind_collection(prepared.stmt, parameters);
-
-    // fast path for exec statements
-    if (mode === "exec") {
-      while ((await sqlite.step(prepared.stmt)) === SQLITE_ROW);
-      return [];
-    }
-
-    const rows: Record<string, SQLiteCompatibleType>[] = [];
-    while ((await sqlite.step(prepared.stmt)) === SQLITE_ROW) {
-      const row = sqlite.row(prepared.stmt);
-      const acc: Record<string, SQLiteCompatibleType> = {};
-      row.forEach((v, i) => (acc[prepared.columns[i]] = v));
-      rows.push(acc);
-    }
-
-    return rows;
-  } catch (e) {
-    if (e instanceof Error || e instanceof SQLiteError)
-      e.message += ` (query: ${sql})`;
-    throw e;
-  } finally {
-    await sqlite
-      .reset(prepared.stmt)
-      // we must clear/destruct the prepared statement if it can't be reset
-      .catch(() =>
-        sqlite
-          .finalize(prepared.stmt)
-          // ignore error (we will just prepare a new statement)
-          .catch(console.error)
-          .finally(() => preparedStatements.delete(sql))
-      );
-  }
-}
-
-async function exec<R>(
-  mode: RunMode,
-  sql: string,
-  parameters?: SQLiteCompatibleType[]
-): Promise<QueryResult<R>> {
-  if (!sql.startsWith("PRAGMA key")) {
-    await waitForDatabase();
-  }
-  if (!db) throw new Error("No database is not opened.");
-
-  const rows = (await run(sql, mode, parameters)) as R[];
-  if (mode === "query") return { rows };
-
-  // initialize the database after it has been successfully decrypted.
-  // all queries prior to that must wait otherwise we get the
-  // "file is not a database" error
-  if (sql.startsWith("PRAGMA key")) await initialize();
-
-  return {
-    insertId: BigInt(sqlite.last_insert_rowid(db)),
-    numAffectedRows: BigInt(sqlite.changes(db)),
-    rows: mode === "raw" ? rows : []
-  };
-}
-
-async function close() {
-  if (!db) return;
-
-  for (const [_, prepared] of preparedStatements) {
-    await sqlite.finalize(prepared.stmt);
-  }
-  preparedStatements.clear();
-  await sqlite.close(db);
-  await vfs?.close();
-
-  db = undefined;
+class _SQLiteWorker {
+  sqlite!: SQLiteAPI;
+  db: number | undefined = undefined;
+  vfs: IDBBatchAtomicVFS | AccessHandlePoolVFS | null = null;
   initialized = false;
-}
+  preparedStatements: Map<string, PreparedStatement> = new Map();
+  retryCounter: Record<string, number> = {};
+  constructor(
+    private readonly dbName: string,
+    private readonly encrypted: boolean
+  ) {
+    console.log("new sqlite worker", dbName, encrypted);
+  }
 
-async function exportDatabase(dbName: string, async: boolean) {
-  const vfs = await getVFS(dbName, async);
-  const stream = new ReadableStream(new DatabaseSource(vfs, dbName));
-  return transfer(stream, [stream]);
-}
+  async open(async: boolean, url?: string) {
+    if (this.db) {
+      console.error("Database is already initialized", this.db);
+      return;
+    }
 
-async function deleteDatabase(dbName: string, async: boolean) {
-  await close();
-  if (vfs) await vfs.delete();
-  else await (await getVFS(dbName, async)).delete();
-}
+    const option = url ? { locateFile: () => url } : {};
+    const sqliteModule = async
+      ? await import("./wa-sqlite-async").then(
+          ({ default: SQLiteAsyncESMFactory }) => SQLiteAsyncESMFactory(option)
+        )
+      : await import("./wa-sqlite").then(({ default: SQLiteSyncESMFactory }) =>
+          SQLiteSyncESMFactory(option)
+        );
+    this.sqlite = Factory(sqliteModule);
+    this.vfs = await this.getVFS(this.dbName, async);
 
-async function getVFS(dbName: string, async: boolean) {
-  const vfs = async
-    ? await import("./IDBBatchAtomicVFS").then(
-        ({ IDBBatchAtomicVFS }) =>
-          new IDBBatchAtomicVFS(dbName, { durability: "strict" })
-      )
-    : await import("./AccessHandlePoolVFS").then(
-        ({ AccessHandlePoolVFS }) => new AccessHandlePoolVFS(dbName)
+    this.sqlite.vfs_register(this.vfs, false);
+    this.db = await this.sqlite.open_v2(
+      this.dbName,
+      undefined,
+      `multipleciphers-${this.vfs.name}`
+    );
+  }
+
+  /**
+   * Wrapper function for preparing SQL statements with caching
+   * to avoid unnecessary computations.
+   */
+  async prepare(sql: string): Promise<PreparedStatement | undefined> {
+    if (!this.db) throw new Error("Database is not initialized.");
+    try {
+      const cached = this.preparedStatements.get(sql);
+      if (cached !== undefined) return cached;
+
+      const str = this.sqlite.str_new(this.db, sql);
+      const prepared = await this.sqlite.prepare_v2(
+        this.db,
+        this.sqlite.str_value(str)
       );
-  if ("isReady" in vfs) await vfs.isReady;
-  return vfs;
+      if (!prepared) return;
+
+      const statement: PreparedStatement = {
+        stmt: prepared.stmt,
+        columns: this.sqlite.column_names(prepared.stmt)
+      };
+      this.preparedStatements.set(sql, statement);
+
+      this.sqlite.str_finish(str);
+
+      // reset retry count on success
+      this.retryCounter[sql] = 0;
+      return statement;
+    } catch (ex) {
+      console.error(ex);
+
+      // statement prepare process can be flaky so retry at least 5 times
+      // before giving up.
+      if (this.retryCounter[sql] < 5) {
+        this.retryCounter[sql] = (this.retryCounter[sql] || 0) + 1;
+        console.warn("Failed to prepare statement. Retrying:", sql);
+        return this.prepare(sql);
+      } else this.retryCounter[sql] = 0;
+
+      if (ex instanceof Error || ex instanceof SQLiteError)
+        ex.message += ` (query: ${sql})`;
+      throw ex;
+    }
+  }
+
+  async exec(sql: string, mode: RunMode, parameters?: SQLiteCompatibleType[]) {
+    const prepared = await this.prepare(sql);
+    if (!prepared) return [];
+    try {
+      if (parameters) this.sqlite.bind_collection(prepared.stmt, parameters);
+
+      // fast path for exec statements
+      if (mode === "exec") {
+        while ((await this.sqlite.step(prepared.stmt)) === SQLITE_ROW);
+        return [];
+      }
+
+      const rows: Record<string, SQLiteCompatibleType>[] = [];
+      while ((await this.sqlite.step(prepared.stmt)) === SQLITE_ROW) {
+        const row = this.sqlite.row(prepared.stmt);
+        const acc: Record<string, SQLiteCompatibleType> = {};
+        row.forEach((v, i) => (acc[prepared.columns[i]] = v));
+        rows.push(acc);
+      }
+
+      return rows;
+    } catch (e) {
+      if (e instanceof Error || e instanceof SQLiteError)
+        e.message += ` (query: ${sql})`;
+      throw e;
+    } finally {
+      await this.sqlite
+        .reset(prepared.stmt)
+        // we must clear/destruct the prepared statement if it can't be reset
+        .catch(() =>
+          this.sqlite
+            .finalize(prepared.stmt)
+            // ignore error (we will just prepare a new statement)
+            .catch(console.error)
+            .finally(() => this.preparedStatements.delete(sql))
+        );
+    }
+  }
+
+  async run<R>(
+    mode: RunMode,
+    sql: string,
+    parameters?: SQLiteCompatibleType[]
+  ): Promise<QueryResult<R>> {
+    if (this.encrypted && !sql.startsWith("PRAGMA key")) {
+      await this.waitForDatabase();
+    }
+    if (!this.db) throw new Error("No database is not opened.");
+
+    const rows = (await this.exec(sql, mode, parameters)) as R[];
+    if (mode === "query") return { rows };
+
+    // initialize the database after it has been successfully decrypted.
+    // all queries prior to that must wait otherwise we get the
+    // "file is not a database" error
+    if (this.encrypted && sql.startsWith("PRAGMA key")) await this.initialize();
+
+    return {
+      insertId: BigInt(this.sqlite.last_insert_rowid(this.db)),
+      numAffectedRows: BigInt(this.sqlite.changes(this.db)),
+      rows: mode === "raw" ? rows : []
+    };
+  }
+
+  async close() {
+    if (!this.db) return;
+
+    for (const [_, prepared] of this.preparedStatements) {
+      await this.sqlite.finalize(prepared.stmt);
+    }
+    this.preparedStatements.clear();
+    await this.sqlite.close(this.db);
+    await this.vfs?.close();
+
+    this.db = undefined;
+    this.initialized = false;
+  }
+
+  async export(dbName: string, async: boolean) {
+    const vfs = await this.getVFS(dbName, async);
+    const stream = new ReadableStream(new DatabaseSource(vfs, dbName));
+    return transfer(stream, [stream]);
+  }
+
+  async delete(dbName: string, async: boolean) {
+    await this.close();
+    if (this.vfs) await this.vfs.delete();
+    else await (await this.getVFS(dbName, async)).delete();
+  }
+
+  async getVFS(dbName: string, async: boolean) {
+    const vfs = async
+      ? await import("./IDBBatchAtomicVFS").then(
+          ({ IDBBatchAtomicVFS }) =>
+            new IDBBatchAtomicVFS(dbName, { durability: "strict" })
+        )
+      : await import("./AccessHandlePoolVFS").then(
+          ({ AccessHandlePoolVFS }) => new AccessHandlePoolVFS(dbName)
+        );
+    if ("isReady" in vfs) await vfs.isReady;
+    return vfs;
+  }
+
+  async initialize() {
+    self.dispatchEvent(
+      new MessageEvent("message", {
+        data: { type: "databaseInitialized", dbName: this.dbName }
+      })
+    );
+    console.log("Database initialized", this.db);
+    this.initialized = true;
+  }
+
+  async waitForDatabase() {
+    // if the database hasn't yet been initialized.
+    if (!this.initialized) {
+      console.log("Waiting for database to be initialized...", this.db);
+      return await new Promise<boolean>((resolve) =>
+        self.addEventListener("message", (ev) => {
+          if (
+            ev.data.type === "databaseInitialized" &&
+            ev.data.dbName === this.dbName
+          )
+            resolve(true);
+        })
+      );
+    }
+    return true;
+  }
 }
 
-async function initialize() {
-  self.dispatchEvent(
-    new MessageEvent("message", { data: { type: "databaseInitialized" } })
-  );
-  console.log("Database initialized", db);
-  initialized = true;
-}
-
-const worker = {
-  close,
-  open,
-  run: exec,
-  export: exportDatabase,
-  delete: deleteDatabase
-};
-
-export type SQLiteWorker = typeof worker;
+export type SQLiteWorker = typeof _SQLiteWorker.prototype;
 
 addEventListener("message", async (event) => {
   if (!event.data.type) {
-    await worker.open(event.data.dbName, event.data.async, event.data.uri);
+    const worker = new _SQLiteWorker(event.data.dbName, event.data.encrypted);
+    await worker.open(event.data.async, event.data.uri);
     const providerPort = createSharedServicePort(worker);
     postMessage(null, [providerPort]);
 
     self.addEventListener("beforeunload", () => worker.close());
   }
 });
-
-async function waitForDatabase() {
-  // if the database hasn't yet been initialized.
-  if (!initialized) {
-    console.log("Waiting for database to be initialized...", db);
-    return await new Promise<boolean>((resolve) =>
-      self.addEventListener("message", (ev) => {
-        if (ev.data.type === "databaseInitialized") resolve(true);
-      })
-    );
-  }
-  return true;
-}

apps/web/src/common/sqlite/wa-sqlite-kysely-driver.ts

@@ -26,7 +26,12 @@ import SQLiteAsyncURI from "./wa-sqlite-async.wasm?url";
 import { Mutex } from "async-mutex";
 import { SharedService } from "./shared-service";
 
-type Config = { dbName: string; async: boolean; init?: () => Promise<void> };
+type Config = {
+  dbName: string;
+  async: boolean;
+  encrypted: boolean;
+  init?: () => Promise<void>;
+};
 
 const servicePool = new Map<
   string,
@@ -55,7 +60,6 @@ export class WaSqliteWorkerDriver implements Driver {
       if (closed) {
         console.log("Already activated. Reinitializing...");
         await service.proxy.open(
-          this.config.dbName,
           this.config.async,
           this.config.async ? SQLiteAsyncURI : SQLiteSyncURI
         );
@@ -89,6 +93,7 @@ export class WaSqliteWorkerDriver implements Driver {
           worker.postMessage({
             dbName: this.config.dbName,
             async: this.config.async,
+            encrypted: this.config.encrypted,
             uri: this.config.async ? SQLiteAsyncURI : SQLiteSyncURI
           });
         }),

apps/web/src/utils/logger.ts

@@ -22,15 +22,32 @@ import {
   logger as _logger,
   logManager
 } from "@notesnook/core/dist/logger";
-import { LogMessage } from "@notesnook/logger";
-import { DatabasePersistence, NNStorage } from "../interfaces/storage";
+import { LogMessage, NoopLogger, format } from "@notesnook/logger";
 import { ZipFile, createZipStream } from "./streams/zip-stream";
 import { createWriteStream } from "./stream-saver";
 import { sanitizeFilename } from "@notesnook/common";
+import { createDialect } from "../common/sqlite";
+import { isFeatureSupported } from "./feature-check";
 
-let logger: typeof _logger;
-async function initializeLogger(persistence: DatabasePersistence = "db") {
-  initialize(new NNStorage("Logs", () => null, persistence), false);
+let logger: typeof _logger = new NoopLogger();
+async function initializeLogger() {
+  await initialize(
+    {
+      dialect: (name, init) => createDialect(name, false, init),
+      ...(IS_DESKTOP_APP || isFeatureSupported("opfs")
+        ? { journalMode: "WAL", lockingMode: "exclusive" }
+        : {
+            journalMode: "MEMORY",
+            lockingMode: "exclusive"
+          }),
+      tempStore: "memory",
+      synchronous: "normal",
+      pageSize: 8192,
+      cacheSize: -32000,
+      skipInitialization: !IS_DESKTOP_APP
+    },
+    false
+  );
   logger = _logger.scope("notesnook-web");
 }
 
@@ -47,11 +64,9 @@ async function downloadLogs() {
         return;
       }
       controller.enqueue({
-        path: sanitizeFilename(log.key, { replacement: "-" }),
+        path: sanitizeFilename(log.key, { replacement: "-" }) + ".log",
         data: textEncoder.encode(
-          (log.logs as LogMessage[])
-            .map((line) => JSON.stringify(line))
-            .join("\n")
+          (log.logs as LogMessage[]).map((line) => format(line)).join("\n")
         )
       });
     }

packages/core/src/api/index.ts

@@ -63,6 +63,7 @@ import { Settings } from "../collections/settings";
 import {
   DatabaseAccessor,
   DatabaseSchema,
+  RawDatabaseSchema,
   SQLiteOptions,
   changeDatabasePassword,
   createDatabase,
@@ -74,7 +75,8 @@ import { Vaults } from "../collections/vaults";
 import { KVStorage } from "../database/kv";
 import { QueueValue } from "../utils/queue-value";
 import { Sanitizer } from "../database/sanitizer";
-import { dropTriggers } from "../database/triggers";
+import { createTriggers, dropTriggers } from "../database/triggers";
+import { NNMigrationProvider } from "../database/migrations";
 
 type EventSourceConstructor = new (
   uri: string,
@@ -244,7 +246,10 @@ class Database {
       await sql.raw(statement).execute(this.sql());
     }
 
-    await initializeDatabase(this.sql().withTables());
+    await initializeDatabase(
+      this.sql().withTables(),
+      new NNMigrationProvider()
+    );
     await this.initCollections();
     return true;
   }
@@ -274,10 +279,11 @@ class Database {
       this.disconnectSSE();
     });
 
-    this._sql = (await createDatabase(
-      "notesnook",
-      this.options.sqliteOptions
-    )) as unknown as Kysely<DatabaseSchema>;
+    this._sql = (await createDatabase<RawDatabaseSchema>("notesnook", {
+      ...this.options.sqliteOptions,
+      migrationProvider: new NNMigrationProvider(),
+      onInit: (db) => createTriggers(db)
+    })) as unknown as Kysely<DatabaseSchema>;
 
     await this.sanitizer.init();
 

packages/core/src/collections/attachments.ts

@@ -100,9 +100,6 @@ export class Attachments implements ICollection {
 
   async init() {
     await this.collection.init();
-    logger.debug("attachments initialized", {
-      total: await this.collection.count()
-    });
   }
 
   async add(

packages/core/src/database/index.ts

@@ -34,7 +34,8 @@ import {
   ColumnType,
   ExpressionBuilder,
   ReferenceExpression,
-  Dialect
+  Dialect,
+  MigrationProvider
 } from "kysely";
 import {
   Attachment,
@@ -58,8 +59,6 @@ import {
   Vault,
   isDeleted
 } from "../types";
-import { NNMigrationProvider } from "./migrations";
-import { createTriggers } from "./triggers";
 import { logger } from "../logger";
 
 // type FilteredKeys<T, U> = {
@@ -258,8 +257,8 @@ const DataMappers: Partial<Record<ItemType, (row: any) => void>> = {
   }
 };
 
-async function setupDatabase(
-  db: Kysely<RawDatabaseSchema>,
+async function setupDatabase<Schema>(
+  db: Kysely<Schema>,
   options: SQLiteOptions
 ) {
   if (options.password)
@@ -295,11 +294,14 @@ async function setupDatabase(
     );
 }
 
-export async function initializeDatabase(db: Kysely<RawDatabaseSchema>) {
+export async function initializeDatabase<Schema>(
+  db: Kysely<Schema>,
+  migrationProvider: MigrationProvider
+) {
   try {
     const migrator = new Migrator({
       db,
-      provider: new NNMigrationProvider()
+      provider: migrationProvider
     });
     const { error, results } = await migrator.migrateToLatest();
 
@@ -314,8 +316,6 @@ export async function initializeDatabase(db: Kysely<RawDatabaseSchema>) {
           .join(", ")}`
       );
 
-    await createTriggers(db);
-
     return db;
   } catch (e) {
     logger.error(e, "Failed to initialized database.");
@@ -336,8 +336,14 @@ export type SQLiteOptions = {
 
   skipInitialization?: boolean;
 };
-export async function createDatabase(name: string, options: SQLiteOptions) {
-  const db = new Kysely<RawDatabaseSchema>({
+export async function createDatabase<Schema>(
+  name: string,
+  options: SQLiteOptions & {
+    migrationProvider: MigrationProvider;
+    onInit?: (db: Kysely<Schema>) => Promise<void>;
+  }
+) {
+  const db = new Kysely<Schema>({
     // log: (event) => {
     //   if (event.queryDurationMillis > 5)
     //     console.warn(event.query.sql, event.queryDurationMillis);
@@ -345,7 +351,8 @@ export async function createDatabase(name: string, options: SQLiteOptions) {
     dialect: options.dialect(name, async () => {
       await db.connection().execute(async (db) => {
         await setupDatabase(db, options);
-        await initializeDatabase(db);
+        await initializeDatabase(db, options.migrationProvider);
+        if (options.onInit) await options.onInit(db);
       });
     }),
     plugins: [new SqliteBooleanPlugin()]
@@ -353,7 +360,8 @@ export async function createDatabase(name: string, options: SQLiteOptions) {
   if (!options.skipInitialization)
     await db.connection().execute(async (db) => {
       await setupDatabase(db, options);
-      await initializeDatabase(db);
+      await initializeDatabase(db, options.migrationProvider);
+      if (options.onInit) await options.onInit(db);
     });
 
   return db;

packages/core/src/logger.ts

@@ -28,7 +28,9 @@ import {
   format,
   ILogger
 } from "@notesnook/logger";
-import { IStorage } from "./interfaces";
+import { Kysely, Migration, MigrationProvider } from "kysely";
+import { SQLiteOptions, createDatabase } from "./database";
+import { toChunks } from "./utils/array";
 
 const WEEK = 86400000 * 7;
 
@@ -38,10 +40,46 @@ const WEEK = 86400000 * 7;
 // 3. Keep 7 days of logs
 // 4. Implement functions for log retrieval & filtering
 
+type SQLiteItem<T> = {
+  [P in keyof T]?: T[P] | null;
+};
+
+type LogMessageWithDate = LogMessage & { date: string };
+export type LogDatabaseSchema = {
+  logs: SQLiteItem<LogMessageWithDate>;
+};
+
+class NNLogsMigrationProvider implements MigrationProvider {
+  async getMigrations(): Promise<Record<string, Migration>> {
+    return {
+      "1": {
+        async up(db) {
+          await db.schema
+            .createTable("logs")
+            .addColumn("timestamp", "integer", (c) => c.notNull())
+            .addColumn("message", "text", (c) => c.notNull())
+            .addColumn("level", "integer", (c) => c.notNull())
+            .addColumn("date", "text")
+            .addColumn("scope", "text")
+            .addColumn("extras", "text")
+            .addColumn("elapsed", "integer")
+            .execute();
+
+          await db.schema
+            .createIndex("log_timestamp_index")
+            .on("logs")
+            .column("timestamp")
+            .execute();
+        }
+      }
+    };
+  }
+}
+
 class DatabaseLogReporter {
   writer: DatabaseLogWriter;
-  constructor(storage: IStorage) {
-    this.writer = new DatabaseLogWriter(storage);
+  constructor(db: Kysely<LogDatabaseSchema>) {
+    this.writer = new DatabaseLogWriter(db);
   }
 
   write(log: LogMessage) {
@@ -50,10 +88,10 @@ class DatabaseLogReporter {
 }
 
 class DatabaseLogWriter {
-  private queue: Map<string, LogMessage> = new Map();
+  private queue: LogMessageWithDate[] = [];
   private hasCleared = false;
 
-  constructor(private readonly storage: IStorage) {
+  constructor(private readonly db: Kysely<LogDatabaseSchema>) {
     setInterval(() => {
       setTimeout(() => {
         if (!this.hasCleared) {
@@ -66,86 +104,86 @@ class DatabaseLogWriter {
   }
 
   push(message: LogMessage) {
-    const key = new Date(message.timestamp).toLocaleDateString();
-    this.queue.set(`${key}:${message.timestamp}`, message);
+    const date = new Date(message.timestamp);
+    (message as LogMessageWithDate).date = `${date.getFullYear()}-${
+      date.getMonth() + 1
+    }-${date.getDate()}`;
+    this.queue.push(message as LogMessageWithDate);
   }
 
   async flush() {
-    if (this.queue.size === 0) return;
-    const queueCopy = Array.from(this.queue.entries());
-    this.queue = new Map();
-
-    await this.storage.writeMulti(queueCopy);
+    if (this.queue.length === 0) return;
+    const queueCopy = this.queue.slice();
+    this.queue = [];
+    for (const chunk of toChunks(queueCopy, 1000)) {
+      await this.db.insertInto("logs").values(chunk).execute();
+    }
   }
 
   async rotate() {
-    const logKeys = (await this.storage.getAllKeys()).sort();
-    const keysToRemove = [];
-    for (const key of logKeys) {
-      const keyParts = key.split(":");
-      if (keyParts.length === 1 || parseInt(keyParts[1]) < Date.now() - WEEK) {
-        keysToRemove.push(key);
-      }
-    }
-
-    if (keysToRemove.length) await this.storage.removeMulti(keysToRemove);
+    const range = Date.now() - WEEK;
+    await this.db.deleteFrom("logs").where("timestamp", "<", range).execute();
   }
 }
 
 class DatabaseLogManager {
-  constructor(private readonly storage: IStorage) {}
+  constructor(private readonly db: Kysely<LogDatabaseSchema>) {}
 
   async get() {
-    const logKeys = await this.storage.getAllKeys();
-    const logEntries = await this.storage.readMulti<LogMessage>(logKeys);
-    const logs: Record<string, LogMessage[]> = {};
+    const logs = await this.db
+      .selectFrom("logs")
+      .select([
+        "timestamp",
+        "message",
+        "level",
+        "scope",
+        "extras",
+        "elapsed",
+        "date"
+      ])
+      .execute();
+    const groupedLogs: Record<string, LogMessage[]> = {};
 
-    for (const [logKey, log] of logEntries) {
-      const keyParts = logKey.split(":");
-      if (keyParts.length === 1) continue;
-
-      const key = keyParts[0];
-      if (!logs[key]) logs[key] = [];
-      logs[key].push(log);
+    for (const log of logs) {
+      const key = log.date!;
+      if (!groupedLogs[key]) groupedLogs[key] = [];
+      groupedLogs[key].push(log as LogMessage);
     }
 
-    return Object.keys(logs)
+    return Object.keys(groupedLogs)
       .sort((a, b) => b.localeCompare(a, undefined, { numeric: true }))
       .map((key) => ({
         key,
-        logs: logs[key]?.sort((a, b) => a.timestamp - b.timestamp)
+        logs: groupedLogs[key]?.sort((a, b) => a.timestamp - b.timestamp)
       }));
   }
 
   async clear() {
-    const logKeys = await this.storage.getAllKeys();
-    await this.storage.removeMulti(logKeys);
+    await this.db.deleteFrom("logs").execute();
   }
 
   async delete(key: string) {
-    const logKeys = await this.storage.getAllKeys();
-    const keysToRemove = [];
-    for (const logKey of logKeys) {
-      const keyParts = logKey.split(":");
-      if (keyParts.length === 1) continue;
-      const currKey = keyParts[0];
-      if (currKey === key) keysToRemove.push(logKey);
-    }
-    if (keysToRemove.length) await this.storage.removeMulti(keysToRemove);
+    await this.db.deleteFrom("logs").where("date", "==", key).execute();
   }
 }
 
-function initialize(storage: IStorage, disableConsoleLogs?: boolean) {
-  if (storage) {
-    const reporters: ILogReporter[] = [new DatabaseLogReporter(storage)];
-    if (process.env.NODE_ENV !== "production" && !disableConsoleLogs)
-      reporters.push(consoleReporter);
-    logger = new Logger({
-      reporter: combineReporters(reporters),
-      lastTime: Date.now()
-    });
-    logManager = new DatabaseLogManager(storage);
-  }
+async function initialize(
+  options: SQLiteOptions,
+  disableConsoleLogs?: boolean
+) {
+  const db = await createDatabase<LogDatabaseSchema>("notesnook-logs", {
+    ...options,
+    migrationProvider: new NNLogsMigrationProvider()
+  });
+
+  const reporters: ILogReporter[] = [new DatabaseLogReporter(db)];
+  if (process.env.NODE_ENV !== "production" && !disableConsoleLogs)
+    reporters.push(consoleReporter);
+  logger = new Logger({
+    reporter: combineReporters(reporters),
+    lastTime: Date.now()
+  });
+  logManager = new DatabaseLogManager(db);
 }
 
 let logger: ILogger = new NoopLogger();

packages/logger/src/index.ts

@@ -118,7 +118,9 @@ function errorLogLevelFactory(level: LogLevel, config: LoggerConfig) {
       extras:
         error instanceof Error
           ? { ...extras, fallbackMessage }
-          : { ...extras, error },
+          : error
+          ? { ...extras, error }
+          : extras,
       scope: config.scope,
       elapsed: now - config.lastTime
     });