core: move inbox item processing outside the sync loop

Signed-off-by: 01zulfi <85733202+01zulfi@users.noreply.github.com>

packages/core/src/api/index.ts

@@ -58,6 +58,7 @@ import {
   IFileStorage,
   IStorage,
   KVStorageAccessor,
+  PendingSyncItemsAccessor,
   StorageAccessor
 } from "../interfaces.js";
 import TokenManager from "./token-manager.js";
@@ -85,6 +86,7 @@ import { LazyPromise } from "../utils/lazy-promise.js";
 import { InboxApiKeys } from "./inbox-api-keys.js";
 import { Circle } from "./circle.js";
 import { Wrapped } from "./wrapped.js";
+import { PendingSyncItems } from "../database/pending-sync-items.js";
 
 type EventSourceConstructor = new (
   uri: string,
@@ -164,6 +166,8 @@ class Database {
 
   private _kv = new KVStorage(this.databaseReady.promise);
   kv: KVStorageAccessor = () => this._kv;
+  private _pendingSyncItems = new PendingSyncItems(this.databaseReady.promise);
+  pendingSyncItems: PendingSyncItemsAccessor = () => this._pendingSyncItems;
   private _config: ConfigStorage = new ConfigStorage(
     this.databaseReady.promise
   );

packages/core/src/api/sync/index.ts

@@ -306,6 +306,19 @@ export class Sync {
   }
 
   async stop(options: SyncOptions) {
+    const pendingInboxItems = await this.db
+      .pendingSyncItems()
+      .getByType("inbox-item");
+    if (pendingInboxItems.length > 0) {
+      const items = pendingInboxItems.map(
+        (item) => JSON.parse(item.data) as SyncInboxItem
+      );
+      await handleInboxItems(items, this.db);
+      await this.db
+        .pendingSyncItems()
+        .remove(pendingInboxItems.map((item) => item.id));
+    }
+
     if (
       (options.type === "send" || options.type === "full") &&
       (await this.collector.hasUnsyncedChanges())
@@ -584,7 +597,18 @@ export class Sync {
           return false;
         }
 
-        await handleInboxItems(inboxItems, this.db);
+        /**
+         * We will process the inbox items after sync is completed.
+         * TODO: do this for other items as well
+         */
+        for (const item of inboxItems) {
+          await this.db.pendingSyncItems().add({
+            id: item.id,
+            type: "inbox-item",
+            data: JSON.stringify(item),
+            dateCreated: Date.now()
+          });
+        }
 
         return true;
       }

packages/core/src/database/index.ts

@@ -50,6 +50,7 @@ import {
   Monograph,
   Note,
   Notebook,
+  PendingSyncItem,
   Relation,
   Reminder,
   SessionContentItem,
@@ -97,6 +98,7 @@ export interface DatabaseSchema {
 }
 
 export type RawDatabaseSchema = DatabaseSchema & {
+  pendingsyncitems: PendingSyncItem;
   kv: {
     key: string;
     value?: string | null;

packages/core/src/database/migrations.ts

@@ -496,6 +496,18 @@ export class NNMigrationProvider implements MigrationProvider {
             .addColumn("errorContext", "text")
             .execute();
         }
+      },
+      "a-2026-07-06": {
+        async up(db) {
+          await db.schema
+            .createTable("pendingsyncitems")
+            .ifNotExists()
+            .addColumn("id", "text", (c) => c.primaryKey().unique().notNull())
+            .addColumn("type", "text")
+            .addColumn("data", "text")
+            .addColumn("dateCreated", "integer")
+            .execute();
+        }
       }
     };
   }

packages/core/src/database/pending-sync-items.ts

@@ -0,0 +1,53 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { LazyDatabaseAccessor, RawDatabaseSchema } from "./index.js";
+import { PendingSyncItem } from "../types.js";
+
+export class PendingSyncItems {
+  private readonly db: LazyDatabaseAccessor<RawDatabaseSchema>;
+  constructor(db: LazyDatabaseAccessor) {
+    this.db = db as unknown as LazyDatabaseAccessor<RawDatabaseSchema>;
+  }
+
+  async add(item: PendingSyncItem) {
+    await this.db.then((db) =>
+      db.replaceInto("pendingsyncitems").values(item).execute()
+    );
+  }
+
+  async getByType(type: PendingSyncItem["type"]) {
+    const result = await this.db.then((db) =>
+      db
+        .selectFrom("pendingsyncitems")
+        .where("type", "==", type)
+        .selectAll()
+        .execute()
+    );
+    return result as PendingSyncItem[];
+  }
+
+  async remove(ids: string[]) {
+    if (ids.length === 0) return;
+
+    await this.db.then((db) =>
+      db.deleteFrom("pendingsyncitems").where("id", "in", ids).execute()
+    );
+  }
+}

packages/core/src/interfaces.ts

@@ -25,6 +25,7 @@ import {
 } from "@notesnook/crypto";
 import { KVStorage } from "./database/kv.js";
 import { ConfigStorage } from "./database/config.js";
+import { PendingSyncItems } from "./database/pending-sync-items.js";
 
 export type Output<TOutputFormat extends DataFormat> =
   TOutputFormat extends "uint8array" ? Uint8Array : string;
@@ -145,4 +146,5 @@ export interface IFileStorage {
 export type StorageAccessor = () => IStorage;
 export type KVStorageAccessor = () => KVStorage;
 export type ConfigStorageAccessor = () => ConfigStorage;
+export type PendingSyncItemsAccessor = () => PendingSyncItems;
 export type CompressorAccessor = () => Promise<ICompressor>;

packages/core/src/types.ts

@@ -19,6 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { Cipher } from "@notesnook/crypto";
 import { isCipher } from "./utils/index.js";
+import { SyncableItemType } from "./api/sync/types.js";
 
 export type TimeFormat = "12-hour" | "24-hour";
 export type DayFormat = "short" | "long";
@@ -547,6 +548,13 @@ export interface InboxItemHistory extends BaseItem<"inboxitemhistory"> {
   errorContext?: string;
 }
 
+export interface PendingSyncItem {
+  id: string;
+  type: SyncableItemType | "inbox-item";
+  data: string;
+  dateCreated: number;
+}
+
 export type Match = {
   prefix: string;
   match: string;