diff --git a/packages/core/__e2e__/sync.test.js b/packages/core/__e2e__/sync.test.js index 83bc744cc..1e02818fe 100644 --- a/packages/core/__e2e__/sync.test.js +++ b/packages/core/__e2e__/sync.test.js @@ -27,7 +27,7 @@ import { delay } from "../__tests__/utils"; import { test, expect, vitest } from "vitest"; import { login } from "./utils"; -const TEST_TIMEOUT = 30 * 1000; +const TEST_TIMEOUT = 60 * 1000; test( "case 1: device A & B should only download the changes from device C (no uploading)", @@ -311,12 +311,16 @@ test( expect(deviceA.notebooks.topics(id).has("Topic 1")).toBeTruthy(); expect(deviceB.notebooks.topics(id).has("Topic 2")).toBeTruthy(); + expect( + deviceB.notebooks.topics(id).topic("Topic 2").dateModified > + deviceA.notebooks.topics(id).topic("Topic 1").dateModified + ).toBeTruthy(); + expect( + deviceB.notebooks.notebook(id).dateModified > + deviceA.notebooks.notebook(id).dateModified + ).toBeTruthy(); - await syncAndWait(deviceA, deviceB, false); - - // await delay(1000); - - // await syncAndWait(deviceB, deviceB, false); + await syncAndWait(deviceB, deviceA, false); expect(deviceA.notebooks.topics(id).has("Topic 1")).toBeTruthy(); expect(deviceB.notebooks.topics(id).has("Topic 1")).toBeTruthy(); @@ -360,15 +364,27 @@ test( expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(1); - await syncAndWait(deviceB, deviceA, false); - await syncAndWait(deviceA, deviceB, false); + ctx.onTestFailed(() => { + console.log(deviceA.notes.topicReferences.get(topic.id), noteA); + console.log(deviceB.notes.topicReferences.get(topic.id), noteB); - expect(deviceA.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2); - expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2); + deviceB.notes.topicReferences.rebuild(); + deviceA.notes.topicReferences.rebuild(); + + console.log(deviceA.notes.topicReferences.get(topic.id), noteA); + console.log(deviceB.notes.topicReferences.get(topic.id), noteB); + }); + await syncAndWait(deviceB, deviceA, false); + + expect(deviceA.notes.note(noteB)).toBeDefined(); + expect(deviceB.notes.note(noteA)).toBeDefined(); expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(1); expect(deviceA.notes.note(noteB).data.notebooks).toHaveLength(1); + expect(deviceA.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2); + expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2); + await cleanup(deviceA, deviceB); }, TEST_TIMEOUT @@ -435,12 +451,22 @@ async function cleanup(...devices) { * @returns */ function syncAndWait(deviceA, deviceB, force = false) { - return new Promise((resolve) => { + return new Promise((resolve, reject) => { + const ref2 = deviceB.eventManager.subscribe( + EVENTS.databaseSyncRequested, + (full, force, lastSynced) => { + console.log("sync requested by device A", full, force, lastSynced); + ref2.unsubscribe(); + deviceB.sync(full, force, lastSynced).catch(reject); + } + ); + const ref = deviceB.eventManager.subscribe(EVENTS.syncCompleted, () => { ref.unsubscribe(); console.log("sync completed."); resolve(); }); + console.log( "waiting for sync...", "Device A:", @@ -448,6 +474,7 @@ function syncAndWait(deviceA, deviceB, force = false) { "Device B:", deviceB.syncer.sync.syncing ); - deviceA.sync(true, force); + + deviceA.sync(true, force).catch(reject); }); } diff --git a/packages/core/__tests__/trash.test.js b/packages/core/__tests__/trash.test.js index 01a3c785a..10b9a0706 100644 --- a/packages/core/__tests__/trash.test.js +++ b/packages/core/__tests__/trash.test.js @@ -45,7 +45,7 @@ test("permanently delete a note", () => expect(await note.content()).toBeDefined(); await db.trash.delete(db.trash.all[0].id); expect(db.trash.all).toHaveLength(0); - const content = await db.content.raw(note.data.contentId); + const content = await db.content.get(note.data.contentId); expect(content).toBeUndefined(); sessions = await db.noteHistory.get(noteId); @@ -229,7 +229,7 @@ test("clear trash should delete note content", () => expect(db.trash.all).toHaveLength(0); - const content = await db.content.raw(note.contentId); + const content = await db.content.get(note.contentId); expect(content).toBeUndefined(); sessions = await db.noteHistory.get(note.id); diff --git a/packages/core/src/api/sync/collector.ts b/packages/core/src/api/sync/collector.ts index 8123500aa..e2e2bd5c3 100644 --- a/packages/core/src/api/sync/collector.ts +++ b/packages/core/src/api/sync/collector.ts @@ -19,103 +19,131 @@ along with this program. If not, see . import { Cipher, SerializedKey } from "@notesnook/crypto"; import Database from ".."; -import { CURRENT_DATABASE_VERSION } from "../../common"; +import { CURRENT_DATABASE_VERSION, EV, EVENTS } from "../../common"; +import { logger } from "../../logger"; +import { SYNC_COLLECTIONS_MAP, SyncItem, SyncTransferItem } from "./types"; import { Item, MaybeDeletedItem } from "../../types"; -export type SyncableItemType = - | "note" - | "shortcut" - | "notebook" - | "content" - | "attachment" - | "reminder" - | "relation" - | "color" - | "tag" - | "settings"; -export type CollectedResult = { - items: (MaybeDeletedItem | Cipher)[]; - types: (SyncableItemType | "vaultKey")[]; -}; - -export type SyncItem = { - id: string; - v: number; -} & Cipher; - +const ASYNC_COLLECTIONS_MAP = { + content: "content" +} as const; class Collector { - private lastSyncedTimestamp = 0; - private key?: SerializedKey; + logger = logger.scope("SyncCollector"); constructor(private readonly db: Database) {} - async collect(lastSyncedTimestamp: number, isForceSync?: boolean) { - await this.db.notes.init(); - - this.lastSyncedTimestamp = lastSyncedTimestamp; - this.key = await this.db.user.getEncryptionKey(); - const vaultKey = await this.db.vault.getKey(); - - const collections = { - note: this.db.notes.raw, - shortcut: this.db.shortcuts.raw, - notebook: this.db.notebooks.raw, - content: await this.db.content.all(), - attachment: this.db.attachments.syncable, - reminder: this.db.reminders.raw, - relation: this.db.relations.raw, - color: this.db.colors.raw, - tag: this.db.tags.raw, - settings: [this.db.settings.raw] - }; - - const result: CollectedResult = { - items: [], - types: [] - }; - for (const type in collections) { - this.collectInternal( - type as SyncableItemType, - collections[type as SyncableItemType], - result, - isForceSync - ); + async *collect( + chunkSize: number, + lastSyncedTimestamp: number, + isForceSync = false + ): AsyncGenerator { + const key = await this.db.user.getEncryptionKey(); + if (!key || !key.key || !key.salt) { + EV.publish(EVENTS.userSessionExpired); + throw new Error("User encryption key not generated. Please relogin."); } - if (vaultKey) { - result.items.push(vaultKey); - result.types.push("vaultKey"); + const settings = await this.prepareChunk( + [this.db.settings.raw], + lastSyncedTimestamp, + isForceSync, + key + ); + if (settings) yield { items: settings, type: "settings" }; + + const attachments = await this.prepareChunk( + this.db.attachments.syncable, + lastSyncedTimestamp, + isForceSync, + key + ); + if (attachments) yield { items: attachments, type: "attachment" }; + + for (const itemType in ASYNC_COLLECTIONS_MAP) { + const collectionKey = + ASYNC_COLLECTIONS_MAP[itemType as keyof typeof ASYNC_COLLECTIONS_MAP]; + const collection = this.db[collectionKey].collection; + for await (const chunk of collection.iterate(chunkSize)) { + const items = await this.prepareChunk( + chunk.map((item) => item[1]), + lastSyncedTimestamp, + isForceSync, + key + ); + if (!items) continue; + yield { items, type: itemType as keyof typeof ASYNC_COLLECTIONS_MAP }; + } } - return result; + for (const itemType in SYNC_COLLECTIONS_MAP) { + const collectionKey = + SYNC_COLLECTIONS_MAP[itemType as keyof typeof SYNC_COLLECTIONS_MAP]; + const collection = this.db[collectionKey].collection; + for (const chunk of collection.iterateSync(chunkSize)) { + const items = await this.prepareChunk( + chunk, + lastSyncedTimestamp, + isForceSync, + key + ); + if (!items) continue; + yield { items, type: itemType as keyof typeof SYNC_COLLECTIONS_MAP }; + } + } } - private serialize(item: MaybeDeletedItem) { - if (!this.key) throw new Error("No encryption key found."); - return this.db.storage().encrypt(this.key, JSON.stringify(item)); - } - - encrypt(array: MaybeDeletedItem[]) { - if (!array.length) return []; - return Promise.all(array.map(this.map, this)); - } - - private collectInternal( - itemType: SyncableItemType, - items: MaybeDeletedItem[], - result: CollectedResult, - isForceSync?: boolean + async prepareChunk( + chunk: MaybeDeletedItem[], + lastSyncedTimestamp: number, + isForceSync: boolean, + key: SerializedKey ) { - if (!items || !items.length) return; + const { ids, items } = filterSyncableItems( + chunk, + lastSyncedTimestamp, + isForceSync + ); + if (!ids.length) return; + const ciphers = await this.db.storage().encryptMulti(key, items); + return toPushItem(ids, ciphers); + } +} +export default Collector; - for (const item of items) { - if (!item) continue; +function toPushItem(ids: string[], ciphers: Cipher<"base64">[]) { + if (ids.length !== ciphers.length) + throw new Error("ids.length must be equal to ciphers.length"); - const isSyncable = !item.synced || isForceSync; - const isUnsynced = - item.dateModified > this.lastSyncedTimestamp || isForceSync; + const items: SyncItem[] = []; + for (let i = 0; i < ids.length; ++i) { + const id = ids[i]; + const cipher = ciphers[i]; + items.push({ ...cipher, v: CURRENT_DATABASE_VERSION, id }); + } + return items; +} - if (isUnsynced && isSyncable) { - result.items.push( +function filterSyncableItems( + items: MaybeDeletedItem[], + lastSyncedTimestamp: number, + isForceSync = false +): { items: string[]; ids: string[] } { + if (!items || !items.length) return { items: [], ids: [] }; + + const ids = []; + const syncableItems = []; + for (const item of items) { + if (!item) continue; + + const isSyncable = !item.synced || isForceSync; + const isUnsynced = item.dateModified > lastSyncedTimestamp || isForceSync; + + // synced is a local only property + delete item.synced; + + if (isUnsynced && isSyncable) { + ids.push(item.id); + syncableItems.push( + JSON.stringify( "localOnly" in item && item.localOnly ? { id: item.id, @@ -124,21 +152,9 @@ class Collector { deleteReason: "localOnly" } : item - ); - result.types.push(itemType); - } + ) + ); } } - - private async map(item: MaybeDeletedItem) { - // synced is a local only property - delete item.synced; - - return { - id: item.id, - v: CURRENT_DATABASE_VERSION, - ...(await this.serialize(item)) - }; - } + return { items: syncableItems, ids }; } -export default Collector; diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.ts index bbc4b39b7..c828e68ef 100644 --- a/packages/core/src/api/sync/index.ts +++ b/packages/core/src/api/sync/index.ts @@ -26,11 +26,7 @@ import { } from "../../common"; import Constants from "../../utils/constants"; import TokenManager from "../token-manager"; -import Collector, { - CollectedResult, - SyncableItemType, - SyncItem -} from "./collector"; +import Collector from "./collector"; import * as signalr from "@microsoft/signalr"; import Merger from "./merger"; import Conflicts from "./conflicts"; @@ -41,6 +37,14 @@ import { Mutex } from "async-mutex"; import Database from ".."; import { migrateItem } from "../../migrations"; import { SerializedKey } from "@notesnook/crypto"; +import { + ItemMap, + MaybeDeletedItem, + Note, + Notebook, + TrashOrItem +} from "../../types"; +import { SyncableItemType, SyncTransferItem } from "./types"; const ITEM_TYPE_TO_COLLECTION_TYPE = { note: "notes", @@ -49,13 +53,10 @@ const ITEM_TYPE_TO_COLLECTION_TYPE = { attachment: "attachments", relation: "relations", reminder: "reminders", - shortcut: "shortcuts" -}; - -type SyncTransferItem = { - items: SyncItem[]; - type: SyncableItemType; -}; + shortcut: "shortcuts", + tag: "tags", + color: "colors" +} as const; export default class SyncManager { sync = new Sync(this.db); @@ -164,6 +165,11 @@ class Sync { }, 15000) as unknown as number; const key = await this.db.user.getEncryptionKey(); + if (!key || !key.key || !key.salt) { + EV.publish(EVENTS.userSessionExpired); + throw new Error("User encryption key not generated. Please relogin."); + } + const dbLastSynced = await this.db.lastSynced(); await this.processChunk(chunk, key, dbLastSynced, true); }); @@ -198,7 +204,7 @@ class Sync { const serverResponse = full ? await this.fetch(lastSynced) : null; this.logger.info("Data fetched", serverResponse || {}); - if (await this.send(lastSynced, force, newLastSynced)) { + if (await this.send(lastSynced, newLastSynced, force)) { this.logger.info("New data sent"); await this.stop(newLastSynced); } else if (serverResponse) { @@ -267,7 +273,7 @@ class Sync { serverResponse.vaultKey.salt !== null && serverResponse.vaultKey.length > 0 ) { - await this.db.vault._setKey(serverResponse.vaultKey); + await this.db.vault.setKey(serverResponse.vaultKey); } this.connection.off("SendItems"); @@ -279,7 +285,11 @@ class Sync { return { lastSynced: serverResponse.lastSynced }; } - async send(oldLastSynced, isForceSync, newLastSynced) { + async send( + oldLastSynced: number, + newLastSynced: number, + isForceSync?: boolean + ) { await this.uploadAttachments(); let isSyncInitialized = false; @@ -290,7 +300,7 @@ class Sync { isForceSync )) { if (!isSyncInitialized) { - const vaultKey = await this.db.vault._getKey(); + const vaultKey = await this.db.vault.getKey(); newLastSynced = await this.connection.invoke("InitializePush", { vaultKey, lastSynced: newLastSynced @@ -315,7 +325,7 @@ class Sync { return true; } - async stop(lastSynced) { + async stop(lastSynced: number) { // refresh topic references this.db.notes.topicReferences.rebuild(); // refresh monographs on sync completed @@ -353,7 +363,7 @@ class Sync { /** * @private */ - async onPushCompleted(lastSynced) { + async onPushCompleted(lastSynced: number) { // refresh topic references this.db.notes.topicReferences.rebuild(); @@ -382,12 +392,14 @@ class Sync { ) ); - let items = []; - if (this.merger.isSyncCollection(chunk.type)) { - items = deserialized.map((item) => - this.merger.mergeItemSync(item, chunk.type, dbLastSynced) - ); - } else if (chunk.type === "content") { + const itemType = chunk.type; + let items: ( + | MaybeDeletedItem< + ItemMap[SyncableItemType] | TrashOrItem | TrashOrItem + > + | undefined + )[] = []; + if (itemType === "content") { const localItems = await this.db.content.multi( chunk.items.map((i) => i.id) ); @@ -396,28 +408,33 @@ class Sync { this.merger.mergeContent(item, localItems[item.id], dbLastSynced) ) ); + } else if (itemType === "settings") { + await this.merger.mergeItem(deserialized[0], itemType, dbLastSynced); + return; } else { - items = await Promise.all( - deserialized.map((item) => - this.merger.mergeItem(item, chunk.type, dbLastSynced) - ) - ); + items = this.merger.isSyncCollection(itemType) + ? deserialized.map((item) => + this.merger.mergeItemSync(item, itemType, dbLastSynced) + ) + : await Promise.all( + deserialized.map((item) => + this.merger.mergeItem(item, itemType, dbLastSynced) + ) + ); } + const collectionType = ITEM_TYPE_TO_COLLECTION_TYPE[itemType]; + await this.db[collectionType].collection.setItems(items as any); + if ( notify && - (chunk.type === "content" || chunk.type === "note") && + (itemType === "note" || itemType === "content") && items.length > 0 ) { items.forEach((item) => this.db.eventManager.publish(EVENTS.syncItemMerged, item) ); } - - const collectionType = this.itemTypeToCollection[chunk.type]; - if (collectionType && this.db[collectionType]) { - await this.db[collectionType]._collection.setItems(items); - } } private async pushItem(item: SyncTransferItem, newLastSynced: number) { @@ -464,7 +481,11 @@ function promiseTimeout(ms: number, promise: Promise) { return Promise.race([promise, timeout]); } -async function deserializeItem(decryptedItem, version, database) { +async function deserializeItem( + decryptedItem: string, + version: number, + database: Database +) { const deserialized = JSON.parse(decryptedItem); deserialized.remote = true; deserialized.synced = true; diff --git a/packages/core/src/api/sync/merger.ts b/packages/core/src/api/sync/merger.ts index 8e44b8bb2..511be0bdb 100644 --- a/packages/core/src/api/sync/merger.ts +++ b/packages/core/src/api/sync/merger.ts @@ -17,292 +17,215 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -import { migrateItem } from "../../migrations"; import { set } from "../../utils/set"; import { logger } from "../../logger"; import { isHTMLEqual } from "../../utils/html-diff"; -import { EV, EVENTS } from "../../common"; import Database from ".."; -import { SyncItem, SyncableItemType } from "./collector"; -import { Item, ItemMap, MaybeDeletedItem, isDeleted } from "../../types"; -import { SerializedKey } from "@notesnook/crypto"; -import { isCipher } from "../../database/crypto"; - -type Conflict

= ( - local: MaybeDeletedItem, - remote: MaybeDeletedItem -) => Promise; - -type Set

= ( - item: MaybeDeletedItem -) => Promise; - -type Get

= ( - id: string -) => - | MaybeDeletedItem - | undefined - | Promise | undefined>; - -type MergeDefinition = { - [P in SyncableItemType]: { - threshold?: number; - get?: Get

; - set: Set

; - conflict?: Conflict

; - }; -}; +import { SYNC_COLLECTIONS_MAP } from "./types"; +import { + Attachment, + ContentItem, + Item, + ItemMap, + MaybeDeletedItem, + Note, + Notebook, + SettingsItem, + TrashOrItem, + isDeleted +} from "../../types"; class Merger { - private mergeDefinition: MergeDefinition; - private logger = logger.scope("Merger"); - private lastSynced = 0; - private key?: SerializedKey; - constructor(private readonly db: Database) { - this.mergeDefinition = { - settings: { - threshold: 1000, - get: () => this.db.settings.raw, - set: (item) => this.db.settings.merge(item), - conflict: (_local, remote) => this.db.settings.merge(remote) - }, - note: { - get: (id) => this.db.notes.note(id)?.data, - set: (item) => this.db.notes.merge(item) - }, - shortcut: { - get: (id) => this.db.shortcuts.shortcut(id), - set: (item) => this.db.shortcuts.merge(item) - }, - reminder: { - get: (id) => this.db.reminders.reminder(id), - set: (item) => this.db.reminders.merge(item) - }, - relation: { - get: (id) => this.db.relations.relation(id), - set: (item) => this.db.relations.merge(item) - }, - tag: { - get: (id) => this.db.tags.tag(id), - set: (item) => this.db.tags.merge(item) - }, - color: { - get: (id) => this.db.colors.color(id), - set: (item) => this.db.colors.merge(item) - }, - notebook: { - threshold: 1000, - get: (id) => this.db.notebooks.notebook(id)?.data, - set: (item) => this.db.notebooks.merge(item), - conflict: (_local, remote) => this.db.notebooks.merge(remote) - }, - content: { - threshold: process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000, - get: (id) => this.db.content.raw(id), - set: async (item) => { - await this.db.content.merge(item); - }, - conflict: async (local, remote) => { - if (isDeleted(local) || isDeleted(remote)) { - if (remote.dateModified > local.dateModified) - await db.content.merge(remote); - return; - } + logger = logger.scope("Merger"); + constructor(private readonly db: Database) {} - const note = this.db.notes.note(local.noteId); - if (!note || !note.data) return; + isSyncCollection(type: string): type is keyof typeof SYNC_COLLECTIONS_MAP { + return type in SYNC_COLLECTIONS_MAP; + } - // if hashes are equal do nothing - if ( - !note.locked && - (!remote || - !local || - !local.data || - !remote.data || - isHTMLEqual(local.data, remote.data)) - ) - return; + isConflicted( + localItem: MaybeDeletedItem, + remoteItem: MaybeDeletedItem, + lastSynced: number, + conflictThreshold: number + ) { + const isResolved = + "dateResolved" in localItem && + localItem.dateResolved === remoteItem.dateModified; + const isModified = + // the local item is modified if it was changed/modified after the last + // sync i.e. it wasn't synced yet. + // However, in case a sync is interrupted the local item's date modified + // will be ahead of last sync. In that case, we also have to check if the + // synced flag is false (it is only false if a user makes edits on the + // local device). + localItem.dateModified > lastSynced && !localItem.synced; + if (isModified && !isResolved) { + // If time difference between local item's edits & remote item's edits + // is less than threshold, we shouldn't trigger a merge conflict; instead + // we will keep the most recently changed item. + const timeDiff = + Math.max(remoteItem.dateModified, localItem.dateModified) - + Math.min(remoteItem.dateModified, localItem.dateModified); - if (note.locked) { - // if note is locked or content is deleted we keep the most recent version. - if (remote.dateModified > local.dateModified) - await this.db.content.merge({ ...remote, id: local.id }); - } else { - // otherwise we trigger the conflicts - await this.db.content.merge({ ...local, conflicted: remote }); - await this.db.notes.add({ id: local.noteId, conflicted: true }); - await this.db.storage().write("hasConflicts", true); - } + if (timeDiff < conflictThreshold) { + if (remoteItem.dateModified > localItem.dateModified) { + return "merge"; } - }, - attachment: { - set: async (remoteAttachment) => { - if (isDeleted(remoteAttachment)) { - await this.db.attachments.merge(remoteAttachment); - return; - } + return; + } - const localAttachment = this.db.attachments.attachment( - remoteAttachment.metadata.hash + return "conflict"; + } else if (!isResolved) { + return "merge"; + } + } + + mergeItemSync( + remoteItem: MaybeDeletedItem< + ItemMap[TType] | TrashOrItem | TrashOrItem + >, + type: TType, + lastSynced: number + ) { + switch (type) { + case "shortcut": + case "reminder": + case "tag": + case "color": + case "note": + case "relation": { + const localItem = this.db[SYNC_COLLECTIONS_MAP[type]].collection.getRaw( + remoteItem.id + ); + if (!localItem || remoteItem.dateModified > localItem.dateModified) { + return remoteItem; + } + break; + } + // case "note": { + // const localItem = this.db.notes.collection.getRaw(remoteItem.id); + // if (!localItem || remoteItem.dateModified > localItem.dateModified) { + // return this.db.notes.merge( + // localItem, + // remoteItem as MaybeDeletedItem> + // ); + // } + // break; + // } + case "notebook": { + const THRESHOLD = 1000; + const localItem = this.db.notebooks.collection.getRaw(remoteItem.id); + if ( + !localItem || + this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD) + ) { + return this.db.notebooks.merge( + localItem, + remoteItem as MaybeDeletedItem>, + lastSynced ); - if ( - localAttachment && - localAttachment.dateUploaded !== remoteAttachment.dateUploaded - ) { - const noteIds = localAttachment.noteIds.slice(); - const isRemoved = await this.db.attachments.remove( - localAttachment.metadata.hash, - true - ); - if (!isRemoved) - throw new Error( - "Conflict could not be resolved in one of the attachments." - ); - remoteAttachment.noteIds = set.union( - remoteAttachment.noteIds, - noteIds - ); - } - await this.db.attachments.merge(remoteAttachment); } + break; } - }; - } - - async _migrate(deserialized: Item, version: number) { - // it is a locked note, bail out. - if (isCipher(deserialized) && deserialized.alg && deserialized.cipher) - return deserialized; - - return migrateItem(deserialized, version, deserialized.type, this.db); - } - - async _deserialize(item: SyncItem, migrate = true) { - if (!this.key) throw new Error("User encryption key not found."); - - const decrypted = await this.db.storage().decrypt(this.key, item); - if (!decrypted) { - throw new Error("Decrypted item cannot be undefined or empty."); - } - - const deserialized = JSON.parse(decrypted); - deserialized.remote = true; - deserialized.synced = true; - if (!migrate) return deserialized; - await this._migrate(deserialized, item.v); - return deserialized; - } - - async _mergeItem( - syncItem: SyncItem, - get: Get, - add: Set - ) { - const remoteItem = (await this._deserialize(syncItem)) as MaybeDeletedItem< - ItemMap[TItemType] - >; - const localItem = await get(remoteItem.id); - if (!localItem || remoteItem.dateModified > localItem.dateModified) { - await add(remoteItem); - return remoteItem; } } - async _mergeItemWithConflicts( - syncItem: SyncItem, - get: Get, - add: Set, - markAsConflicted: Conflict, - threshold: number + async mergeContent( + remoteItem: MaybeDeletedItem, + localItem: MaybeDeletedItem, + lastSynced: number ) { - const remoteItem = (await this._deserialize(syncItem)) as MaybeDeletedItem< - ItemMap[TItemType] - >; - const localItem = await get(remoteItem.id); + if (localItem && "localOnly" in localItem && localItem.localOnly) return; - if (!localItem || isDeleted(localItem)) { - await add(remoteItem); + const THRESHOLD = process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000; + const conflicted = + localItem && + this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD); + if (!localItem || conflicted === "merge") { return remoteItem; - } else { - const isResolved = - "dateResolved" in localItem && - localItem.dateResolved === remoteItem.dateModified; - const isModified = - // the local item is modified if it was changed/modified after the last sync - // i.e. it wasn't synced yet. - // However, in case a sync is interrupted the local item's date modified will - // be ahead of last sync. In that case, we also have to check if the synced flag - // is false (it is only false if a user makes edits on the local device). - localItem.dateModified > this.lastSynced && !localItem.synced; - if (isModified && !isResolved) { - // If time difference between local item's edits & remote item's edits - // is less than threshold, we shouldn't trigger a merge conflict; instead - // we will keep the most recently changed item. - const timeDiff = - Math.max(remoteItem.dateModified, localItem.dateModified) - - Math.min(remoteItem.dateModified, localItem.dateModified); + } else if (conflicted === "conflict") { + if (isDeleted(localItem) || isDeleted(remoteItem)) { + if (remoteItem.dateModified > localItem.dateModified) return remoteItem; + return; + } - if (timeDiff < threshold) { - if (remoteItem.dateModified > localItem.dateModified) { - await add(remoteItem); - return remoteItem; - } - return; - } + const note = this.db.notes.collection.get(localItem.noteId); + if (!note) return; - this.logger.info("Conflict detected", { - itemId: remoteItem.id, - isResolved, - isModified, - timeDiff, - remote: remoteItem.dateModified, - local: localItem.dateModified, - lastSynced: this.lastSynced + // if hashes are equal do nothing + if ( + !note.locked && + (!remoteItem || + !remoteItem || + !localItem.data || + !remoteItem.data || + isHTMLEqual(localItem.data, remoteItem.data)) + ) + return; + + if (note.locked) { + // if note is locked or content is deleted we keep the most recent version. + if (remoteItem.dateModified > localItem.dateModified) return remoteItem; + } else { + // otherwise we trigger the conflicts + await this.db.notes.add({ + id: localItem.noteId, + conflicted: true }); - - await markAsConflicted(localItem, remoteItem); - } else if (!isResolved) { - await add(remoteItem); - return remoteItem; + await this.db.storage().write("hasConflicts", true); + return { + ...localItem, + conflicted: remoteItem + }; } } } - async mergeItem( - type: TItemType | "vaultKey", - item: SyncItem + async mergeItem( + remoteItem: SettingsItem | MaybeDeletedItem, + type: "settings" | "attachment", + lastSynced: number ) { - this.lastSynced = await this.db.lastSynced(); + switch (type) { + case "settings": { + if (isDeleted(remoteItem) || remoteItem.type !== "settings") return; - if (!this.key) this.key = await this.db.user.getEncryptionKey(); - if (!this.key || !this.key.key || !this.key.salt) { - EV.publish(EVENTS.userSessionExpired); - throw new Error("User encryption key not generated. Please relogin."); - } + const localItem = this.db.settings.raw; + if ( + !localItem || + this.isConflicted(localItem, remoteItem, lastSynced, 1000) + ) { + await this.db.settings.merge(remoteItem, lastSynced); + } + break; + } + case "attachment": { + if (isDeleted(remoteItem)) { + return this.db.attachments.merge(undefined, remoteItem); + } + if (remoteItem.type !== "attachment") return; - if (type === "vaultKey") { - await this.db.vault.setKey(await this._deserialize(item, false)); - return; - } - - const definition = this.mergeDefinition[type]; - if (definition.conflict && definition.get && definition.threshold) { - return await this._mergeItemWithConflicts( - item, - definition.get, - definition.set, - definition.conflict, - definition.threshold - ); - } else if (definition.get && definition.set) { - return await this._mergeItem( - item, - definition.get, - definition.set - ); - } else if (!definition.get && !!definition.set) { - const remote = await this._deserialize(item); - await definition.set(remote); + const localAttachment = this.db.attachments.attachment( + remoteItem.metadata.hash + ); + if ( + localAttachment && + localAttachment.dateUploaded !== remoteItem.dateUploaded + ) { + const noteIds = localAttachment.noteIds.slice(); + const isRemoved = await this.db.attachments.remove( + localAttachment.metadata.hash, + true + ); + if (!isRemoved) + throw new Error( + "Conflict could not be resolved in one of the attachments." + ); + remoteItem.noteIds = set.union(remoteItem.noteIds, noteIds); + } + return this.db.attachments.merge(undefined, remoteItem); + } } } } diff --git a/packages/core/src/api/sync/types.ts b/packages/core/src/api/sync/types.ts new file mode 100644 index 000000000..4d53748e5 --- /dev/null +++ b/packages/core/src/api/sync/types.ts @@ -0,0 +1,52 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import { Cipher } from "@notesnook/crypto"; + +export type SyncableItemType = + | "note" + | "shortcut" + | "notebook" + | "content" + | "attachment" + | "reminder" + | "relation" + | "color" + | "tag" + | "settings"; + +export type SyncItem = { + id: string; + v: number; +} & Cipher<"base64">; + +export const SYNC_COLLECTIONS_MAP = { + note: "notes", + notebook: "notebooks", + shortcut: "shortcuts", + reminder: "reminders", + relation: "relations", + tag: "tags", + color: "colors" +} as const; + +export type SyncTransferItem = { + items: SyncItem[]; + type: SyncableItemType; +}; diff --git a/packages/core/src/api/user-manager.ts b/packages/core/src/api/user-manager.ts index 366275ce6..6f3fe308f 100644 --- a/packages/core/src/api/user-manager.ts +++ b/packages/core/src/api/user-manager.ts @@ -32,7 +32,7 @@ export type User = { email: string; isEmailConfirmed: boolean; salt: string; - attachmentsKey?: Cipher; + attachmentsKey?: Cipher<"base64">; marketingConsent?: boolean; mfa: { isEnabled: boolean; diff --git a/packages/core/src/api/vault.ts b/packages/core/src/api/vault.ts index 55a7a4067..5abc4bd50 100644 --- a/packages/core/src/api/vault.ts +++ b/packages/core/src/api/vault.ts @@ -209,7 +209,7 @@ export default class Vault { return await this.lockNote(note, await this.getVaultPassword()); } - async exists(vaultKey?: Cipher) { + async exists(vaultKey?: Cipher<"base64">) { if (!vaultKey) vaultKey = await this.getKey(); return vaultKey && isCipher(vaultKey); } @@ -370,10 +370,10 @@ export default class Vault { } async getKey() { - return await this.db.storage().read("vaultKey"); + return await this.db.storage().read>("vaultKey"); } - async setKey(vaultKey: Cipher) { + async setKey(vaultKey: Cipher<"base64">) { await this.db.storage().write("vaultKey", vaultKey); } } diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.ts index 2b1f6313d..2befc8150 100644 --- a/packages/core/src/collections/attachments.ts +++ b/packages/core/src/collections/attachments.ts @@ -43,7 +43,7 @@ import Database from "../api"; export class Attachments implements ICollection { name = "attachments"; key: Cipher<"base64"> | null = null; - private readonly collection: CachedCollection<"attachments", Attachment>; + readonly collection: CachedCollection<"attachments", Attachment>; constructor(private readonly db: Database) { this.collection = new CachedCollection( db.storage, @@ -109,7 +109,7 @@ export class Attachments implements ICollection { } merge( - localAttachment: MaybeDeletedItem, + localAttachment: MaybeDeletedItem | undefined, remoteAttachment: MaybeDeletedItem ) { if (isDeleted(remoteAttachment)) return remoteAttachment; @@ -323,7 +323,10 @@ export class Attachments implements ICollection { return ( outputType === "base64" - ? dataurl.fromObject({ type: attachment.metadata.type, data }) + ? dataurl.fromObject({ + type: attachment.metadata.type, + data + }) : data ) as Output; } diff --git a/packages/core/src/collections/colors.ts b/packages/core/src/collections/colors.ts index a1710a86b..ded823f59 100644 --- a/packages/core/src/collections/colors.ts +++ b/packages/core/src/collections/colors.ts @@ -36,7 +36,7 @@ export const DefaultColors: Record = { export class Colors implements ICollection { name = "colors"; - private readonly collection: CachedCollection<"colors", Color>; + readonly collection: CachedCollection<"colors", Color>; constructor(private readonly db: Database) { this.collection = new CachedCollection( db.storage, diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.ts index ef119c9d8..efd437b6f 100644 --- a/packages/core/src/collections/content.ts +++ b/packages/core/src/collections/content.ts @@ -49,7 +49,7 @@ export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({ export class Content implements ICollection { name = "content"; - private readonly collection: IndexedCollection<"content", ContentItem>; + readonly collection: IndexedCollection<"content", ContentItem>; constructor(private readonly db: Database) { this.collection = new IndexedCollection( db.storage, @@ -130,7 +130,7 @@ export class Content implements ICollection { async get(id: string) { const content = await this.raw(id); if (!content || isDeleted(content)) return; - return content.data; + return content; } async raw(id: string) { diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.ts index 1fd5d5f1d..2142ca14d 100644 --- a/packages/core/src/collections/notebooks.ts +++ b/packages/core/src/collections/notebooks.ts @@ -27,7 +27,6 @@ import { MaybeDeletedItem, Notebook, Topic, - TrashItem, TrashOrItem, isDeleted, isTrashItem @@ -52,15 +51,23 @@ export class Notebooks implements ICollection { return this.collection.init(); } - async merge(remoteNotebook: MaybeDeletedItem>) { + merge( + localNotebook: MaybeDeletedItem> | undefined, + remoteNotebook: MaybeDeletedItem>, + lastSyncedTimestamp: number + ) { if (isDeleted(remoteNotebook) || isTrashItem(remoteNotebook)) - return await this.collection.add(remoteNotebook); + return remoteNotebook; - const id = remoteNotebook.id; - const localNotebook = this.collection.get(id); + if ( + localNotebook && + (isTrashItem(localNotebook) || isDeleted(localNotebook)) + ) { + if (localNotebook.dateModified > remoteNotebook.dateModified) return; + return remoteNotebook; + } if (localNotebook && localNotebook.topics?.length) { - const lastSyncedTimestamp = await this.db.lastSynced(); let isChanged = false; // merge new and old topics for (const oldTopic of localNotebook.topics) { @@ -93,7 +100,7 @@ export class Notebooks implements ICollection { } remoteNotebook.remote = !isChanged; } - return await this.collection.add(remoteNotebook); + return remoteNotebook; } async add( diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.ts index 8eaff927e..dcbb2ff57 100644 --- a/packages/core/src/collections/notes.ts +++ b/packages/core/src/collections/notes.ts @@ -27,13 +27,7 @@ import { Tiptap } from "../content-types/tiptap"; import { EMPTY_CONTENT, isUnencryptedContent } from "./content"; import { CHECK_IDS, checkIsUserPremium } from "../common"; import { buildFromTemplate } from "../utils/templates"; -import { - Note, - TrashOrItem, - isTrashItem, - MaybeDeletedItem, - isDeleted -} from "../types"; +import { Note, TrashOrItem, isTrashItem, isDeleted } from "../types"; import Database from "../api"; import { CachedCollection } from "../database/cached-collection"; import { ICollection } from "./collection"; @@ -67,17 +61,6 @@ export class Notes implements ICollection { this.topicReferences.rebuild(); } - async merge(remoteNote: MaybeDeletedItem>) { - if (!remoteNote) return; - - const id = remoteNote.id; - const localNote = this.collection.get(id); - - if (localNote && localNote.localOnly) return; - - return await this.collection.add(remoteNote); - } - async add( item: Partial; sessionId: string }> ): Promise { @@ -430,6 +413,9 @@ export class Notes implements ICollection { this.topicReferences.rebuild(); } + /** + * @internal + */ async _clearAllNotebookReferences(notebookId: string) { const notes = this.db.notes.all; @@ -482,7 +468,7 @@ class NoteIdCache { for (const note of notes) { const { notebooks } = note; - if (!notebooks) continue; + if (!notebooks) return; for (const notebook of notebooks) { for (const topic of notebook.topics) { diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.ts index b5050ed56..9ba70a8c2 100644 --- a/packages/core/src/collections/relations.ts +++ b/packages/core/src/collections/relations.ts @@ -29,7 +29,7 @@ type RelationsArray = Relation[] & { export class Relations implements ICollection { name = "relations"; - private readonly collection: CachedCollection<"relations", Relation>; + readonly collection: CachedCollection<"relations", Relation>; constructor(private readonly db: Database) { this.collection = new CachedCollection( db.storage, diff --git a/packages/core/src/collections/reminders.ts b/packages/core/src/collections/reminders.ts index dc78388a9..0f75b5a1d 100644 --- a/packages/core/src/collections/reminders.ts +++ b/packages/core/src/collections/reminders.ts @@ -36,7 +36,7 @@ dayjs.extend(isToday); export class Reminders implements ICollection { name = "reminders"; - private readonly collection: CachedCollection<"reminders", Reminder>; + readonly collection: CachedCollection<"reminders", Reminder>; constructor(private readonly db: Database) { this.collection = new CachedCollection( db.storage, diff --git a/packages/core/src/collections/session-content.ts b/packages/core/src/collections/session-content.ts index 16d5942b7..0936ab6f7 100644 --- a/packages/core/src/collections/session-content.ts +++ b/packages/core/src/collections/session-content.ts @@ -27,7 +27,7 @@ import Database from "../api"; import { ContentType, SessionContentItem, isDeleted } from "../types"; export type NoteContent = { - data: TLocked extends true ? Cipher : string; + data: TLocked extends true ? Cipher<"base64"> : string; type: ContentType; }; diff --git a/packages/core/src/collections/shortcuts.ts b/packages/core/src/collections/shortcuts.ts index ecd4a8ea7..e032e71aa 100644 --- a/packages/core/src/collections/shortcuts.ts +++ b/packages/core/src/collections/shortcuts.ts @@ -25,7 +25,7 @@ import { ICollection } from "./collection"; const ALLOWED_SHORTCUT_TYPES = ["notebook", "topic", "tag"]; export class Shortcuts implements ICollection { name = "shortcuts"; - private readonly collection: CachedCollection<"shortcuts", Shortcut>; + readonly collection: CachedCollection<"shortcuts", Shortcut>; constructor(private readonly db: Database) { this.collection = new CachedCollection( db.storage, diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.ts index 730c4fb07..92b90ba56 100644 --- a/packages/core/src/collections/tags.ts +++ b/packages/core/src/collections/tags.ts @@ -25,7 +25,7 @@ import { ICollection } from "./collection"; export class Tags implements ICollection { name = "tags"; - private readonly collection: CachedCollection<"tags", Tag>; + readonly collection: CachedCollection<"tags", Tag>; constructor(private readonly db: Database) { this.collection = new CachedCollection(db.storage, "tags", db.eventManager); } diff --git a/packages/core/src/database/cached-collection.ts b/packages/core/src/database/cached-collection.ts index 12a85df87..1c6a13726 100644 --- a/packages/core/src/database/cached-collection.ts +++ b/packages/core/src/database/cached-collection.ts @@ -28,7 +28,7 @@ import { } from "../types"; import { StorageAccessor } from "../interfaces"; import EventManager from "../utils/event-manager"; -import { toChunks } from "../utils/array"; +import { chunkedIterate } from "../utils/array"; export class CachedCollection< TCollectionType extends CollectionType, @@ -109,6 +109,11 @@ export class CachedCollection< return item; } + getRaw(id: string) { + const item = this.cache.get(id); + return item; + } + raw() { return Array.from(this.cache.values()); } @@ -140,10 +145,7 @@ export class CachedCollection< } *iterateSync(chunkSize: number) { - const chunks = toChunks(Array.from(this.cache.values()), chunkSize); - for (const chunk of chunks) { - yield chunk; - } + yield* chunkedIterate(Array.from(this.cache.values()), chunkSize); } invalidateCache() { diff --git a/packages/core/src/database/crypto.ts b/packages/core/src/database/crypto.ts index 1128e0b91..4673c2d51 100644 --- a/packages/core/src/database/crypto.ts +++ b/packages/core/src/database/crypto.ts @@ -31,7 +31,7 @@ export class Crypto { } } -export function isCipher(item: any): item is Cipher { +export function isCipher(item: any): item is Cipher<"base64"> { return ( typeof item === "object" && "cipher" in item && diff --git a/packages/core/src/database/indexed-collection.ts b/packages/core/src/database/indexed-collection.ts index ac9b1c151..99168e291 100644 --- a/packages/core/src/database/indexed-collection.ts +++ b/packages/core/src/database/indexed-collection.ts @@ -108,7 +108,7 @@ export class IndexedCollection< return Object.fromEntries(data); } - setItems(items) { + setItems(items: (MaybeDeletedItem | undefined)[]) { const entries = items.reduce((array, item) => { if (!item) return array; @@ -120,7 +120,7 @@ export class IndexedCollection< array.push([item.id, item]); return array; - }, []); + }, [] as [string, MaybeDeletedItem][]); return this.indexer.writeMulti(entries); } diff --git a/packages/core/src/database/indexer.ts b/packages/core/src/database/indexer.ts index 8625b083b..8d8813ad5 100644 --- a/packages/core/src/database/indexer.ts +++ b/packages/core/src/database/indexer.ts @@ -86,13 +86,15 @@ export default class Indexer { * @param {any[]} items * @returns */ - async writeMulti(items) { - const entries = items.map(([id, item]) => { - if (!this.indices.includes(id)) this.indices.push(id); - return [this.makeId(id), item]; - }); + async writeMulti(items: [string, MaybeDeletedItem][]) { + const entries: [string, MaybeDeletedItem | string[]][] = items.map( + ([id, item]) => { + if (!this.indices.includes(id)) this.indices.push(id); + return [this.makeId(id), item]; + } + ); entries.push([this.type, this.indices]); - await super.writeMulti(entries); + await this.storage().writeMulti(entries); } async migrateIndices() { diff --git a/packages/core/src/utils/array.js b/packages/core/src/utils/array.ts similarity index 64% rename from packages/core/src/utils/array.js rename to packages/core/src/utils/array.ts index 4c92b0cdb..9f20fd5b0 100644 --- a/packages/core/src/utils/array.js +++ b/packages/core/src/utils/array.ts @@ -17,48 +17,58 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -export function findItemAndDelete(array, predicate) { +export function findItemAndDelete( + array: T[], + predicate: (item: T) => boolean +) { return deleteAtIndex(array, array.findIndex(predicate)); } -export function addItem(array, item) { +export function addItem(array: T[], item: T) { const index = array.indexOf(item); if (index > -1) return false; array.push(item); return true; } -export function deleteItem(array, item) { +export function deleteItem(array: T[], item: T) { return deleteAtIndex(array, array.indexOf(item)); } -export function deleteItems(array, ...items) { - for (let item of items) { +export function deleteItems(array: T[], ...items: T[]) { + for (const item of items) { deleteItem(array, item); } } -export function findById(array, id) { +export function findById(array: T[], id: string) { if (!array) return false; return array.find((item) => item.id === id); } -export function hasItem(array, item) { +export function hasItem(array: T[], item: T) { if (!array) return false; return array.indexOf(item) > -1; } -function deleteAtIndex(array, index) { +function deleteAtIndex(array: T[], index: number) { if (index === -1) return false; array.splice(index, 1); return true; } -export function toChunks(array, chunkSize) { - let chunks = []; +export function toChunks(array: T[], chunkSize: number) { + const chunks: T[][] = []; for (let i = 0; i < array.length; i += chunkSize) { const chunk = array.slice(i, i + chunkSize); chunks.push(chunk); } return chunks; } + +export function* chunkedIterate(array: T[], chunkSize: number) { + for (let i = 0; i < array.length; i += chunkSize) { + const chunk = array.slice(i, i + chunkSize); + yield chunk; + } +}