Mirrors/notesnook
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook.git synced 2026-09-01 19:49:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

core: fix types & syncing issues

Browse Source
This commit is contained in:
Abdullah Atta
2023-09-11 11:16:09 +05:00
parent c33ef6f70f
commit f73c02a1db
23 changed files with 517 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
packages/core/__e2e__/sync.test.js
View File

@@ -27,7 +27,7 @@ import { delay } from "../__tests__/utils";
import { test, expect, vitest } from "vitest";
import { login } from "./utils";
const TEST_TIMEOUT = 30 * 1000;
const TEST_TIMEOUT = 60 * 1000;
test(
"case 1: device A & B should only download the changes from device C (no uploading)",
@@ -311,12 +311,16 @@ test(
expect(deviceA.notebooks.topics(id).has("Topic 1")).toBeTruthy();
expect(deviceB.notebooks.topics(id).has("Topic 2")).toBeTruthy();
expect(
deviceB.notebooks.topics(id).topic("Topic 2").dateModified >
deviceA.notebooks.topics(id).topic("Topic 1").dateModified
).toBeTruthy();
expect(
deviceB.notebooks.notebook(id).dateModified >
deviceA.notebooks.notebook(id).dateModified
).toBeTruthy();
await syncAndWait(deviceA, deviceB, false);
// await delay(1000);
// await syncAndWait(deviceB, deviceB, false);
await syncAndWait(deviceB, deviceA, false);
expect(deviceA.notebooks.topics(id).has("Topic 1")).toBeTruthy();
expect(deviceB.notebooks.topics(id).has("Topic 1")).toBeTruthy();
@@ -360,15 +364,27 @@ test(
expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(1);
await syncAndWait(deviceB, deviceA, false);
await syncAndWait(deviceA, deviceB, false);
ctx.onTestFailed(() => {
console.log(deviceA.notes.topicReferences.get(topic.id), noteA);
console.log(deviceB.notes.topicReferences.get(topic.id), noteB);
expect(deviceA.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);
expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);
deviceB.notes.topicReferences.rebuild();
deviceA.notes.topicReferences.rebuild();
console.log(deviceA.notes.topicReferences.get(topic.id), noteA);
console.log(deviceB.notes.topicReferences.get(topic.id), noteB);
});
await syncAndWait(deviceB, deviceA, false);
expect(deviceA.notes.note(noteB)).toBeDefined();
expect(deviceB.notes.note(noteA)).toBeDefined();
expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(1);
expect(deviceA.notes.note(noteB).data.notebooks).toHaveLength(1);
expect(deviceA.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);
expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);
await cleanup(deviceA, deviceB);
},
TEST_TIMEOUT
@@ -435,12 +451,22 @@ async function cleanup(...devices) {
* @returns
*/
function syncAndWait(deviceA, deviceB, force = false) {
return new Promise((resolve) => {
return new Promise((resolve, reject) => {
const ref2 = deviceB.eventManager.subscribe(
EVENTS.databaseSyncRequested,
(full, force, lastSynced) => {
console.log("sync requested by device A", full, force, lastSynced);
ref2.unsubscribe();
deviceB.sync(full, force, lastSynced).catch(reject);
}
);
const ref = deviceB.eventManager.subscribe(EVENTS.syncCompleted, () => {
ref.unsubscribe();
console.log("sync completed.");
resolve();
});
console.log(
"waiting for sync...",
"Device A:",
@@ -448,6 +474,7 @@ function syncAndWait(deviceA, deviceB, force = false) {
"Device B:",
deviceB.syncer.sync.syncing
);
deviceA.sync(true, force);
deviceA.sync(true, force).catch(reject);
});
}

4
packages/core/__tests__/trash.test.js
View File

@@ -45,7 +45,7 @@ test("permanently delete a note", () =>
expect(await note.content()).toBeDefined();
await db.trash.delete(db.trash.all[0].id);
expect(db.trash.all).toHaveLength(0);
const content = await db.content.raw(note.data.contentId);
const content = await db.content.get(note.data.contentId);
expect(content).toBeUndefined();
sessions = await db.noteHistory.get(noteId);
@@ -229,7 +229,7 @@ test("clear trash should delete note content", () =>
expect(db.trash.all).toHaveLength(0);
const content = await db.content.raw(note.contentId);
const content = await db.content.get(note.contentId);
expect(content).toBeUndefined();
sessions = await db.noteHistory.get(note.id);

210
packages/core/src/api/sync/collector.ts
View File

@@ -19,103 +19,131 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { Cipher, SerializedKey } from "@notesnook/crypto";
import Database from "..";
import { CURRENT_DATABASE_VERSION } from "../../common";
import { CURRENT_DATABASE_VERSION, EV, EVENTS } from "../../common";
import { logger } from "../../logger";
import { SYNC_COLLECTIONS_MAP, SyncItem, SyncTransferItem } from "./types";
import { Item, MaybeDeletedItem } from "../../types";
export type SyncableItemType =
| "note"
| "shortcut"
| "notebook"
| "content"
| "attachment"
| "reminder"
| "relation"
| "color"
| "tag"
| "settings";
export type CollectedResult = {
items: (MaybeDeletedItem<Item> | Cipher)[];
types: (SyncableItemType | "vaultKey")[];
};
export type SyncItem = {
id: string;
v: number;
} & Cipher;
const ASYNC_COLLECTIONS_MAP = {
content: "content"
} as const;
class Collector {
private lastSyncedTimestamp = 0;
private key?: SerializedKey;
logger = logger.scope("SyncCollector");
constructor(private readonly db: Database) {}
async collect(lastSyncedTimestamp: number, isForceSync?: boolean) {
await this.db.notes.init();
this.lastSyncedTimestamp = lastSyncedTimestamp;
this.key = await this.db.user.getEncryptionKey();
const vaultKey = await this.db.vault.getKey();
const collections = {
note: this.db.notes.raw,
shortcut: this.db.shortcuts.raw,
notebook: this.db.notebooks.raw,
content: await this.db.content.all(),
attachment: this.db.attachments.syncable,
reminder: this.db.reminders.raw,
relation: this.db.relations.raw,
color: this.db.colors.raw,
tag: this.db.tags.raw,
settings: [this.db.settings.raw]
};
const result: CollectedResult = {
items: [],
types: []
};
for (const type in collections) {
this.collectInternal(
type as SyncableItemType,
collections[type as SyncableItemType],
result,
isForceSync
);
async *collect(
chunkSize: number,
lastSyncedTimestamp: number,
isForceSync = false
): AsyncGenerator<SyncTransferItem, void, unknown> {
const key = await this.db.user.getEncryptionKey();
if (!key || !key.key || !key.salt) {
EV.publish(EVENTS.userSessionExpired);
throw new Error("User encryption key not generated. Please relogin.");
}
if (vaultKey) {
result.items.push(vaultKey);
result.types.push("vaultKey");
const settings = await this.prepareChunk(
[this.db.settings.raw],
lastSyncedTimestamp,
isForceSync,
key
);
if (settings) yield { items: settings, type: "settings" };
const attachments = await this.prepareChunk(
this.db.attachments.syncable,
lastSyncedTimestamp,
isForceSync,
key
);
if (attachments) yield { items: attachments, type: "attachment" };
for (const itemType in ASYNC_COLLECTIONS_MAP) {
const collectionKey =
ASYNC_COLLECTIONS_MAP[itemType as keyof typeof ASYNC_COLLECTIONS_MAP];
const collection = this.db[collectionKey].collection;
for await (const chunk of collection.iterate(chunkSize)) {
const items = await this.prepareChunk(
chunk.map((item) => item[1]),
lastSyncedTimestamp,
isForceSync,
key
);
if (!items) continue;
yield { items, type: itemType as keyof typeof ASYNC_COLLECTIONS_MAP };
}
}
return result;
for (const itemType in SYNC_COLLECTIONS_MAP) {
const collectionKey =
SYNC_COLLECTIONS_MAP[itemType as keyof typeof SYNC_COLLECTIONS_MAP];
const collection = this.db[collectionKey].collection;
for (const chunk of collection.iterateSync(chunkSize)) {
const items = await this.prepareChunk(
chunk,
lastSyncedTimestamp,
isForceSync,
key
);
if (!items) continue;
yield { items, type: itemType as keyof typeof SYNC_COLLECTIONS_MAP };
}
}
}
private serialize(item: MaybeDeletedItem<Item>) {
if (!this.key) throw new Error("No encryption key found.");
return this.db.storage().encrypt(this.key, JSON.stringify(item));
}
encrypt(array: MaybeDeletedItem<Item>[]) {
if (!array.length) return [];
return Promise.all(array.map(this.map, this));
}
private collectInternal(
itemType: SyncableItemType,
items: MaybeDeletedItem<Item>[],
result: CollectedResult,
isForceSync?: boolean
async prepareChunk(
chunk: MaybeDeletedItem<Item>[],
lastSyncedTimestamp: number,
isForceSync: boolean,
key: SerializedKey
) {
if (!items || !items.length) return;
const { ids, items } = filterSyncableItems(
chunk,
lastSyncedTimestamp,
isForceSync
);
if (!ids.length) return;
const ciphers = await this.db.storage().encryptMulti(key, items);
return toPushItem(ids, ciphers);
}
}
export default Collector;
for (const item of items) {
if (!item) continue;
function toPushItem(ids: string[], ciphers: Cipher<"base64">[]) {
if (ids.length !== ciphers.length)
throw new Error("ids.length must be equal to ciphers.length");
const isSyncable = !item.synced || isForceSync;
const isUnsynced =
item.dateModified > this.lastSyncedTimestamp || isForceSync;
const items: SyncItem[] = [];
for (let i = 0; i < ids.length; ++i) {
const id = ids[i];
const cipher = ciphers[i];
items.push({ ...cipher, v: CURRENT_DATABASE_VERSION, id });
}
return items;
}
if (isUnsynced && isSyncable) {
result.items.push(
function filterSyncableItems(
items: MaybeDeletedItem<Item>[],
lastSyncedTimestamp: number,
isForceSync = false
): { items: string[]; ids: string[] } {
if (!items || !items.length) return { items: [], ids: [] };
const ids = [];
const syncableItems = [];
for (const item of items) {
if (!item) continue;
const isSyncable = !item.synced || isForceSync;
const isUnsynced = item.dateModified > lastSyncedTimestamp || isForceSync;
// synced is a local only property
delete item.synced;
if (isUnsynced && isSyncable) {
ids.push(item.id);
syncableItems.push(
JSON.stringify(
"localOnly" in item && item.localOnly
? {
id: item.id,
@@ -124,21 +152,9 @@ class Collector {
deleteReason: "localOnly"
}
: item
);
result.types.push(itemType);
}
)
);
}
}
private async map(item: MaybeDeletedItem<Item>) {
// synced is a local only property
delete item.synced;
return {
id: item.id,
v: CURRENT_DATABASE_VERSION,
...(await this.serialize(item))
};
}
return { items: syncableItems, ids };
}
export default Collector;

93
packages/core/src/api/sync/index.ts
View File

@@ -26,11 +26,7 @@ import {
} from "../../common";
import Constants from "../../utils/constants";
import TokenManager from "../token-manager";
import Collector, {
CollectedResult,
SyncableItemType,
SyncItem
} from "./collector";
import Collector from "./collector";
import * as signalr from "@microsoft/signalr";
import Merger from "./merger";
import Conflicts from "./conflicts";
@@ -41,6 +37,14 @@ import { Mutex } from "async-mutex";
import Database from "..";
import { migrateItem } from "../../migrations";
import { SerializedKey } from "@notesnook/crypto";
import {
ItemMap,
MaybeDeletedItem,
Note,
Notebook,
TrashOrItem
} from "../../types";
import { SyncableItemType, SyncTransferItem } from "./types";
const ITEM_TYPE_TO_COLLECTION_TYPE = {
note: "notes",
@@ -49,13 +53,10 @@ const ITEM_TYPE_TO_COLLECTION_TYPE = {
attachment: "attachments",
relation: "relations",
reminder: "reminders",
shortcut: "shortcuts"
};
type SyncTransferItem = {
items: SyncItem[];
type: SyncableItemType;
};
shortcut: "shortcuts",
tag: "tags",
color: "colors"
} as const;
export default class SyncManager {
sync = new Sync(this.db);
@@ -164,6 +165,11 @@ class Sync {
}, 15000) as unknown as number;
const key = await this.db.user.getEncryptionKey();
if (!key || !key.key || !key.salt) {
EV.publish(EVENTS.userSessionExpired);
throw new Error("User encryption key not generated. Please relogin.");
}
const dbLastSynced = await this.db.lastSynced();
await this.processChunk(chunk, key, dbLastSynced, true);
});
@@ -198,7 +204,7 @@ class Sync {
const serverResponse = full ? await this.fetch(lastSynced) : null;
this.logger.info("Data fetched", serverResponse || {});
if (await this.send(lastSynced, force, newLastSynced)) {
if (await this.send(lastSynced, newLastSynced, force)) {
this.logger.info("New data sent");
await this.stop(newLastSynced);
} else if (serverResponse) {
@@ -267,7 +273,7 @@ class Sync {
serverResponse.vaultKey.salt !== null &&
serverResponse.vaultKey.length > 0
) {
await this.db.vault._setKey(serverResponse.vaultKey);
await this.db.vault.setKey(serverResponse.vaultKey);
}
this.connection.off("SendItems");
@@ -279,7 +285,11 @@ class Sync {
return { lastSynced: serverResponse.lastSynced };
}
async send(oldLastSynced, isForceSync, newLastSynced) {
async send(
oldLastSynced: number,
newLastSynced: number,
isForceSync?: boolean
) {
await this.uploadAttachments();
let isSyncInitialized = false;
@@ -290,7 +300,7 @@ class Sync {
isForceSync
)) {
if (!isSyncInitialized) {
const vaultKey = await this.db.vault._getKey();
const vaultKey = await this.db.vault.getKey();
newLastSynced = await this.connection.invoke("InitializePush", {
vaultKey,
lastSynced: newLastSynced
@@ -315,7 +325,7 @@ class Sync {
return true;
}
async stop(lastSynced) {
async stop(lastSynced: number) {
// refresh topic references
this.db.notes.topicReferences.rebuild();
// refresh monographs on sync completed
@@ -353,7 +363,7 @@ class Sync {
/**
* @private
*/
async onPushCompleted(lastSynced) {
async onPushCompleted(lastSynced: number) {
// refresh topic references
this.db.notes.topicReferences.rebuild();
@@ -382,12 +392,14 @@ class Sync {
)
);
let items = [];
if (this.merger.isSyncCollection(chunk.type)) {
items = deserialized.map((item) =>
this.merger.mergeItemSync(item, chunk.type, dbLastSynced)
);
} else if (chunk.type === "content") {
const itemType = chunk.type;
let items: (
| MaybeDeletedItem<
ItemMap[SyncableItemType] | TrashOrItem<Note> | TrashOrItem<Notebook>
>
| undefined
)[] = [];
if (itemType === "content") {
const localItems = await this.db.content.multi(
chunk.items.map((i) => i.id)
);
@@ -396,28 +408,33 @@ class Sync {
this.merger.mergeContent(item, localItems[item.id], dbLastSynced)
)
);
} else if (itemType === "settings") {
await this.merger.mergeItem(deserialized[0], itemType, dbLastSynced);
return;
} else {
items = await Promise.all(
deserialized.map((item) =>
this.merger.mergeItem(item, chunk.type, dbLastSynced)
)
);
items = this.merger.isSyncCollection(itemType)
? deserialized.map((item) =>
this.merger.mergeItemSync(item, itemType, dbLastSynced)
)
: await Promise.all(
deserialized.map((item) =>
this.merger.mergeItem(item, itemType, dbLastSynced)
)
);
}
const collectionType = ITEM_TYPE_TO_COLLECTION_TYPE[itemType];
await this.db[collectionType].collection.setItems(items as any);
if (
notify &&
(chunk.type === "content" || chunk.type === "note") &&
(itemType === "note" || itemType === "content") &&
items.length > 0
) {
items.forEach((item) =>
this.db.eventManager.publish(EVENTS.syncItemMerged, item)
);
}
const collectionType = this.itemTypeToCollection[chunk.type];
if (collectionType && this.db[collectionType]) {
await this.db[collectionType]._collection.setItems(items);
}
}
private async pushItem(item: SyncTransferItem, newLastSynced: number) {
@@ -464,7 +481,11 @@ function promiseTimeout(ms: number, promise: Promise<unknown>) {
return Promise.race([promise, timeout]);
}
async function deserializeItem(decryptedItem, version, database) {
async function deserializeItem(
decryptedItem: string,
version: number,
database: Database
) {
const deserialized = JSON.parse(decryptedItem);
deserialized.remote = true;
deserialized.synced = true;

435
packages/core/src/api/sync/merger.ts
View File

@@ -17,292 +17,215 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { migrateItem } from "../../migrations";
import { set } from "../../utils/set";
import { logger } from "../../logger";
import { isHTMLEqual } from "../../utils/html-diff";
import { EV, EVENTS } from "../../common";
import Database from "..";
import { SyncItem, SyncableItemType } from "./collector";
import { Item, ItemMap, MaybeDeletedItem, isDeleted } from "../../types";
import { SerializedKey } from "@notesnook/crypto";
import { isCipher } from "../../database/crypto";
type Conflict<P extends SyncableItemType> = (
local: MaybeDeletedItem<ItemMap[P]>,
remote: MaybeDeletedItem<ItemMap[P]>
) => Promise<void>;
type Set<P extends SyncableItemType> = (
item: MaybeDeletedItem<ItemMap[P]>
) => Promise<void>;
type Get<P extends SyncableItemType> = (
id: string
) =>
| MaybeDeletedItem<ItemMap[P]>
| undefined
| Promise<MaybeDeletedItem<ItemMap[P]> | undefined>;
type MergeDefinition = {
[P in SyncableItemType]: {
threshold?: number;
get?: Get<P>;
set: Set<P>;
conflict?: Conflict<P>;
};
};
import { SYNC_COLLECTIONS_MAP } from "./types";
import {
Attachment,
ContentItem,
Item,
ItemMap,
MaybeDeletedItem,
Note,
Notebook,
SettingsItem,
TrashOrItem,
isDeleted
} from "../../types";
class Merger {
private mergeDefinition: MergeDefinition;
private logger = logger.scope("Merger");
private lastSynced = 0;
private key?: SerializedKey;
constructor(private readonly db: Database) {
this.mergeDefinition = {
settings: {
threshold: 1000,
get: () => this.db.settings.raw,
set: (item) => this.db.settings.merge(item),
conflict: (_local, remote) => this.db.settings.merge(remote)
},
note: {
get: (id) => this.db.notes.note(id)?.data,
set: (item) => this.db.notes.merge(item)
},
shortcut: {
get: (id) => this.db.shortcuts.shortcut(id),
set: (item) => this.db.shortcuts.merge(item)
},
reminder: {
get: (id) => this.db.reminders.reminder(id),
set: (item) => this.db.reminders.merge(item)
},
relation: {
get: (id) => this.db.relations.relation(id),
set: (item) => this.db.relations.merge(item)
},
tag: {
get: (id) => this.db.tags.tag(id),
set: (item) => this.db.tags.merge(item)
},
color: {
get: (id) => this.db.colors.color(id),
set: (item) => this.db.colors.merge(item)
},
notebook: {
threshold: 1000,
get: (id) => this.db.notebooks.notebook(id)?.data,
set: (item) => this.db.notebooks.merge(item),
conflict: (_local, remote) => this.db.notebooks.merge(remote)
},
content: {
threshold: process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000,
get: (id) => this.db.content.raw(id),
set: async (item) => {
await this.db.content.merge(item);
},
conflict: async (local, remote) => {
if (isDeleted(local) || isDeleted(remote)) {
if (remote.dateModified > local.dateModified)
await db.content.merge(remote);
return;
}
logger = logger.scope("Merger");
constructor(private readonly db: Database) {}
const note = this.db.notes.note(local.noteId);
if (!note || !note.data) return;
isSyncCollection(type: string): type is keyof typeof SYNC_COLLECTIONS_MAP {
return type in SYNC_COLLECTIONS_MAP;
}
// if hashes are equal do nothing
if (
!note.locked &&
(!remote ||
!local ||
!local.data ||
!remote.data ||
isHTMLEqual(local.data, remote.data))
)
return;
isConflicted(
localItem: MaybeDeletedItem<Item>,
remoteItem: MaybeDeletedItem<Item>,
lastSynced: number,
conflictThreshold: number
) {
const isResolved =
"dateResolved" in localItem &&
localItem.dateResolved === remoteItem.dateModified;
const isModified =
// the local item is modified if it was changed/modified after the last
// sync i.e. it wasn't synced yet.
// However, in case a sync is interrupted the local item's date modified
// will be ahead of last sync. In that case, we also have to check if the
// synced flag is false (it is only false if a user makes edits on the
// local device).
localItem.dateModified > lastSynced && !localItem.synced;
if (isModified && !isResolved) {
// If time difference between local item's edits & remote item's edits
// is less than threshold, we shouldn't trigger a merge conflict; instead
// we will keep the most recently changed item.
const timeDiff =
Math.max(remoteItem.dateModified, localItem.dateModified) -
Math.min(remoteItem.dateModified, localItem.dateModified);
if (note.locked) {
// if note is locked or content is deleted we keep the most recent version.
if (remote.dateModified > local.dateModified)
await this.db.content.merge({ ...remote, id: local.id });
} else {
// otherwise we trigger the conflicts
await this.db.content.merge({ ...local, conflicted: remote });
await this.db.notes.add({ id: local.noteId, conflicted: true });
await this.db.storage().write("hasConflicts", true);
}
if (timeDiff < conflictThreshold) {
if (remoteItem.dateModified > localItem.dateModified) {
return "merge";
}
},
attachment: {
set: async (remoteAttachment) => {
if (isDeleted(remoteAttachment)) {
await this.db.attachments.merge(remoteAttachment);
return;
}
return;
}
const localAttachment = this.db.attachments.attachment(
remoteAttachment.metadata.hash
return "conflict";
} else if (!isResolved) {
return "merge";
}
}
mergeItemSync<TType extends keyof typeof SYNC_COLLECTIONS_MAP>(
remoteItem: MaybeDeletedItem<
ItemMap[TType] | TrashOrItem<Note> | TrashOrItem<Notebook>
>,
type: TType,
lastSynced: number
) {
switch (type) {
case "shortcut":
case "reminder":
case "tag":
case "color":
case "note":
case "relation": {
const localItem = this.db[SYNC_COLLECTIONS_MAP[type]].collection.getRaw(
remoteItem.id
);
if (!localItem || remoteItem.dateModified > localItem.dateModified) {
return remoteItem;
}
break;
}
// case "note": {
// const localItem = this.db.notes.collection.getRaw(remoteItem.id);
// if (!localItem || remoteItem.dateModified > localItem.dateModified) {
// return this.db.notes.merge(
// localItem,
// remoteItem as MaybeDeletedItem<TrashOrItem<Note>>
// );
// }
// break;
// }
case "notebook": {
const THRESHOLD = 1000;
const localItem = this.db.notebooks.collection.getRaw(remoteItem.id);
if (
!localItem ||
this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD)
) {
return this.db.notebooks.merge(
localItem,
remoteItem as MaybeDeletedItem<TrashOrItem<Notebook>>,
lastSynced
);
if (
localAttachment &&
localAttachment.dateUploaded !== remoteAttachment.dateUploaded
) {
const noteIds = localAttachment.noteIds.slice();
const isRemoved = await this.db.attachments.remove(
localAttachment.metadata.hash,
true
);
if (!isRemoved)
throw new Error(
"Conflict could not be resolved in one of the attachments."
);
remoteAttachment.noteIds = set.union(
remoteAttachment.noteIds,
noteIds
);
}
await this.db.attachments.merge(remoteAttachment);
}
break;
}
};
}
async _migrate(deserialized: Item, version: number) {
// it is a locked note, bail out.
if (isCipher(deserialized) && deserialized.alg && deserialized.cipher)
return deserialized;
return migrateItem(deserialized, version, deserialized.type, this.db);
}
async _deserialize(item: SyncItem, migrate = true) {
if (!this.key) throw new Error("User encryption key not found.");
const decrypted = await this.db.storage().decrypt(this.key, item);
if (!decrypted) {
throw new Error("Decrypted item cannot be undefined or empty.");
}
const deserialized = JSON.parse(decrypted);
deserialized.remote = true;
deserialized.synced = true;
if (!migrate) return deserialized;
await this._migrate(deserialized, item.v);
return deserialized;
}
async _mergeItem<TItemType extends SyncableItemType>(
syncItem: SyncItem,
get: Get<TItemType>,
add: Set<TItemType>
) {
const remoteItem = (await this._deserialize(syncItem)) as MaybeDeletedItem<
ItemMap[TItemType]
>;
const localItem = await get(remoteItem.id);
if (!localItem || remoteItem.dateModified > localItem.dateModified) {
await add(remoteItem);
return remoteItem;
}
}
async _mergeItemWithConflicts<TItemType extends SyncableItemType>(
syncItem: SyncItem,
get: Get<TItemType>,
add: Set<TItemType>,
markAsConflicted: Conflict<TItemType>,
threshold: number
async mergeContent(
remoteItem: MaybeDeletedItem<ContentItem>,
localItem: MaybeDeletedItem<ContentItem>,
lastSynced: number
) {
const remoteItem = (await this._deserialize(syncItem)) as MaybeDeletedItem<
ItemMap[TItemType]
>;
const localItem = await get(remoteItem.id);
if (localItem && "localOnly" in localItem && localItem.localOnly) return;
if (!localItem || isDeleted(localItem)) {
await add(remoteItem);
const THRESHOLD = process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000;
const conflicted =
localItem &&
this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD);
if (!localItem || conflicted === "merge") {
return remoteItem;
} else {
const isResolved =
"dateResolved" in localItem &&
localItem.dateResolved === remoteItem.dateModified;
const isModified =
// the local item is modified if it was changed/modified after the last sync
// i.e. it wasn't synced yet.
// However, in case a sync is interrupted the local item's date modified will
// be ahead of last sync. In that case, we also have to check if the synced flag
// is false (it is only false if a user makes edits on the local device).
localItem.dateModified > this.lastSynced && !localItem.synced;
if (isModified && !isResolved) {
// If time difference between local item's edits & remote item's edits
// is less than threshold, we shouldn't trigger a merge conflict; instead
// we will keep the most recently changed item.
const timeDiff =
Math.max(remoteItem.dateModified, localItem.dateModified) -
Math.min(remoteItem.dateModified, localItem.dateModified);
} else if (conflicted === "conflict") {
if (isDeleted(localItem) || isDeleted(remoteItem)) {
if (remoteItem.dateModified > localItem.dateModified) return remoteItem;
return;
}
if (timeDiff < threshold) {
if (remoteItem.dateModified > localItem.dateModified) {
await add(remoteItem);
return remoteItem;
}
return;
}
const note = this.db.notes.collection.get(localItem.noteId);
if (!note) return;
this.logger.info("Conflict detected", {
itemId: remoteItem.id,
isResolved,
isModified,
timeDiff,
remote: remoteItem.dateModified,
local: localItem.dateModified,
lastSynced: this.lastSynced
// if hashes are equal do nothing
if (
!note.locked &&
(!remoteItem ||
!remoteItem ||
!localItem.data ||
!remoteItem.data ||
isHTMLEqual(localItem.data, remoteItem.data))
)
return;
if (note.locked) {
// if note is locked or content is deleted we keep the most recent version.
if (remoteItem.dateModified > localItem.dateModified) return remoteItem;
} else {
// otherwise we trigger the conflicts
await this.db.notes.add({
id: localItem.noteId,
conflicted: true
});
await markAsConflicted(localItem, remoteItem);
} else if (!isResolved) {
await add(remoteItem);
return remoteItem;
await this.db.storage().write("hasConflicts", true);
return {
...localItem,
conflicted: remoteItem
};
}
}
}
async mergeItem<TItemType extends SyncableItemType>(
type: TItemType | "vaultKey",
item: SyncItem
async mergeItem(
remoteItem: SettingsItem | MaybeDeletedItem<Attachment>,
type: "settings" | "attachment",
lastSynced: number
) {
this.lastSynced = await this.db.lastSynced();
switch (type) {
case "settings": {
if (isDeleted(remoteItem) || remoteItem.type !== "settings") return;
if (!this.key) this.key = await this.db.user.getEncryptionKey();
if (!this.key || !this.key.key || !this.key.salt) {
EV.publish(EVENTS.userSessionExpired);
throw new Error("User encryption key not generated. Please relogin.");
}
const localItem = this.db.settings.raw;
if (
!localItem ||
this.isConflicted(localItem, remoteItem, lastSynced, 1000)
) {
await this.db.settings.merge(remoteItem, lastSynced);
}
break;
}
case "attachment": {
if (isDeleted(remoteItem)) {
return this.db.attachments.merge(undefined, remoteItem);
}
if (remoteItem.type !== "attachment") return;
if (type === "vaultKey") {
await this.db.vault.setKey(await this._deserialize(item, false));
return;
}
const definition = this.mergeDefinition[type];
if (definition.conflict && definition.get && definition.threshold) {
return await this._mergeItemWithConflicts<TItemType>(
item,
definition.get,
definition.set,
definition.conflict,
definition.threshold
);
} else if (definition.get && definition.set) {
return await this._mergeItem<TItemType>(
item,
definition.get,
definition.set
);
} else if (!definition.get && !!definition.set) {
const remote = await this._deserialize(item);
await definition.set(remote);
const localAttachment = this.db.attachments.attachment(
remoteItem.metadata.hash
);
if (
localAttachment &&
localAttachment.dateUploaded !== remoteItem.dateUploaded
) {
const noteIds = localAttachment.noteIds.slice();
const isRemoved = await this.db.attachments.remove(
localAttachment.metadata.hash,
true
);
if (!isRemoved)
throw new Error(
"Conflict could not be resolved in one of the attachments."
);
remoteItem.noteIds = set.union(remoteItem.noteIds, noteIds);
}
return this.db.attachments.merge(undefined, remoteItem);
}
}
}
}

52
packages/core/src/api/sync/types.ts Normal file
View File

@@ -0,0 +1,52 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Cipher } from "@notesnook/crypto";
export type SyncableItemType =
| "note"
| "shortcut"
| "notebook"
| "content"
| "attachment"
| "reminder"
| "relation"
| "color"
| "tag"
| "settings";
export type SyncItem = {
id: string;
v: number;
} & Cipher<"base64">;
export const SYNC_COLLECTIONS_MAP = {
note: "notes",
notebook: "notebooks",
shortcut: "shortcuts",
reminder: "reminders",
relation: "relations",
tag: "tags",
color: "colors"
} as const;
export type SyncTransferItem = {
items: SyncItem[];
type: SyncableItemType;
};

2
packages/core/src/api/user-manager.ts
View File

@@ -32,7 +32,7 @@ export type User = {
email: string;
isEmailConfirmed: boolean;
salt: string;
attachmentsKey?: Cipher;
attachmentsKey?: Cipher<"base64">;
marketingConsent?: boolean;
mfa: {
isEnabled: boolean;

6
packages/core/src/api/vault.ts
View File

@@ -209,7 +209,7 @@ export default class Vault {
return await this.lockNote(note, await this.getVaultPassword());
}
async exists(vaultKey?: Cipher) {
async exists(vaultKey?: Cipher<"base64">) {
if (!vaultKey) vaultKey = await this.getKey();
return vaultKey && isCipher(vaultKey);
}
@@ -370,10 +370,10 @@ export default class Vault {
}
async getKey() {
return await this.db.storage().read<Cipher>("vaultKey");
return await this.db.storage().read<Cipher<"base64">>("vaultKey");
}
async setKey(vaultKey: Cipher) {
async setKey(vaultKey: Cipher<"base64">) {
await this.db.storage().write("vaultKey", vaultKey);
}
}

9
packages/core/src/collections/attachments.ts
View File

@@ -43,7 +43,7 @@ import Database from "../api";
export class Attachments implements ICollection {
name = "attachments";
key: Cipher<"base64"> | null = null;
private readonly collection: CachedCollection<"attachments", Attachment>;
readonly collection: CachedCollection<"attachments", Attachment>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
@@ -109,7 +109,7 @@ export class Attachments implements ICollection {
}
merge(
localAttachment: MaybeDeletedItem<Attachment>,
localAttachment: MaybeDeletedItem<Attachment> | undefined,
remoteAttachment: MaybeDeletedItem<Attachment>
) {
if (isDeleted(remoteAttachment)) return remoteAttachment;
@@ -323,7 +323,10 @@ export class Attachments implements ICollection {
return (
outputType === "base64"
? dataurl.fromObject({ type: attachment.metadata.type, data })
? dataurl.fromObject({
type: attachment.metadata.type,
data
})
: data
) as Output<TOutputFormat>;
}

2
packages/core/src/collections/colors.ts
View File

@@ -36,7 +36,7 @@ export const DefaultColors: Record<string, string> = {
export class Colors implements ICollection {
name = "colors";
private readonly collection: CachedCollection<"colors", Color>;
readonly collection: CachedCollection<"colors", Color>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,

4
packages/core/src/collections/content.ts
View File

@@ -49,7 +49,7 @@ export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({
export class Content implements ICollection {
name = "content";
private readonly collection: IndexedCollection<"content", ContentItem>;
readonly collection: IndexedCollection<"content", ContentItem>;
constructor(private readonly db: Database) {
this.collection = new IndexedCollection(
db.storage,
@@ -130,7 +130,7 @@ export class Content implements ICollection {
async get(id: string) {
const content = await this.raw(id);
if (!content || isDeleted(content)) return;
return content.data;
return content;
}
async raw(id: string) {

21
packages/core/src/collections/notebooks.ts
View File

@@ -27,7 +27,6 @@ import {
MaybeDeletedItem,
Notebook,
Topic,
TrashItem,
TrashOrItem,
isDeleted,
isTrashItem
@@ -52,15 +51,23 @@ export class Notebooks implements ICollection {
return this.collection.init();
}
async merge(remoteNotebook: MaybeDeletedItem<TrashOrItem<Notebook>>) {
merge(
localNotebook: MaybeDeletedItem<TrashOrItem<Notebook>> | undefined,
remoteNotebook: MaybeDeletedItem<TrashOrItem<Notebook>>,
lastSyncedTimestamp: number
) {
if (isDeleted(remoteNotebook) || isTrashItem(remoteNotebook))
return await this.collection.add(remoteNotebook);
return remoteNotebook;
const id = remoteNotebook.id;
const localNotebook = this.collection.get(id);
if (
localNotebook &&
(isTrashItem(localNotebook) || isDeleted(localNotebook))
) {
if (localNotebook.dateModified > remoteNotebook.dateModified) return;
return remoteNotebook;
}
if (localNotebook && localNotebook.topics?.length) {
const lastSyncedTimestamp = await this.db.lastSynced();
let isChanged = false;
// merge new and old topics
for (const oldTopic of localNotebook.topics) {
@@ -93,7 +100,7 @@ export class Notebooks implements ICollection {
}
remoteNotebook.remote = !isChanged;
}
return await this.collection.add(remoteNotebook);
return remoteNotebook;
}
async add(

24
packages/core/src/collections/notes.ts
View File

@@ -27,13 +27,7 @@ import { Tiptap } from "../content-types/tiptap";
import { EMPTY_CONTENT, isUnencryptedContent } from "./content";
import { CHECK_IDS, checkIsUserPremium } from "../common";
import { buildFromTemplate } from "../utils/templates";
import {
Note,
TrashOrItem,
isTrashItem,
MaybeDeletedItem,
isDeleted
} from "../types";
import { Note, TrashOrItem, isTrashItem, isDeleted } from "../types";
import Database from "../api";
import { CachedCollection } from "../database/cached-collection";
import { ICollection } from "./collection";
@@ -67,17 +61,6 @@ export class Notes implements ICollection {
this.topicReferences.rebuild();
}
async merge(remoteNote: MaybeDeletedItem<TrashOrItem<Note>>) {
if (!remoteNote) return;
const id = remoteNote.id;
const localNote = this.collection.get(id);
if (localNote && localNote.localOnly) return;
return await this.collection.add(remoteNote);
}
async add(
item: Partial<Note & { content: NoteContent<false>; sessionId: string }>
): Promise<string | undefined> {
@@ -430,6 +413,9 @@ export class Notes implements ICollection {
this.topicReferences.rebuild();
}
/**
* @internal
*/
async _clearAllNotebookReferences(notebookId: string) {
const notes = this.db.notes.all;
@@ -482,7 +468,7 @@ class NoteIdCache {
for (const note of notes) {
const { notebooks } = note;
if (!notebooks) continue;
if (!notebooks) return;
for (const notebook of notebooks) {
for (const topic of notebook.topics) {

2
packages/core/src/collections/relations.ts
View File

@@ -29,7 +29,7 @@ type RelationsArray<TType extends keyof ItemMap> = Relation[] & {
export class Relations implements ICollection {
name = "relations";
private readonly collection: CachedCollection<"relations", Relation>;
readonly collection: CachedCollection<"relations", Relation>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,

2
packages/core/src/collections/reminders.ts
View File

@@ -36,7 +36,7 @@ dayjs.extend(isToday);
export class Reminders implements ICollection {
name = "reminders";
private readonly collection: CachedCollection<"reminders", Reminder>;
readonly collection: CachedCollection<"reminders", Reminder>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,

2
packages/core/src/collections/session-content.ts
View File

@@ -27,7 +27,7 @@ import Database from "../api";
import { ContentType, SessionContentItem, isDeleted } from "../types";
export type NoteContent<TLocked extends boolean> = {
data: TLocked extends true ? Cipher : string;
data: TLocked extends true ? Cipher<"base64"> : string;
type: ContentType;
};

2
packages/core/src/collections/shortcuts.ts
View File

@@ -25,7 +25,7 @@ import { ICollection } from "./collection";
const ALLOWED_SHORTCUT_TYPES = ["notebook", "topic", "tag"];
export class Shortcuts implements ICollection {
name = "shortcuts";
private readonly collection: CachedCollection<"shortcuts", Shortcut>;
readonly collection: CachedCollection<"shortcuts", Shortcut>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,

2
packages/core/src/collections/tags.ts
View File

@@ -25,7 +25,7 @@ import { ICollection } from "./collection";
export class Tags implements ICollection {
name = "tags";
private readonly collection: CachedCollection<"tags", Tag>;
readonly collection: CachedCollection<"tags", Tag>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(db.storage, "tags", db.eventManager);
}

12
packages/core/src/database/cached-collection.ts
View File

@@ -28,7 +28,7 @@ import {
} from "../types";
import { StorageAccessor } from "../interfaces";
import EventManager from "../utils/event-manager";
import { toChunks } from "../utils/array";
import { chunkedIterate } from "../utils/array";
export class CachedCollection<
TCollectionType extends CollectionType,
@@ -109,6 +109,11 @@ export class CachedCollection<
return item;
}
getRaw(id: string) {
const item = this.cache.get(id);
return item;
}
raw() {
return Array.from(this.cache.values());
}
@@ -140,10 +145,7 @@ export class CachedCollection<
}
*iterateSync(chunkSize: number) {
const chunks = toChunks(Array.from(this.cache.values()), chunkSize);
for (const chunk of chunks) {
yield chunk;
}
yield* chunkedIterate(Array.from(this.cache.values()), chunkSize);
}
invalidateCache() {

2
packages/core/src/database/crypto.ts
View File

@@ -31,7 +31,7 @@ export class Crypto {
}
}
export function isCipher(item: any): item is Cipher {
export function isCipher(item: any): item is Cipher<"base64"> {
return (
typeof item === "object" &&
"cipher" in item &&

4
packages/core/src/database/indexed-collection.ts
View File

@@ -108,7 +108,7 @@ export class IndexedCollection<
return Object.fromEntries(data);
}
setItems(items) {
setItems(items: (MaybeDeletedItem<T> | undefined)[]) {
const entries = items.reduce((array, item) => {
if (!item) return array;
@@ -120,7 +120,7 @@ export class IndexedCollection<
array.push([item.id, item]);
return array;
}, []);
}, [] as [string, MaybeDeletedItem<T>][]);
return this.indexer.writeMulti(entries);
}

14
packages/core/src/database/indexer.ts
View File

@@ -86,13 +86,15 @@ export default class Indexer<T> {
* @param {any[]} items
* @returns
*/
async writeMulti(items) {
const entries = items.map(([id, item]) => {
if (!this.indices.includes(id)) this.indices.push(id);
return [this.makeId(id), item];
});
async writeMulti(items: [string, MaybeDeletedItem<T>][]) {
const entries: [string, MaybeDeletedItem<T> | string[]][] = items.map(
([id, item]) => {
if (!this.indices.includes(id)) this.indices.push(id);
return [this.makeId(id), item];
}
);
entries.push([this.type, this.indices]);
await super.writeMulti(entries);
await this.storage().writeMulti(entries);
}
async migrateIndices() {

30
packages/core/src/utils/array.js → packages/core/src/utils/array.ts
View File

@@ -17,48 +17,58 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
export function findItemAndDelete(array, predicate) {
export function findItemAndDelete<T>(
array: T[],
predicate: (item: T) => boolean
) {
return deleteAtIndex(array, array.findIndex(predicate));
}
export function addItem(array, item) {
export function addItem<T>(array: T[], item: T) {
const index = array.indexOf(item);
if (index > -1) return false;
array.push(item);
return true;
}
export function deleteItem(array, item) {
export function deleteItem<T>(array: T[], item: T) {
return deleteAtIndex(array, array.indexOf(item));
}
export function deleteItems(array, ...items) {
for (let item of items) {
export function deleteItems<T>(array: T[], ...items: T[]) {
for (const item of items) {
deleteItem(array, item);
}
}
export function findById(array, id) {
export function findById<T extends { id: string }>(array: T[], id: string) {
if (!array) return false;
return array.find((item) => item.id === id);
}
export function hasItem(array, item) {
export function hasItem<T>(array: T[], item: T) {
if (!array) return false;
return array.indexOf(item) > -1;
}
function deleteAtIndex(array, index) {
function deleteAtIndex<T>(array: T[], index: number) {
if (index === -1) return false;
array.splice(index, 1);
return true;
}
export function toChunks(array, chunkSize) {
let chunks = [];
export function toChunks<T>(array: T[], chunkSize: number) {
const chunks: T[][] = [];
for (let i = 0; i < array.length; i += chunkSize) {
const chunk = array.slice(i, i + chunkSize);
chunks.push(chunk);
}
return chunks;
}
export function* chunkedIterate<T>(array: T[], chunkSize: number) {
for (let i = 0; i < array.length; i += chunkSize) {
const chunk = array.slice(i, i + chunkSize);
yield chunk;
}
}