Mirrors/notesnook
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook.git synced 2025-12-23 06:59:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

sync: prepare -> collector, other optimizations

Browse Source
This commit is contained in:
thecodrr
2020-04-16 03:04:44 +05:00
parent 1828728f77
commit 8ad2bc5457
9 changed files with 119 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
packages/core/api/conflicts.js
View File

@@ -1,16 +0,0 @@
class Conflicts {
/**
*
* @param {import('./index').default} db
*/
constructor(db) {
this._db = db;
}
async recalculate() {
if (this._db.notes.conflicted.length <= 0) {
await this._db.context.write("hasConflicts", false);
}
}
}
export default Conflicts;

2
packages/core/api/index.js
View File

@@ -8,7 +8,7 @@ import Sync from "./sync";
import Vault from "./vault";
import Lookup from "./lookup";
import Content from "../collections/content";
import Conflicts from "./conflicts";
import Conflicts from "./sync/conflicts";
class Database {
constructor(context) {

10
packages/core/api/sync/__tests__/prepare.test.js → packages/core/api/sync/__tests__/collector.test.js
View File

@@ -1,5 +1,5 @@
import StorageInterface from "../../../__mocks__/storage.mock";
import Prepare from "../prepare";
import Collector from "../collector";
import { enableFetchMocks, disableFetchMocks } from "jest-fetch-mock";
import {
noteTest,
@@ -73,8 +73,8 @@ describe.each(tests)("%s preparation", (collection, add, addExtra) => {
.fill(0)
.map(() => add(db, collection))
);
const prepare = new Prepare(db);
const data = await prepare.get(0);
const collector = new Collector(db);
const data = await collector.collect(0);
expect(data[collection].length).toBe(MAX_ITEMS);
expect(
data[collection].every((item) => !!item.iv && !!item.cipher)
@@ -91,8 +91,8 @@ describe.each(tests)("%s preparation", (collection, add, addExtra) => {
.map(() => add(db, collection))
);
await addExtra(db, collection);
const prepare = new Prepare(db);
const data = await prepare.get(10);
const collector = new Collector(db);
const data = await collector.collect(10);
expect(data[collection].length).toBe(MAX_ITEMS);
expect(
data[collection].every((item) => !!item.iv && !!item.cipher)

36
packages/core/api/sync/__tests__/merger.test.js
View File

@@ -56,13 +56,16 @@ describe.each(tests)(
test(`merge ${collection} into empty database`, () =>
databaseTest().then(async (db) => {
await login(db);
const merger = new Merger(db, 0);
const result = await merger.merge({
const merger = new Merger(db);
const result = await merger.merge(
{
[collection]: [
{ id: testItem.id, ...(await getEncrypted(testItem)) },
],
synced: false,
});
},
0
);
expect(result).toBe(true);
expect(db[collection].all[0].id).toStrictEqual(testItem.id);
expect(db[collection].all[0].dateEdited).toStrictEqual(
@@ -73,13 +76,16 @@ describe.each(tests)(
test(`merge local and remote ${collection}`, () =>
databaseTest().then(async (db) => {
await login(db);
const merger = new Merger(db, 0);
const merger = new Merger(db);
const item = await add(db);
item.title = "Google";
const result = await merger.merge({
const result = await merger.merge(
{
[collection]: [{ id: item.id, ...(await getEncrypted(item)) }],
synced: false,
});
},
0
);
expect(result).toBe(true);
expect(db[collection].all.length).toBe(1);
expect(db[collection].all[0]).toStrictEqual(item);
@@ -88,14 +94,17 @@ describe.each(tests)(
test(`local ${collection} are more updated than remote ones`, () =>
databaseTest().then(async (db) => {
await login(db);
const merger = new Merger(db, 0);
const merger = new Merger(db);
const item = await add(db);
await edit(db, item);
item.title = "Google";
const result = await merger.merge({
const result = await merger.merge(
{
[collection]: [{ id: item.id, ...(await getEncrypted(item)) }],
synced: false,
});
},
0
);
expect(result).toBe(true);
expect(db[collection].all.length).toBe(1);
expect(db[collection].all[0]).toStrictEqual(get(db, item));
@@ -111,8 +120,9 @@ test("local delta updated after lastSyncedTimestamp should cause merge conflict"
delta: { ops: [{ insert: "my name is abdullah" }] },
};
const deltaId = db.notes.note(id).data.content.delta;
const merger = new Merger(db, 200);
const result = await merger.merge({
const merger = new Merger(db);
const result = await merger.merge(
{
delta: [
{
id: deltaId,
@@ -126,7 +136,9 @@ test("local delta updated after lastSyncedTimestamp should cause merge conflict"
})),
},
],
});
},
200
);
const localDelta = await db.delta.raw(deltaId);
expect(localDelta.conflicted.id).toBe(deltaId);
expect(localDelta.conflicted.noteId).toBe(id);

2
packages/core/api/sync/__tests__/sync.test.js
View File

@@ -2,7 +2,7 @@
import { enableFetchMocks, disableFetchMocks } from "jest-fetch-mock";
import StorageInterface from "../../../__mocks__/storage.mock";
//import Sync from "../sync";
//import Prepare from "../prepare";
//import Collector from "../prepare";
import { databaseTest, TEST_NOTE } from "../../../__tests__/utils";
import { login, getEncrypted } from "./utils";

22
packages/core/api/sync/prepare.js → packages/core/api/sync/collector.js
View File

@@ -4,7 +4,7 @@ if (!tfun) {
tfun = global.tfun;
}
class Prepare {
class Collector {
/**
*
* @param {Database} db
@@ -13,17 +13,17 @@ class Prepare {
this._db = db;
}
async get(lastSyncedTimestamp) {
async collect(lastSyncedTimestamp) {
this._lastSyncedTimestamp = lastSyncedTimestamp;
this.key = await this._db.user.key();
return {
notes: await this._prepareForServer(this._db.notes.raw),
notebooks: await this._prepareForServer(this._db.notebooks.raw),
delta: await this._prepareForServer(await this._db.delta.all()),
text: await this._prepareForServer(await this._db.text.all()),
tags: await this._prepareForServer(this._db.tags.raw),
colors: await this._prepareForServer(this._db.colors.raw),
trash: await this._prepareForServer(this._db.trash.raw),
notes: await this._collect(this._db.notes.raw),
notebooks: await this._collect(this._db.notebooks.raw),
delta: await this._collect(await this._db.delta.all()),
text: await this._collect(await this._db.text.all()),
tags: await this._collect(this._db.tags.raw),
colors: await this._collect(this._db.colors.raw),
trash: await this._collect(this._db.trash.raw),
vaultKey: await this._serialize(await this._db.vault._getKey()),
};
}
@@ -33,7 +33,7 @@ class Prepare {
return this._db.context.encrypt(this.key, JSON.stringify(item));
}
_prepareForServer(array) {
_collect(array) {
return Promise.all(
tfun
.filter((item) => item.dateEdited > this._lastSyncedTimestamp)
@@ -49,4 +49,4 @@ class Prepare {
);
}
}
export default Prepare;
export default Collector;

27
packages/core/api/sync/conflicts.js Normal file
View File

@@ -0,0 +1,27 @@
class Conflicts {
/**
*
* @param {import('../index').default} db
*/
constructor(db) {
this._db = db;
}
async recalculate() {
if (this._db.notes.conflicted.length <= 0) {
await this._db.context.write("hasConflicts", false);
}
}
async check() {
let hasConflicts = await this._db.context.read("hasConflicts");
if (hasConflicts) {
const mergeConflictError = new Error(
"Merge conflicts detected. Please resolve all conflicts to continue syncing."
);
mergeConflictError.code = "MERGE_CONFLICT";
throw mergeConflictError;
}
}
}
export default Conflicts;

68
packages/core/api/sync/index.js
View File

@@ -1,6 +1,6 @@
/**
* GENERAL PROCESS:
* make a get request to server with current lastSyncedTimestamp
* make a get request to server with current lastSynced
* parse the response. the response should contain everything that user has on the server
* decrypt the response
* merge everything into the database and look for conflicts
@@ -10,12 +10,12 @@
/**
* MERGING:
* Locally, get everything that was editted/added after the lastSyncedTimestamp
* Locally, get everything that was editted/added after the lastSynced
* Run forEach loop on the server response.
* Add items that do not exist in the local collections
* Remove items (without asking) that need to be removed
* Update items that were editted before the lastSyncedTimestamp
* Try to merge items that were edited after the lastSyncedTimestamp
* Update items that were editted before the lastSynced
* Try to merge items that were edited after the lastSynced
* Items in which the content has changed, send them for conflict resolution
* Otherwise, keep the most recently updated copy.
*/
@@ -25,9 +25,8 @@
* Syncing should pause until all the conflicts have been resolved
* And then it should continue.
*/
import Database from "../index";
import { HOST, HEADERS } from "../../utils/constants";
import Prepare from "./prepare";
import Collector from "./collector";
import Merger from "./merger";
import { areAllEmpty } from "./utils";
var tfun = require("transfun/transfun.js").tfun;
@@ -38,63 +37,50 @@ if (!tfun) {
export default class Sync {
/**
*
* @param {Database} db
* @param {import("../index").default} db
*/
constructor(db) {
this.db = db;
this._db = db;
this._collector = new Collector(this._db);
this._merger = new Merger(this._db);
}
async _fetch(lastSyncedTimestamp) {
let token = await this.db.user.token();
if (!token) throw new Error("You are not logged in");
let response = await fetch(`${HOST}sync?lst=${lastSyncedTimestamp}`, {
async _fetch(lastSynced, token) {
let response = await fetch(`${HOST}sync?lst=${lastSynced}`, {
headers: { ...HEADERS, Authorization: `Bearer ${token}` },
});
//TODO decrypt the response.
return await response.json();
}
async throwOnConflicts() {
let hasConflicts = await this.db.context.read("hasConflicts");
if (hasConflicts) {
const mergeConflictError = new Error(
"Merge conflicts detected. Please resolve all conflicts to continue syncing."
);
mergeConflictError.code = "MERGE_CONFLICT";
throw mergeConflictError;
}
}
async start() {
let user = await this.db.user.get();
if (!user) throw new Error("You need to login to sync.");
let user = await this._db.user.get();
let token = await this._db.user.token();
if (!user || !token) throw new Error("You need to login to sync.");
await this.db.conflicts.recalculate();
await this.throwOnConflicts();
// update the conflicts status and if find any, throw
await this._db.conflicts.recalculate();
await this._db.conflicts.check();
let lastSyncedTimestamp = user.lastSynced || 0;
let serverResponse = await this._fetch(lastSyncedTimestamp);
let lastSynced = user.lastSynced || 0;
let serverResponse = await this._fetch(lastSynced, token);
// we prepare local data before merging so we always have correct data
const prepare = new Prepare(this.db, user);
const data = await prepare.get(lastSyncedTimestamp);
const data = await this._collector.collect(lastSynced);
// merge the server response
const merger = new Merger(this.db, lastSyncedTimestamp);
await merger.merge(serverResponse);
await this.throwOnConflicts();
await this._merger.merge(serverResponse, lastSynced);
// check for conflicts and throw
await this._db.conflicts.check();
// send the data back to server
const lastSynced = await this._send(data);
lastSynced = await this._send(data, token);
// update our lastSynced time
if (lastSynced) await this.db.user.set({ lastSynced });
if (lastSynced) await this._db.user.set({ lastSynced });
}
async _send(data) {
//TODO encrypt the payload
let token = await this.db.user.token();
if (!token) return;
async _send(data, token) {
let response = await fetch(`${HOST}sync`, {
method: "POST",
headers: { ...HEADERS, Authorization: `Bearer ${token}` },

6
packages/core/api/sync/merger.js
View File

@@ -6,9 +6,8 @@ class Merger {
*
* @param {Database} db
*/
constructor(db, lastSynced) {
constructor(db) {
this._db = db;
this._lastSynced = lastSynced;
}
async _deserialize(item) {
@@ -59,8 +58,9 @@ class Merger {
);
}
async merge(serverResponse) {
async merge(serverResponse, lastSynced) {
if (!serverResponse) return false;
this._lastSynced = lastSynced;
const {
notes,
synced,