diff --git a/packages/core/api/index.js b/packages/core/api/index.js index e4744b654..5ea6cdb7d 100644 --- a/packages/core/api/index.js +++ b/packages/core/api/index.js @@ -2,6 +2,7 @@ import Notes from "../collections/notes"; import Notebooks from "../collections/notebooks"; import Trash from "../collections/trash"; import User from "../models/user"; +import Sync from "./sync"; class Database { constructor(context) { @@ -15,6 +16,11 @@ class Database { await this.notes.init(this.notebooks, this.trash); await this.notebooks.init(this.notes, this.trash); await this.trash.init(this.notes, this.notebooks); + this.syncer = new Sync(this); + } + + sync() { + return this.syncer.start(); } } diff --git a/packages/core/api/sync.js b/packages/core/api/sync.js new file mode 100644 index 000000000..c7e8a23b8 --- /dev/null +++ b/packages/core/api/sync.js @@ -0,0 +1,96 @@ +/** + * GENERAL PROCESS: + * make a get request to server with current lastSyncedTimestamp + * parse the response. the response should contain everything that user has on the server + * decrypt the response + * merge everything into the database and look for conflicts + * send the conflicts (if any) to the end-user for resolution + * once the conflicts have been resolved, send the updated data back to the server + */ + +/** + * MERGING: + * Locally, get everything that was editted/added after the lastSyncedTimestamp + * Run forEach loop on the server response. + * Add items that do not exist in the local collections + * Remove items (without asking) that need to be removed + * Update items that were editted before the lastSyncedTimestamp + * Try to merge items that were edited after the lastSyncedTimestamp + * Items in which the content has changed, send them for conflict resolution + * Otherwise, keep the most recently updated copy. + */ + +/** + * CONFLICTS: + * Syncing should pause until all the conflicts have been resolved + * And then it should continue. + */ +import Database from "./index"; +import { HOST } from "../utils/constants"; +import fetch from "node-fetch"; + +export default class Sync { + /** + * + * @param {Database} db + */ + constructor(db) { + this.db = db; + } + + async _fetch(lastSyncedTimestamp) { + let token = await this.db.user.token(); + if (!token) return; + let response = await fetch(`${HOST}sync?lst=${lastSyncedTimestamp}`, { + headers: { ...HEADERS, Authorization: `Bearer ${token}` } + }); + //TODO decrypt the response. + return await response.json(); + } + + async start() { + let user = await this.db.user.get(); + if (!user) return false; + let lastSyncedTimestamp = user.lastSyncedTimestamp || 0; + let serverResponse = await this._fetch(lastSyncedTimestamp); + let data = this._merge(serverResponse, lastSyncedTimestamp); + await this._send(data); + return true; + } + + _merge(serverResponse, lastSyncedTimestamp) { + const { notes, notebooks /* tags, colors, trash */ } = serverResponse; + notes.forEach(async note => { + let localNote = this.db.notes.note(note.id); + if (!localNote || note.dateEdited > localNote.data.dateEdited) { + await this.db.notes.add({ ...note, remote: true }); + } + }); + notebooks.forEach(async nb => { + let localNb = this.db.notebooks.notebook(nb.id); + if (!localNb || nb.dateEdited > localNb.data.dateEdited) { + await this.db.notebooks.add({ ...nb, remote: true }); + } + }); + // TODO trash, colors, tags + return { + notes: this.db.notes.filter(v => v.dateEdited > lastSyncedTimestamp), + notebooks: this.db.notebooks.filter( + v => v.dateEdited > lastSyncedTimestamp + ), + tags: [], + colors: [], + tags: [] + }; + } + + async _send(data) { + //TODO encrypt the payload + let response = await fetch(`${HOST}sync`, { + method: "POST", + headers: { ...HEADERS, Authorization: `Bearer ${token}` }, + body: JSON.stringify(data) + }); + return response.ok; + } +} diff --git a/packages/core/collections/notebooks.js b/packages/core/collections/notebooks.js index de6d467b3..4a435cec6 100644 --- a/packages/core/collections/notebooks.js +++ b/packages/core/collections/notebooks.js @@ -91,8 +91,8 @@ export default class Notebooks { filter(query) { if (!query) return []; - return tfun.filter(v => fuzzysearch(query, v.title + " " + v.description))( - this.all - ); + let queryFn = v => fuzzysearch(query, v.title + " " + v.description); + if (query instanceof Function) queryFn = query; + return tfun.filter(queryFn)(this.all); } } diff --git a/packages/core/collections/notes.js b/packages/core/collections/notes.js index 9d1f2f825..104b3fa59 100644 --- a/packages/core/collections/notes.js +++ b/packages/core/collections/notes.js @@ -117,9 +117,9 @@ export default class Notes { filter(query) { if (!query) return []; - return tfun.filter(v => fuzzysearch(query, v.title + " " + v.content.text))( - this.all - ); + let queryFn = v => fuzzysearch(query, v.title + " " + v.content.text); + if (query instanceof Function) queryFn = query; + return tfun.filter(queryFn)(this.all); } group(by, special = false) { diff --git a/packages/core/database/cached-collection.js b/packages/core/database/cached-collection.js index dfa8beb55..adfef6fac 100644 --- a/packages/core/database/cached-collection.js +++ b/packages/core/database/cached-collection.js @@ -30,7 +30,7 @@ export default class CachedCollection { let exists = this.map.has(item.id); await this.updateItem(item); if (!exists) { - item.dateCreated = Date.now(); + item.dateCreated = item.dateCreated || Date.now(); await this.indexer.index(item.id); } } @@ -38,7 +38,10 @@ export default class CachedCollection { async updateItem(item) { if (this.transactionOpen) return; if (!item.id) throw new Error("The item must contain the id field."); - item.dateEdited = Date.now(); + // if item is newly synced, remote will be true. + item.dateEdited = item.remote ? item.dateEdited : Date.now(); + // the item has become local now, so remove the flag. + delete item.remote; this.map.set(item.id, item); await this.indexer.write(item.id, item); } diff --git a/packages/core/models/user.js b/packages/core/models/user.js index 960e2c424..bb92f90bc 100644 --- a/packages/core/models/user.js +++ b/packages/core/models/user.js @@ -11,7 +11,7 @@ export default class User { this.context = context; } - async user() { + async get() { return this.context.read("user"); } @@ -25,11 +25,11 @@ export default class User { await this.context.write("user", user); } - async refreshToken() { + async token() { let user = await this.user(); - if (!user) return false; + if (!user) return; if (user.expiry < Date.now()) { - return true; + return user.accessToken; } let response = await authRequest("oauth/token", { refresh_token: user.refreshToken, @@ -44,7 +44,7 @@ export default class User { expiry: dt.getTime() }; await this.context.write("user", user); - return true; + return user.accessToken; } logout() { diff --git a/packages/core/package.json b/packages/core/package.json index d904d5814..91a0733d1 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -10,7 +10,8 @@ "babel-jest": "^24.9.0", "babel-polyfill": "^6.26.0", "babel-preset-env": "^1.7.0", - "jest": "^24.9.0" + "jest": "^24.9.0", + "node-fetch": "^2.6.0" }, "scripts": { "test": "jest" @@ -18,9 +19,7 @@ "dependencies": { "fast-sort": "^2.0.1", "fuzzysearch": "^1.0.3", - "node-fetch": "^2.6.0", "qclone": "^1.0.4", - "teleport-javascript": "^1.0.0", "transfun": "^1.0.2" } } diff --git a/packages/core/yarn.lock b/packages/core/yarn.lock index 602404bb0..dd3e5b3bb 100644 --- a/packages/core/yarn.lock +++ b/packages/core/yarn.lock @@ -4437,11 +4437,6 @@ tar@^4: safe-buffer "^5.1.2" yallist "^3.0.3" -teleport-javascript@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/teleport-javascript/-/teleport-javascript-1.0.0.tgz#c9397fad598d662027e4d3a5fa7e7da1c8361547" - integrity sha512-j1llvWVFyEn/6XIFDfX5LAU43DXe0GCt3NfXDwJ8XpRRMkS+i50SAkonAONBy+vxwPFBd50MFU8a2uj8R/ccLg== - test-exclude@^5.2.3: version "5.2.3" resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-5.2.3.tgz#c3d3e1e311eb7ee405e092dac10aefd09091eac0"