feat: add basic syncing (untested)

packages/core/api/index.js

@@ -2,6 +2,7 @@ import Notes from "../collections/notes";
 import Notebooks from "../collections/notebooks";
 import Trash from "../collections/trash";
 import User from "../models/user";
+import Sync from "./sync";
 
 class Database {
   constructor(context) {
@@ -15,6 +16,11 @@ class Database {
     await this.notes.init(this.notebooks, this.trash);
     await this.notebooks.init(this.notes, this.trash);
     await this.trash.init(this.notes, this.notebooks);
+    this.syncer = new Sync(this);
+  }
+
+  sync() {
+    return this.syncer.start();
   }
 }
 

packages/core/api/sync.js

@@ -0,0 +1,96 @@
+/**
+ * GENERAL PROCESS:
+ * make a get request to server with current lastSyncedTimestamp
+ * parse the response. the response should contain everything that user has on the server
+ * decrypt the response
+ * merge everything into the database and look for conflicts
+ * send the conflicts (if any) to the end-user for resolution
+ * once the conflicts have been resolved, send the updated data back to the server
+ */
+
+/**
+ * MERGING:
+ * Locally, get everything that was editted/added after the lastSyncedTimestamp
+ * Run forEach loop on the server response.
+ * Add items that do not exist in the local collections
+ * Remove items (without asking) that need to be removed
+ * Update items that were editted before the lastSyncedTimestamp
+ * Try to merge items that were edited after the lastSyncedTimestamp
+ * Items in which the content has changed, send them for conflict resolution
+ * Otherwise, keep the most recently updated copy.
+ */
+
+/**
+ * CONFLICTS:
+ * Syncing should pause until all the conflicts have been resolved
+ * And then it should continue.
+ */
+import Database from "./index";
+import { HOST } from "../utils/constants";
+import fetch from "node-fetch";
+
+export default class Sync {
+  /**
+   *
+   * @param {Database} db
+   */
+  constructor(db) {
+    this.db = db;
+  }
+
+  async _fetch(lastSyncedTimestamp) {
+    let token = await this.db.user.token();
+    if (!token) return;
+    let response = await fetch(`${HOST}sync?lst=${lastSyncedTimestamp}`, {
+      headers: { ...HEADERS, Authorization: `Bearer ${token}` }
+    });
+    //TODO decrypt the response.
+    return await response.json();
+  }
+
+  async start() {
+    let user = await this.db.user.get();
+    if (!user) return false;
+    let lastSyncedTimestamp = user.lastSyncedTimestamp || 0;
+    let serverResponse = await this._fetch(lastSyncedTimestamp);
+    let data = this._merge(serverResponse, lastSyncedTimestamp);
+    await this._send(data);
+    return true;
+  }
+
+  _merge(serverResponse, lastSyncedTimestamp) {
+    const { notes, notebooks /* tags, colors, trash */ } = serverResponse;
+    notes.forEach(async note => {
+      let localNote = this.db.notes.note(note.id);
+      if (!localNote || note.dateEdited > localNote.data.dateEdited) {
+        await this.db.notes.add({ ...note, remote: true });
+      }
+    });
+    notebooks.forEach(async nb => {
+      let localNb = this.db.notebooks.notebook(nb.id);
+      if (!localNb || nb.dateEdited > localNb.data.dateEdited) {
+        await this.db.notebooks.add({ ...nb, remote: true });
+      }
+    });
+    // TODO trash, colors, tags
+    return {
+      notes: this.db.notes.filter(v => v.dateEdited > lastSyncedTimestamp),
+      notebooks: this.db.notebooks.filter(
+        v => v.dateEdited > lastSyncedTimestamp
+      ),
+      tags: [],
+      colors: [],
+      tags: []
+    };
+  }
+
+  async _send(data) {
+    //TODO encrypt the payload
+    let response = await fetch(`${HOST}sync`, {
+      method: "POST",
+      headers: { ...HEADERS, Authorization: `Bearer ${token}` },
+      body: JSON.stringify(data)
+    });
+    return response.ok;
+  }
+}

packages/core/collections/notebooks.js

@@ -91,8 +91,8 @@ export default class Notebooks {
 
   filter(query) {
     if (!query) return [];
-    return tfun.filter(v => fuzzysearch(query, v.title + " " + v.description))(
-      this.all
-    );
+    let queryFn = v => fuzzysearch(query, v.title + " " + v.description);
+    if (query instanceof Function) queryFn = query;
+    return tfun.filter(queryFn)(this.all);
   }
 }

packages/core/collections/notes.js

@@ -117,9 +117,9 @@ export default class Notes {
 
   filter(query) {
     if (!query) return [];
-    return tfun.filter(v => fuzzysearch(query, v.title + " " + v.content.text))(
-      this.all
-    );
+    let queryFn = v => fuzzysearch(query, v.title + " " + v.content.text);
+    if (query instanceof Function) queryFn = query;
+    return tfun.filter(queryFn)(this.all);
   }
 
   group(by, special = false) {

packages/core/database/cached-collection.js

@@ -30,7 +30,7 @@ export default class CachedCollection {
     let exists = this.map.has(item.id);
     await this.updateItem(item);
     if (!exists) {
-      item.dateCreated = Date.now();
+      item.dateCreated = item.dateCreated || Date.now();
       await this.indexer.index(item.id);
     }
   }
@@ -38,7 +38,10 @@ export default class CachedCollection {
   async updateItem(item) {
     if (this.transactionOpen) return;
     if (!item.id) throw new Error("The item must contain the id field.");
-    item.dateEdited = Date.now();
+    // if item is newly synced, remote will be true.
+    item.dateEdited = item.remote ? item.dateEdited : Date.now();
+    // the item has become local now, so remove the flag.
+    delete item.remote;
     this.map.set(item.id, item);
     await this.indexer.write(item.id, item);
   }

packages/core/models/user.js

@@ -11,7 +11,7 @@ export default class User {
     this.context = context;
   }
 
-  async user() {
+  async get() {
     return this.context.read("user");
   }
 
@@ -25,11 +25,11 @@ export default class User {
     await this.context.write("user", user);
   }
 
-  async refreshToken() {
+  async token() {
     let user = await this.user();
-    if (!user) return false;
+    if (!user) return;
     if (user.expiry < Date.now()) {
-      return true;
+      return user.accessToken;
     }
     let response = await authRequest("oauth/token", {
       refresh_token: user.refreshToken,
@@ -44,7 +44,7 @@ export default class User {
       expiry: dt.getTime()
     };
     await this.context.write("user", user);
-    return true;
+    return user.accessToken;
   }
 
   logout() {

packages/core/package.json

@@ -10,7 +10,8 @@
     "babel-jest": "^24.9.0",
     "babel-polyfill": "^6.26.0",
     "babel-preset-env": "^1.7.0",
-    "jest": "^24.9.0"
+    "jest": "^24.9.0",
+    "node-fetch": "^2.6.0"
   },
   "scripts": {
     "test": "jest"
@@ -18,9 +19,7 @@
   "dependencies": {
     "fast-sort": "^2.0.1",
     "fuzzysearch": "^1.0.3",
-    "node-fetch": "^2.6.0",
     "qclone": "^1.0.4",
-    "teleport-javascript": "^1.0.0",
     "transfun": "^1.0.2"
   }
 }

packages/core/yarn.lock

@@ -4437,11 +4437,6 @@ tar@^4:
     safe-buffer "^5.1.2"
     yallist "^3.0.3"
 
-teleport-javascript@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/teleport-javascript/-/teleport-javascript-1.0.0.tgz#c9397fad598d662027e4d3a5fa7e7da1c8361547"
-  integrity sha512-j1llvWVFyEn/6XIFDfX5LAU43DXe0GCt3NfXDwJ8XpRRMkS+i50SAkonAONBy+vxwPFBd50MFU8a2uj8R/ccLg==
-
 test-exclude@^5.2.3:
   version "5.2.3"
   resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-5.2.3.tgz#c3d3e1e311eb7ee405e092dac10aefd09091eac0"