inital work

packages/core/api/index.js

@@ -19,6 +19,7 @@ import UserManager from "./user-manager";
 import http from "../utils/http";
 import Monographs from "./monographs";
 import Offers from "./offers";
+import Attachments from "../collections/attachments";
 
 /**
  * @type {EventSource}
@@ -30,8 +31,9 @@ class Database {
    * @param {any} context
    * @param {EventSource} eventsource
    */
-  constructor(context, eventsource) {
+  constructor(context, eventsource, fs) {
     this.context = new Storage(context);
+    this.fs = fs;
     NNEventSource = eventsource;
     this._syncTimeout = 0;
   }
@@ -86,6 +88,8 @@ class Database {
     this.colors = await Tags.new(this, "colors");
     /** @type {Content} */
     this.content = await Content.new(this, "content", false);
+    /** @type {Attachments} */
+    this.attachments = await Attachments.new(this, "attachments");
 
     this.trash = new Trash(this);
 

packages/core/api/sync/collector.js

@@ -1,4 +1,6 @@
 import { CURRENT_DATABASE_VERSION } from "../../common";
+import { getContentFromData } from "../../content-types";
+import { diff } from "../../utils/array";
 import Database from "../index";
 
 class Collector {
@@ -8,32 +10,22 @@ class Collector {
    */
   constructor(db) {
     this._db = db;
-
-    this._map = async (i) => {
-      const item = { ...i };
-      // in case of resolved content
-      delete item.resolved;
-      // turn the migrated flag off so we don't keep syncing this item repeated
-      delete item.migrated;
-
-      return {
-        id: item.id,
-        v: CURRENT_DATABASE_VERSION,
-        ...(await this._serialize(item)),
-      };
-    };
   }
 
   async collect(lastSyncedTimestamp) {
     this._lastSyncedTimestamp = lastSyncedTimestamp;
     this.key = await this._db.user.getEncryptionKey();
 
+    const contents = await this._db.content.extractAttachments(
+      this._collect(await this._db.content.all())
+    );
+
     return {
-      notes: await this._collect(this._db.notes.raw),
-      notebooks: await this._collect(this._db.notebooks.raw),
-      content: await this._collect(await this._db.content.all()),
-      // trash: await this._collect(this._db.trash.raw),
-      settings: await this._collect([this._db.settings.raw]),
+      notes: await this._encrypt(this._collect(this._db.notes.raw)),
+      notebooks: await this._encrypt(this._collect(this._db.notebooks.raw)),
+      content: await this._encrypt(contents),
+      attachments: await this._encrypt(this._collect(this._db.attachments.all)),
+      settings: await this._encrypt(this._collect([this._db.settings.raw])),
       vaultKey: await this._serialize(await this._db.vault._getKey()),
     };
   }
@@ -43,15 +35,37 @@ class Collector {
     return this._db.context.encrypt(this.key, JSON.stringify(item));
   }
 
+  _encrypt(array) {
+    if (!array.length) return [];
+    return Promise.all(array.map(this._map, this));
+  }
+
+  /**
+   *
+   * @param {Array} array
+   * @returns {Array}
+   */
   _collect(array) {
-    return Promise.all(
-      array.reduce((prev, item) => {
-        if (!item || item.localOnly) return prev;
-        if (item.dateEdited > this._lastSyncedTimestamp || item.migrated)
-          prev.push(this._map(item));
-        return prev;
-      }, [])
-    );
+    if (!array.length) return [];
+    return array.reduce((prev, item) => {
+      if (!item || item.localOnly) return prev;
+      if (item.dateEdited > this._lastSyncedTimestamp || item.migrated)
+        prev.push(item);
+      return prev;
+    }, []);
+  }
+
+  async _map(item) {
+    // in case of resolved content
+    delete item.resolved;
+    // turn the migrated flag off so we don't keep syncing this item repeated
+    delete item.migrated;
+
+    return {
+      id: item.id,
+      v: CURRENT_DATABASE_VERSION,
+      ...(await this._serialize(item)),
+    };
   }
 }
 export default Collector;

packages/core/api/sync/index.js

@@ -97,6 +97,7 @@ export default class Sync {
 
       return true;
     } catch (e) {
+      this._isSyncing = false;
       throw e;
     } finally {
       this._isSyncing = false;

packages/core/api/vault.js

@@ -207,7 +207,10 @@ export default class Vault {
       note = this._db.notes.note(id).data;
       if (note.locked) return;
       contentId = note.contentId;
-      let content = await this._db.content.raw(contentId);
+      let content = await this._db.content.raw(contentId, false);
+      // TODO:
+      // 1. extract and empty out all attachments from note contents
+      // 2. encrypt them with vault password
       data = content.data;
       type = content.type;
     }

packages/core/collections/attachments.js

@@ -0,0 +1,109 @@
+import Collection from "./collection";
+import id from "../utils/id";
+import SparkMD5 from "spark-md5";
+import { deleteItem } from "../utils/array";
+
+export default class Attachments extends Collection {
+  constructor(db, name, cached) {
+    super(db, name, cached);
+    this.key = null;
+  }
+
+  async _initEncryptionKey() {
+    if (!this.key) this.key = await this._db.user.getEncryptionKey();
+    if (!this.key)
+      throw new Error(
+        "Failed to get user encryption key. Cannot cache attachments."
+      );
+  }
+
+  /**
+   *
+   * @param {{
+   *  iv: string,
+   *  salt: string,
+   *  length: number,
+   *  alg: string,
+   *  hash: string,
+   *  hashType: string,
+   *  filename: string,
+   *  type: string
+   * }} attachment
+   * @param {string} noteId Optional as attachments will be parsed at extraction time
+   * @returns
+   */
+  add(attachment, noteId) {
+    if (!attachment || !noteId)
+      return console.error("attachment or noteId cannot be null");
+
+    if (!attachment.hash) throw new Error("Please provide attachment hash.");
+
+    const oldAttachment = this.all.find(
+      (a) => a.metadata.hash === attachment.hash
+    );
+    if (oldAttachment) {
+      if (oldAttachment.noteIds.includes(noteId)) return;
+
+      oldAttachment.noteIds.push(noteId);
+      return this._collection.updateItem(oldAttachment);
+    }
+
+    const { iv, salt, length, alg, hash, hashType, filename, type } =
+      attachment;
+    const attachmentItem = {
+      id: id(),
+      noteIds: noteId ? [noteId] : [],
+      iv,
+      salt,
+      length,
+      alg,
+      metadata: {
+        hash,
+        hashType,
+        filename,
+        type,
+      },
+      dateCreated: Date.now(),
+      dateEdited: undefined,
+      dateUploaded: undefined,
+      dateDeleted: undefined,
+    };
+    return this._collection.addItem(attachmentItem);
+  }
+
+  delete(hash, noteId) {
+    const attachment = this.all.find((a) => a.metadata.hash === hash);
+    if (!attachment || !deleteItem(attachment.noteIds, noteId)) return;
+    if (!attachment.noteIds.length) attachment.dateDeleted = Date.now();
+
+    return this._collection.updateItem(attachment);
+  }
+
+  async get(hash) {
+    const attachment = this.all.find((a) => a.metadata.hash === hash);
+    if (!attachment) return;
+    await this._initEncryptionKey();
+    const data = await this._db.fs.readEncrypted(attachment.hash, this.key, {
+      iv: attachment.iv,
+      salt: attachment.salt,
+      length: attachment.length,
+      alg: attachment.alg,
+      outputType: "base64",
+    });
+    attachment.data = data;
+    return attachment;
+  }
+
+  async save(data, type) {
+    await this._initEncryptionKey();
+    await this._db.fs.writeEncrypted(null, { data, type, key: this.key });
+  }
+
+  get pending() {
+    return this.all.filter((attachment) => !attachment.dateUploaded);
+  }
+
+  get all() {
+    return this._collection.getItems();
+  }
+}

packages/core/collections/content.js

@@ -1,5 +1,7 @@
 import Collection from "./collection";
 import getId from "../utils/id";
+import { getContentFromData } from "../content-types";
+import { diff, hasItem } from "../utils/array";
 
 export default class Content extends Collection {
   async add(content) {
@@ -37,10 +39,10 @@ export default class Content extends Collection {
     return content.data;
   }
 
-  async raw(id) {
+  async raw(id, withAttachments = true) {
     const content = await this._collection.getItem(id);
     if (!content) return;
-    return content;
+    return withAttachments ? await this.insertAttachments(content) : content;
   }
 
   remove(id) {
@@ -56,22 +58,37 @@ export default class Content extends Collection {
     return this._collection.getItems(this._collection.indexer.indices);
   }
 
-  async cleanup() {
-    const indices = this._collection.indexer.indices;
-    await this._db.notes.init();
-    const notes = this._db.notes._collection.getRaw();
-    if (!notes.length && indices.length > 0) return [];
-    let ids = [];
-    for (let contentId of indices) {
-      const noteIndex = notes.findIndex((note) => note.contentId === contentId);
-      const isOrphaned = noteIndex === -1;
-      if (isOrphaned) {
-        ids.push(contentId);
-        await this._collection.deleteItem(contentId);
-      } else if (notes[noteIndex].localOnly) {
-        ids.push(contentId);
-      }
+  async insertAttachments(contentItem) {
+    const content = getContentFromData(contentItem.type, contentItem.data);
+    contentItem.data = await content.insertAttachments((hash) => {
+      return this._db.attachments.get(hash);
+    });
+    return contentItem;
+  }
+
+  async extractAttachments(contents) {
+    const allAttachments = this._db.attachments.all;
+    for (let contentItem of contents) {
+      const content = getContentFromData(contentItem.type, contentItem.data);
+      const { data, attachments } = await content.extractAttachments(
+        (data, type) => this._db.attachments.save(data, type)
+      );
+
+      await diff(allAttachments, attachments, async (attachment, action) => {
+        if (hasItem(attachment.noteIds, contentItem.noteId)) return;
+
+        if (action === "delete") {
+          await this._db.attachments.delete(
+            attachment.hash,
+            contentItem.noteId
+          );
+        } else if (action === "insert") {
+          await this._db.attachments.add(attachment, contentItem.noteId);
+        }
+      });
+
+      contentItem.data = data;
     }
-    return ids;
+    return contents;
   }
 }

packages/core/content-types/tiny.js

@@ -1,5 +1,7 @@
 import showdown from "showdown";
 import decode from "lean-he/decode";
+import SparkMD5 from "spark-md5";
+import dataurl from "../utils/dataurl";
 
 var converter = new showdown.Converter();
 converter.setFlavor("original");
@@ -56,6 +58,80 @@ class Tiny {
     const lowercase = this.toTXT().toLowerCase();
     return tokens.some((token) => lowercase.indexOf(token) > -1);
   }
+
+  async insertAttachments(get) {
+    if (!("DOMParser" in window || "DOMParser" in global)) return;
+
+    let doc = new DOMParser().parseFromString(this.data, "text/html");
+    const attachmentElements = doc.querySelectorAll("img");
+
+    for (let attachment of attachmentElements) {
+      switch (attachment.tagName) {
+        case "IMG": {
+          const hash = attachment.dataset["hash"];
+
+          const attachmentItem = await get(hash);
+          if (!attachmentItem) continue;
+
+          attachment.setAttribute(
+            "src",
+            dataurl.fromObject({
+              data: attachmentItem.data,
+              type: attachmentItem.metadata.type,
+            })
+          );
+          break;
+        }
+      }
+    }
+    return doc.body.innerHTML;
+  }
+
+  async extractAttachments(store) {
+    if (!("DOMParser" in window || "DOMParser" in global)) return;
+
+    let doc = new DOMParser().parseFromString(this.data, "text/html");
+    const attachmentElements = doc.querySelectorAll("img,.attachment");
+
+    const attachments = [];
+    for (let attachment of attachmentElements) {
+      switch (attachment.tagName) {
+        case "IMG": {
+          if (!attachment.dataset.hash) {
+            const src = attachment.getAttribute("src");
+            if (!src) continue;
+
+            const { data, mime } = dataurl.toObject(src);
+            if (!data) continue;
+
+            const type = attachment.dataset.mime || mime || "image/jpeg";
+            const metadata = await store(data, "base64");
+            attachment.dataset.hash = metadata.hash;
+
+            attachments.push({
+              type,
+              filename: attachment.dataset.filename,
+              ...metadata,
+            });
+          } else {
+            attachments.push({
+              hash: attachment.dataset.hash,
+            });
+          }
+          attachment.removeAttribute("src");
+          break;
+        }
+        default: {
+          if (!attachment.dataset.hash) return;
+          attachments.push({
+            hash: attachment.dataset.hash,
+          });
+          break;
+        }
+      }
+    }
+    return { data: doc.body.innerHTML, attachments };
+  }
 }
 export default Tiny;
 

packages/core/utils/array.js

@@ -20,13 +20,30 @@ export function deleteItems(array, ...items) {
 }
 
 export function findById(array, id) {
+  if (!array) return false;
   return array.find((item) => item.id === id);
 }
 
 export function hasItem(array, item) {
+  if (!array) return false;
   return array.indexOf(item) > -1;
 }
 
+export async function diff(arr1, arr2, action) {
+  let length = arr1.length + arr2.length;
+  for (var i = 0; i < length; ++i) {
+    var actionKey = "delete";
+    var item = arr1[i];
+
+    if (i >= arr1.length) {
+      var actionKey = "insert";
+      var item = arr2[i - arr1.length];
+    }
+
+    await action(item, actionKey);
+  }
+}
+
 function deleteAtIndex(array, index) {
   if (index === -1) return false;
   array.splice(index, 1);

packages/core/utils/dataurl.js

@@ -0,0 +1,14 @@
+const REGEX = /^data:(?<mime>image\/.+);base64,(?<data>.+)/;
+
+function toObject(dataurl) {
+  const { groups } = REGEX.exec(dataurl);
+  return groups || {};
+}
+
+function fromObject({ type, data }) {
+  //const { groups } = REGEX.exec(dataurl);
+  return `data:${type};base64,${data}`;
+}
+
+const dataurl = { toObject, fromObject };
+export default dataurl;