Revert "feat: impl fully indexed & searchable data stores"

This reverts commit 56297da18b.

packages/core/__tests__/notebooks.test.js

@@ -2,7 +2,7 @@ import {
   StorageInterface,
   notebookTest,
   TEST_NOTEBOOK,
-  TEST_NOTE,
+  TEST_NOTE
 } from "./utils";
 
 beforeEach(async () => {
@@ -26,7 +26,7 @@ test("search all notebooks", () =>
   notebookTest({
     ...TEST_NOTEBOOK,
     title: "I will be searched.",
-    description: "searched description",
+    description: "searched description"
   }).then(({ db }) => {
     let filtered = db.notebooks.filter("searhed");
     expect(filtered.length).toBeGreaterThan(0);
@@ -54,10 +54,16 @@ test("unpin a notebook", () =>
 test("delete a notebook", () =>
   notebookTest().then(async ({ db, id }) => {
     let noteId = await db.notes.add(TEST_NOTE);
-    await db.notebooks.notebook(id).topics.topic("General").add(noteId);
-    expect(db.notebooks.notebook(id).topics.topic("General").has(noteId)).toBe(
-      true
-    );
+    await db.notebooks
+      .notebook(id)
+      .topics.topic("General")
+      .add(noteId);
+    expect(
+      db.notebooks
+        .notebook(id)
+        .topics.topic("General")
+        .has(noteId)
+    ).toBe(true);
     let note = db.notes.note(noteId);
     expect(note.notebook.id).toBe(id);
     await db.notebooks.delete(id);

packages/core/__tests__/topics.test.js

@@ -79,12 +79,13 @@ test("delete a topic", () =>
   }));
 
 test("delete note from edited topic", () =>
-  notebookTest().then(async ({ db, id }) => {
-    const noteId = await db.notes.add(TEST_NOTE);
+  notebookTest().then(async ({ id }) =>
+    noteTest().then(async ({ db, id: noteId }) => {
       let topics = db.notebooks.notebook(id).topics;
       await topics.add("Home");
       let topic = topics.topic("Home");
       await db.notes.move({ id, topic: topic._topic.title }, noteId);
       await topics.add({ id: topic._topic.id, title: "Hello22" });
       await db.notes.delete(noteId);
-  }));
+    })
+  ));

packages/core/api/lookup.js

@@ -1,5 +1,4 @@
 import fuzzysearch from "fuzzysearch";
-import sm from "../utils/set";
 var tfun = require("transfun/transfun.js").tfun;
 if (!tfun) {
   tfun = global.tfun;
@@ -15,22 +14,23 @@ export default class Lookup {
   }
 
   notes(notes, query) {
-    return sm.union(
-      this._db.content._collection.search.search(query, {
-        map: (elem) => {
-          return notes.find((note) => note.contentId === elem);
-        },
-      }),
-      this._db.notes._collection.search.search(query)
+    let contentIds = this._db.content._collection.search.searchDocs(query);
+    let noteIds = this._db.notes._collection.search.searchDocs(query);
+    return notes.filter((note) => {
+      return (
+        contentIds.findIndex((content) => note.id === content.noteId) > -1 ||
+        noteIds.findIndex((n) => n.id === note.id) > -1
       );
+    });
   }
 
   notebooks(array, query) {
-    return this._db.notebooks._collection.search.search(query, {
-      map: (elem) => {
-        return array.find((nb) => nb.id === elem);
-      },
-    });
+    const notebooksIds = this._db.notebooks._collection.search.searchDocs(
+      query
+    );
+    return tfun.filter(
+      (nb) => notebooksIds.findIndex((notebook) => notebook.id === nb.id) > -1
+    )(array);
   }
 
   topics(array, query) {

packages/core/collections/content.js

@@ -44,7 +44,11 @@ export default class Content extends Collection {
     return this._collection.removeItem(id);
   }
 
+  multi(ids) {
+    return this._collection.getItems(ids);
+  }
+
   all() {
-    return this._collection.getItems();
+    return this._collection.getItems(this._collection.indexer.indices);
   }
 }

packages/core/collections/notebooks.js

@@ -41,6 +41,7 @@ export default class Notebooks extends Collection {
       description: notebook.description,
       dateCreated: notebook.dateCreated,
       pinned: !!notebook.pinned,
+      favorite: !!notebook.favorite,
       topics: notebook.topics || [],
       totalNotes: 0,
     };

packages/core/database/backup.js

@@ -34,23 +34,26 @@ export default class Backup {
       (key) => !invalidKeys.some((t) => t === key)
     );
 
-    let data = Object.fromEntries(await this._db.context.readMulti(keys));
+    const db = Object.fromEntries(await this._db.context.readMulti(keys));
+    db.h = md5.hex(JSON.stringify(db));
+    db.ht = "md5";
 
     if (encrypt) {
       const key = await this._db.user.key();
-      data = await this._db.context.encrypt(key, JSON.stringify(data));
+      return JSON.stringify({
+        type,
+        date: Date.now(),
+        data: await this._db.context.encrypt(key, JSON.stringify(db)),
+      });
     }
 
     // save backup time
     await this._db.context.write("lastBackupTime", Date.now());
 
     return JSON.stringify({
-      version: 2,
       type,
       date: Date.now(),
-      data,
-      hash: md5.hex(JSON.stringify(data)),
-      hash_type: "md5",
+      data: db,
     });
   }
 
@@ -72,26 +75,16 @@ export default class Backup {
       db = JSON.parse(await this._db.context.decrypt(key, db));
     }
 
-    if (!this._verify(backup))
+    if (!this._verify(db))
       throw new Error("Backup file has been tempered, aborting...");
-    // TODO add a proper restoration system.
-    // for (let key in db) {
-    //   let value = db[key];
-    //   if (value && value.dateEdited) {
-    //     value.dateEdited = Date.now();
-    //   }
 
-    //   const oldValue = await this._db.context.read(oldValue);
-
-    //   let finalValue = oldValue || value;
-    //   if (typeof value === "object") {
-    //     finalValue = Array.isArray(value)
-    //       ? [...value, ...oldValue]
-    //       : { ...value, ...oldValue };
-    //   }
-
-    //   await this._db.context.write(key, finalValue);
-    // }
+    for (let key in db) {
+      let value = db[key];
+      if (value && value.dateEdited) {
+        value.dateEdited = Date.now();
+      }
+      await this._db.context.write(key, value);
+    }
   }
 
   _validate(backup) {
@@ -103,11 +96,14 @@ export default class Backup {
     );
   }
 
-  _verify(backup) {
-    const { hash, hash_type, data: db } = backup;
+  _verify(db) {
+    const hash = db.h;
+    const hash_type = db.ht;
+    delete db.h;
+    delete db.ht;
     switch (hash_type) {
       case "md5": {
-        return hash === md5.hex(JSON.stringify(db));
+        return hash == md5.hex(JSON.stringify(db));
       }
       default: {
         return false;

packages/core/database/cached-collection.js

@@ -1,36 +1,45 @@
-import Storage from "./storage";
-import HyperSearch from "hypersearch";
-import { getSchema } from "./schemas";
-import PersistentCachedMap from "./persistentcachedmap";
+import Indexer from "./indexer";
 import sort from "fast-sort";
+import { EV } from "../common";
 import IndexedCollection from "./indexed-collection";
 
 export default class CachedCollection extends IndexedCollection {
+  constructor(context, type) {
+    super(context, type);
+  }
+
   async init() {
-    const index = new PersistentCachedMap(`${this.type}Index`, this.storage);
-    const store = new PersistentCachedMap(`${this.type}Store`, this.storage);
-    await index.init();
-    await store.init();
-    this.search = new HyperSearch({
-      schema: getSchema(this.type),
-      tokenizer: "forward",
-      index,
-      store,
-    });
+    await super.init();
+    const data = await this.indexer.readMulti(this.indexer.indices);
+    this.map = new Map(data);
+  }
+
+  async clear() {
+    await super.clear();
+    this.map.clear();
+  }
+
+  async updateItem(item, index = true) {
+    await super.updateItem(item, index);
+    this.map.set(item.id, item);
+    EV.publish("db:write", item);
   }
 
   exists(id) {
-    const item = this.getItem(id);
-    return item && !item.deleted;
+    return this.map.has(id) && !this.map.get(id).deleted;
+  }
+
+  getItem(id) {
+    return this.map.get(id);
   }
 
   getRaw() {
-    return Array.from(this.search.getAllDocs());
+    return Array.from(this.map.values());
   }
 
   getItems(sortFn = (u) => u.dateCreated) {
     let items = [];
-    this.search.options.store.forEach((value) => {
+    this.map.forEach((value) => {
       if (!value || value.deleted) return;
       items[items.length] = value;
     });

packages/core/database/indexed-collection.js

@@ -1,78 +1,77 @@
-import Storage from "./storage";
+import Indexer from "./indexer";
 import HyperSearch from "hypersearch";
 import { getSchema } from "./schemas";
-import PersistentCachedMap from "./persistentcachedmap";
-import PersistentMap from "./persistentmap";
-import sort from "fast-sort";
 
 export default class IndexedCollection {
-  /**
-   *
-   * @param {Storage} storage
-   * @param {string} type
-   */
-  constructor(storage, type) {
+  constructor(context, type) {
+    this.indexer = new Indexer(context, type);
     this.type = type;
-    this.storage = storage;
-  }
-
-  clear() {
-    return this.search.clear();
-  }
-
-  async init() {
-    const index = new PersistentCachedMap(`${this.type}Index`, this.storage);
-    const store = new PersistentMap(`${this.type}Store`, this.storage);
-    await index.init();
-    await store.init();
     this.search = new HyperSearch({
-      schema: getSchema(this.type),
+      schema: getSchema(type),
       tokenizer: "forward",
-      index,
-      store,
     });
   }
 
-  async addItem(item) {
-    const exists = this.exists(item.id);
-    if (!exists) item.dateCreated = item.dateCreated || Date.now();
-    await this._upsertItem(item);
+  clear() {
+    return this.indexer.clear();
   }
 
-  /**
-   * @protected
-   */
-  async _upsertItem(item) {
+  async init() {
+    await this.indexer.init();
+    const index = await this.indexer.read(`${this.type}-index`);
+    if (index) this.search.import(index);
+  }
+
+  async addItem(item) {
+    const exists = await this.exists(item.id);
+    if (!exists) item.dateCreated = item.dateCreated || Date.now();
+    await this.updateItem(item);
+    if (!exists) {
+      await this.indexer.index(item.id);
+    }
+  }
+
+  async updateItem(item, index = true) {
     if (!item.id) throw new Error("The item must contain the id field.");
     // if item is newly synced, remote will be true.
     item.dateEdited = item.remote ? item.dateEdited : Date.now();
     // the item has become local now, so remove the flag.
     delete item.remote;
-    const exists = this.exists(item.id);
-    if (!exists) await this.search.addDoc(item);
-    else await this.search.updateDoc(item.id, item);
+    await this.indexer.write(item.id, item);
+
+    if (index && (this.type === "notes" || this.type === "notebooks")) {
+      this.search.addDoc(item);
+      this.indexer.write(`${this.type}-index`, this.search.export());
+    }
   }
 
   async removeItem(id) {
-    await this.search.remove(id);
-    await this._upsertItem({
+    await this.updateItem(
+      {
         id,
         deleted: true,
         dateCreated: Date.now(),
         dateEdited: Date.now(),
-    });
+      },
+      false
+    );
+    if (this.type === "notes" || this.type === "notebooks")
+      this.search.remove(id);
   }
 
-  async exists(id) {
-    const item = await this.getItem(id);
-    return item && !item.deleted;
+  exists(id) {
+    return this.indexer.exists(id);
   }
 
   getItem(id) {
-    return this.search.getById(id);
+    return this.indexer.read(id);
   }
 
-  async getItems() {
-    return await this.search.getAllDocs();
+  async getItems(indices) {
+    const data = await this.indexer.readMulti(indices);
+    return data.reduce((total, current) => {
+      total.push(current[1]);
+      return total;
+    }, []);
   }
 }

packages/core/database/indexer.js

@@ -11,7 +11,7 @@ export default class Indexer extends Storage {
     this.indices = (await this.read(this.type, true)) || [];
   }
 
-  exists(key) {
+  async exists(key) {
     return this.indices.includes(key);
   }
 
@@ -30,10 +30,7 @@ export default class Indexer extends Storage {
   }
 
   async clear() {
-    for (var i = 0; i < this.indices.length; ++i) {
-      let key = this.indices[i];
-      await this.remove(key);
-    }
     this.indices = [];
+    await super.clear();
   }
 }

packages/core/database/persistentcachedmap.js

@@ -1,53 +0,0 @@
-import Storage from "./storage";
-import Indexer from "./indexer";
-
-export default class PersistentCachedMap {
-  /**
-   *
-   * @param {string} key
-   * @param {Storage} storage
-   */
-  constructor(key, storage) {
-    this.key = key;
-    this.indexer = new Indexer(storage, key);
-  }
-
-  async init() {
-    await this.indexer.init();
-    const data = await this.indexer.readMulti(this.indexer.indices);
-    this.map = new Map(data);
-  }
-
-  async set(key, value) {
-    this.map.set(key, value);
-    await this.indexer.write(key, value);
-    await this.indexer.index(key);
-  }
-
-  async delete(key) {
-    await this.indexer.remove(key);
-    await this.indexer.deindex(key);
-    return this.map.delete(key);
-  }
-
-  get(key) {
-    return this.map.get(key);
-  }
-
-  has(key) {
-    return this.map.has(key);
-  }
-
-  async clear() {
-    await this.indexer.clear();
-    this.map.clear();
-  }
-
-  values() {
-    return this.map.values();
-  }
-
-  forEach(callbackFn) {
-    return this.map.forEach(callbackFn);
-  }
-}

packages/core/database/persistentmap.js

@@ -1,48 +0,0 @@
-import Indexer from "./indexer";
-import Storage from "./storage";
-
-export default class PersistentMap {
-  /**
-   *
-   * @param {string} key
-   * @param {Storage} storage
-   */
-  constructor(key, storage) {
-    this.key = key;
-    this.indexer = new Indexer(storage, key);
-  }
-
-  init() {
-    return this.indexer.init();
-  }
-
-  async set(key, value) {
-    await this.indexer.write(key, value);
-    await this.indexer.index(key);
-  }
-
-  async delete(key) {
-    await this.indexer.remove(key);
-    await this.indexer.deindex(key);
-  }
-
-  get(key) {
-    return this.indexer.read(key);
-  }
-
-  has(key) {
-    return this.indexer.exists(key);
-  }
-
-  async clear() {
-    await this.indexer.clear();
-  }
-
-  async values() {
-    const data = await this.indexer.readMulti(this.indexer.indices);
-    return data.reduce((total, current) => {
-      total.push(current[1]);
-      return total;
-    }, []);
-  }
-}

packages/core/database/schemas.js

@@ -1,46 +1,22 @@
 const { getContentFromData } = require("../contenttypes");
-const { qclone } = require("qclone");
-
-const onlyStore = { store: true, index: false };
-const indexAndStore = { store: true, index: true };
-const asId = { asId: true };
-
-const basicItem = {
-  id: asId,
-  deleted: onlyStore,
-  dateCreated: onlyStore,
-  dateEdited: onlyStore,
-  type: onlyStore,
-};
 
 const schemas = {
   content: {
-    ...basicItem,
-    noteId: onlyStore,
-    conflicted: onlyStore,
-    resolved: onlyStore,
+    id: { asId: true, store: false },
+    noteId: { store: true, index: false },
     data: {
       resolve: (doc) => {
-        if (!doc.data || doc.data.iv) return "";
+        if (doc.data.iv) return "";
         const content = getContentFromData(doc.type, doc.data);
         return content._text;
       },
-      store: true,
+      store: false,
       index: true,
     },
   },
   notes: {
-    ...basicItem,
-    locked: onlyStore,
-    colors: onlyStore,
-    tags: onlyStore,
-    conflicted: onlyStore,
-    contentId: onlyStore,
-    pinned: onlyStore,
-    favorite: onlyStore,
-    title: indexAndStore,
-    headline: onlyStore,
-    notebook: onlyStore,
+    id: { asId: true, store: false },
+    title: true,
     // pinned: {
     //   resolve: (doc) => (doc.pinned ? "pinned:true" : "pinned:false"),
     //   index: true,
@@ -49,46 +25,18 @@ const schemas = {
     // },
   },
   notebooks: {
-    ...basicItem,
-    title: indexAndStore,
-    description: indexAndStore,
-    totalNotes: onlyStore,
-    pinned: onlyStore,
+    id: { asId: true, store: false },
+    title: true,
+    description: true,
     topics: {
       resolve: (doc) => {
         if (!doc.topics) return "";
         return doc.topics.map((v) => v.title || v).join(" ");
       },
-      ...indexAndStore,
     },
   },
-  colors: {
-    ...basicItem,
-    merge: ["tags"],
-  },
-  tags: {
-    ...basicItem,
-    noteIds: onlyStore,
-    deletedIds: onlyStore,
-    title: indexAndStore,
-  },
-  trash: {
-    id: asId,
-    dateDeleted: onlyStore,
-    itemType: onlyStore,
-    itemId: onlyStore,
-    merge: ["notes", "notebooks"],
-  },
 };
 
 export function getSchema(type) {
-  let schema = qclone(schemas[type]);
-  if (schema.merge) {
-    const mergeSchemas = schema.merge;
-    delete schema.merge;
-    mergeSchemas.forEach((key) => {
-      schema = { ...schema, ...schemas[key] };
-    });
-  }
-  return schema;
+  return schemas[type];
 }