Mirrors/notesnook
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook.git synced 2025-12-22 22:49:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Revert "feat: impl fully indexed & searchable data stores"

Browse Source 
This reverts commit 56297da18b.
This commit is contained in:
thecodrr
2020-11-16 15:00:52 +05:00
parent fd0d84b3ab
commit c81c0feda8
12 changed files with 152 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
packages/core/__tests__/notebooks.test.js
View File

@@ -2,7 +2,7 @@ import {
StorageInterface,
notebookTest,
TEST_NOTEBOOK,
TEST_NOTE,
TEST_NOTE
} from "./utils";
beforeEach(async () => {
@@ -26,7 +26,7 @@ test("search all notebooks", () =>
notebookTest({
...TEST_NOTEBOOK,
title: "I will be searched.",
description: "searched description",
description: "searched description"
}).then(({ db }) => {
let filtered = db.notebooks.filter("searhed");
expect(filtered.length).toBeGreaterThan(0);
@@ -54,10 +54,16 @@ test("unpin a notebook", () =>
test("delete a notebook", () =>
notebookTest().then(async ({ db, id }) => {
let noteId = await db.notes.add(TEST_NOTE);
await db.notebooks.notebook(id).topics.topic("General").add(noteId);
expect(db.notebooks.notebook(id).topics.topic("General").has(noteId)).toBe(
true
);
await db.notebooks
.notebook(id)
.topics.topic("General")
.add(noteId);
expect(
db.notebooks
.notebook(id)
.topics.topic("General")
.has(noteId)
).toBe(true);
let note = db.notes.note(noteId);
expect(note.notebook.id).toBe(id);
await db.notebooks.delete(id);

7
packages/core/__tests__/topics.test.js
View File

@@ -79,12 +79,13 @@ test("delete a topic", () =>
}));
test("delete note from edited topic", () =>
notebookTest().then(async ({ db, id }) => {
const noteId = await db.notes.add(TEST_NOTE);
notebookTest().then(async ({ id }) =>
noteTest().then(async ({ db, id: noteId }) => {
let topics = db.notebooks.notebook(id).topics;
await topics.add("Home");
let topic = topics.topic("Home");
await db.notes.move({ id, topic: topic._topic.title }, noteId);
await topics.add({ id: topic._topic.id, title: "Hello22" });
await db.notes.delete(noteId);
}));
})
));

26
packages/core/api/lookup.js
View File

@@ -1,5 +1,4 @@
import fuzzysearch from "fuzzysearch";
import sm from "../utils/set";
var tfun = require("transfun/transfun.js").tfun;
if (!tfun) {
tfun = global.tfun;
@@ -15,22 +14,23 @@ export default class Lookup {
}
notes(notes, query) {
return sm.union(
this._db.content._collection.search.search(query, {
map: (elem) => {
return notes.find((note) => note.contentId === elem);
},
}),
this._db.notes._collection.search.search(query)
let contentIds = this._db.content._collection.search.searchDocs(query);
let noteIds = this._db.notes._collection.search.searchDocs(query);
return notes.filter((note) => {
return (
contentIds.findIndex((content) => note.id === content.noteId) > -1 ||
noteIds.findIndex((n) => n.id === note.id) > -1
);
});
}
notebooks(array, query) {
return this._db.notebooks._collection.search.search(query, {
map: (elem) => {
return array.find((nb) => nb.id === elem);
},
});
const notebooksIds = this._db.notebooks._collection.search.searchDocs(
query
);
return tfun.filter(
(nb) => notebooksIds.findIndex((notebook) => notebook.id === nb.id) > -1
)(array);
}
topics(array, query) {

6
packages/core/collections/content.js
View File

@@ -44,7 +44,11 @@ export default class Content extends Collection {
return this._collection.removeItem(id);
}
multi(ids) {
return this._collection.getItems(ids);
}
all() {
return this._collection.getItems();
return this._collection.getItems(this._collection.indexer.indices);
}
}

1
packages/core/collections/notebooks.js
View File

@@ -41,6 +41,7 @@ export default class Notebooks extends Collection {
description: notebook.description,
dateCreated: notebook.dateCreated,
pinned: !!notebook.pinned,
favorite: !!notebook.favorite,
topics: notebook.topics || [],
totalNotes: 0,
};

50
packages/core/database/backup.js
View File

@@ -34,23 +34,26 @@ export default class Backup {
(key) => !invalidKeys.some((t) => t === key)
);
let data = Object.fromEntries(await this._db.context.readMulti(keys));
const db = Object.fromEntries(await this._db.context.readMulti(keys));
db.h = md5.hex(JSON.stringify(db));
db.ht = "md5";
if (encrypt) {
const key = await this._db.user.key();
data = await this._db.context.encrypt(key, JSON.stringify(data));
return JSON.stringify({
type,
date: Date.now(),
data: await this._db.context.encrypt(key, JSON.stringify(db)),
});
}
// save backup time
await this._db.context.write("lastBackupTime", Date.now());
return JSON.stringify({
version: 2,
type,
date: Date.now(),
data,
hash: md5.hex(JSON.stringify(data)),
hash_type: "md5",
data: db,
});
}
@@ -72,26 +75,16 @@ export default class Backup {
db = JSON.parse(await this._db.context.decrypt(key, db));
}
if (!this._verify(backup))
if (!this._verify(db))
throw new Error("Backup file has been tempered, aborting...");
// TODO add a proper restoration system.
// for (let key in db) {
// let value = db[key];
// if (value && value.dateEdited) {
// value.dateEdited = Date.now();
// }
// const oldValue = await this._db.context.read(oldValue);
// let finalValue = oldValue || value;
// if (typeof value === "object") {
// finalValue = Array.isArray(value)
// ? [...value, ...oldValue]
// : { ...value, ...oldValue };
// }
// await this._db.context.write(key, finalValue);
// }
for (let key in db) {
let value = db[key];
if (value && value.dateEdited) {
value.dateEdited = Date.now();
}
await this._db.context.write(key, value);
}
}
_validate(backup) {
@@ -103,11 +96,14 @@ export default class Backup {
);
}
_verify(backup) {
const { hash, hash_type, data: db } = backup;
_verify(db) {
const hash = db.h;
const hash_type = db.ht;
delete db.h;
delete db.ht;
switch (hash_type) {
case "md5": {
return hash === md5.hex(JSON.stringify(db));
return hash == md5.hex(JSON.stringify(db));
}
default: {
return false;

45
packages/core/database/cached-collection.js
View File

@@ -1,36 +1,45 @@
import Storage from "./storage";
import HyperSearch from "hypersearch";
import { getSchema } from "./schemas";
import PersistentCachedMap from "./persistentcachedmap";
import Indexer from "./indexer";
import sort from "fast-sort";
import { EV } from "../common";
import IndexedCollection from "./indexed-collection";
export default class CachedCollection extends IndexedCollection {
constructor(context, type) {
super(context, type);
}
async init() {
const index = new PersistentCachedMap(`${this.type}Index`, this.storage);
const store = new PersistentCachedMap(`${this.type}Store`, this.storage);
await index.init();
await store.init();
this.search = new HyperSearch({
schema: getSchema(this.type),
tokenizer: "forward",
index,
store,
});
await super.init();
const data = await this.indexer.readMulti(this.indexer.indices);
this.map = new Map(data);
}
async clear() {
await super.clear();
this.map.clear();
}
async updateItem(item, index = true) {
await super.updateItem(item, index);
this.map.set(item.id, item);
EV.publish("db:write", item);
}
exists(id) {
const item = this.getItem(id);
return item && !item.deleted;
return this.map.has(id) && !this.map.get(id).deleted;
}
getItem(id) {
return this.map.get(id);
}
getRaw() {
return Array.from(this.search.getAllDocs());
return Array.from(this.map.values());
}
getItems(sortFn = (u) => u.dateCreated) {
let items = [];
this.search.options.store.forEach((value) => {
this.map.forEach((value) => {
if (!value || value.deleted) return;
items[items.length] = value;
});

89
packages/core/database/indexed-collection.js
View File

@@ -1,78 +1,77 @@
import Storage from "./storage";
import Indexer from "./indexer";
import HyperSearch from "hypersearch";
import { getSchema } from "./schemas";
import PersistentCachedMap from "./persistentcachedmap";
import PersistentMap from "./persistentmap";
import sort from "fast-sort";
export default class IndexedCollection {
/**
*
* @param {Storage} storage
* @param {string} type
*/
constructor(storage, type) {
constructor(context, type) {
this.indexer = new Indexer(context, type);
this.type = type;
this.storage = storage;
}
clear() {
return this.search.clear();
}
async init() {
const index = new PersistentCachedMap(`${this.type}Index`, this.storage);
const store = new PersistentMap(`${this.type}Store`, this.storage);
await index.init();
await store.init();
this.search = new HyperSearch({
schema: getSchema(this.type),
schema: getSchema(type),
tokenizer: "forward",
index,
store,
});
}
async addItem(item) {
const exists = this.exists(item.id);
if (!exists) item.dateCreated = item.dateCreated || Date.now();
await this._upsertItem(item);
clear() {
return this.indexer.clear();
}
/**
* @protected
*/
async _upsertItem(item) {
async init() {
await this.indexer.init();
const index = await this.indexer.read(`${this.type}-index`);
if (index) this.search.import(index);
}
async addItem(item) {
const exists = await this.exists(item.id);
if (!exists) item.dateCreated = item.dateCreated || Date.now();
await this.updateItem(item);
if (!exists) {
await this.indexer.index(item.id);
}
}
async updateItem(item, index = true) {
if (!item.id) throw new Error("The item must contain the id field.");
// if item is newly synced, remote will be true.
item.dateEdited = item.remote ? item.dateEdited : Date.now();
// the item has become local now, so remove the flag.
delete item.remote;
const exists = this.exists(item.id);
if (!exists) await this.search.addDoc(item);
else await this.search.updateDoc(item.id, item);
await this.indexer.write(item.id, item);
if (index && (this.type === "notes" || this.type === "notebooks")) {
this.search.addDoc(item);
this.indexer.write(`${this.type}-index`, this.search.export());
}
}
async removeItem(id) {
await this.search.remove(id);
await this._upsertItem({
await this.updateItem(
{
id,
deleted: true,
dateCreated: Date.now(),
dateEdited: Date.now(),
});
},
false
);
if (this.type === "notes" || this.type === "notebooks")
this.search.remove(id);
}
async exists(id) {
const item = await this.getItem(id);
return item && !item.deleted;
exists(id) {
return this.indexer.exists(id);
}
getItem(id) {
return this.search.getById(id);
return this.indexer.read(id);
}
async getItems() {
return await this.search.getAllDocs();
async getItems(indices) {
const data = await this.indexer.readMulti(indices);
return data.reduce((total, current) => {
total.push(current[1]);
return total;
}, []);
}
}

7
packages/core/database/indexer.js
View File

@@ -11,7 +11,7 @@ export default class Indexer extends Storage {
this.indices = (await this.read(this.type, true)) || [];
}
exists(key) {
async exists(key) {
return this.indices.includes(key);
}
@@ -30,10 +30,7 @@ export default class Indexer extends Storage {
}
async clear() {
for (var i = 0; i < this.indices.length; ++i) {
let key = this.indices[i];
await this.remove(key);
}
this.indices = [];
await super.clear();
}
}

53
packages/core/database/persistentcachedmap.js
View File

@@ -1,53 +0,0 @@
import Storage from "./storage";
import Indexer from "./indexer";
export default class PersistentCachedMap {
/**
*
* @param {string} key
* @param {Storage} storage
*/
constructor(key, storage) {
this.key = key;
this.indexer = new Indexer(storage, key);
}
async init() {
await this.indexer.init();
const data = await this.indexer.readMulti(this.indexer.indices);
this.map = new Map(data);
}
async set(key, value) {
this.map.set(key, value);
await this.indexer.write(key, value);
await this.indexer.index(key);
}
async delete(key) {
await this.indexer.remove(key);
await this.indexer.deindex(key);
return this.map.delete(key);
}
get(key) {
return this.map.get(key);
}
has(key) {
return this.map.has(key);
}
async clear() {
await this.indexer.clear();
this.map.clear();
}
values() {
return this.map.values();
}
forEach(callbackFn) {
return this.map.forEach(callbackFn);
}
}

48
packages/core/database/persistentmap.js
View File

@@ -1,48 +0,0 @@
import Indexer from "./indexer";
import Storage from "./storage";
export default class PersistentMap {
/**
*
* @param {string} key
* @param {Storage} storage
*/
constructor(key, storage) {
this.key = key;
this.indexer = new Indexer(storage, key);
}
init() {
return this.indexer.init();
}
async set(key, value) {
await this.indexer.write(key, value);
await this.indexer.index(key);
}
async delete(key) {
await this.indexer.remove(key);
await this.indexer.deindex(key);
}
get(key) {
return this.indexer.read(key);
}
has(key) {
return this.indexer.exists(key);
}
async clear() {
await this.indexer.clear();
}
async values() {
const data = await this.indexer.readMulti(this.indexer.indices);
return data.reduce((total, current) => {
total.push(current[1]);
return total;
}, []);
}
}

72
packages/core/database/schemas.js
View File

@@ -1,46 +1,22 @@
const { getContentFromData } = require("../contenttypes");
const { qclone } = require("qclone");
const onlyStore = { store: true, index: false };
const indexAndStore = { store: true, index: true };
const asId = { asId: true };
const basicItem = {
id: asId,
deleted: onlyStore,
dateCreated: onlyStore,
dateEdited: onlyStore,
type: onlyStore,
};
const schemas = {
content: {
...basicItem,
noteId: onlyStore,
conflicted: onlyStore,
resolved: onlyStore,
id: { asId: true, store: false },
noteId: { store: true, index: false },
data: {
resolve: (doc) => {
if (!doc.data || doc.data.iv) return "";
if (doc.data.iv) return "";
const content = getContentFromData(doc.type, doc.data);
return content._text;
},
store: true,
store: false,
index: true,
},
},
notes: {
...basicItem,
locked: onlyStore,
colors: onlyStore,
tags: onlyStore,
conflicted: onlyStore,
contentId: onlyStore,
pinned: onlyStore,
favorite: onlyStore,
title: indexAndStore,
headline: onlyStore,
notebook: onlyStore,
id: { asId: true, store: false },
title: true,
// pinned: {
// resolve: (doc) => (doc.pinned ? "pinned:true" : "pinned:false"),
// index: true,
@@ -49,46 +25,18 @@ const schemas = {
// },
},
notebooks: {
...basicItem,
title: indexAndStore,
description: indexAndStore,
totalNotes: onlyStore,
pinned: onlyStore,
id: { asId: true, store: false },
title: true,
description: true,
topics: {
resolve: (doc) => {
if (!doc.topics) return "";
return doc.topics.map((v) => v.title || v).join(" ");
},
...indexAndStore,
},
},
colors: {
...basicItem,
merge: ["tags"],
},
tags: {
...basicItem,
noteIds: onlyStore,
deletedIds: onlyStore,
title: indexAndStore,
},
trash: {
id: asId,
dateDeleted: onlyStore,
itemType: onlyStore,
itemId: onlyStore,
merge: ["notes", "notebooks"],
},
};
export function getSchema(type) {
let schema = qclone(schemas[type]);
if (schema.merge) {
const mergeSchemas = schema.merge;
delete schema.merge;
mergeSchemas.forEach((key) => {
schema = { ...schema, ...schemas[key] };
});
}
return schema;
return schemas[type];
}