Mirrors/notesnook
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook.git synced 2025-12-16 19:57:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

core: convert almost everything to typescript

Browse Source
This commit is contained in:
Abdullah Atta
2023-08-21 13:32:06 +05:00
parent 3b15717150
commit 41ea254301
125 changed files with 9258 additions and 9132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
packages/core/__e2e__/offers.test.js
View File

@@ -18,21 +18,19 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import hosts from "../src/utils/constants";
import Offers from "../src/api/offers";
import { Offers } from "../src/api/offers";
import { test, expect } from "vitest";
test("get offer code", async () => {
const offers = new Offers();
hosts.SUBSCRIPTIONS_HOST = "https://subscriptions.streetwriters.co";
expect(await offers.getCode("TESTOFFER", "android")).toMatchSnapshot(
expect(await Offers.getCode("TESTOFFER", "android")).toMatchSnapshot(
"offer-code"
);
});
test("get invalid offer code", async () => {
const offers = new Offers();
hosts.SUBSCRIPTIONS_HOST = "https://subscriptions.streetwriters.co";
await expect(() => offers.getCode("INVALIDOFFER", "android")).rejects.toThrow(
await expect(() => Offers.getCode("INVALIDOFFER", "android")).rejects.toThrow(
/Not found/i
);
});

8
packages/core/__e2e__/pricing.test.js
View File

@@ -17,12 +17,11 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Pricing from "../src/api/pricing";
import { Pricing } from "../src/api/pricing";
import { test, expect, describe } from "vitest";
test.each(["monthly", "yearly", undefined])(`get %s price`, async (period) => {
const pricing = new Pricing();
const price = await pricing.price(period);
const price = await Pricing.price(period);
expect(price).toMatchSnapshot(
{
country: expect.any(String),
@@ -38,8 +37,7 @@ describe.each(["android", "ios", "web"])(`get %s pricing tier`, (platform) => {
test.each(["monthly", "yearly"])(
`get %s ${platform} tier`,
async (period) => {
const pricing = new Pricing();
const price = await pricing.sku(platform, period);
const price = await Pricing.sku(platform, period);
expect(price).toMatchSnapshot(
{
country: expect.any(String),

19
packages/core/__mocks__/fs.mock.ts
View File

@@ -18,6 +18,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { DataFormat, SerializedKey } from "@notesnook/crypto";
import {
FileEncryptionMetadataWithOutputType,
IFileStorage,
RequestOptions
} from "../src/interfaces";
import { xxhash64 } from "hash-wasm";
import { IDataType } from "hash-wasm/dist/lib/util";
@@ -64,7 +69,7 @@ export async function hashBuffer(data: IDataType) {
async function readEncrypted<TOutputFormat extends DataFormat>(
filename: string,
_key: SerializedKey,
_cipherData: any
_cipherData: FileEncryptionMetadataWithOutputType<TOutputFormat>
) {
const cipher = fs[filename];
if (!cipher) {
@@ -74,17 +79,17 @@ async function readEncrypted<TOutputFormat extends DataFormat>(
return cipher.data;
}
async function uploadFile(filename: string, _requestOptions: any) {
async function uploadFile(filename: string, _requestOptions: RequestOptions) {
const cipher = fs[filename];
if (!cipher) throw new Error(`File not found. Filename: ${filename}`);
return true;
}
async function downloadFile(filename: string, _requestOptions: any) {
async function downloadFile(filename: string, _requestOptions: RequestOptions) {
return hasItem(filename);
}
async function deleteFile(filename: string, _requestOptions: any) {
async function deleteFile(filename: string, _requestOptions: RequestOptions) {
if (!hasItem(filename)) return true;
delete fs[filename];
return true;
@@ -98,7 +103,7 @@ async function clearFileStorage() {
fs = {};
}
export const FS = {
export const FS: IFileStorage = {
writeEncryptedBase64,
readEncrypted,
uploadFile: cancellable(uploadFile),
@@ -110,9 +115,9 @@ export const FS = {
};
function cancellable<T>(
operation: (filename: string, requestOptions: any) => Promise<T>
operation: (filename: string, requestOptions: RequestOptions) => Promise<T>
) {
return function (filename: string, requestOptions: any) {
return function (filename: string, requestOptions: RequestOptions) {
const abortController = new AbortController();
return {
execute: () => operation(filename, requestOptions),

3
packages/core/__mocks__/node-storage.mock.ts
View File

@@ -18,8 +18,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Cipher, NNCrypto, SerializedKey } from "@notesnook/crypto";
import { IStorage } from "../src/interfaces";
export class NodeStorageInterface {
export class NodeStorageInterface implements IStorage {
storage = {};
crypto = new NNCrypto();

1
packages/core/__tests__/__fixtures__/backup.v5.8.json Normal file
View File

@@ -0,0 +1 @@
{"version":5.8,"type":"node","date":1692020856821,"data":{"settings":{"type":"settings","id":"64da307856565633109999c3","groupOptions":{},"toolbarConfig":{},"aliases":{"bda9643ac6601722a28f238714274da4":"red","e9bae3ce1d7ac00b0b1aa2fbddc50cfb":"tag1","d487dd0b55dfcacdd920ccbdaeafa351":"yellow","ac00dfc2c5c111870601a37c60d31626":"tag3","9f27410725ab8cc8854a2769c7a516b8":"green","bb7aedfa61007447dd6efaf9f37641e3":"purple","f32af7d8e6b19f67a63af85e5e7b8a82":"tag2"},"dateModified":1692020856820,"dateCreated":0,"trashCleanupInterval":7,"titleFormat":"Note $date$ $time$","timeFormat":"12-hour","dateFormat":"DD-MM-YYYY","synced":false},"64da307856565633109999c4_notes":{"id":"64da307856565633109999c4","type":"note","title":"Test note 1","tags":["tag1"],"color":"red","pinned":false,"locked":false,"favorite":false,"localOnly":false,"conflicted":false,"readonly":false,"dateCreated":1692020856818,"dateEdited":1692020856818,"dateModified":1692020856818,"synced":false},"notes":["64da307856565633109999c4","64da307856565633109999c5","64da307856565633109999c6","64da307856565633109999c7"],"bda9643ac6601722a28f238714274da4_colors":{"type":"tag","id":"bda9643ac6601722a28f238714274da4","title":"red","noteIds":["64da307856565633109999c4"],"localOnly":true,"dateCreated":1692020856819,"dateModified":1692020856819,"synced":false,"alias":"red"},"e9bae3ce1d7ac00b0b1aa2fbddc50cfb_tags":{"type":"tag","id":"e9bae3ce1d7ac00b0b1aa2fbddc50cfb","title":"tag1","noteIds":["64da307856565633109999c4","64da307856565633109999c6","64da307856565633109999c7"],"localOnly":true,"dateModified":1692020856820,"synced":false,"alias":"tag1"},"64da307856565633109999c5_notes":{"id":"64da307856565633109999c5","type":"note","title":"Test note 2","tags":["tag3"],"color":"yellow","pinned":false,"locked":false,"favorite":false,"localOnly":false,"conflicted":false,"readonly":false,"dateCreated":1692020856819,"dateEdited":1692020856819,"dateModified":1692020856819,"synced":false},"d487dd0b55dfcacdd920ccbdaeafa351_colors":{"type":"tag","id":"d487dd0b55dfcacdd920ccbdaeafa351","title":"yellow","noteIds":["64da307856565633109999c5"],"localOnly":true,"dateCreated":1692020856819,"dateModified":1692020856819,"synced":false,"alias":"yellow"},"ac00dfc2c5c111870601a37c60d31626_tags":{"type":"tag","id":"ac00dfc2c5c111870601a37c60d31626","title":"tag3","noteIds":["64da307856565633109999c5","64da307856565633109999c6"],"localOnly":true,"dateModified":1692020856820,"synced":false,"alias":"tag3"},"64da307856565633109999c6_notes":{"id":"64da307856565633109999c6","type":"note","title":"Test note 3","tags":["tag1","tag3"],"color":"green","pinned":false,"locked":false,"favorite":false,"localOnly":false,"conflicted":false,"readonly":false,"dateCreated":1692020856819,"dateEdited":1692020856819,"dateModified":1692020856819,"synced":false},"9f27410725ab8cc8854a2769c7a516b8_colors":{"type":"tag","id":"9f27410725ab8cc8854a2769c7a516b8","title":"green","noteIds":["64da307856565633109999c6"],"localOnly":true,"dateCreated":1692020856819,"dateModified":1692020856819,"synced":false,"alias":"green"},"64da307856565633109999c7_notes":{"id":"64da307856565633109999c7","type":"note","title":"Test note 4","tags":["tag1","tag2"],"color":"purple","pinned":false,"locked":false,"favorite":false,"localOnly":false,"conflicted":false,"readonly":false,"dateCreated":1692020856820,"dateEdited":1692020856820,"dateModified":1692020856820,"synced":false},"bb7aedfa61007447dd6efaf9f37641e3_colors":{"type":"tag","id":"bb7aedfa61007447dd6efaf9f37641e3","title":"purple","noteIds":["64da307856565633109999c7"],"localOnly":true,"dateCreated":1692020856820,"dateModified":1692020856820,"synced":false},"f32af7d8e6b19f67a63af85e5e7b8a82_tags":{"type":"tag","id":"f32af7d8e6b19f67a63af85e5e7b8a82","title":"tag2","noteIds":["64da307856565633109999c7"],"localOnly":true,"dateCreated":1692020856820,"dateModified":1692020856820,"synced":false}},"hash":"5fe160b7983f4c8a8694a8cba3702fa5","hash_type":"md5"}

69
packages/core/__tests__/backup.test.js
View File

@@ -17,37 +17,37 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import {
TEST_NOTE,
databaseTest,
loginFakeUser,
noteTest,
notebookTest
} from "./utils";
import { TEST_NOTE, databaseTest, loginFakeUser, notebookTest } from "./utils";
import v52Backup from "./__fixtures__/backup.v5.2.json";
import v52BackupCopy from "./__fixtures__/backup.v5.2.copy.json";
import v56BackupCopy from "./__fixtures__/backup.v5.6.json";
import v58BackupCopy from "./__fixtures__/backup.v5.8.json";
import qclone from "qclone";
import { test, expect, describe } from "vitest";
import { makeId } from "../src/utils/id";
test("export backup", () =>
noteTest().then(() =>
notebookTest().then(async ({ db }) => {
const exp = [];
for await (const file of db.backup.export("node", false)) {
exp.push(file);
}
notebookTest().then(async ({ db }) => {
const id = await db.notes.add(TEST_NOTE);
const exp = [];
for await (const file of db.backup.export("node", false)) {
exp.push(file);
}
let backup = JSON.parse(exp[1].data);
expect(exp.length).toBe(2);
expect(exp[0].path).toBe(".nnbackup");
expect(backup.type).toBe("node");
expect(backup.date).toBeGreaterThan(0);
expect(backup.data).toBeTypeOf("string");
expect(backup.compressed).toBe(true);
expect(backup.encrypted).toBe(false);
})
));
let backup = JSON.parse(exp[1].data);
expect(exp.length).toBe(2);
expect(exp[0].path).toBe(".nnbackup");
expect(backup.type).toBe("node");
expect(backup.date).toBeGreaterThan(0);
expect(backup.data).toBeTypeOf("string");
expect(backup.compressed).toBe(true);
expect(backup.encrypted).toBe(false);
expect(
JSON.parse(await db.compressor().decompress(backup.data)).find(
(i) => i.id === id
)
).toBeDefined();
}));
test("export encrypted backup", () =>
notebookTest().then(async ({ db }) => {
@@ -117,7 +117,8 @@ test("import tempered backup", () =>
describe.each([
["v5.2", v52Backup],
["v5.2 copy", v52BackupCopy],
["v5.6", v56BackupCopy]
["v5.6", v56BackupCopy],
["v5.8", v58BackupCopy]
])("testing backup version: %s", (version, data) => {
test(`import ${version} backup`, () => {
return databaseTest().then(async (db) => {
@@ -131,12 +132,17 @@ describe.each([
expect(
db.notes.all.every((v) => {
const doesNotHaveContent = !v.content;
const doesNotHaveColors = !v.colors && (!v.color || v.color.length);
const doesNotHaveColors = !v.colors; // && (!v.color || v.color.length);
const hasTopicsInAllNotebooks =
!v.notebooks ||
v.notebooks.every((nb) => !!nb.id && !!nb.topics && !nb.topic);
const hasDateModified = v.dateModified > 0;
const doesNotHaveTags = !v.tags;
const doesNotHaveColor = !v.color;
if (!doesNotHaveTags) console.log(v);
return (
doesNotHaveTags &&
doesNotHaveColor &&
doesNotHaveContent &&
!v.notebook &&
hasTopicsInAllNotebooks &&
@@ -146,6 +152,16 @@ describe.each([
})
).toBeTruthy();
expect(
db.tags.all.every((t) => makeId(t.title) !== t.id && !t.noteIds)
).toBeTruthy();
expect(
db.colors.all.every(
(t) => makeId(t.title) !== t.id && !t.noteIds && !!t.colorCode
)
).toBeTruthy();
expect(
db.notebooks.all.every((v) => v.title != null && v.dateModified > 0)
).toBeTruthy();
@@ -158,7 +174,8 @@ describe.each([
db.attachments.all.every((v) => v.dateModified > 0 && !v.dateEdited)
).toBeTruthy();
expect(db.shortcuts.all).toHaveLength(data.data.settings.pins.length);
if (data.data.settings.pins)
expect(db.shortcuts.all).toHaveLength(data.data.settings.pins.length);
const allContent = await db.content.all();
expect(

45
packages/core/__tests__/content.test.js
View File

@@ -1,45 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { databaseTest } from "./utils";
import { test, expect } from "vitest";
test("adding a deleted content should not throw", () =>
databaseTest().then(async (db) => {
await expect(
db.content.add({
deleted: true,
dateEdited: new Date(),
id: "hello",
data: "YOYO!"
})
).resolves.toBeUndefined();
}));
test("tagging an empty note should not create an invalid content item", () =>
databaseTest().then(async (db) => {
const id = await db.notes.add({ title: "Hello" });
const contentId = await db.notes.note(id)._note.contentId;
expect(contentId).toBeUndefined();
expect(await db.content.all()).toHaveLength(0);
await db.notes.note(id).tag("tag1");
expect(await db.content.all()).toHaveLength(0);
}));

2
packages/core/__tests__/lookup.test.js
View File

@@ -67,7 +67,7 @@ test("search notebooks", () =>
test("search topics", () =>
notebookTest().then(async ({ db, id }) => {
const topics = db.notebooks.notebook(id).topics.all;
const topics = db.notebooks.topics(id).all;
let filtered = db.lookup.topics(topics, "hello");
expect(filtered).toHaveLength(1);
}));

22
packages/core/__tests__/note-history.test.js
View File

@@ -108,20 +108,6 @@ test("date created of session should not change on edit", () =>
expect(newDateModified).toBeGreaterThan(dateModified);
}));
test("serialized session data should get deserialized", () =>
noteTest({ ...TEST_NOTE, sessionId: "session" }).then(async ({ db, id }) => {
let json = await db.noteHistory.serialize();
await db.noteHistory.clearSessions(id);
await db.noteHistory.deserialize(json);
let history = await db.noteHistory.get(id);
expect(history).toHaveLength(1);
let content = await db.noteHistory.content(history[0].id);
expect(content).toMatchObject(TEST_NOTE.content);
}));
test("clear a note's sessions", () =>
noteTest({ ...TEST_NOTE, sessionId: "session" }).then(async ({ db, id }) => {
await db.noteHistory.clearSessions(id);
@@ -147,12 +133,6 @@ test("return empty array if no history available", () =>
expect(history).toHaveLength(0);
}));
test("session should not be added to history if values are null or undefined", () =>
noteTest().then(async ({ db }) => {
let history = await db.noteHistory.add();
expect(history).toBeUndefined();
}));
test("auto clear sessions if they exceed the limit", () =>
noteTest({ ...TEST_NOTE, sessionId: Date.now() }).then(async ({ db, id }) => {
let editedContent = {
@@ -169,7 +149,7 @@ test("auto clear sessions if they exceed the limit", () =>
let sessions = await db.noteHistory.get(id);
expect(sessions).toHaveLength(2);
await db.noteHistory._cleanup(id, 1);
await db.noteHistory.cleanup(id, 1);
sessions = await db.noteHistory.get(id);
expect(await db.noteHistory.get(id)).toHaveLength(1);

4
packages/core/__tests__/notebooks.test.js
View File

@@ -65,7 +65,7 @@ test("merge notebook with new topics", () =>
const newNotebook = db.notebooks.merge(notebook.data, {
...notebook.data,
topics: [...notebook.data.topics, makeTopic("Home", id)],
topics: [...notebook.data.topics, makeTopic({ title: "Home" }, id)],
remote: true
});
@@ -83,7 +83,7 @@ test("merge notebook with topics removed", () =>
const newNotebook = db.notebooks.merge(notebook.data, {
...notebook.data,
topics: [makeTopic("Home", id)],
topics: [makeTopic({ title: "Home" }, id)],
remote: true
});

233
packages/core/__tests__/notes.test.js → packages/core/__tests__/notes.test.ts
View File

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Database from "../src/api";
import { groupArray } from "../src/utils/grouping";
import {
databaseTest,
@@ -29,46 +30,77 @@ import {
} from "./utils";
import { test, expect } from "vitest";
async function createAndAddNoteToNotebook(
db: Database,
noteId: string,
options: {
notebookTitle: string;
topicTitle: string;
}
) {
const { notebookTitle, topicTitle } = options;
const notebookId = await db.notebooks.add({ title: notebookTitle });
if (!notebookId) throw new Error("Could not create notebook");
const topics = db.notebooks.topics(notebookId);
await topics.add({ title: topicTitle });
const topic = topics.topic(topicTitle);
if (!topic) throw new Error("Could not find topic.");
await db.notes.addToNotebook({ id: notebookId, topic: topic.id }, noteId);
return { topic, topics, notebookId };
}
test("add invalid note", () =>
databaseTest().then(async (db) => {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
let id = await db.notes.add();
expect(id).toBeUndefined();
id = await db.notes.add({});
expect(id).toBeUndefined();
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
id = await db.notes.add({ hello: "world" });
expect(id).toBeUndefined();
}));
test("add note", () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
expect(note.data).toBeDefined();
expect(await note.content()).toStrictEqual(TEST_NOTE.content.data);
const note = db.notes.note(id);
expect(note).toBeDefined();
expect(await note?.content()).toStrictEqual(TEST_NOTE.content.data);
}));
test("get note content", () =>
noteTest().then(async ({ db, id }) => {
let content = await db.notes.note(id).content();
const content = await db.notes.note(id)?.content();
expect(content).toStrictEqual(TEST_NOTE.content.data);
}));
test("delete note", () =>
noteTest().then(async ({ db, id }) => {
let notebookId = await db.notebooks.add(TEST_NOTEBOOK);
let topics = db.notebooks.notebook(notebookId).topics;
const notebookId = await db.notebooks.add(TEST_NOTEBOOK);
if (!notebookId) throw new Error("Could not create notebook.");
const topics = db.notebooks.topics(notebookId);
let topic = topics.topic("hello");
if (!topic) throw new Error("Could not find topic.");
await db.notes.addToNotebook({ id: notebookId, topic: topic.id }, id);
topic = topics.topic("hello");
if (!topic) throw new Error("Could not find topic.");
expect(topic.all.findIndex((v) => v.id === id)).toBeGreaterThan(-1);
await db.notes.delete(id);
expect(db.notes.note(id)).toBeUndefined();
expect(topic.all.findIndex((v) => v.id === id)).toBe(-1);
expect(db.notebooks.notebook(notebookId).totalNotes).toBe(0);
expect(topics.topic("hello").totalNotes).toBe(0);
expect(db.notebooks.totalNotes(notebookId)).toBe(0);
expect(topics.topic("hello")?.totalNotes).toBe(0);
}));
test("get all notes", () =>
@@ -78,8 +110,8 @@ test("get all notes", () =>
test("note without a title should get a premade title", () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
expect(note.title.startsWith("Note ")).toBe(true);
const note = db.notes.note(id);
expect(note?.title.startsWith("Note ")).toBe(true);
}));
test("note should get headline from content", () =>
@@ -90,8 +122,8 @@ test("note should get headline from content", () =>
data: "<p>This is a very colorful existence.</p>"
}
}).then(async ({ db, id }) => {
let note = db.notes.note(id);
expect(note.headline).toBe("This is a very colorful existence.");
const note = db.notes.note(id);
expect(note?.headline).toBe("This is a very colorful existence.");
}));
test("note should not get headline if there is no p tag", () =>
@@ -102,29 +134,29 @@ test("note should not get headline if there is no p tag", () =>
data: `<ol style="list-style-type: decimal;" data-mce-style="list-style-type: decimal;"><li>Hello I won't be a headline :(</li><li>Me too.</li><li>Gold.</li></ol>`
}
}).then(async ({ db, id }) => {
let note = db.notes.note(id);
expect(note.headline).toBe("");
const note = db.notes.note(id);
expect(note?.headline).toBe("");
}));
test("note title should allow trailing space", () =>
noteTest({ title: "Hello ", content: TEST_NOTE.content }).then(
async ({ db, id }) => {
let note = db.notes.note(id);
expect(note.title).toBe("Hello ");
const note = db.notes.note(id);
expect(note?.title).toBe("Hello ");
}
));
test("note title should not allow newlines", () =>
noteTest({ title: "Hello\nhello", content: TEST_NOTE.content }).then(
async ({ db, id }) => {
let note = db.notes.note(id);
expect(note.title).toBe("Hello hello");
const note = db.notes.note(id);
expect(note?.title).toBe("Hello hello");
}
));
test("update note", () =>
noteTest().then(async ({ db, id }) => {
let noteData = {
const noteData = {
id,
title: "I am a new title",
content: {
@@ -135,12 +167,12 @@ test("update note", () =>
favorite: true
// colors: ["red", "blue"]
};
id = await db.notes.add(noteData);
let note = db.notes.note(id);
expect(note.title).toBe(noteData.title);
expect(await note.content()).toStrictEqual(noteData.content.data);
expect(note.data.pinned).toBe(true);
expect(note.data.favorite).toBe(true);
await db.notes.add(noteData);
const note = db.notes.note(id);
expect(note?.title).toBe(noteData.title);
expect(await note?.content()).toStrictEqual(noteData.content.data);
expect(note?.data.pinned).toBe(true);
expect(note?.data.favorite).toBe(true);
}));
test("get favorite notes", () =>
@@ -167,139 +199,153 @@ test("get grouped notes by year", () => groupedTest("year"));
test("get grouped notes by weak", () => groupedTest("week"));
test("get grouped notes default", () => groupedTest());
test("get grouped notes default", () => groupedTest("default"));
test("pin note", () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
await note.pin();
await note?.pin();
note = db.notes.note(id);
expect(note.data.pinned).toBe(true);
expect(note?.data.pinned).toBe(true);
}));
test("favorite note", () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
await note.favorite();
await note?.favorite();
note = db.notes.note(id);
expect(note.data.favorite).toBe(true);
expect(note?.data.favorite).toBe(true);
}));
test("add note to topic", () =>
noteTest().then(async ({ db, id }) => {
let notebookId = await db.notebooks.add({ title: "Hello" });
let topics = db.notebooks.notebook(notebookId).topics;
await topics.add("Home");
let topic = topics.topic("Home");
const { topic, notebookId } = await createAndAddNoteToNotebook(db, id, {
notebookTitle: "Hello",
topicTitle: "Home"
});
await db.notes.addToNotebook({ id: notebookId, topic: topic.id }, id);
topic = topics.topic("Home");
expect(topic.all).toHaveLength(1);
expect(topic.totalNotes).toBe(1);
expect(db.notebooks.notebook(notebookId).totalNotes).toBe(1);
let note = db.notes.note(id);
expect(note.notebooks.some((n) => n.id === notebookId)).toBe(true);
expect(db.notebooks.totalNotes(notebookId)).toBe(1);
expect(db.notes.note(id)?.notebooks?.some((n) => n.id === notebookId)).toBe(
true
);
}));
test("duplicate note to topic should not be added", () =>
noteTest().then(async ({ db, id }) => {
let notebookId = await db.notebooks.add({ title: "Hello" });
let topics = db.notebooks.notebook(notebookId).topics;
await topics.add("Home");
let topic = topics.topic("Home");
await db.notes.addToNotebook({ id: notebookId, topic: topic.id }, id);
topic = topics.topic("Home");
expect(topic.all).toHaveLength(1);
const { topics } = await createAndAddNoteToNotebook(db, id, {
notebookTitle: "Hello",
topicTitle: "Home"
});
expect(topics.topic("Home")?.all).toHaveLength(1);
}));
test("add the same note to 2 notebooks", () =>
noteTest().then(async ({ db, id }) => {
let notebookId = await db.notebooks.add({ title: "Hello" });
let topics = db.notebooks.notebook(notebookId).topics;
await topics.add("Home");
let topic = topics.topic("Home")._topic;
await db.notes.addToNotebook({ id: notebookId, topic: topic.id }, id);
const nb1 = await createAndAddNoteToNotebook(db, id, {
notebookTitle: "Hello",
topicTitle: "Home"
});
const nb2 = await createAndAddNoteToNotebook(db, id, {
notebookTitle: "Hello2",
topicTitle: "Home2"
});
expect(topics.topic(topic.id).has(id)).toBe(true);
let notebookId2 = await db.notebooks.add({ title: "Hello2" });
let topics2 = db.notebooks.notebook(notebookId2).topics;
await topics2.add("Home2");
let topic2 = topics2.topic("Home2")._topic;
await db.notes.addToNotebook({ id: notebookId2, topic: topic2.id }, id);
let note = db.notes.note(id);
expect(note.notebooks).toHaveLength(2);
expect(topics2.topic(topic2.id).has(id)).toBe(true);
expect(nb1.topics.topic(nb1.topic.id)?.has(id)).toBe(true);
expect(nb2.topics.topic(nb2.topic.id)?.has(id)).toBe(true);
expect(db.notes.note(id)?.notebooks).toHaveLength(2);
}));
test("moving note to same notebook and topic should do nothing", () =>
noteTest().then(async ({ db, id }) => {
const notebookId = await db.notebooks.add({ title: "Hello" });
let topics = db.notebooks.notebook(notebookId).topics;
await topics.add("Home");
let topic = topics.topic("Home");
await db.notes.addToNotebook({ id: notebookId, topic: topic.id }, id);
const { notebookId, topic } = await createAndAddNoteToNotebook(db, id, {
notebookTitle: "Home",
topicTitle: "Hello"
});
await db.notes.addToNotebook({ id: notebookId, topic: topic.id }, id);
let note = db.notes.note(id);
expect(note.notebooks.some((n) => n.id === notebookId)).toBe(true);
expect(db.notes.note(id)?.notebooks?.some((n) => n.id === notebookId)).toBe(
true
);
}));
test("export note to html", () =>
noteTest().then(async ({ db, id }) => {
const html = await db.notes.note(id).export("html");
const html = await db.notes.export(id, { format: "html" });
if (!html) throw new Error("Failed to export.");
expect(html.includes(TEST_NOTE.content.data)).toBeTruthy();
}));
test("export note to md", () =>
noteTest().then(async ({ db, id }) => {
const md = await db.notes.note(id).export("md");
const md = await db.notes.export(id, { format: "md" });
if (!md) throw new Error("Failed to export.");
expect(md).toBeTypeOf("string");
expect(md.includes(`Hello This is colorful\n`)).toBeTruthy();
}));
test("export note to txt", () =>
noteTest().then(async ({ db, id }) => {
const txt = await db.notes.note(id).export("txt");
const txt = await db.notes.export(id, { format: "txt" });
if (!txt) throw new Error("Failed to export.");
expect(txt.includes("Hello")).toBeTruthy();
}));
test("deleting a colored note should remove it from that color", () =>
noteTest().then(async ({ db, id }) => {
await db.notes.note(id).color("yellow");
let color = db.colors.tag("yellow");
const colorId = await db.colors.add({
title: "yellow",
colorCode: "#ffff22"
});
const color = db.colors.color(colorId);
if (!color) throw new Error("Failed to add color.");
expect(color).toBeTruthy();
expect(color.noteIds.indexOf(id)).toBeGreaterThan(-1);
await db.relations.add(
{ id: colorId, type: "color" },
{ id, type: "note" }
);
expect(
db.relations
.from({ id: colorId, type: "color" }, "note")
.findIndex((r) => r.to.id === id)
).toBe(0);
await db.notes.delete(id);
color = db.colors.tag("yellow");
expect(color).toBeFalsy();
expect(
db.relations
.from({ id: colorId, type: "color" }, "note")
.findIndex((r) => r.to.id === id)
).toBe(-1);
// TODO expect(color.noteIds.indexOf(id)).toBe(-1);
}));
test("note's content should follow note's localOnly property", () =>
noteTest().then(async ({ db, id }) => {
await db.notes.note(id).localOnly();
await db.notes.note(id)?.localOnly();
let note = db.notes.note(id);
expect(note._note.localOnly).toBe(true);
let content = await db.content.raw(note._note.contentId);
expect(content.localOnly).toBe(true);
if (!note?.contentId) throw new Error("No content in note.");
await db.notes.note(id).localOnly();
expect(note?.data.localOnly).toBe(true);
let content = await db.content.raw(note.contentId);
expect(content?.localOnly).toBe(true);
await db.notes.note(id)?.localOnly();
note = db.notes.note(id);
expect(note._note.localOnly).toBe(false);
content = await db.content.raw(note._note.contentId);
expect(content.localOnly).toBe(false);
if (!note?.contentId) throw new Error("No content in note.");
expect(note?.data.localOnly).toBe(false);
content = await db.content.raw(note.contentId);
expect(content?.localOnly).toBe(false);
}));
test("grouping items where item.title is empty or undefined shouldn't throw", () => {
expect(
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
groupArray([{ title: "" }], {
groupBy: "abc",
sortBy: "title",
@@ -314,7 +360,7 @@ test("note content should not contain image base64 data after save", () =>
await db.notes.add({ id, content: { type: "tiptap", data: IMG_CONTENT } });
const note = db.notes.note(id);
const content = await note.content();
const content = await note?.content();
expect(content).not.toContain(`src="data:image/png;`);
expect(content).not.toContain(`src=`);
@@ -330,6 +376,7 @@ test("adding a note with an invalid tag should clean the tag array", () =>
})
).resolves.toBe("helloworld");
const note = db.notes.note("helloworld");
expect(note.tags).toHaveLength(0);
expect(
db.relations.to({ id: "helloworld", type: "note" }, "tag")
).toHaveLength(0);
}));

16
packages/core/__tests__/settings.test.js
View File

@@ -22,9 +22,9 @@ import { test, expect } from "vitest";
test("settings' dateModified should not update on init", () =>
databaseTest().then(async (db) => {
const beforeDateModified = db.settings._settings.dateModified;
const beforeDateModified = db.settings.raw.dateModified;
await db.settings.init();
const afterDateModified = db.settings._settings.dateModified;
const afterDateModified = db.settings.raw.dateModified;
expect(beforeDateModified).toBe(afterDateModified);
}));
@@ -37,18 +37,6 @@ test("settings' dateModified should update after merge conflict resolve", () =>
expect(afterDateModified).toBeGreaterThan(beforeDateModified);
}));
test("tag alias should update if aliases in settings update", () =>
databaseTest().then(async (db) => {
const tag = await db.tags.add("hello");
await db.settings.merge({
groupOptions: {},
aliases: {
[tag.id]: "hello232"
}
});
expect(db.tags.tag(tag.id).alias).toBe("hello232");
}));
test("save group options", () =>
databaseTest().then(async (db) => {
const groupOptions = {

12
packages/core/__tests__/shortcuts.test.js
View File

@@ -45,7 +45,7 @@ test("create a duplicate shortcut of notebook", () =>
test("create shortcut of a topic", () =>
notebookTest().then(async ({ db, id }) => {
const notebook = db.notebooks.notebook(id)._notebook;
const notebook = db.notebooks.notebook(id).data;
const topic = notebook.topics[0];
await db.shortcuts.add({
item: { type: "topic", id: topic.id, notebookId: id }
@@ -57,18 +57,18 @@ test("create shortcut of a topic", () =>
test("pin a tag", () =>
databaseTest().then(async (db) => {
const tag = await db.tags.add("HELLO!");
await db.shortcuts.add({ item: { type: "tag", id: tag.id } });
const tagId = await db.tags.add({ title: "HELLO!" });
await db.shortcuts.add({ item: { type: "tag", id: tagId } });
expect(db.shortcuts.all).toHaveLength(1);
expect(db.shortcuts.all[0].item.id).toBe(tag.id);
expect(db.shortcuts.all[0].item.id).toBe(tagId);
}));
test("remove shortcut", () =>
databaseTest().then(async (db) => {
const tag = await db.tags.add("HELLO!");
const tagId = await db.tags.add({ title: "HELLO!" });
const shortcutId = await db.shortcuts.add({
item: { type: "tag", id: tag.id }
item: { type: "tag", id: tagId }
});
expect(db.shortcuts.all).toHaveLength(1);

7
packages/core/__tests__/storage.test.js
View File

@@ -18,11 +18,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { NodeStorageInterface } from "../__mocks__/node-storage.mock";
import Storage from "../src/database/storage";
import { test, expect } from "vitest";
test("add a value", async () => {
const storage = new Storage(new NodeStorageInterface());
const storage = new NodeStorageInterface();
await storage.write("hello", "world");
let value = await storage.read("hello");
@@ -31,7 +30,7 @@ test("add a value", async () => {
});
test("remove", async () => {
const storage = new Storage(new NodeStorageInterface());
const storage = new NodeStorageInterface();
await storage.write("hello", "world");
await storage.remove("hello");
@@ -41,7 +40,7 @@ test("remove", async () => {
});
test("clear", async () => {
const storage = new Storage(new NodeStorageInterface());
const storage = new NodeStorageInterface();
await storage.write("hello", "world");
storage.clear();

132
packages/core/__tests__/tags.test.js
View File

@@ -1,132 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { noteTest, TEST_NOTE } from "./utils";
import { test, expect, describe } from "vitest";
function checkColorValue(note, value) {
expect(note.data.color).toBe(value);
}
function checkTagValue(note, value) {
expect(note.data.tags[0]).toBe(value);
}
describe.each([
["tag", "untag", "tagged", "hello"],
["color", "uncolor", "colored", "red"]
])("%s", (action, unaction, filter, value) => {
let check = action === "tag" ? checkTagValue : checkColorValue;
let collection = action === "tag" ? "tags" : "colors";
// let key = action === "tag" ? "tags" : "color";
test(`${action} a note`, () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
await note[action](value);
note = db.notes.note(id);
check(note, value);
expect(db[collection].all[0].title).toBe(value);
expect(db[collection].all[0].noteIds).toHaveLength(1);
}));
test(`${action} 2 notes`, () =>
noteTest().then(async ({ db, id }) => {
const id2 = await db.notes.add(TEST_NOTE);
let note = db.notes.note(id);
await note[action](value);
note = db.notes.note(id2);
await note[action](value);
expect(db[collection].all[0].title).toBe(value);
expect(db[collection].all[0].noteIds).toHaveLength(2);
}));
test(`${unaction} from note`, () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
await note[action](value);
note = db.notes.note(id);
check(note, value);
await note[unaction](value);
note = db.notes.note(id);
check(note, undefined);
expect(db[collection].all).toHaveLength(0);
}));
test(`get ${collection}`, () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
await note[action](value);
expect(db[collection].all.length).toBeGreaterThan(0);
}));
test(`get notes in ${action}`, () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
await note[action](value);
const tag = db[collection].all.find((v) => v.title === value);
const filteredNotes = db.notes[filter](tag.id);
check(db.notes.note(filteredNotes[0]), value);
}));
test(`rename a ${action}`, () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
await note[action](value);
let tag = db[collection].tag(value);
await db[collection].rename(tag.id, value + " new");
tag = db[collection].tag(tag.id);
expect(db[collection].alias(tag.id)).toBe(value + " new");
}));
test(`remove a ${action}`, () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
await note[action](value);
let tag = db[collection].tag(value);
await db[collection].remove(tag.id);
expect(db[collection].tag(value)).toBeUndefined();
}));
test(`elements in ${collection}.all contain alias property`, () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
await note[action](value);
expect(db[collection].all.every((item) => !!item.alias)).toBe(true);
}));
test(`invalid characters from ${action} title are removed`, () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
await note[action](`${value.toUpperCase()} \t\t\t\t\t\r\n\r\n`);
note = db.notes.note(id);
check(note, value);
}));
test(`accented characters from ${action} title are not removed`, () =>
noteTest().then(async ({ db, id }) => {
let note = db.notes.note(id);
const _value = `échantillo`;
await note[action](_value);
note = db.notes.note(id);
check(note, _value);
}));
});

90
packages/core/__tests__/tags.test.ts Normal file
View File

@@ -0,0 +1,90 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { databaseTest, noteTest, TEST_NOTE } from "./utils";
import { test, expect } from "vitest";
function tag(title: string) {
return { title };
}
function color(title: string) {
return { title, colorCode: "#ffff22" };
}
for (const type of ["tag", "color"] as const) {
const collection = type === "color" ? "colors" : "tags";
const item = type === "color" ? color : tag;
test(`${type} a note`, () =>
noteTest().then(async ({ db, id }) => {
const tagId = await db[collection].add(item("hello"));
await db.relations.add({ id: tagId, type }, { id, type: "note" });
expect(db[collection].all[0].title).toBe("hello");
expect(db.relations.from({ id: tagId, type }, "note")).toHaveLength(1);
}));
test(`${type} 2 notes`, () =>
noteTest().then(async ({ db, id }) => {
const id2 = await db.notes.add(TEST_NOTE);
if (!id2) throw new Error("Failed to create note.");
const tagId = await db[collection].add(item("hello"));
await db.relations.add({ id: tagId, type }, { id, type: "note" });
await db.relations.add({ id: tagId, type }, { id: id2, type: "note" });
expect(db[collection].all[0].title).toBe("hello");
expect(db.relations.from({ id: tagId, type }, "note")).toHaveLength(2);
}));
test(`rename a ${type}`, () =>
databaseTest().then(async (db) => {
const tagId = await db[collection].add(item("hello"));
await db[collection].add({ id: tagId, title: `hello (new)` });
expect(db[collection].all[0].title).toBe("hello (new)");
}));
test(`remove a ${type}`, () =>
noteTest().then(async ({ db, id }) => {
const tagId = await db[collection].add(item("hello"));
await db.relations.add({ id: tagId, type }, { id, type: "note" });
await db[collection].remove(tagId);
expect(db[collection].all).toHaveLength(0);
expect(db.relations.from({ id: tagId, type }, "note")).toHaveLength(0);
}));
test(`invalid characters from ${type} title are removed`, () =>
databaseTest().then(async (db) => {
await db[collection].add(
item(" \n\n\n\t\t\thello l\n\n\n\t\t ")
);
expect(db[collection].all[0].title).toBe("hello l");
}));
test(`remove a note from ${type}`, () =>
noteTest().then(async ({ db, id }) => {
const tagId = await db[collection].add(item("hello"));
await db.relations.add({ id: tagId, type }, { id, type: "note" });
await db.relations.unlink({ id: tagId, type }, { id, type: "note" });
expect(db.relations.from({ id: tagId, type }, "note")).toHaveLength(0);
}));
}

43
packages/core/__tests__/topics.test.js
View File

@@ -22,26 +22,26 @@ import { test, expect } from "vitest";
test("get empty topic", () =>
notebookTest().then(({ db, id }) => {
let topic = db.notebooks.notebook(id).topics.topic("hello");
let topic = db.notebooks.topics(id).topic("hello");
expect(topic.all).toHaveLength(0);
}));
test("getting invalid topic should return undefined", () =>
notebookTest().then(({ db, id }) => {
expect(db.notebooks.notebook(id).topics.topic("invalid")).toBeUndefined();
expect(db.notebooks.topics(id).topic("invalid")).toBeUndefined();
}));
test("add topic to notebook", () =>
notebookTest().then(async ({ db, id }) => {
let topics = db.notebooks.notebook(id).topics;
await topics.add("Home");
let topics = db.notebooks.topics(id);
await topics.add({ title: "Home" });
expect(topics.all.length).toBeGreaterThan(1);
expect(topics.all.findIndex((v) => v.title === "Home")).toBeGreaterThan(-1);
}));
test("add note to topic", () =>
notebookTest().then(async ({ db, id }) => {
let topics = db.notebooks.notebook(id).topics;
let topics = db.notebooks.topics(id);
let topic = topics.topic("hello");
let noteId = await db.notes.add(TEST_NOTE);
await db.notes.addToNotebook({ id, topic: topic.id }, noteId);
@@ -53,7 +53,7 @@ test("add note to topic", () =>
test("delete note of a topic", () =>
notebookTest().then(async ({ db, id }) => {
let topics = db.notebooks.notebook(id).topics;
let topics = db.notebooks.topics(id);
let topic = topics.topic("hello");
let noteId = await db.notes.add(TEST_NOTE);
await db.notes.addToNotebook({ id, topic: topic.id }, noteId);
@@ -71,9 +71,9 @@ test("delete note of a topic", () =>
test("edit topic title", () =>
notebookTest().then(async ({ db, id }) => {
let topics = db.notebooks.notebook(id).topics;
let topics = db.notebooks.topics(id);
await topics.add("Home");
await topics.add({ title: "Home" });
let topic = topics.topic("Home");
@@ -92,19 +92,10 @@ test("edit topic title", () =>
);
}));
test("duplicate topic to notebook should not be added", () =>
notebookTest().then(async ({ db, id }) => {
let topics = db.notebooks.notebook(id).topics;
await topics.add("Home");
let len = topics.all.length;
await topics.add("Home");
expect(topics.all).toHaveLength(len);
}));
test("get topic", () =>
notebookTest().then(async ({ db, id }) => {
let topics = db.notebooks.notebook(id).topics;
await topics.add("Home");
let topics = db.notebooks.topics(id);
await topics.add({ title: "Home" });
let topic = topics.topic("Home");
let noteId = await db.notes.add({
content: TEST_NOTE.content
@@ -118,17 +109,17 @@ test("get topic", () =>
test("delete a topic", () =>
notebookTest().then(async ({ db, id }) => {
let topics = db.notebooks.notebook(id).topics;
await topics.add("Home");
await topics.delete(topics.topic("Home")._topic.id);
let topics = db.notebooks.topics(id);
await topics.add({ title: "Home" });
await topics.delete(topics.topic("Home").id);
expect(topics.all.findIndex((v) => v.title === "Home")).toBe(-1);
}));
test("delete note from edited topic", () =>
notebookTest().then(async ({ db, id }) => {
const noteId = await db.notes.add(TEST_NOTE);
let topics = db.notebooks.notebook(id).topics;
await topics.add("Home");
let topics = db.notebooks.topics(id);
await topics.add({ title: "Home" });
let topic = topics.topic("Home");
await db.notes.addToNotebook({ id, topic: topic._topic.title }, noteId);
await topics.add({ id: topic._topic.id, title: "Hello22" });
@@ -137,9 +128,9 @@ test("delete note from edited topic", () =>
test("editing one topic should not update dateEdited of all", () =>
notebookTest().then(async ({ db, id }) => {
let topics = db.notebooks.notebook(id).topics;
let topics = db.notebooks.topics(id);
await topics.add("Home");
await topics.add({ title: "Home" });
await topics.add("Home2");
await topics.add("Home3");

10
packages/core/__tests__/trash.test.js
View File

@@ -46,7 +46,7 @@ test("permanently delete a note", () =>
await db.trash.delete(db.trash.all[0].id);
expect(db.trash.all).toHaveLength(0);
const content = await db.content.raw(note.data.contentId);
expect(content.deleted).toBe(true);
expect(content).toBeUndefined();
sessions = await db.noteHistory.get(noteId);
expect(sessions).toHaveLength(0);
@@ -175,12 +175,14 @@ test("permanently delete items older than 7 days", () =>
await db.notebooks.delete(notebookId);
await db.notes.delete(noteId);
await db.notes._collection.updateItem({
await db.notes.collection.update({
type: "trash",
id: noteId,
dateDeleted: sevenDaysEarlier
});
await db.notebooks._collection.updateItem({
await db.notebooks.collection.update({
type: "trash",
id: notebookId,
dateDeleted: sevenDaysEarlier
});
@@ -228,7 +230,7 @@ test("clear trash should delete note content", () =>
expect(db.trash.all).toHaveLength(0);
const content = await db.content.raw(note.contentId);
expect(content.deleted).toBe(true);
expect(content).toBeUndefined();
sessions = await db.noteHistory.get(note.id);
expect(sessions).toHaveLength(0);

1
packages/core/__tests__/utils.test.js
View File

@@ -32,6 +32,7 @@ test("group alphabetically", () => {
sortDirection: "asc",
sortBy: "title"
}).filter((v) => !v.item);
expect(
sortedAlphabet.every((alpha, index) => {
return ret[index].title === alpha.title.toUpperCase();

57
packages/core/__tests__/utils/index.js → packages/core/__tests__/utils/index.ts
View File

@@ -24,34 +24,45 @@ import { groupArray } from "../../src/utils/grouping";
import { FS } from "../../__mocks__/fs.mock";
import Compressor from "../../__mocks__/compressor.mock";
import { expect } from "vitest";
import EventSource from "eventsource";
import { EventSourcePolyfill as EventSource } from "event-source-polyfill";
import { randomBytes } from "../../src/utils/random";
import { GroupOptions, Note, Notebook, Topic } from "../../src/types";
import { NoteContent } from "../../src/collections/session-content";
const TEST_NOTEBOOK = {
const TEST_NOTEBOOK: Partial<
Omit<Notebook, "topics"> & { topics: Partial<Topic>[] }
> = {
title: "Test Notebook",
description: "Test Description",
topics: ["hello", "hello", " "]
topics: [{ title: "hello" }]
};
const TEST_NOTEBOOK2 = {
const TEST_NOTEBOOK2: Partial<
Omit<Notebook, "topics"> & { topics: Partial<Topic>[] }
> = {
title: "Test Notebook 2",
description: "Test Description 2",
topics: ["Home2"]
topics: [{ title: "Home2" }]
};
function databaseTest() {
let db = new DB();
db.setup(new NodeStorageInterface(), EventSource, FS, Compressor);
const db = new DB();
db.setup({
storage: new NodeStorageInterface(),
eventsource: EventSource,
fs: FS,
compressor: Compressor
});
return db.init().then(() => db);
}
const notebookTest = (notebook = TEST_NOTEBOOK) =>
databaseTest().then(async (db) => {
let id = await db.notebooks.add(notebook);
const id = await db.notebooks.add(notebook);
return { db, id };
});
var TEST_NOTE = {
const TEST_NOTE: { content: NoteContent<false> } = {
content: {
type: "tiptap",
data: `<p>Hello <span style="color:#f00">This is colorful</span></p>`
@@ -64,13 +75,18 @@ const IMG_CONTENT_WITHOUT_HASH = `<p>This is a note for me.j</p>\n<p><img src="d
const LONG_TEXT =
"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.";
const noteTest = (note = TEST_NOTE) =>
const noteTest = (
note: Partial<
Note & { content: NoteContent<false>; sessionId: string }
> = TEST_NOTE
) =>
databaseTest().then(async (db) => {
let id = await db.notes.add(note);
const id = await db.notes.add(note);
if (!id) throw new Error("Failed to add note.");
return { db, id };
});
const groupedTest = (type) =>
const groupedTest = (type: GroupOptions["groupBy"]) =>
noteTest().then(async ({ db }) => {
await db.notes.add({ ...TEST_NOTE, title: "HELLO WHAT!" });
await db.notes.add({
@@ -83,7 +99,7 @@ const groupedTest = (type) =>
title: "Some title and title title",
dateCreated: dayjs().subtract(2, "weeks").unix()
});
let grouped = groupArray(db.notes.all, {
const grouped = groupArray(db.notes.all, {
groupBy: type,
sortDirection: "desc",
sortBy: "dateCreated"
@@ -92,25 +108,24 @@ const groupedTest = (type) =>
expect(grouped.some((i) => i.type === "header")).toBe(true);
});
function delay(ms) {
function delay(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function loginFakeUser(db) {
const email = "johndoe@example.com";
const userSalt = randomBytes(16).toString("base64");
await db.storage.deriveCryptoKey(`_uk_@${email}`, {
await db.storage().deriveCryptoKey(`_uk_@${email}`, {
password: "password",
salt: userSalt
});
const userEncryptionKey = await db.storage.getCryptoKey(`_uk_@${email}`);
const userEncryptionKey = await db.storage().getCryptoKey(`_uk_@${email}`);
const key = await db.storage.generateRandomKey();
const attachmentsKey = await db.storage.encrypt(
{ password: userEncryptionKey },
JSON.stringify(key)
);
const key = await db.crypto().generateRandomKey();
const attachmentsKey = await db
.storage()
.encrypt({ password: userEncryptionKey }, JSON.stringify(key));
await db.user.setUser({
email,

2
packages/core/__tests__/vault.test.js
View File

@@ -23,7 +23,7 @@ import { test, expect } from "vitest";
test("create vault", () =>
databaseTest().then(async (db) => {
await expect(db.vault.create("password")).resolves.toBe(true);
const vaultKey = await db.storage.read("vaultKey");
const vaultKey = await db.storage().read("vaultKey");
expect(vaultKey).toBeDefined();
expect(vaultKey.iv).toBeDefined();
expect(vaultKey.cipher).toBeDefined();

2
packages/core/globals.setup.js
View File

@@ -20,7 +20,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import "isomorphic-fetch";
import dotenv from "dotenv";
import { DOMParser } from "linkedom";
import WebSocket from "ws";
globalThis.DOMParser = DOMParser;
globalThis.WebSocket = WebSocket;
require("abortcontroller-polyfill/dist/polyfill-patch-fetch");
dotenv.config();

4917
packages/core/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

9
packages/core/package.json
View File

@@ -9,10 +9,13 @@
},
"devDependencies": {
"@notesnook/crypto": "file:../crypto",
"@types/event-source-polyfill": "^1.0.1",
"@types/html-to-text": "^9.0.0",
"@types/katex": "^0.16.1",
"@types/katex": "^0.16.2",
"@types/prismjs": "^1.26.0",
"@types/spark-md5": "^3.0.2",
"@types/streetwriters__showdown": "npm:@types/showdown@^2.0.6",
"@types/ws": "^8.5.5",
"@vitest/coverage-v8": "^0.34.1",
"abortcontroller-polyfill": "^1.7.3",
"cross-env": "^7.0.3",
@@ -26,7 +29,8 @@
"otplib": "^12.0.1",
"refractor": "^4.8.1",
"vitest": "^0.34.1",
"vitest-fetch-mock": "^0.2.2"
"vitest-fetch-mock": "^0.2.2",
"ws": "^8.13.0"
},
"scripts": {
"prebuild": "node scripts/prebuild.mjs",
@@ -54,6 +58,7 @@
"mime-db": "1.52.0",
"prismjs": "^1.29.0",
"qclone": "^1.2.0",
"rfdc": "^1.3.0",
"spark-md5": "^3.0.2"
},
"overrides": {

13
packages/core/src/api/__tests__/__snapshots__/debug.test.js.snap
View File

@@ -1,13 +0,0 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`strip note > stripped-note 1`] = `"{\\"title\\":true,\\"description\\":false,\\"headline\\":true,\\"colored\\":false,\\"type\\":\\"note\\",\\"tags\\":[],\\"id\\":\\"hello\\",\\"contentId\\":\\"hello2\\",\\"dateModified\\":123,\\"dateEdited\\":123,\\"dateCreated\\":123}"`;
exports[`strip note with content > stripped-note-with-content 1`] = `"{\\"title\\":true,\\"description\\":false,\\"headline\\":true,\\"colored\\":false,\\"type\\":\\"note\\",\\"tags\\":[],\\"id\\":\\"hello\\",\\"contentId\\":\\"hello2\\",\\"dateModified\\":123,\\"dateEdited\\":123,\\"dateCreated\\":123,\\"additionalData\\":{\\"content\\":\\"{\\\\\\"title\\\\\\":false,\\\\\\"description\\\\\\":false,\\\\\\"headline\\\\\\":false,\\\\\\"colored\\\\\\":false,\\\\\\"type\\\\\\":\\\\\\"tiptap\\\\\\",\\\\\\"id\\\\\\":\\\\\\"hello\\\\\\",\\\\\\"dateModified\\\\\\":123,\\\\\\"dateEdited\\\\\\":123,\\\\\\"dateCreated\\\\\\":123}\\"}}"`;
exports[`strip notebook > stripped-notebook 1`] = `"{\\"title\\":true,\\"description\\":true,\\"headline\\":false,\\"colored\\":false,\\"type\\":\\"notebook\\",\\"id\\":\\"hello\\",\\"dateModified\\":123,\\"dateEdited\\":123,\\"dateCreated\\":123,\\"additionalData\\":[{\\"type\\":\\"topic\\",\\"id\\":\\"hello\\",\\"notebookId\\":\\"hello23\\",\\"title\\":\\"hello\\",\\"dateCreated\\":123,\\"dateEdited\\":123,\\"dateModified\\":123}]}"`;
exports[`strip tag > stripped-tag 1`] = `"{\\"title\\":true,\\"description\\":false,\\"headline\\":false,\\"colored\\":false,\\"type\\":\\"tag\\",\\"noteIds\\":[],\\"id\\":\\"hello\\",\\"dateModified\\":123,\\"dateEdited\\":123,\\"dateCreated\\":123}"`;
exports[`strip topic > stripped-topic 1`] = `"{\\"title\\":true,\\"description\\":false,\\"headline\\":false,\\"colored\\":false,\\"type\\":\\"topic\\",\\"id\\":\\"hello\\",\\"dateModified\\":123,\\"dateEdited\\":123,\\"dateCreated\\":123}"`;
exports[`strip trashed note > stripped-trashed-note 1`] = `"{\\"title\\":true,\\"description\\":false,\\"headline\\":true,\\"colored\\":false,\\"type\\":\\"trash\\",\\"tags\\":[],\\"id\\":\\"hello\\",\\"contentId\\":\\"hello2\\",\\"dateModified\\":123,\\"dateEdited\\":123,\\"dateDeleted\\":123,\\"dateCreated\\":123}"`;

92
packages/core/src/api/__tests__/debug.test.js
View File

@@ -17,82 +17,11 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Debug from "../debug";
import { noteTest, notebookTest, databaseTest } from "../../../__tests__/utils";
import { Debug } from "../debug";
import createFetchMock from "vitest-fetch-mock";
import { vi, test, expect } from "vitest";
const fetchMocker = createFetchMock(vi);
test("strip empty item shouldn't throw", () => {
const debug = new Debug();
expect(debug.strip()).toBe("{}");
});
test("strip note", () =>
noteTest().then(({ db, id }) => {
const note = db.notes.note(id)._note;
const debug = new Debug();
expect(debug.strip(normalizeItem(note))).toMatchSnapshot("stripped-note");
}));
test("strip trashed note", () =>
noteTest().then(async ({ db, id }) => {
await db.notes.delete(id);
const note = db.trash.all[0];
const debug = new Debug();
expect(debug.strip(normalizeItem(note))).toMatchSnapshot(
"stripped-trashed-note"
);
}));
test("strip note with content", () =>
noteTest().then(async ({ db, id }) => {
const note = db.notes.note(id)._note;
const debug = new Debug();
const content = await db.content.raw(note.contentId);
note.additionalData = {
content: db.debug.strip(normalizeItem(content))
};
expect(debug.strip(normalizeItem(note))).toMatchSnapshot(
"stripped-note-with-content"
);
}));
test("strip notebook", () =>
notebookTest().then(async ({ db, id }) => {
const notebook = db.notebooks.notebook(id)._notebook;
const debug = new Debug();
notebook.additionalData = notebook.topics.map((topic) =>
normalizeItem(topic)
);
expect(debug.strip(normalizeItem(notebook))).toMatchSnapshot(
"stripped-notebook"
);
}));
test("strip topic", () =>
notebookTest().then(async ({ db, id }) => {
const notebook = db.notebooks.notebook(id)._notebook;
const debug = new Debug();
expect(debug.strip(normalizeItem(notebook.topics[0]))).toMatchSnapshot(
"stripped-topic"
);
}));
test("strip tag", () =>
databaseTest().then(async (db) => {
const tag = await db.tags.add("Hello tag");
const debug = new Debug();
expect(debug.strip(normalizeItem(tag))).toMatchSnapshot("stripped-tag");
}));
test("reporting empty issue should return undefined", async () => {
const debug = new Debug();
expect(await debug.report()).toBeUndefined();
});
const SUCCESS_REPORT_RESPONSE = {
url: "https://reported/"
};
@@ -100,14 +29,12 @@ const SUCCESS_REPORT_RESPONSE = {
test("reporting issue should return issue url", async () => {
fetchMocker.enableMocks();
const debug = new Debug();
fetch.mockResponseOnce(JSON.stringify(SUCCESS_REPORT_RESPONSE), {
headers: { "Content-Type": "application/json" }
});
expect(
await debug.report({
await Debug.report({
title: "I am title",
body: "I am body",
userId: "anything"
@@ -120,8 +47,6 @@ test("reporting issue should return issue url", async () => {
test("reporting invalid issue should return undefined", async () => {
fetchMocker.enableMocks();
const debug = new Debug();
fetch.mockResponseOnce(
JSON.stringify({
error_description: "Invalid issue."
@@ -129,18 +54,7 @@ test("reporting invalid issue should return undefined", async () => {
{ status: 400, headers: { "Content-Type": "application/json" } }
);
expect(await debug.report({})).toBeUndefined();
expect(await Debug.report({})).toBeUndefined();
fetchMocker.disableMocks();
});
function normalizeItem(item) {
item.id = "hello";
item.notebookId = "hello23";
item.dateModified = 123;
item.dateEdited = 123;
item.dateCreated = 123;
if (item.dateDeleted) item.dateDeleted = 123;
if (item.contentId) item.contentId = "hello2";
return item;
}

67
packages/core/src/api/debug.js
View File

@@ -1,67 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import hosts from "../utils/constants";
export default class Debug {
strip(item) {
if (!item) return "{}";
return JSON.stringify({
title: !!item.title,
description: !!item.description,
headline: !!item.headline,
colored: !!item.color,
type: item.type,
notebooks: item.notebooks,
notes: item.notes,
noteIds: item.noteIds,
tags: item.tags,
id: item.id,
contentId: item.contentId,
dateModified: item.dateModified,
dateEdited: item.dateEdited,
dateDeleted: item.dateDeleted,
dateCreated: item.dateCreated,
additionalData: item.additionalData
});
}
/**
*
* @param {{
* title: string,
* body: string,
* userId: string | undefined
* }} reportData
* @returns {Promise<string>} link to the github issue
*/
async report(reportData) {
if (!reportData) return;
const { title, body, userId } = reportData;
const response = await fetch(`${hosts.ISSUES_HOST}/create/notesnook`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ title, body, userId })
});
if (!response.ok) return;
const json = await response.json();
return json.url;
}
}

38
packages/core/src/api/debug.ts Normal file
View File

@@ -0,0 +1,38 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import hosts from "../utils/constants";
export class Debug {
static async report(reportData: {
title: string;
body: string;
userId?: string;
}): Promise<string | undefined> {
const { title, body, userId } = reportData;
const response = await fetch(`${hosts.ISSUES_HOST}/create/notesnook`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ title, body, userId })
});
if (!response.ok) return;
const json = await response.json();
return json.url;
}
}

19
packages/core/src/api/healthcheck.js → packages/core/src/api/healthcheck.ts
View File

@@ -16,16 +16,21 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import hosts from "../utils/constants";
import http from "../utils/http";
export class HealthCheck {
static async isAuthServerHealthy() {
try {
const response = await http.get(`${hosts.AUTH_HOST}/health`);
return response.trim() === "Healthy";
} catch {
return false;
}
static async auth() {
return check(hosts.AUTH_HOST);
}
}
async function check(host: string) {
try {
const response = await http.get(`${host}/health`);
return response.trim() === "Healthy";
} catch {
return false;
}
}

315
packages/core/src/api/index.js
View File

@@ -1,315 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Notes from "../collections/notes";
import Storage from "../database/storage";
import FileStorage from "../database/fs";
import Notebooks from "../collections/notebooks";
import Trash from "../collections/trash";
import Tags from "../collections/tags";
import Sync from "./sync";
import Vault from "./vault";
import Lookup from "./lookup";
import Content from "../collections/content";
import Backup from "../database/backup";
import Session from "./session";
import Constants from "../utils/constants";
import { EV, EVENTS } from "../common";
import Settings from "./settings";
import Migrations from "./migrations";
import UserManager from "./user-manager";
import http from "../utils/http";
import Monographs from "./monographs";
import Offers from "./offers";
import Attachments from "../collections/attachments";
import Debug from "./debug";
import { Mutex } from "async-mutex";
import NoteHistory from "../collections/note-history";
import MFAManager from "./mfa-manager";
import EventManager from "../utils/event-manager";
import Pricing from "./pricing";
import { logger } from "../logger";
import Shortcuts from "../collections/shortcuts";
import Reminders from "../collections/reminders";
import Relations from "../collections/relations";
import Subscriptions from "./subscriptions";
/**
* @type {EventSource}
*/
var NNEventSource;
// const DIFFERENCE_THRESHOLD = 20 * 1000;
// const MAX_TIME_ERROR_FAILURES = 5;
class Database {
isInitialized = false;
/**
*
* @param {any} storage
* @param {EventSource} eventsource
*/
constructor() {
/**
* @type {EventSource}
*/
this.evtSource = null;
this.sseMutex = new Mutex();
this.lastHeartbeat = undefined; // { local: 0, server: 0 };
this.timeErrorFailures = 0;
this.eventManager = new EventManager();
}
setup(storage, eventsource, fs, compressor) {
this.compressor = compressor;
this.storage = storage ? new Storage(storage) : null;
this.fs = fs && storage ? new FileStorage(fs, storage) : null;
NNEventSource = eventsource;
this.session = new Session(this.storage);
this.user = new UserManager(this.storage, this);
this.mfa = new MFAManager(this.storage, this);
this.syncer = new Sync(this);
this.vault = new Vault(this);
this.lookup = new Lookup(this);
this.backup = new Backup(this);
this.settings = new Settings(this);
this.migrations = new Migrations(this);
this.monographs = new Monographs(this);
this.offers = new Offers();
this.debug = new Debug();
this.pricing = new Pricing();
this.subscriptions = new Subscriptions(this.user.tokenManager);
this.trash = new Trash(this);
}
async _validate() {
if (!(await this.session.valid())) {
throw new Error(
"Your system clock is not setup correctly. Please adjust your date and time and then retry."
);
}
await this.session.set();
}
async init() {
EV.subscribeMulti(
[EVENTS.userLoggedIn, EVENTS.userFetched, EVENTS.tokenRefreshed],
this.connectSSE,
this
);
EV.subscribe(EVENTS.attachmentDeleted, async (attachment) => {
await this.fs.cancel(attachment.metadata?.hash);
});
EV.subscribe(EVENTS.userLoggedOut, async () => {
await this.monographs.deinit();
await this.fs.clear();
this.disconnectSSE();
});
await this._validate();
await this.initCollections();
await this.migrations.init();
this.isInitialized = true;
if (this.migrations.required()) {
logger.warn("Database migration is required.");
}
}
async initCollections() {
await this.settings.init();
// collections
/** @type {Notebooks} */
this.notebooks = await Notebooks.new(this, "notebooks");
/** @type {Tags} */
this.tags = await Tags.new(this, "tags");
/** @type {Tags} */
this.colors = await Tags.new(this, "colors");
/** @type {Content} */
this.content = await Content.new(this, "content", false);
/** @type {Attachments} */
this.attachments = await Attachments.new(this, "attachments");
/**@type {NoteHistory} */
this.noteHistory = await NoteHistory.new(this, "notehistory", false);
/**@type {Shortcuts} */
this.shortcuts = await Shortcuts.new(this, "shortcuts");
/**@type {Reminders} */
this.reminders = await Reminders.new(this, "reminders");
/**@type {Relations} */
this.relations = await Relations.new(this, "relations");
/** @type {Notes} */
this.notes = await Notes.new(this, "notes");
await this.trash.init();
this.monographs.init().catch(console.error);
}
disconnectSSE() {
if (!this.evtSource) return;
this.evtSource.onopen = null;
this.evtSource.onmessage = null;
this.evtSource.onerror = null;
this.evtSource.close();
this.evtSource = null;
}
/**
*
* @param {{force: boolean, error: any}} args
*/
async connectSSE(args) {
if (args && !!args.error) return;
await this.sseMutex.runExclusive(async () => {
this.eventManager.publish(EVENTS.databaseSyncRequested, true, false);
const forceReconnect = args && args.force;
if (
!NNEventSource ||
(!forceReconnect &&
this.evtSource &&
this.evtSource.readyState === this.evtSource.OPEN)
)
return;
this.disconnectSSE();
const token = await this.user.tokenManager.getAccessToken();
if (!token) return;
this.evtSource = new NNEventSource(`${Constants.SSE_HOST}/sse`, {
headers: { Authorization: `Bearer ${token}` }
});
this.evtSource.onopen = async () => {
console.log("SSE: opened channel successfully!");
};
this.evtSource.onerror = function (error) {
console.log("SSE: error:", error);
};
this.evtSource.onmessage = async (event) => {
try {
var { type, data } = JSON.parse(event.data);
data = JSON.parse(data);
} catch (e) {
console.log("SSE: Unsupported message. Message = ", event.data);
return;
}
switch (type) {
// TODO: increase reliablity for this.
// case "heartbeat": {
// const { t: serverTime } = data;
// const localTime = Date.now();
// if (!this.lastHeartbeat) {
// this.lastHeartbeat = { local: localTime, server: serverTime };
// break;
// }
// const timeElapsed = {
// local: localTime - this.lastHeartbeat.local,
// server: serverTime - this.lastHeartbeat.server,
// };
// const travelTime = timeElapsed.local - timeElapsed.server;
// const actualTime = localTime - travelTime;
// const diff = actualTime - serverTime;
// // Fail several times consecutively before raising an error. This is done to root out
// // false positives.
// if (Math.abs(diff) > DIFFERENCE_THRESHOLD) {
// if (this.timeErrorFailures >= MAX_TIME_ERROR_FAILURES) {
// EV.publish(EVENTS.systemTimeInvalid, { serverTime, localTime });
// } else this.timeErrorFailures++;
// } else this.timeErrorFailures = 0;
// this.lastHeartbeat.local = localTime;
// this.lastHeartbeat.server = serverTime;
// break;
// }
case "upgrade": {
const user = await this.user.getUser();
user.subscription = data;
await this.user.setUser(user);
EV.publish(EVENTS.userSubscriptionUpdated, data);
break;
}
case "logout": {
await this.user.logout(true, data.reason || "Unknown.");
break;
}
case "emailConfirmed": {
const token = await this.storage.read("token");
await this.user.tokenManager._refreshToken(token);
await this.user.fetchUser(true);
EV.publish(EVENTS.userEmailConfirmed);
break;
}
}
};
});
}
async lastSynced() {
return (await this.storage.read("lastSynced")) || 0;
}
/**
*
* @param {{
* type: "full" | "fetch" | "send";
* force?: boolean;
* serverLastSynced?: number;
* }} options
* @returns
*/
sync(options) {
return this.syncer.start(options);
}
/**
*
* @param {{AUTH_HOST: string, API_HOST: string, SSE_HOST: string, SUBSCRIPTIONS_HOST: string, ISSUES_HOST: string}} hosts
*/
host(hosts) {
if (process.env.NODE_ENV !== "production") {
Constants.AUTH_HOST = hosts.AUTH_HOST || Constants.AUTH_HOST;
Constants.API_HOST = hosts.API_HOST || Constants.API_HOST;
Constants.SSE_HOST = hosts.SSE_HOST || Constants.SSE_HOST;
Constants.SUBSCRIPTIONS_HOST =
hosts.SUBSCRIPTIONS_HOST || Constants.SUBSCRIPTIONS_HOST;
Constants.ISSUES_HOST = hosts.ISSUES_HOST || Constants.ISSUES_HOST;
}
}
version() {
return http.get(`${Constants.API_HOST}/version`);
}
async announcements() {
let url = `${Constants.API_HOST}/announcements/active`;
const user = await this.user.getUser();
if (user) url += `?userId=${user.id}`;
return http.get(url);
}
}
export default Database;

316
packages/core/src/api/index.ts Normal file
View File

@@ -0,0 +1,316 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Notes } from "../collections/notes";
import { Crypto, CryptoAccessor } from "../database/crypto";
import { FileStorage, FileStorageAccessor } from "../database/fs";
import { Notebooks } from "../collections/notebooks";
import Trash from "../collections/trash";
import { Tags } from "../collections/tags";
import { Colors } from "../collections/colors";
import Sync, { SyncOptions } from "./sync";
import Vault from "./vault";
import Lookup from "./lookup";
import { Content } from "../collections/content";
import Backup from "../database/backup";
import Session from "./session";
import Hosts from "../utils/constants";
import { EV, EVENTS } from "../common";
import Settings from "../collections/settings";
import Migrations from "./migrations";
import UserManager from "./user-manager";
import http from "../utils/http";
import Monographs from "./monographs";
import { Offers } from "./offers";
import { Attachments } from "../collections/attachments";
import { Debug } from "./debug";
import { Mutex } from "async-mutex";
import { NoteHistory } from "../collections/note-history";
import MFAManager from "./mfa-manager";
import EventManager from "../utils/event-manager";
import { Pricing } from "./pricing";
import { logger } from "../logger";
import { Shortcuts } from "../collections/shortcuts";
import { Reminders } from "../collections/reminders";
import { Relations } from "../collections/relations";
import Subscriptions from "./subscriptions";
import {
CompressorAccessor,
ICompressor,
IFileStorage,
IStorage,
StorageAccessor
} from "../interfaces";
import TokenManager from "./token-manager";
import { Attachment } from "../types";
type EventSourceConstructor = new (
uri: string,
init: EventSourceInit & { headers?: Record<string, string> }
) => EventSource;
type Options = {
storage: IStorage;
eventsource?: EventSourceConstructor;
fs: IFileStorage;
compressor: ICompressor;
};
// const DIFFERENCE_THRESHOLD = 20 * 1000;
// const MAX_TIME_ERROR_FAILURES = 5;
class Database {
isInitialized = false;
eventManager = new EventManager();
sseMutex = new Mutex();
storage: StorageAccessor = () => {
if (!this.options?.storage)
throw new Error(
"Database not initialized. Did you forget to call db.setup()?"
);
return this.options.storage;
};
fs: FileStorageAccessor = () => {
if (!this.options?.fs)
throw new Error(
"Database not initialized. Did you forget to call db.setup()?"
);
return new FileStorage(this.options.fs, this.storage);
};
crypto: CryptoAccessor = () => {
if (!this.options)
throw new Error(
"Database not initialized. Did you forget to call db.setup()?"
);
return new Crypto(this.storage);
};
compressor: CompressorAccessor = () => {
if (!this.options?.compressor)
throw new Error(
"Database not initialized. Did you forget to call db.setup()?"
);
return this.options.compressor;
};
private options?: Options;
EventSource?: EventSourceConstructor;
eventSource?: EventSource | null;
session = new Session(this.storage);
mfa = new MFAManager(this.storage);
tokenManager = new TokenManager(this.storage);
subscriptions = new Subscriptions(this.tokenManager);
offers = new Offers();
debug = new Debug();
pricing = new Pricing();
user = new UserManager(this);
syncer = new Sync(this);
vault = new Vault(this);
lookup = new Lookup(this);
backup = new Backup(this);
settings = new Settings(this);
migrations = new Migrations(this);
monographs = new Monographs(this);
trash = new Trash(this);
notebooks = new Notebooks(this);
tags = new Tags(this);
colors = new Colors(this);
content = new Content(this);
attachments = new Attachments(this);
noteHistory = new NoteHistory(this);
shortcuts = new Shortcuts(this);
reminders = new Reminders(this);
relations = new Relations(this);
notes = new Notes(this);
// constructor() {
// this.sseMutex = new Mutex();
// // this.lastHeartbeat = undefined; // { local: 0, server: 0 };
// // this.timeErrorFailures = 0;
// }
setup(options: Options) {
this.options = options;
}
async _validate() {
if (!(await this.session.valid())) {
throw new Error(
"Your system clock is not setup correctly. Please adjust your date and time and then retry."
);
}
await this.session.set();
}
async init() {
EV.subscribeMulti(
[EVENTS.userLoggedIn, EVENTS.userFetched, EVENTS.tokenRefreshed],
this.connectSSE,
this
);
EV.subscribe(EVENTS.attachmentDeleted, async (attachment: Attachment) => {
await this.fs().cancel(attachment.metadata.hash, "upload");
await this.fs().cancel(attachment.metadata.hash, "download");
});
EV.subscribe(EVENTS.userLoggedOut, async () => {
await this.monographs.deinit();
await this.fs().clear();
this.disconnectSSE();
});
await this._validate();
await this.initCollections();
await this.migrations.init();
this.isInitialized = true;
if (this.migrations.required()) {
logger.warn("Database migration is required.");
}
}
async initCollections() {
await this.settings.init();
// collections
await this.notebooks.init();
await this.tags.init();
await this.colors.init();
await this.content.init();
await this.attachments.init();
await this.noteHistory.init();
await this.shortcuts.init();
await this.reminders.init();
await this.relations.init();
await this.notes.init();
await this.trash.init();
this.monographs.init().catch(console.error);
}
disconnectSSE() {
if (!this.eventSource) return;
this.eventSource.onopen = null;
this.eventSource.onmessage = null;
this.eventSource.onerror = null;
this.eventSource.close();
this.eventSource = null;
}
/**
*
* @param {{force: boolean, error: any}} args
*/
async connectSSE(args?: { force: boolean }) {
await this.sseMutex.runExclusive(async () => {
this.eventManager.publish(EVENTS.databaseSyncRequested, true, false);
const forceReconnect = args && args.force;
if (
!this.EventSource ||
(!forceReconnect &&
this.eventSource &&
this.eventSource.readyState === this.eventSource.OPEN)
)
return;
this.disconnectSSE();
const token = await this.tokenManager.getAccessToken();
if (!token) return;
this.eventSource = new this.EventSource(`${Hosts.SSE_HOST}/sse`, {
headers: { Authorization: `Bearer ${token}` }
});
this.eventSource.onopen = async () => {
console.log("SSE: opened channel successfully!");
};
this.eventSource.onerror = function (error) {
console.log("SSE: error:", error);
};
this.eventSource.onmessage = async (event) => {
try {
const message = JSON.parse(event.data);
const data = JSON.parse(message.data);
switch (message.type) {
case "upgrade": {
const user = await this.user.getUser();
if (!user) break;
user.subscription = data;
await this.user.setUser(user);
EV.publish(EVENTS.userSubscriptionUpdated, data);
break;
}
case "logout": {
await this.user.logout(true, data.reason || "Unknown.");
break;
}
case "emailConfirmed": {
await this.tokenManager._refreshToken(true);
await this.user.fetchUser();
EV.publish(EVENTS.userEmailConfirmed);
break;
}
}
} catch (e) {
console.log("SSE: Unsupported message. Message = ", event.data);
return;
}
};
});
}
async lastSynced() {
return (await this.storage().read<number | undefined>("lastSynced")) || 0;
}
sync(options: SyncOptions) {
return this.syncer.start(options);
}
host(hosts: typeof Hosts) {
if (process.env.NODE_ENV !== "production") {
Hosts.AUTH_HOST = hosts.AUTH_HOST || Hosts.AUTH_HOST;
Hosts.API_HOST = hosts.API_HOST || Hosts.API_HOST;
Hosts.SSE_HOST = hosts.SSE_HOST || Hosts.SSE_HOST;
Hosts.SUBSCRIPTIONS_HOST =
hosts.SUBSCRIPTIONS_HOST || Hosts.SUBSCRIPTIONS_HOST;
Hosts.ISSUES_HOST = hosts.ISSUES_HOST || Hosts.ISSUES_HOST;
}
}
version() {
return http.get(`${Hosts.API_HOST}/version`);
}
async announcements() {
let url = `${Hosts.API_HOST}/announcements/active`;
const user = await this.user.getUser();
if (user) url += `?userId=${user.id}`;
return http.get(url);
}
}
export default Database;

66
packages/core/src/api/lookup.js → packages/core/src/api/lookup.ts
View File

@@ -18,30 +18,42 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { filter, parse } from "liqe";
import Database from ".";
import {
Attachment,
Note,
Notebook,
Reminder,
Tag,
Topic,
TrashItem,
isDeleted
} from "../types";
import { isUnencryptedContent } from "../collections/content";
export default class Lookup {
/**
*
* @param {import('./index').default} db
*/
constructor(db) {
this._db = db;
}
constructor(private readonly db: Database) {}
async notes(notes, query) {
const contents = await this._db.content.multi(
async notes(notes: Note[], query: string) {
const contents = await this.db.content.multi(
notes.map((note) => note.contentId || "")
);
return search(notes, query, (note) => {
let text = note.title;
if (!note.locked && !!note.contentId && !!contents[note.contentId])
text += contents[note.contentId]["data"];
const noteContent = note.contentId ? contents[note.contentId] : "";
if (
!note.locked &&
noteContent &&
!isDeleted(noteContent) &&
isUnencryptedContent(noteContent)
)
text += noteContent.data;
return text;
});
}
notebooks(array, query) {
notebooks(array: Notebook[], query: string) {
return search(
array,
query,
@@ -50,36 +62,36 @@ export default class Lookup {
);
}
topics(array, query) {
return this._byTitle(array, query);
topics(array: Topic[], query: string) {
return this.byTitle(array, query);
}
tags(array, query) {
return this._byTitle(array, query);
tags(array: Tag[], query: string) {
return this.byTitle(array, query);
}
reminders(array, query) {
reminders(array: Reminder[], query: string) {
return search(array, query, (n) => `${n.title} ${n.description || ""}`);
}
trash(array, query) {
return this._byTitle(array, query);
trash(array: TrashItem[], query: string) {
return this.byTitle(array, query);
}
attachments(array, query) {
return search(array, query, (n) =>
n.metadata
? `${n.metadata.filename} ${n.metadata.type} ${n.metadata.hash}`
: ""
attachments(array: Attachment[], query: string) {
return search(
array,
query,
(n) => `${n.metadata.filename} ${n.metadata.type} ${n.metadata.hash}`
);
}
_byTitle(array, query) {
return search(array, query, (n) => n.alias || n.title);
private byTitle<T extends { title: string }>(array: T[], query: string) {
return search(array, query, (n) => n.title);
}
}
function search(items, query, selector) {
function search<T>(items: T[], query: string, selector: (item: T) => string) {
try {
return filter(
parse(`text:"${query.toLowerCase()}"`),

54
packages/core/src/api/mfa-manager.js → packages/core/src/api/mfa-manager.ts
View File

@@ -20,6 +20,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import http from "../utils/http";
import constants from "../utils/constants";
import TokenManager from "./token-manager";
import { StorageAccessor } from "../interfaces";
const ENDPOINTS = {
setup: "/mfa",
@@ -30,24 +31,12 @@ const ENDPOINTS = {
};
class MFAManager {
/**
*
* @param {import("../database/storage").default} storage
* @param {import("../api/index").default} db
*/
constructor(storage, db) {
this._storage = storage;
this._db = db;
tokenManager: TokenManager;
constructor(private readonly storage: StorageAccessor) {
this.tokenManager = new TokenManager(storage);
}
/**
*
* @param {"app" | "sms" | "email"} type
* @param {string} phoneNumber
* @returns
*/
async setup(type, phoneNumber = undefined) {
async setup(type: "app" | "sms" | "email", phoneNumber?: string) {
const token = await this.tokenManager.getAccessToken();
if (!token) return;
return await http.post(
@@ -60,13 +49,7 @@ class MFAManager {
);
}
/**
*
* @param {"app" | "sms" | "email"} type
* @param {string} code
* @returns
*/
async enable(type, code) {
async enable(type: "app" | "sms" | "email", code: string) {
return this._enable(type, code, false);
}
@@ -76,19 +59,15 @@ class MFAManager {
* @param {string} code
* @returns
*/
async enableFallback(type, code) {
async enableFallback(type: "app" | "sms" | "email", code: string) {
return this._enable(type, code, true);
}
/**
*
* @param {"app" | "sms" | "email"} type
* @param {string} code
* @param {boolean} isFallback
* @private
* @returns
*/
async _enable(type, code, isFallback) {
async _enable(
type: "app" | "sms" | "email",
code: string,
isFallback: boolean
) {
const token = await this.tokenManager.getAccessToken();
if (!token) return;
return await http.patch(
@@ -107,11 +86,6 @@ class MFAManager {
);
}
/**
* Generate new 2FA recovery codes or get count of valid recovery codes.
* @param {boolean} generate
* @returns
*/
async codes() {
const token = await this.tokenManager.getAccessToken();
if (!token) return;
@@ -121,11 +95,7 @@ class MFAManager {
);
}
/**
* @param {"sms" | "email"} method
* @returns
*/
async sendCode(method) {
async sendCode(method: "sms" | "email") {
const token = await this.tokenManager.getAccessToken();
if (!token) throw new Error("Unauthorized.");

116
packages/core/src/api/migrations.js
View File

@@ -1,116 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { CURRENT_DATABASE_VERSION } from "../common";
import Migrator from "../database/migrator";
class Migrations {
/**
*
* @param {import("./index").default} db
*/
constructor(db) {
this._db = db;
this._migrator = new Migrator();
this._isMigrating = false;
}
async init() {
this.dbVersion =
(await this._db.storage.read("v")) || CURRENT_DATABASE_VERSION;
this._db.storage.write("v", this.dbVersion);
}
required() {
return this.dbVersion < CURRENT_DATABASE_VERSION;
}
async migrate() {
try {
if (!this.required() || this._isMigrating) return;
this._isMigrating = true;
await this._db.notes.init();
const collections = [
{
index: () => this._db.attachments.all,
dbCollection: this._db.attachments
},
{
index: () => this._db.notebooks.raw,
dbCollection: this._db.notebooks
},
{
index: () => this._db.tags.raw,
dbCollection: this._db.tags
},
{
index: () => this._db.colors.raw,
dbCollection: this._db.colors
},
{
iterate: true,
dbCollection: this._db.content
},
{
index: () => [this._db.settings.raw],
dbCollection: this._db.settings,
type: "settings"
},
{
index: () => this._db.shortcuts.raw,
dbCollection: this._db.shortcuts
},
{
index: () => this._db.reminders.raw,
dbCollection: this._db.reminders
},
{
index: () => this._db.relations.raw,
dbCollection: this._db.relations
},
{
iterate: true,
dbCollection: this._db.noteHistory
},
{
iterate: true,
dbCollection: this._db.noteHistory.sessionContent
},
{
index: () => this._db.notes.raw,
dbCollection: this._db.notes
}
];
await this._migrator.migrate(
this._db,
collections,
(item) => item,
this.dbVersion
);
await this._db.storage.write("v", CURRENT_DATABASE_VERSION);
this.dbVersion = CURRENT_DATABASE_VERSION;
} finally {
this._isMigrating = false;
}
}
}
export default Migrations;

106
packages/core/src/api/migrations.ts Normal file
View File

@@ -0,0 +1,106 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Database from ".";
import { CURRENT_DATABASE_VERSION } from "../common";
import Migrator, { MigratableCollections } from "../database/migrator";
class Migrations {
private readonly migrator = new Migrator();
private migrating = false;
version = CURRENT_DATABASE_VERSION;
constructor(private readonly db: Database) {}
async init() {
this.version =
(await this.db.storage().read("v")) || CURRENT_DATABASE_VERSION;
this.db.storage().write("v", this.version);
}
required() {
return this.version < CURRENT_DATABASE_VERSION;
}
async migrate() {
try {
if (!this.required() || this.migrating) return;
this.migrating = true;
await this.db.notes.init();
const collections: MigratableCollections = [
{
items: () => this.db.attachments.all,
type: "attachments"
},
{
items: () => this.db.notebooks.raw,
type: "notebooks"
},
{
items: () => this.db.tags.raw,
type: "tags"
},
{
items: () => this.db.colors.raw,
type: "colors"
},
{
iterate: true,
type: "content"
},
{
items: () => [this.db.settings.raw],
type: "settings"
},
{
items: () => this.db.shortcuts.raw,
type: "shortcuts"
},
{
items: () => this.db.reminders.raw,
type: "reminders"
},
{
items: () => this.db.relations.raw,
type: "relations"
},
{
iterate: true,
type: "notehistory"
},
{
iterate: true,
type: "sessioncontent"
},
{
items: () => this.db.notes.raw,
type: "notes"
}
];
await this.migrator.migrate(this.db, collections, this.version);
await this.db.storage().write("v", CURRENT_DATABASE_VERSION);
this.version = CURRENT_DATABASE_VERSION;
} finally {
this.migrating = false;
}
}
}
export default Migrations;

16
packages/core/src/api/monographs.js
View File

@@ -32,7 +32,7 @@ class Monographs {
async deinit() {
this.monographs = undefined;
await this._db.storage.write("monographs", this.monographs);
await this._db.storage().write("monographs", this.monographs);
}
async init() {
@@ -40,9 +40,9 @@ class Monographs {
const user = await this._db.user.getUser();
const token = await this._db.user.tokenManager.getAccessToken();
if (!user || !token || !user.isEmailConfirmed) return;
let monographs = await this._db.storage.read("monographs", true);
let monographs = await this._db.storage().read("monographs", true);
monographs = await http.get(`${Constants.API_HOST}/monographs`, token);
await this._db.storage.write("monographs", monographs);
await this._db.storage().write("monographs", monographs);
if (monographs) this.monographs = monographs;
} catch (e) {
@@ -103,10 +103,12 @@ class Monographs {
};
if (opts.password) {
monograph.encryptedContent = await this._db.storage.encrypt(
{ password: opts.password },
JSON.stringify({ type: content.type, data: content.data })
);
monograph.encryptedContent = await this._db
.storage()
.encrypt(
{ password: opts.password },
JSON.stringify({ type: content.type, data: content.data })
);
} else {
monograph.content = JSON.stringify({
type: content.type,

4
packages/core/src/api/offers.js → packages/core/src/api/offers.ts
View File

@@ -21,8 +21,8 @@ import { CLIENT_ID } from "../common";
import hosts from "../utils/constants";
import http from "../utils/http";
export default class Offers {
async getCode(promo, platform) {
export class Offers {
static async getCode(promo: string, platform: "ios" | "android" | "web") {
const result = await http.get(
`${hosts.SUBSCRIPTIONS_HOST}/offers?promoCode=${promo}&clientId=${CLIENT_ID}&platformId=${platform}`
);

39
packages/core/src/api/pricing.js → packages/core/src/api/pricing.ts
View File

@@ -19,35 +19,24 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import http from "../utils/http";
type Product = {
country: string;
countryCode: string;
sku?: string;
price?: string;
discount: number;
};
const BASE_URL = `https://notesnook.com/api/v1/prices`;
class Pricing {
/**
*
* @param {"android"|"ios"|"web"} platform
* @param {"monthly"|"yearly"} period
* @returns {Promise<{
* country: string,
* countryCode: string,
* sku: string,
* discount: number
* }>}
*/
sku(platform, period) {
export class Pricing {
static sku(
platform: "android" | "ios" | "web",
period: "monthly" | "yearly"
): Promise<Product> {
return http.get(`${BASE_URL}/skus/${platform}/${period}`);
}
/**
*
* @param {"monthly"|"yearly"} period
* @returns {Promise<{
* country: string,
* countryCode: string,
* price: string,
* discount: number
* }>}
*/
price(period = "monthly") {
static price(period: "monthly" | "yearly" = "monthly"): Promise<Product> {
return http.get(`${BASE_URL}/${period}`);
}
}
export default Pricing;

14
packages/core/src/api/session.js → packages/core/src/api/session.ts
View File

@@ -17,21 +17,17 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { StorageAccessor } from "../interfaces";
class Session {
/**
*
* @param {import("../database/storage").default} context
*/
constructor(context) {
this._storage = context;
}
constructor(private readonly storage: StorageAccessor) {}
get() {
return this._storage.read("t");
return this.storage().read<number>("t");
}
set() {
return this._storage.write("t", Date.now());
return this.storage().write("t", Date.now());
}
async valid() {

232
packages/core/src/api/settings.js
View File

@@ -1,232 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { EV, EVENTS } from "../common";
import { getId } from "../utils/id";
import "../types";
class Settings {
/**
*
* @param {import("./index").default} db
*/
constructor(db) {
this._db = db;
}
async init() {
var settings = await this._db.storage.read("settings");
this._initSettings(settings);
await this._saveSettings(false);
EV.subscribe(EVENTS.userLoggedOut, async () => {
this._initSettings();
await this._saveSettings(false);
});
}
get raw() {
return this._settings;
}
async merge(remoteItem, lastSynced) {
if (this._settings.dateModified > lastSynced) {
this._settings = {
...this._settings,
...(remoteItem.deleted
? {}
: {
...remoteItem,
groupOptions: {
...this._settings.groupOptions,
...remoteItem.groupOptions
},
toolbarConfig: {
...this._settings.toolbarConfig,
...remoteItem.toolbarConfig
},
aliases: {
...this._settings.aliases,
...remoteItem.aliases
}
})
};
this._settings.dateModified = Date.now();
} else {
this._initSettings(remoteItem);
}
await this._saveSettings(false);
}
/**
*
* @param {GroupingKey} key
* @param {GroupOptions} groupOptions
*/
async setGroupOptions(key, groupOptions) {
if (!this._settings.groupOptions) this._settings.groupOptions = {};
this._settings.groupOptions[key] = groupOptions;
await this._saveSettings();
}
/**
*
* @param {GroupingKey} key
* @returns {GroupOptions}
*/
getGroupOptions(key) {
return (
(this._settings.groupOptions && this._settings.groupOptions[key]) || {
groupBy: "default",
sortBy:
key === "trash"
? "dateDeleted"
: key === "tags"
? "dateCreated"
: key === "reminders"
? "dueDate"
: "dateEdited",
sortDirection: key === "reminders" ? "asc" : "desc"
}
);
}
/**
*
* @param {string} key
* @param {{preset: string, config?: any[]}} config
*/
async setToolbarConfig(key, config) {
if (!this._settings.toolbarConfig) this._settings.toolbarConfig = {};
this._settings.toolbarConfig[key] = config;
await this._saveSettings();
}
/**
*
* @param {string} key
* @returns {{preset: string, config: any[]}}
*/
getToolbarConfig(key) {
return this._settings.toolbarConfig && this._settings.toolbarConfig[key];
}
async setAlias(id, name) {
if (!this._settings.aliases) this._settings.aliases = {};
this._settings.aliases[id] = name;
await this._saveSettings();
}
getAlias(id) {
return this._settings.aliases && this._settings.aliases[id];
}
/**
* Setting to -1 means never clear trash.
* @param {1 | 7 | 30 | 365 | -1} time
*/
async setTrashCleanupInterval(time) {
this._settings.trashCleanupInterval = time;
await this._saveSettings();
}
/**
* @returns {7 | 30 | 365 | -1}
*/
getTrashCleanupInterval() {
return this._settings.trashCleanupInterval || 7;
}
/**
*
* @param {{id: string, topic?: string} | undefined} item
*/
async setDefaultNotebook(item) {
this._settings.defaultNotebook = !item
? undefined
: {
id: item.id,
topic: item.topic
};
await this._saveSettings();
}
/**
*
* @returns {{id: string, topic?: string} | undefined}
*/
getDefaultNotebook() {
return this._settings.defaultNotebook;
}
async setTitleFormat(format) {
this._settings.titleFormat = format;
await this._saveSettings();
}
getTitleFormat() {
return this._settings.titleFormat || "Note $date$ $time$";
}
getDateFormat() {
return this._settings.dateFormat || "DD-MM-YYYY";
}
async setDateFormat(format) {
this._settings.dateFormat = format;
await this._saveSettings();
}
/**
*
* @returns {"12-hour" | "24-hour"}
*/
getTimeFormat() {
return this._settings.timeFormat || "12-hour";
}
async setTimeFormat(format) {
this._settings.timeFormat = format;
await this._saveSettings();
}
_initSettings(settings) {
this._settings = {
type: "settings",
id: getId(),
dateModified: 0,
dateCreated: 0,
...(settings || {})
};
}
async _saveSettings(updateDateModified = true) {
this._db.eventManager.publish(
EVENTS.databaseUpdated,
"settings",
this._settings
);
if (updateDateModified) {
this._settings.dateModified = Date.now();
this._settings.synced = false;
}
delete this._settings.remote;
await this._db.storage.write("settings", this._settings);
}
}
export default Settings;

53
packages/core/src/api/subscriptions.js → packages/core/src/api/subscriptions.ts
View File

@@ -19,23 +19,52 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import hosts from "../utils/constants";
import http from "../utils/http";
import TokenManager from "./token-manager";
export type TransactionStatus =
| "completed"
| "refunded"
| "partially_refunded"
| "disputed";
export type Transaction = {
order_id: string;
checkout_id: string;
amount: string;
currency: string;
status: TransactionStatus;
created_at: string;
passthrough: null;
product_id: number;
is_subscription: boolean;
is_one_off: boolean;
subscription: Subscription;
user: User;
receipt_url: string;
};
type Subscription = {
subscription_id: number;
status: string;
};
type User = {
user_id: number;
email: string;
marketing_consent: boolean;
};
export default class Subscriptions {
/**
* @param {import("../api/token-manager").default} tokenManager
*/
constructor(tokenManager) {
this._tokenManager = tokenManager;
}
constructor(private readonly tokenManager: TokenManager) {}
async cancel() {
const token = await this._tokenManager.getAccessToken();
const token = await this.tokenManager.getAccessToken();
if (!token) return;
await http.delete(`${hosts.SUBSCRIPTIONS_HOST}/subscriptions`, token);
}
async refund() {
const token = await this._tokenManager.getAccessToken();
const token = await this.tokenManager.getAccessToken();
if (!token) return;
await http.post(
`${hosts.SUBSCRIPTIONS_HOST}/subscriptions/refund`,
@@ -44,8 +73,8 @@ export default class Subscriptions {
);
}
async transactions() {
const token = await this._tokenManager.getAccessToken();
async transactions(): Promise<Transaction[] | undefined> {
const token = await this.tokenManager.getAccessToken();
if (!token) return;
return await http.get(
`${hosts.SUBSCRIPTIONS_HOST}/subscriptions/transactions`,
@@ -53,8 +82,8 @@ export default class Subscriptions {
);
}
async updateUrl() {
const token = await this._tokenManager.getAccessToken();
async updateUrl(): Promise<string | undefined> {
const token = await this.tokenManager.getAccessToken();
if (!token) return;
return await http.get(
`${hosts.SUBSCRIPTIONS_HOST}/subscriptions/update`,

2
packages/core/src/api/sync/__tests__/collector.test.js
View File

@@ -26,7 +26,7 @@ import {
import Collector from "../collector";
import { test, expect } from "vitest";
test("newly created note should get included in collector", () =>
test.only("newly created note should get included in collector", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const collector = new Collector(db);

449
packages/core/src/api/sync/__tests__/sync.test.js Normal file
View File

@@ -0,0 +1,449 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Database from "../../index";
import { NodeStorageInterface } from "../../../../__mocks__/node-storage.mock";
import { FS } from "../../../../__mocks__/fs.mock";
import Compressor from "../../../../__mocks__/compressor.mock";
import { CHECK_IDS, EV, EVENTS } from "../../../common";
import { EventSource } from "event-source-polyfill";
import { delay } from "../../../../__tests__/utils";
import { test, expect, vitest } from "vitest";
import { login } from "../../../../__e2e__/utils";
const TEST_TIMEOUT = 30 * 1000;
test(
"case 1: device A & B should only download the changes from device C (no uploading)",
async () => {
const types = [];
function onSyncProgress({ type }) {
types.push(type);
}
const [deviceA, deviceB, deviceC] = await Promise.all([
initializeDevice("deviceA"),
initializeDevice("deviceB"),
initializeDevice("deviceC")
]);
deviceA.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
deviceB.eventManager.subscribe(EVENTS.syncProgress, onSyncProgress);
await deviceC.notes.add({ title: "new note 1" });
await syncAndWait(deviceC, deviceC);
expect(types.every((t) => t === "download")).toBe(true);
await cleanup(deviceA, deviceB, deviceC);
},
TEST_TIMEOUT
);
test(
"case 3: Device A & B have unsynced changes but server has nothing",
async () => {
const [deviceA, deviceB] = await Promise.all([
initializeDevice("deviceA"),
initializeDevice("deviceB")
]);
const note1Id = await deviceA.notes.add({
title: "Test note from device A"
});
const note2Id = await deviceB.notes.add({
title: "Test note from device B"
});
await syncAndWait(deviceA, deviceB);
expect(deviceA.notes.note(note2Id)).toBeTruthy();
expect(deviceB.notes.note(note1Id)).toBeTruthy();
expect(deviceA.notes.note(note1Id)).toBeTruthy();
expect(deviceB.notes.note(note2Id)).toBeTruthy();
await cleanup(deviceA, deviceA);
},
TEST_TIMEOUT
);
// test(
// "case 4: Device A's sync is interrupted halfway and Device B makes some changes afterwards and syncs.",
// async () => {
// const deviceA = await initializeDevice("deviceA");
// const deviceB = await initializeDevice("deviceB");
// const unsyncedNoteIds = [];
// for (let i = 0; i < 10; ++i) {
// const id = await deviceA.notes.add({
// title: `Test note ${i} from device A`,
// });
// unsyncedNoteIds.push(id);
// }
// const half = unsyncedNoteIds.length / 2 + 1;
// deviceA.eventManager.subscribe(
// EVENTS.syncProgress,
// async ({ type, current }) => {
// if (type === "upload" && current === half) {
// await deviceA.syncer.stop();
// }
// }
// );
// await expect(deviceA.sync(true)).rejects.toThrow();
// let syncedNoteIds = [];
// for (let i = 0; i < unsyncedNoteIds.length; ++i) {
// const expectedNoteId = unsyncedNoteIds[i];
// if (deviceB.notes.note(expectedNoteId))
// syncedNoteIds.push(expectedNoteId);
// }
// expect(
// syncedNoteIds.length === half - 1 || syncedNoteIds.length === half
// ).toBe(true);
// const deviceBNoteId = await deviceB.notes.add({
// title: "Test note of case 4 from device B",
// });
// await deviceB.sync(true);
// await syncAndWait(deviceA, deviceB);
// expect(deviceA.notes.note(deviceBNoteId)).toBeTruthy();
// expect(
// unsyncedNoteIds
// .map((id) => !!deviceB.notes.note(id))
// .every((res) => res === true)
// ).toBe(true);
// await cleanup(deviceA, deviceB);
// },
//
// );
// test.only(
// "case 5: Device A's sync is interrupted halfway and Device B makes changes on the same note's content that didn't get synced on Device A due to interruption.",
// async () => {
// const deviceA = await initializeDevice("deviceA");
// const deviceB = await initializeDevice("deviceB");
// const noteIds = [];
// for (let i = 0; i < 10; ++i) {
// const id = await deviceA.notes.add({
// content: {
// type: "tiptap",
// data: `<p>deviceA=true</p>`,
// },
// });
// noteIds.push(id);
// }
// await deviceA.sync(true);
// await deviceB.sync(true);
// const unsyncedNoteIds = [];
// for (let id of noteIds) {
// const noteId = await deviceA.notes.add({
// id,
// content: {
// type: "tiptap",
// data: `<p>deviceA=true+changed=true</p>`,
// },
// });
// unsyncedNoteIds.push(noteId);
// }
// deviceA.eventManager.subscribe(
// EVENTS.syncProgress,
// async ({ type, total, current }) => {
// const half = total / 2 + 1;
// if (type === "upload" && current === half) {
// await deviceA.syncer.stop();
// }
// }
// );
// await expect(deviceA.sync(true)).rejects.toThrow();
// await delay(10 * 1000);
// for (let id of unsyncedNoteIds) {
// await deviceB.notes.add({
// id,
// content: {
// type: "tiptap",
// data: "<p>changes from device B</p>",
// },
// });
// }
// const error = await withError(async () => {
// await deviceB.sync(true);
// await deviceA.sync(true);
// });
// expect(error).not.toBeInstanceOf(NoErrorThrownError);
// expect(error.message.includes("Merge")).toBeTruthy();
// await cleanup(deviceA, deviceB);
// },
//
// );
test(
"issue: running force sync from device A makes device B always download everything",
async () => {
const [deviceA, deviceB] = await Promise.all([
initializeDevice("deviceA"),
initializeDevice("deviceB")
]);
await syncAndWait(deviceA, deviceB, true);
const handler = vitest.fn();
deviceB.eventManager.subscribe(EVENTS.syncProgress, handler);
await deviceB.sync(true);
expect(handler).not.toHaveBeenCalled();
await cleanup(deviceB);
},
TEST_TIMEOUT
);
test(
"issue: colors are not properly created if multiple notes are synced together",
async () => {
const [deviceA, deviceB] = await Promise.all([
initializeDevice("deviceA", [CHECK_IDS.noteColor]),
initializeDevice("deviceB", [CHECK_IDS.noteColor])
]);
const noteIds = [];
for (let i = 0; i < 3; ++i) {
const id = await deviceA.notes.add({
content: {
type: "tiptap",
data: `<p>deviceA=true</p>`
}
});
noteIds.push(id);
}
await syncAndWait(deviceA, deviceB);
const colorId = await deviceA.colors.add({
title: "yellow",
colorCode: "#ffff22"
});
for (let noteId of noteIds) {
expect(deviceB.notes.note(noteId)).toBeTruthy();
expect(
deviceB.relations
.from({ id: colorId, type: "color" }, "note")
.findIndex((a) => a.to.id === noteId)
).toBe(-1);
await deviceA.relations.add(
{ id: colorId, type: "color" },
{ id: noteId, type: "note" }
);
}
await syncAndWait(deviceA, deviceB);
expect(deviceB.colors.exists(colorId)).toBeTruthy();
const purpleNotes = deviceB.relations
.from({ id: colorId, type: "color" }, "note")
.resolved();
expect(
noteIds.every((id) => purpleNotes.findIndex((p) => p.id === id) > -1)
).toBe(true);
await cleanup(deviceA, deviceB);
},
TEST_TIMEOUT
);
test(
"issue: new topic on device A gets replaced by the new topic on device B",
async () => {
const [deviceA, deviceB] = await Promise.all([
initializeDevice("deviceA"),
initializeDevice("deviceB")
]);
// const deviceA = await initializeDevice("deviceA");
// const deviceB = await initializeDevice("deviceB");
const id = await deviceA.notebooks.add({ title: "Notebook 1" });
await syncAndWait(deviceA, deviceB, false);
expect(deviceB.notebooks.notebook(id)).toBeDefined();
await deviceA.notebooks.topics(id).add({ title: "Topic 1" });
// to create a conflict
await delay(1500);
await deviceB.notebooks.topics(id).add({ title: "Topic 2" });
expect(deviceA.notebooks.topics(id).has("Topic 1")).toBeTruthy();
expect(deviceB.notebooks.topics(id).has("Topic 2")).toBeTruthy();
await syncAndWait(deviceA, deviceB, false);
// await delay(1000);
// await syncAndWait(deviceB, deviceB, false);
expect(deviceA.notebooks.topics(id).has("Topic 1")).toBeTruthy();
expect(deviceB.notebooks.topics(id).has("Topic 1")).toBeTruthy();
expect(deviceA.notebooks.topics(id).has("Topic 2")).toBeTruthy();
expect(deviceB.notebooks.topics(id).has("Topic 2")).toBeTruthy();
await cleanup(deviceA, deviceB);
},
TEST_TIMEOUT
);
test(
"issue: remove notebook reference from notes that are removed from topic during merge",
async () => {
const [deviceA, deviceB] = await Promise.all([
initializeDevice("deviceA"),
initializeDevice("deviceB")
]);
const id = await deviceA.notebooks.add({
title: "Notebook 1",
topics: [{ title: "Topic 1" }]
});
const topic = deviceA.notebooks.topics(id).topic("Topic 1");
await syncAndWait(deviceA, deviceB, false);
expect(deviceB.notebooks.notebook(id)).toBeDefined();
const noteA = await deviceA.notes.add({ title: "Note 1" });
await deviceA.notes.addToNotebook({ id, topic: topic.id }, noteA);
expect(topic.totalNotes).toBe(1);
await delay(2000);
const noteB = await deviceB.notes.add({ title: "Note 2" });
await deviceB.notes.addToNotebook({ id, topic: topic.id }, noteB);
expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(1);
await syncAndWait(deviceB, deviceA, false);
await syncAndWait(deviceA, deviceB, false);
expect(deviceA.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);
expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);
expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(1);
expect(deviceA.notes.note(noteB).data.notebooks).toHaveLength(1);
await cleanup(deviceA, deviceB);
},
TEST_TIMEOUT
);
/**
*
* @param {string} id
* @returns {Promise<Database>}
*/
async function initializeDevice(id, capabilities = []) {
console.time(`Init ${id}`);
EV.subscribe(EVENTS.userCheckStatus, async (type) => {
return {
type,
result: capabilities.indexOf(type) > -1
};
});
EV.subscribe(EVENTS.syncCheckStatus, async (type) => {
return {
type,
result: true
};
});
const device = new Database();
device.setup({
storage: new NodeStorageInterface(),
eventsource: EventSource,
fs: FS,
compressor: Compressor
});
await device.init();
await login(device);
await device.user.resetUser(false);
await device.sync(true, false);
console.timeEnd(`Init ${id}`);
return device;
}
/**
*
* @param {...Database} devices
*/
async function cleanup(...devices) {
await Promise.all([
devices.map(async (device) => {
await device.syncer.stop();
await device.user.logout();
device.eventManager.unsubscribeAll();
})
]);
EV.unsubscribeAll();
}
/**
*
* @param {Database} deviceA
* @param {Database} deviceB
* @returns
*/
function syncAndWait(deviceA, deviceB, force = false) {
return new Promise((resolve) => {
const ref = deviceB.eventManager.subscribe(EVENTS.syncCompleted, () => {
ref.unsubscribe();
console.log("sync completed.");
resolve();
});
console.log(
"waiting for sync...",
"Device A:",
deviceA.syncer.sync.syncing,
"Device B:",
deviceB.syncer.sync.syncing
);
deviceA.sync(true, force);
});
}

34
packages/core/src/api/sync/auto-sync.js → packages/core/src/api/sync/auto-sync.ts
View File

@@ -17,22 +17,21 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Database from "..";
import { EVENTS } from "../../common";
import { logger } from "../../logger";
import { Item } from "../../types";
export class AutoSync {
/**
*
* @param {import("../index").default} db
* @param {number} interval
*/
constructor(db, interval) {
this.db = db;
this.interval = interval;
this.timeout = null;
this.isAutoSyncing = false;
this.logger = logger.scope("AutoSync");
}
timeout = 0;
isAutoSyncing = false;
logger = logger.scope("AutoSync");
databaseUpdatedEvent?: { unsubscribe: () => boolean };
constructor(
private readonly db: Database,
private readonly interval: number
) {}
async start() {
this.logger.info(`Auto sync requested`);
@@ -55,13 +54,12 @@ export class AutoSync {
this.logger.info(`Auto sync stopped`);
}
/**
* @private
*/
schedule(id, item) {
private schedule(id: string, item?: Item) {
if (
item &&
(item.remote || item.localOnly || item.failed || !!item.dateUploaded)
(item.remote ||
("localOnly" in item && item.localOnly) ||
("failed" in item && item.failed) || ("dateUploaded" in item && item.dateUploaded))
)
return;
@@ -76,6 +74,6 @@ export class AutoSync {
this.timeout = setTimeout(() => {
this.logger.info(`Sync requested by: ${id}`);
this.db.eventManager.publish(EVENTS.databaseSyncRequested, false, false);
}, interval);
}, interval) as unknown as number;
}
}

156
packages/core/src/api/sync/collector.js
View File

@@ -1,156 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { CURRENT_DATABASE_VERSION } from "../../common";
import { logger } from "../../logger";
const SYNC_COLLECTIONS_MAP = {
note: "notes",
notebook: "notebooks",
shortcut: "shortcuts",
reminder: "reminders",
relation: "relations"
};
const ASYNC_COLLECTIONS_MAP = {
content: "content"
};
class Collector {
/**
*
* @param {import("../index").default} db
*/
constructor(db) {
this._db = db;
this.logger = logger.scope("SyncCollector");
}
async *collect(chunkSize, lastSyncedTimestamp, isForceSync) {
const key = await this._db.user.getEncryptionKey();
const settings = await this.prepareChunk(
[this._db.settings.raw],
lastSyncedTimestamp,
isForceSync,
key,
"settings"
);
if (settings) yield settings;
const attachments = await this.prepareChunk(
this._db.attachments.syncable,
lastSyncedTimestamp,
isForceSync,
key,
"attachment"
);
if (attachments) yield attachments;
for (const itemType in ASYNC_COLLECTIONS_MAP) {
const collectionKey = ASYNC_COLLECTIONS_MAP[itemType];
const collection = this._db[collectionKey]._collection;
for await (const chunk of collection.iterate(chunkSize)) {
const items = await this.prepareChunk(
chunk.map((item) => item[1]),
lastSyncedTimestamp,
isForceSync,
key,
itemType
);
if (!items) continue;
yield items;
}
}
for (const itemType in SYNC_COLLECTIONS_MAP) {
const collectionKey = SYNC_COLLECTIONS_MAP[itemType];
const collection = this._db[collectionKey]._collection;
for (const chunk of collection.iterateSync(chunkSize)) {
const items = await this.prepareChunk(
chunk,
lastSyncedTimestamp,
isForceSync,
key,
itemType
);
if (!items) continue;
yield items;
}
}
}
async prepareChunk(chunk, lastSyncedTimestamp, isForceSync, key, itemType) {
const { ids, items } = filterSyncableItems(
chunk,
lastSyncedTimestamp,
isForceSync
);
if (!ids.length) return;
const ciphers = await this._db.storage.encryptMulti(key, items);
return toPushItem(itemType, ids, ciphers);
}
}
export default Collector;
function toPushItem(type, ids, ciphers) {
const items = ciphers.map((cipher, index) => {
cipher.v = CURRENT_DATABASE_VERSION;
cipher.id = ids[index];
return cipher;
});
return {
items,
type
};
}
function filterSyncableItems(items, lastSyncedTimestamp, isForceSync) {
if (!items || !items.length) return { items: [], ids: [] };
const ids = [];
const syncableItems = [];
for (const item of items) {
if (!item) continue;
const isSyncable = !item.synced || isForceSync;
const isUnsynced = item.dateModified > lastSyncedTimestamp || isForceSync;
// in case of resolved content
delete item.resolved;
// synced is a local only property
delete item.synced;
if (isUnsynced && isSyncable) {
ids.push(item.id);
syncableItems.push(
JSON.stringify(
item.localOnly
? {
id: item.id,
deleted: true,
dateModified: item.dateModified,
deleteReason: "localOnly"
}
: item
)
);
}
}
return { items: syncableItems, ids };
}

144
packages/core/src/api/sync/collector.ts Normal file
View File

@@ -0,0 +1,144 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Cipher, SerializedKey } from "@notesnook/crypto";
import Database from "..";
import { CURRENT_DATABASE_VERSION } from "../../common";
import { Item, MaybeDeletedItem } from "../../types";
export type SyncableItemType =
| "note"
| "shortcut"
| "notebook"
| "content"
| "attachment"
| "reminder"
| "relation"
| "color"
| "tag"
| "settings";
export type CollectedResult = {
items: (MaybeDeletedItem<Item> | Cipher)[];
types: (SyncableItemType | "vaultKey")[];
};
export type SyncItem = {
id: string;
v: number;
} & Cipher;
class Collector {
private lastSyncedTimestamp = 0;
private key?: SerializedKey;
constructor(private readonly db: Database) {}
async collect(lastSyncedTimestamp: number, isForceSync?: boolean) {
await this.db.notes.init();
this.lastSyncedTimestamp = lastSyncedTimestamp;
this.key = await this.db.user.getEncryptionKey();
const vaultKey = await this.db.vault.getKey();
const collections = {
note: this.db.notes.raw,
shortcut: this.db.shortcuts.raw,
notebook: this.db.notebooks.raw,
content: await this.db.content.all(),
attachment: this.db.attachments.syncable,
reminder: this.db.reminders.raw,
relation: this.db.relations.raw,
color: this.db.colors.raw,
tag: this.db.tags.raw,
settings: [this.db.settings.raw]
};
const result: CollectedResult = {
items: [],
types: []
};
for (const type in collections) {
this.collectInternal(
type as SyncableItemType,
collections[type as SyncableItemType],
result,
isForceSync
);
}
if (vaultKey) {
result.items.push(vaultKey);
result.types.push("vaultKey");
}
return result;
}
private serialize(item: MaybeDeletedItem<Item>) {
if (!this.key) throw new Error("No encryption key found.");
return this.db.storage().encrypt(this.key, JSON.stringify(item));
}
encrypt(array: MaybeDeletedItem<Item>[]) {
if (!array.length) return [];
return Promise.all(array.map(this.map, this));
}
private collectInternal(
itemType: SyncableItemType,
items: MaybeDeletedItem<Item>[],
result: CollectedResult,
isForceSync?: boolean
) {
if (!items || !items.length) return;
for (const item of items) {
if (!item) continue;
const isSyncable = !item.synced || isForceSync;
const isUnsynced =
item.dateModified > this.lastSyncedTimestamp || isForceSync;
if (isUnsynced && isSyncable) {
result.items.push(
"localOnly" in item && item.localOnly
? {
id: item.id,
deleted: true,
dateModified: item.dateModified,
deleteReason: "localOnly"
}
: item
);
result.types.push(itemType);
}
}
}
private async map(item: MaybeDeletedItem<Item>) {
// synced is a local only property
delete item.synced;
return {
id: item.id,
v: CURRENT_DATABASE_VERSION,
...(await this.serialize(item))
};
}
}
export default Collector;

20
packages/core/src/api/sync/conflicts.js → packages/core/src/api/sync/conflicts.ts
View File

@@ -17,29 +17,25 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Database from "..";
class Conflicts {
/**
*
* @param {import('../index').default} db
*/
constructor(db) {
this._db = db;
}
constructor(private readonly db: Database) {}
async recalculate() {
if (this._db.notes.conflicted.length <= 0) {
await this._db.storage.write("hasConflicts", false);
if (this.db.notes.conflicted.length <= 0) {
await this.db.storage().write("hasConflicts", false);
}
}
check() {
return this._db.storage.read("hasConflicts");
return this.db.storage().read("hasConflicts");
}
throw() {
throw new Error(
"Merge conflicts detected. Please resolve all conflicts to continue syncing.",
{ cause: "MERGE_CONFLICT" }
"Merge conflicts detected. Please resolve all conflicts to continue syncing."
// { cause: "MERGE_CONFLICT" }
);
}
}

162
packages/core/src/api/sync/index.js → packages/core/src/api/sync/index.ts
View File

@@ -26,43 +26,59 @@ import {
} from "../../common";
import Constants from "../../utils/constants";
import TokenManager from "../token-manager";
import Collector from "./collector";
import Collector, {
CollectedResult,
SyncableItemType,
SyncItem
} from "./collector";
import * as signalr from "@microsoft/signalr";
import Merger from "./merger";
import Conflicts from "./conflicts";
import { AutoSync } from "./auto-sync";
import { logger } from "../../logger";
import { Mutex } from "async-mutex";
import Database from "..";
import { migrateItem } from "../../migrations";
import { SerializedKey } from "@notesnook/crypto";
/**
* @typedef {{
* items: any[],
* type: string,
* }} SyncTransferItem
*/
const ITEM_TYPE_TO_COLLECTION_TYPE = {
note: "notes",
notebook: "notebooks",
content: "content",
attachment: "attachments",
relation: "relations",
reminder: "reminders",
shortcut: "shortcuts"
};
export type SyncOptions = {
type: "full" | "fetch" | "send";
force?: boolean;
serverLastSynced?: number;
};
type SyncTransferItem = {
items: SyncItem[];
type: SyncableItemType;
};
export default class SyncManager {
/**
*
* @param {import("../index").default} db
*/
constructor(db) {
this.sync = new Sync(db);
this._db = db;
}
sync = new Sync(this.db);
constructor(private readonly db: Database) {}
async start(options) {
async start(options: SyncOptions) {
try {
await this.sync.autoSync.start();
await this.sync.start(options);
return true;
} catch (e) {
var isHubException = e.message.includes("HubException:");
const isHubException = (e as Error).message.includes("HubException:");
if (isHubException) {
var actualError = /HubException: (.*)/gm.exec(e.message);
var actualError = /HubException: (.*)/gm.exec((e as Error).message);
const errorText =
actualError && actualError.length > 1 ? actualError[1] : e.message;
actualError && actualError.length > 1
? actualError[1]
: (e as Error).message;
// NOTE: sometimes there's the case where the user has already
// confirmed their email but the server still thinks that it
@@ -71,9 +87,9 @@ export default class SyncManager {
if (
(errorText.includes("Please confirm your email ") ||
errorText.includes("Invalid token.")) &&
(await this._db.user.getUser()).isEmailConfirmed
(await this.db.user.getUser())?.isEmailConfirmed
) {
await this._db.user.tokenManager._refreshToken(true);
await this.db.tokenManager._refreshToken(true);
return false;
}
@@ -83,7 +99,7 @@ export default class SyncManager {
}
}
async acquireLock(callback) {
async acquireLock(callback: () => Promise<void>) {
try {
this.sync.autoSync.stop();
await callback();
@@ -98,33 +114,25 @@ export default class SyncManager {
}
class Sync {
/**
*
* @param {import("../index").default} db
*/
constructor(db) {
this.db = db;
this.conflicts = new Conflicts(db);
this.collector = new Collector(db);
this.merger = new Merger(db);
this.autoSync = new AutoSync(db, 1000);
this.logger = logger.scope("Sync");
this.syncConnectionMutex = new Mutex();
this.itemTypeToCollection = {
note: "notes",
notebook: "notebooks",
content: "content",
attachment: "attachments",
relation: "relations",
reminder: "reminders",
shortcut: "shortcuts"
};
conflicts = new Conflicts(this.db);
collector = new Collector(this.db);
merger = new Merger(this.db);
autoSync = new AutoSync(this.db, 1000);
logger = logger.scope("Sync");
syncConnectionMutex = new Mutex();
connection: signalr.HubConnection;
constructor(private readonly db: Database) {
let remoteSyncTimeout = 0;
const tokenManager = new TokenManager(db.storage);
this.connection = new signalr.HubConnectionBuilder()
.withUrl(`${Constants.API_HOST}/hubs/sync`, {
accessTokenFactory: () => tokenManager.getAccessToken(),
accessTokenFactory: async () => {
const token = await tokenManager.getAccessToken();
if (!token) throw new Error("Failed to get access token.");
return token;
},
skipNegotiation: true,
transport: signalr.HttpTransportType.WebSockets,
logger: {
@@ -151,7 +159,6 @@ class Sync {
this.autoSync.stop();
});
let count = 0;
this.connection.on("PushItems", async (chunk) => {
if (this.connection.state !== signalr.HubConnectionState.Connected)
return;
@@ -159,7 +166,7 @@ class Sync {
clearTimeout(remoteSyncTimeout);
remoteSyncTimeout = setTimeout(() => {
this.db.eventManager.publish(EVENTS.syncAborted);
}, 15000);
}, 15000) as unknown as number;
const key = await this.db.user.getEncryptionKey();
const dbLastSynced = await this.db.lastSynced();
@@ -167,21 +174,12 @@ class Sync {
});
this.connection.on("PushCompleted", (lastSynced) => {
count = 0;
clearTimeout(remoteSyncTimeout);
this.onPushCompleted(lastSynced);
});
}
/**
*
* @param {{
* type: "full" | "fetch" | "send";
* force?: boolean;
* serverLastSynced?: number;
* }} options
*/
async start(options) {
async start(options: SyncOptions) {
if (!(await checkSyncStatus(SYNC_CHECK_IDS.sync))) {
await this.connection.stop();
return;
@@ -206,7 +204,7 @@ class Sync {
options.type === "fetch" || options.type === "full"
? await this.fetch(lastSynced)
: null;
this.logger.info("Data fetched", serverResponse);
this.logger.info("Data fetched", serverResponse || {});
if (
(options.type === "send" || options.type === "full") &&
@@ -227,7 +225,7 @@ class Sync {
}
}
async init(isForceSync) {
async init(isForceSync?: boolean) {
await this.checkConnection();
await this.conflicts.recalculate();
@@ -242,7 +240,7 @@ class Sync {
return { lastSynced, oldLastSynced };
}
async fetch(lastSynced) {
async fetch(lastSynced: number) {
await this.checkConnection();
const key = await this.db.user.getEncryptionKey();
@@ -339,7 +337,7 @@ class Sync {
this.logger.info("Stopping sync", { lastSynced });
const storedLastSynced = await this.db.lastSynced();
if (lastSynced > storedLastSynced)
await this.db.storage.write("lastSynced", lastSynced);
await this.db.storage().write("lastSynced", lastSynced);
this.db.eventManager.publish(EVENTS.syncCompleted);
}
@@ -355,8 +353,12 @@ class Sync {
const attachments = this.db.attachments.pending;
this.logger.info("Uploading attachments...", { total: attachments.length });
await this.db.fs.queueUploads(
attachments.map((a) => ({ filename: a.metadata.hash })),
await this.db.fs().queueUploads(
attachments.map((a) => ({
filename: a.metadata.hash,
chunkSize: a.chunkSize,
metadata: a.metadata
})),
"sync-uploads"
);
}
@@ -379,8 +381,13 @@ class Sync {
await this.db.monographs.init();
}
async processChunk(chunk, key, dbLastSynced, notify = false) {
const decrypted = await this.db.storage.decryptMulti(key, chunk.items);
async processChunk(
chunk: SyncTransferItem,
key: SerializedKey,
dbLastSynced: number,
notify = false
) {
const decrypted = await this.db.storage().decryptMulti(key, chunk.items);
const deserialized = await Promise.all(
decrypted.map((item, index) =>
@@ -426,20 +433,14 @@ class Sync {
}
}
/**
*
* @param {SyncTransferItem} item
* @returns {Promise<boolean>}
* @private
*/
async pushItem(item, newLastSynced) {
private async pushItem(item: SyncTransferItem, newLastSynced: number) {
await this.checkConnection();
return (
(await this.connection.invoke("PushItems", item, newLastSynced)) === 1
);
}
async checkConnection() {
private async checkConnection() {
await this.syncConnectionMutex.runExclusive(async () => {
try {
if (this.connection.state !== signalr.HubConnectionState.Connected) {
@@ -452,19 +453,22 @@ class Sync {
await promiseTimeout(30000, this.connection.start());
}
} catch (e) {
this.logger.warn(e.message);
throw new Error(
"Could not connect to the Sync server. Please try again."
);
console.error(e);
if (e instanceof Error) {
this.logger.warn(e.message);
throw new Error(
"Could not connect to the Sync server. Please try again."
);
}
}
});
}
}
function promiseTimeout(ms, promise) {
function promiseTimeout(ms: number, promise: Promise<unknown>) {
// Create a promise that rejects in <ms> milliseconds
let timeout = new Promise((resolve, reject) => {
let id = setTimeout(() => {
const timeout = new Promise((resolve, reject) => {
const id = setTimeout(() => {
clearTimeout(id);
reject(new Error("Sync timed out in " + ms + "ms."));
}, ms);

199
packages/core/src/api/sync/merger.js
View File

@@ -1,199 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import setManipulator from "../../utils/set";
import { logger } from "../../logger";
import { isHTMLEqual } from "../../utils/html-diff";
class Merger {
/**
*
* @param {import("../index").default} db
*/
constructor(db) {
this._db = db;
this.logger = logger.scope("Merger");
this.syncCollectionMap = {
shortcut: "shortcuts",
reminder: "reminders",
relation: "relations",
notebook: "notebooks"
};
}
isSyncCollection(type) {
return !!this.syncCollectionMap[type];
}
isConflicted(localItem, remoteItem, lastSynced, conflictThreshold) {
const isResolved = localItem.dateResolved === remoteItem.dateModified;
const isModified =
// the local item is modified if it was changed/modified after the last
// sync i.e. it wasn't synced yet.
// However, in case a sync is interrupted the local item's date modified
// will be ahead of last sync. In that case, we also have to check if the
// synced flag is false (it is only false if a user makes edits on the
// local device).
localItem.dateModified > lastSynced && !localItem.synced;
if (isModified && !isResolved) {
// If time difference between local item's edits & remote item's edits
// is less than threshold, we shouldn't trigger a merge conflict; instead
// we will keep the most recently changed item.
const timeDiff =
Math.max(remoteItem.dateModified, localItem.dateModified) -
Math.min(remoteItem.dateModified, localItem.dateModified);
if (timeDiff < conflictThreshold) {
if (remoteItem.dateModified > localItem.dateModified) {
return "merge";
}
return;
}
return "conflict";
} else if (!isResolved) {
return "merge";
}
}
mergeItemSync(remoteItem, type, lastSynced) {
switch (type) {
case "shortcut":
case "reminder":
case "relation": {
const localItem = this._db[
this.syncCollectionMap[type]
]._collection.getItem(remoteItem.id);
if (!localItem || remoteItem.dateModified > localItem.dateModified) {
return remoteItem;
}
break;
}
case "notebook": {
const THRESHOLD = 1000;
const localItem = this._db.notebooks._collection.getItem(remoteItem.id);
if (
!localItem ||
this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD)
) {
return this._db.notebooks.merge(localItem, remoteItem, lastSynced);
}
break;
}
}
}
async mergeContent(remoteItem, localItem, lastSynced) {
if (localItem && localItem.localOnly) return;
const THRESHOLD = process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000;
const conflicted =
localItem &&
this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD);
if (!localItem || conflicted === "merge") {
return remoteItem;
} else if (conflicted === "conflict") {
const note = this._db.notes._collection.getItem(localItem.noteId);
if (!note || note.deleted) return;
// if hashes are equal do nothing
if (
!note.locked &&
(!remoteItem ||
!remoteItem ||
!localItem.data ||
!remoteItem.data ||
isHTMLEqual(localItem.data, remoteItem.data))
)
return;
if (remoteItem.deleted || localItem.deleted || note.locked) {
// if note is locked or content is deleted we keep the most recent version.
if (remoteItem.dateModified > localItem.dateModified) return remoteItem;
} else {
// otherwise we trigger the conflicts
await this._db.notes.add({
id: localItem.noteId,
conflicted: true
});
await this._db.storage.write("hasConflicts", true);
return {
...localItem,
conflicted: remoteItem
};
}
}
}
async mergeItem(remoteItem, type, lastSynced) {
switch (type) {
case "note": {
const localItem = this._db.notes._collection.getItem(remoteItem.id);
if (!localItem || remoteItem.dateModified > localItem.dateModified) {
return await this._db.notes.merge(localItem, remoteItem);
}
break;
}
case "settings": {
const localItem = this._db.settings.raw;
if (
!localItem ||
this.isConflicted(localItem, remoteItem, lastSynced, 1000)
) {
await this._db.settings.merge(remoteItem, lastSynced);
}
break;
}
case "attachment": {
if (remoteItem.deleted)
return this._db.attachments.merge(null, remoteItem);
// rare case but if remote attachment doesn't have metadata it's probably broken so we should just skip it in this case.
if (!remoteItem.metadata) break;
const localItem = this._db.attachments.attachment(
remoteItem.metadata?.hash
);
if (
localItem &&
localItem.metadata &&
localItem.dateUploaded !== remoteItem.dateUploaded
) {
const noteIds = localItem.noteIds.slice();
const isRemoved = await this._db.attachments.remove(
localItem.metadata.hash,
true
);
if (!isRemoved)
throw new Error(
"Conflict could not be resolved in one of the attachments."
);
remoteItem.noteIds = setManipulator.union(
remoteItem.noteIds,
noteIds
);
remoteItem.remote = false;
}
return this._db.attachments.merge(localItem, remoteItem);
}
}
}
}
export default Merger;

309
packages/core/src/api/sync/merger.ts Normal file
View File

@@ -0,0 +1,309 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { migrateItem } from "../../migrations";
import { set } from "../../utils/set";
import { logger } from "../../logger";
import { isHTMLEqual } from "../../utils/html-diff";
import { EV, EVENTS } from "../../common";
import Database from "..";
import { SyncItem, SyncableItemType } from "./collector";
import { Item, ItemMap, MaybeDeletedItem, isDeleted } from "../../types";
import { SerializedKey } from "@notesnook/crypto";
import { isCipher } from "../../database/crypto";
type Conflict<P extends SyncableItemType> = (
local: MaybeDeletedItem<ItemMap[P]>,
remote: MaybeDeletedItem<ItemMap[P]>
) => Promise<void>;
type Set<P extends SyncableItemType> = (
item: MaybeDeletedItem<ItemMap[P]>
) => Promise<void>;
type Get<P extends SyncableItemType> = (
id: string
) =>
| MaybeDeletedItem<ItemMap[P]>
| undefined
| Promise<MaybeDeletedItem<ItemMap[P]> | undefined>;
type MergeDefinition = {
[P in SyncableItemType]: {
threshold?: number;
get?: Get<P>;
set: Set<P>;
conflict?: Conflict<P>;
};
};
class Merger {
private mergeDefinition: MergeDefinition;
private logger = logger.scope("Merger");
private lastSynced = 0;
private key?: SerializedKey;
constructor(private readonly db: Database) {
this.mergeDefinition = {
settings: {
threshold: 1000,
get: () => this.db.settings.raw,
set: (item) => this.db.settings.merge(item),
conflict: (_local, remote) => this.db.settings.merge(remote)
},
note: {
get: (id) => this.db.notes.note(id)?.data,
set: (item) => this.db.notes.merge(item)
},
shortcut: {
get: (id) => this.db.shortcuts.shortcut(id),
set: (item) => this.db.shortcuts.merge(item)
},
reminder: {
get: (id) => this.db.reminders.reminder(id),
set: (item) => this.db.reminders.merge(item)
},
relation: {
get: (id) => this.db.relations.relation(id),
set: (item) => this.db.relations.merge(item)
},
tag: {
get: (id) => this.db.tags.tag(id),
set: (item) => this.db.tags.merge(item)
},
color: {
get: (id) => this.db.colors.color(id),
set: (item) => this.db.colors.merge(item)
},
notebook: {
threshold: 1000,
get: (id) => this.db.notebooks.notebook(id)?.data,
set: (item) => this.db.notebooks.merge(item),
conflict: (_local, remote) => this.db.notebooks.merge(remote)
},
content: {
threshold: process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000,
get: (id) => this.db.content.raw(id),
set: async (item) => {
await this.db.content.merge(item);
},
conflict: async (local, remote) => {
if (isDeleted(local) || isDeleted(remote)) {
if (remote.dateModified > local.dateModified)
await db.content.merge(remote);
return;
}
const note = this.db.notes.note(local.noteId);
if (!note || !note.data) return;
// if hashes are equal do nothing
if (
!note.locked &&
(!remote ||
!local ||
!local.data ||
!remote.data ||
isHTMLEqual(local.data, remote.data))
)
return;
if (note.locked) {
// if note is locked or content is deleted we keep the most recent version.
if (remote.dateModified > local.dateModified)
await this.db.content.merge({ ...remote, id: local.id });
} else {
// otherwise we trigger the conflicts
await this.db.content.merge({ ...local, conflicted: remote });
await this.db.notes.add({ id: local.noteId, conflicted: true });
await this.db.storage().write("hasConflicts", true);
}
}
},
attachment: {
set: async (remoteAttachment) => {
if (isDeleted(remoteAttachment)) {
await this.db.attachments.merge(remoteAttachment);
return;
}
const localAttachment = this.db.attachments.attachment(
remoteAttachment.metadata.hash
);
if (
localAttachment &&
localAttachment.dateUploaded !== remoteAttachment.dateUploaded
) {
const noteIds = localAttachment.noteIds.slice();
const isRemoved = await this.db.attachments.remove(
localAttachment.metadata.hash,
true
);
if (!isRemoved)
throw new Error(
"Conflict could not be resolved in one of the attachments."
);
remoteAttachment.noteIds = set.union(
remoteAttachment.noteIds,
noteIds
);
}
await this.db.attachments.merge(remoteAttachment);
}
}
};
}
async _migrate(deserialized: Item, version: number) {
// it is a locked note, bail out.
if (isCipher(deserialized) && deserialized.alg && deserialized.cipher)
return deserialized;
return migrateItem(deserialized, version, deserialized.type, this.db);
}
async _deserialize(item: SyncItem, migrate = true) {
if (!this.key) throw new Error("User encryption key not found.");
const decrypted = await this.db.storage().decrypt(this.key, item);
if (!decrypted) {
throw new Error("Decrypted item cannot be undefined or empty.");
}
const deserialized = JSON.parse(decrypted);
deserialized.remote = true;
deserialized.synced = true;
if (!migrate) return deserialized;
await this._migrate(deserialized, item.v);
return deserialized;
}
async _mergeItem<TItemType extends SyncableItemType>(
syncItem: SyncItem,
get: Get<TItemType>,
add: Set<TItemType>
) {
const remoteItem = (await this._deserialize(syncItem)) as MaybeDeletedItem<
ItemMap[TItemType]
>;
const localItem = await get(remoteItem.id);
if (!localItem || remoteItem.dateModified > localItem.dateModified) {
await add(remoteItem);
return remoteItem;
}
}
async _mergeItemWithConflicts<TItemType extends SyncableItemType>(
syncItem: SyncItem,
get: Get<TItemType>,
add: Set<TItemType>,
markAsConflicted: Conflict<TItemType>,
threshold: number
) {
const remoteItem = (await this._deserialize(syncItem)) as MaybeDeletedItem<
ItemMap[TItemType]
>;
const localItem = await get(remoteItem.id);
if (!localItem || isDeleted(localItem)) {
await add(remoteItem);
return remoteItem;
} else {
const isResolved =
"dateResolved" in localItem &&
localItem.dateResolved === remoteItem.dateModified;
const isModified =
// the local item is modified if it was changed/modified after the last sync
// i.e. it wasn't synced yet.
// However, in case a sync is interrupted the local item's date modified will
// be ahead of last sync. In that case, we also have to check if the synced flag
// is false (it is only false if a user makes edits on the local device).
localItem.dateModified > this.lastSynced && !localItem.synced;
if (isModified && !isResolved) {
// If time difference between local item's edits & remote item's edits
// is less than threshold, we shouldn't trigger a merge conflict; instead
// we will keep the most recently changed item.
const timeDiff =
Math.max(remoteItem.dateModified, localItem.dateModified) -
Math.min(remoteItem.dateModified, localItem.dateModified);
if (timeDiff < threshold) {
if (remoteItem.dateModified > localItem.dateModified) {
await add(remoteItem);
return remoteItem;
}
return;
}
this.logger.info("Conflict detected", {
itemId: remoteItem.id,
isResolved,
isModified,
timeDiff,
remote: remoteItem.dateModified,
local: localItem.dateModified,
lastSynced: this.lastSynced
});
await markAsConflicted(localItem, remoteItem);
} else if (!isResolved) {
await add(remoteItem);
return remoteItem;
}
}
}
async mergeItem<TItemType extends SyncableItemType>(
type: TItemType | "vaultKey",
item: SyncItem
) {
this.lastSynced = await this.db.lastSynced();
if (!this.key) this.key = await this.db.user.getEncryptionKey();
if (!this.key || !this.key.key || !this.key.salt) {
EV.publish(EVENTS.userSessionExpired);
throw new Error("User encryption key not generated. Please relogin.");
}
if (type === "vaultKey") {
await this.db.vault.setKey(await this._deserialize(item, false));
return;
}
const definition = this.mergeDefinition[type];
if (definition.conflict && definition.get && definition.threshold) {
return await this._mergeItemWithConflicts<TItemType>(
item,
definition.get,
definition.set,
definition.conflict,
definition.threshold
);
} else if (definition.get && definition.set) {
return await this._mergeItem<TItemType>(
item,
definition.get,
definition.set
);
} else if (!definition.get && !!definition.set) {
const remote = await this._deserialize(item);
await definition.set(remote);
}
}
}
export default Merger;

95
packages/core/src/api/sync/sync-queue.js
View File

@@ -1,95 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import set from "../../utils/set";
import qclone from "qclone";
import { logger } from "../../logger";
export class SyncQueue {
/**
*
* @param {import("../../database/storage").default} storage
*/
constructor(storage) {
this.storage = storage;
this.logger = logger.scope("SyncQueue");
}
async new(data, syncedAt) {
const itemIds = mapToIds(data);
const syncData = { itemIds, syncedAt };
await this.save(syncData);
return syncData;
}
async merge(data, syncedAt) {
const syncQueue = await this.get();
if (!syncQueue.itemIds) return;
const itemIds = set.union(syncQueue.itemIds, mapToIds(data));
const syncData = { itemIds, syncedAt };
await this.save(syncData);
this.logger.info("Ids after merge", { itemIds });
return syncData;
}
async dequeue(...ids) {
const syncQueue = await this.get();
if (!syncQueue || !syncQueue.itemIds) return;
const { itemIds } = syncQueue;
for (let id of ids) {
const index = itemIds.findIndex((i) => i === id);
if (index <= -1) continue;
itemIds.splice(index, 1);
}
this.logger.info("Ids after dequeue", { itemIds });
if (itemIds.length <= 0) await this.save(null);
else await this.save(syncQueue);
}
/**
*
* @returns {Promise<{ itemIds: string[]; syncedAt: number; }>}
*/
async get() {
const syncQueue = await this.storage.read("syncQueue");
if (!syncQueue || syncQueue.itemIds.length <= 0) return {};
return qclone(syncQueue);
}
async save(syncQueue) {
await this.storage.write("syncQueue", syncQueue);
}
}
function mapToIds(data) {
const ids = [];
const keys = ["attachments", "content", "notes", "notebooks", "settings"];
for (let key of keys) {
const array = data[key];
if (!array || !Array.isArray(array)) continue;
for (let item of array) {
ids.push(`${key}:${item.id}`);
}
}
return ids;
}

6
packages/core/src/api/sync/utils.js → packages/core/src/api/sync/utils.ts
View File

@@ -17,12 +17,10 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
function areAllEmpty(obj) {
for (let key in obj) {
export function areAllEmpty(obj: Record<string, unknown>) {
for (const key in obj) {
const value = obj[key];
if (Array.isArray(value) && value.length > 0) return false;
}
return true;
}
export { areAllEmpty };

48
packages/core/src/api/token-manager.js → packages/core/src/api/token-manager.ts
View File

@@ -22,6 +22,15 @@ import constants from "../utils/constants";
import { EV, EVENTS } from "../common";
import { withTimeout, Mutex } from "async-mutex";
import { logger } from "../logger";
import { StorageAccessor } from "../interfaces";
type Token = {
access_token: string;
t: number;
expires_in: number;
scope: string;
refresh_token: string;
};
const ENDPOINTS = {
token: "/connect/token",
@@ -31,18 +40,12 @@ const ENDPOINTS = {
};
class TokenManager {
/**
*
* @param {import("../database/storage").default} storage
*/
constructor(storage) {
this._storage = storage;
this._refreshTokenMutex = withTimeout(new Mutex(), 10 * 1000);
this.logger = logger.scope("TokenManager");
}
mutex = withTimeout(new Mutex(), 10 * 1000);
logger = logger.scope("TokenManager");
constructor(private readonly storage: StorageAccessor) {}
async getToken(renew = true, forceRenew = false) {
let token = await this._storage.read("token");
async getToken(renew = true, forceRenew = false): Promise<Token | undefined> {
const token = await this.storage().read<Token>("token");
if (!token || !token.access_token) return;
this.logger.info("Access token requested", {
@@ -58,13 +61,13 @@ class TokenManager {
return token;
}
_isTokenExpired(token) {
_isTokenExpired(token: Token) {
const { t, expires_in } = token;
const expiryMs = t + expires_in * 1000;
return Date.now() >= expiryMs;
}
_isTokenRefreshable(token) {
_isTokenRefreshable(token: Token) {
const { scope, refresh_token } = token;
if (!refresh_token || !scope) return false;
@@ -81,7 +84,7 @@ class TokenManager {
}
async _refreshToken(forceRenew = false) {
await this._refreshTokenMutex.runExclusive(async () => {
await this.mutex.runExclusive(async () => {
this.logger.info("Refreshing access token");
const token = await this.getToken(false, false);
@@ -116,7 +119,7 @@ class TokenManager {
if (!token) return;
const { access_token } = token;
await this._storage.remove("token");
await this.storage().remove("token");
await http.post(
`${constants.AUTH_HOST}${ENDPOINTS.logout}`,
null,
@@ -124,14 +127,14 @@ class TokenManager {
);
}
saveToken(tokenResponse) {
saveToken(tokenResponse: Omit<Token, "t">) {
this.logger.info("Saving new token", tokenResponse);
if (!tokenResponse || !tokenResponse.access_token) return;
let token = { ...tokenResponse, t: Date.now() };
return this._storage.write("token", token);
const token: Token = { ...tokenResponse, t: Date.now() };
return this.storage().write("token", token);
}
async getAccessTokenFromAuthorizationCode(userId, authCode) {
async getAccessTokenFromAuthorizationCode(userId: string, authCode: string) {
return await this.saveToken(
await http.post(`${constants.AUTH_HOST}${ENDPOINTS.temporaryToken}`, {
authorization_code: authCode,
@@ -143,12 +146,15 @@ class TokenManager {
}
export default TokenManager;
async function getSafeToken(action, errorMessage) {
async function getSafeToken<T>(action: () => Promise<T>, errorMessage: string) {
try {
return await action();
} catch (e) {
console.error(errorMessage, e);
if (e.message === "invalid_grant" || e.message === "invalid_client") {
if (
e instanceof Error &&
(e.message === "invalid_grant" || e.message === "invalid_client")
) {
EV.publish(EVENTS.userSessionExpired);
}
throw e;

224
packages/core/src/api/user-manager.js → packages/core/src/api/user-manager.ts
View File

@@ -23,6 +23,32 @@ import constants from "../utils/constants";
import TokenManager from "./token-manager";
import { EV, EVENTS } from "../common";
import { HealthCheck } from "./healthcheck";
import Database from ".";
import { Cipher, SerializedKey } from "@notesnook/crypto";
export type User = {
id: string;
email: string;
isEmailConfirmed: boolean;
salt: string;
attachmentsKey?: Cipher<"base64">;
mfa: {
isEnabled: boolean;
primaryMethod: string;
secondaryMethod: string;
remainingValidCodes: number;
};
subscription: {
appId: 0;
cancelURL: string | null;
expiry: number;
productId: string;
provider: 0 | 1 | 2 | 3;
start: number;
type: 0 | 1 | 2 | 5 | 6 | 7;
updateURL: string | null;
};
};
const ENDPOINTS = {
signup: "/users",
@@ -38,26 +64,19 @@ const ENDPOINTS = {
};
class UserManager {
/**
*
* @param {import("../database/storage").default} storage
* @param {import("../api/index").default} db
*/
constructor(storage, db) {
this._storage = storage;
this._db = db;
this.tokenManager = new TokenManager(storage);
private tokenManager: TokenManager;
constructor(private readonly db: Database) {
this.tokenManager = new TokenManager(this.db.storage);
EV.subscribe(EVENTS.userUnauthorized, async (url) => {
if (
url.includes("/connect/token") ||
!(await HealthCheck.isAuthServerHealthy())
)
return;
EV.subscribe(EVENTS.userUnauthorized, async (url: string) => {
if (url.includes("/connect/token") || !(await HealthCheck.auth())) return;
try {
await this.tokenManager._refreshToken(true);
} catch (e) {
if (e.message === "invalid_grant" || e.message === "invalid_client") {
if (
e instanceof Error &&
(e.message === "invalid_grant" || e.message === "invalid_client")
) {
await this.logout(
false,
`Your token has been revoked. Error: ${e.message}.`
@@ -72,10 +91,10 @@ class UserManager {
if (!user) return;
}
async signup(email, password) {
async signup(email: string, password: string) {
email = email.toLowerCase();
const hashedPassword = await this._storage.hash(password, email);
const hashedPassword = await this.db.storage().hash(password, email);
await http.post(`${constants.API_HOST}${ENDPOINTS.signup}`, {
email,
password: hashedPassword,
@@ -85,7 +104,7 @@ class UserManager {
return await this._login({ email, password, hashedPassword });
}
async authenticateEmail(email) {
async authenticateEmail(email: string) {
if (!email) throw new Error("Email is required.");
email = email.toLowerCase();
@@ -100,7 +119,7 @@ class UserManager {
return result.additional_data;
}
async authenticateMultiFactorCode(code, method) {
async authenticateMultiFactorCode(code: string, method: string) {
if (!code || !method) throw new Error("code & method are required.");
const token = await this.tokenManager.getAccessToken();
@@ -122,10 +141,10 @@ class UserManager {
}
async authenticatePassword(
email,
password,
hashedPassword = null,
sessionExpired = false
email: string,
password: string,
hashedPassword?: string,
sessionExpired?: boolean
) {
if (!email || !password) throw new Error("email & password are required.");
@@ -134,7 +153,7 @@ class UserManager {
email = email.toLowerCase();
if (!hashedPassword) {
hashedPassword = await this._storage.hash(password, email);
hashedPassword = await this.db.storage().hash(password, email);
}
try {
await this.tokenManager.saveToken(
@@ -154,10 +173,10 @@ class UserManager {
if (!user) throw new Error("Unauthorized.");
if (!sessionExpired) {
await this._storage.write("lastSynced", 0);
await this.db.storage().write("lastSynced", 0);
}
await this._storage.deriveCryptoKey(`_uk_@${user.email}`, {
await this.db.storage().deriveCryptoKey(`_uk_@${user.email}`, {
password,
salt: user.salt
});
@@ -168,14 +187,23 @@ class UserManager {
}
}
/**
* @private
*/
async _login({ email, password, hashedPassword, code, method }) {
async _login({
email,
password,
hashedPassword,
code,
method
}: {
email: string;
password: string;
hashedPassword?: string;
code?: string;
method?: string;
}) {
email = email && email.toLowerCase();
if (!hashedPassword && password) {
hashedPassword = await this._storage.hash(password, email);
hashedPassword = await this.db.storage().hash(password, email);
}
await this.tokenManager.saveToken(
@@ -191,11 +219,13 @@ class UserManager {
);
const user = await this.fetchUser();
await this._storage.deriveCryptoKey(`_uk_@${user.email}`, {
if (!user) return;
await this.db.storage().deriveCryptoKey(`_uk_@${user.email}`, {
password,
salt: user.salt
});
await this._storage.write("lastSynced", 0);
await this.db.storage().write("lastSynced", 0);
EV.publish(EVENTS.userLoggedIn, user);
}
@@ -228,33 +258,28 @@ class UserManager {
return true;
}
async logout(revoke = true, reason) {
async logout(revoke = true, reason?: string) {
try {
if (revoke) await this.tokenManager.revokeToken();
} catch (e) {
console.error(e);
} finally {
await this._storage.clear();
await this.db.storage().clear();
EV.publish(EVENTS.userLoggedOut, reason);
EV.publish(EVENTS.appRefreshRequested);
}
}
setUser(user) {
if (!user) return;
return this._storage.write("user", user);
setUser(user: User) {
return this.db.storage().write("user", user);
}
/**
*
* @returns {Promise<User>}
*/
getUser() {
return this._storage.read("user");
return this.db.storage().read<User>("user");
}
async resetUser(removeAttachments = true) {
let token = await this.tokenManager.getAccessToken();
const token = await this.tokenManager.getAccessToken();
if (!token) return;
await http.post(
`${constants.API_HOST}${ENDPOINTS.resetUser}`,
@@ -264,10 +289,8 @@ class UserManager {
return true;
}
async updateUser(user) {
if (!user) return;
let token = await this.tokenManager.getAccessToken();
async updateUser(user: User) {
const token = await this.tokenManager.getAccessToken();
await http.patch.json(
`${constants.API_HOST}${ENDPOINTS.user}`,
user,
@@ -277,26 +300,23 @@ class UserManager {
await this.setUser(user);
}
async deleteUser(password) {
let token = await this.tokenManager.getAccessToken();
if (!token) return;
async deleteUser(password: string) {
const token = await this.tokenManager.getAccessToken();
const user = await this.getUser();
if (!token || !user) return;
await http.post(
`${constants.API_HOST}${ENDPOINTS.deleteUser}`,
{ password: await this._storage.hash(password, user.email) },
{ password: await this.db.storage().hash(password, user.email) },
token
);
await this.logout(false, "Account deleted.");
return true;
}
/**
*
* @returns {Promise<User | undefined>}
*/
async fetchUser() {
async fetchUser(): Promise<User | undefined> {
try {
let token = await this.tokenManager.getAccessToken();
const token = await this.tokenManager.getAccessToken();
if (!token) return;
const user = await http.get(
`${constants.API_HOST}${ENDPOINTS.user}`,
@@ -315,15 +335,15 @@ class UserManager {
}
}
changePassword(oldPassword, newPassword) {
changePassword(oldPassword: string, newPassword: string) {
return this._updatePassword("change_password", {
old_password: oldPassword,
new_password: newPassword
});
}
async changeMarketingConsent(enabled) {
let token = await this.tokenManager.getAccessToken();
async changeMarketingConsent(enabled: boolean) {
const token = await this.tokenManager.getAccessToken();
if (!token) return;
await http.patch(
@@ -336,16 +356,16 @@ class UserManager {
);
}
resetPassword(newPassword) {
resetPassword(newPassword: string) {
return this._updatePassword("reset_password", {
new_password: newPassword
});
}
async getEncryptionKey() {
async getEncryptionKey(): Promise<SerializedKey | undefined> {
const user = await this.getUser();
if (!user) return;
const key = await this._storage.getCryptoKey(`_uk_@${user.email}`);
const key = await this.db.storage().getCryptoKey(`_uk_@${user.email}`);
if (!key) return;
return { key, salt: user.salt };
}
@@ -356,39 +376,40 @@ class UserManager {
if (!user) return;
if (!user.attachmentsKey) {
let token = await this.tokenManager.getAccessToken();
const token = await this.tokenManager.getAccessToken();
user = await http.get(`${constants.API_HOST}${ENDPOINTS.user}`, token);
}
if (!user) return;
const userEncryptionKey = await this.getEncryptionKey();
if (!userEncryptionKey) return;
if (!user.attachmentsKey) {
const key = await this._storage.generateRandomKey();
user.attachmentsKey = await this._storage.encrypt(
userEncryptionKey,
JSON.stringify(key)
);
const key = await this.db.crypto().generateRandomKey();
user.attachmentsKey = await this.db
.storage()
.encrypt(userEncryptionKey, JSON.stringify(key));
await this.updateUser(user);
return key;
}
const plainData = await this._storage.decrypt(
userEncryptionKey,
user.attachmentsKey
);
const plainData = await this.db
.storage()
.decrypt(userEncryptionKey, user.attachmentsKey);
if (!plainData) return;
return JSON.parse(plainData);
} catch (e) {
console.error(e);
throw new Error(
`Could not get attachments encryption key. Please make sure you have Internet access. Error: ${e.message}`
);
if (e instanceof Error)
throw new Error(
`Could not get attachments encryption key. Please make sure you have Internet access. Error: ${e.message}`
);
}
}
async sendVerificationEmail(newEmail) {
let token = await this.tokenManager.getAccessToken();
async sendVerificationEmail(newEmail: string) {
const token = await this.tokenManager.getAccessToken();
if (!token) return;
await http.post(
`${constants.AUTH_HOST}${ENDPOINTS.verifyUser}`,
@@ -397,7 +418,7 @@ class UserManager {
);
}
async changeEmail(newEmail, password, code) {
async changeEmail(newEmail: string, password: string, code: string) {
const token = await this.tokenManager.getAccessToken();
if (!token) return;
@@ -411,39 +432,47 @@ class UserManager {
{
type: "change_email",
new_email: newEmail,
password: await this._storage.hash(password, email),
password: await this.db.storage().hash(password, email),
verification_code: code
},
token
);
await this._storage.deriveCryptoKey(`_uk_@${newEmail}`, {
await this.db.storage().deriveCryptoKey(`_uk_@${newEmail}`, {
password,
salt: user.salt
});
}
recoverAccount(email) {
recoverAccount(email: string) {
return http.post(`${constants.AUTH_HOST}${ENDPOINTS.recoverAccount}`, {
email,
client_id: "notesnook"
});
}
async verifyPassword(password) {
async verifyPassword(password: string) {
try {
const user = await this.getUser();
if (!user) return false;
const key = await this.getEncryptionKey();
const cipher = await this._storage.encrypt(key, "notesnook");
const plainText = await this._storage.decrypt({ password }, cipher);
if (!user || !key) return false;
const cipher = await this.db.storage().encrypt(key, "notesnook");
const plainText = await this.db.storage().decrypt({ password }, cipher);
return plainText === "notesnook";
} catch (e) {
return false;
}
}
async _updatePassword(type, data) {
async _updatePassword(
type: "change_password" | "reset_password",
data: {
new_password: string;
old_password?: string;
encryptionKey?: SerializedKey;
}
) {
const token = await this.tokenManager.getAccessToken();
const user = await this.getUser();
if (!token || !user) throw new Error("You are not logged in.");
@@ -461,31 +490,30 @@ class UserManager {
await this.clearSessions();
if (data.encryptionKey) await this._db.sync({ type: "fetch", force: true });
if (data.encryptionKey) await this.db.sync({ type: "fetch", force: true });
await this._storage.deriveCryptoKey(`_uk_@${email}`, {
await this.db.storage().deriveCryptoKey(`_uk_@${email}`, {
password: new_password,
salt
});
if (!(await this.resetUser(false))) return;
await this._db.sync({ type: "send", force: true });
await this.db.sync({ type: "send", force: true });
if (attachmentsKey) {
const userEncryptionKey = await this.getEncryptionKey();
if (!userEncryptionKey) return;
user.attachmentsKey = await this._storage.encrypt(
userEncryptionKey,
JSON.stringify(attachmentsKey)
);
user.attachmentsKey = await this.db
.storage()
.encrypt(userEncryptionKey, JSON.stringify(attachmentsKey));
await this.updateUser(user);
}
if (old_password)
old_password = await this._storage.hash(old_password, email);
old_password = await this.db.storage().hash(old_password, email);
if (new_password)
new_password = await this._storage.hash(new_password, email);
new_password = await this.db.storage().hash(new_password, email);
await http.patch(
`${constants.AUTH_HOST}${ENDPOINTS.patchUser}`,

365
packages/core/src/api/vault.js
View File

@@ -1,365 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { CHECK_IDS, EV, EVENTS, checkIsUserPremium } from "../common";
import { tinyToTiptap } from "../migrations";
const ERASE_TIME = 1000 * 60 * 30;
var ERASER_TIMEOUT = null;
export default class Vault {
get _password() {
return this._vaultPassword;
}
set _password(value) {
this._vaultPassword = value;
if (value) {
this._startEraser();
}
}
_startEraser() {
clearTimeout(ERASER_TIMEOUT);
ERASER_TIMEOUT = setTimeout(() => {
this._password = null;
EV.publish(EVENTS.vaultLocked);
}, ERASE_TIME);
}
/**
*
* @param {import('./index').default} db
*/
constructor(db) {
this._db = db;
this._storage = db.storage;
this._key = "svvaads1212#2123";
this._vaultPassword = null;
this.ERRORS = {
noVault: "ERR_NO_VAULT",
vaultLocked: "ERR_VAULT_LOCKED",
wrongPassword: "ERR_WRONG_PASSWORD"
};
EV.subscribe(EVENTS.userLoggedOut, () => {
this._password = null;
});
}
get unlocked() {
return !!this._vaultPassword;
}
/**
* Creates a new vault
* @param {string} password The password
* @returns {Promise<Boolean>}
*/
async create(password) {
if (!(await checkIsUserPremium(CHECK_IDS.vaultAdd))) return;
const vaultKey = await this._getKey();
if (!vaultKey || !vaultKey.cipher || !vaultKey.iv) {
const encryptedData = await this._storage.encrypt(
{ password },
this._key
);
await this._setKey(encryptedData);
this._password = password;
}
return true;
}
/**
* Unlocks the vault with the given password
* @param {string} password The password
* @throws ERR_NO_VAULT | ERR_WRONG_PASSWORD
* @returns {Promise<Boolean>}
*/
async unlock(password) {
const vaultKey = await this._getKey();
if (!(await this.exists(vaultKey))) throw new Error(this.ERRORS.noVault);
try {
await this._storage.decrypt({ password }, vaultKey);
} catch (e) {
throw new Error(this.ERRORS.wrongPassword);
}
this._password = password;
return true;
}
async changePassword(oldPassword, newPassword) {
if (await this.unlock(oldPassword)) {
await this._db.notes.init();
const contentItems = [];
for (const note of this._db.notes.locked) {
try {
let encryptedContent = await this._db.content.raw(note.contentId);
let content = await this.decryptContent(
encryptedContent,
oldPassword
);
contentItems.push({
...content,
id: note.contentId,
noteId: note.id
});
} catch (e) {
throw new Error(
`Could not decrypt content of note ${note.id}. Error: ${e.message}`
);
}
}
for (const content of contentItems) {
await this._encryptContent(
content.id,
null,
content.data,
content.type,
newPassword
);
}
await this._storage.remove("vaultKey");
await this.create(newPassword);
}
}
async clear(password) {
if (await this.unlock(password)) {
await this._db.notes.init();
for (var note of this._db.notes.locked) {
await this._unlockNote(note, password, true);
}
}
}
async delete(deleteAllLockedNotes = false) {
if (deleteAllLockedNotes) {
await this._db.notes.init();
await this._db.notes.remove(
...this._db.notes.locked.map((note) => note.id)
);
}
await this._storage.remove("vaultKey");
this._password = null;
}
/**
* Locks (add to vault) a note
* @param {string} noteId The id of the note to lock
*/
async add(noteId) {
if (!(await checkIsUserPremium(CHECK_IDS.vaultAdd))) return;
await this._check();
await this._lockNote({ id: noteId }, this._password);
await this._db.noteHistory.clearSessions(noteId);
}
/**
* Permanently unlocks (remove from vault) a note
* @param {string} noteId The note id
* @param {string} password The password to unlock note with
*/
async remove(noteId, password) {
const note = this._db.notes.note(noteId);
if (!note) return;
await this._unlockNote(note.data, password, true);
if (!(await this.exists())) await this.create(this.password);
}
/**
* Temporarily unlock (open) a note
* @param {string} noteId The note id
* @param {string} password The password to open note with
*/
async open(noteId, password) {
const note = this._db.notes.note(noteId);
if (!note) return;
const unlockedNote = await this._unlockNote(note.data, password, false);
this._password = password;
if (!(await this.exists())) await this.create(password);
return unlockedNote;
}
/**
* Saves a note in the vault
* @param {{Object}} note The note to save into the vault
*/
async save(note) {
if (!note) return;
await this._check();
// roll over erase timer
this._startEraser();
return await this._lockNote(note, this._password);
}
async exists(vaultKey = undefined) {
if (!vaultKey) vaultKey = await this._getKey();
return vaultKey && vaultKey.cipher && vaultKey.iv;
}
// Private & internal methods
/** @private */
_locked() {
return !this._password || !this._password.length;
}
/** @private */
async _check() {
if (!(await this.exists())) {
throw new Error(this.ERRORS.noVault);
}
if (this._locked()) {
throw new Error(this.ERRORS.vaultLocked);
}
}
/** @private */
async _encryptContent(contentId, sessionId, content, type, password) {
let encryptedContent = await this._storage.encrypt(
{ password },
JSON.stringify(content)
);
await this._db.content.add({
id: contentId,
sessionId,
data: encryptedContent,
type
});
}
async decryptContent(encryptedContent, password = null) {
if (!password) {
await this._check();
password = this._password;
}
if (encryptedContent.noteId && typeof encryptedContent.data !== "object") {
await this._db.notes.add({
id: encryptedContent.noteId,
locked: false
});
return encryptedContent;
}
let decryptedContent = await this._storage.decrypt(
{ password },
encryptedContent.data
);
const content = {
type: encryptedContent.type,
data: JSON.parse(decryptedContent)
};
// #MIGRATION: convert tiny to tiptap
if (content.type === "tiny") {
content.type = "tiptap";
content.data = tinyToTiptap(content.data);
}
return content;
}
/** @private */
async _lockNote(note, password) {
let { id, content: { type, data } = {}, sessionId, title } = note;
note = this._db.notes.note(id);
if (!note) return;
note = note.data;
const contentId = note.contentId;
if (!contentId) throw new Error("Cannot lock note because it is empty.");
// Case: when note is being newly locked
if (!note.locked && (!data || !type)) {
let content = await this._db.content.raw(contentId, false);
// NOTE:
// At this point, the note already has all the attachments extracted
// so we should just encrypt it as normal.
data = content.data;
type = content.type;
} else if (data && type) {
const content = await this._db.content.extractAttachments({
data,
type,
noteId: id
});
data = content.data;
type = content.type;
}
if (data && type)
await this._encryptContent(contentId, sessionId, data, type, password);
return await this._db.notes.add({
id,
locked: true,
headline: "",
title: title || note.title,
favorite: note.favorite,
localOnly: note.localOnly,
readonly: note.readonly,
dateEdited: Date.now()
});
}
/** @private */
async _unlockNote(note, password, perm = false) {
let encryptedContent = await this._db.content.raw(note.contentId);
let content = await this.decryptContent(encryptedContent, password);
if (perm) {
await this._db.notes.add({
id: note.id,
locked: false,
headline: note.headline,
contentId: note.contentId,
content
});
// await this._db.content.add({ id: note.contentId, data: content });
return;
}
return {
...note,
content
};
}
/** @inner */
async _getKey() {
return await this._storage.read("vaultKey");
}
/** @inner */
async _setKey(vaultKey) {
if (!vaultKey) return;
await this._storage.write("vaultKey", vaultKey);
}
}

378
packages/core/src/api/vault.ts Normal file
View File

@@ -0,0 +1,378 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Cipher } from "@notesnook/crypto";
import Database from ".";
import { CHECK_IDS, EV, EVENTS, checkIsUserPremium } from "../common";
import { tinyToTiptap } from "../migrations";
import { isCipher } from "../database/crypto";
import { EncryptedContentItem, Note, isDeleted } from "../types";
import {
EMPTY_CONTENT,
isEncryptedContent,
isUnencryptedContent
} from "../collections/content";
import { NoteContent } from "../collections/session-content";
const ERRORS = {
noVault: "ERR_NO_VAULT",
vaultLocked: "ERR_VAULT_LOCKED",
wrongPassword: "ERR_WRONG_PASSWORD"
};
const ERASE_TIME = 1000 * 60 * 30;
export default class Vault {
vaultPassword?: string;
erasureTimeout = 0;
key = "svvaads1212#2123";
private get password() {
return this.vaultPassword;
}
private set password(value) {
this.vaultPassword = value;
if (value) {
this.startEraser();
}
}
private startEraser() {
clearTimeout(this.erasureTimeout);
this.erasureTimeout = setTimeout(() => {
this.password = undefined;
EV.publish(EVENTS.vaultLocked);
}, ERASE_TIME) as unknown as number;
}
constructor(private readonly db: Database) {
this.password = undefined;
EV.subscribe(EVENTS.userLoggedOut, () => {
this.password = undefined;
});
}
get unlocked() {
return !!this.vaultPassword;
}
async create(password: string) {
if (!(await checkIsUserPremium(CHECK_IDS.vaultAdd))) return;
const vaultKey = await this.getKey();
if (!vaultKey || !isCipher(vaultKey)) {
const encryptedData = await this.db
.storage()
.encrypt({ password }, this.key);
await this.setKey(encryptedData);
this.password = password;
}
return true;
}
async unlock(password: string) {
const vaultKey = await this.getKey();
if (!vaultKey || !(await this.exists(vaultKey)))
throw new Error(ERRORS.noVault);
try {
await this.db.storage().decrypt({ password }, vaultKey);
} catch (e) {
throw new Error(ERRORS.wrongPassword);
}
this.password = password;
return true;
}
async changePassword(oldPassword: string, newPassword: string) {
if (await this.unlock(oldPassword)) {
const contentItems = [];
for (const note of this.db.notes.locked) {
if (!note.contentId) continue;
const encryptedContent = await this.db.content.raw(note.contentId);
if (
!encryptedContent ||
isDeleted(encryptedContent) ||
!isEncryptedContent(encryptedContent)
)
continue;
try {
const content = await this.decryptContent(
encryptedContent,
oldPassword
);
contentItems.push({
...content,
id: note.contentId,
noteId: note.id
});
} catch (e) {
console.error(e);
throw new Error(
`Could not decrypt content of note ${note.id}. Error: ${
(e as Error).message
}`
);
}
}
for (const content of contentItems) {
await this.encryptContent(content, newPassword, content.id);
}
await this.db.storage().remove("vaultKey");
await this.create(newPassword);
}
}
async clear(password: string) {
if (await this.unlock(password)) {
await this.db.notes.init();
for (const note of this.db.notes.locked) {
await this.unlockNote(note, password, true);
}
}
}
async delete(deleteAllLockedNotes = false) {
if (deleteAllLockedNotes) {
await this.db.notes.init();
await this.db.notes.remove(
...this.db.notes.locked.map((note) => note.id)
);
}
await this.db.storage().remove("vaultKey");
this.password = undefined;
}
/**
* Locks (add to vault) a note
*/
async add(noteId: string) {
if (!(await checkIsUserPremium(CHECK_IDS.vaultAdd))) return;
await this.lockNote({ id: noteId }, await this.getVaultPassword());
await this.db.noteHistory.clearSessions(noteId);
}
/**
* Permanently unlocks (remove from vault) a note
*/
async remove(noteId: string, password: string) {
const note = this.db.notes.note(noteId);
if (!note) return;
await this.unlockNote(note.data, password, true);
if (!(await this.exists())) await this.create(password);
}
/**
* Temporarily unlock (open) a note
*/
async open(noteId: string, password: string) {
const note = this.db.notes.note(noteId);
if (!note) return;
const unlockedNote = await this.unlockNote(note.data, password, false);
this.password = password;
if (!(await this.exists())) await this.create(password);
return unlockedNote;
}
/**
* Saves a note in the vault
*/
async save(
note: Partial<Note & { content: NoteContent<false>; sessionId: string }> & {
id: string;
}
) {
if (!note) return;
// roll over erase timer
this.startEraser();
return await this.lockNote(note, await this.getVaultPassword());
}
async exists(vaultKey?: Cipher) {
if (!vaultKey) vaultKey = await this.getKey();
return vaultKey && isCipher(vaultKey);
}
// Private & internal methods
private async getVaultPassword() {
if (!(await this.exists())) {
throw new Error(ERRORS.noVault);
}
if (!this.password || !this.password.length) {
throw new Error(ERRORS.vaultLocked);
}
return this.password;
}
private async encryptContent(
content: NoteContent<false>,
password: string,
contentId?: string,
sessionId?: string
) {
const encryptedContent = await this.db
.storage()
.encrypt({ password }, JSON.stringify(content.data));
await this.db.content.add({
id: contentId,
sessionId,
data: encryptedContent,
type: content.type
});
}
async decryptContent(
encryptedContent: EncryptedContentItem,
password?: string
) {
if (!password) password = await this.getVaultPassword();
if (
encryptedContent.noteId &&
typeof encryptedContent.data !== "object" &&
!isCipher(encryptedContent.data)
) {
await this.db.notes.add({
id: encryptedContent.noteId,
locked: false
});
return { data: encryptedContent.data, type: encryptedContent.type };
}
const decryptedContent = await this.db
.storage()
.decrypt({ password }, encryptedContent.data);
const content: NoteContent<false> = {
type: encryptedContent.type,
data: JSON.parse(decryptedContent)
};
// #MIGRATION: convert tiny to tiptap
if (content.type === "tiny") {
content.type = "tiptap";
content.data = tinyToTiptap(content.data);
}
return content;
}
private async lockNote(
item: Partial<Note & { content: NoteContent<false>; sessionId: string }> & {
id: string;
},
password: string
) {
const { id, content, sessionId, title } = item;
let { type, data } = content || {};
const note = this.db.notes.note(id);
if (!note) return;
const contentId = note.contentId;
// if (!contentId) throw new Error("Cannot lock note because it is empty.");
// Case: when note is being newly locked
if (!note.locked && (!data || !type) && !!contentId) {
const rawContent = await this.db.content.raw(contentId);
if (
!rawContent ||
isDeleted(rawContent) ||
!isUnencryptedContent(rawContent)
)
return await this.db.notes.add({
id,
locked: true
});
// NOTE:
// At this point, the note already has all the attachments extracted
// so we should just encrypt it as normal.
data = rawContent.data;
type = rawContent.type;
} else if (data && type) {
const content = await this.db.content.extractAttachments({
...EMPTY_CONTENT(id),
data,
type
});
data = content.data;
type = content.type;
}
if (data && type)
await this.encryptContent({ data, type }, password, contentId, sessionId);
return await this.db.notes.add({
id,
locked: true,
headline: "",
title: title || note.title,
favorite: note.data.favorite,
localOnly: note.data.localOnly,
readonly: note.data.readonly,
dateEdited: Date.now()
});
}
private async unlockNote(note: Note, password: string, perm = false) {
if (!note.contentId) return;
const encryptedContent = await this.db.content.raw(note.contentId);
if (
!encryptedContent ||
isDeleted(encryptedContent) ||
!isEncryptedContent(encryptedContent)
)
return;
const content = await this.decryptContent(encryptedContent, password);
if (perm) {
await this.db.notes.add({
id: note.id,
locked: false,
headline: note.headline,
contentId: note.contentId,
content
});
// await this.db.content.add({ id: note.contentId, data: content });
return;
}
return {
...note,
content
};
}
async getKey() {
return await this.db.storage().read<Cipher>("vaultKey");
}
async setKey(vaultKey: Cipher) {
await this.db.storage().write("vaultKey", vaultKey);
}
}

343
packages/core/src/collections/attachments.js → packages/core/src/collections/attachments.ts
View File

@@ -17,27 +17,54 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Collection from "./collection";
import { ICollection } from "./collection";
import { getId } from "../utils/id";
import { deleteItem, hasItem } from "../utils/array";
import { EV, EVENTS } from "../common";
import dataurl from "../utils/dataurl";
import dayjs from "dayjs";
import setManipulator from "../utils/set";
import { set } from "../utils/set";
import {
getFileNameWithExtension,
isImage,
isWebClip
} from "../utils/filename";
import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";
import { CachedCollection } from "../database/cached-collection";
import { Output } from "../interfaces";
import {
Attachment,
AttachmentMetadata,
MaybeDeletedItem,
isDeleted
} from "../types";
import Database from "../api";
export default class Attachments extends Collection {
constructor(db, name, cached) {
super(db, name, cached);
export class Attachments implements ICollection {
name = "attachments";
key: Cipher<"base64"> | null = null;
private readonly collection: CachedCollection<"attachments", Attachment>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
"attachments",
db.eventManager
);
this.key = null;
EV.subscribe(
EVENTS.fileDownloaded,
async ({ success, filename, groupId, eventData }) => {
async ({
success,
filename,
groupId,
eventData
}: {
success: boolean;
filename: string;
groupId: string;
eventData: Record<string, unknown>;
}) => {
if (!success || !eventData || !eventData.readOnDownload) return;
const attachment = this.attachment(filename);
if (!attachment || !attachment.metadata) return;
@@ -54,29 +81,45 @@ export default class Attachments extends Collection {
}
);
EV.subscribe(EVENTS.fileUploaded, async ({ success, error, filename }) => {
const attachment = this.attachment(filename);
if (!attachment) return;
if (success) await this.markAsUploaded(attachment.id);
else
await this.markAsFailed(
attachment.id,
error || "Failed to upload attachment."
);
});
EV.subscribe(
EVENTS.fileUploaded,
async ({
success,
error,
filename
}: {
success: boolean;
filename: string;
error: string;
}) => {
const attachment = this.attachment(filename);
if (!attachment) return;
if (success) await this.markAsUploaded(attachment.id);
else
await this.markAsFailed(
attachment.id,
error || "Failed to upload attachment."
);
}
);
}
merge(localAttachment, remoteAttachment) {
if (remoteAttachment.deleted) return remoteAttachment;
async init() {
await this.collection.init();
}
merge(
localAttachment: MaybeDeletedItem<Attachment>,
remoteAttachment: MaybeDeletedItem<Attachment>
) {
if (isDeleted(remoteAttachment)) return remoteAttachment;
if (
!localAttachment ||
remoteAttachment.dateModified > localAttachment.dateModified
)
return remoteAttachment;
if (localAttachment && localAttachment.noteIds) {
remoteAttachment.noteIds = setManipulator.union(
localAttachment &&
!isDeleted(localAttachment) &&
localAttachment.noteIds
) {
remoteAttachment.noteIds = set.union(
remoteAttachment.noteIds,
localAttachment.noteIds
);
@@ -86,40 +129,33 @@ export default class Attachments extends Collection {
return remoteAttachment;
}
/**
*
* @param {{
* iv: string,
* length: number,
* alg: string,
* hash: string,
* hashType: string,
* filename: string,
* type: string,
* salt: string,
* chunkSize: number,
* key: {}
* }} attachmentArg
* @param {string} noteId Optional as attachments will be parsed at extraction time
* @returns
*/
async add(attachmentArg, noteId = undefined) {
if (!attachmentArg) return console.error("attachment cannot be undefined");
if (!attachmentArg.hash) throw new Error("Please provide attachment hash.");
async add(
item: Partial<
Omit<Attachment, "key" | "metadata"> & {
key: SerializedKey;
}
> & {
metadata: Partial<AttachmentMetadata> & { hash: string };
}
) {
if (!item) return console.error("attachment cannot be undefined");
if (!item.metadata.hash) throw new Error("Please provide attachment hash.");
const oldAttachment =
this.all.find((a) => a.metadata?.hash === attachmentArg.hash) || {};
let id = oldAttachment.id || getId();
const noteIds = oldAttachment.noteIds || [];
if (noteId && !noteIds.includes(noteId)) noteIds.push(noteId);
const oldAttachment = this.all.find(
(a) => a.metadata.hash === item.metadata?.hash
);
const id = oldAttachment?.id || getId();
const noteIds = set.union(oldAttachment?.noteIds || [], item.noteIds || []);
const encryptedKey = item.key
? await this.encryptKey(item.key)
: oldAttachment?.key;
const attachment = {
...oldAttachment,
...oldAttachment.metadata,
...attachmentArg,
...oldAttachment?.metadata,
...item,
noteIds,
key: attachmentArg.key || oldAttachment.key
key: encryptedKey
};
const {
@@ -154,10 +190,7 @@ export default class Attachments extends Collection {
return;
}
const encryptedKey = attachmentArg.key
? await this._encryptKey(attachmentArg.key)
: key;
const attachmentItem = {
return this.collection.add({
type: "attachment",
id,
noteIds,
@@ -165,7 +198,7 @@ export default class Attachments extends Collection {
salt,
length,
alg,
key: encryptedKey,
key,
chunkSize,
metadata: {
hash,
@@ -174,36 +207,36 @@ export default class Attachments extends Collection {
type: type || "application/octet-stream"
},
dateCreated: attachment.dateCreated || Date.now(),
dateModified: attachment.dateModified,
dateModified: attachment.dateModified || Date.now(),
dateUploaded: attachment.dateUploaded,
dateDeleted: undefined,
failed: attachment.failed
};
return this._collection.addItem(attachmentItem);
});
}
async generateKey() {
await this._getEncryptionKey();
return await this._db.storage.generateRandomKey();
return await this.db.crypto().generateRandomKey();
}
async decryptKey(key) {
async decryptKey(key: Cipher<"base64">): Promise<SerializedKey | null> {
const encryptionKey = await this._getEncryptionKey();
const plainData = await this._db.storage.decrypt(encryptionKey, key);
const plainData = await this.db.storage().decrypt(encryptionKey, key);
if (!plainData) return null;
return JSON.parse(plainData);
}
async delete(hashOrId, noteId) {
async delete(hashOrId: string, noteId: string) {
const attachment = this.attachment(hashOrId);
if (!attachment || !deleteItem(attachment.noteIds, noteId)) return;
if (!attachment.noteIds.length) {
attachment.dateDeleted = Date.now();
EV.publish(EVENTS.attachmentDeleted, attachment);
}
return await this._collection.updateItem(attachment);
return await this.collection.update(attachment);
}
async remove(hashOrId, localOnly) {
async remove(hashOrId: string, localOnly: boolean) {
const attachment = this.attachment(hashOrId);
if (!attachment || !attachment.metadata) return false;
@@ -211,50 +244,46 @@ export default class Attachments extends Collection {
throw new Error("This attachment is inside a locked note.");
if (
await this._db.fs.deleteFile(
attachment.metadata.hash,
localOnly || !attachment.dateUploaded
)
await this.db
.fs()
.deleteFile(
attachment.metadata.hash,
localOnly || !attachment.dateUploaded
)
) {
if (!localOnly) {
await this.detach(attachment);
}
await this._collection.removeItem(attachment.id);
await this.collection.remove(attachment.id);
return true;
}
return false;
}
async detach(attachment) {
await this._db.notes.init();
async detach(attachment: Attachment) {
for (const noteId of attachment.noteIds) {
const note = this._db.notes.note(noteId);
if (!note) continue;
const contentId = note.data.contentId;
await this._db.content.removeAttachments(contentId, [
const note = this.db.notes.note(noteId);
if (!note || !note.data.contentId) continue;
await this.db.content.removeAttachments(note.data.contentId, [
attachment.metadata.hash
]);
}
}
async _canDetach(attachment) {
await this._db.notes.init();
async _canDetach(attachment: Attachment) {
for (const noteId of attachment.noteIds) {
const note = this._db.notes.note(noteId);
const note = this.db.notes?.note(noteId);
if (note && note.data.locked) return false;
}
return true;
}
/**
* Get specified type of attachments of a note
* @param {string} noteId
* @param {"files"|"images"|"webclips"|"all"} type
* @returns {Array}
*/
ofNote(noteId, type) {
let attachments = [];
ofNote(
noteId: string,
type: "files" | "images" | "webclips" | "all"
): Attachment[] {
let attachments: Attachment[] = [];
if (type === "files") attachments = this.files;
else if (type === "images") attachments = this.images;
@@ -266,86 +295,95 @@ export default class Attachments extends Collection {
);
}
exists(hash) {
const attachment = this.all.find((a) => a.metadata?.hash === hash);
exists(hash: string) {
const attachment = this.all.find((a) => a.metadata.hash === hash);
return !!attachment;
}
/**
* @param {string} hash
* @param {"base64" | "text"} outputType
* @returns {Promise<string>} dataurl formatted string
*/
async read(hash, outputType) {
const attachment = this.all.find((a) => a.metadata?.hash === hash);
async read<TOutputFormat extends DataFormat>(
hash: string,
outputType: TOutputFormat
): Promise<Output<TOutputFormat> | undefined> {
const attachment = this.all.find((a) => a.metadata.hash === hash);
if (!attachment) return;
const key = await this.decryptKey(attachment.key);
const data = await this._db.fs.readEncrypted(
attachment.metadata.hash,
key,
{
if (!key) return;
const data = await this.db
.fs()
.readEncrypted(attachment.metadata.hash, key, {
chunkSize: attachment.chunkSize,
iv: attachment.iv,
salt: attachment.salt,
length: attachment.length,
alg: attachment.alg,
outputType
}
);
});
if (!data) return;
return outputType === "base64"
? dataurl.fromObject({ type: attachment.metadata.type, data })
: data;
return (
outputType === "base64"
? dataurl.fromObject({ type: attachment.metadata.type, data })
: data
) as Output<TOutputFormat>;
}
attachment(hashOrId) {
attachment(hashOrId: string) {
return this.all.find(
(a) => a.id === hashOrId || a.metadata?.hash === hashOrId
);
}
markAsUploaded(id) {
markAsUploaded(id: string) {
const attachment = this.attachment(id);
if (!attachment) return;
attachment.dateUploaded = Date.now();
attachment.failed = undefined;
return this._collection.updateItem(attachment);
return this.collection.update(attachment);
}
reset(id) {
reset(id: string) {
const attachment = this.attachment(id);
if (!attachment) return;
attachment.dateUploaded = undefined;
return this._collection.updateItem(attachment);
return this.collection.update(attachment);
}
markAsFailed(id, reason) {
markAsFailed(id: string, reason: string) {
const attachment = this.attachment(id);
if (!attachment) return;
attachment.failed = reason;
return this._collection.updateItem(attachment);
return this.collection.update(attachment);
}
async save(data, mimeType) {
const { hash } = await this._db.fs.hashBase64(data);
const attachment = this.attachment(hash);
if (attachment)
return {
metadata: attachment.metadata
};
async save(
data: string,
mimeType: string,
filename?: string
): Promise<string | undefined> {
const hashResult = await this.db.fs().hashBase64(data);
if (!hashResult) return;
if (this.exists(hashResult.hash)) return hashResult.hash;
const key = await this._db.attachments.generateKey();
const metadata = await this._db.fs.writeEncryptedBase64(
data,
const key = await this.generateKey();
const { hash, hashType, ...encryptionMetadata } = await this.db
.fs()
.writeEncryptedBase64(data, key, mimeType);
await this.add({
...encryptionMetadata,
key,
mimeType
);
return { key, metadata };
metadata: {
filename: filename || hash,
hash,
hashType,
type: mimeType || "application/octet-stream"
}
});
return hash;
}
async downloadMedia(noteId, hashesToLoad) {
async downloadMedia(noteId: string, hashesToLoad?: string[]) {
const attachments = this.media.filter(
(attachment) =>
!!attachment.metadata &&
@@ -353,7 +391,7 @@ export default class Attachments extends Collection {
(!hashesToLoad || hasItem(hashesToLoad, attachment.metadata?.hash))
);
await this._db.fs.queueDownloads(
await this.db.fs().queueDownloads(
attachments.map((a) => ({
filename: a.metadata.hash,
metadata: a.metadata,
@@ -373,35 +411,33 @@ export default class Attachments extends Collection {
)
continue;
const isDeleted = await this._db.fs.deleteFile(attachment.metadata.hash);
const isDeleted = await this.db.fs().deleteFile(attachment.metadata.hash);
if (!isDeleted) continue;
await this._collection.removeItem(attachment.id);
await this.collection.remove(attachment.id);
}
}
get pending() {
return this.all.filter(
(attachment) =>
attachment.metadata &&
(attachment.dateUploaded <= 0 || !attachment.dateUploaded)
(attachment) => !attachment.dateUploaded || attachment.dateUploaded <= 0
);
}
get uploaded() {
return this.all.filter((attachment) => attachment.dateUploaded > 0);
return this.all.filter((attachment) => !!attachment.dateUploaded);
}
get syncable() {
return this._collection
.getRaw()
return this.collection
.raw()
.filter(
(attachment) => attachment.dateUploaded > 0 || attachment.deleted
(attachment) => isDeleted(attachment) || !!attachment.dateUploaded
);
}
get deleted() {
return this.all.filter((attachment) => attachment.dateDeleted > 0);
return this.all.filter((attachment) => !!attachment.dateDeleted);
}
get images() {
@@ -434,30 +470,20 @@ export default class Attachments extends Collection {
);
}
/**
* @returns {any[]}
*/
get all() {
return this._collection.getItems();
return this.collection.items();
}
/**
* @private
*/
async _encryptKey(key) {
private async encryptKey(key: SerializedKey) {
const encryptionKey = await this._getEncryptionKey();
const encryptedKey = await this._db.storage.encrypt(
encryptionKey,
JSON.stringify(key)
);
const encryptedKey = await this.db
.storage()
.encrypt(encryptionKey, JSON.stringify(key));
return encryptedKey;
}
/**
* @private
*/
async _getEncryptionKey() {
this.key = await this._db.user.getAttachmentsKey();
this.key = await this.db.user?.getAttachmentsKey();
if (!this.key)
throw new Error(
"Failed to get user encryption key. Cannot cache attachments."
@@ -466,13 +492,14 @@ export default class Attachments extends Collection {
}
}
export function getOutputType(attachment) {
export function getOutputType(attachment: Attachment): DataFormat {
if (attachment.metadata.type === "application/vnd.notesnook.web-clip")
return "text";
else if (attachment.metadata.type.startsWith("image/")) return "base64";
return "uint8array";
}
function getAttachmentType(attachment) {
function getAttachmentType(attachment: Attachment) {
if (attachment.metadata.type === "application/vnd.notesnook.web-clip")
return "webclip";
else if (attachment.metadata.type.startsWith("image/")) return "image";

75
packages/core/src/collections/collection.js
View File

@@ -1,75 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { EV, EVENTS } from "../common";
import CachedCollection from "../database/cached-collection";
import IndexedCollection from "../database/indexed-collection";
class Collection {
static async new(db, name, cached = true, deferred = false) {
const collection = new this(db, name, cached);
if (!deferred) await collection.init();
else await collection._collection.indexer.init();
if (collection._collection.clear)
EV.subscribe(
EVENTS.userLoggedOut,
async () => await collection._collection.clear()
);
return collection;
}
async init() {
if (this.initialized) return;
await this._collection.init();
EV.publish(EVENTS.databaseCollectionInitiated, this.collectionName);
this.initialized = true;
}
/**
*
* @param {import("../api").default} db
*/
constructor(db, name, cached) {
this._db = db;
this.collectionName = name;
if (cached)
this._collection = new CachedCollection(
this._db.storage,
name,
this._db.eventManager
);
else
this._collection = new IndexedCollection(
this._db.storage,
name,
this._db.eventManager
);
}
async encrypted() {
const data = await this._collection.indexer.readMulti(
this._collection.indexer.getIndices()
);
return data.map((d) => d[1]);
}
}
export default Collection;

8
packages/core/src/utils/templates/markdown/builder.js → packages/core/src/collections/collection.ts
View File

@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
export {
template as buildMarkdown,
templateWithFrontmatter as buildMarkdownWithFrontmatter
} from "./template";
export interface ICollection {
name: string;
init(): Promise<void>;
}

104
packages/core/src/collections/colors.ts Normal file
View File

@@ -0,0 +1,104 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { ICollection } from "./collection";
import { getId } from "../utils/id";
import { Color, MaybeDeletedItem } from "../types";
import Database from "../api";
import { CachedCollection } from "../database/cached-collection";
import { Tags } from "./tags";
export class Colors implements ICollection {
name = "colors";
private readonly collection: CachedCollection<"colors", Color>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
"colors",
db.eventManager
);
}
init() {
return this.collection.init();
}
color(id: string) {
return this.collection.get(id);
}
async merge(remoteColor: MaybeDeletedItem<Color>) {
if (!remoteColor) return;
const localColor = this.collection.get(remoteColor.id);
if (!localColor || remoteColor.dateModified > localColor.dateModified)
await this.collection.add(remoteColor);
}
async add(item: Partial<Color>) {
if (item.remote)
throw new Error("Please use db.colors.merge to merge remote colors.");
const id = item.id || getId(item.dateCreated);
const oldColor = this.color(id);
item.title = item.title ? Tags.sanitize(item.title) : item.title;
if (!item.title && !oldColor?.title) throw new Error("Title is required.");
if (!item.colorCode && !oldColor?.colorCode)
throw new Error("Color code is required.");
const color: Color = {
id,
dateCreated: item.dateCreated || oldColor?.dateCreated || Date.now(),
dateModified: item.dateModified || oldColor?.dateModified || Date.now(),
title: item.title || oldColor?.title || "",
colorCode: item.colorCode || oldColor?.colorCode || "",
type: "color",
remote: false
};
await this.collection.add(color);
return color.id;
}
get raw() {
return this.collection.raw();
}
get all(): Color[] {
return this.collection.items();
}
async remove(id: string) {
await this.collection.remove(id);
await this.db.relations.cleanup();
}
async delete(id: string) {
await this.collection.delete(id);
await this.db.relations.cleanup();
}
exists(id: string) {
return this.collection.exists(id);
}
find(idOrTitle: string) {
return this.all.find((t) => t.title === idOrTitle || t.id === idOrTitle);
}
}

245
packages/core/src/collections/content.js
View File

@@ -1,245 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Collection from "./collection";
import { getId } from "../utils/id";
import { getContentFromData } from "../content-types";
import { hasItem } from "../utils/array";
import { getOutputType } from "./attachments";
export default class Content extends Collection {
async add(content) {
if (!content) return;
if (content.remote)
throw new Error(
"Please do not use this method for merging. Instead add the item directly to database."
);
if (content.deleted) return await this._collection.addItem(content);
if (typeof content.data === "object") {
if (typeof content.data.data === "string")
content.data = content.data.data;
else if (!content.data.iv && !content.data.cipher)
content.data = `<p>Content is invalid: ${JSON.stringify(
content.data
)}</p>`;
}
const oldContent = await this.raw(content.id);
if (content.id && oldContent) {
content = {
...oldContent,
...content,
dateEdited: content.dateEdited || Date.now()
};
}
const id = content.id || getId();
const contentItem = await this.extractAttachments({
noteId: content.noteId,
id,
type: content.type,
data: content.data || content,
dateEdited: content.dateEdited,
dateCreated: content.dateCreated,
dateModified: content.dateModified,
localOnly: !!content.localOnly,
conflicted: content.conflicted,
dateResolved: content.dateResolved
});
await this._collection.addItem(contentItem);
if (content.sessionId) {
await this._db.noteHistory.add(contentItem.noteId, content.sessionId, {
data: contentItem.data,
type: contentItem.type
});
}
return id;
}
async get(id) {
const content = await this.raw(id);
if (!content) return;
return content.data;
}
async raw(id) {
if (!id) return;
const content = await this._collection.getItem(id);
if (!content) return;
return content;
}
remove(id) {
if (!id) return;
return this._collection.removeItem(id);
}
multi(ids) {
return this._collection.getItems(ids);
}
exists(id) {
return this._collection.exists(id);
}
async all() {
return Object.values(
await this._collection.getItems(this._collection.indexer.indices)
);
}
insertMedia(contentItem) {
return this._insert(contentItem, this._db.attachments.read);
}
insertPlaceholders(contentItem, placeholder) {
return this._insert(contentItem, () => placeholder);
}
async downloadMedia(groupId, contentItem, notify = true) {
if (!contentItem) return contentItem;
const content = getContentFromData(contentItem.type, contentItem.data);
if (!content) console.log(contentItem);
contentItem.data = await content.insertMedia(async (hashes) => {
const attachments = hashes
.map((h) => this._db.attachments.attachment(h))
.filter((a) => !!a && !!a.metadata);
await this._db.fs.queueDownloads(
attachments.map((a) => ({
filename: a.metadata.hash,
metadata: a.metadata,
chunkSize: a.chunkSize
})),
groupId,
notify ? { readOnDownload: false } : undefined
);
const sources = {};
for (const attachment of attachments) {
if (!attachment.metadata) continue;
const src = await this._db.attachments.read(
attachment.metadata.hash,
getOutputType(attachment)
);
if (!src) continue;
sources[attachment.metadata.hash] = src;
}
return sources;
});
return contentItem;
}
async _insert(contentItem, getData) {
if (!contentItem || !getData) return;
const content = getContentFromData(contentItem.type, contentItem.data);
contentItem.data = await content.insertMedia(getData);
return contentItem;
}
/**
*
* @param {string} id
* @param {string[]} hashes
* @returns {Promise<any>}
*/
async removeAttachments(id, hashes) {
const contentItem = await this.raw(id);
const content = getContentFromData(contentItem.type, contentItem.data);
contentItem.data = content.removeAttachments(hashes);
await this.add(contentItem);
}
/**
*
* @param {any} contentItem
* @returns {Promise<any>}
*/
async extractAttachments(contentItem) {
if (contentItem.localOnly || typeof contentItem.data !== "string")
return contentItem;
const allAttachments = this._db.attachments.all;
const content = getContentFromData(contentItem.type, contentItem.data);
if (!content) return contentItem;
const { data, attachments } = await content.extractAttachments(
(data, type, mimeType) => this._db.attachments.save(data, type, mimeType)
);
const noteAttachments = allAttachments.filter((attachment) =>
hasItem(attachment.noteIds, contentItem.noteId)
);
const toDelete = noteAttachments.filter((attachment) => {
return attachments.every(
(a) => a && a.hash && a.hash !== attachment.metadata?.hash
);
});
const toAdd = attachments.filter((attachment) => {
return (
attachment &&
attachment.hash &&
noteAttachments.every((a) => attachment.hash !== a.metadata?.hash)
);
});
for (let attachment of toDelete) {
if (!attachment.metadata) continue;
await this._db.attachments.delete(
attachment.metadata?.hash,
contentItem.noteId
);
}
for (let attachment of toAdd) {
await this._db.attachments.add(attachment, contentItem.noteId);
}
if (toAdd.length > 0) {
contentItem.dateModified = Date.now();
}
contentItem.data = data;
return contentItem;
}
async cleanup() {
const indices = this._collection.indexer.indices;
await this._db.notes.init();
const notes = this._db.notes._collection.getRaw();
if (!notes.length && indices.length > 0) return [];
let ids = [];
for (let contentId of indices) {
const noteIndex = notes.findIndex((note) => note.contentId === contentId);
const isOrphaned = noteIndex === -1;
if (isOrphaned) {
ids.push(contentId);
await this._collection.deleteItem(contentId);
} else if (notes[noteIndex].localOnly) {
ids.push(contentId);
}
}
return ids;
}
}

311
packages/core/src/collections/content.ts Normal file
View File

@@ -0,0 +1,311 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { ICollection } from "./collection";
import { getId } from "../utils/id";
import { getContentFromData } from "../content-types";
import { hasItem } from "../utils/array";
import { ResolveHashes } from "../content-types/tiptap";
import { isCipher } from "../database/crypto";
import {
Attachment,
ContentItem,
ContentType,
EncryptedContentItem,
MaybeDeletedItem,
UnencryptedContentItem,
isDeleted
} from "../types";
import { IndexedCollection } from "../database/indexed-collection";
import Database from "../api";
import { getOutputType } from "./attachments";
export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({
noteId,
dateCreated: Date.now(),
dateEdited: Date.now(),
dateModified: Date.now(),
id: getId(),
localOnly: true,
type: "tiptap",
data: "<p></p>"
});
export class Content implements ICollection {
name = "content";
private readonly collection: IndexedCollection<"content", ContentItem>;
constructor(private readonly db: Database) {
this.collection = new IndexedCollection(
db.storage,
"content",
db.eventManager
);
}
async init() {
await this.collection.init();
}
async merge(content: MaybeDeletedItem<ContentItem>) {
return await this.collection.addItem(
isDeleted(content) || !isUnencryptedContent(content)
? content
: await this.extractAttachments(content)
);
}
async add(content: Partial<ContentItem>) {
if (typeof content.data === "object") {
if ("data" in content.data && typeof content.data.data === "string")
content.data = content.data.data;
else if (!content.data.iv && !content.data.cipher)
content.data = `<p>Content is invalid: ${JSON.stringify(
content.data
)}</p>`;
}
if (content.remote)
throw new Error(
"Please use db.content.merge for merging remote content."
);
const oldContent = content.id ? await this.raw(content.id) : undefined;
if (content.id && oldContent) {
content = {
...oldContent,
...content
};
}
if (!content.noteId) return;
const id = content.id || getId();
const contentItem: ContentItem = {
noteId: content.noteId,
id,
type: content.type || "tiptap",
data: content.data || "<p></p>",
dateEdited: content.dateEdited || Date.now(),
dateCreated: content.dateCreated || Date.now(),
dateModified: content.dateModified || Date.now(),
localOnly: !!content.localOnly,
conflicted: content.conflicted,
dateResolved: content.dateResolved
};
await this.collection.addItem(
isUnencryptedContent(contentItem)
? await this.extractAttachments(contentItem)
: contentItem
);
if (content.sessionId) {
await this.db.noteHistory?.add(
contentItem.noteId,
content.sessionId,
isCipher(contentItem.data),
{
data: contentItem.data,
type: contentItem.type
}
);
}
return id;
}
async get(id: string) {
const content = await this.raw(id);
if (!content || isDeleted(content)) return;
return content.data;
}
async raw(id: string) {
const content = await this.collection.getItem(id);
if (!content) return;
return content;
}
remove(id: string) {
if (!id) return;
return this.collection.removeItem(id);
}
multi(ids: string[]) {
return this.collection.getItems(ids);
}
exists(id: string) {
return this.collection.exists(id);
}
async all() {
return Object.values(
await this.collection.getItems(this.collection.indexer.indices)
);
}
insertMedia(contentItem: UnencryptedContentItem) {
return this.insert(contentItem, async (hashes) => {
const sources: Record<string, string> = {};
for (const hash of hashes) {
const src = await this.db.attachments.read(hash, "base64");
if (!src) continue;
sources[hash] = src;
}
return sources;
});
}
insertPlaceholders(contentItem: UnencryptedContentItem, placeholder: string) {
return this.insert(contentItem, async (hashes) => {
return Object.fromEntries(hashes.map((h) => [h, placeholder]));
});
}
async downloadMedia(
groupId: string,
contentItem: { type: ContentType; data: string },
notify = true
) {
const content = getContentFromData(contentItem.type, contentItem.data);
if (!content) return contentItem;
contentItem.data = await content.insertMedia(async (hashes) => {
const attachments = hashes.reduce((attachments, hash) => {
const attachment = this.db.attachments.attachment(hash);
if (!attachment) return attachments;
attachments.push(attachment);
return attachments;
}, [] as Attachment[]);
await this.db.fs().queueDownloads(
attachments.map((a) => ({
filename: a.metadata.hash,
metadata: a.metadata,
chunkSize: a.chunkSize
})),
groupId,
notify ? { readOnDownload: false } : undefined
);
const sources: Record<string, string> = {};
for (const attachment of attachments) {
const src = await this.db.attachments.read(
attachment.metadata.hash,
getOutputType(attachment)
);
if (!src) continue;
sources[attachment.metadata.hash] = src;
}
return sources;
});
return contentItem;
}
private async insert(
contentItem: UnencryptedContentItem,
getData: ResolveHashes
) {
const content = getContentFromData(contentItem.type, contentItem.data);
if (!content) return contentItem;
contentItem.data = await content.insertMedia(getData);
return contentItem;
}
async removeAttachments(id: string, hashes: string[]) {
const contentItem = await this.raw(id);
if (!contentItem || isDeleted(contentItem) || isCipher(contentItem.data))
return;
const content = getContentFromData(contentItem.type, contentItem.data);
if (!content) return;
contentItem.data = content.removeAttachments(hashes);
await this.add(contentItem);
}
async extractAttachments(contentItem: UnencryptedContentItem) {
if (contentItem.localOnly) return contentItem;
const allAttachments = this.db.attachments?.all;
const content = getContentFromData(contentItem.type, contentItem.data);
if (!content) return contentItem;
const { data, hashes } = await content.extractAttachments(
this.db.attachments.save
);
const noteAttachments = allAttachments.filter((attachment) =>
hasItem(attachment.noteIds, contentItem.noteId)
);
const toDelete = noteAttachments.filter((attachment) => {
return hashes.every((hash) => hash !== attachment.metadata.hash);
});
const toAdd = hashes.filter((hash) => {
return hash && noteAttachments.every((a) => hash !== a.metadata.hash);
});
for (const attachment of toDelete) {
await this.db.attachments.delete(
attachment.metadata.hash,
contentItem.noteId
);
}
for (const hash of toAdd) {
await this.db.attachments.add({
noteIds: [contentItem.noteId],
metadata: { hash }
});
}
if (toAdd.length > 0) {
contentItem.dateModified = Date.now();
}
contentItem.data = data;
return contentItem;
}
async cleanup() {
const indices = this.collection.indexer.indices;
await this.db.notes.init();
const notes = this.db.notes.all;
if (!notes.length && indices.length > 0) return [];
const ids = [];
for (const contentId of indices) {
const noteIndex = notes.findIndex((note) => note.contentId === contentId);
const isOrphaned = noteIndex === -1;
if (isOrphaned) {
ids.push(contentId);
await this.collection.deleteItem(contentId);
} else if (notes[noteIndex].localOnly) {
ids.push(contentId);
}
}
return ids;
}
}
export function isUnencryptedContent(
content: ContentItem
): content is UnencryptedContentItem {
return !isCipher(content.data);
}
export function isEncryptedContent(
content: ContentItem
): content is EncryptedContentItem {
return isCipher(content.data);
}

244
packages/core/src/collections/note-history.js
View File

@@ -1,244 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { makeSessionContentId } from "../utils/id";
import Collection from "./collection";
import SessionContent from "./session-content";
/**
* @typedef Session
* @property {string} id
* @property {string} noteId
* @property {string} sessionContentId
* @property {string} dateModified
* @property {string} dateCreated
* @property {boolean} locked
*/
/**
* @typedef Content
* @property {string} data
* @property {string} type
*/
export default class NoteHistory extends Collection {
async init() {
await super.init();
this.versionsLimit = 100;
/**
* @type {SessionContent}
*/
this.sessionContent = await SessionContent.new(
this._db,
"sessioncontent",
false
);
}
async merge(item) {
await this._collection.addItem(item);
}
/**
* Get complete session history of a note.
* @param noteId id of the note
* @returns {Promise<Session[]>} An array of session ids of a note
*/
async get(noteId) {
if (!noteId) return [];
let indices = this._collection.indexer.getIndices();
let sessionIds = indices.filter((id) => id.startsWith(noteId));
if (sessionIds.length === 0) return [];
let history = (await this._getSessions(sessionIds)) || [];
return history.sort(function (a, b) {
return b.dateModified - a.dateModified;
});
}
/**
* Add and update a session content
* @param {string} noteId id of the note
* @param {string} dateEdited edited date of the note
* @param {Content} content
*
* @returns {Promise<Session>}
*/
async add(noteId, dateEdited, content) {
if (!noteId || !dateEdited || !content) return;
let sessionId = `${noteId}_${dateEdited}`;
let oldSession = await this._collection.getItem(sessionId);
let session = {
type: "session",
id: sessionId,
sessionContentId: makeSessionContentId(sessionId),
noteId,
dateCreated: oldSession ? oldSession.dateCreated : Date.now(),
localOnly: true
};
const note = this._db.notes.note(noteId);
if (note && note.data.locked) {
session.locked = true;
}
await this._collection.addItem(session);
await this.sessionContent.add(sessionId, content, session.locked);
await this._cleanup(noteId);
return session;
}
async _cleanup(noteId, limit = this.versionsLimit) {
let history = await this.get(noteId);
if (history.length === 0 || history.length < limit) return;
history.sort(function (a, b) {
return a.dateModified - b.dateModified;
});
let deleteCount = history.length - limit;
for (let i = 0; i < deleteCount; i++) {
let session = history[i];
await this._remove(session);
}
}
/**
* Get content of a session
* @param {string} sessionId session id
*
* @returns {Promise<Content>}
*/
async content(sessionId) {
if (!sessionId) return;
/**
* @type {Session}
*/
let session = await this._collection.getItem(sessionId);
return await this.sessionContent.get(session.sessionContentId);
}
/**
* Remove a session from storage
* @param {string} sessionId
*/
async remove(sessionId) {
if (!sessionId) return;
/**
* @type {Session}
*/
let session = await this._collection.getItem(sessionId);
await this._remove(session);
}
/**
* Remove all sessions of a note from storage
* @param {string} noteId
*/
async clearSessions(noteId) {
if (!noteId) return;
let history = await this.get(noteId);
for (let item of history) {
await this._remove(item);
}
}
/**
*
* @param {Session} session
*/
async _remove(session) {
await this._collection.deleteItem(session.id);
await this.sessionContent.remove(session.sessionContentId);
}
/**
*
* @param {string} sessionId
*/
async restore(sessionId) {
/**
* @type {Session}
*/
const session = await this._collection.getItem(sessionId);
const content = await this.sessionContent.get(session.sessionContentId);
const note = this._db.notes.note(session.noteId);
if (!note) return;
if (session.locked) {
await this._db.content.add({
id: note.data.contentId,
data: content.data,
type: content.type
});
} else {
await this._db.notes.add({
id: session.noteId,
content: {
data: content.data,
type: content.type
}
});
}
}
/**
*
* @returns A json string containing all sessions with content
*/
async serialize() {
return JSON.stringify({
sessions: await this.all(),
sessionContents: await this.sessionContent.all()
});
}
async all() {
return this._getSessions(this._collection.indexer.getIndices());
}
async _getSessions(sessionIds) {
let items = await this._collection.getItems(sessionIds);
return Object.values(items);
}
/**
* Restore session history from a serialized json string.
* @param {string} data
* @returns
*/
async deserialize(data) {
if (!data) return;
let deserialized = JSON.parse(data);
if (!deserialized.sessions || !deserialized.sessionContents) return;
for (let session of deserialized.sessions) {
let sessionContent = deserialized.sessionContents.find((v) =>
v.id.includes(session.id)
);
if (sessionContent) {
await this._collection.addItem(session);
await this.sessionContent._collection.addItem(sessionContent);
}
}
}
}

166
packages/core/src/collections/note-history.ts Normal file
View File

@@ -0,0 +1,166 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Database from "../api";
import { isCipher } from "../database/crypto";
import { IndexedCollection } from "../database/indexed-collection";
import { HistorySession, isDeleted } from "../types";
import { makeSessionContentId } from "../utils/id";
import { ICollection } from "./collection";
import { SessionContent, NoteContent } from "./session-content";
export class NoteHistory implements ICollection {
name = "notehistory";
versionsLimit = 100;
sessionContent = new SessionContent(this.db);
private readonly collection: IndexedCollection<"notehistory", HistorySession>;
constructor(private readonly db: Database) {
this.collection = new IndexedCollection(
db.storage,
"notehistory",
db.eventManager
);
}
async init() {
await this.collection.init();
await this.sessionContent.init();
}
async merge(item: HistorySession) {
await this.collection.addItem(item);
}
async get(noteId: string) {
if (!noteId) return [];
const indices = this.collection.indexer.indices;
const sessionIds = indices.filter((id) => id.startsWith(noteId));
if (sessionIds.length === 0) return [];
const history = await this.getSessions(sessionIds);
return history.sort(function (a, b) {
return b.dateModified - a.dateModified;
});
}
async add(
noteId: string,
sessionId: string,
locked: boolean,
content: NoteContent<boolean>
) {
sessionId = `${noteId}_${sessionId}`;
const oldSession = await this.collection.getItem(sessionId);
if (oldSession && isDeleted(oldSession)) return;
const session: HistorySession = {
type: "session",
id: sessionId,
sessionContentId: makeSessionContentId(sessionId),
noteId,
dateCreated: oldSession ? oldSession.dateCreated : Date.now(),
dateModified: Date.now(),
localOnly: true,
locked
};
await this.collection.addItem(session);
await this.sessionContent.add(sessionId, content, locked);
await this.cleanup(noteId);
return session;
}
private async cleanup(noteId: string, limit = this.versionsLimit) {
const history = await this.get(noteId);
if (history.length === 0 || history.length < limit) return;
history.sort(function (a, b) {
return a.dateModified - b.dateModified;
});
const deleteCount = history.length - limit;
for (let i = 0; i < deleteCount; i++) {
const session = history[i];
await this._remove(session);
}
}
async content(sessionId: string) {
const session = await this.collection.getItem(sessionId);
if (!session || isDeleted(session)) return;
return await this.sessionContent.get(session.sessionContentId);
}
async remove(sessionId: string) {
const session = await this.collection.getItem(sessionId);
if (!session || isDeleted(session)) return;
await this._remove(session);
}
async clearSessions(noteId: string) {
if (!noteId) return;
const history = await this.get(noteId);
for (const item of history) {
await this._remove(item);
}
}
private async _remove(session: HistorySession) {
await this.collection.deleteItem(session.id);
await this.sessionContent.remove(session.sessionContentId);
}
async restore(sessionId: string) {
const session = await this.collection.getItem(sessionId);
if (!session || isDeleted(session)) return;
const content = await this.sessionContent.get(session.sessionContentId);
const note = this.db.notes.note(session.noteId);
if (!note || !content) return;
if (session.locked && isCipher(content.data)) {
await this.db.content.add({
id: note.contentId,
data: content.data,
type: content.type
});
} else if (content.data && !isCipher(content.data)) {
await this.db.notes.add({
id: session.noteId,
content: {
data: content.data,
type: content.type
}
});
}
}
async all() {
return this.getSessions(this.collection.indexer.indices);
}
private async getSessions(sessionIds: string[]): Promise<HistorySession[]> {
const items = await this.collection.getItems(sessionIds);
return Object.values(items).filter(
(a) => !isDeleted(a)
) as HistorySession[];
}
}

170
packages/core/src/collections/notebooks.js
View File

@@ -1,170 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Collection from "./collection";
import Notebook from "../models/notebook";
import { getId } from "../utils/id";
import { CHECK_IDS, checkIsUserPremium } from "../common";
import qclone from "qclone";
export default class Notebooks extends Collection {
merge(localNotebook, remoteNotebook, lastSyncedTimestamp) {
if (remoteNotebook.deleted) return remoteNotebook;
if (
localNotebook &&
(localNotebook.type === "trash" || localNotebook.deleted)
) {
if (localNotebook.dateModified > remoteNotebook.dateModified) return;
return remoteNotebook;
}
if (localNotebook && localNotebook.topics?.length) {
let isChanged = false;
// merge new and old topics
for (let oldTopic of localNotebook.topics) {
const newTopicIndex = remoteNotebook.topics.findIndex(
(t) => t.id === oldTopic.id
);
const newTopic = remoteNotebook.topics[newTopicIndex];
// CASE 1: if topic exists in old notebook but not in new notebook, it's deleted.
// However, if the dateEdited of topic in the old notebook is > lastSyncedTimestamp
// it was newly added or edited so add it to the new notebook.
if (!newTopic && oldTopic.dateEdited > lastSyncedTimestamp) {
remoteNotebook.topics.push({ ...oldTopic, dateEdited: Date.now() });
isChanged = true;
}
// CASE 2: if topic exists in new notebook but not in old notebook, it's new.
// This case will be automatically handled as the new notebook is our source of truth.
// CASE 3: if topic exists in both notebooks:
// if oldTopic.dateEdited > newTopic.dateEdited: we keep oldTopic
// and merge the notes of both topics.
else if (newTopic && oldTopic.dateEdited > newTopic.dateEdited) {
remoteNotebook.topics[newTopicIndex] = {
...oldTopic,
dateEdited: Date.now()
};
isChanged = true;
}
}
remoteNotebook.remote = !isChanged;
}
return remoteNotebook;
}
async add(notebookArg) {
if (!notebookArg) throw new Error("Notebook cannot be undefined or null.");
if (notebookArg.remote)
throw new Error(
"Please use db.notebooks.merge to merge remote notebooks"
);
//TODO reliably and efficiently check for duplicates.
const id = notebookArg.id || getId();
let oldNotebook = this._collection.getItem(id);
if (
!oldNotebook &&
this.all.length >= 3 &&
!(await checkIsUserPremium(CHECK_IDS.notebookAdd))
)
return;
let notebook = {
...oldNotebook,
...notebookArg,
topics: oldNotebook?.topics || []
};
if (!notebook.title) throw new Error("Notebook must contain a title.");
notebook = {
id,
type: "notebook",
title: notebook.title,
description: notebook.description,
pinned: !!notebook.pinned,
topics: notebook.topics || [],
dateCreated: notebook.dateCreated,
dateModified: notebook.dateModified,
dateEdited: Date.now()
};
await this._collection.addItem(notebook);
if (!oldNotebook && notebookArg.topics) {
await this.notebook(id).topics.add(...notebookArg.topics);
}
return id;
}
get raw() {
return this._collection.getRaw();
}
/**
* @return {any[]}
*/
get all() {
return this._collection.getItems();
}
get pinned() {
return this.all.filter((item) => item.pinned === true);
}
get deleted() {
return this.raw.filter((item) => item.dateDeleted > 0);
}
/**
*
* @param {string} id The id of the notebook
* @returns {Notebook} The notebook of the given id
*/
notebook(id) {
if (!id) return;
let notebook =
typeof id === "object" && "type" in id
? id
: this._collection.getItem(id);
if (!notebook || notebook.deleted) return;
return new Notebook(notebook, this._db);
}
exists(id) {
return this._collection.exists(id);
}
async delete(...ids) {
for (let id of ids) {
let notebook = this.notebook(id);
if (!notebook) continue;
const notebookData = qclone(notebook.data);
// await notebook.topics.delete(...notebook.data.topics);
await this._collection.removeItem(id);
await this._db.shortcuts.remove(id);
await this._db.trash.add(notebookData);
}
}
}

221
packages/core/src/collections/notebooks.ts Normal file
View File

@@ -0,0 +1,221 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { createNotebookModel } from "../models/notebook";
import { getId } from "../utils/id";
import { CHECK_IDS, checkIsUserPremium } from "../common";
import { CachedCollection } from "../database/cached-collection";
import Topics from "./topics";
import Database from "../api";
import {
MaybeDeletedItem,
Notebook,
Topic,
TrashOrItem,
isDeleted,
isTrashItem
} from "../types";
import { ICollection } from "./collection";
export class Notebooks implements ICollection {
name = "notebooks";
/**
* @internal
*/
collection: CachedCollection<"notebooks", TrashOrItem<Notebook>>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
"notebooks",
db.eventManager
);
}
init() {
return this.collection.init();
}
async merge(remoteNotebook: MaybeDeletedItem<TrashOrItem<Notebook>>) {
if (isDeleted(remoteNotebook) || isTrashItem(remoteNotebook))
return await this.collection.add(remoteNotebook);
const id = remoteNotebook.id;
const localNotebook = this.collection.get(id);
if (localNotebook && localNotebook.topics?.length) {
const lastSyncedTimestamp = await this.db.lastSynced();
let isChanged = false;
// merge new and old topics
for (const oldTopic of localNotebook.topics) {
const newTopicIndex = remoteNotebook.topics.findIndex(
(t) => t.id === oldTopic.id
);
const newTopic = remoteNotebook.topics[newTopicIndex];
// CASE 1: if topic exists in old notebook but not in new notebook, it's deleted.
// However, if the dateEdited of topic in the old notebook is > lastSyncedTimestamp
// it was newly added or edited so add it to the new notebook.
if (!newTopic && oldTopic.dateEdited > lastSyncedTimestamp) {
remoteNotebook.topics.push({ ...oldTopic, dateEdited: Date.now() });
isChanged = true;
}
// CASE 2: if topic exists in new notebook but not in old notebook, it's new.
// This case will be automatically handled as the new notebook is our source of truth.
// CASE 3: if topic exists in both notebooks:
// if oldTopic.dateEdited > newTopic.dateEdited: we keep oldTopic
// and merge the notes of both topics.
else if (newTopic && oldTopic.dateEdited > newTopic.dateEdited) {
remoteNotebook.topics[newTopicIndex] = {
...oldTopic,
dateEdited: Date.now()
};
isChanged = true;
}
}
remoteNotebook.remote = !isChanged;
}
return await this.collection.add(remoteNotebook);
}
async add(
notebookArg: Partial<
Omit<Notebook, "topics"> & { topics: Partial<Topic>[] }
>
) {
if (!notebookArg) throw new Error("Notebook cannot be undefined or null.");
if (notebookArg.remote)
throw new Error(
"Please use db.notebooks.merge to merge remote notebooks"
);
//TODO reliably and efficiently check for duplicates.
const id = notebookArg.id || getId();
const oldNotebook = this.collection.get(id);
if (oldNotebook && isTrashItem(oldNotebook))
throw new Error("Cannot modify trashed notebooks.");
if (
!oldNotebook &&
this.all.length >= 3 &&
!(await checkIsUserPremium(CHECK_IDS.notebookAdd))
)
return;
const mergedNotebook: Partial<Notebook> = {
...oldNotebook,
...notebookArg,
topics: oldNotebook?.topics || []
};
if (!mergedNotebook.title)
throw new Error("Notebook must contain a title.");
const notebook: Notebook = {
id,
type: "notebook",
title: mergedNotebook.title,
description: mergedNotebook.description,
pinned: !!mergedNotebook.pinned,
topics: mergedNotebook.topics || [],
dateCreated: mergedNotebook.dateCreated || Date.now(),
dateModified: mergedNotebook.dateModified || Date.now(),
dateEdited: Date.now()
};
await this.collection.add(notebook);
if (!oldNotebook && notebookArg.topics) {
await this.topics(id).add(...notebookArg.topics);
}
return id;
}
get raw() {
return this.collection.raw();
}
get all() {
return this.collection.items((note) =>
isTrashItem(note) ? undefined : note
) as Notebook[];
}
get pinned() {
return this.all.filter((item) => item.pinned === true);
}
get trashed() {
return this.raw.filter((item) => isTrashItem(item));
}
async pin(...ids: string[]) {
for (const id of ids) {
if (!this.exists(id)) continue;
await this.add({ id, pinned: true });
}
}
async unpin(...ids: string[]) {
for (const id of ids) {
if (!this.exists(id)) continue;
await this.add({ id, pinned: false });
}
}
topics(id: string) {
return new Topics(id, this.db);
}
totalNotes(id: string) {
const notebook = this.collection.get(id);
if (!notebook || isTrashItem(notebook)) return 0;
let count = 0;
for (const topic of notebook.topics) {
count += this.db.notes.topicReferences.count(topic.id);
}
return count + this.db.relations.from(notebook, "note").resolved().length;
}
notebook(idOrNotebook: string | Notebook) {
const notebook =
typeof idOrNotebook === "string"
? this.collection.get(idOrNotebook)
: idOrNotebook;
if (!notebook || isTrashItem(notebook)) return;
return createNotebookModel(notebook, this.db);
}
exists(id: string) {
return this.collection.exists(id);
}
async delete(...ids: string[]) {
for (const id of ids) {
const notebook = this.collection.get(id);
if (!notebook || isTrashItem(notebook)) continue;
await this.collection.remove(id);
await this.db.shortcuts?.remove(id);
await this.db.trash?.add(notebook);
}
}
}

487
packages/core/src/collections/notes.js
View File

@@ -1,487 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Collection from "./collection";
import Note from "../models/note";
import { getId } from "../utils/id";
import { getContentFromData } from "../content-types";
import qclone from "qclone";
import { deleteItem, findById } from "../utils/array";
import { NEWLINE_STRIP_REGEX, formatTitle } from "../utils/title-format";
/**
* @typedef {{ id: string, topic?: string, rebuildCache?: boolean }} NotebookReference
*/
export default class Notes extends Collection {
constructor(db, name, cached) {
super(db, name, cached);
this.topicReferences = new NoteIdCache(this);
}
async init() {
await super.init();
this.topicReferences.rebuild();
}
trashed(id) {
return this.raw.find((item) => item.dateDeleted > 0 && item.id === id);
}
async merge(localNote, remoteNote) {
const id = remoteNote.id;
if (localNote) {
if (localNote.localOnly) return;
if (localNote.color) await this._db.colors.untag(localNote.color, id);
for (let tag of localNote.tags || []) {
await this._db.tags.untag(tag, id);
}
}
await this._resolveColorAndTags(remoteNote);
return remoteNote;
}
async add(noteArg) {
if (!noteArg) return;
if (noteArg.remote)
throw new Error("Please use db.notes.merge to merge remote notes.");
let id = noteArg.id || getId();
let oldNote = this._collection.getItem(id);
let note = {
...oldNote,
...noteArg
};
if (oldNote) note.contentId = oldNote.contentId;
if (!oldNote && !noteArg.content && !noteArg.contentId && !noteArg.title)
return;
if (noteArg.content && noteArg.content.data && noteArg.content.type) {
const { type, data } = noteArg.content;
let content = getContentFromData(type, data);
if (!content) throw new Error("Invalid content type.");
note.contentId = await this._db.content.add({
noteId: id,
sessionId: note.sessionId,
id: note.contentId,
type,
data,
localOnly: !!note.localOnly
});
note.headline = getNoteHeadline(note, content);
if (oldNote) note.dateEdited = Date.now();
}
if (note.contentId && noteArg.localOnly !== undefined) {
await this._db.content.add({
id: note.contentId,
localOnly: !!noteArg.localOnly
});
}
const noteTitle = this._getNoteTitle(note, oldNote, note.headline);
if (oldNote && oldNote.title !== noteTitle) note.dateEdited = Date.now();
note = {
id,
contentId: note.contentId,
type: "note",
title: noteTitle,
headline: note.headline,
tags: note.tags || [],
notebooks: note.notebooks || undefined,
color: note.color,
pinned: !!note.pinned,
locked: !!note.locked,
favorite: !!note.favorite,
localOnly: !!note.localOnly,
conflicted: !!note.conflicted,
readonly: !!note.readonly,
dateCreated: note.dateCreated,
dateEdited:
noteArg.dateEdited || note.dateEdited || note.dateCreated || Date.now(),
dateModified: note.dateModified
};
await this._collection.addItem(note);
await this._resolveColorAndTags(note);
return note.id;
}
/**
*
* @param {string} id The id of note
* @returns {Note} The note of the given id
*/
note(id) {
if (!id) return;
let note =
typeof id === "object" && "type" in id
? id
: this._collection.getItem(id);
if (!note || note.deleted) return;
return new Note(note, this._db);
}
get raw() {
return this._collection.getRaw();
}
/**
* @returns {any[]}
*/
get all() {
const items = this._collection.getItems();
return items;
}
get pinned() {
return this.all.filter((item) => item.pinned === true);
}
get conflicted() {
return this.all.filter((item) => item.conflicted === true);
}
get favorites() {
return this.all.filter((item) => item.favorite === true);
}
get deleted() {
return this.raw.filter((item) => item.dateDeleted > 0);
}
get locked() {
return this.all.filter((item) => item.locked === true);
}
tagged(tagId) {
return this._getTagItems(tagId, "tags");
}
colored(colorId) {
return this._getTagItems(colorId, "colors");
}
exists(id) {
return this._collection.exists(id);
}
/**
* @private
*/
_getTagItems(tagId, collection) {
const tag = this._db[collection].tag(tagId);
if (!tag || tag.noteIds.length <= 0) return [];
const array = tag.noteIds.reduce((arr, id) => {
const item = this.note(id);
if (item) arr.push(item.data);
return arr;
}, []);
return array.sort((a, b) => b.dateCreated - a.dateCreated);
}
delete(...ids) {
return this._delete(true, ...ids);
}
remove(...ids) {
return this._delete(false, ...ids);
}
/**
* @private
*/
async _delete(moveToTrash = true, ...ids) {
for (let id of ids) {
let item = this.note(id);
if (!item) continue;
const itemData = qclone(item.data);
if (itemData.notebooks && !moveToTrash) {
for (let notebook of itemData.notebooks) {
for (let topicId of notebook.topics) {
await this.removeFromNotebook(
{ id: notebook.id, topic: topicId, rebuildCache: false },
id
);
}
}
}
for (let tag of itemData.tags) {
await this._db.tags.untag(tag, id);
}
if (itemData.color) {
await this._db.colors.untag(itemData.color, id);
}
const attachments = this._db.attachments.ofNote(itemData.id, "all");
for (let attachment of attachments) {
if (!attachment || !attachment.metadata) continue;
await this._db.attachments.delete(
attachment.metadata.hash,
itemData.id
);
}
// await this._collection.removeItem(id);
if (moveToTrash) await this._db.trash.add(itemData);
else {
await this._collection.removeItem(id);
await this._db.content.remove(itemData.contentId);
}
}
this.topicReferences.rebuild();
}
async _resolveColorAndTags(note) {
const { color, tags, id } = note;
if (color) {
const addedColor = await this._db.colors.add(color, id);
if (addedColor) note.color = addedColor.title;
}
if (tags && tags.length) {
for (let i = 0; i < tags.length; ++i) {
const tag = tags[i];
const addedTag = await this._db.tags.add(tag, id).catch(() => void 0);
if (!addedTag) {
tags.splice(i, 1);
continue;
}
if (addedTag.title !== tag) tags[i] = addedTag.title;
}
}
}
/**
* @param {NotebookReference} to
*/
async addToNotebook(to, ...noteIds) {
if (!to) throw new Error("The destination notebook cannot be undefined.");
if (!to.id) throw new Error("The destination notebook must contain id.");
const { id: notebookId, topic: topicId } = to;
for (let noteId of noteIds) {
let note = this._db.notes.note(noteId);
if (!note || note.data.deleted) continue;
if (topicId) {
const notebooks = note.notebooks || [];
const noteNotebook = notebooks.find((nb) => nb.id === notebookId);
const noteHasNotebook = !!noteNotebook;
const noteHasTopic =
noteHasNotebook && noteNotebook.topics.indexOf(topicId) > -1;
if (noteHasNotebook && !noteHasTopic) {
// 1 note can be inside multiple topics
noteNotebook.topics.push(topicId);
} else if (!noteHasNotebook) {
notebooks.push({
id: notebookId,
topics: [topicId]
});
}
if (!noteHasNotebook || !noteHasTopic) {
await this._db.notes.add({
id: noteId,
notebooks
});
this.topicReferences.add(topicId, noteId);
}
} else {
await this._db.relations.add(
{ id: notebookId, type: "notebook" },
note.data
);
}
}
}
/**
* @param {NotebookReference} to
*/
async removeFromNotebook(to, ...noteIds) {
if (!to) throw new Error("The destination notebook cannot be undefined.");
if (!to.id) throw new Error("The destination notebook must contain id.");
const { id: notebookId, topic: topicId, rebuildCache = true } = to;
for (const noteId of noteIds) {
const note = this.note(noteId);
if (!note || note.deleted) {
continue;
}
if (topicId) {
if (!note.notebooks) continue;
const { notebooks } = note;
const notebook = findById(notebooks, notebookId);
if (!notebook) continue;
const { topics } = notebook;
if (!deleteItem(topics, topicId)) continue;
if (topics.length <= 0) deleteItem(notebooks, notebook);
await this._db.notes.add({
id: noteId,
notebooks
});
} else {
await this._db.relations.unlink(
{ id: notebookId, type: "notebook" },
note.data
);
}
}
if (rebuildCache) this.topicReferences.rebuild();
}
async removeFromAllNotebooks(...noteIds) {
for (const noteId of noteIds) {
const note = this.note(noteId);
if (!note || note.deleted) {
continue;
}
await this._db.notes.add({
id: noteId,
notebooks: []
});
await this._db.relations.unlinkAll(note.data, "notebook");
}
this.topicReferences.rebuild();
}
async _clearAllNotebookReferences(notebookId) {
const notes = this._db.notes.all;
for (const note of notes) {
const { notebooks } = note;
if (!notebooks) continue;
for (let notebook of notebooks) {
if (notebook.id !== notebookId) continue;
if (!deleteItem(notebooks, notebook)) continue;
}
await this._collection.updateItem(note);
}
}
_getNoteTitle(note, oldNote, headline) {
if (note.title && note.title.trim().length > 0) {
return note.title.replace(NEWLINE_STRIP_REGEX, " ");
} else if (
oldNote &&
oldNote.title &&
oldNote.title.trim().length > 0 &&
(note.title === undefined || note.title === null)
) {
return oldNote.title.replace(NEWLINE_STRIP_REGEX, " ");
}
return formatTitle(
this._db.settings.getTitleFormat(),
this._db.settings.getDateFormat(),
this._db.settings.getTimeFormat(),
headline?.split(" ").splice(0, 10).join(" "),
this._collection.count()
);
}
}
function getNoteHeadline(note, content) {
if (note.locked) return "";
return content.toHeadline();
}
class NoteIdCache {
/**
*
* @param {Notes} notes
*/
constructor(notes) {
this.notes = notes;
this.cache = new Map();
}
rebuild() {
this.cache = new Map();
const notes = this.notes.all;
for (const note of notes) {
const { notebooks } = note;
if (!notebooks) continue;
for (let notebook of notebooks) {
for (let topic of notebook.topics) {
this.add(topic, note.id);
}
}
}
}
add(topicId, noteId) {
let noteIds = this.cache.get(topicId);
if (!noteIds) noteIds = [];
if (noteIds.includes(noteId)) return;
noteIds.push(noteId);
this.cache.set(topicId, noteIds);
}
has(topicId, noteId) {
let noteIds = this.cache.get(topicId);
if (!noteIds) return false;
return noteIds.includes(noteId);
}
count(topicId) {
let noteIds = this.cache.get(topicId);
if (!noteIds) return 0;
return noteIds.length;
}
get(topicId) {
return this.cache.get(topicId) || [];
}
}

514
packages/core/src/collections/notes.ts Normal file
View File

@@ -0,0 +1,514 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { createNoteModel } from "../models/note";
import { getId } from "../utils/id";
import { getContentFromData } from "../content-types";
import { deleteItem, findById } from "../utils/array";
import { NEWLINE_STRIP_REGEX, formatTitle } from "../utils/title-format";
import { clone } from "../utils/clone";
import { Tiptap } from "../content-types/tiptap";
import { EMPTY_CONTENT, isUnencryptedContent } from "./content";
import { CHECK_IDS, checkIsUserPremium } from "../common";
import { buildFromTemplate } from "../utils/templates";
import {
Note,
UnencryptedContentItem,
TrashOrItem,
isTrashItem,
MaybeDeletedItem,
isDeleted
} from "../types";
import Database from "../api";
import { CachedCollection } from "../database/cached-collection";
import { ICollection } from "./collection";
import { NoteContent } from "./session-content";
type NotebookReference = { id: string; topic?: string; rebuildCache?: boolean };
type ExportOptions = {
format: "html" | "md" | "txt" | "md-frontmatter";
contentItem?: UnencryptedContentItem;
rawContent?: string;
};
export class Notes implements ICollection {
name = "notes";
topicReferences = new NoteIdCache(this);
/**
* @internal
*/
collection: CachedCollection<"notes", TrashOrItem<Note>>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
"notes",
db.eventManager
);
}
async init() {
await this.collection.init();
this.topicReferences.rebuild();
}
async merge(remoteNote: MaybeDeletedItem<TrashOrItem<Note>>) {
if (!remoteNote) return;
const id = remoteNote.id;
const localNote = this.collection.get(id);
if (localNote && localNote.localOnly) return;
return await this.collection.add(remoteNote);
}
async add(
item: Partial<Note & { content: NoteContent<false>; sessionId: string }>
): Promise<string | undefined> {
if (!item) return;
if (item.remote)
throw new Error("Please use db.notes.merge to merge remote notes.");
const id = item.id || getId();
const oldNote = this.collection.get(id);
if (oldNote && isTrashItem(oldNote))
throw new Error("Cannot modify trashed notes.");
const note = {
...oldNote,
...item
};
if (oldNote) note.contentId = oldNote.contentId;
if (!oldNote && !item.content && !item.contentId && !item.title) return;
if (item.content && item.content.data && item.content.type) {
const { type, data } = item.content;
const content = getContentFromData(type, data);
if (!content) throw new Error("Invalid content type.");
note.contentId = await this.db.content.add({
noteId: id,
sessionId: note.sessionId,
id: note.contentId,
type,
data,
localOnly: !!note.localOnly
});
note.headline = note.locked ? "" : getNoteHeadline(content);
if (oldNote) note.dateEdited = Date.now();
}
if (item.localOnly !== undefined) {
await this.db.content.add({
id: note.contentId,
localOnly: !!item.localOnly
});
}
const noteTitle = this.getNoteTitle(note, oldNote, note.headline);
if (oldNote && oldNote.title !== noteTitle) note.dateEdited = Date.now();
await this.collection.add({
id,
contentId: note.contentId,
type: "note",
title: noteTitle,
headline: note.headline,
notebooks: note.notebooks || undefined,
pinned: !!note.pinned,
locked: !!note.locked,
favorite: !!note.favorite,
localOnly: !!note.localOnly,
conflicted: !!note.conflicted,
readonly: !!note.readonly,
dateCreated: note.dateCreated || Date.now(),
dateEdited:
item.dateEdited || note.dateEdited || note.dateCreated || Date.now(),
dateModified: note.dateModified || Date.now()
});
return id;
}
note(idOrNote: string | Note) {
if (!idOrNote) return;
const note =
typeof idOrNote === "object" ? idOrNote : this.collection.get(idOrNote);
if (!note || isTrashItem(note)) return;
return createNoteModel(note, this.db);
}
get raw() {
return this.collection.raw();
}
get all() {
return this.collection.items((note) =>
isTrashItem(note) ? undefined : note
) as Note[];
}
isTrashed(id: string) {
return this.raw.find((item) => item.id === id && isTrashItem(item));
}
get trashed() {
return this.raw.filter((item) => isTrashItem(item));
}
get pinned() {
return this.all.filter((item) => item.pinned === true);
}
get conflicted() {
return this.all.filter((item) => item.conflicted === true);
}
get favorites() {
return this.all.filter((item) => item.favorite === true);
}
get locked(): Note[] {
return this.all.filter(
(item) => !isTrashItem(item) && item.locked === true
) as Note[];
}
exists(id: string) {
return this.collection.exists(id);
}
delete(...ids: string[]) {
return this._delete(true, ...ids);
}
remove(...ids: string[]) {
return this._delete(false, ...ids);
}
async export(id: string, options: ExportOptions) {
const { format, rawContent } = options;
if (format !== "txt" && !(await checkIsUserPremium(CHECK_IDS.noteExport)))
return false;
const note = this.note(id);
if (!note) return false;
if (!options.contentItem) {
const rawContent = note.contentId
? await this.db.content.raw(note.contentId)
: undefined;
if (
rawContent &&
(isDeleted(rawContent) || !isUnencryptedContent(rawContent))
)
return false;
options.contentItem = rawContent || EMPTY_CONTENT(note.id);
}
const { data, type } =
format === "txt"
? options.contentItem
: await this.db.content.downloadMedia(
`export-${note.id}`,
options.contentItem,
false
);
const content = getContentFromData(type, data);
return buildFromTemplate(format, {
...note.data,
content:
rawContent ||
(format === "html"
? content.toHTML()
: format === "md"
? content.toMD()
: content.toTXT())
});
}
async duplicate(...ids: string[]) {
for (const id of ids) {
const note = this.collection.get(id);
if (!note || isTrashItem(note)) continue;
const content = note.contentId
? await this.db.content.raw(note.contentId)
: undefined;
if (content && (isDeleted(content) || !isUnencryptedContent(content)))
throw new Error("Cannot duplicate a locked or deleted note.");
const duplicateId = await this.db.notes.add({
...clone(note),
id: undefined,
content: content
? {
type: content.type,
data: content.data
}
: undefined,
readonly: false,
favorite: false,
pinned: false,
contentId: undefined,
title: note.title + " (Copy)",
dateEdited: undefined,
dateCreated: undefined,
dateModified: undefined
});
if (!duplicateId) return;
for (const notebook of this.db.relations
.to(note, "notebook")
.resolved()) {
await this.db.relations.add(notebook, {
id: duplicateId,
type: "note"
});
}
return duplicateId;
}
}
private async _delete(moveToTrash = true, ...ids: string[]) {
for (const id of ids) {
const item = this.collection.get(id);
if (!item) continue;
const itemData = clone(item);
if (itemData.notebooks && !moveToTrash) {
for (const notebook of itemData.notebooks) {
for (const topicId of notebook.topics) {
await this.removeFromNotebook(
{ id: notebook.id, topic: topicId, rebuildCache: false },
id
);
}
}
}
await this.db.relations.unlinkAll(item, "tag");
await this.db.relations.unlinkAll(item, "color");
const attachments = this.db.attachments.ofNote(itemData.id, "all");
for (const attachment of attachments) {
await this.db.attachments.delete(attachment.metadata.hash, itemData.id);
}
if (moveToTrash && !isTrashItem(itemData))
await this.db.trash.add(itemData);
else {
await this.collection.remove(id);
if (itemData.contentId)
await this.db.content.remove(itemData.contentId);
}
}
this.topicReferences.rebuild();
}
async addToNotebook(to: NotebookReference, ...noteIds: string[]) {
if (!to) throw new Error("The destination notebook cannot be undefined.");
if (!to.id) throw new Error("The destination notebook must contain id.");
const { id: notebookId, topic: topicId } = to;
for (const noteId of noteIds) {
const note = this.collection.get(noteId);
if (!note || isTrashItem(note)) continue;
if (topicId) {
const notebooks = note.notebooks || [];
const noteNotebook = notebooks.find((nb) => nb.id === notebookId);
const noteHasNotebook = !!noteNotebook;
const noteHasTopic =
noteHasNotebook && noteNotebook.topics.indexOf(topicId) > -1;
if (noteHasNotebook && !noteHasTopic) {
// 1 note can be inside multiple topics
noteNotebook.topics.push(topicId);
} else if (!noteHasNotebook) {
notebooks.push({
id: notebookId,
topics: [topicId]
});
}
if (!noteHasNotebook || !noteHasTopic) {
await this.db.notes.add({
id: noteId,
notebooks
});
this.topicReferences.add(topicId, noteId);
}
} else {
await this.db.relations.add({ id: notebookId, type: "notebook" }, note);
}
}
}
async removeFromNotebook(to: NotebookReference, ...noteIds: string[]) {
if (!to) throw new Error("The destination notebook cannot be undefined.");
if (!to.id) throw new Error("The destination notebook must contain id.");
const { id: notebookId, topic: topicId, rebuildCache = true } = to;
for (const noteId of noteIds) {
const note = this.collection.get(noteId);
if (!note || isTrashItem(note)) {
continue;
}
if (topicId) {
if (!note.notebooks) continue;
const { notebooks } = note;
const notebook = findById(notebooks, notebookId);
if (!notebook) continue;
const { topics } = notebook;
if (!deleteItem(topics, topicId)) continue;
if (topics.length <= 0) deleteItem(notebooks, notebook);
await this.db.notes.add({
id: noteId,
notebooks
});
} else {
await this.db.relations.unlink(
{ id: notebookId, type: "notebook" },
note
);
}
}
if (rebuildCache) this.topicReferences.rebuild();
}
async removeFromAllNotebooks(...noteIds: string[]) {
for (const noteId of noteIds) {
const note = this.collection.get(noteId);
if (!note || isTrashItem(note)) {
continue;
}
await this.db.notes.add({
id: noteId,
notebooks: []
});
await this.db.relations.unlinkAll(note, "notebook");
}
this.topicReferences.rebuild();
}
async _clearAllNotebookReferences(notebookId: string) {
const notes = this.db.notes.all;
for (const note of notes) {
const { notebooks } = note;
if (!notebooks) continue;
for (const notebook of notebooks) {
if (notebook.id !== notebookId) continue;
if (!deleteItem(notebooks, notebook)) continue;
}
await this.collection.update(note);
}
}
private getNoteTitle(note: Partial<Note>, oldNote?: Note, headline?: string) {
if (note.title && note.title.trim().length > 0) {
return note.title.replace(NEWLINE_STRIP_REGEX, " ");
} else if (
oldNote &&
oldNote.title &&
oldNote.title.trim().length > 0 &&
(note.title === undefined || note.title === null)
) {
return oldNote.title.replace(NEWLINE_STRIP_REGEX, " ");
}
return formatTitle(
this.db.settings.getTitleFormat(),
this.db.settings.getDateFormat(),
this.db.settings.getTimeFormat(),
headline?.split(" ").splice(0, 10).join(" "),
this.collection.count()
);
}
}
function getNoteHeadline(content: Tiptap) {
return content.toHeadline();
}
class NoteIdCache {
private cache = new Map<string, string[]>();
constructor(private readonly notes: Notes) {}
rebuild() {
this.cache = new Map();
const notes = this.notes.all;
for (const note of notes) {
const { notebooks } = note;
if (!notebooks) continue;
for (const notebook of notebooks) {
for (const topic of notebook.topics) {
this.add(topic, note.id);
}
}
}
}
add(topicId: string, noteId: string) {
let noteIds = this.cache.get(topicId);
if (!noteIds) noteIds = [];
if (noteIds.includes(noteId)) return;
noteIds.push(noteId);
this.cache.set(topicId, noteIds);
}
has(topicId: string, noteId: string) {
const noteIds = this.cache.get(topicId);
if (!noteIds) return false;
return noteIds.includes(noteId);
}
count(topicId: string) {
const noteIds = this.cache.get(topicId);
if (!noteIds) return 0;
return noteIds.length;
}
get(topicId: string) {
return this.cache.get(topicId) || [];
}
}

167
packages/core/src/collections/relations.js → packages/core/src/collections/relations.ts
View File

@@ -17,37 +17,36 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { CachedCollection } from "../database/cached-collection";
import { makeId } from "../utils/id";
import Collection from "./collection";
import { ICollection } from "./collection";
import { Relation, ItemMap, ItemReference, MaybeDeletedItem } from "../types";
import Database from "../api";
/**
* @typedef {{
* id: string;
* type: string;
* }} ItemReference
*
* @typedef {{
* id: string;
* type: string;
* from: ItemReference;
* to: ItemReference;
* dateCreated: number;
* dateModified: number;
* }} Relation
*/
type RelationsArray<TType extends keyof ItemMap> = Relation[] & {
resolved: (limit?: number) => ItemMap[TType][];
};
export default class Relations extends Collection {
async merge(relation) {
if (!relation) return;
return relation;
export class Relations implements ICollection {
name = "relations";
private readonly collection: CachedCollection<"relations", Relation>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
"relations",
db.eventManager
);
}
/**
*
* @param {ItemReference} from
* @param {ItemReference} to
*/
async add(from, to) {
init() {
return this.collection.init();
}
async merge(relation: MaybeDeletedItem<Relation>) {
await this.collection.add(relation);
}
async add(from: ItemReference, to: ItemReference) {
if (
this.all.find(
(a) =>
@@ -56,7 +55,7 @@ export default class Relations extends Collection {
)
return;
const relation = {
const relation: Relation = {
id: generateId(from, to),
type: "relation",
dateCreated: Date.now(),
@@ -65,74 +64,66 @@ export default class Relations extends Collection {
to: { id: to.id, type: to.type }
};
await this._collection.addItem(relation);
await this.collection.add(relation);
}
/**
*
* @param {ItemReference} reference
* @param {string} type
*/
from(reference, type) {
from<TType extends keyof ItemMap>(
reference: ItemReference,
type: TType
): RelationsArray<TType> {
const relations = this.all.filter(
(a) => compareItemReference(a.from, reference) && a.to.type === type
);
return this.resolve(relations, "to");
Object.defineProperties(relations, {
resolved: {
writable: false,
enumerable: false,
configurable: false,
value: (limit?: number) =>
this.resolve(limit ? relations.slice(0, limit) : relations, "to")
}
});
return relations as RelationsArray<TType>;
}
/**
*
* @param {ItemReference} reference
* @param {string} type
*/
to(reference, type) {
to<TType extends keyof ItemMap>(
reference: ItemReference,
type: TType
): RelationsArray<TType> {
const relations = this.all.filter(
(a) => compareItemReference(a.to, reference) && a.from.type === type
);
return this.resolve(relations, "from");
}
/**
* Count number of from -> to relations
* @param {ItemReference} reference
* @param {string} type
*/
count(reference, type) {
return this.all.filter(
(a) => compareItemReference(a.from, reference) && a.to.type === type
).length;
Object.defineProperties(relations, {
resolved: {
writable: false,
enumerable: false,
configurable: false,
value: (limit?: number) =>
this.resolve(limit ? relations.slice(0, limit) : relations, "from")
}
});
return relations as RelationsArray<TType>;
}
get raw() {
return this._collection.getRaw();
return this.collection.raw();
}
/**
* @return {Relation[]}
*/
get all() {
return this._collection.getItems();
get all(): Relation[] {
return this.collection.items();
}
/**
* @return {Relation}
*/
relation(id) {
return this._collection.getItem(id);
relation(id: string) {
return this.collection.get(id);
}
async remove(...ids) {
async remove(...ids: string[]) {
for (const id of ids) {
await this._collection.removeItem(id);
await this.collection.remove(id);
}
}
/**
*
* @param {ItemReference} from
* @param {ItemReference} to
*/
async unlink(from, to) {
async unlink(from: ItemReference, to: ItemReference) {
const relation = this.all.find(
(a) =>
compareItemReference(a.from, from) && compareItemReference(a.to, to)
@@ -142,12 +133,7 @@ export default class Relations extends Collection {
await this.remove(relation.id);
}
/**
*
* @param {ItemReference} from
* @param {string} type
*/
async unlinkAll(to, type) {
async unlinkAll(to: ItemReference, type: keyof ItemMap) {
for (const relation of this.all.filter(
(a) => compareItemReference(a.to, to) && a.from.type === type
)) {
@@ -155,30 +141,29 @@ export default class Relations extends Collection {
}
}
/**
* @param {Relation[]} relations
* @param {"from" | "to"} resolveType
* @private
*
* @returns {any[]}
*/
resolve(relations, resolveType) {
private resolve(relations: Relation[], resolveType: "from" | "to") {
const items = [];
for (const relation of relations) {
const reference = resolveType === "from" ? relation.from : relation.to;
let item = null;
switch (reference.type) {
case "tag":
item = this.db.tags.tag(reference.id);
break;
case "color":
item = this.db.colors.color(reference.id);
break;
case "reminder":
item = this._db.reminders.reminder(reference.id);
item = this.db.reminders.reminder(reference.id);
break;
case "note": {
const note = this._db.notes.note(reference.id);
const note = this.db.notes.note(reference.id);
if (!note) continue;
item = note.data;
break;
}
case "notebook": {
const notebook = this._db.notebooks.notebook(reference.id);
const notebook = this.db.notebooks.notebook(reference.id);
if (!notebook) continue;
item = notebook.data;
break;
@@ -195,7 +180,7 @@ export default class Relations extends Collection {
* @param {ItemReference} a
* @param {ItemReference} b
*/
function compareItemReference(a, b) {
function compareItemReference(a: ItemReference, b: ItemReference) {
return a.id === b.id && a.type === b.type;
}
@@ -204,7 +189,7 @@ function compareItemReference(a, b) {
* @param {ItemReference} a
* @param {ItemReference} b
*/
function generateId(a, b) {
function generateId(a: ItemReference, b: ItemReference) {
const str = `${a.id}${b.id}${a.type}${b.type}`;
return makeId(str);
}

112
packages/core/src/collections/reminders.js → packages/core/src/collections/reminders.ts
View File

@@ -22,64 +22,54 @@ import isSameOrBefore from "dayjs/plugin/isSameOrBefore";
import isToday from "dayjs/plugin/isToday";
import isTomorrow from "dayjs/plugin/isTomorrow";
import isYesterday from "dayjs/plugin/isYesterday";
import { formatDate } from "../utils/date";
import { TimeFormat, formatDate } from "../utils/date";
import { getId } from "../utils/id";
import Collection from "./collection";
import { ICollection } from "./collection";
import { CachedCollection } from "../database/cached-collection";
import { Reminder } from "../types";
import Database from "../api";
dayjs.extend(isTomorrow);
dayjs.extend(isSameOrBefore);
dayjs.extend(isYesterday);
dayjs.extend(isToday);
/**
* @typedef {{
* id: string;
* type: string;
* title: string;
* description?: string;
* priority: "silent" | "vibrate" | "urgent";
* date: number;
* mode: "repeat" | "once" | "permanent";
* recurringMode?: "week" | "month" | "day";
* selectedDays?: number[];
* dateCreated: number;
* dateModified: number;
* localOnly?: boolean;
* disabled?: boolean;
* snoozeUntil?: number;
* }} Reminder
*
*/
export default class Reminders extends Collection {
async merge(reminder) {
if (!reminder) return;
return reminder;
export class Reminders implements ICollection {
name = "reminders";
private readonly collection: CachedCollection<"reminders", Reminder>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
"reminders",
db.eventManager
);
}
/**
*
* @param {Partial<Reminder>} reminder
* @returns
*/
async add(reminder) {
async init() {
await this.collection.init();
}
async add(reminder: Partial<Reminder>) {
if (!reminder) return;
if (reminder.remote)
throw new Error("Please use db.reminders.merge to merge reminders.");
const id = reminder.id || getId();
let oldReminder = this._collection.getItem(id);
const oldReminder = this.collection.get(id);
reminder = {
...oldReminder,
...reminder
};
reminder = {
if (!reminder.date || !reminder.title)
throw new Error("date and title are required in a reminder.");
await this.collection.add({
id,
type: "reminder",
dateCreated: reminder.dateCreated,
dateModified: reminder.dateModified,
dateCreated: reminder.dateCreated || Date.now(),
dateModified: reminder.dateModified || Date.now(),
date: reminder.date,
description: reminder.description,
mode: reminder.mode || "once",
@@ -90,45 +80,37 @@ export default class Reminders extends Collection {
localOnly: reminder.localOnly,
disabled: reminder.disabled,
snoozeUntil: reminder.snoozeUntil
};
await this._collection.addItem(reminder);
});
return reminder.id;
}
get raw() {
return this._collection.getRaw();
return this.collection.raw();
}
/**
* @return {Reminder[]}
*/
get all() {
return this._collection.getItems();
return this.collection.items();
}
exists(itemId) {
return !!this.reminder(itemId);
exists(itemId: string) {
return this.collection.exists(itemId);
}
reminder(id) {
return this.all.find((reminder) => reminder.id === id);
reminder(id: string) {
return this.collection.get(id);
}
async remove(...reminderIds) {
async remove(...reminderIds: string[]) {
for (const id of reminderIds) {
await this._collection.removeItem(id);
await this.collection.remove(id);
}
}
}
/**
* @param {Reminder} reminder
*/
export function formatReminderTime(
reminder,
reminder: Reminder,
short = false,
options = {
options: { timeFormat: TimeFormat; dateFormat: string } = {
timeFormat: "12-hour",
dateFormat: "DD-MM-YYYY"
}
@@ -181,10 +163,7 @@ export function formatReminderTime(
return short ? text : `${tag}: ${text}`;
}
/**
* @param {Reminder} reminder
*/
export function isReminderToday(reminder) {
export function isReminderToday(reminder: Reminder) {
const { date } = reminder;
let time = date;
@@ -197,10 +176,7 @@ export function isReminderToday(reminder) {
return dayjs(time).isToday();
}
/**
* @param {Reminder} reminder
*/
export function getUpcomingReminderTime(reminder) {
export function getUpcomingReminderTime(reminder: Reminder) {
if (reminder.mode === "once") return reminder.date;
// this is only the time (hour & minutes); date is not included
const time = dayjs(reminder.date);
@@ -241,7 +217,7 @@ export function getUpcomingReminderTime(reminder) {
return relativeTime.valueOf();
}
export function getUpcomingReminder(reminders) {
export function getUpcomingReminder(reminders: Reminder[]) {
const sorted = reminders.sort((a, b) => {
const d1 = a.mode === "repeat" ? getUpcomingReminderTime(a) : a.date;
const d2 = b.mode === "repeat" ? getUpcomingReminderTime(b) : b.date;
@@ -250,11 +226,7 @@ export function getUpcomingReminder(reminders) {
return sorted[0];
}
/**
* @param {Reminder} reminder
*/
export function isReminderActive(reminder) {
export function isReminderActive(reminder: Reminder) {
const time =
reminder.mode === "once"
? reminder.date
@@ -264,6 +236,6 @@ export function isReminderActive(reminder) {
!reminder.disabled &&
(reminder.mode !== "once" ||
time > Date.now() ||
reminder.snoozeUntil > Date.now())
(reminder.snoozeUntil && reminder.snoozeUntil > Date.now()))
);
}

90
packages/core/src/collections/session-content.js
View File

@@ -1,90 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { tinyToTiptap } from "../migrations";
import { makeSessionContentId } from "../utils/id";
import Collection from "./collection";
export default class SessionContent extends Collection {
async merge(item) {
await this._collection.addItem(item);
}
/**
*
* @param {string} sessionId
* @param {{content:string:data:string}} content
*/
async add(sessionId, content, locked) {
if (!sessionId || !content) return;
let data = locked
? content.data
: await this._db.compressor.compress(content.data);
await this._collection.addItem({
type: "sessioncontent",
id: makeSessionContentId(sessionId),
data,
contentType: content.type,
compressed: !locked,
localOnly: true,
locked
});
}
/**
*
* @param {string} sessionId
* @returns {Promise<{content:string;data:string}>}
*/
async get(sessionContentId) {
if (!sessionContentId) return;
let session = await this._collection.getItem(sessionContentId);
if (session.contentType === "tiny" && session.compressed) {
session.compressed = await this._db.compressor.compress(
tinyToTiptap(await this._db.compressor.decompress(session.data))
);
session.contentType = "tiptap";
await this._collection.addItem(session);
}
return {
data: session.compressed
? await this._db.compressor.decompress(session.data)
: session.data,
type: session.contentType
};
}
/**
*
* @param {string} sessionContentId
*/
async remove(sessionContentId) {
await this._collection.deleteItem(sessionContentId);
}
async all() {
let indices = this._collection.indexer.getIndices();
let items = await this._collection.getItems(indices);
return Object.values(items);
}
}

118
packages/core/src/collections/session-content.ts Normal file
View File

@@ -0,0 +1,118 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Cipher } from "@notesnook/crypto";
import { tinyToTiptap } from "../migrations";
import { makeSessionContentId } from "../utils/id";
import { ICollection } from "./collection";
import { isCipher } from "../database/crypto";
import { IndexedCollection } from "../database/indexed-collection";
import Database from "../api";
import { ContentType, SessionContentItem, isDeleted } from "../types";
export type NoteContent<TLocked extends boolean> = {
data: TLocked extends true ? Cipher : string;
type: ContentType;
};
export class SessionContent implements ICollection {
name = "sessioncontent";
private readonly collection: IndexedCollection<
"sessioncontent",
SessionContentItem
>;
constructor(private readonly db: Database) {
this.collection = new IndexedCollection(
db.storage,
"sessioncontent",
db.eventManager
);
}
async init() {
await this.collection.init();
}
async merge(item: SessionContentItem) {
await this.collection.addItem(item);
}
async add<TLocked extends boolean>(
sessionId: string,
content: NoteContent<TLocked>,
locked: TLocked
) {
if (!sessionId || !content) return;
const data =
locked || isCipher(content.data)
? content.data
: await this.db.compressor().compress(content.data);
await this.collection.addItem({
type: "sessioncontent",
id: makeSessionContentId(sessionId),
data,
contentType: content.type,
compressed: !locked,
localOnly: true,
locked,
dateCreated: Date.now(),
dateModified: Date.now()
});
}
async get(sessionContentId: string) {
const session = await this.collection.getItem(sessionContentId);
if (!session || isDeleted(session)) return;
if (
session.contentType === "tiny" &&
session.compressed &&
!session.locked &&
!isCipher(session.data)
) {
session.data = await this.db
.compressor()
.compress(
tinyToTiptap(await this.db.compressor().decompress(session.data))
);
session.contentType = "tiptap";
await this.collection.addItem(session);
}
return {
data:
session.compressed && !isCipher(session.data)
? await this.db.compressor().decompress(session.data)
: session.data,
type: session.contentType
};
}
async remove(sessionContentId: string) {
await this.collection.deleteItem(sessionContentId);
}
async all() {
const indices = this.collection.indexer.indices;
const items = await this.collection.getItems(indices);
return Object.values(items);
}
}

200
packages/core/src/collections/settings.ts Normal file
View File

@@ -0,0 +1,200 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { EV, EVENTS } from "../common";
import { getId } from "../utils/id";
import Database from "../api";
import {
DefaultNotebook,
GroupOptions,
GroupingKey,
SettingsItem,
ToolbarConfig,
TrashCleanupInterval
} from "../types";
import { ICollection } from "./collection";
import { TimeFormat } from "../utils/date";
class Settings implements ICollection {
name = "settings";
private settings: SettingsItem = {
type: "settings",
dateModified: 0,
dateCreated: 0,
id: getId()
};
constructor(private readonly db: Database) {}
async init() {
const settings = await this.db.storage().read<SettingsItem>("settings");
this.reset(settings);
await this.save(false);
EV.subscribe(EVENTS.userLoggedOut, async () => {
this.reset();
await this.save(false);
});
}
get raw() {
return this.settings;
}
async merge(remoteItem: SettingsItem, lastSynced: number) {
if (this.settings.dateModified > lastSynced) {
this.settings.id = remoteItem.id;
this.settings.groupOptions = {
...this.settings.groupOptions,
...remoteItem.groupOptions
};
this.settings.toolbarConfig = {
...this.settings.toolbarConfig,
...remoteItem.toolbarConfig
};
this.settings.aliases = {
...this.settings.aliases,
...remoteItem.aliases
};
this.settings.dateModified = Date.now();
} else {
this.reset(remoteItem);
}
await this.save(false);
}
async setGroupOptions(key: GroupingKey, groupOptions: GroupOptions) {
if (!this.settings.groupOptions) this.settings.groupOptions = {};
this.settings.groupOptions[key] = groupOptions;
await this.save();
}
getGroupOptions(key: GroupingKey) {
return (
(this.settings.groupOptions && this.settings.groupOptions[key]) || {
groupBy: "default",
sortBy:
key === "trash"
? "dateDeleted"
: key === "tags"
? "dateCreated"
: key === "reminders"
? "dueDate"
: "dateEdited",
sortDirection: key === "reminders" ? "asc" : "desc"
}
);
}
async setToolbarConfig(key: string, config: ToolbarConfig) {
if (!this.settings.toolbarConfig) this.settings.toolbarConfig = {};
this.settings.toolbarConfig[key] = config;
await this.save();
}
getToolbarConfig(key: string) {
return this.settings.toolbarConfig && this.settings.toolbarConfig[key];
}
/**
* Setting to -1 means never clear trash.
*/
async setTrashCleanupInterval(interval: TrashCleanupInterval) {
this.settings.trashCleanupInterval = interval;
await this.save();
}
getTrashCleanupInterval() {
return this.settings.trashCleanupInterval || 7;
}
async setDefaultNotebook(item: DefaultNotebook | undefined) {
this.settings.defaultNotebook = !item
? undefined
: {
id: item.id,
topic: item.topic
};
await this.save();
}
getDefaultNotebook() {
return this.settings.defaultNotebook;
}
async setTitleFormat(format: string) {
this.settings.titleFormat = format;
await this.save();
}
getTitleFormat() {
return this.settings.titleFormat || "Note $date$ $time$";
}
getDateFormat() {
return this.settings.dateFormat || "DD-MM-YYYY";
}
async setDateFormat(format: string) {
this.settings.dateFormat = format;
await this.save();
}
getTimeFormat() {
return this.settings.timeFormat || "12-hour";
}
async setTimeFormat(format: TimeFormat) {
this.settings.timeFormat = format || "12-hour";
await this.save();
}
/**
* @deprecated only kept here for migration purposes.
*/
getAlias(id: string) {
return this.settings.aliases && this.settings.aliases[id];
}
private reset(settings?: Partial<SettingsItem>) {
this.settings = {
type: "settings",
id: getId(),
dateModified: 0,
dateCreated: 0,
...(settings || {})
};
}
private async save(updateDateModified = true) {
this.db.eventManager.publish(
EVENTS.databaseUpdated,
"settings",
this.settings
);
if (updateDateModified) {
this.settings.dateModified = Date.now();
this.settings.synced = false;
}
delete this.settings.remote;
await this.db.storage().write("settings", this.settings);
}
}
export default Settings;

114
packages/core/src/collections/shortcuts.js → packages/core/src/collections/shortcuts.ts
View File

@@ -17,52 +17,41 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Collection from "./collection";
/**
* @typedef {{
* id: string,
* type: "tag" | "notebook" | "topic",
* notebookId?: string
* }} ShortcutRef
*
* @typedef {{
* id: string,
* type: "shortcut",
* item: ShortcutRef,
* dateCreated: number,
* dateModified: number,
* sortIndex: number
* }} Shortcut
*
*/
import Database from "../api";
import { CachedCollection } from "../database/cached-collection";
import { Notebook, Shortcut, Tag, Topic } from "../types";
import { ICollection } from "./collection";
const ALLOWED_SHORTCUT_TYPES = ["notebook", "topic", "tag"];
export default class Shortcuts extends Collection {
async merge(shortcut) {
if (!shortcut) return;
return shortcut;
export class Shortcuts implements ICollection {
name = "shortcuts";
private readonly collection: CachedCollection<"shortcuts", Shortcut>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(
db.storage,
"shortcuts",
db.eventManager
);
}
/**
*
* @param {Partial<Shortcut>} shortcut
* @returns
*/
async add(shortcut) {
init() {
return this.collection.init();
}
async add(shortcut: Partial<Shortcut>) {
if (!shortcut) return;
if (shortcut.remote)
throw new Error(
"Please use db.shortcuts.merge to merge remote shortcuts."
);
if (!ALLOWED_SHORTCUT_TYPES.includes(shortcut.item.type))
if (shortcut.item && !ALLOWED_SHORTCUT_TYPES.includes(shortcut.item.type))
throw new Error("Cannot create a shortcut for this type of item.");
let oldShortcut = shortcut.item
const oldShortcut = shortcut.item
? this.shortcut(shortcut.item.id)
: shortcut.id
? this._collection.getItem(shortcut.id)
? this.shortcut(shortcut.id)
: null;
shortcut = {
@@ -70,84 +59,77 @@ export default class Shortcuts extends Collection {
...shortcut
};
if (!shortcut.item)
throw new Error("Cannot create a shortcut without an item.");
const id = shortcut.id || shortcut.item.id;
shortcut = {
await this.collection.add({
id,
type: "shortcut",
item: {
type: shortcut.item.type,
id: shortcut.item.id,
notebookId: shortcut.item.notebookId
},
dateCreated: shortcut.dateCreated,
dateModified: shortcut.dateModified,
sortIndex: this._collection.count()
};
await this._collection.addItem(shortcut);
return shortcut.id;
item: shortcut.item,
dateCreated: shortcut.dateCreated || Date.now(),
dateModified: shortcut.dateModified || Date.now(),
sortIndex: this.collection.count()
});
return id;
}
get raw() {
return this._collection.getRaw();
return this.collection.raw();
}
/**
* @return {Shortcut[]}
*/
get all() {
return this._collection.getItems();
return this.collection.items();
}
get resolved() {
return this.all.reduce((prev, shortcut) => {
const {
item: { id, type, notebookId }
item: { id }
} = shortcut;
let item = null;
switch (type) {
let item: Notebook | Topic | Tag | null | undefined = null;
switch (shortcut.item.type) {
case "notebook": {
const notebook = this._db.notebooks.notebook(id);
const notebook = this.db.notebooks.notebook(id);
item = notebook ? notebook.data : null;
break;
}
case "topic": {
const notebook = this._db.notebooks.notebook(notebookId);
if (notebook) {
const topic = notebook.topics.topic(id);
if (topic) item = topic._topic;
}
const topic = this.db.notebooks
.topics(shortcut.item.notebookId)
.topic(id);
if (topic) item = topic._topic;
break;
}
case "tag":
item = this._db.tags.tag(id);
item = this.db.tags.tag(id);
break;
}
if (item) prev.push(item);
return prev;
}, []);
}, [] as (Notebook | Topic | Tag)[]);
}
exists(itemId) {
return !!this.shortcut(itemId);
exists(id: string) {
return !!this.shortcut(id);
}
shortcut(id) {
shortcut(id: string) {
return this.all.find(
(shortcut) => shortcut.item.id === id || shortcut.id === id
);
}
async remove(...shortcutIds) {
async remove(...shortcutIds: string[]) {
const shortcuts = this.all.filter(
(shortcut) =>
shortcutIds.includes(shortcut.item.id) ||
shortcutIds.includes(shortcut.id)
);
for (const { id } of shortcuts) {
await this._collection.removeItem(id);
await this.collection.remove(id);
}
}
}

150
packages/core/src/collections/tags.js
View File

@@ -1,150 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Collection from "./collection";
import { makeId } from "../utils/id";
import { deleteItems, hasItem } from "../utils/array";
import setManipulator from "../utils/set";
import { Mutex } from "async-mutex";
export default class Tags extends Collection {
constructor(db, name, cached) {
super(db, name, cached);
this.mutex = new Mutex();
}
tag(id) {
const tagItem = this.all.find((t) => t.id === id || t.title === id);
return tagItem;
}
async merge(tag) {
if (!tag) return;
await this._collection.addItem(tag);
}
async add(tagId, ...noteIds) {
return this.mutex.runExclusive(async () => {
tagId = this.sanitize(tagId);
if (!tagId) throw new Error("Tag title cannot be empty.");
let tag = this.tag(tagId);
if (tag && !noteIds.length)
throw new Error("A tag with this id already exists.");
tag = tag || {
title: tagId
};
let id = tag.id || makeId(tag.title.toLowerCase());
let notes = tag.noteIds || [];
tag = {
type: "tag",
id,
title: tag.title,
noteIds: setManipulator.union(notes, noteIds),
localOnly: true
};
await this._collection.addItem(tag);
if (!this._db.settings.getAlias(tag.id))
await this._db.settings.setAlias(tag.id, tag.title);
return tag;
});
}
async rename(tagId, newName) {
let tag = this.tag(tagId);
if (!tag) {
console.error(`No tag found. Tag id:`, tagId);
return;
}
await this._db.settings.setAlias(tagId, newName);
await this._collection.addItem({ ...tag, alias: newName });
}
alias(tagId) {
let tag = this.tag(tagId);
if (!tag) {
console.error(`No tag found. Tag id:`, tagId);
return;
}
return this._db.settings.getAlias(tag.id) || tag.title;
}
get raw() {
return this._collection.getRaw();
}
/**
* @return {any[]}
*/
get all() {
return this._collection.getItems((item) => {
item.alias = this._db.settings.getAlias(item.id) || item.title;
return item;
});
}
async remove(tagId) {
let tag = this.tag(tagId);
if (!tag) {
console.error(`No tag found. Tag id:`, tagId);
return;
}
for (let noteId of tag.noteIds) {
const note = this._db.notes.note(noteId);
if (!note) continue;
if (hasItem(note.tags, tag.title)) await note.untag(tag.title);
}
await this._db.shortcuts.remove(tagId);
await this._collection.deleteItem(tagId);
}
async untag(tagId, ...noteIds) {
let tag = this.tag(tagId);
if (!tag) {
console.error(`No such tag found. Tag title:`, tagId);
return;
}
deleteItems(tag.noteIds, ...noteIds);
if (tag.noteIds.length > 0) await this._collection.addItem(tag);
else {
await this._db.shortcuts.remove(tag.id);
await this._collection.deleteItem(tag.id);
}
}
sanitize(tag) {
if (!tag) return;
let sanitized = tag.toLocaleLowerCase();
sanitized = sanitized.replace(/[\s]+/g, "");
// sanitized = sanitized.replace(/[+!@#$%^&*()+{}\][:;'"<>?/.\s=,]+/g, "");
return sanitized.trim();
}
}

96
packages/core/src/collections/tags.ts Normal file
View File

@@ -0,0 +1,96 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { getId } from "../utils/id";
import { CachedCollection } from "../database/cached-collection";
import { MaybeDeletedItem, Tag } from "../types";
import Database from "../api";
import { ICollection } from "./collection";
export class Tags implements ICollection {
name = "tags";
private readonly collection: CachedCollection<"tags", Tag>;
constructor(private readonly db: Database) {
this.collection = new CachedCollection(db.storage, "tags", db.eventManager);
}
init() {
return this.collection.init();
}
tag(id: string) {
return this.collection.get(id);
}
async merge(remoteTag: MaybeDeletedItem<Tag>) {
if (!remoteTag) return;
const localTag = this.collection.get(remoteTag.id);
if (!localTag || remoteTag.dateModified > localTag.dateModified)
await this.collection.add(remoteTag);
}
async add(item: Partial<Tag>) {
if (item.remote)
throw new Error("Please use db.tags.merge to merge remote tags.");
const id = item.id || getId(item.dateCreated);
const oldTag = this.tag(id);
item.title = item.title ? Tags.sanitize(item.title) : item.title;
if (!item.title && !oldTag?.title) throw new Error("Title is required.");
const tag: Tag = {
id,
dateCreated: item.dateCreated || oldTag?.dateCreated || Date.now(),
dateModified: item.dateModified || oldTag?.dateModified || Date.now(),
title: item.title || oldTag?.title || "",
type: "tag",
remote: false
};
await this.collection.add(tag);
return tag.id;
}
get raw() {
return this.collection.raw();
}
get all() {
return this.collection.items();
}
async remove(id: string) {
await this.collection.remove(id);
await this.db.relations.cleanup();
}
async delete(id: string) {
await this.collection.delete(id);
await this.db.relations.cleanup();
}
exists(id: string) {
return this.collection.exists(id);
}
static sanitize(title: string) {
return title.replace(/^\s+|\s+$/gm, "");
}
}

144
packages/core/src/collections/topics.js
View File

@@ -1,144 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Topic from "../models/topic";
import qclone from "qclone";
import { getId } from "../utils/id";
export default class Topics {
/**
*
* @param {import('../api').default} db
* @param {string} notebookId
*/
constructor(notebookId, db) {
this._db = db;
this._notebookId = notebookId;
}
has(topic) {
return (
this.all.findIndex(
(v) => v.id === topic || v.title === (topic.title || topic)
) > -1
);
}
/* _dedupe(source) {
let length = source.length,
seen = new Map();
for (let index = 0; index < length; index++) {
let value = source[index];
if (value.id) {
seen.set(value.id, {
...seen.get(value.id),
...value,
});
continue;
}
let title = value.title || value;
if (title.trim().length <= 0) continue;
seen.set(title, value);
}
return seen;
} */
async add(...topics) {
let notebook = qclone(this._db.notebooks.notebook(this._notebookId).data);
let allTopics = [...notebook.topics, ...topics];
notebook.topics = [];
for (let t of allTopics) {
let topic = makeTopic(t, this._notebookId);
if (notebook.topics.findIndex((_topic) => _topic.title === t) > -1)
continue;
if (topic.title.length <= 0) continue;
if (topics.findIndex((t) => topic.id === t.id) > -1)
topic.dateEdited = Date.now();
let index = notebook.topics.findIndex((t) => t.id === topic.id);
if (index > -1) {
notebook.topics[index] = {
...notebook.topics[index],
...topic
};
} else {
notebook.topics.push(topic);
}
}
return this._db.notebooks._collection.updateItem(notebook);
}
/**
* @returns {Array} an array containing all the topics
*/
get all() {
return this._db.notebooks.notebook(this._notebookId).data.topics;
}
/**
*
* @param {string | Object} topic can be an object or string containing the topic title.
* @returns {Topic} The topic by the given title
*/
topic(topic) {
if (typeof topic === "string") {
topic = this.all.find((t) => t.id === topic || t.title === topic);
}
if (!topic) return;
return new Topic(topic, this._notebookId, this._db);
}
async delete(...topicIds) {
const notebook = qclone(this._db.notebooks.notebook(this._notebookId).data);
let allTopics = notebook.topics;
for (let topicId of topicIds) {
const topic = this.topic(topicId);
if (!topic) continue;
await topic.clear();
await this._db.shortcuts.remove(topicId);
const topicIndex = allTopics.findIndex(
(t) => t.id === topicId || t.title === topicId
);
allTopics.splice(topicIndex, 1);
}
await this._db.notebooks._collection.updateItem(notebook);
}
}
// we export this for testing.
export function makeTopic(topic, notebookId) {
if (typeof topic !== "string") return topic;
return {
type: "topic",
id: getId(), //topic,
notebookId,
title: topic.trim(),
dateCreated: Date.now(),
dateEdited: Date.now()
};
}

114
packages/core/src/collections/topics.ts Normal file
View File

@@ -0,0 +1,114 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { createTopicModel } from "../models/topic";
import { getId } from "../utils/id";
import Database from "../api";
import { clone } from "../utils/clone";
import { Topic } from "../types";
export default class Topics {
constructor(
private readonly notebookId: string,
private readonly db: Database
) {}
has(topic: string) {
return this.all.findIndex((v) => v.id === topic || v.title === topic) > -1;
}
async add(...topics: Partial<Topic>[]) {
const notebook = clone(this.db.notebooks.notebook(this.notebookId)?.data);
if (!notebook) return;
const allTopics = [...notebook.topics, ...topics];
notebook.topics = [];
for (const t of allTopics) {
const topic = makeTopic(t, this.notebookId);
if (!topic) continue;
if (topics.findIndex((t) => t.id === topic.id) > -1)
topic.dateEdited = Date.now();
const index = notebook.topics.findIndex((t) => t.id === topic.id);
if (index > -1) {
notebook.topics[index] = {
...notebook.topics[index],
...topic
};
} else {
notebook.topics.push(topic);
}
}
return this.db.notebooks.collection.update(notebook);
}
get all() {
return this.db.notebooks.notebook(this.notebookId)?.data.topics || [];
}
topic(idOrTitleOrTopic: string | Topic) {
const topic =
typeof idOrTitleOrTopic === "string"
? this.all.find(
(t) => t.id === idOrTitleOrTopic || t.title === idOrTitleOrTopic
)
: idOrTitleOrTopic;
if (!topic) return;
return createTopicModel(topic, this.notebookId, this.db);
}
async delete(...topicIds: string[]) {
const notebook = clone(this.db.notebooks.notebook(this.notebookId)?.data);
if (!notebook) return;
const allTopics = notebook.topics;
for (const topicId of topicIds) {
const topic = this.topic(topicId);
if (!topic) continue;
await topic.clear();
await this.db.shortcuts.remove(topicId);
const topicIndex = allTopics.findIndex(
(t) => t.id === topicId || t.title === topicId
);
allTopics.splice(topicIndex, 1);
}
return this.db.notebooks.collection.update(notebook);
}
}
// we export this for testing.
export function makeTopic(
topic: Partial<Topic>,
notebookId: string
): Topic | undefined {
if (!topic.title) return;
return {
type: "topic",
id: topic.id || getId(),
notebookId: topic.notebookId || notebookId,
title: topic.title.trim(),
dateCreated: topic.dateCreated || Date.now(),
dateEdited: topic.dateEdited || Date.now(),
dateModified: Date.now()
};
}

146
packages/core/src/collections/trash.js
View File

@@ -1,146 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import dayjs from "dayjs";
export default class Trash {
/**
*
* @param {import("../api").default} db
*/
constructor(db) {
this._db = db;
this.collections = ["notes", "notebooks"];
}
async init() {
await this.cleanup();
}
async cleanup() {
const now = dayjs().unix();
const duration = this._db.settings.getTrashCleanupInterval();
if (duration === -1 || !duration) return;
for (const item of this.all) {
if (dayjs(item.dateDeleted).add(duration, "days").unix() > now) continue;
await this.delete(item.id);
}
}
get all() {
let trashItems = [];
for (const key of this.collections) {
const collection = this._db[key];
trashItems.push(...collection.deleted);
}
return trashItems;
}
_getItem(id) {
for (const key of this.collections) {
const collection = this._db[key]._collection;
if (collection.has(id)) return [collection.getItem(id), collection];
}
return [];
}
async add(item) {
const collection = collectionNameFromItem(item);
if (!item || !item.type || !collection) return;
await this._db[collection]._collection.updateItem({
...item,
id: item.itemId || item.id,
type: "trash",
itemType: item.itemType || item.type,
dateDeleted: item.dateDeleted || Date.now(),
deleted: true
});
}
async delete(...ids) {
for (let id of ids) {
if (!id) continue;
let [item, collection] = this._getItem(id);
if (!item) continue;
if (item.itemType === "note") {
await this._db.content.remove(item.contentId);
await this._db.noteHistory.clearSessions(id);
} else if (item.itemType === "notebook") {
await this._db.notes._clearAllNotebookReferences(item.id);
}
await collection.removeItem(id);
}
this._db.notes.topicReferences.rebuild();
}
async restore(...ids) {
for (let id of ids) {
let [item] = this._getItem(id);
if (!item) continue;
item = { ...item };
delete item.dateDeleted;
delete item.deleted;
item.type = item.itemType;
delete item.itemType;
if (item.type === "note") {
await this._db.notes.add(item);
} else if (item.type === "notebook") {
await this._db.notebooks.add(item);
}
}
this._db.notes.topicReferences.rebuild();
}
async clear() {
for (let item of this.all) {
await this.delete(item.id);
}
}
synced(id) {
let [item] = this._getItem(id);
if (item.itemType === "note") {
const { contentId } = item;
return !contentId || this._db.content.exists(contentId);
} else return true;
}
/**
*
* @param {string} id
*/
exists(id) {
return this.all.findIndex((item) => item.id === id) > -1;
}
}
function collectionNameFromItem(item) {
const { type, itemType } = item;
let typeToCompare = type === "trash" ? itemType : type;
switch (typeToCompare) {
case "note":
return "notes";
case "notebook":
return "notebooks";
default:
return null;
}
}

136
packages/core/src/collections/trash.ts Normal file
View File

@@ -0,0 +1,136 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import dayjs from "dayjs";
import Database from "../api";
import { BaseTrashItem, Note, Notebook, isTrashItem } from "../types";
function toTrashItem<T extends Note | Notebook>(item: T): BaseTrashItem<T> {
return {
...item,
id: item.id,
type: "trash",
itemType: item.type,
dateDeleted: Date.now()
};
}
export default class Trash {
collections = ["notes", "notebooks"] as const;
constructor(private readonly db: Database) {}
async init() {
await this.cleanup();
}
async cleanup() {
const now = dayjs().unix();
const duration = this.db.settings.getTrashCleanupInterval();
if (duration === -1 || !duration) return;
for (const item of this.all) {
if (
isTrashItem(item) &&
item.dateDeleted &&
dayjs(item.dateDeleted).add(duration, "days").unix() > now
)
continue;
await this.delete(item.id);
}
}
get all() {
const trashItems = [];
for (const key of this.collections) {
const collection = this.db[key];
trashItems.push(...collection.trashed);
}
return trashItems;
}
private getItem(id: string) {
for (const key of this.collections) {
const collection = this.db[key].collection;
const item = collection.get(id);
if (item && isTrashItem(item)) return [item, collection] as const;
}
return [] as const;
}
async add(item: Note | Notebook) {
if (item.type === "note") {
await this.db.notes.collection.update(toTrashItem(item));
} else if (item.type === "notebook") {
await this.db.notebooks.collection.update(toTrashItem(item));
}
}
async delete(...ids: string[]) {
for (const id of ids) {
if (!id) continue;
const [item, collection] = this.getItem(id);
if (!item || !collection) continue;
if (item.itemType === "note") {
if (item.contentId) await this.db.content.remove(item.contentId);
await this.db.noteHistory.clearSessions(id);
} else if (item.itemType === "notebook") {
await this.db.notes._clearAllNotebookReferences(item.id);
}
await collection.remove(id);
}
this.db.notes.topicReferences.rebuild();
}
async restore(...ids: string[]) {
for (const id of ids) {
const [item] = this.getItem(id);
if (!item) continue;
if (item.itemType === "note") {
await this.db.notes.collection.update({ ...item, type: "note" });
} else if (item.itemType === "notebook") {
await this.db.notebooks.collection.update({
...item,
type: "notebook"
});
}
}
this.db.notes.topicReferences.rebuild();
}
async clear() {
for (const item of this.all) {
await this.delete(item.id);
}
}
synced(id: string) {
const [item] = this.getItem(id);
if (item && item.itemType === "note") {
const { contentId } = item;
return !contentId || this.db.content.exists(contentId);
} else return true;
}
/**
*
* @param {string} id
*/
exists(id: string) {
return this.all.findIndex((item) => item.id === id) > -1;
}
}

19
packages/core/src/common.js → packages/core/src/common.ts
View File

@@ -21,7 +21,7 @@ import EventManager from "./utils/event-manager";
export const EV = new EventManager();
export async function checkIsUserPremium(type) {
export async function checkIsUserPremium(type: string) {
// if (process.env.NODE_ENV === "test") return true;
const results = await EV.publishWithResult(EVENTS.userCheckStatus, type);
@@ -29,21 +29,30 @@ export async function checkIsUserPremium(type) {
return results.some((r) => r.type === type && r.result === true);
}
export async function checkSyncStatus(type) {
export async function checkSyncStatus(type: string) {
const results = await EV.publishWithResult(EVENTS.syncCheckStatus, type);
if (typeof results === "boolean") return results;
else if (typeof results === "undefined") return true;
return results.some((r) => r.type === type && r.result === true);
}
export function sendSyncProgressEvent(EV, type, current) {
export function sendSyncProgressEvent(
EV: EventManager,
type: string,
current: number
) {
EV.publish(EVENTS.syncProgress, {
type,
current
});
}
export function sendMigrationProgressEvent(EV, collection, total, current) {
export function sendMigrationProgressEvent(
EV: EventManager,
collection: string,
total: number,
current?: number
) {
EV.publish(EVENTS.migrationProgress, {
collection,
total,
@@ -121,4 +130,4 @@ export const DATE_FORMATS = [
export const TIME_FORMATS = ["12-hour", "24-hour"];
export const CURRENT_DATABASE_VERSION = 5.9;
export const CURRENT_DATABASE_VERSION = 6.0;

4
packages/core/src/content-types/__tests__/tiptap.test.js
View File

@@ -27,9 +27,9 @@ import { test, expect } from "vitest";
test("img src is empty after extract attachments", async () => {
const tiptap = new Tiptap(IMG_CONTENT_WITHOUT_HASH);
const result = await tiptap.extractAttachments(async () => {
return { key: "hello", metadata: { hash: "helloworld" } };
return "helloworld";
});
expect(result.attachments).toHaveLength(1);
expect(result.hashes).toHaveLength(1);
expect(result.data).not.toContain(`src="data:image/png;`);
expect(result.data).not.toContain(`src=""`);
expect(result.data).toContain(`data-hash="helloworld"`);

5
packages/core/src/content-types/index.js → packages/core/src/content-types/index.ts
View File

@@ -17,11 +17,10 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { ContentType } from "../types";
import { Tiptap } from "./tiptap";
export function getContentFromData(type, data) {
if (!type) return null;
export function getContentFromData(type: ContentType, data: string) {
switch (type) {
case "tiptap":
return new Tiptap(data);

112
packages/core/src/content-types/tiptap.js → packages/core/src/content-types/tiptap.ts
View File

@@ -20,8 +20,19 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import showdown from "@streetwriters/showdown";
import dataurl from "../utils/dataurl";
import { extractFirstParagraph, getDummyDocument } from "../utils/html-parser";
import { HTMLParser, HTMLRewriter } from "../utils/html-rewriter";
import { convert } from "html-to-text";
import { HTMLRewriter } from "../utils/html-rewriter";
import { HTMLParser } from "../utils/html-parser";
import {
DomNode,
FormatOptions,
RecursiveCallback,
convert
} from "html-to-text";
import { BlockTextBuilder } from "html-to-text/lib/block-text-builder";
export type ResolveHashes = (
hashes: string[]
) => Promise<Record<string, string>>;
const ATTRIBUTES = {
hash: "data-hash",
@@ -30,15 +41,13 @@ const ATTRIBUTES = {
src: "src"
};
showdown.helper.document = getDummyDocument();
var converter = new showdown.Converter();
(showdown.helper as any).document = getDummyDocument();
const converter = new showdown.Converter();
converter.setFlavor("original");
const splitter = /\W+/gm;
export class Tiptap {
constructor(data) {
this.data = data;
}
constructor(private readonly data: string) {}
toHTML() {
return this.data;
@@ -91,17 +100,14 @@ export class Tiptap {
// return this.toTXT().trim().length <= 0;
// }
/**
* @returns {Boolean}
*/
search(query) {
search(query: string) {
const tokens = query.toLowerCase().split(splitter);
const lowercase = this.toTXT().toLowerCase();
return tokens.some((token) => lowercase.indexOf(token) > -1);
}
async insertMedia(resolve) {
let hashes = [];
async insertMedia(resolve: ResolveHashes) {
const hashes: string[] = [];
new HTMLParser({
ontag: (name, attr) => {
const hash = attr[ATTRIBUTES.hash];
@@ -127,7 +133,7 @@ export class Tiptap {
* @param {string[]} hashes
* @returns
*/
removeAttachments(hashes) {
removeAttachments(hashes: string[]) {
return new HTMLRewriter({
ontag: (_name, attr) => {
if (hashes.includes(attr[ATTRIBUTES.hash])) return false;
@@ -135,17 +141,28 @@ export class Tiptap {
}).transform(this.data);
}
async extractAttachments(store) {
async extractAttachments(
store: (
data: string,
mime: string,
filename?: string
) => Promise<string | undefined>
) {
if (
!this.data.includes(ATTRIBUTES.src) &&
!this.data.includes(ATTRIBUTES.hash)
)
return {
data: this.data,
attachments: []
hashes: []
};
let sources = [];
const sources: {
src: string;
filename?: string;
mime?: string;
id: string;
}[] = [];
new HTMLParser({
ontag: (name, attr, pos) => {
const hash = attr[ATTRIBUTES.hash];
@@ -153,28 +170,30 @@ export class Tiptap {
if (name === "img" && !hash && src) {
sources.push({
src,
filename: attr[ATTRIBUTES.filename],
mime: attr[ATTRIBUTES.mime],
id: `${pos.start}${pos.end}`
});
}
}
}).parse(this.data);
const images = {};
const images: Record<string, string | false> = {};
for (const image of sources) {
try {
const { data, mime } = dataurl.toObject(image.src);
if (!data) continue;
const storeResult = await store(data, mime);
if (!storeResult) continue;
const hash = await store(data, mime, image.filename);
if (!hash) continue;
images[image.id] = { ...storeResult, mime };
images[image.id] = hash;
} catch (e) {
console.error(e);
images[image.id] = false;
}
}
let attachments = [];
const hashes: string[] = [];
const html = new HTMLRewriter({
ontag: (name, attr, pos) => {
switch (name) {
@@ -182,28 +201,15 @@ export class Tiptap {
const hash = attr[ATTRIBUTES.hash];
if (hash) {
attachments.push({
hash
});
hashes.push(hash);
delete attr[ATTRIBUTES.src];
} else {
const imageData = images[`${pos.start}${pos.end}`];
if (!imageData) return imageData;
const hash = images[`${pos.start}${pos.end}`];
if (!hash) return;
const { key, metadata, mime } = imageData;
if (!metadata.hash) return;
hashes.push(hash);
const type = attr[ATTRIBUTES.mime] || mime || "image/jpeg";
const filename = attr[ATTRIBUTES.filename] || metadata.hash;
attachments.push({
type,
filename,
...metadata,
key
});
attr[ATTRIBUTES.hash] = metadata.hash;
attr[ATTRIBUTES.hash] = hash;
delete attr[ATTRIBUTES.src];
}
break;
@@ -212,9 +218,7 @@ export class Tiptap {
case "span": {
const hash = attr[ATTRIBUTES.hash];
if (!hash) return;
attachments.push({
hash
});
hashes.push(hash);
break;
}
}
@@ -223,19 +227,18 @@ export class Tiptap {
return {
data: html,
attachments
hashes
};
}
}
/**
* @param { import("html-to-text").DomNode } elem List items with their prefixes.
* @param { import("html-to-text").RecursiveCallback } walk Recursive callback to process child nodes.
* @param { import("html-to-text/lib/block-text-builder").BlockTextBuilder } builder Passed around to accumulate output text.
* @param { import("html-to-text").FormatOptions } formatOptions Options specific to a formatter.
* @param { (elem: import("html-to-text").DomNode) => string } nextPrefixCallback Function that returns increasing index each time it is called.
*/
function formatList(elem, walk, builder, formatOptions, nextPrefixCallback) {
function formatList(
elem: DomNode,
walk: RecursiveCallback,
builder: BlockTextBuilder,
formatOptions: FormatOptions,
nextPrefixCallback: (elem: DomNode) => string
) {
const isNestedList = elem?.parent?.name === "li";
// With Roman numbers, index length is not as straightforward as with Arabic numbers or letters,
@@ -243,7 +246,10 @@ function formatList(elem, walk, builder, formatOptions, nextPrefixCallback) {
let maxPrefixLength = 0;
const listItems = (elem.children || [])
// it might be more accurate to check only for html spaces here, but no significant benefit
.filter((child) => child.type !== "text" || !/^\s*$/.test(child.data))
.filter(
(child) =>
child.type !== "text" || (child.data && !/^\s*$/.test(child.data))
)
.map(function (child) {
if (child.name !== "li") {
return { node: child, prefix: "" };

345
packages/core/src/database/backup.js
View File

@@ -1,345 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import SparkMD5 from "spark-md5";
import { CURRENT_DATABASE_VERSION } from "../common.js";
import Migrator from "./migrator.js";
import { toChunks } from "../utils/array.js";
import { migrateItem } from "../migrations.js";
import Indexer from "./indexer.js";
import setManipulator from "../utils/set.js";
import { logger } from "../logger.js";
const COLORS = [
"red",
"orange",
"yellow",
"green",
"blue",
"purple",
"gray",
"black",
"white"
];
const invalidKeys = [
"user",
"t",
"v",
"lastBackupTime",
"lastSynced",
// all indexes
"notes",
"notebooks",
"content",
"tags",
"colors",
"attachments",
"relations",
"reminders",
"sessioncontent",
"notehistory",
"shortcuts",
"vaultKey",
"hasConflict",
"token",
"monographs"
];
const itemTypeToCollectionKey = {
note: "notes",
notebook: "notebooks",
tiptap: "content",
tiny: "content",
tag: "tags",
color: "colors",
attachment: "attachments",
relation: "relations",
reminder: "reminders",
sessioncontent: "sessioncontent",
session: "notehistory",
notehistory: "notehistory",
content: "content",
shortcut: "shortcuts"
};
const validTypes = ["mobile", "web", "node"];
export default class Backup {
/**
*
* @param {import("../api/index.js").default} db
*/
constructor(db) {
this._db = db;
this._migrator = new Migrator();
this.logger = logger.scope("Backup");
}
lastBackupTime() {
return this._db.storage.read("lastBackupTime");
}
async updateBackupTime() {
await this._db.storage.write("lastBackupTime", Date.now());
}
/**
*
* @param {"web"|"mobile"|"node"} type
* @param {boolean} encrypt
*/
async *export(type, encrypt = false) {
if (!validTypes.some((t) => t === type))
throw new Error("Invalid type. It must be one of 'mobile' or 'web'.");
if (encrypt && !(await this._db.user.getUser()))
throw new Error("Please login to create encrypted backups.");
yield {
path: ".nnbackup",
data: ""
};
let keys = await this._db.storage.getAllKeys();
const key = await this._db.user.getEncryptionKey();
const chunks = toChunks(keys, 20);
let buffer = [];
let bufferLength = 0;
const MAX_CHUNK_SIZE = 10 * 1024 * 1024;
let chunkIndex = 0;
while (chunks.length > 0) {
const chunk = chunks.pop();
const items = await this._db.storage.readMulti(chunk);
items.forEach(([id, item]) => {
if (
!item ||
invalidKeys.includes(id) ||
(item.deleted && !item.type) ||
id.startsWith("_uk_")
)
return;
const data = JSON.stringify(item);
buffer.push(data);
bufferLength += data.length;
});
if (bufferLength >= MAX_CHUNK_SIZE || chunks.length === 0) {
let itemsJSON = `[${buffer.join(",")}]`;
buffer = [];
bufferLength = 0;
itemsJSON = await this._db.compressor.compress(itemsJSON);
const hash = SparkMD5.hash(itemsJSON);
if (encrypt) itemsJSON = await this._db.storage.encrypt(key, itemsJSON);
yield {
path: `${chunkIndex++}-${encrypt ? "encrypted" : "plain"}-${hash}`,
data: `{
"version": ${CURRENT_DATABASE_VERSION},
"type": "${type}",
"date": ${Date.now()},
"data": ${JSON.stringify(itemsJSON)},
"hash": "${hash}",
"hash_type": "md5",
"compressed": true,
"encrypted": ${encrypt ? "true" : "false"}
}`
};
}
}
if (bufferLength > 0 || buffer.length > 0)
throw new Error("Buffer not empty.");
await this.updateBackupTime();
}
/**
*
* @param {any} backup the backup data
* @param {string} [password]
* @param {string} [key]
*/
async import(backup, password, key) {
if (!backup) return;
if (!this._validate(backup)) throw new Error("Invalid backup.");
backup = this._migrateBackup(backup);
let db = backup.data;
const isEncrypted = db.salt && db.iv && db.cipher;
if (backup.encrypted || isEncrypted) {
if (!password && !key)
throw new Error(
"Please provide a password or an encryption key to decrypt this backup & restore it."
);
key = key
? { key, salt: db.salt }
: await this._db.storage.generateCryptoKey(password, db.salt);
if (!key)
throw new Error("Could not generate encryption key for backup.");
try {
backup.data = await this._db.storage.decrypt(key, db);
} catch (e) {
if (
e.message.includes("ciphertext cannot be decrypted") ||
e.message === "FAILURE"
)
throw new Error("Incorrect password.");
throw new Error(`Could not decrypt backup: ${e.message}`);
}
}
if (backup.hash && !this._verify(backup))
throw new Error("Backup file has been tempered, aborting...");
if (backup.compressed)
backup.data = await this._db.compressor.decompress(backup.data);
backup.data =
typeof backup.data === "string" ? JSON.parse(backup.data) : backup.data;
await this._migrateData(backup);
}
_migrateBackup(backup) {
const { version = 0 } = backup;
if (version > CURRENT_DATABASE_VERSION)
throw new Error(
"This backup was made from a newer version of Notesnook. Cannot restore."
);
switch (version) {
case CURRENT_DATABASE_VERSION:
case 5.8:
case 5.7:
case 5.6:
case 5.5:
case 5.4:
case 5.3:
case 5.2:
case 5.1:
case 5.0: {
return backup;
}
default:
throw new Error("Unknown backup version.");
}
}
async _migrateData(backup) {
const { data, version = 0 } = backup;
const toAdd = {};
for (let item of Array.isArray(data) ? data : Object.values(data)) {
// we do not want to restore deleted items
if (!item || (!item.type && item.deleted) || typeof item !== "object")
continue;
// in v5.6 of the database, we did not set note history session's type
if (!item.type && item.sessionContentId) item.type = "notehistory";
await migrateItem(item, version, item.type, this._db, "backup");
// since items in trash can have their own set of migrations,
// we have to run the migration again to account for that.
if (item.type === "trash" && item.itemType)
await migrateItem(item, version, item.itemType, this._db, "backup");
// colors are naively of type "tag" instead of "color" so we have to fix that.
const itemType =
item.type === "tag" && COLORS.includes(item.title.toLowerCase())
? "color"
: item.itemType || item.type;
if (itemType === "attachment" && item.metadata && item.metadata.hash) {
const attachment = this._db.attachments.attachment(item.metadata.hash);
if (attachment) {
const isNewGeneric =
item.metadata.type === "application/octet-stream";
const isOldGeneric =
attachment.metadata.type === "application/octet-stream";
item = {
...attachment,
metadata: {
...attachment.metadata,
type:
// we keep whichever mime type is more specific
isNewGeneric && !isOldGeneric
? attachment.metadata.type
: item.metadata.type,
filename:
// we keep the filename based on which item's mime type we kept
isNewGeneric && !isOldGeneric
? attachment.metadata.filename
: item.metadata.filename
},
noteIds: setManipulator.union(attachment.noteIds, item.noteIds)
};
} else {
item.dateUploaded = undefined;
item.failed = undefined;
}
}
// items should sync immediately after getting restored
item.dateModified = Date.now();
item.synced = false;
const collectionKey = itemTypeToCollectionKey[itemType];
if (collectionKey) {
toAdd[collectionKey] = toAdd[collectionKey] || [];
toAdd[collectionKey].push([item.id, item]);
} else if (item.type === "settings")
await this._db.storage.write("settings", item);
}
for (const collectionKey in toAdd) {
const indexer = new Indexer(this._db.storage, collectionKey);
await indexer.init();
await indexer.writeMulti(toAdd[collectionKey]);
}
}
_validate(backup) {
return (
!!backup.date &&
!!backup.data &&
!!backup.type &&
validTypes.some((t) => t === backup.type)
);
}
_verify(backup) {
const { compressed, hash, hash_type, data: db } = backup;
switch (hash_type) {
case "md5": {
return hash === SparkMD5.hash(compressed ? db : JSON.stringify(db));
}
default: {
return false;
}
}
}
}

444
packages/core/src/database/backup.ts Normal file
View File

@@ -0,0 +1,444 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import SparkMD5 from "spark-md5";
import { CURRENT_DATABASE_VERSION } from "../common.js";
import Migrator from "./migrator.js";
import Database from "../api/index.js";
import {
Item,
MaybeDeletedItem,
Note,
Notebook,
ValueOf,
isDeleted
} from "../types.js";
import { Cipher } from "@notesnook/crypto";
import { isCipher } from "./crypto.js";
import { toChunks } from "../utils/array";
import { migrateItem } from "../migrations";
import Indexer from "./indexer";
import { set } from "../utils/set.js";
type BackupDataItem = MaybeDeletedItem<Item> | string[];
type BackupPlatform = "web" | "mobile" | "node";
type BaseBackupFile = {
version: number;
type: BackupPlatform;
date: number;
// encrypted?: boolean;
// compressed?: boolean;
};
type LegacyUnencryptedBackupFile = BaseBackupFile & {
data: Record<string, BackupDataItem> | string;
hash: string;
hash_type: "md5";
};
type LegacyEncryptedBackupFile = BaseBackupFile & {
data: Cipher<"base64">;
};
type UnencryptedBackupFile = BaseBackupFile & {
data: string;
hash: string;
hash_type: "md5";
compressed: true;
encrypted: false;
};
type EncryptedBackupFile = BaseBackupFile & {
data: Cipher<"base64">;
hash: string;
hash_type: "md5";
compressed: true;
encrypted: true;
};
type BackupFile = UnencryptedBackupFile | EncryptedBackupFile;
type LegacyBackupFile = LegacyUnencryptedBackupFile | LegacyEncryptedBackupFile;
function isEncryptedBackup(
backup: LegacyBackupFile | BackupFile
): backup is EncryptedBackupFile | LegacyEncryptedBackupFile {
return "encrypted" in backup ? backup.encrypted : isCipher(backup.data);
}
function isLegacyBackupFile(
backup: LegacyBackupFile | BackupFile
): backup is LegacyBackupFile {
return backup.version <= 5.8;
}
const COLORS = [
"red",
"orange",
"yellow",
"green",
"blue",
"purple",
"gray",
"black",
"white"
];
const invalidKeys = [
"user",
"t",
"v",
"lastBackupTime",
"lastSynced",
// all indexes
"notes",
"notebooks",
"content",
"tags",
"colors",
"attachments",
"relations",
"reminders",
"sessioncontent",
"notehistory",
"shortcuts",
"vaultKey",
"hasConflict",
"token",
"monographs"
];
const itemTypeToCollectionKey = {
note: "notes",
notebook: "notebooks",
tiptap: "content",
tiny: "content",
tag: "tags",
color: "colors",
attachment: "attachments",
relation: "relations",
reminder: "reminders",
sessioncontent: "sessioncontent",
session: "notehistory",
notehistory: "notehistory",
content: "content",
shortcut: "shortcuts",
// to make ts happy
topic: "topics"
} as const;
const validTypes = ["mobile", "web", "node"];
export default class Backup {
migrator = new Migrator();
constructor(private readonly db: Database) {}
lastBackupTime() {
return this.db.storage().read("lastBackupTime");
}
async updateBackupTime() {
await this.db.storage().write("lastBackupTime", Date.now());
}
async *export(type: BackupPlatform, encrypt = false) {
if (!validTypes.some((t) => t === type))
throw new Error("Invalid type. It must be one of 'mobile' or 'web'.");
if (encrypt && !(await this.db.user.getUser()))
throw new Error("Please login to create encrypted backups.");
const key = await this.db.user.getEncryptionKey();
if (encrypt && !key) throw new Error("No encryption key found.");
yield {
path: ".nnbackup",
data: ""
};
const keys = await this.db.storage().getAllKeys();
const chunks = toChunks(keys, 20);
let buffer: string[] = [];
let bufferLength = 0;
const MAX_CHUNK_SIZE = 10 * 1024 * 1024;
let chunkIndex = 0;
while (chunks.length > 0) {
const chunk = chunks.pop();
const items = await this.db.storage().readMulti(chunk);
items.forEach(([id, item]) => {
const isDeleted =
item &&
typeof item === "object" &&
"deleted" in item &&
!("type" in item);
if (
!item ||
invalidKeys.includes(id) ||
isDeleted ||
id.startsWith("_uk_")
)
return;
const data = JSON.stringify(item);
buffer.push(data);
bufferLength += data.length;
});
if (bufferLength >= MAX_CHUNK_SIZE || chunks.length === 0) {
let itemsJSON = `[${buffer.join(",")}]`;
buffer = [];
bufferLength = 0;
itemsJSON = await this.db.compressor().compress(itemsJSON);
const hash = SparkMD5.hash(itemsJSON);
if (encrypt && key)
itemsJSON = JSON.stringify(
await this.db.storage().encrypt(key, itemsJSON)
);
else itemsJSON = JSON.stringify(itemsJSON);
yield {
path: `${chunkIndex++}-${encrypt ? "encrypted" : "plain"}-${hash}`,
data: `{
"version": ${CURRENT_DATABASE_VERSION},
"type": "${type}",
"date": ${Date.now()},
"data": ${itemsJSON},
"hash": "${hash}",
"hash_type": "md5",
"compressed": true,
"encrypted": ${encrypt ? "true" : "false"}
}`
};
}
}
if (bufferLength > 0 || buffer.length > 0)
throw new Error("Buffer not empty.");
await this.updateBackupTime();
}
async import(
backup: LegacyBackupFile | BackupFile,
password?: string,
encryptionKey?: string
) {
if (!this.validate(backup)) throw new Error("Invalid backup.");
backup = this.migrateBackup(backup);
let decryptedData: string | Record<string, BackupDataItem> | undefined =
undefined;
if (isEncryptedBackup(backup)) {
if (!password && !encryptionKey)
throw new Error(
"Please provide a password to decrypt this backup & restore it."
);
const key = encryptionKey
? { key: encryptionKey, salt: backup.data.salt }
: password
? await this.db.storage().generateCryptoKey(password, backup.data.salt)
: undefined;
if (!key)
throw new Error("Could not generate encryption key for backup.");
try {
decryptedData = await this.db.storage().decrypt(key, backup.data);
} catch (e) {
console.error(e);
if (e instanceof Error) {
if (
e.message.includes("ciphertext cannot be decrypted") ||
e.message === "FAILURE"
)
throw new Error("Incorrect password.");
throw new Error(`Could not decrypt backup: ${e.message}`);
}
}
} else {
decryptedData = backup.data;
}
if (!decryptedData) return;
if ("hash" in backup && !this.verify(backup))
throw new Error("Backup file has been tempered, aborting...");
if ("compressed" in backup && typeof decryptedData === "string")
decryptedData = await this.db.compressor().decompress(decryptedData);
await this.migrateData(
typeof decryptedData === "string"
? (JSON.parse(decryptedData) as BackupDataItem[])
: Object.values(decryptedData),
backup.version
);
}
private migrateBackup(backup: BackupFile | LegacyBackupFile) {
const { version = 0 } = backup;
if (version > CURRENT_DATABASE_VERSION)
throw new Error(
"This backup was made from a newer version of Notesnook. Cannot migrate."
);
switch (version) {
case CURRENT_DATABASE_VERSION:
case 5.9:
case 5.8:
case 5.7:
case 5.6:
case 5.5:
case 5.4:
case 5.3:
case 5.2:
case 5.1:
case 5.0: {
return backup;
}
default:
throw new Error("Unknown backup version.");
}
}
private async migrateData(data: BackupDataItem[], version: number) {
const toAdd: Partial<
Record<
ValueOf<typeof itemTypeToCollectionKey>,
[string, MaybeDeletedItem<Item>][]
>
> = {};
for (let item of data) {
// we do not want to restore deleted items
if (
!item ||
typeof item !== "object" ||
Array.isArray(item) ||
isDeleted(item)
)
continue;
// in v5.6 of the database, we did not set note history session's type
if ("sessionContentId" in item && item.type !== "session")
(item as any).type = "notehistory";
// colors are naively of type "tag" instead of "color" so we have to fix that.
if (item.type === "tag" && COLORS.includes(item.title.toLowerCase()))
(item as any).type = "color";
await migrateItem(item, version, item.type, this.db, "backup");
// since items in trash can have their own set of migrations,
// we have to run the migration again to account for that.
if (item.type === "trash" && item.itemType)
await migrateItem(
item as unknown as Note | Notebook,
version,
item.itemType,
this.db,
"backup"
);
if (item.type === "attachment" && item.metadata && item.metadata.hash) {
const attachment = this.db.attachments.attachment(item.metadata.hash);
if (attachment) {
const isNewGeneric =
item.metadata.type === "application/octet-stream";
const isOldGeneric =
attachment.metadata.type === "application/octet-stream";
item = {
...attachment,
metadata: {
...attachment.metadata,
type:
// we keep whichever mime type is more specific
isNewGeneric && !isOldGeneric
? attachment.metadata.type
: item.metadata.type,
filename:
// we keep the filename based on which item's mime type we kept
isNewGeneric && !isOldGeneric
? attachment.metadata.filename
: item.metadata.filename
},
noteIds: set.union(attachment.noteIds, item.noteIds)
};
} else {
item.dateUploaded = undefined;
item.failed = undefined;
}
}
// items should sync immediately after getting restored
item.dateModified = Date.now();
item.synced = false;
if (item.type === "settings")
await this.db.storage().write("settings", item);
else {
const itemType = "itemType" in item ? item.itemType : item.type;
const collectionKey = itemTypeToCollectionKey[itemType];
if (collectionKey) {
toAdd[collectionKey] = toAdd[collectionKey] || [];
toAdd[collectionKey]?.push([item.id, item]);
}
}
}
for (const collectionKey in toAdd) {
const items =
toAdd[collectionKey as ValueOf<typeof itemTypeToCollectionKey>];
if (!items) continue;
const indexer = new Indexer(this.db.storage, collectionKey);
await indexer.init();
await indexer.writeMulti(items);
}
}
private validate(backup: LegacyBackupFile | BackupFile) {
return (
!!backup.date &&
!!backup.data &&
!!backup.type &&
validTypes.some((t) => t === backup.type)
);
}
private verify(backup: BackupFile | LegacyUnencryptedBackupFile) {
const { hash, hash_type, data } = backup;
switch (hash_type) {
case "md5": {
return (
hash ===
SparkMD5.hash(
"compressed" in backup && backup.compressed
? data
: JSON.stringify(data)
)
);
}
default: {
return false;
}
}
}
}

127
packages/core/src/database/cached-collection.js
View File

@@ -1,127 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import IndexedCollection from "./indexed-collection";
import MapStub from "../utils/map";
import { toChunks } from "../utils/array";
export default class CachedCollection extends IndexedCollection {
constructor(context, type, eventManager) {
super(context, type, eventManager);
this.type = type;
this.map = new Map();
this.items = undefined;
// this.eventManager = eventManager;
// this.encryptionKeyFactory = encryptionKeyFactory;
}
async init() {
await super.init();
let data = await this.indexer.readMulti(this.indexer.indices);
if (this.map && this.map.dispose) this.map.dispose();
// const encryptionKey =
// this.encryptionKeyFactory && (await this.encryptionKeyFactory());
// if (encryptionKey) {
// for (let item of data) {
// const [_key, value] = item;
// const decryptedValue = JSON.parse(
// await this.indexer.decrypt(encryptionKey, value)
// );
// item[1] = decryptedValue;
// }
// }
this.map = new MapStub.Map(data, this.type);
}
async clear() {
await super.clear();
this.map.clear();
this.invalidateCache();
}
async updateItem(item) {
await super.updateItem(item);
this.map.set(item.id, item);
this.invalidateCache();
}
exists(id) {
const item = this.getItem(id);
return item && !item.deleted;
}
has(id) {
return this.map.has(id);
}
count() {
return this.map.size;
}
getItem(id) {
return this.map.get(id);
}
async deleteItem(id) {
this.map.delete(id);
await super.deleteItem(id);
this.invalidateCache();
}
getRaw() {
return Array.from(this.map.values());
}
getItems(map = undefined) {
if (this.items && this.items.length === this.map.size) return this.items;
this.items = [];
this.map.forEach((value) => {
if (!value || value.deleted || !value.id) return;
value = map ? map(value) : value;
this.items.push(value);
});
this.items.sort((a, b) => b.dateCreated - a.dateCreated);
return this.items;
}
async setItems(items) {
await super.setItems(items);
for (let item of items) {
if (item) {
this.map.set(item.id, item);
}
}
this.invalidateCache();
}
*iterateSync(chunkSize) {
const chunks = toChunks(Array.from(this.map.values()), chunkSize);
for (const chunk of chunks) {
yield chunk;
}
}
invalidateCache() {
this.items = undefined;
}
}

152
packages/core/src/database/cached-collection.ts Normal file
View File

@@ -0,0 +1,152 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { IndexedCollection } from "./indexed-collection";
import MapStub from "../utils/map";
import {
BaseItem,
CollectionType,
Collections,
MaybeDeletedItem,
isDeleted
} from "../types";
import { StorageAccessor } from "../interfaces";
import EventManager from "../utils/event-manager";
import { toChunks } from "../utils/array";
export class CachedCollection<
TCollectionType extends CollectionType,
T extends BaseItem<Collections[TCollectionType]>
> {
private collection: IndexedCollection<TCollectionType, T>;
private cache = new Map<string, MaybeDeletedItem<T>>();
private cachedItems?: T[];
constructor(
storage: StorageAccessor,
type: TCollectionType,
eventManager: EventManager
) {
this.collection = new IndexedCollection(storage, type, eventManager);
}
async init() {
await this.collection.init();
const data = await this.collection.indexer.readMulti(
this.collection.indexer.indices
);
if ("dispose" in this.cache && typeof this.cache.dispose === "function")
this.cache.dispose();
this.cache = new MapStub.Map(data);
}
async add(item: MaybeDeletedItem<T>) {
await this.collection.addItem(item);
this.cache.set(item.id, item);
this.invalidateCache();
}
async clear() {
await this.collection.clear();
this.cache.clear();
this.invalidateCache();
}
async update(item: T) {
await this.collection.updateItem(item);
this.cache.set(item.id, item);
this.invalidateCache();
}
async delete(id: string) {
this.cache.delete(id);
await this.collection.deleteItem(id);
this.invalidateCache();
}
async remove(id: string) {
this.cache.set(id, {
id,
deleted: true,
dateModified: Date.now()
});
await this.collection.removeItem(id);
this.invalidateCache();
}
exists(id: string) {
const item = this.cache.get(id);
return this.collection.exists(id) && !!item && !isDeleted(item);
}
has(id: string) {
return this.cache.has(id);
}
count() {
return this.cache.size;
}
get(id: string) {
const item = this.cache.get(id);
if (!item || isDeleted(item)) return;
return item;
}
raw() {
return Array.from(this.cache.values());
}
items(map?: (item: T) => T | undefined) {
if (this.cachedItems && this.cachedItems.length === this.cache.size)
return this.cachedItems;
this.cachedItems = [];
this.cache.forEach((value) => {
if (isDeleted(value)) return;
const mapped = map ? map(value) : value;
if (!mapped) return;
this.cachedItems?.push(mapped);
});
this.cachedItems.sort((a, b) => b.dateCreated - a.dateCreated);
return this.cachedItems;
}
async setItems(items: (MaybeDeletedItem<T> | undefined)[]) {
await this.collection.setItems(items);
for (const item of items) {
if (item) {
this.cache.set(item.id, item);
}
}
this.invalidateCache();
}
*iterateSync(chunkSize: number) {
const chunks = toChunks(Array.from(this.cache.values()), chunkSize);
for (const chunk of chunks) {
yield chunk;
}
}
invalidateCache() {
this.cachedItems = undefined;
}
}

41
packages/core/src/database/crypto.ts Normal file
View File

@@ -0,0 +1,41 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Cipher } from "@notesnook/crypto";
import { StorageAccessor } from "../interfaces";
import { randomBytes } from "../utils/random";
export type CryptoAccessor = () => Crypto;
export class Crypto {
constructor(private readonly storage: StorageAccessor) {}
async generateRandomKey() {
const passwordBytes = randomBytes(124);
const password = passwordBytes.toString("base64");
return await this.storage().generateCryptoKey(password);
}
}
export function isCipher(item: any): item is Cipher {
return (
typeof item === "object" &&
"cipher" in item &&
"iv" in item &&
"salt" in item
);
}

96
packages/core/src/database/fs.js → packages/core/src/database/fs.ts
View File

@@ -19,22 +19,44 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import hosts from "../utils/constants";
import TokenManager from "../api/token-manager";
import {
FileEncryptionMetadataWithOutputType,
IFileStorage,
StorageAccessor
} from "../interfaces";
import { DataFormat, SerializedKey } from "@notesnook/crypto/dist/src/types";
import { AttachmentMetadata } from "../types";
import { EV, EVENTS } from "../common";
export default class FileStorage {
constructor(fs, storage) {
this.fs = fs;
export type FileStorageAccessor = () => FileStorage;
export type DownloadableFile = {
filename: string;
metadata: AttachmentMetadata;
chunkSize: number;
};
export type QueueItem = DownloadableFile & {
cancel?: (reason?: string) => Promise<void>;
};
export class FileStorage {
private readonly tokenManager: TokenManager;
downloads = new Map<string, QueueItem[]>();
uploads = new Map<string, QueueItem[]>();
constructor(private readonly fs: IFileStorage, storage: StorageAccessor) {
this.tokenManager = new TokenManager(storage);
this.downloads = new Map();
this.uploads = new Map();
}
async queueDownloads(files, groupId, eventData) {
async queueDownloads(
files: DownloadableFile[],
groupId: string,
eventData?: Record<string, unknown>
) {
const token = await this.tokenManager.getAccessToken();
const total = files.length;
let current = 0;
this.downloads.set(groupId, files);
for (const file of files) {
for (const file of files as QueueItem[]) {
const { filename, metadata, chunkSize } = file;
if (await this.exists(filename)) {
current++;
@@ -77,16 +99,18 @@ export default class FileStorage {
this.downloads.delete(groupId);
}
async queueUploads(files, groupId) {
async queueUploads(files: DownloadableFile[], groupId: string) {
const token = await this.tokenManager.getAccessToken();
const total = files.length;
let current = 0;
this.uploads.set(groupId, files);
for (const file of files) {
const { filename } = file;
for (const file of files as QueueItem[]) {
const { filename, chunkSize, metadata } = file;
const url = `${hosts.API_HOST}/s3?name=${filename}`;
const { execute, cancel } = this.fs.uploadFile(filename, {
chunkSize,
metadata,
url,
headers: { Authorization: `Bearer ${token}` }
});
@@ -118,7 +142,12 @@ export default class FileStorage {
this.uploads.delete(groupId);
}
async downloadFile(groupId, filename, chunkSize, metadata) {
async downloadFile(
groupId: string,
filename: string,
chunkSize: number,
metadata: AttachmentMetadata
) {
const url = `${hosts.API_HOST}/s3?name=${filename}`;
const token = await this.tokenManager.getAccessToken();
const { execute, cancel } = this.fs.downloadFile(filename, {
@@ -127,19 +156,20 @@ export default class FileStorage {
chunkSize,
headers: { Authorization: `Bearer ${token}` }
});
this.downloads.set(groupId, [{ cancel }]);
this.downloads.set(groupId, [{ cancel, filename, chunkSize, metadata }]);
const result = await execute();
this.downloads.delete(groupId);
return result;
}
async cancel(groupId) {
async cancel(groupId: string) {
const queues = [
{ type: "download", files: this.downloads.get(groupId) },
{ type: "upload", files: this.uploads.get(groupId) }
].filter((a) => !!a.files);
for (const queue of queues) {
if (!queue.files) continue;
for (let i = 0; i < queue.files.length; ++i) {
const file = queue.files[i];
if (file.cancel) await file.cancel("Operation canceled.");
@@ -156,51 +186,43 @@ export default class FileStorage {
}
}
readEncrypted(filename, encryptionKey, cipherData) {
readEncrypted<TOutputFormat extends DataFormat>(
filename: string,
encryptionKey: SerializedKey,
cipherData: FileEncryptionMetadataWithOutputType<TOutputFormat>
) {
return this.fs.readEncrypted(filename, encryptionKey, cipherData);
}
writeEncryptedBase64(data, encryptionKey, mimeType) {
return this.fs.writeEncryptedBase64({
data,
key: encryptionKey,
mimeType
});
writeEncryptedBase64(
data: string,
encryptionKey: SerializedKey,
mimeType: string
) {
return this.fs.writeEncryptedBase64(data, encryptionKey, mimeType);
}
async deleteFile(filename, localOnly) {
async deleteFile(filename: string, localOnly = false) {
if (localOnly) return await this.fs.deleteFile(filename);
const token = await this.tokenManager.getAccessToken();
const url = `${hosts.API_HOST}/s3?name=${filename}`;
return await this.fs.deleteFile(filename, {
url,
headers: { Authorization: `Bearer ${token}` }
headers: { Authorization: `Bearer ${token}` },
chunkSize: 0
});
}
/**
*
* @param {string} filename
* @returns {Promise<boolean>}
*/
exists(filename) {
exists(filename: string) {
return this.fs.exists(filename);
}
/**
*
* @returns {Promise<void>}
*/
clear() {
return this.fs.clearFileStorage();
}
/**
* @param {string} data
* @returns {Promise<{hash: string, type: string}>}
*/
hashBase64(data) {
hashBase64(data: string) {
return this.fs.hashBase64(data);
}
}

75
packages/core/src/database/indexed-collection.js → packages/core/src/database/indexed-collection.ts
View File

@@ -19,13 +19,29 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { EVENTS } from "../common";
import { toChunks } from "../utils/array";
import { StorageAccessor } from "../interfaces";
import {
CollectionType,
Collections,
ItemMap,
MaybeDeletedItem,
isDeleted
} from "../types";
import EventManager from "../utils/event-manager";
import Indexer from "./indexer";
export default class IndexedCollection {
constructor(context, type, eventManager) {
this.indexer = new Indexer(context, type);
this.eventManager = eventManager;
// this.encryptionKeyFactory = encryptionKeyFactory;
export class IndexedCollection<
TCollectionType extends CollectionType = CollectionType,
T extends ItemMap[Collections[TCollectionType]] = ItemMap[Collections[TCollectionType]]
> {
readonly indexer: Indexer<T>;
constructor(
storage: StorageAccessor,
type: TCollectionType,
private readonly eventManager: EventManager
) {
this.indexer = new Indexer(storage, type);
}
clear() {
@@ -36,18 +52,19 @@ export default class IndexedCollection {
await this.indexer.init();
}
async addItem(item) {
async addItem(item: MaybeDeletedItem<T>) {
if (!item.id) throw new Error("The item must contain the id field.");
const exists = this.exists(item.id);
if (!exists) item.dateCreated = item.dateCreated || Date.now();
if (!exists && !isDeleted(item))
item.dateCreated = item.dateCreated || Date.now();
await this.updateItem(item);
if (!exists) {
await this.indexer.index(item.id);
}
}
async updateItem(item) {
async updateItem(item: MaybeDeletedItem<T>) {
if (!item.id) throw new Error("The item must contain the id field.");
this.eventManager.publish(EVENTS.databaseUpdated, item.id, item);
@@ -58,52 +75,35 @@ export default class IndexedCollection {
}
// the item has become local now, so remove the flags
delete item.remote;
// if (await this.getEncryptionKey()) {
// const encrypted = await this.indexer.encrypt(
// await this.getEncryptionKey(),
// JSON.stringify(item)
// );
// encrypted.dateModified = item.dateModified;
// encrypted.localOnly = item.localOnly;
// encrypted.id = item.id;
// await this.indexer.write(item.id, encrypted);
// } else
await this.indexer.write(item.id, item);
}
removeItem(id) {
removeItem(id: string) {
this.eventManager.publish(EVENTS.databaseUpdated, id);
return this.updateItem({
return this.indexer.write(id, {
id,
deleted: true
deleted: true,
dateModified: Date.now()
});
}
async deleteItem(id) {
async deleteItem(id: string) {
this.eventManager.publish(EVENTS.databaseUpdated, id);
await this.indexer.deindex(id);
return await this.indexer.remove(id);
}
exists(id) {
exists(id: string) {
return this.indexer.exists(id);
}
async getItem(id) {
async getItem(id: string) {
const item = await this.indexer.read(id);
if (!item) return;
// if ((await this.getEncryptionKey()) && item.iv && item.cipher) {
// return JSON.parse(
// await this.indexer.decrypt(await this.getEncryptionKey(), item)
// );
// } else
return item;
}
async getItems(indices) {
async getItems(indices: string[]) {
const data = await this.indexer.readMulti(indices);
return Object.fromEntries(data);
}
@@ -124,14 +124,7 @@ export default class IndexedCollection {
return this.indexer.writeMulti(entries);
}
async getEncryptionKey() {
if (!this.encryptionKeyFactory) return;
if (this.encryptionKey) return this.encryptionKey;
this.encryptionKey = await this.encryptionKeyFactory();
return this.encryptionKey;
}
async *iterate(chunkSize) {
async *iterate(chunkSize: number) {
const chunks = toChunks(this.indexer.indices, chunkSize);
for (const chunk of chunks) {
yield await this.indexer.readMulti(chunk);

66
packages/core/src/database/indexer.js → packages/core/src/database/indexer.ts
View File

@@ -17,58 +17,64 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Storage from "./storage";
import { StorageAccessor } from "../interfaces";
import { MaybeDeletedItem } from "../types";
export default class Indexer extends Storage {
constructor(storage, type) {
super(storage);
this.type = type;
this.indices = [];
}
export default class Indexer<T> {
private _indices: string[] = [];
constructor(
private readonly storage: StorageAccessor,
private readonly type: string
) {}
async init() {
this.indices = (await super.read(this.type, true)) || [];
this._indices = (await this.storage().read(this.type, true)) || [];
}
exists(key) {
exists(key: string) {
return this.indices.includes(key);
}
async index(key) {
async index(key: string) {
if (this.exists(key)) return;
this.indices.push(key);
await super.write(this.type, this.indices);
await this.storage().write(this.type, this.indices);
}
getIndices() {
return this.indices;
get indices() {
return this._indices;
}
async deindex(key) {
async deindex(key: string) {
if (!this.exists(key)) return;
this.indices.splice(this.indices.indexOf(key), 1);
await super.write(this.type, this.indices);
await this.storage().write(this.type, this.indices);
}
async clear() {
this.indices = [];
await super.clear();
this._indices = [];
await this.storage().clear();
}
read(key, isArray = false) {
return super.read(this.makeId(key), isArray);
async read(
key: string,
isArray = false
): Promise<MaybeDeletedItem<T> | undefined> {
return await this.storage().read(this.makeId(key), isArray);
}
write(key, data) {
return super.write(this.makeId(key), data);
write(key: string, data: MaybeDeletedItem<T>) {
return this.storage().write(this.makeId(key), data);
}
remove(key) {
return super.remove(this.makeId(key));
remove(key: string) {
return this.storage().remove(this.makeId(key));
}
async readMulti(keys) {
const entries = await super.readMulti(keys.map(this.makeId, this));
async readMulti(keys: string[]) {
const entries = await this.storage().readMulti<MaybeDeletedItem<T>>(
keys.map(this.makeId, this)
);
entries.forEach((entry) => {
entry[0] = entry[0].replace(`_${this.type}`, "");
});
@@ -90,11 +96,11 @@ export default class Indexer extends Storage {
}
async migrateIndices() {
const keys = (await super.getAllKeys()).filter(
const keys = (await this.storage().getAllKeys()).filter(
(key) => !key.endsWith(`_${this.type}`) && this.exists(key)
);
for (const id of keys) {
const item = await super.read(id);
const item = await this.storage().read<T>(id);
if (!item) continue;
await this.write(id, item);
@@ -102,11 +108,11 @@ export default class Indexer extends Storage {
// remove old ids once they have been moved
for (const id of keys) {
await super.remove(id);
await this.storage().remove(id);
}
}
makeId = (id) => {
private makeId(id: string) {
return `${id}_${this.type}`;
};
}
}

123
packages/core/src/database/migrator.js
View File

@@ -1,123 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { sendMigrationProgressEvent } from "../common";
import { migrateCollection, migrateItem } from "../migrations";
class Migrator {
async migrate(db, collections, get, version) {
for (let collection of collections) {
if (
(!collection.iterate && !collection.index) ||
!collection.dbCollection
)
continue;
if (collection.dbCollection.collectionName)
sendMigrationProgressEvent(
db.eventManager,
collection.dbCollection.collectionName,
0,
0
);
await migrateCollection(collection.dbCollection, version);
if (collection.index) {
await this.migrateItems(
db,
collection,
collection.index(),
get,
version
);
} else if (collection.iterate) {
for await (const index of collection.dbCollection._collection.iterate(
100
)) {
await this.migrateItems(
db,
collection,
index.map((item) => item[1]),
get,
version
);
}
}
}
return true;
}
async migrateItems(db, collection, index, get, version) {
const toAdd = [];
for (var i = 0; i < index.length; ++i) {
let id = index[i];
let item = get(id, collection.dbCollection.collectionName);
if (!item) {
continue;
}
// check if item is permanently deleted or just a soft delete
if (item.deleted && !item.type) {
await collection.dbCollection?._collection?.addItem(item);
continue;
}
const itemId = item.id;
const migrated = await migrateItem(
item,
version,
item.type || collection.type || collection.dbCollection.type,
db,
"local"
);
if (migrated) {
if (collection.type === "settings") {
await collection.dbCollection.merge(item);
} else if (item.type === "note") {
toAdd.push(await db.notes.merge(null, item));
} else if (collection.dbCollection._collection) {
toAdd.push(item);
} else {
throw new Error(
`No idea how to handle this kind of item: ${item.type}.`
);
}
// if id changed after migration, we need to delete the old one.
if (item.id !== itemId) {
await collection.dbCollection._collection.deleteItem(itemId);
}
}
}
if (toAdd.length > 0) {
await collection.dbCollection._collection.setItems(toAdd);
if (collection.dbCollection.collectionName)
sendMigrationProgressEvent(
db.eventManager,
collection.dbCollection.collectionName,
toAdd.length,
toAdd.length
);
}
}
}
export default Migrator;

133
packages/core/src/database/migrator.ts Normal file
View File

@@ -0,0 +1,133 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Database from "../api";
import { sendMigrationProgressEvent } from "../common";
import { migrateCollection, migrateItem } from "../migrations";
import {
CollectionType,
Collections,
Item,
MaybeDeletedItem,
isDeleted,
isTrashItem
} from "../types";
import { IndexedCollection } from "./indexed-collection";
export type RawItem = MaybeDeletedItem<Item>;
type MigratableCollection = {
iterate?: boolean;
items?: () => (RawItem | undefined)[];
type: CollectionType;
};
export type MigratableCollections = MigratableCollection[];
class Migrator {
async migrate(
db: Database,
collections: MigratableCollections,
version: number
) {
for (const collection of collections) {
sendMigrationProgressEvent(db.eventManager, collection.type, 0, 0);
const indexedCollection = new IndexedCollection(
db.storage,
collection.type,
db.eventManager
);
await migrateCollection(indexedCollection, version);
if (collection.items) {
await this.migrateItems(
db,
collection.type,
indexedCollection,
collection.items(),
version
);
} else if (collection.iterate) {
await indexedCollection.init();
for await (const entries of indexedCollection.iterate(100)) {
await this.migrateItems(
db,
collection.type,
indexedCollection,
entries.map((i) => i[1]),
version
);
}
}
}
await db.initCollections();
return true;
}
async migrateItems(
db: Database,
type: keyof Collections,
collection: IndexedCollection,
items: (RawItem | undefined)[],
version: number
) {
const toAdd = [];
for (let i = 0; i < items.length; ++i) {
const item = items[i];
if (!item) continue;
// check if item is permanently deleted or just a soft delete
if (isDeleted(item) && !isTrashItem(item)) {
toAdd.push(item);
continue;
}
const itemId = item.id;
const migrated = await migrateItem(
item,
version,
item.type || type,
db,
"local"
);
if (migrated) {
if (item.type === "settings") {
await db.settings.merge(item, Infinity);
} else toAdd.push(item);
// if id changed after migration, we need to delete the old one.
if (item.id !== itemId) {
await collection.deleteItem(itemId);
}
}
}
if (toAdd.length > 0) {
await collection.setItems(toAdd);
sendMigrationProgressEvent(
db.eventManager,
type,
toAdd.length,
toAdd.length
);
}
}
}
export default Migrator;

96
packages/core/src/database/storage.js
View File

@@ -1,96 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { randomBytes } from "../utils/random";
export default class Storage {
constructor(storage) {
this.storage = storage;
}
write(key, data) {
return this.storage.write(key, data);
}
readMulti(keys) {
return this.storage.readMulti(keys);
}
writeMulti(entries) {
return this.storage.writeMulti(entries);
}
read(key, isArray = false) {
return this.storage.read(key, isArray);
}
clear() {
return this.storage.clear();
}
remove(key) {
return this.storage.remove(key);
}
removeMulti(keys) {
return this.storage.removeMulti(keys);
}
getAllKeys() {
return this.storage.getAllKeys();
}
encrypt(password, data) {
return this.storage.encrypt(password, data);
}
encryptMulti(password, data) {
return this.storage.encryptMulti(password, data);
}
decrypt(password, cipher) {
return this.storage.decrypt(password, cipher);
}
decryptMulti(password, items) {
return this.storage.decryptMulti(password, items);
}
deriveCryptoKey(name, data) {
return this.storage.deriveCryptoKey(name, data);
}
hash(password, userId) {
return this.storage.hash(password, userId);
}
getCryptoKey(name) {
return this.storage.getCryptoKey(name);
}
generateCryptoKey(password, salt) {
return this.storage.generateCryptoKey(password, salt);
}
async generateRandomKey() {
const passwordBytes = randomBytes(124);
const password = passwordBytes.toString("base64");
return await this.storage.generateCryptoKey(password);
}
}

114
packages/core/src/interfaces.ts Normal file
View File

@@ -0,0 +1,114 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";
import { AttachmentMetadata } from "./types";
export type Output<TOutputFormat extends DataFormat> =
TOutputFormat extends Omit<DataFormat, "uint8array"> ? string : Uint8Array;
export type FileEncryptionMetadata = {
chunkSize: number;
iv: string;
length: number;
salt: string;
alg: string;
};
export type FileEncryptionMetadataWithOutputType<
TOutputFormat extends DataFormat
> = FileEncryptionMetadata & { outputType: TOutputFormat };
export type FileEncryptionMetadataWithHash = FileEncryptionMetadata & {
hash: string;
hashType: string;
};
export interface IStorage {
write<T>(key: string, data: T): Promise<void>;
writeMulti<T>(entries: [string, T][]): Promise<void>;
readMulti<T>(keys: string[]): Promise<[string, T][]>;
read<T>(key: string, isArray?: boolean): Promise<T | undefined>;
remove(key: string): Promise<void>;
clear(): Promise<void>;
getAllKeys(): Promise<string[]>;
encrypt(key: SerializedKey, plainText: string): Promise<Cipher<"base64">>;
encryptMulti(
key: SerializedKey,
items: string[]
): Promise<Cipher<"base64">[]>;
decrypt(key: SerializedKey, cipherData: Cipher<"base64">): Promise<string>;
decryptMulti(
key: SerializedKey,
items: Cipher<"base64">[]
): Promise<string[]>;
deriveCryptoKey(name: string, credentials: SerializedKey): Promise<void>;
hash(password: string, email: string): Promise<string>;
getCryptoKey(name: string): Promise<string | undefined>;
generateCryptoKey(password: string, salt?: string): Promise<SerializedKey>;
// async generateRandomKey() {
// const passwordBytes = randomBytes(124);
// const password = passwordBytes.toString("base64");
// return await this.storage.generateCryptoKey(password);
// }
}
export interface ICompressor {
compress(data: string): Promise<string>;
decompress(data: string): Promise<string>;
}
export type RequestOptions = {
url: string;
metadata?: AttachmentMetadata;
chunkSize: number;
headers: { Authorization: string };
};
type Cancellable<T> = {
execute(): Promise<T>;
cancel(reason?: string): Promise<void>;
};
export interface IFileStorage {
downloadFile(
filename: string,
requestOptions: RequestOptions
): Cancellable<boolean>;
uploadFile(
filename: string,
requestOptions: RequestOptions
): Cancellable<boolean>;
readEncrypted<TOutputFormat extends DataFormat>(
filename: string,
encryptionKey: SerializedKey,
cipherData: FileEncryptionMetadataWithOutputType<TOutputFormat>
): Promise<Output<TOutputFormat> | undefined>;
writeEncryptedBase64(
data: string,
encryptionKey: SerializedKey,
mimeType: string
): Promise<FileEncryptionMetadataWithHash>;
deleteFile(
filename: string,
requestOptions?: RequestOptions
): Promise<boolean>;
exists(filename: string): Promise<boolean>;
clearFileStorage(): Promise<void>;
hashBase64(data: string): Promise<{ hash: string; type: string }>;
}
export type StorageAccessor = () => IStorage;
export type CompressorAccessor = () => ICompressor;

10
packages/core/src/logger.js
View File

@@ -37,7 +37,7 @@ const WEEK = 86400000 * 7;
class DatabaseLogReporter {
/**
*
* @param {import("./database/storage").default} storage
* @param {import("./database/crypto").default} storage
*/
constructor(storage) {
this.writer = new DatabaseLogWriter(storage);
@@ -55,7 +55,7 @@ class DatabaseLogReporter {
class DatabaseLogWriter {
/**
*
* @param {import("./database/storage").default} storage
* @param {import("./database/crypto").default} storage
*/
constructor(storage) {
this.storage = storage;
@@ -102,7 +102,7 @@ class DatabaseLogWriter {
class DatabaseLogManager {
/**
*
* @param {import("./database/storage").default} storage
* @param {import("./database/crypto").default} storage
*/
constructor(storage) {
this.storage = storage;
@@ -148,7 +148,7 @@ class DatabaseLogManager {
}
}
function initalize(storage, disableConsoleLogs) {
function initialize(storage, disableConsoleLogs) {
if (storage) {
let reporters = [new DatabaseLogReporter(storage)];
if (process.env.NODE_ENV !== "production" && !disableConsoleLogs)
@@ -171,4 +171,4 @@ var logger = new NoopLogger();
*/
var logManager;
export { LogLevel, format, initalize, logManager, logger };
export { LogLevel, format, initialize, logManager, logger };

200
packages/core/src/migrations.js → packages/core/src/migrations.ts
View File

@@ -20,8 +20,51 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { parseHTML } from "./utils/html-parser";
import { decodeHTML5 } from "entities";
import { CURRENT_DATABASE_VERSION } from "./common";
import Database from "./api";
import { getId, makeId } from "./utils/id";
import {
Color,
ContentItem,
HistorySession,
Item,
ItemMap,
ItemType
} from "./types";
import { isCipher } from "./database/crypto";
import { IndexedCollection } from "./database/indexed-collection";
const migrations = [
const ColorToHexCode: Record<string, string> = {
red: "#f44336",
orange: "#FF9800",
yellow: "#FFD600",
green: "#4CAF50",
blue: "#2196F3",
purple: "#673AB7",
gray: "#9E9E9E",
black: "#000000",
white: "#ffffff"
};
type MigrationType = "local" | "sync" | "backup";
type MigrationItemType = ItemType | "notehistory" | "content" | "all";
type MigrationItemMap = ItemMap & {
notehistory: HistorySession;
content: ContentItem;
all: Item;
};
type Migration = {
version: number;
items: {
[P in MigrationItemType]?: (
item: MigrationItemMap[P],
db: Database,
migrationType: MigrationType
) => boolean | Promise<boolean> | void;
};
collection?: (collection: IndexedCollection) => Promise<void> | void;
};
const migrations: Migration[] = [
{ version: 5.0, items: {} },
{ version: 5.1, items: {} },
{
@@ -35,7 +78,7 @@ const migrations = [
tiny: (item) => {
replaceDateEditedWithDateModified(false)(item);
if (!item.data || item.data.iv) return true;
if (!item.data || isCipher(item.data)) return true;
item.data = removeToxClassFromChecklist(wrapTablesWithDiv(item.data));
return true;
@@ -47,7 +90,7 @@ const migrations = [
version: 5.3,
items: {
tiny: (item) => {
if (!item.data || item.data.iv) return false;
if (!item.data || isCipher(item.data)) return false;
item.data = decodeWrappedTableHtml(item.data);
return true;
}
@@ -57,7 +100,7 @@ const migrations = [
version: 5.4,
items: {
tiny: (item) => {
if (!item.data || item.data.iv) return false;
if (!item.data || isCipher(item.data)) return false;
item.type = "tiptap";
item.data = tinyToTiptap(item.data);
return true;
@@ -102,12 +145,12 @@ const migrations = [
version: 5.7,
items: {
tiny: (item) => {
if (!item.data || item.data.iv) return false;
if (!item.data || isCipher(item.data)) return false;
item.type = "tiptap";
return changeSessionContentType(item);
},
content: (item) => {
if (!item.data || item.data.iv) return false;
if (!item.data || isCipher(item.data)) return false;
const oldType = item.type;
item.type = "tiptap";
return oldType !== item.type;
@@ -127,10 +170,7 @@ const migrations = [
}
},
collection: async (collection) => {
if (collection._collection) {
const indexer = collection._collection.indexer;
await indexer.migrateIndices();
}
await collection.indexer.migrateIndices();
}
},
{
@@ -144,15 +184,86 @@ const migrations = [
}
}
},
{ version: 5.9, items: {} }
{
version: 5.9,
items: {
tag: async (item, db) => {
const alias = db.settings?.getAlias(item.id);
item.title = alias || item.title;
item.id = getId(item.dateCreated);
const colorCode = ColorToHexCode[item.title];
if (colorCode) {
(item as unknown as Color).type = "color";
(item as unknown as Color).colorCode = colorCode;
}
delete item.localOnly;
delete item.noteIds;
delete item.alias;
return true;
},
note: async (item, db) => {
for (const tag of item.tags || []) {
const oldTagId = makeId(tag);
const oldTag = db.tags.tag(oldTagId);
const alias = db.settings.getAlias(oldTagId);
const newTag = db.tags.all.find(
(t) => [alias, tag].includes(t.title) && t.id !== oldTagId
);
const newTagId =
newTag?.id ||
(await db.tags.add({
dateCreated: oldTag?.dateCreated,
dateModified: oldTag?.dateModified,
title: alias || tag,
type: "tag"
}));
if (!newTagId) continue;
await db.relations.add({ type: "tag", id: newTagId }, item);
await db.tags.delete(oldTagId);
}
if (item.color) {
const oldColorId = makeId(item.color);
const oldColor = db.tags.tag(oldColorId);
const alias = db.settings.getAlias(oldColorId);
const newColor = db.tags.all.find(
(t) => [alias, item.color].includes(t.title) && t.id !== oldColorId
);
const newColorId =
newColor?.id ||
(await db.colors.add({
dateCreated: oldColor?.dateCreated,
dateModified: oldColor?.dateModified,
title: alias || item.color,
colorCode: ColorToHexCode[item.color],
type: "color"
}));
if (newColorId) {
await db.relations.add({ type: "color", id: newColorId }, item);
await db.colors.delete(oldColorId);
}
}
delete item.tags;
delete item.color;
return true;
}
}
},
{
version: 6.0,
items: {}
}
];
export async function migrateItem(
item,
version,
type,
database,
migrationType
export async function migrateItem<TItemType extends MigrationItemType>(
item: MigrationItemMap[TItemType],
version: number,
type: TItemType,
database: Database,
migrationType: MigrationType
) {
let migrationStartIndex = migrations.findIndex((m) => m.version === version);
if (migrationStartIndex <= -1) {
@@ -182,7 +293,10 @@ export async function migrateItem(
return count > 0;
}
export async function migrateCollection(collection, version) {
export async function migrateCollection(
collection: IndexedCollection,
version: number
) {
let migrationStartIndex = migrations.findIndex((m) => m.version === version);
if (migrationStartIndex <= -1) {
throw new Error(
@@ -198,14 +312,11 @@ export async function migrateCollection(collection, version) {
if (!migration.collection) continue;
await migration.collection(collection);
if (collection._collection && collection._collection.init)
await collection._collection.init();
}
}
function replaceDateEditedWithDateModified(removeDateEditedProperty = false) {
return function (item) {
return function (item: any) {
item.dateModified = item.dateEdited;
if (removeDateEditedProperty) delete item.dateEdited;
delete item.persistDateEdited;
@@ -213,10 +324,10 @@ function replaceDateEditedWithDateModified(removeDateEditedProperty = false) {
};
}
function wrapTablesWithDiv(html) {
function wrapTablesWithDiv(html: string) {
const document = parseHTML(html);
const tables = document.getElementsByTagName("table");
for (let table of tables) {
for (const table of tables) {
table.setAttribute("contenteditable", "true");
const div = document.createElement("div");
div.setAttribute("contenteditable", "false");
@@ -224,27 +335,31 @@ function wrapTablesWithDiv(html) {
div.classList.add("table-container");
table.replaceWith(div);
}
return document.outerHTML || document.body.innerHTML;
return "outerHTML" in document
? (document.outerHTML as string)
: document.body.innerHTML;
}
function removeToxClassFromChecklist(html) {
function removeToxClassFromChecklist(html: string): string {
const document = parseHTML(html);
const checklists = document.querySelectorAll(
".tox-checklist,.tox-checklist--checked"
);
for (let item of checklists) {
for (const item of checklists) {
if (item.classList.contains("tox-checklist--checked"))
item.classList.replace("tox-checklist--checked", "checked");
else if (item.classList.contains("tox-checklist"))
item.classList.replace("tox-checklist", "checklist");
}
return document.outerHTML || document.body.innerHTML;
return "outerHTML" in document
? (document.outerHTML as string)
: document.body.innerHTML;
}
const regex = /&lt;div class="table-container".*&lt;\/table&gt;&lt;\/div&gt;/gm;
function decodeWrappedTableHtml(html) {
return html.replaceAll(regex, (match) => {
function decodeWrappedTableHtml(html: string) {
return html.replace(regex, (match) => {
const html = decodeHTML5(match);
return html;
});
@@ -254,20 +369,23 @@ const NEWLINE_REPLACEMENT_REGEX = /\n|<br>|<br\/>/gm;
const PREBLOCK_REGEX = /(<pre.*?>)(.*?)(<\/pre>)/gm;
const SPAN_REGEX = /<span class=.*?>(.*?)<\/span>/gm;
export function tinyToTiptap(html) {
export function tinyToTiptap(html: string) {
if (typeof html !== "string") return html;
// Preserve newlines in pre blocks
html = html
.replace(/\n/gm, "<br/>")
.replace(PREBLOCK_REGEX, (_pre, start, inner, end) => {
let codeblock = start;
codeblock += inner
.replace(NEWLINE_REPLACEMENT_REGEX, "<br/>")
.replace(SPAN_REGEX, (_span, inner) => inner);
codeblock += end;
return codeblock;
});
.replace(
PREBLOCK_REGEX,
(_pre, start: string, inner: string, end: string) => {
let codeblock = start;
codeblock += inner
.replace(NEWLINE_REPLACEMENT_REGEX, "<br/>")
.replace(SPAN_REGEX, (_span, inner) => inner);
codeblock += end;
return codeblock;
}
);
const document = parseHTML(html);
@@ -284,7 +402,7 @@ export function tinyToTiptap(html) {
const images = document.querySelectorAll("p > img");
for (const image of images) {
image.parentElement.replaceWith(image.cloneNode());
image.parentElement?.replaceWith(image.cloneNode());
}
const bogus = document.querySelectorAll("[data-mce-bogus]");
@@ -303,7 +421,7 @@ export function tinyToTiptap(html) {
return document.body.innerHTML;
}
function changeSessionContentType(item) {
function changeSessionContentType(item: any) {
if (item.id.endsWith("_content")) {
item.contentType = item.type;
item.type = "sessioncontent";

242
packages/core/src/models/note.js
View File

@@ -1,242 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import * as MarkdownBuilder from "../utils/templates/markdown/builder";
import HTMLBuilder from "../utils/templates/html/builder";
import TextBuilder from "../utils/templates/text/builder";
import { getContentFromData } from "../content-types";
import { CHECK_IDS, checkIsUserPremium } from "../common";
import { addItem, deleteItem } from "../utils/array";
import { formatDate } from "../utils/date";
import qclone from "qclone";
export default class Note {
/**
*
* @param {import('../api').default} db
* @param {any} note
*/
constructor(note, db) {
this._note = note;
this._db = db;
}
get data() {
return this._note;
}
get headline() {
return this._note.headline;
}
get title() {
return this._note.title;
}
get tags() {
return this._note.tags;
}
get colors() {
return this._note.colors;
}
get id() {
return this._note.id;
}
get notebooks() {
return this._note.notebooks;
}
get deleted() {
return this._note.deleted;
}
get dateEdited() {
return this._note.dateEdited;
}
get dateModified() {
return this._note.dateModified;
}
/**
*
* @param {"html"|"md"|"md-frontmatter"|"txt"} format - Format to export into
* @param {string} [contentItem=undefined]
* @param {boolean} [template=true]
* @param {string} [rawHTML=undefined] rawHTML - Use this raw content instead of generating itself
* @returns {Promise<string | false | undefined>}
*/
async export(to = "html", contentItem, template = true, rawHTML) {
if (to !== "txt" && !(await checkIsUserPremium(CHECK_IDS.noteExport)))
return false;
if (!this.data) return false;
const templateData = {
metadata: this.data,
title: this.title,
editedOn: formatDate(this.dateEdited),
headline: this.headline,
createdOn: formatDate(this.data.dateCreated),
tags: this.tags.join(", ")
};
contentItem = contentItem ||
(await this._db.content.raw(this._note.contentId)) || {
type: "tiptap",
data: "<p></p>"
};
let content;
if (to !== "txt") {
const { data, type } = await this._db.content.downloadMedia(
`export-${this.id}`,
contentItem,
false
);
content = getContentFromData(type, data);
} else {
content = getContentFromData(contentItem.type, contentItem.data);
}
switch (to) {
case "html":
templateData.content = rawHTML || content.toHTML();
return template
? await HTMLBuilder.buildHTML(templateData)
: templateData.content;
case "txt":
templateData.content = rawHTML || content.toTXT();
return template
? TextBuilder.buildText(templateData)
: templateData.content;
case "md":
templateData.content = rawHTML || content.toMD();
return template
? MarkdownBuilder.buildMarkdown(templateData)
: templateData.content;
case "md-frontmatter":
templateData.content = rawHTML || content.toMD();
return template
? MarkdownBuilder.buildMarkdownWithFrontmatter(templateData)
: templateData.content;
default:
throw new Error("Export format not supported.");
}
}
async content() {
const content = await this._db.content.raw(this._note.contentId);
return content ? content.data : null;
}
async duplicate() {
const content = await this._db.content.raw(this._note.contentId);
const duplicateId = await this._db.notes.add({
...qclone(this._note),
id: undefined,
content: content
? {
type: content.type,
data: content.data
}
: undefined,
readonly: false,
favorite: false,
pinned: false,
contentId: null,
title: this._note.title + " (Copy)",
dateEdited: null,
dateCreated: null,
dateModified: null
});
for (const notebook of this._db.relations.to(this._note, "notebook")) {
await this._db.relations.add(notebook, { id: duplicateId, type: "note" });
}
return duplicateId;
}
async color(color) {
if (!(await checkIsUserPremium(CHECK_IDS.noteColor))) return;
if (this._note.color)
await this._db.colors.untag(this._note.color, this._note.id);
await this._db.notes.add({
id: this.id,
color: this._db.colors.sanitize(color)
});
}
async uncolor() {
if (!this._note.color) return;
await this._db.colors.untag(this._note.color, this._note.id);
await this._db.notes.add({
id: this.id,
color: undefined
});
}
async tag(tag) {
if (
!this._db.tags.tag(tag) &&
this._db.tags.all.length >= 5 &&
!(await checkIsUserPremium(CHECK_IDS.noteTag))
)
return;
let tagItem = await this._db.tags.add(tag, this._note.id);
if (addItem(this._note.tags, tagItem.title))
await this._db.notes.add(this._note);
}
async untag(tag) {
const tagItem = this._db.tags.tag(tag);
if (tagItem && deleteItem(this._note.tags, tagItem.title)) {
await this._db.notes.add(this._note);
} else console.error("This note is not tagged by the specified tag.", tag);
await this._db.tags.untag(tag, this._note.id);
}
_toggle(prop) {
return this._db.notes.add({ id: this._note.id, [prop]: !this._note[prop] });
}
localOnly() {
return this._toggle("localOnly");
}
favorite() {
return this._toggle("favorite");
}
pin() {
return this._toggle("pinned");
}
readonly() {
return this._toggle("readonly");
}
synced() {
return !this.data.contentId || this._db.content.exists(this.data.contentId);
}
}

52
packages/core/src/models/note.ts Normal file
View File

@@ -0,0 +1,52 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Database from "../api";
import { isDeleted, type Note } from "../types";
export function createNoteModel(note: Note, db: Database) {
return {
...note,
data: note,
async content() {
if (!note.contentId) return null;
const content = await db.content.raw(note.contentId);
return content && !isDeleted(content) ? content.data : null;
},
synced() {
return !note.contentId || db.content.exists(note.contentId);
},
localOnly() {
return toggleProperty(db, note, "localOnly");
},
favorite() {
return toggleProperty(db, note, "favorite");
},
pin() {
return toggleProperty(db, note, "pinned");
},
readonly() {
return toggleProperty(db, note, "readonly");
}
};
}
function toggleProperty(db: Database, note: Note, property: keyof Note) {
return db.notes.add({ id: note.id, [property]: !note[property] });
}

Some files were not shown because too many files have changed in this diff Show More