diff --git a/packages/core/__benches__/relations.bench.ts b/packages/core/__benches__/relations.bench.ts
new file mode 100644
index 000000000..e28e4f01d
--- /dev/null
+++ b/packages/core/__benches__/relations.bench.ts
@@ -0,0 +1,57 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+*/
+
+import { bench, describe } from "vitest";
+import { databaseTest } from "../__tests__/utils";
+
+describe("relations", async () => {
+ const db = await databaseTest();
+ // const totalNotebooks = 10;
+
+ // let parentNotebookId: string | undefined = undefined;
+ // for (let i = 1; i <= 100; ++i) {
+ // const id = await db.notebooks.add({ title: `notebook-somethign-${i}` });
+ // if (parentNotebookId)
+ // await db.relations.add(
+ // { id: parentNotebookId, type: "notebook" },
+ // { id, type: "notebook" }
+ // );
+ // parentNotebookId = id;
+ // for (let j = 1; j <= 100; ++j) {
+ // await db.relations.add(
+ // { type: "notebook", id },
+ // { id: `${j * i}-note`, type: "note" }
+ // );
+ // }
+ // }
+
+ //
+
+ // const id2 = await db.notebooks.add({ title: `notebook-somethign` });
+
+ console.log(await db.notebooks.totalNotes("6516a04a35a073f359e7e801"));
+
+ // console.log(
+ // await db.relations.from({ id: "8-note", type: "note" }, "notebook").unlink()
+ // );
+
+ // bench("get some relations from 10k relations", async () => {
+ // await db.notebooks.totalNotes("6516a04a35a073f359e7e801");
+ // });
+});
diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.ts
index dc62615af..31f31c7ed 100644
--- a/packages/core/__tests__/notes.test.ts
+++ b/packages/core/__tests__/notes.test.ts
@@ -68,11 +68,12 @@ test("add invalid note", () =>
expect(db.notes.add({ hello: "world" })).rejects.toThrow();
}));
-test("add note", () =>
+test.only("add note", () =>
noteTest().then(async ({ db, id }) => {
- const note = db.notes.note(id);
+ const note = await db.notes.note$(id);
expect(note).toBeDefined();
- expect(await note?.content()).toStrictEqual(TEST_NOTE.content.data);
+ const content = await db.content.get(note!.contentId!);
+ expect(content!.data).toStrictEqual(TEST_NOTE.content.data);
}));
test("get note content", () =>
diff --git a/packages/core/nn.db b/packages/core/nn.db
new file mode 100644
index 000000000..2701cf90a
Binary files /dev/null and b/packages/core/nn.db differ
diff --git a/packages/core/package-lock.json b/packages/core/package-lock.json
index 94a35ef6e..2258b8bed 100644
--- a/packages/core/package-lock.json
+++ b/packages/core/package-lock.json
@@ -20,6 +20,7 @@
"html-to-text": "^9.0.5",
"htmlparser2": "^8.0.1",
"katex": "0.16.2",
+ "kysely": "^0.26.3",
"linkedom": "^0.14.17",
"liqe": "^1.13.0",
"mime-db": "1.52.0",
@@ -30,6 +31,7 @@
},
"devDependencies": {
"@notesnook/crypto": "file:../crypto",
+ "@types/better-sqlite3": "^7.6.5",
"@types/event-source-polyfill": "^1.0.1",
"@types/html-to-text": "^9.0.0",
"@types/katex": "^0.16.2",
@@ -40,6 +42,7 @@
"@types/ws": "^8.5.5",
"@vitest/coverage-v8": "^0.34.1",
"abortcontroller-polyfill": "^1.7.3",
+ "better-sqlite3": "^8.6.0",
"cross-env": "^7.0.3",
"dotenv": "^16.0.1",
"event-source-polyfill": "^1.0.31",
@@ -2173,6 +2176,15 @@
"node": ">= 10"
}
},
+ "node_modules/@types/better-sqlite3": {
+ "version": "7.6.9",
+ "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.9.tgz",
+ "integrity": "sha512-FvktcujPDj9XKMJQWFcl2vVl7OdRIqsSRX9b0acWwTmwLK9CF2eqo/FRcmMLNpugKoX/avA6pb7TorDLmpgTnQ==",
+ "dev": true,
+ "dependencies": {
+ "@types/node": "*"
+ }
+ },
"node_modules/@types/chai": {
"version": "4.3.14",
"resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.14.tgz",
@@ -2444,6 +2456,57 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/base64-js": {
+ "version": "1.5.1",
+ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+ "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ]
+ },
+ "node_modules/better-sqlite3": {
+ "version": "8.7.0",
+ "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-8.7.0.tgz",
+ "integrity": "sha512-99jZU4le+f3G6aIl6PmmV0cxUIWqKieHxsiF7G34CVFiE+/UabpYqkU0NJIkY/96mQKikHeBjtR27vFfs5JpEw==",
+ "dev": true,
+ "hasInstallScript": true,
+ "dependencies": {
+ "bindings": "^1.5.0",
+ "prebuild-install": "^7.1.1"
+ }
+ },
+ "node_modules/bindings": {
+ "version": "1.5.0",
+ "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+ "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+ "dev": true,
+ "dependencies": {
+ "file-uri-to-path": "1.0.0"
+ }
+ },
+ "node_modules/bl": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+ "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+ "dev": true,
+ "dependencies": {
+ "buffer": "^5.5.0",
+ "inherits": "^2.0.4",
+ "readable-stream": "^3.4.0"
+ }
+ },
"node_modules/boolbase": {
"version": "1.0.0",
"license": "ISC"
@@ -2457,6 +2520,30 @@
"concat-map": "0.0.1"
}
},
+ "node_modules/buffer": {
+ "version": "5.7.1",
+ "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+ "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ],
+ "dependencies": {
+ "base64-js": "^1.3.1",
+ "ieee754": "^1.1.13"
+ }
+ },
"node_modules/cac": {
"version": "6.7.14",
"resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz",
@@ -2523,6 +2610,12 @@
"node": "*"
}
},
+ "node_modules/chownr": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+ "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
+ "dev": true
+ },
"node_modules/combined-stream": {
"version": "1.0.8",
"dev": true,
@@ -2667,6 +2760,21 @@
"url": "https://github.com/sponsors/wooorm"
}
},
+ "node_modules/decompress-response": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+ "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+ "dev": true,
+ "dependencies": {
+ "mimic-response": "^3.1.0"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/deep-eql": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.3.tgz",
@@ -2679,6 +2787,15 @@
"node": ">=6"
}
},
+ "node_modules/deep-extend": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+ "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
+ "dev": true,
+ "engines": {
+ "node": ">=4.0.0"
+ }
+ },
"node_modules/deepmerge": {
"version": "4.3.1",
"license": "MIT",
@@ -2694,6 +2811,15 @@
"node": ">=0.4.0"
}
},
+ "node_modules/detect-libc": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz",
+ "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==",
+ "dev": true,
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/diff-sequences": {
"version": "29.6.3",
"resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz",
@@ -2773,6 +2899,15 @@
"node": ">=12"
}
},
+ "node_modules/end-of-stream": {
+ "version": "1.4.4",
+ "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
+ "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
+ "dev": true,
+ "dependencies": {
+ "once": "^1.4.0"
+ }
+ },
"node_modules/entities": {
"version": "4.5.0",
"license": "BSD-2-Clause",
@@ -2840,6 +2975,15 @@
"node": ">=12.0.0"
}
},
+ "node_modules/expand-template": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
+ "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
+ "dev": true,
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/fetch-cookie": {
"version": "2.1.0",
"license": "Unlicense",
@@ -2848,6 +2992,12 @@
"tough-cookie": "^4.0.0"
}
},
+ "node_modules/file-uri-to-path": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+ "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
+ "dev": true
+ },
"node_modules/form-data": {
"version": "4.0.0",
"dev": true,
@@ -2861,25 +3011,17 @@
"node": ">= 6"
}
},
+ "node_modules/fs-constants": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+ "dev": true
+ },
"node_modules/fs.realpath": {
"version": "1.0.0",
"dev": true,
"license": "ISC"
},
- "node_modules/fsevents": {
- "version": "2.3.3",
- "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
- "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
- "dev": true,
- "hasInstallScript": true,
- "optional": true,
- "os": [
- "darwin"
- ],
- "engines": {
- "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
- }
- },
"node_modules/get-func-name": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz",
@@ -2889,6 +3031,12 @@
"node": "*"
}
},
+ "node_modules/github-from-package": {
+ "version": "0.0.0",
+ "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+ "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
+ "dev": true
+ },
"node_modules/glob": {
"version": "7.2.3",
"dev": true,
@@ -3032,6 +3180,26 @@
"node": ">=0.10.0"
}
},
+ "node_modules/ieee754": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+ "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ]
+ },
"node_modules/inflight": {
"version": "1.0.6",
"dev": true,
@@ -3046,6 +3214,12 @@
"dev": true,
"license": "ISC"
},
+ "node_modules/ini": {
+ "version": "1.3.8",
+ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
+ "dev": true
+ },
"node_modules/is-alphabetical": {
"version": "2.0.1",
"dev": true,
@@ -3252,6 +3426,14 @@
"node": ">= 12"
}
},
+ "node_modules/kysely": {
+ "version": "0.26.3",
+ "resolved": "https://registry.npmjs.org/kysely/-/kysely-0.26.3.tgz",
+ "integrity": "sha512-yWSgGi9bY13b/W06DD2OCDDHQmq1kwTGYlQ4wpZkMOJqMGCstVCFIvxCCVG4KfY1/3G0MhDAcZsip/Lw8/vJWw==",
+ "engines": {
+ "node": ">=14.0.0"
+ }
+ },
"node_modules/leac": {
"version": "0.6.0",
"license": "MIT",
@@ -3360,6 +3542,18 @@
"node": ">= 0.6"
}
},
+ "node_modules/mimic-response": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
+ "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
+ "dev": true,
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/minimatch": {
"version": "3.1.2",
"dev": true,
@@ -3371,6 +3565,21 @@
"node": "*"
}
},
+ "node_modules/minimist": {
+ "version": "1.2.8",
+ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+ "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+ "dev": true,
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/mkdirp-classic": {
+ "version": "0.5.3",
+ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
+ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
+ "dev": true
+ },
"node_modules/mlly": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/mlly/-/mlly-1.6.1.tgz",
@@ -3397,6 +3606,12 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/napi-build-utils": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
+ "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==",
+ "dev": true
+ },
"node_modules/nearley": {
"version": "2.20.1",
"license": "MIT",
@@ -3421,6 +3636,18 @@
"version": "2.20.3",
"license": "MIT"
},
+ "node_modules/node-abi": {
+ "version": "3.56.0",
+ "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.56.0.tgz",
+ "integrity": "sha512-fZjdhDOeRcaS+rcpve7XuwHBmktS1nS1gzgghwKUQQ8nTy2FdSDr6ZT8k6YhvlJeHmmQMYiT/IH9hfco5zeW2Q==",
+ "dev": true,
+ "dependencies": {
+ "semver": "^7.3.5"
+ },
+ "engines": {
+ "node": ">=10"
+ }
+ },
"node_modules/node-fetch": {
"version": "2.6.7",
"license": "MIT",
@@ -3644,6 +3871,32 @@
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
}
},
+ "node_modules/prebuild-install": {
+ "version": "7.1.2",
+ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz",
+ "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==",
+ "dev": true,
+ "dependencies": {
+ "detect-libc": "^2.0.0",
+ "expand-template": "^2.0.3",
+ "github-from-package": "0.0.0",
+ "minimist": "^1.2.3",
+ "mkdirp-classic": "^0.5.3",
+ "napi-build-utils": "^1.0.1",
+ "node-abi": "^3.3.0",
+ "pump": "^3.0.0",
+ "rc": "^1.2.7",
+ "simple-get": "^4.0.0",
+ "tar-fs": "^2.0.0",
+ "tunnel-agent": "^0.6.0"
+ },
+ "bin": {
+ "prebuild-install": "bin.js"
+ },
+ "engines": {
+ "node": ">=10"
+ }
+ },
"node_modules/pretty-format": {
"version": "29.7.0",
"resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz",
@@ -3678,6 +3931,16 @@
"version": "1.9.0",
"license": "MIT"
},
+ "node_modules/pump": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
+ "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+ "dev": true,
+ "dependencies": {
+ "end-of-stream": "^1.1.0",
+ "once": "^1.3.1"
+ }
+ },
"node_modules/punycode": {
"version": "2.3.1",
"license": "MIT",
@@ -3708,12 +3971,41 @@
"node": ">=0.12"
}
},
+ "node_modules/rc": {
+ "version": "1.2.8",
+ "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+ "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+ "dev": true,
+ "dependencies": {
+ "deep-extend": "^0.6.0",
+ "ini": "~1.3.0",
+ "minimist": "^1.2.0",
+ "strip-json-comments": "~2.0.1"
+ },
+ "bin": {
+ "rc": "cli.js"
+ }
+ },
"node_modules/react-is": {
"version": "18.2.0",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
"integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
"dev": true
},
+ "node_modules/readable-stream": {
+ "version": "3.6.2",
+ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
+ "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+ "dev": true,
+ "dependencies": {
+ "inherits": "^2.0.3",
+ "string_decoder": "^1.1.1",
+ "util-deprecate": "^1.0.1"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
"node_modules/refractor": {
"version": "4.8.1",
"dev": true,
@@ -3781,6 +4073,26 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/safe-buffer": {
+ "version": "5.2.1",
+ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+ "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ]
+ },
"node_modules/safer-buffer": {
"version": "2.1.2",
"dev": true,
@@ -3849,6 +4161,51 @@
"dev": true,
"license": "ISC"
},
+ "node_modules/simple-concat": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
+ "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ]
+ },
+ "node_modules/simple-get": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
+ "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ],
+ "dependencies": {
+ "decompress-response": "^6.0.0",
+ "once": "^1.3.1",
+ "simple-concat": "^1.0.0"
+ }
+ },
"node_modules/source-map": {
"version": "0.6.1",
"dev": true,
@@ -3889,6 +4246,24 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/string_decoder": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
+ "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
+ "dev": true,
+ "dependencies": {
+ "safe-buffer": "~5.2.0"
+ }
+ },
+ "node_modules/strip-json-comments": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
+ "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
+ "dev": true,
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/strip-literal": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-1.3.0.tgz",
@@ -3917,6 +4292,34 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/tar-fs": {
+ "version": "2.1.1",
+ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
+ "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
+ "dev": true,
+ "dependencies": {
+ "chownr": "^1.1.1",
+ "mkdirp-classic": "^0.5.2",
+ "pump": "^3.0.0",
+ "tar-stream": "^2.1.4"
+ }
+ },
+ "node_modules/tar-stream": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
+ "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+ "dev": true,
+ "dependencies": {
+ "bl": "^4.0.3",
+ "end-of-stream": "^1.4.1",
+ "fs-constants": "^1.0.0",
+ "inherits": "^2.0.3",
+ "readable-stream": "^3.1.1"
+ },
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/test-exclude": {
"version": "6.0.0",
"dev": true,
@@ -3992,6 +4395,18 @@
"version": "2.4.1",
"license": "0BSD"
},
+ "node_modules/tunnel-agent": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+ "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
+ "dev": true,
+ "dependencies": {
+ "safe-buffer": "^5.0.1"
+ },
+ "engines": {
+ "node": "*"
+ }
+ },
"node_modules/type-detect": {
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz",
@@ -4026,6 +4441,12 @@
"requires-port": "^1.0.0"
}
},
+ "node_modules/util-deprecate": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
+ "dev": true
+ },
"node_modules/v8-to-istanbul": {
"version": "9.2.0",
"dev": true,
diff --git a/packages/core/package.json b/packages/core/package.json
index ef7814a58..c8137e13a 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -9,6 +9,7 @@
},
"devDependencies": {
"@notesnook/crypto": "file:../crypto",
+ "@types/better-sqlite3": "^7.6.5",
"@types/event-source-polyfill": "^1.0.1",
"@types/html-to-text": "^9.0.0",
"@types/katex": "^0.16.2",
@@ -19,6 +20,7 @@
"@types/ws": "^8.5.5",
"@vitest/coverage-v8": "^0.34.1",
"abortcontroller-polyfill": "^1.7.3",
+ "better-sqlite3": "^8.6.0",
"cross-env": "^7.0.3",
"dotenv": "^16.0.1",
"event-source-polyfill": "^1.0.31",
@@ -54,6 +56,7 @@
"html-to-text": "^9.0.5",
"htmlparser2": "^8.0.1",
"katex": "0.16.2",
+ "kysely": "^0.26.3",
"linkedom": "^0.14.17",
"liqe": "^1.13.0",
"mime-db": "1.52.0",
diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts
index c41e4da85..ad92931de 100644
--- a/packages/core/src/api/index.ts
+++ b/packages/core/src/api/index.ts
@@ -60,6 +60,9 @@ import {
import TokenManager from "./token-manager";
import { Attachment } from "../types";
import { Settings } from "../collections/settings";
+import { DatabaseAccessor, DatabaseSchema, createDatabase } from "../database";
+import { Kysely, SqliteDriver, Transaction } from "kysely";
+import BetterSQLite3 from "better-sqlite3";
type EventSourceConstructor = new (
uri: string,
@@ -111,6 +114,32 @@ class Database {
return this.options.compressor;
};
+ private _sql?: Kysely;
+ sql: DatabaseAccessor = () => {
+ if (this._transaction) return this._transaction;
+
+ if (!this._sql)
+ throw new Error(
+ "Database not initialized. Did you forget to call db.init()?"
+ );
+ return this._sql;
+ };
+
+ private _transaction?: Transaction;
+ transaction = (
+ executor: (tr: Transaction) => void | Promise
+ ) => {
+ if (this._transaction) return executor(this._transaction);
+ return this.sql()
+ .transaction()
+ .execute(async (tr) => {
+ this._transaction = tr;
+ await executor(tr);
+ this._transaction = undefined;
+ })
+ .finally(() => (this._transaction = undefined));
+ };
+
private options?: Options;
EventSource?: EventSourceConstructor;
eventSource?: EventSource | null;
@@ -144,6 +173,7 @@ class Database {
reminders = new Reminders(this);
relations = new Relations(this);
notes = new Notes(this);
+
// constructor() {
// this.sseMutex = new Mutex();
// // this.lastHeartbeat = undefined; // { local: 0, server: 0 };
@@ -170,8 +200,8 @@ class Database {
this
);
EV.subscribe(EVENTS.attachmentDeleted, async (attachment: Attachment) => {
- await this.fs().cancel(attachment.metadata.hash, "upload");
- await this.fs().cancel(attachment.metadata.hash, "download");
+ await this.fs().cancel(attachment.hash, "upload");
+ await this.fs().cancel(attachment.hash, "download");
});
EV.subscribe(EVENTS.userLoggedOut, async () => {
await this.monographs.clear();
@@ -179,6 +209,10 @@ class Database {
this.disconnectSSE();
});
+ this._sql = await createDatabase(
+ new SqliteDriver({ database: BetterSQLite3("nn.db") })
+ );
+
await this._validate();
await this.initCollections();
diff --git a/packages/core/src/api/sync/collector.ts b/packages/core/src/api/sync/collector.ts
index 199cbcc76..204632984 100644
--- a/packages/core/src/api/sync/collector.ts
+++ b/packages/core/src/api/sync/collector.ts
@@ -66,21 +66,21 @@ class Collector {
}
}
- for (const itemType in SYNC_COLLECTIONS_MAP) {
- const collectionKey =
- SYNC_COLLECTIONS_MAP[itemType as keyof typeof SYNC_COLLECTIONS_MAP];
- const collection = this.db[collectionKey].collection;
- for (const chunk of collection.iterateSync(chunkSize)) {
- const items = await this.prepareChunk(
- chunk,
- lastSyncedTimestamp,
- isForceSync,
- key
- );
- if (!items) continue;
- yield { items, type: itemType as keyof typeof SYNC_COLLECTIONS_MAP };
- }
- }
+ // for (const itemType in SYNC_COLLECTIONS_MAP) {
+ // const collectionKey =
+ // SYNC_COLLECTIONS_MAP[itemType as keyof typeof SYNC_COLLECTIONS_MAP];
+ // const collection = this.db[collectionKey].collection;
+ // for (const chunk of collection.iterateSync(chunkSize)) {
+ // const items = await this.prepareChunk(
+ // chunk,
+ // lastSyncedTimestamp,
+ // isForceSync,
+ // key
+ // );
+ // if (!items) continue;
+ // yield { items, type: itemType as keyof typeof SYNC_COLLECTIONS_MAP };
+ // }
+ // }
}
async prepareChunk(
diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.ts
index e95e0ada9..4c8d04244 100644
--- a/packages/core/src/collections/attachments.ts
+++ b/packages/core/src/collections/attachments.ts
@@ -29,21 +29,18 @@ import {
isWebClip
} from "../utils/filename";
import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";
-import { CachedCollection } from "../database/cached-collection";
import { Output } from "../interfaces";
-import { Attachment, AttachmentMetadata, isDeleted } from "../types";
+import { Attachment } from "../types";
import Database from "../api";
+import { SQLCollection } from "../database/sql-collection";
+import { isCipher } from "../database/crypto";
export class Attachments implements ICollection {
name = "attachments";
key: Cipher<"base64"> | null = null;
- readonly collection: CachedCollection<"attachments", Attachment>;
+ readonly collection: SQLCollection<"attachments", Attachment>;
constructor(private readonly db: Database) {
- this.collection = new CachedCollection(
- db.storage,
- "attachments",
- db.eventManager
- );
+ this.collection = new SQLCollection(db.sql, "attachments", db.eventManager);
this.key = null;
EV.subscribe(
@@ -60,15 +57,15 @@ export class Attachments implements ICollection {
eventData: Record;
}) => {
if (!success || !eventData || !eventData.readOnDownload) return;
- const attachment = this.attachment(filename);
- if (!attachment || !attachment.metadata) return;
+ const attachment = await this.attachment(filename);
+ if (!attachment) return;
const src = await this.read(filename, getOutputType(attachment));
if (!src) return;
EV.publish(EVENTS.mediaAttachmentDownloaded, {
groupId,
- hash: attachment.metadata.hash,
+ hash: attachment.hash,
attachmentType: getAttachmentType(attachment),
src
});
@@ -86,7 +83,7 @@ export class Attachments implements ICollection {
filename: string;
error: string;
}) => {
- const attachment = this.attachment(filename);
+ const attachment = await this.attachment(filename);
if (!attachment) return;
if (success) await this.markAsUploaded(attachment.id);
else
@@ -104,54 +101,50 @@ export class Attachments implements ICollection {
async add(
item: Partial<
- Omit & {
+ Omit & {
key: SerializedKey;
}
- > & {
- metadata: Partial & { hash: string };
- }
+ >
) {
if (!item) return console.error("attachment cannot be undefined");
- if (!item.metadata.hash) throw new Error("Please provide attachment hash.");
+ if (!item.hash) throw new Error("Please provide attachment hash.");
- const oldAttachment = this.all.find(
- (a) => a.metadata.hash === item.metadata?.hash
- );
+ const oldAttachment = await this.attachment(item.hash);
const id = oldAttachment?.id || getId();
const encryptedKey = item.key
- ? await this.encryptKey(item.key)
- : oldAttachment?.key;
+ ? JSON.stringify(await this.encryptKey(item.key))
+ : oldAttachment?.encryptionKey;
const attachment = {
...oldAttachment,
- ...oldAttachment?.metadata,
...item,
- key: encryptedKey
+ encryptionKey: encryptedKey
};
const {
iv,
- length,
+ size,
alg,
hash,
hashType,
filename,
+ mimeType,
salt,
- type,
chunkSize,
- key
+ encryptionKey
} = attachment;
if (
!iv ||
- !length ||
+ !size ||
!alg ||
!hash ||
!hashType ||
- !filename ||
+ // !filename ||
+ // !mimeType ||
!salt ||
!chunkSize ||
- !key
+ !encryptionKey
) {
console.error(
"Attachment is invalid because all properties are required:",
@@ -161,27 +154,33 @@ export class Attachments implements ICollection {
return;
}
- return this.collection.add({
+ await this.collection.upsert({
type: "attachment",
id,
iv,
salt,
- length,
+ size,
alg,
- key,
+ encryptionKey,
chunkSize,
- metadata: {
- hash,
- hashType,
- filename: getFileNameWithExtension(filename, type),
- type: type || "application/octet-stream"
- },
+
+ filename:
+ filename ||
+ getFileNameWithExtension(
+ filename || hash,
+ mimeType || "application/octet-stream"
+ ),
+ hash,
+ hashType,
+ mimeType: mimeType || "application/octet-stream",
+
dateCreated: attachment.dateCreated || Date.now(),
dateModified: attachment.dateModified || Date.now(),
dateUploaded: attachment.dateUploaded,
- dateDeleted: undefined,
failed: attachment.failed
});
+
+ return id;
}
async generateKey() {
@@ -189,7 +188,9 @@ export class Attachments implements ICollection {
return await this.db.crypto().generateRandomKey();
}
- async decryptKey(key: Cipher<"base64">): Promise {
+ async decryptKey(keyJSON: string): Promise {
+ const key = JSON.parse(keyJSON);
+ if (!isCipher(key)) return null;
const encryptionKey = await this._getEncryptionKey();
const plainData = await this.db.storage().decrypt(encryptionKey, key);
if (!plainData) return null;
@@ -197,8 +198,8 @@ export class Attachments implements ICollection {
}
async remove(hashOrId: string, localOnly: boolean) {
- const attachment = this.attachment(hashOrId);
- if (!attachment || !attachment.metadata) return false;
+ const attachment = await this.attachment(hashOrId);
+ if (!attachment) return false;
if (!localOnly && !(await this.canDetach(attachment)))
throw new Error("This attachment is inside a locked note.");
@@ -206,116 +207,115 @@ export class Attachments implements ICollection {
if (
await this.db
.fs()
- .deleteFile(
- attachment.metadata.hash,
- localOnly || !attachment.dateUploaded
- )
+ .deleteFile(attachment.hash, localOnly || !attachment.dateUploaded)
) {
if (!localOnly) {
await this.detach(attachment);
}
- await this.collection.remove(attachment.id);
+ await this.collection.softDelete([attachment.id]);
return true;
}
return false;
}
async detach(attachment: Attachment) {
- for (const note of this.db.relations.from(attachment, "note").resolved()) {
+ for (const note of await this.db.relations
+ .from(attachment, "note")
+ .resolve()) {
if (!note || !note.contentId) continue;
await this.db.content.removeAttachments(note.contentId, [
- attachment.metadata.hash
+ attachment.hash
]);
}
}
private async canDetach(attachment: Attachment) {
- return this.db.relations
- .from(attachment, "note")
- .resolved()
- .every((note) => !note.locked);
+ return (await this.db.relations.from(attachment, "note").resolve()).every(
+ (note) => !note.locked
+ );
}
- ofNote(
+ async ofNote(
noteId: string,
...types: ("files" | "images" | "webclips" | "all")[]
- ): Attachment[] {
- const noteAttachments = this.db.relations
+ ): Promise {
+ const noteAttachments = await this.db.relations
.from({ type: "note", id: noteId }, "attachment")
- .resolved();
+ .resolve();
if (types.includes("all")) return noteAttachments;
return noteAttachments.filter((a) => {
- if (isImage(a.metadata.type) && types.includes("images")) return true;
- else if (isWebClip(a.metadata.type) && types.includes("webclips"))
- return true;
+ if (isImage(a.mimeType) && types.includes("images")) return true;
+ else if (isWebClip(a.mimeType) && types.includes("webclips")) return true;
else if (types.includes("files")) return true;
});
}
- exists(hash: string) {
- const attachment = this.all.find((a) => a.metadata.hash === hash);
- return !!attachment;
+ async exists(hash: string) {
+ return !!(await this.attachment(hash));
}
async read(
hash: string,
outputType: TOutputFormat
): Promise | undefined> {
- const attachment = this.all.find((a) => a.metadata.hash === hash);
+ const attachment = await this.attachment(hash);
if (!attachment) return;
- const key = await this.decryptKey(attachment.key);
+ const key = await this.decryptKey(attachment.encryptionKey);
if (!key) return;
- const data = await this.db
- .fs()
- .readEncrypted(attachment.metadata.hash, key, {
- chunkSize: attachment.chunkSize,
- iv: attachment.iv,
- salt: attachment.salt,
- length: attachment.length,
- alg: attachment.alg,
- outputType
- });
+ const data = await this.db.fs().readEncrypted(attachment.hash, key, {
+ chunkSize: attachment.chunkSize,
+ iv: attachment.iv,
+ salt: attachment.salt,
+ size: attachment.size,
+ alg: attachment.alg,
+ outputType
+ });
if (!data) return;
return (
outputType === "base64"
? dataurl.fromObject({
- type: attachment.metadata.type,
+ mimeType: attachment.mimeType,
data
})
: data
) as Output;
}
- attachment(hashOrId: string) {
- return this.all.find(
- (a) => a.id === hashOrId || a.metadata?.hash === hashOrId
- );
+ async attachment(hashOrId: string): Promise {
+ return await this.db
+ .sql()
+ .selectFrom("attachments")
+ .selectAll()
+ .where((eb) =>
+ eb.or([eb("id", "==", hashOrId), eb("hash", "==", hashOrId)])
+ )
+ .where("deleted", "is", null)
+ .$narrowType()
+ .executeTakeFirst();
}
markAsUploaded(id: string) {
- const attachment = this.attachment(id);
- if (!attachment) return;
- attachment.dateUploaded = Date.now();
- attachment.failed = undefined;
- return this.collection.update(attachment);
+ return this.collection.update([id], {
+ dateUploaded: Date.now(),
+ failed: null
+ });
}
reset(id: string) {
- const attachment = this.attachment(id);
- if (!attachment) return;
- attachment.dateUploaded = undefined;
- return this.collection.update(attachment);
+ return this.collection.update([id], {
+ dateUploaded: null
+ });
}
markAsFailed(id: string, reason: string) {
- const attachment = this.attachment(id);
- if (!attachment) return;
- attachment.failed = reason;
- return this.collection.update(attachment);
+ return this.collection.update([id], {
+ dateUploaded: null,
+ failed: reason
+ });
}
async save(
@@ -325,7 +325,7 @@ export class Attachments implements ICollection {
): Promise {
const hashResult = await this.db.fs().hashBase64(data);
if (!hashResult) return;
- if (this.exists(hashResult.hash)) return hashResult.hash;
+ if (await this.exists(hashResult.hash)) return hashResult.hash;
const key = await this.generateKey();
const { hash, hashType, ...encryptionMetadata } = await this.db
@@ -335,27 +335,23 @@ export class Attachments implements ICollection {
await this.add({
...encryptionMetadata,
key,
- metadata: {
- filename: filename || hash,
- hash,
- hashType,
- type: mimeType || "application/octet-stream"
- }
+
+ filename: filename || hash,
+ hash,
+ hashType,
+ mimeType: mimeType || "application/octet-stream"
});
return hash;
}
async downloadMedia(noteId: string, hashesToLoad?: string[]) {
- let attachments = this.ofNote(noteId, "images", "webclips");
+ let attachments = await this.ofNote(noteId, "images", "webclips");
if (hashesToLoad)
- attachments = attachments.filter((a) =>
- hasItem(hashesToLoad, a.metadata.hash)
- );
+ attachments = attachments.filter((a) => hasItem(hashesToLoad, a.hash));
await this.db.fs().queueDownloads(
attachments.map((a) => ({
- filename: a.metadata.hash,
- metadata: a.metadata,
+ filename: a.hash,
chunkSize: a.chunkSize
})),
noteId,
@@ -375,65 +371,60 @@ export class Attachments implements ICollection {
const isDeleted = await this.db.fs().deleteFile(attachment.metadata.hash);
if (!isDeleted) continue;
- await this.collection.remove(attachment.id);
+ await this.collection.softDelete(attachment.id);
}
}
- get pending() {
- return this.all.filter(
- (attachment) => !attachment.dateUploaded || attachment.dateUploaded <= 0
- );
- }
+ // get pending() {
+ // return this.all.filter(
+ // (attachment) =>
+ // (!attachment.dateUploaded || attachment.dateUploaded <= 0) &&
+ // this.db.relations.to(attachment, "note").length > 0
+ // );
+ // }
- get uploaded() {
- return this.all.filter((attachment) => !!attachment.dateUploaded);
- }
+ // get uploaded() {
+ // return this.all.filter((attachment) => !!attachment.dateUploaded);
+ // }
- get syncable() {
- return this.collection
- .raw()
- .filter(
- (attachment) => isDeleted(attachment) || !!attachment.dateUploaded
- );
- }
+ // get syncable() {
+ // return this.collection
+ // .raw()
+ // .filter(
+ // (attachment) => isDeleted(attachment) || !!attachment.dateUploaded
+ // );
+ // }
- get deleted() {
- return this.all.filter((attachment) => !!attachment.dateDeleted);
- }
+ // get deleted() {
+ // return this.all.filter((attachment) => !!attachment.dateDeleted);
+ // }
- get images() {
- return this.all.filter(
- (attachment) => attachment.metadata && isImage(attachment.metadata.type)
- );
- }
+ // get images() {
+ // return this.all.filter((attachment) => isImage(attachment.metadata.type));
+ // }
- get webclips() {
- return this.all.filter(
- (attachment) => attachment.metadata && isWebClip(attachment.metadata.type)
- );
- }
+ // get webclips() {
+ // return this.all.filter((attachment) => isWebClip(attachment.metadata.type));
+ // }
- get media() {
- return this.all.filter(
- (attachment) =>
- attachment.metadata &&
- (isImage(attachment.metadata.type) ||
- isWebClip(attachment.metadata.type))
- );
- }
+ // get media() {
+ // return this.all.filter(
+ // (attachment) =>
+ // isImage(attachment.metadata.type) || isWebClip(attachment.metadata.type)
+ // );
+ // }
- get files() {
- return this.all.filter(
- (attachment) =>
- attachment.metadata &&
- !isImage(attachment.metadata.type) &&
- !isWebClip(attachment.metadata.type)
- );
- }
+ // get files() {
+ // return this.all.filter(
+ // (attachment) =>
+ // !isImage(attachment.metadata.type) &&
+ // !isWebClip(attachment.metadata.type)
+ // );
+ // }
- get all() {
- return this.collection.items();
- }
+ // get all() {
+ // return this.collection.items();
+ // }
private async encryptKey(key: SerializedKey) {
const encryptionKey = await this._getEncryptionKey();
@@ -454,15 +445,15 @@ export class Attachments implements ICollection {
}
export function getOutputType(attachment: Attachment): DataFormat {
- if (attachment.metadata.type === "application/vnd.notesnook.web-clip")
+ if (attachment.mimeType === "application/vnd.notesnook.web-clip")
return "text";
- else if (attachment.metadata.type.startsWith("image/")) return "base64";
+ else if (attachment.mimeType.startsWith("image/")) return "base64";
return "uint8array";
}
function getAttachmentType(attachment: Attachment) {
- if (attachment.metadata.type === "application/vnd.notesnook.web-clip")
+ if (attachment.mimeType === "application/vnd.notesnook.web-clip")
return "webclip";
- else if (attachment.metadata.type.startsWith("image/")) return "image";
+ else if (attachment.mimeType.startsWith("image/")) return "image";
else return "generic";
}
diff --git a/packages/core/src/collections/colors.ts b/packages/core/src/collections/colors.ts
index ded823f59..edfc1c52e 100644
--- a/packages/core/src/collections/colors.ts
+++ b/packages/core/src/collections/colors.ts
@@ -19,10 +19,10 @@ along with this program. If not, see .
import { ICollection } from "./collection";
import { getId } from "../utils/id";
-import { Color, MaybeDeletedItem } from "../types";
+import { Color } from "../types";
import Database from "../api";
-import { CachedCollection } from "../database/cached-collection";
import { Tags } from "./tags";
+import { SQLCollection } from "../database/sql-collection";
export const DefaultColors: Record = {
red: "#f44336",
@@ -36,13 +36,9 @@ export const DefaultColors: Record = {
export class Colors implements ICollection {
name = "colors";
- readonly collection: CachedCollection<"colors", Color>;
+ readonly collection: SQLCollection<"colors", Color>;
constructor(private readonly db: Database) {
- this.collection = new CachedCollection(
- db.storage,
- "colors",
- db.eventManager
- );
+ this.collection = new SQLCollection(db.sql, "colors", db.eventManager);
}
init() {
@@ -53,27 +49,27 @@ export class Colors implements ICollection {
return this.collection.get(id);
}
- async merge(remoteColor: MaybeDeletedItem) {
- if (!remoteColor) return;
+ // async merge(remoteColor: MaybeDeletedItem) {
+ // if (!remoteColor) return;
- const localColor = this.collection.get(remoteColor.id);
- if (!localColor || remoteColor.dateModified > localColor.dateModified)
- await this.collection.add(remoteColor);
- }
+ // const localColor = this.collection.get(remoteColor.id);
+ // if (!localColor || remoteColor.dateModified > localColor.dateModified)
+ // await this.collection.add(remoteColor);
+ // }
async add(item: Partial) {
if (item.remote)
throw new Error("Please use db.colors.merge to merge remote colors.");
const id = item.id || getId(item.dateCreated);
- const oldColor = this.color(id);
+ const oldColor = await this.color(id);
item.title = item.title ? Tags.sanitize(item.title) : item.title;
if (!item.title && !oldColor?.title) throw new Error("Title is required.");
if (!item.colorCode && !oldColor?.colorCode)
throw new Error("Color code is required.");
- const color: Color = {
+ await this.collection.upsert({
id,
dateCreated: item.dateCreated || oldColor?.dateCreated || Date.now(),
dateModified: item.dateModified || oldColor?.dateModified || Date.now(),
@@ -81,34 +77,35 @@ export class Colors implements ICollection {
colorCode: item.colorCode || oldColor?.colorCode || "",
type: "color",
remote: false
- };
- await this.collection.add(color);
- return color.id;
+ });
+ return id;
}
- get raw() {
- return this.collection.raw();
+ // get raw() {
+ // return this.collection.raw();
+ // }
+
+ // get all(): Color[] {
+ // return this.collection.items();
+ // }
+
+ async remove(...ids: string[]) {
+ await this.db.transaction(async () => {
+ await this.db.relations.unlinkOfType("color", ids);
+ await this.collection.softDelete(ids);
+ });
}
- get all(): Color[] {
- return this.collection.items();
- }
-
- async remove(id: string) {
- await this.collection.remove(id);
- await this.db.relations.cleanup();
- }
-
- async delete(id: string) {
- await this.collection.delete(id);
- await this.db.relations.cleanup();
- }
+ // async delete(id: string) {
+ // await this.collection.delete(id);
+ // await this.db.relations.cleanup();
+ // }
exists(id: string) {
return this.collection.exists(id);
}
- find(idOrTitle: string) {
- return this.all.find((t) => t.title === idOrTitle || t.id === idOrTitle);
- }
+ // find(idOrTitle: string) {
+ // return this.all.find((t) => t.title === idOrTitle || t.id === idOrTitle);
+ // }
}
diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.ts
index 10b8fcc50..ec439ba14 100644
--- a/packages/core/src/collections/content.ts
+++ b/packages/core/src/collections/content.ts
@@ -27,13 +27,12 @@ import {
ContentItem,
ContentType,
EncryptedContentItem,
- MaybeDeletedItem,
UnencryptedContentItem,
isDeleted
} from "../types";
-import { IndexedCollection } from "../database/indexed-collection";
import Database from "../api";
import { getOutputType } from "./attachments";
+import { SQLCollection } from "../database/sql-collection";
export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({
noteId,
@@ -48,27 +47,15 @@ export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({
export class Content implements ICollection {
name = "content";
- readonly collection: IndexedCollection<"content", ContentItem>;
+ readonly collection: SQLCollection<"content", ContentItem>;
constructor(private readonly db: Database) {
- this.collection = new IndexedCollection(
- db.storage,
- "content",
- db.eventManager
- );
+ this.collection = new SQLCollection(db.sql, "content", db.eventManager);
}
async init() {
await this.collection.init();
}
- async merge(content: MaybeDeletedItem) {
- return await this.collection.addItem(
- isDeleted(content) || !isUnencryptedContent(content)
- ? content
- : await this.extractAttachments(content)
- );
- }
-
async add(content: Partial) {
if (typeof content.data === "object") {
if ("data" in content.data && typeof content.data.data === "string")
@@ -106,7 +93,7 @@ export class Content implements ICollection {
conflicted: content.conflicted,
dateResolved: content.dateResolved
};
- await this.collection.addItem(
+ await this.collection.upsert(
isUnencryptedContent(contentItem)
? await this.extractAttachments(contentItem)
: contentItem
@@ -133,29 +120,46 @@ export class Content implements ICollection {
}
async raw(id: string) {
- const content = await this.collection.getItem(id);
+ const content = await this.collection.get(id);
if (!content) return;
return content;
}
- remove(id: string) {
- if (!id) return;
- return this.collection.removeItem(id);
+ remove(...ids: string[]) {
+ return this.collection.softDelete(ids);
}
- multi(ids: string[]) {
- return this.collection.getItems(ids);
+ removeByNoteId(...ids: string[]) {
+ return this.db
+ .sql()
+ .replaceInto("content")
+ .columns(["id", "dateModified", "deleted"])
+ .expression((eb) =>
+ eb
+ .selectFrom("content")
+ .where("noteId", "in", ids)
+ .select((eb) => [
+ "content.id",
+ eb.lit(Date.now()).as("dateModified"),
+ eb.lit(1).as("deleted")
+ ])
+ )
+ .execute();
}
+ // multi(ids: string[]) {
+ // return this.collection.getItems(ids);
+ // }
+
exists(id: string) {
return this.collection.exists(id);
}
- async all() {
- return Object.values(
- await this.collection.getItems(this.collection.indexer.indices)
- );
- }
+ // async all() {
+ // return Object.values(
+ // await this.collection.getItems(this.collection.indexer.indices)
+ // );
+ // }
insertMedia(contentItem: UnencryptedContentItem) {
return this.insert(contentItem, async (hashes) => {
@@ -183,17 +187,16 @@ export class Content implements ICollection {
const content = getContentFromData(contentItem.type, contentItem.data);
if (!content) return contentItem;
contentItem.data = await content.insertMedia(async (hashes) => {
- const attachments = hashes.reduce((attachments, hash) => {
- const attachment = this.db.attachments.attachment(hash);
- if (!attachment) return attachments;
+ const attachments: Attachment[] = [];
+ for (const hash of hashes) {
+ const attachment = await this.db.attachments.attachment(hash);
+ if (!attachment) continue;
attachments.push(attachment);
- return attachments;
- }, [] as Attachment[]);
+ }
await this.db.fs().queueDownloads(
attachments.map((a) => ({
- filename: a.metadata.hash,
- metadata: a.metadata,
+ filename: a.hash,
chunkSize: a.chunkSize
})),
groupId,
@@ -203,11 +206,11 @@ export class Content implements ICollection {
const sources: Record = {};
for (const attachment of attachments) {
const src = await this.db.attachments.read(
- attachment.metadata.hash,
+ attachment.hash,
getOutputType(attachment)
);
if (!src) continue;
- sources[attachment.metadata.hash] = src;
+ sources[attachment.hash] = src;
}
return sources;
});
@@ -243,16 +246,16 @@ export class Content implements ICollection {
this.db.attachments.save
);
- const noteAttachments = this.db.relations
+ const noteAttachments = await this.db.relations
.from({ type: "note", id: contentItem.noteId }, "attachment")
- .resolved();
+ .resolve();
const toDelete = noteAttachments.filter((attachment) => {
- return hashes.every((hash) => hash !== attachment.metadata.hash);
+ return hashes.every((hash) => hash !== attachment.hash);
});
const toAdd = hashes.filter((hash) => {
- return hash && noteAttachments.every((a) => hash !== a.metadata.hash);
+ return hash && noteAttachments.every((a) => hash !== a.hash);
});
for (const attachment of toDelete) {
@@ -266,7 +269,7 @@ export class Content implements ICollection {
}
for (const hash of toAdd) {
- const attachment = this.db.attachments.attachment(hash);
+ const attachment = await this.db.attachments.attachment(hash);
if (!attachment) continue;
await this.db.relations.add(
{
@@ -284,24 +287,24 @@ export class Content implements ICollection {
return contentItem;
}
- async cleanup() {
- const indices = this.collection.indexer.indices;
- await this.db.notes.init();
- const notes = this.db.notes.all;
- if (!notes.length && indices.length > 0) return [];
- const ids = [];
- for (const contentId of indices) {
- const noteIndex = notes.findIndex((note) => note.contentId === contentId);
- const isOrphaned = noteIndex === -1;
- if (isOrphaned) {
- ids.push(contentId);
- await this.collection.deleteItem(contentId);
- } else if (notes[noteIndex].localOnly) {
- ids.push(contentId);
- }
- }
- return ids;
- }
+ // async cleanup() {
+ // const indices = this.collection.indexer.indices;
+ // await this.db.notes.init();
+ // const notes = this.db.notes.all;
+ // if (!notes.length && indices.length > 0) return [];
+ // const ids = [];
+ // for (const contentId of indices) {
+ // const noteIndex = notes.findIndex((note) => note.contentId === contentId);
+ // const isOrphaned = noteIndex === -1;
+ // if (isOrphaned) {
+ // ids.push(contentId);
+ // await this.collection.deleteItem(contentId);
+ // } else if (notes[noteIndex].localOnly) {
+ // ids.push(contentId);
+ // }
+ // }
+ // return ids;
+ // }
}
export function isUnencryptedContent(
diff --git a/packages/core/src/collections/note-history.ts b/packages/core/src/collections/note-history.ts
index 2464e80be..ee7b4effd 100644
--- a/packages/core/src/collections/note-history.ts
+++ b/packages/core/src/collections/note-history.ts
@@ -19,7 +19,7 @@ along with this program. If not, see .
import Database from "../api";
import { isCipher } from "../database/crypto";
-import { IndexedCollection } from "../database/indexed-collection";
+import { SQLCollection } from "../database/sql-collection";
import { HistorySession, isDeleted } from "../types";
import { makeSessionContentId } from "../utils/id";
import { ICollection } from "./collection";
@@ -29,13 +29,9 @@ export class NoteHistory implements ICollection {
name = "notehistory";
versionsLimit = 100;
sessionContent = new SessionContent(this.db);
- private readonly collection: IndexedCollection<"notehistory", HistorySession>;
+ private readonly collection: SQLCollection<"notehistory", HistorySession>;
constructor(private readonly db: Database) {
- this.collection = new IndexedCollection(
- db.storage,
- "notehistory",
- db.eventManager
- );
+ this.collection = new SQLCollection(db.sql, "notehistory", db.eventManager);
}
async init() {
@@ -43,10 +39,6 @@ export class NoteHistory implements ICollection {
await this.sessionContent.init();
}
- async merge(item: HistorySession) {
- await this.collection.addItem(item);
- }
-
async get(noteId: string) {
if (!noteId) return [];
@@ -67,7 +59,7 @@ export class NoteHistory implements ICollection {
content: NoteContent
) {
sessionId = `${noteId}_${sessionId}`;
- const oldSession = await this.collection.getItem(sessionId);
+ const oldSession = await this.collection.get(sessionId);
if (oldSession && isDeleted(oldSession)) return;
@@ -82,7 +74,7 @@ export class NoteHistory implements ICollection {
locked
};
- await this.collection.addItem(session);
+ await this.collection.upsert(session);
await this.sessionContent.add(sessionId, content, locked);
await this.cleanup(noteId);
@@ -104,19 +96,18 @@ export class NoteHistory implements ICollection {
}
async content(sessionId: string) {
- const session = await this.collection.getItem(sessionId);
+ const session = await this.collection.get(sessionId);
if (!session || isDeleted(session)) return;
return await this.sessionContent.get(session.sessionContentId);
}
async remove(sessionId: string) {
- const session = await this.collection.getItem(sessionId);
+ const session = await this.collection.get(sessionId);
if (!session || isDeleted(session)) return;
await this._remove(session);
}
- async clearSessions(noteId: string) {
- if (!noteId) return;
+ async clearSessions(...noteIds: string[]) {
const history = await this.get(noteId);
for (const item of history) {
await this._remove(item);
@@ -124,12 +115,12 @@ export class NoteHistory implements ICollection {
}
private async _remove(session: HistorySession) {
- await this.collection.deleteItem(session.id);
+ await this.collection.delete(session.id);
await this.sessionContent.remove(session.sessionContentId);
}
async restore(sessionId: string) {
- const session = await this.collection.getItem(sessionId);
+ const session = await this.collection.get(sessionId);
if (!session || isDeleted(session)) return;
const content = await this.sessionContent.get(session.sessionContentId);
@@ -153,14 +144,14 @@ export class NoteHistory implements ICollection {
}
}
- async all() {
- return this.getSessions(this.collection.indexer.indices);
- }
+ // async all() {
+ // return this.getSessions(this.collection.indexer.indices);
+ // }
- private async getSessions(sessionIds: string[]): Promise {
- const items = await this.collection.getItems(sessionIds);
- return Object.values(items).filter(
- (a) => !isDeleted(a)
- ) as HistorySession[];
- }
+ // private async getSessions(sessionIds: string[]): Promise {
+ // const items = await this.collection.getItems(sessionIds);
+ // return Object.values(items).filter(
+ // (a) => !isDeleted(a)
+ // ) as HistorySession[];
+ // }
}
diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.ts
index 4c41a9391..a9ac92fbc 100644
--- a/packages/core/src/collections/notebooks.ts
+++ b/packages/core/src/collections/notebooks.ts
@@ -17,25 +17,20 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
-import { createNotebookModel } from "../models/notebook";
import { getId } from "../utils/id";
-import { CachedCollection } from "../database/cached-collection";
import Database from "../api";
-import { BaseTrashItem, Notebook, TrashOrItem, isTrashItem } from "../types";
+import { Notebook, TrashOrItem, isTrashItem } from "../types";
import { ICollection } from "./collection";
+import { SQLCollection } from "../database/sql-collection";
export class Notebooks implements ICollection {
name = "notebooks";
/**
* @internal
*/
- collection: CachedCollection<"notebooks", TrashOrItem>;
+ collection: SQLCollection<"notebooks", TrashOrItem>;
constructor(private readonly db: Database) {
- this.collection = new CachedCollection(
- db.storage,
- "notebooks",
- db.eventManager
- );
+ this.collection = new SQLCollection(db.sql, "notebooks", db.eventManager);
}
init() {
@@ -51,7 +46,7 @@ export class Notebooks implements ICollection {
//TODO reliably and efficiently check for duplicates.
const id = notebookArg.id || getId();
- const oldNotebook = this.collection.get(id);
+ const oldNotebook = await this.notebook(id);
if (oldNotebook && isTrashItem(oldNotebook))
throw new Error("Cannot modify trashed notebooks.");
@@ -64,7 +59,7 @@ export class Notebooks implements ICollection {
if (!mergedNotebook.title)
throw new Error("Notebook must contain a title.");
- const notebook: Notebook = {
+ await this.collection.upsert({
id,
type: "notebook",
title: mergedNotebook.title,
@@ -74,79 +69,87 @@ export class Notebooks implements ICollection {
dateCreated: mergedNotebook.dateCreated || Date.now(),
dateModified: mergedNotebook.dateModified || Date.now(),
dateEdited: Date.now()
- };
-
- await this.collection.add(notebook);
+ });
return id;
}
- get raw() {
- return this.collection.raw();
- }
+ // get raw() {
+ // return this.collection.raw();
+ // }
- get all() {
- return this.collection.items((note) =>
- isTrashItem(note) ? undefined : note
- ) as Notebook[];
- }
+ // get all() {
+ // return this.collection.items((note) =>
+ // isTrashItem(note) ? undefined : note
+ // ) as Notebook[];
+ // }
- get pinned() {
- return this.all.filter((item) => item.pinned === true);
- }
+ // get pinned() {
+ // return this.all.filter((item) => item.pinned === true);
+ // }
- get trashed() {
- return this.raw.filter((item) =>
- isTrashItem(item)
- ) as BaseTrashItem[];
- }
+ // get trashed() {
+ // return this.raw.filter((item) =>
+ // isTrashItem(item)
+ // ) as BaseTrashItem[];
+ // }
async pin(...ids: string[]) {
- for (const id of ids) {
- if (!this.exists(id)) continue;
- await this.add({ id, pinned: true });
- }
+ await this.collection.update(ids, { pinned: true });
}
async unpin(...ids: string[]) {
- for (const id of ids) {
- if (!this.exists(id)) continue;
- await this.add({ id, pinned: false });
- }
+ await this.collection.update(ids, { pinned: false });
}
- totalNotes(id: string) {
- let count = 0;
- const subNotebooks = this.db.relations.from(
- { type: "notebook", id },
- "notebook"
- );
- for (const notebook of subNotebooks) {
- count += this.totalNotes(notebook.to.id);
- }
- count += this.db.relations.from({ type: "notebook", id }, "note").length;
- return count;
+ async totalNotes(id: string) {
+ const result = await this.db
+ .sql()
+ .withRecursive(`subNotebooks(id)`, (eb) =>
+ eb
+ .selectNoFrom((eb) => eb.val(id).as("id"))
+ .unionAll((eb) =>
+ eb
+ .selectFrom(["relations", "subNotebooks"])
+ .select("relations.toId as id")
+ .where("toType", "==", "notebook")
+ .where("fromType", "==", "notebook")
+ .whereRef("fromId", "==", "subNotebooks.id")
+ .where("toId", "not in", this.db.trash.cache)
+ .$narrowType<{ id: string }>()
+ )
+ )
+ .selectFrom("relations")
+ .where("toType", "==", "note")
+ .where("fromType", "==", "notebook")
+ .where("fromId", "in", (eb) =>
+ eb.selectFrom("subNotebooks").select("subNotebooks.id")
+ )
+ .where("toId", "not in", this.db.trash.cache)
+ .select((eb) => eb.fn.count("id").as("totalNotes"))
+ .executeTakeFirst();
+
+ if (!result) return 0;
+ return result.totalNotes;
}
- notebook(idOrNotebook: string | Notebook) {
- const notebook =
- typeof idOrNotebook === "string"
- ? this.collection.get(idOrNotebook)
- : idOrNotebook;
+ async notebook(id: string) {
+ const notebook = await this.collection.get(id);
if (!notebook || isTrashItem(notebook)) return;
- return createNotebookModel(notebook, this.db);
+ return notebook;
}
exists(id: string) {
return this.collection.exists(id);
}
+ async remove(...ids: string[]) {
+ await this.db.trash.add("notebook", ids);
+ }
+
async delete(...ids: string[]) {
- for (const id of ids) {
- const notebook = this.collection.get(id);
- if (!notebook || isTrashItem(notebook)) continue;
- await this.collection.remove(id);
- await this.db.shortcuts?.remove(id);
- await this.db.trash?.add(notebook);
- }
+ await this.db.transaction(async () => {
+ await this.db.relations.unlinkOfType("notebook", ids);
+ await this.collection.softDelete(ids);
+ });
}
}
diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.ts
index 2d163d62a..2a668a000 100644
--- a/packages/core/src/collections/notes.ts
+++ b/packages/core/src/collections/notes.ts
@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
-import { createNoteModel } from "../models/note";
import { getId } from "../utils/id";
import { getContentFromData } from "../content-types";
import { NEWLINE_STRIP_REGEX, formatTitle } from "../utils/title-format";
@@ -26,17 +25,11 @@ import { Tiptap } from "../content-types/tiptap";
import { EMPTY_CONTENT, isUnencryptedContent } from "./content";
import { CHECK_IDS, checkIsUserPremium } from "../common";
import { buildFromTemplate } from "../utils/templates";
-import {
- Note,
- TrashOrItem,
- isTrashItem,
- isDeleted,
- BaseTrashItem
-} from "../types";
+import { Note, TrashOrItem, isTrashItem, isDeleted } from "../types";
import Database from "../api";
-import { CachedCollection } from "../database/cached-collection";
import { ICollection } from "./collection";
import { NoteContent } from "./session-content";
+import { SQLCollection } from "../database/sql-collection";
type ExportOptions = {
format: "html" | "md" | "txt" | "md-frontmatter";
@@ -50,17 +43,15 @@ export class Notes implements ICollection {
/**
* @internal
*/
- collection: CachedCollection<"notes", TrashOrItem>;
+ collection: SQLCollection<"notes", TrashOrItem>;
+ totalNotes = 0;
constructor(private readonly db: Database) {
- this.collection = new CachedCollection(
- db.storage,
- "notes",
- db.eventManager
- );
+ this.collection = new SQLCollection(db.sql, "notes", db.eventManager);
}
async init() {
await this.collection.init();
+ this.totalNotes = await this.collection.count();
}
async add(
@@ -70,9 +61,7 @@ export class Notes implements ICollection {
throw new Error("Please use db.notes.merge to merge remote notes.");
const id = item.id || getId();
- const oldNote = this.collection.get(id);
- if (oldNote && isTrashItem(oldNote))
- throw new Error("Cannot modify trashed notes.");
+ const oldNote = await this.note(id);
const note = {
...oldNote,
@@ -110,10 +99,10 @@ export class Notes implements ICollection {
});
}
- const noteTitle = this.getNoteTitle(note, oldNote, note.headline);
+ const noteTitle = await this.getNoteTitle(note, oldNote, note.headline);
if (oldNote && oldNote.title !== noteTitle) note.dateEdited = Date.now();
- await this.collection.add({
+ await this.collection.upsert({
id,
contentId: note.contentId,
type: "note",
@@ -136,54 +125,61 @@ export class Notes implements ICollection {
dateModified: note.dateModified || Date.now()
});
+ if (!oldNote) this.totalNotes++;
return id;
}
- note(idOrNote: string | Note) {
- if (!idOrNote) return;
- const note =
- typeof idOrNote === "object" ? idOrNote : this.collection.get(idOrNote);
- if (!note || isTrashItem(note)) return;
- return createNoteModel(note, this.db);
+ async note(idOrNote: string) {
+ const note = await this.collection.get(idOrNote);
+ if (!note || isTrashItem(note) || isDeleted(note)) return;
+ return note;
}
- get raw() {
- return this.collection.raw();
- }
+ // note(idOrNote: string | Note) {
+ // if (!idOrNote) return;
+ // const note =
+ // typeof idOrNote === "object" ? idOrNote : this.collection.get(idOrNote);
+ // if (!note || isTrashItem(note)) return;
+ // return createNoteModel(note, this.db);
+ // }
- get all() {
- return this.collection.items((note) =>
- isTrashItem(note) ? undefined : note
- ) as Note[];
- }
+ // get raw() {
+ // return this.collection.raw();
+ // }
- isTrashed(id: string) {
- return this.raw.find((item) => item.id === id && isTrashItem(item));
- }
+ // get all() {
+ // return this.collection.items((note) =>
+ // isTrashItem(note) ? undefined : note
+ // ) as Note[];
+ // }
- get trashed() {
- return this.raw.filter((item) =>
- isTrashItem(item)
- ) as BaseTrashItem[];
- }
+ // isTrashed(id: string) {
+ // return this.raw.find((item) => item.id === id && isTrashItem(item));
+ // }
- get pinned() {
- return this.all.filter((item) => item.pinned === true);
- }
+ // get trashed() {
+ // return this.raw.filter((item) =>
+ // isTrashItem(item)
+ // ) as BaseTrashItem[];
+ // }
- get conflicted() {
- return this.all.filter((item) => item.conflicted === true);
- }
+ // get pinned() {
+ // return this.all.filter((item) => item.pinned === true);
+ // }
- get favorites() {
- return this.all.filter((item) => item.favorite === true);
- }
+ // get conflicted() {
+ // return this.all.filter((item) => item.conflicted === true);
+ // }
- get locked(): Note[] {
- return this.all.filter(
- (item) => !isTrashItem(item) && item.locked === true
- ) as Note[];
- }
+ // get favorites() {
+ // return this.all.filter((item) => item.favorite === true);
+ // }
+
+ // get locked(): Note[] {
+ // return this.all.filter(
+ // (item) => !isTrashItem(item) && item.locked === true
+ // ) as Note[];
+ // }
exists(id: string) {
return this.collection.exists(id);
@@ -202,7 +198,7 @@ export class Notes implements ICollection {
if (format !== "txt" && !(await checkIsUserPremium(CHECK_IDS.noteExport)))
return false;
- const note = this.note(id);
+ const note = await this.note(id);
if (!note) return false;
if (!options.contentItem) {
@@ -239,15 +235,15 @@ export class Notes implements ICollection {
return options?.disableTemplate
? contentString
: buildFromTemplate(format, {
- ...note.data,
+ ...note,
content: contentString
});
}
async duplicate(...ids: string[]) {
for (const id of ids) {
- const note = this.collection.get(id);
- if (!note || isTrashItem(note)) continue;
+ const note = await this.note(id);
+ if (!note) continue;
const content = note.contentId
? await this.db.content.raw(note.contentId)
@@ -274,13 +270,16 @@ export class Notes implements ICollection {
});
if (!duplicateId) return;
- for (const notebook of this.db.relations
+ for (const notebook of await this.db.relations
.to(note, "notebook")
- .resolved()) {
- await this.db.relations.add(notebook, {
- id: duplicateId,
- type: "note"
- });
+ .get()) {
+ await this.db.relations.add(
+ { type: "notebook", id: notebook },
+ {
+ id: duplicateId,
+ type: "note"
+ }
+ );
}
return duplicateId;
@@ -288,24 +287,17 @@ export class Notes implements ICollection {
}
private async _delete(moveToTrash = true, ...ids: string[]) {
- for (const id of ids) {
- const item = this.collection.get(id);
- if (!item) continue;
- const itemData = clone(item);
-
- if (moveToTrash && !isTrashItem(itemData))
- await this.db.trash.add(itemData);
- else {
- await this.db.relations.unlinkAll(item, "tag");
- await this.db.relations.unlinkAll(item, "color");
- await this.db.relations.unlinkAll(item, "attachment");
- await this.db.relations.unlinkAll(item, "notebook");
-
- await this.collection.remove(id);
- if (itemData.contentId)
- await this.db.content.remove(itemData.contentId);
- }
+ if (moveToTrash) {
+ await this.db.trash.add("note", ids);
+ } else {
+ await this.db.transaction(async () => {
+ await this.db.relations.unlinkOfType("note", ids);
+ await this.collection.softDelete(ids);
+ await this.db.content.removeByNoteId(...ids);
+ });
}
+
+ this.totalNotes = Math.max(0, this.totalNotes - ids.length);
}
async addToNotebook(notebookId: string, ...noteIds: string[]) {
@@ -318,24 +310,31 @@ export class Notes implements ICollection {
}
async removeFromNotebook(notebookId: string, ...noteIds: string[]) {
- for (const noteId of noteIds) {
- await this.db.relations.unlink(
- { id: notebookId, type: "notebook" },
- { type: "note", id: noteId }
- );
- }
+ await this.db.transaction(async () => {
+ for (const noteId of noteIds) {
+ await this.db.relations.unlink(
+ { id: notebookId, type: "notebook" },
+ { type: "note", id: noteId }
+ );
+ }
+ });
}
async removeFromAllNotebooks(...noteIds: string[]) {
- for (const noteId of noteIds) {
- await this.db.relations.unlinkAll(
- { type: "note", id: noteId },
- "notebook"
- );
- }
+ await this.db.transaction(async () => {
+ for (const noteId of noteIds) {
+ await this.db.relations
+ .to({ type: "note", id: noteId }, "notebook")
+ .unlink();
+ }
+ });
}
- private getNoteTitle(note: Partial, oldNote?: Note, headline?: string) {
+ private async getNoteTitle(
+ note: Partial,
+ oldNote?: Note,
+ headline?: string
+ ) {
if (note.title && note.title.trim().length > 0) {
return note.title.replace(NEWLINE_STRIP_REGEX, " ");
} else if (
@@ -352,7 +351,7 @@ export class Notes implements ICollection {
this.db.settings.getDateFormat(),
this.db.settings.getTimeFormat(),
headline?.split(" ").splice(0, 10).join(" "),
- this.collection.count()
+ this.totalNotes
);
}
}
diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.ts
index 1f5fd388c..6f4095752 100644
--- a/packages/core/src/collections/relations.ts
+++ b/packages/core/src/collections/relations.ts
@@ -17,197 +17,113 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
-import { CachedCollection } from "../database/cached-collection";
import { makeId } from "../utils/id";
import { ICollection } from "./collection";
-import { Relation, ItemMap, ItemReference, MaybeDeletedItem } from "../types";
+import {
+ Relation,
+ ItemMap,
+ ItemReference,
+ ValueOf,
+ MaybeDeletedItem
+} from "../types";
import Database from "../api";
-
-type RelationsArray = Relation[] & {
- resolved: (limit?: number) => ItemMap[TType][];
- has: (id: string) => boolean;
-};
+import { SQLCollection } from "../database/sql-collection";
+import { DatabaseAccessor, DatabaseSchema } from "../database";
+import { SelectQueryBuilder } from "kysely";
export class Relations implements ICollection {
name = "relations";
- readonly collection: CachedCollection<"relations", Relation>;
+ readonly collection: SQLCollection<"relations", Relation>;
constructor(private readonly db: Database) {
- this.collection = new CachedCollection(
- db.storage,
- "relations",
- db.eventManager
- );
+ this.collection = new SQLCollection(db.sql, "relations", db.eventManager);
}
- init() {
- return this.collection.init();
- }
-
- async merge(relation: MaybeDeletedItem) {
- await this.collection.add(relation);
+ async init() {
+ // return this.collection.init();
}
async add(from: ItemReference, to: ItemReference) {
- if (
- this.all.find(
- (a) =>
- compareItemReference(a.from, from) && compareItemReference(a.to, to)
- )
- )
- return;
-
- const relation: Relation = {
+ await this.collection.upsert({
id: generateId(from, to),
type: "relation",
dateCreated: Date.now(),
dateModified: Date.now(),
- from: { id: from.id, type: from.type },
- to: { id: to.id, type: to.type }
- };
-
- await this.collection.add(relation);
+ fromId: from.id,
+ fromType: from.type,
+ toId: to.id,
+ toType: to.type
+ });
}
- from(
+ from(
reference: ItemReference,
type: TType
- ): RelationsArray {
- const relations =
- type === "note" || type === "notebook"
- ? this.all.filter(
- (a) =>
- compareItemReference(a.from, reference) &&
- a.to.type === type &&
- !this.db.trash.exists(a.to.id)
- )
- : this.all.filter(
- (a) => compareItemReference(a.from, reference) && a.to.type === type
- );
-
- Object.defineProperties(relations, {
- resolved: {
- writable: false,
- enumerable: false,
- configurable: false,
- value: (limit?: number) =>
- this.resolve(limit ? relations.slice(0, limit) : relations, "to")
- },
- has: {
- writable: false,
- enumerable: false,
- configurable: false,
- value: (id: string) => relations.some((rel) => rel.to.id === id)
- }
- });
- return relations as RelationsArray;
+ ) {
+ return new RelationsArray(
+ this.db.sql,
+ this.db.trash.cache,
+ reference,
+ type,
+ "from"
+ );
}
- to(
+ to(
reference: ItemReference,
type: TType
- ): RelationsArray {
- const relations =
- type === "note" || type === "notebook"
- ? this.all.filter(
- (a) =>
- compareItemReference(a.to, reference) &&
- a.from.type === type &&
- !this.db.trash.exists(a.from.id)
- )
- : this.all.filter(
- (a) => compareItemReference(a.to, reference) && a.from.type === type
- );
- Object.defineProperties(relations, {
- resolved: {
- writable: false,
- enumerable: false,
- configurable: false,
- value: (limit?: number) =>
- this.resolve(limit ? relations.slice(0, limit) : relations, "from")
- },
- has: {
- writable: false,
- enumerable: false,
- configurable: false,
- value: (id: string) => relations.some((rel) => rel.from.id === id)
- }
- });
- return relations as RelationsArray;
+ ) {
+ return new RelationsArray(
+ this.db.sql,
+ this.db.trash.cache,
+ reference,
+ type,
+ "to"
+ );
}
- get raw() {
- return this.collection.raw();
- }
+ // get raw() {
+ // return this.collection.raw();
+ // }
- get all(): Relation[] {
- return this.collection.items();
- }
+ // get all(): Relation[] {
+ // return this.collection.items();
+ // }
- relation(id: string) {
- return this.collection.get(id);
- }
+ // relation(id: string) {
+ // return this.collection.get(id);
+ // }
async remove(...ids: string[]) {
- for (const id of ids) {
- await this.collection.remove(id);
- }
+ await this.collection.softDelete(ids);
}
- async unlink(from: ItemReference, to: ItemReference) {
- const relation = this.all.find(
- (a) =>
- compareItemReference(a.from, from) && compareItemReference(a.to, to)
- );
- if (!relation) return;
-
- await this.remove(relation.id);
+ unlink(from: ItemReference, to: ItemReference) {
+ return this.remove(generateId(from, to));
}
- async unlinkAll(to: ItemReference, type?: keyof ItemMap) {
- for (const relation of this.all.filter(
- (a) => compareItemReference(a.to, to) && (!type || a.from.type === type)
- )) {
- await this.remove(relation.id);
- }
- }
-
- private resolve(relations: Relation[], resolveType: "from" | "to") {
- const items = [];
- for (const relation of relations) {
- const reference = resolveType === "from" ? relation.from : relation.to;
- let item = null;
- switch (reference.type) {
- case "tag":
- item = this.db.tags.tag(reference.id);
- break;
- case "color":
- item = this.db.colors.color(reference.id);
- break;
- case "reminder":
- item = this.db.reminders.reminder(reference.id);
- break;
- case "note": {
- const note = this.db.notes.note(reference.id);
- if (!note) continue;
- item = note.data;
- break;
- }
- case "notebook": {
- const notebook = this.db.notebooks.notebook(reference.id);
- if (!notebook) continue;
- item = notebook.data;
- break;
- }
- case "attachment": {
- const attachment = this.db.attachments.attachment(reference.id);
- if (!attachment) continue;
- item = attachment;
- break;
- }
- }
- if (item) items.push(item);
- }
- return items;
+ async unlinkOfType(type: keyof RelatableTable, ids?: string[]) {
+ await this.db
+ .sql()
+ .replaceInto("relations")
+ .columns(["id", "dateModified", "deleted"])
+ .expression((eb) =>
+ eb
+ .selectFrom("relations")
+ .where((eb) =>
+ eb.or([eb("fromType", "==", type), eb("toType", "==", type)])
+ )
+ .$if(ids !== undefined && ids.length > 0, (eb) =>
+ eb.where((eb) =>
+ eb.or([eb("fromId", "in", ids), eb("toId", "in", ids)])
+ )
+ )
+ .select((eb) => [
+ "relations.id",
+ eb.lit(Date.now()).as("dateModified"),
+ eb.lit(1).as("deleted")
+ ])
+ )
+ .execute();
}
}
@@ -229,3 +145,132 @@ function generateId(a: ItemReference, b: ItemReference) {
const str = `${a.id}${b.id}${a.type}${b.type}`;
return makeId(str);
}
+
+const TABLE_MAP = {
+ note: "notes",
+ notebook: "notebooks",
+ reminder: "reminders",
+ tag: "tags",
+ color: "colors",
+ attachment: "attachments"
+} as const;
+
+type RelatableTable = typeof TABLE_MAP;
+
+class RelationsArray {
+ private table: ValueOf = TABLE_MAP[this.type];
+
+ constructor(
+ private readonly sql: DatabaseAccessor,
+ private readonly trashIds: string[],
+ private readonly reference: ItemReference,
+ private readonly type: TType,
+ private readonly direction: "from" | "to"
+ ) {}
+
+ async resolve(limit?: number): Promise {
+ const items = await this.sql()
+ .selectFrom(this.table)
+ .where("id", "in", (b) =>
+ b
+ .selectFrom("relations")
+ .$call((eb) =>
+ this.buildRelationsQuery()(
+ eb as SelectQueryBuilder
+ )
+ )
+ )
+ .$if(limit !== undefined && limit > 0, (b) => b.limit(limit!))
+ .selectAll()
+ // TODO: check if we need to index deleted field.
+ .where("deleted", "is", null)
+ .execute();
+ return items as unknown as ItemMap[TType][];
+ }
+
+ async unlink() {
+ await this.sql()
+ .replaceInto("relations")
+ .columns(["id", "dateModified", "deleted"])
+ .expression((eb) =>
+ eb
+ .selectFrom("relations")
+ .$call(this.buildRelationsQuery())
+ .clearSelect()
+ .select((eb) => [
+ "relations.id",
+ eb.lit(Date.now()).as("dateModified"),
+ eb.lit(1).as("deleted")
+ ])
+ )
+ .execute();
+ }
+
+ async get() {
+ const ids = await this.sql()
+ .selectFrom("relations")
+ .$call(this.buildRelationsQuery())
+ .execute();
+ return ids.map((i) => i.id);
+ }
+
+ async count() {
+ const result = await this.sql()
+ .selectFrom("relations")
+ .$call(this.buildRelationsQuery())
+ .clearSelect()
+ .select((b) => b.fn.count("relations.id").as("count"))
+ .executeTakeFirst();
+ if (!result) return 0;
+ return result.count;
+ }
+
+ async has(id: string) {
+ const result = await this.sql()
+ .selectFrom("relations")
+ .$call(this.buildRelationsQuery())
+ .clearSelect()
+ .where(this.direction === "from" ? "toId" : "fromId", "==", id)
+ .select((b) => b.fn.count("id").as("count"))
+ .executeTakeFirst();
+ if (!result) return false;
+ return result.count > 0;
+ }
+
+ /**
+ * Build an optimized query for obtaining relations based on the given
+ * parameters. The resulting query uses a covering index (the most
+ * optimizable index) for obtaining relations.
+ */
+ private buildRelationsQuery() {
+ return (
+ builder: SelectQueryBuilder
+ ) => {
+ if (this.direction === "to") {
+ return builder
+ .where("fromType", "==", this.type)
+ .where("toType", "==", this.reference.type)
+ .where("toId", "==", this.reference.id)
+ .$if(
+ (this.type === "note" || this.type === "notebook") &&
+ this.trashIds.length > 0,
+ (b) => b.where("fromId", "not in", this.trashIds)
+ )
+ .select("relations.fromId as id")
+ .$narrowType<{ id: string }>();
+ } else {
+ return builder
+ .where("toType", "==", this.type)
+ .where("fromType", "==", this.reference.type)
+ .where("fromId", "==", this.reference.id)
+ .$if(
+ (this.type === "note" || this.type === "notebook") &&
+ this.trashIds.length > 0,
+ (b) => b.where("toId", "not in", this.trashIds)
+ )
+ .select("relations.toId as id")
+ .$narrowType<{ id: string }>();
+ }
+ };
+ }
+}
diff --git a/packages/core/src/collections/reminders.ts b/packages/core/src/collections/reminders.ts
index ff3e2e41b..e7c570c50 100644
--- a/packages/core/src/collections/reminders.ts
+++ b/packages/core/src/collections/reminders.ts
@@ -25,9 +25,9 @@ import isYesterday from "dayjs/plugin/isYesterday";
import { TimeFormat, formatDate } from "../utils/date";
import { getId } from "../utils/id";
import { ICollection } from "./collection";
-import { CachedCollection } from "../database/cached-collection";
import { Reminder } from "../types";
import Database from "../api";
+import { SQLCollection } from "../database/sql-collection";
dayjs.extend(isTomorrow);
dayjs.extend(isSameOrBefore);
@@ -36,13 +36,9 @@ dayjs.extend(isToday);
export class Reminders implements ICollection {
name = "reminders";
- readonly collection: CachedCollection<"reminders", Reminder>;
+ readonly collection: SQLCollection<"reminders", Reminder>;
constructor(private readonly db: Database) {
- this.collection = new CachedCollection(
- db.storage,
- "reminders",
- db.eventManager
- );
+ this.collection = new SQLCollection(db.sql, "reminders", db.eventManager);
}
async init() {
@@ -65,7 +61,7 @@ export class Reminders implements ICollection {
if (!reminder.date || !reminder.title)
throw new Error("date and title are required in a reminder.");
- await this.collection.add({
+ await this.collection.upsert({
id,
type: "reminder",
dateCreated: reminder.dateCreated || Date.now(),
@@ -81,16 +77,16 @@ export class Reminders implements ICollection {
disabled: reminder.disabled,
snoozeUntil: reminder.snoozeUntil
});
- return reminder.id;
+ return id;
}
- get raw() {
- return this.collection.raw();
- }
+ // get raw() {
+ // return this.collection.raw();
+ // }
- get all() {
- return this.collection.items();
- }
+ // get all() {
+ // return this.collection.items();
+ // }
exists(itemId: string) {
return this.collection.exists(itemId);
@@ -101,9 +97,7 @@ export class Reminders implements ICollection {
}
async remove(...reminderIds: string[]) {
- for (const id of reminderIds) {
- await this.collection.remove(id);
- }
+ await this.collection.softDelete(reminderIds);
}
}
diff --git a/packages/core/src/collections/session-content.ts b/packages/core/src/collections/session-content.ts
index 0936ab6f7..3c3d90263 100644
--- a/packages/core/src/collections/session-content.ts
+++ b/packages/core/src/collections/session-content.ts
@@ -22,9 +22,9 @@ import { tinyToTiptap } from "../migrations";
import { makeSessionContentId } from "../utils/id";
import { ICollection } from "./collection";
import { isCipher } from "../database/crypto";
-import { IndexedCollection } from "../database/indexed-collection";
import Database from "../api";
import { ContentType, SessionContentItem, isDeleted } from "../types";
+import { SQLCollection } from "../database/sql-collection";
export type NoteContent = {
data: TLocked extends true ? Cipher<"base64"> : string;
@@ -33,13 +33,13 @@ export type NoteContent = {
export class SessionContent implements ICollection {
name = "sessioncontent";
- private readonly collection: IndexedCollection<
+ private readonly collection: SQLCollection<
"sessioncontent",
SessionContentItem
>;
constructor(private readonly db: Database) {
- this.collection = new IndexedCollection(
- db.storage,
+ this.collection = new SQLCollection(
+ db.sql,
"sessioncontent",
db.eventManager
);
@@ -49,10 +49,6 @@ export class SessionContent implements ICollection {
await this.collection.init();
}
- async merge(item: SessionContentItem) {
- await this.collection.addItem(item);
- }
-
async add(
sessionId: string,
content: NoteContent,
@@ -64,7 +60,7 @@ export class SessionContent implements ICollection {
? content.data
: await this.db.compressor().compress(content.data);
- await this.collection.addItem({
+ await this.collection.upsert({
type: "sessioncontent",
id: makeSessionContentId(sessionId),
data,
@@ -78,7 +74,7 @@ export class SessionContent implements ICollection {
}
async get(sessionContentId: string) {
- const session = await this.collection.getItem(sessionContentId);
+ const session = await this.collection.get(sessionContentId);
if (!session || isDeleted(session)) return;
if (
@@ -93,7 +89,7 @@ export class SessionContent implements ICollection {
tinyToTiptap(await this.db.compressor().decompress(session.data))
);
session.contentType = "tiptap";
- await this.collection.addItem(session);
+ await this.collection.upsert(session);
}
return {
@@ -106,13 +102,13 @@ export class SessionContent implements ICollection {
}
async remove(sessionContentId: string) {
- await this.collection.deleteItem(sessionContentId);
+ await this.collection.delete(sessionContentId);
}
- async all() {
- const indices = this.collection.indexer.indices;
- const items = await this.collection.getItems(indices);
+ // async all() {
+ // const indices = this.collection.indexer.indices;
+ // const items = await this.collection.getItems(indices);
- return Object.values(items);
- }
+ // return Object.values(items);
+ // }
}
diff --git a/packages/core/src/collections/settings.ts b/packages/core/src/collections/settings.ts
index 6cb0042fb..29d3367a4 100644
--- a/packages/core/src/collections/settings.ts
+++ b/packages/core/src/collections/settings.ts
@@ -29,8 +29,8 @@ import {
TrashCleanupInterval
} from "../types";
import { ICollection } from "./collection";
-import { CachedCollection } from "../database/cached-collection";
import { TimeFormat } from "../utils/date";
+import { SQLCachedCollection } from "../database/sql-cached-collection";
const DEFAULT_GROUP_OPTIONS = (key: GroupingKey) =>
({
@@ -66,12 +66,12 @@ const defaultSettings: SettingItemMap = {
};
export class Settings implements ICollection {
- name = "settingsv2";
- readonly collection: CachedCollection<"settingsv2", SettingItem>;
+ name = "settings";
+ readonly collection: SQLCachedCollection<"settings", SettingItem>;
constructor(db: Database) {
- this.collection = new CachedCollection(
- db.storage,
- "settingsv2",
+ this.collection = new SQLCachedCollection(
+ db.sql,
+ "settings",
db.eventManager
);
}
@@ -80,19 +80,19 @@ export class Settings implements ICollection {
return this.collection.init();
}
- get raw() {
- return this.collection.raw();
- }
+ // get raw() {
+ // return this.collection.raw();
+ // }
private async set(
key: TKey,
value: SettingItemMap[TKey]
) {
const id = makeId(key);
- const oldItem = this.collection.get(id);
+ const oldItem = await this.collection.get(id);
if (oldItem && oldItem.key !== key) throw new Error("Key conflict.");
- await this.collection.add({
+ await this.collection.upsert({
id,
key,
value,
diff --git a/packages/core/src/collections/shortcuts.ts b/packages/core/src/collections/shortcuts.ts
index d6cf7abb5..aa4269761 100644
--- a/packages/core/src/collections/shortcuts.ts
+++ b/packages/core/src/collections/shortcuts.ts
@@ -18,20 +18,16 @@ along with this program. If not, see .
*/
import Database from "../api";
-import { CachedCollection } from "../database/cached-collection";
-import { Notebook, Shortcut, Tag, Topic } from "../types";
+import { SQLCollection } from "../database/sql-collection";
+import { Shortcut } from "../types";
import { ICollection } from "./collection";
const ALLOWED_SHORTCUT_TYPES = ["notebook", "topic", "tag"];
export class Shortcuts implements ICollection {
name = "shortcuts";
- readonly collection: CachedCollection<"shortcuts", Shortcut>;
+ readonly collection: SQLCollection<"shortcuts", Shortcut>;
constructor(private readonly db: Database) {
- this.collection = new CachedCollection(
- db.storage,
- "shortcuts",
- db.eventManager
- );
+ this.collection = new SQLCollection(db.sql, "shortcuts", db.eventManager);
}
init() {
@@ -45,11 +41,15 @@ export class Shortcuts implements ICollection {
"Please use db.shortcuts.merge to merge remote shortcuts."
);
- if (shortcut.item && !ALLOWED_SHORTCUT_TYPES.includes(shortcut.item.type))
+ if (
+ shortcut.itemId &&
+ shortcut.itemType &&
+ !ALLOWED_SHORTCUT_TYPES.includes(shortcut.itemType)
+ )
throw new Error("Cannot create a shortcut for this type of item.");
- const oldShortcut = shortcut.item
- ? this.shortcut(shortcut.item.id)
+ const oldShortcut = shortcut.itemId
+ ? this.shortcut(shortcut.itemId)
: shortcut.id
? this.shortcut(shortcut.id)
: null;
@@ -64,65 +64,72 @@ export class Shortcuts implements ICollection {
const id = shortcut.id || shortcut.item.id;
- await this.collection.add({
+ await this.collection.upsert({
id,
type: "shortcut",
- item: shortcut.item,
+ itemId: shortcut.itemId,
+ itemType: shortcut.itemType,
+ // item: shortcut.item,
dateCreated: shortcut.dateCreated || Date.now(),
dateModified: shortcut.dateModified || Date.now(),
- sortIndex: this.collection.count()
+ sortIndex: await this.collection.count()
});
return id;
}
- get raw() {
- return this.collection.raw();
- }
+ // get raw() {
+ // return this.collection.raw();
+ // }
- get all() {
- return this.collection.items();
- }
+ // get all() {
+ // return this.collection.items();
+ // }
- get resolved() {
- return this.all.reduce((prev, shortcut) => {
- const {
- item: { id }
- } = shortcut;
-
- let item: Notebook | Topic | Tag | null | undefined = null;
- switch (shortcut.item.type) {
- case "notebook": {
- const notebook = this.db.notebooks.notebook(id);
- item = notebook ? notebook.data : null;
- break;
- }
- case "tag":
- item = this.db.tags.tag(id);
- break;
- }
- if (item) prev.push(item);
- return prev;
- }, [] as (Notebook | Topic | Tag)[]);
+ async get() {
+ // return this.all.reduce((prev, shortcut) => {
+ // const {
+ // item: { id }
+ // } = shortcut;
+ // let item: Notebook | Topic | Tag | null | undefined = null;
+ // switch (shortcut.item.type) {
+ // case "notebook": {
+ // const notebook = this.db.notebooks.notebook(id);
+ // item = notebook ? notebook.data : null;
+ // break;
+ // }
+ // case "tag":
+ // item = this.db.tags.tag(id);
+ // break;
+ // }
+ // if (item) prev.push(item);
+ // return prev;
+ // }, [] as (Notebook | Topic | Tag)[]);
}
exists(id: string) {
- return !!this.shortcut(id);
+ return this.collection.exists(id);
}
shortcut(id: string) {
- return this.all.find(
- (shortcut) => shortcut.item.id === id || shortcut.id === id
- );
+ return this.collection.get(id);
}
async remove(...shortcutIds: string[]) {
- const shortcuts = this.all.filter(
- (shortcut) =>
- shortcutIds.includes(shortcut.item.id) ||
- shortcutIds.includes(shortcut.id)
- );
- for (const { id } of shortcuts) {
- await this.collection.remove(id);
- }
+ await this.collection.softDelete(shortcutIds);
+ // await this.db
+ // .sql()
+ // .deleteFrom("shortcuts")
+ // .where((eb) =>
+ // eb.or([eb("id", "in", shortcutIds), eb("itemId", "in", shortcutIds)])
+ // )
+ // .execute();
+ // const shortcuts = this.all.filter(
+ // (shortcut) =>
+ // shortcutIds.includes(shortcut.item.id) ||
+ // shortcutIds.includes(shortcut.id)
+ // );
+ // for (const { id } of shortcuts) {
+ // await this.collection.remove(id);
+ // }
}
}
diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.ts
index 3a8ee704d..5b7e8a3af 100644
--- a/packages/core/src/collections/tags.ts
+++ b/packages/core/src/collections/tags.ts
@@ -18,16 +18,16 @@ along with this program. If not, see .
*/
import { getId } from "../utils/id";
-import { CachedCollection } from "../database/cached-collection";
-import { MaybeDeletedItem, Tag } from "../types";
+import { Tag } from "../types";
import Database from "../api";
import { ICollection } from "./collection";
+import { SQLCollection } from "../database/sql-collection";
export class Tags implements ICollection {
name = "tags";
- readonly collection: CachedCollection<"tags", Tag>;
+ readonly collection: SQLCollection<"tags", Tag>;
constructor(private readonly db: Database) {
- this.collection = new CachedCollection(db.storage, "tags", db.eventManager);
+ this.collection = new SQLCollection(db.sql, "tags", db.eventManager);
}
init() {
@@ -38,58 +38,46 @@ export class Tags implements ICollection {
return this.collection.get(id);
}
- find(idOrTitle: string) {
- return this.all.find(
- (tag) => tag.title === idOrTitle || tag.id === idOrTitle
- );
- }
-
- async merge(remoteTag: MaybeDeletedItem) {
- if (!remoteTag) return;
-
- const localTag = this.collection.get(remoteTag.id);
- if (!localTag || remoteTag.dateModified > localTag.dateModified)
- await this.collection.add(remoteTag);
- }
+ // find(idOrTitle: string) {
+ // return this.all.find(
+ // (tag) => tag.title === idOrTitle || tag.id === idOrTitle
+ // );
+ // }
async add(item: Partial) {
if (item.remote)
throw new Error("Please use db.tags.merge to merge remote tags.");
const id = item.id || getId(item.dateCreated);
- const oldTag = this.tag(id);
+ const oldTag = await this.tag(id);
item.title = item.title ? Tags.sanitize(item.title) : item.title;
if (!item.title && !oldTag?.title) throw new Error("Title is required.");
- const tag: Tag = {
+ await this.collection.upsert({
id,
dateCreated: item.dateCreated || oldTag?.dateCreated || Date.now(),
dateModified: item.dateModified || oldTag?.dateModified || Date.now(),
title: item.title || oldTag?.title || "",
type: "tag",
remote: false
- };
- await this.collection.add(tag);
- return tag.id;
+ });
+ return id;
}
- get raw() {
- return this.collection.raw();
- }
+ // get raw() {
+ // return this.collection.raw();
+ // }
- get all() {
- return this.collection.items();
- }
+ // get all() {
+ // return this.collection.items();
+ // }
- async remove(id: string) {
- await this.collection.remove(id);
- await this.db.relations.cleanup();
- }
-
- async delete(id: string) {
- await this.collection.delete(id);
- await this.db.relations.cleanup();
+ async remove(...ids: string[]) {
+ await this.db.transaction(async () => {
+ await this.db.relations.unlinkOfType("tag", ids);
+ await this.collection.softDelete(ids);
+ });
}
exists(id: string) {
diff --git a/packages/core/src/collections/trash.ts b/packages/core/src/collections/trash.ts
index dd78c1286..a277e6207 100644
--- a/packages/core/src/collections/trash.ts
+++ b/packages/core/src/collections/trash.ts
@@ -19,23 +19,6 @@ along with this program. If not, see .
import dayjs from "dayjs";
import Database from "../api";
-import {
- BaseTrashItem,
- Note,
- Notebook,
- TrashItem,
- isTrashItem
-} from "../types";
-
-function toTrashItem(item: T): BaseTrashItem {
- return {
- ...item,
- id: item.id,
- type: "trash",
- itemType: item.type,
- dateDeleted: Date.now()
- };
-}
export default class Trash {
collections = ["notes", "notebooks"] as const;
@@ -44,96 +27,122 @@ export default class Trash {
async init() {
await this.cleanup();
- this.cache = this.all.map((t) => t.id);
+ const result = await this.db
+ .sql()
+ .selectNoFrom((eb) => [
+ eb
+ .selectFrom("notes")
+ .where("type", "==", "trash")
+ .select("id")
+ .as("id"),
+ eb
+ .selectFrom("notebooks")
+ .where("type", "==", "trash")
+ .select("id")
+ .as("id")
+ ])
+ .execute();
+
+ this.cache = result.reduce((ids, item) => {
+ if (item.id) ids.push(item.id);
+ return ids;
+ }, [] as string[]);
}
async cleanup() {
- const now = dayjs().unix();
const duration = this.db.settings.getTrashCleanupInterval();
if (duration === -1 || !duration) return;
- for (const item of this.all) {
- if (
- isTrashItem(item) &&
- item.dateDeleted &&
- dayjs(item.dateDeleted).add(duration, "days").unix() > now
- )
- continue;
- await this.delete(item.id);
- }
+
+ const maxMs = dayjs().subtract(duration, "days").toDate().getTime();
+ const expiredItems = await this.db
+ .sql()
+ .selectNoFrom((eb) => [
+ eb
+ .selectFrom("notes")
+ .where("type", "==", "trash")
+ .where("dateDeleted", "<=", maxMs)
+ .select("id")
+ .as("noteId"),
+ eb
+ .selectFrom("notebooks")
+ .where("type", "==", "trash")
+ .where("dateDeleted", "<=", maxMs)
+ .select("id")
+ .as("notebookId")
+ ])
+ .execute();
+ const { noteIds, notebookIds } = expiredItems.reduce(
+ (ids, item) => {
+ if (item.noteId) ids.noteIds.push(item.noteId);
+ if (item.notebookId) ids.notebookIds.push(item.notebookId);
+ return ids;
+ },
+ { noteIds: [] as string[], notebookIds: [] as string[] }
+ );
+ await this.delete("note", noteIds);
+ await this.delete("note", notebookIds);
}
- get all(): TrashItem[] {
- const trashItems: TrashItem[] = [];
- for (const key of this.collections) {
- const collection = this.db[key];
- trashItems.push(...collection.trashed);
+ async add(type: "note" | "notebook", ids: string[]) {
+ if (type === "note") {
+ await this.db.notes.collection.update(ids, {
+ type: "trash",
+ itemType: "note",
+ dateDeleted: Date.now()
+ });
+ } else if (type === "notebook") {
+ await this.db.notebooks.collection.update(ids, {
+ type: "trash",
+ itemType: "notebook",
+ dateDeleted: Date.now()
+ });
}
- return trashItems;
+ this.cache.push(...ids);
}
- private getItem(id: string) {
- for (const key of this.collections) {
- const collection = this.db[key].collection;
- const item = collection.get(id);
- if (item && isTrashItem(item)) return [item, collection] as const;
+ async delete(type: "note" | "notebook", ids: string[]) {
+ if (type === "note") {
+ await this.db.content.removeByNoteId(...ids);
+ await this.db.noteHistory.clearSessions(...ids);
+ await this.db.notes.delete(...ids);
+ } else if (type === "notebook") {
+ await this.db.relations.unlinkOfType("notebook", ids);
+ await this.db.notebooks.delete(...ids);
}
- return [] as const;
+ ids.forEach((id) => this.cache.splice(this.cache.indexOf(id), 1));
}
- async add(item: Note | Notebook) {
- if (item.type === "note") {
- await this.db.notes.collection.update(toTrashItem(item));
- } else if (item.type === "notebook") {
- await this.db.notebooks.collection.update(toTrashItem(item));
- }
- this.cache.push(item.id);
- }
-
- async delete(...ids: string[]) {
- for (const id of ids) {
- const [item, collection] = this.getItem(id);
- if (!item || !collection) continue;
- if (item.itemType === "note") {
- if (item.contentId) await this.db.content.remove(item.contentId);
- await this.db.noteHistory.clearSessions(id);
- } else if (item.itemType === "notebook") {
- await this.db.relations.unlinkAll({ type: "notebook", id: item.id });
- }
- await collection.remove(id);
- this.cache.splice(this.cache.indexOf(id), 1);
- }
- }
-
- async restore(...ids: string[]) {
- for (const id of ids) {
- const [item] = this.getItem(id);
- if (!item) continue;
- if (item.itemType === "note") {
- await this.db.notes.collection.update({ ...item, type: "note" });
- } else if (item.itemType === "notebook") {
- await this.db.notebooks.collection.update({
- ...item,
- type: "notebook"
- });
- }
- this.cache.splice(this.cache.indexOf(id), 1);
+ async restore(type: "note" | "notebook", ids: string[]) {
+ if (type === "note") {
+ await this.db.notes.collection.update(ids, {
+ type: "note",
+ dateDeleted: null,
+ itemType: null
+ });
+ } else {
+ await this.db.notebooks.collection.update(ids, {
+ type: "notebook",
+ dateDeleted: null,
+ itemType: null
+ });
}
+ ids.forEach((id) => this.cache.splice(this.cache.indexOf(id), 1));
}
async clear() {
- for (const item of this.all) {
- await this.delete(item.id);
- }
+ // for (const item of this.all) {
+ // await this.delete(item.id);
+ // }
this.cache = [];
}
- synced(id: string) {
- const [item] = this.getItem(id);
- if (item && item.itemType === "note") {
- const { contentId } = item;
- return !contentId || this.db.content.exists(contentId);
- } else return true;
- }
+ // synced(id: string) {
+ // // const [item] = this.getItem(id);
+ // if (item && item.itemType === "note") {
+ // const { contentId } = item;
+ // return !contentId || this.db.content.exists(contentId);
+ // } else return true;
+ // }
/**
*
diff --git a/packages/core/src/database/fs.ts b/packages/core/src/database/fs.ts
index ae8ae0787..f635d1d4a 100644
--- a/packages/core/src/database/fs.ts
+++ b/packages/core/src/database/fs.ts
@@ -25,13 +25,12 @@ import {
StorageAccessor
} from "../interfaces";
import { DataFormat, SerializedKey } from "@notesnook/crypto/dist/src/types";
-import { AttachmentMetadata } from "../types";
import { EV, EVENTS } from "../common";
export type FileStorageAccessor = () => FileStorage;
export type DownloadableFile = {
filename: string;
- metadata: AttachmentMetadata;
+ // metadata: AttachmentMetadata;
chunkSize: number;
};
export type QueueItem = DownloadableFile & {
@@ -57,7 +56,7 @@ export class FileStorage {
this.downloads.set(groupId, files);
for (const file of files as QueueItem[]) {
- const { filename, metadata, chunkSize } = file;
+ const { filename, chunkSize } = file;
if (await this.exists(filename)) {
current++;
EV.publish(EVENTS.fileDownloaded, {
@@ -71,7 +70,6 @@ export class FileStorage {
const url = `${hosts.API_HOST}/s3?name=${filename}`;
const { execute, cancel } = this.fs.downloadFile(filename, {
- metadata,
url,
chunkSize,
headers: { Authorization: `Bearer ${token}` }
@@ -106,11 +104,10 @@ export class FileStorage {
this.uploads.set(groupId, files);
for (const file of files as QueueItem[]) {
- const { filename, chunkSize, metadata } = file;
+ const { filename, chunkSize } = file;
const url = `${hosts.API_HOST}/s3?name=${filename}`;
const { execute, cancel } = this.fs.uploadFile(filename, {
chunkSize,
- metadata,
url,
headers: { Authorization: `Bearer ${token}` }
});
@@ -142,21 +139,15 @@ export class FileStorage {
this.uploads.delete(groupId);
}
- async downloadFile(
- groupId: string,
- filename: string,
- chunkSize: number,
- metadata: AttachmentMetadata
- ) {
+ async downloadFile(groupId: string, filename: string, chunkSize: number) {
const url = `${hosts.API_HOST}/s3?name=${filename}`;
const token = await this.tokenManager.getAccessToken();
const { execute, cancel } = this.fs.downloadFile(filename, {
- metadata,
url,
chunkSize,
headers: { Authorization: `Bearer ${token}` }
});
- this.downloads.set(groupId, [{ cancel, filename, chunkSize, metadata }]);
+ this.downloads.set(groupId, [{ cancel, filename, chunkSize }]);
const result = await execute();
this.downloads.delete(groupId);
return result;
diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.ts
new file mode 100644
index 000000000..83c8c8d94
--- /dev/null
+++ b/packages/core/src/database/index.ts
@@ -0,0 +1,191 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+*/
+
+import {
+ Migrator,
+ Kysely,
+ SqliteAdapter,
+ SqliteIntrospector,
+ SqliteQueryCompiler,
+ sql,
+ Driver,
+ KyselyPlugin,
+ PluginTransformQueryArgs,
+ PluginTransformResultArgs,
+ QueryResult,
+ UnknownRow,
+ RootOperationNode,
+ OperationNodeTransformer,
+ ValueNode,
+ PrimitiveValueListNode,
+ Transaction,
+ ColumnType
+} from "kysely";
+import {
+ Attachment,
+ Color,
+ ContentItem,
+ HistorySession,
+ Note,
+ Notebook,
+ Relation,
+ Reminder,
+ SessionContentItem,
+ SettingItem,
+ Shortcut,
+ Tag,
+ TrashOrItem,
+ ValueOf
+} from "../types";
+import { NNMigrationProvider } from "./migrations";
+
+// type FilteredKeys = {
+// [P in keyof T]: T[P] extends U ? P : never;
+// }[keyof T];
+type SQLiteValue = T extends string | number | boolean | Array
+ ? T
+ : T extends object | Array
+ ? ColumnType
+ : never;
+export type SQLiteItem = {
+ [P in keyof T]?: T[P] | null;
+} & { id: string };
+
+export interface DatabaseSchema {
+ notes: SQLiteItem>; //| SQLiteItem>;
+ content: SQLiteItem;
+ relations: SQLiteItem;
+ notebooks: SQLiteItem>;
+ attachments: SQLiteItem;
+ tags: SQLiteItem;
+ colors: SQLiteItem;
+ reminders: SQLiteItem;
+ settings: SQLiteItem;
+ notehistory: SQLiteItem;
+ sessioncontent: SQLiteItem;
+ shortcuts: SQLiteItem;
+}
+
+type AsyncOrSyncResult = Async extends true
+ ? Promise
+ : Response;
+
+export interface DatabaseCollection {
+ clear(): Promise;
+ init(): Promise;
+ upsert(item: T): Promise;
+ softDelete(ids: string[]): Promise;
+ delete(ids: string[]): Promise;
+ exists(id: string): AsyncOrSyncResult;
+ count(): AsyncOrSyncResult;
+ get(id: string): AsyncOrSyncResult;
+ put(items: (T | undefined)[]): Promise;
+ update(ids: string[], partial: Partial): Promise;
+}
+
+export type DatabaseAccessor = () =>
+ | Kysely
+ | Transaction;
+
+type FilterBooleanProperties = keyof {
+ [K in keyof T as T[K] extends boolean | undefined | null ? K : never]: T[K];
+};
+
+type BooleanFields = ValueOf<{
+ [D in keyof DatabaseSchema]: FilterBooleanProperties;
+}>;
+
+const BooleanProperties: BooleanFields[] = [
+ "compressed",
+ "conflicted",
+ "deleted",
+ "disabled",
+ "favorite",
+ "localOnly",
+ "locked",
+ "migrated",
+ "pinned",
+ "readonly",
+ "remote",
+ "synced"
+];
+
+export async function createDatabase(driver: Driver) {
+ const db = new Kysely({
+ dialect: {
+ createAdapter: () => new SqliteAdapter(),
+ createDriver: () => driver,
+ createIntrospector: (db) => new SqliteIntrospector(db),
+ createQueryCompiler: () => new SqliteQueryCompiler()
+ },
+ plugins: [new SqliteBooleanPlugin()]
+ });
+
+ const migrator = new Migrator({
+ db,
+ provider: new NNMigrationProvider()
+ });
+
+ await sql`PRAGMA journal_mode = WAL`.execute(db);
+ await sql`PRAGMA synchronous = normal`.execute(db);
+
+ await migrator.migrateToLatest();
+
+ return db;
+}
+
+export class SqliteBooleanPlugin implements KyselyPlugin {
+ readonly #transformer = new SqliteBooleanTransformer();
+
+ transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
+ return this.#transformer.transformNode(args.node);
+ }
+
+ transformResult(
+ args: PluginTransformResultArgs
+ ): Promise> {
+ for (const row of args.result.rows) {
+ for (const key of BooleanProperties) {
+ const value = row[key];
+ row[key] = value === 1 ? true : false;
+ }
+ }
+ return Promise.resolve(args.result);
+ }
+}
+
+class SqliteBooleanTransformer extends OperationNodeTransformer {
+ transformValue(node: ValueNode): ValueNode {
+ return {
+ ...super.transformValue(node),
+ value: typeof node.value === "boolean" ? (node.value ? 1 : 0) : node.value
+ };
+ }
+
+ protected transformPrimitiveValueList(
+ node: PrimitiveValueListNode
+ ): PrimitiveValueListNode {
+ return {
+ ...super.transformPrimitiveValueList(node),
+ values: node.values.map((value) =>
+ typeof value === "boolean" ? (value ? 1 : 0) : value
+ )
+ };
+ }
+}
diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.ts
new file mode 100644
index 000000000..d4f0d3b89
--- /dev/null
+++ b/packages/core/src/database/migrations.ts
@@ -0,0 +1,218 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+*/
+
+import { CreateTableBuilder, Migration, MigrationProvider, sql } from "kysely";
+
+export class NNMigrationProvider implements MigrationProvider {
+ async getMigrations(): Promise> {
+ return {
+ "1": {
+ async up(db) {
+ await db.schema
+ .createTable("notes")
+ .modifyEnd(sql`without rowid`)
+ .$call(addBaseColumns)
+ .addColumn("title", "text")
+ .addColumn("headline", "text")
+ .addColumn("contentId", "text")
+ .addColumn("pinned", "boolean")
+ .addColumn("locked", "boolean")
+ .addColumn("favorite", "boolean")
+ .addColumn("localOnly", "boolean")
+ .addColumn("conflicted", "boolean")
+ .addColumn("readonly", "boolean")
+ .addColumn("dateEdited", "integer")
+ .addColumn("dateDeleted", "integer")
+ .addColumn("itemType", "text")
+ .addForeignKeyConstraint(
+ "note_has_content",
+ ["contentId"],
+ "content",
+ ["id"],
+ (b) => b.onDelete("restrict").onUpdate("restrict")
+ )
+ .execute();
+
+ await db.schema
+ .createTable("content")
+ .modifyEnd(sql`without rowid`)
+ .$call(addBaseColumns)
+ .addColumn("noteId", "text")
+ .addColumn("data", "text")
+ .addColumn("localOnly", "boolean")
+ .addColumn("conflicted", "text")
+ .addColumn("sessionId", "text")
+ .addColumn("dateEdited", "integer")
+ .addColumn("dateResolved", "integer")
+ .execute();
+
+ await db.schema
+ .createTable("notebooks")
+ .modifyEnd(sql`without rowid`)
+ .$call(addBaseColumns)
+ .addColumn("title", "text")
+ .addColumn("description", "text")
+ .addColumn("dateEdited", "text")
+ .addColumn("pinned", "boolean")
+ .execute();
+
+ await db.schema
+ .createTable("tags")
+ .modifyEnd(sql`without rowid`)
+ .$call(addBaseColumns)
+ .addColumn("title", "text")
+ .execute();
+
+ await db.schema
+ .createTable("colors")
+ .modifyEnd(sql`without rowid`)
+ .$call(addBaseColumns)
+ .addColumn("title", "text")
+ .addColumn("colorCode", "text")
+ .execute();
+
+ await db.schema
+ .createTable("relations")
+ .modifyEnd(sql`without rowid`)
+ .$call(addBaseColumns)
+ .addColumn("fromType", "text")
+ .addColumn("fromId", "text")
+ .addColumn("toType", "text")
+ .addColumn("toId", "text")
+ .execute();
+
+ await db.schema
+ .createTable("shortcuts")
+ .modifyEnd(sql`without rowid`)
+ .$call(addBaseColumns)
+ .addColumn("sortIndex", "integer")
+ .addColumn("itemId", "text")
+ .addColumn("itemType", "text")
+ .execute();
+
+ await db.schema
+ .createTable("reminders")
+ .modifyEnd(sql`without rowid`)
+ .$call(addBaseColumns)
+ .addColumn("title", "text")
+ .addColumn("description", "text")
+ .addColumn("priority", "text")
+ .addColumn("date", "integer")
+ .addColumn("mode", "text")
+ .addColumn("recurringMode", "text")
+ .addColumn("selectedDays", "blob")
+ .addColumn("localOnly", "boolean")
+ .addColumn("disabled", "boolean")
+ .addColumn("snoozeUntil", "integer")
+ .execute();
+
+ await db.schema
+ .createTable("attachments")
+ .modifyEnd(sql`without rowid`)
+ .$call(addBaseColumns)
+ .addColumn("iv", "text")
+ .addColumn("salt", "text")
+ .addColumn("size", "integer")
+ .addColumn("alg", "text")
+ .addColumn("encryptionKey", "text")
+ .addColumn("chunkSize", "integer")
+ .addColumn("hash", "text", (c) => c.unique())
+ .addColumn("hashType", "text")
+ .addColumn("mimeType", "text")
+ .addColumn("filename", "text")
+ .addColumn("dateDeleted", "integer")
+ .addColumn("dateUploaded", "integer")
+ .addColumn("failed", "text")
+ .execute();
+
+ await db.schema
+ .createIndex("relation_from_general")
+ .on("relations")
+ .columns(["fromType", "toType", "fromId"])
+ .where("toType", "!=", "note")
+ .where("toType", "!=", "notebook")
+ .execute();
+
+ await db.schema
+ .createIndex("relation_to_general")
+ .on("relations")
+ .columns(["fromType", "toType", "toId"])
+ .where("fromType", "!=", "note")
+ .where("fromType", "!=", "notebook")
+ .execute();
+
+ await db.schema
+ .createIndex("relation_from_note_notebook")
+ .on("relations")
+ .columns(["fromType", "toType", "fromId", "toId"])
+ .where((eb) =>
+ eb.or([
+ eb("toType", "==", "note"),
+ eb("toType", "==", "notebook")
+ ])
+ )
+ .execute();
+
+ await db.schema
+ .createIndex("relation_to_note_notebook")
+ .on("relations")
+ .columns(["fromType", "toType", "toId", "fromId"])
+ .where((eb) =>
+ eb.or([
+ eb("fromType", "==", "note"),
+ eb("fromType", "==", "notebook")
+ ])
+ )
+ .execute();
+
+ await db.schema
+ .createIndex("note_type")
+ .on("notes")
+ .columns(["type"])
+ .execute();
+
+ await db.schema
+ .createIndex("notebook_type")
+ .on("notebooks")
+ .columns(["type"])
+ .execute();
+
+ await db.schema
+ .createIndex("attachment_hash")
+ .on("attachments")
+ .column("hash")
+ .execute();
+ },
+ async down(db) {}
+ }
+ };
+ }
+}
+
+const addBaseColumns = (
+ builder: CreateTableBuilder
+) => {
+ return builder
+ .addColumn("id", "text", (c) => c.primaryKey().unique().notNull())
+ .addColumn("type", "text")
+ .addColumn("dateModified", "integer")
+ .addColumn("dateCreated", "integer")
+ .addColumn("synced", "boolean")
+ .addColumn("deleted", "boolean");
+};
diff --git a/packages/core/src/database/sql-cached-collection.ts b/packages/core/src/database/sql-cached-collection.ts
new file mode 100644
index 000000000..46b315b95
--- /dev/null
+++ b/packages/core/src/database/sql-cached-collection.ts
@@ -0,0 +1,169 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+*/
+
+import { MaybeDeletedItem, isDeleted } from "../types";
+import EventManager from "../utils/event-manager";
+import { DatabaseAccessor, DatabaseCollection, DatabaseSchema } from ".";
+import { SQLCollection } from "./sql-collection";
+
+export class SQLCachedCollection<
+ TCollectionType extends keyof DatabaseSchema,
+ T extends DatabaseSchema[TCollectionType] = DatabaseSchema[TCollectionType]
+> implements DatabaseCollection
+{
+ private collection: SQLCollection;
+ private cache = new Map>();
+ // private cachedItems?: T[];
+
+ constructor(
+ sql: DatabaseAccessor,
+ type: TCollectionType,
+ eventManager: EventManager
+ ) {
+ this.collection = new SQLCollection(sql, type, eventManager);
+ }
+
+ async init() {
+ await this.collection.init();
+ // const data = await this.collection.indexer.readMulti(
+ // this.collection.indexer.indices
+ // );
+ // this.cache = new Map(data);
+ }
+
+ // async add(item: MaybeDeletedItem) {
+ // await this.collection.addItem(item);
+ // this.cache.set(item.id, item);
+ // this.invalidateCache();
+ // }
+
+ async clear() {
+ await this.collection.clear();
+ this.cache.clear();
+ }
+
+ async upsert(item: T) {
+ await this.collection.upsert(item);
+ this.cache.set(item.id, item);
+ }
+
+ async delete(ids: string[]) {
+ ids.forEach((id) => this.cache.delete(id));
+ await this.collection.delete(ids);
+ }
+
+ async softDelete(ids: string[]) {
+ ids.forEach((id) =>
+ this.cache.set(id, {
+ id,
+ deleted: true,
+ dateModified: Date.now()
+ })
+ );
+ await this.collection.softDelete(ids);
+ }
+
+ exists(id: string) {
+ const item = this.cache.get(id);
+ return !!item && !isDeleted(item);
+ }
+
+ count(): number {
+ return this.cache.size;
+ }
+
+ get(id: string): T | undefined {
+ const item = this.cache.get(id);
+ if (!item || isDeleted(item)) return;
+ return item;
+ }
+
+ async put(items: (T | undefined)[]): Promise {
+ await this.collection.put(items);
+ for (const item of items) {
+ if (item) this.cache.set(item.id, item);
+ }
+ }
+
+ async update(ids: string[], partial: Partial): Promise {
+ await this.collection.update(ids, partial);
+ for (const id of ids) {
+ const item = this.cache.get(id);
+ if (!item) continue;
+ this.cache.set(id, { ...item, ...partial, dateModified: Date.now() });
+ }
+ }
+
+ // has(id: string) {
+ // return this.cache.has(id);
+ // }
+
+ // count() {
+ // return this.cache.size;
+ // }
+
+ // get(id: string) {
+ // const item = this.cache.get(id);
+ // if (!item || isDeleted(item)) return;
+ // return item;
+ // }
+
+ // getRaw(id: string) {
+ // const item = this.cache.get(id);
+ // return item;
+ // }
+
+ // raw() {
+ // return Array.from(this.cache.values());
+ // }
+
+ // items(map?: (item: T) => T | undefined) {
+ // if (this.cachedItems && this.cachedItems.length === this.cache.size)
+ // return this.cachedItems;
+
+ // this.cachedItems = [];
+ // this.cache.forEach((value) => {
+ // if (isDeleted(value)) return;
+ // const mapped = map ? map(value) : value;
+ // if (!mapped) return;
+ // this.cachedItems?.push(mapped);
+ // });
+ // this.cachedItems.sort((a, b) => b.dateCreated - a.dateCreated);
+ // return this.cachedItems;
+ // }
+
+ // async setItems(items: (MaybeDeletedItem | undefined)[]) {
+ // await this.collection.setItems(items);
+ // for (const item of items) {
+ // if (item) {
+ // this.cache.set(item.id, item);
+ // }
+ // }
+
+ // this.invalidateCache();
+ // }
+
+ // *iterateSync(chunkSize: number) {
+ // yield* chunkedIterate(Array.from(this.cache.values()), chunkSize);
+ // }
+
+ // invalidateCache() {
+ // this.cachedItems = undefined;
+ // }
+}
diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.ts
new file mode 100644
index 000000000..a77382288
--- /dev/null
+++ b/packages/core/src/database/sql-collection.ts
@@ -0,0 +1,148 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+*/
+
+import { EVENTS } from "../common";
+import { isDeleted } from "../types";
+import EventManager from "../utils/event-manager";
+import {
+ DatabaseAccessor,
+ DatabaseCollection,
+ DatabaseSchema,
+ SQLiteItem
+} from ".";
+
+export class SQLCollection<
+ TCollectionType extends keyof DatabaseSchema,
+ T extends DatabaseSchema[TCollectionType] = DatabaseSchema[TCollectionType]
+> implements DatabaseCollection, true>
+{
+ constructor(
+ private readonly db: DatabaseAccessor,
+ private readonly type: TCollectionType,
+ private readonly eventManager: EventManager
+ ) {}
+
+ async clear() {
+ await this.db().deleteFrom(this.type).execute();
+ }
+
+ async init() {}
+
+ async upsert(item: SQLiteItem) {
+ if (!item.id) throw new Error("The item must contain the id field.");
+ if (!item.deleted) item.dateCreated = item.dateCreated || Date.now();
+ this.eventManager.publish(EVENTS.databaseUpdated, item.id, item);
+
+ // if item is newly synced, remote will be true.
+ if (!item.remote) {
+ item.dateModified = Date.now();
+ item.synced = false;
+ }
+ // the item has become local now, so remove the flags
+ delete item.remote;
+
+ await this.db()
+ .replaceInto(this.type)
+ .values(item)
+ .execute();
+ }
+
+ async softDelete(ids: string[]) {
+ this.eventManager.publish(EVENTS.databaseUpdated, ids);
+ await this.db()
+ .replaceInto(this.type)
+ .values(
+ ids.map((id) => ({
+ id,
+ deleted: true,
+ dateModified: Date.now()
+ }))
+ )
+ .execute();
+ }
+
+ async delete(ids: string[]) {
+ this.eventManager.publish(EVENTS.databaseUpdated, ids);
+ await this.db()
+ .deleteFrom(this.type)
+ .where("id", "in", ids)
+ .execute();
+ }
+
+ async exists(id: string) {
+ const { count } =
+ (await this.db()
+ .selectFrom(this.type)
+ .select((a) => a.fn.count("id").as("count"))
+ .where("id", "==", id)
+ .limit(1)
+ .executeTakeFirst()) || {};
+
+ return count !== undefined && count > 0;
+ }
+
+ async count() {
+ const { count } =
+ (await this.db()
+ .selectFrom(this.type)
+ .select((a) => a.fn.count("id").as("count"))
+ .where("deleted", "is", null)
+ .executeTakeFirst()) || {};
+ return count || 0;
+ }
+
+ async get(id: string) {
+ const item = await this.db()
+ .selectFrom(this.type)
+ .selectAll()
+ .where("id", "==", id)
+ .executeTakeFirst();
+ if (!item || isDeleted(item)) return;
+ return item as T;
+ }
+
+ async put(items: (SQLiteItem | undefined)[]) {
+ const entries = items.reduce((array, item) => {
+ if (!item) return array;
+ if (!item.remote) {
+ item.dateModified = Date.now();
+ item.synced = false;
+ }
+ delete item.remote;
+ array.push(item);
+ return array;
+ }, [] as SQLiteItem[]);
+
+ await this.db()
+ .replaceInto(this.type)
+ .values(entries)
+ .execute();
+ }
+
+ async update(ids: string[], partial: Partial>) {
+ await this.db()
+ .updateTable(this.type)
+ .where("id", "in", ids)
+ .set({
+ ...partial,
+ dateModified: Date.now()
+ })
+ .execute();
+ }
+}
diff --git a/packages/core/src/interfaces.ts b/packages/core/src/interfaces.ts
index e479ac37c..c6fbc13a6 100644
--- a/packages/core/src/interfaces.ts
+++ b/packages/core/src/interfaces.ts
@@ -18,14 +18,13 @@ along with this program. If not, see .
*/
import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";
-import { AttachmentMetadata } from "./types";
export type Output =
TOutputFormat extends Omit ? string : Uint8Array;
export type FileEncryptionMetadata = {
chunkSize: number;
iv: string;
- length: number;
+ size: number;
salt: string;
alg: string;
};
@@ -74,7 +73,7 @@ export interface ICompressor {
export type RequestOptions = {
url: string;
- metadata?: AttachmentMetadata;
+ // metadata?: AttachmentMetadata;
chunkSize: number;
headers: { Authorization: string };
};
diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.ts
index c14e8aaac..4cd65c0d6 100644
--- a/packages/core/src/migrations.ts
+++ b/packages/core/src/migrations.ts
@@ -298,10 +298,17 @@ const migrations: Migration[] = [
return true;
},
shortcut: (item) => {
- if (item.item.type === "topic") {
+ if (item.item?.type === "topic") {
item.item = { type: "notebook", id: item.item.id };
- return true;
}
+
+ if (item.item) {
+ item.itemId = item.item.id;
+ item.itemType = item.item.type;
+ }
+
+ delete item.item;
+ return true;
},
settings: async (item, db) => {
if (item.trashCleanupInterval)
@@ -336,6 +343,16 @@ const migrations: Migration[] = [
}
}
return true;
+ },
+ relation: (item) => {
+ item.fromId = item.from!.id;
+ item.fromType = item.from!.type;
+ item.toId = item.to!.id;
+ item.toType = item.to!.type;
+
+ delete item.to;
+ delete item.from;
+ return true;
}
}
},
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
index 8618a2109..3f9f11b5e 100644
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -131,6 +131,7 @@ export interface BaseItem {
migrated?: boolean;
remote?: boolean;
synced?: boolean;
+ deleted?: boolean;
}
export type NotebookReference = {
@@ -164,6 +165,9 @@ export interface Note extends BaseItem<"note"> {
readonly: boolean;
dateEdited: number;
+
+ dateDeleted: null;
+ itemType: null;
}
export interface Notebook extends BaseItem<"notebook"> {
@@ -171,6 +175,10 @@ export interface Notebook extends BaseItem<"notebook"> {
description?: string;
dateEdited: number;
pinned: boolean;
+
+ dateDeleted: null;
+ itemType: null;
+
/**
* @deprecated only kept here for migration purposes.
*/
@@ -191,6 +199,9 @@ export interface Topic extends BaseItem<"topic"> {
notes?: string[];
}
+/**
+ * @deprecated only kept here for migration purposes
+ */
export type AttachmentMetadata = {
hash: string;
hashType: string;
@@ -201,15 +212,32 @@ export type AttachmentMetadata = {
export interface Attachment extends BaseItem<"attachment"> {
iv: string;
salt: string;
- length: number;
alg: string;
- key: Cipher<"base64">;
chunkSize: number;
- metadata: AttachmentMetadata;
+
dateUploaded?: number;
failed?: string;
dateDeleted?: number;
+ filename: string;
+ size: number;
+ hash: string;
+ hashType: string;
+ mimeType: string;
+ encryptionKey: string;
+
+ /**
+ * @deprecated only kept here for migration purposes
+ */
+ key?: Cipher<"base64">;
+ /**
+ * @deprecated only kept here for migration purposes
+ */
+ length?: number;
+ /**
+ * @deprecated only kept here for migration purposes
+ */
+ metadata?: AttachmentMetadata;
/**
* @deprecated only kept here for migration purposes
*/
@@ -244,14 +272,31 @@ export type ItemReference = {
};
export interface Relation extends BaseItem<"relation"> {
- from: ItemReference;
- to: ItemReference;
+ fromId: string;
+ fromType: keyof ItemMap;
+ toId: string;
+ toType: keyof ItemMap;
+
+ /**
+ * @deprecated only kept here for migration purposes
+ */
+ from?: ItemReference;
+ /**
+ * @deprecated only kept here for migration purposes
+ */
+ to?: ItemReference;
}
+/**
+ * @deprecated only kept here for migration purposes
+ */
type BaseShortcutReference = {
id: string;
};
+/**
+ * @deprecated only kept here for migration purposes
+ */
type TagNotebookShortcutReference = BaseShortcutReference & {
type: "tag" | "notebook";
};
@@ -265,7 +310,14 @@ type TopicShortcutReference = BaseShortcutReference & {
};
export interface Shortcut extends BaseItem<"shortcut"> {
- item: TopicShortcutReference | TagNotebookShortcutReference;
+ itemId: string;
+ itemType: "tag" | "notebook";
+
+ /**
+ * @deprecated only kept here for migration purposes
+ */
+ item?: TopicShortcutReference | TagNotebookShortcutReference;
+
sortIndex: number;
}
@@ -376,17 +428,14 @@ export type TrashOrItem> =
export type BaseTrashItem> =
BaseItem<"trash"> & {
- title: string;
itemType: TItem["type"];
dateDeleted: number;
- } & Omit;
+ } & Omit;
export type TrashItem = BaseTrashItem | BaseTrashItem;
-export function isDeleted>(
- item: MaybeDeletedItem
-): item is DeletedItem {
- return "deleted" in item;
+export function isDeleted(item: object): item is DeletedItem {
+ return "deleted" in item && !!item.deleted;
}
export function isTrashItem(item: MaybeDeletedItem- ): item is TrashItem {
diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts
index 98a695d14..c189446b4 100644
--- a/packages/core/vitest.config.ts
+++ b/packages/core/vitest.config.ts
@@ -27,10 +27,14 @@ export default defineConfig({
coverage: {
reporter: ["text", "html"]
},
+ exclude: ["__benches__/**/*.bench.ts"],
include: [
...(IS_E2E ? ["__e2e__/**/*.test.{js,ts}"] : []),
"__tests__/**/*.test.{js,ts}",
"src/**/*.test.{js,ts}"
- ]
+ ],
+ benchmark: {
+ include: ["__benches__/**/*.bench.ts"]
+ }
}
});