Mirrors/notesnook
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook.git synced 2025-12-23 23:19:40 +01:00
Code Issues Packages Projects Releases Wiki Activity

core: implement new high perf indexed based grouping

Browse Source
This commit is contained in:
Abdullah Atta
2023-12-05 15:34:18 +05:00
parent 29f66d457d
commit 2ceb68ec87
12 changed files with 788 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
packages/core/__benches__/notes.bench.ts Normal file
View File

@@ -0,0 +1,69 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { bench, describe } from "vitest";
import { databaseTest } from "../__tests__/utils";
import Database from "../src/api";
async function addNotes(db: Database) {
const titles = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".split(
""
);
for (let i = 0; i < 40000; ++i) {
await db.notes.add({
title: `${titles[getRandom(0, titles.length)]} Some other title of mine`
});
if (i % 100 === 0) console.log(i);
}
console.log("DONE");
}
describe("notes", async () => {
const db = await databaseTest();
bench("get grouping", async () => {
await db.notes.all.grouped({
groupBy: "abc",
sortBy: "title",
sortDirection: "asc"
});
});
const grouping = await db.notes.all.grouped({
groupBy: "abc",
sortBy: "title",
sortDirection: "asc"
});
bench("get items in adjacent batches (sequential access)", async function () {
await grouping.item(30000);
await grouping.item(30000 + 500 + 1);
await grouping.item(30000 + 500 + 500 + 1);
await grouping.item(30000 + 500 + 500 + 500 + 1);
await grouping.item(30000 + 500 + 500 + 500 + 500 + 1);
});
bench("get item from random batches (random access)", async () => {
await grouping.item(getRandom(0, 40000));
});
});
function getRandom(min: number, max: number) {
return Math.round(Math.random() * (max - min) + min);
}

4
packages/core/__tests__/utils/index.ts
View File

@@ -48,7 +48,9 @@ function databaseTest() {
eventsource: EventSource, eventsource: EventSource,
fs: FS, fs: FS,
compressor: Compressor, compressor: Compressor,
dialect: new SqliteDialect({ database: BetterSQLite3(":memory:") }) sqliteOptions: {
dialect: new SqliteDialect({ database: BetterSQLite3("db.sql") })
}
}); });
return db.init().then(() => db); return db.init().then(() => db);
} }

430
packages/core/package-lock.json generated
View File

@@ -42,7 +42,7 @@
"@types/spark-md5": "^3.0.2", "@types/spark-md5": "^3.0.2",
"@types/streetwriters__showdown": "npm:@types/showdown@^2.0.6", "@types/streetwriters__showdown": "npm:@types/showdown@^2.0.6",
"@types/ws": "^8.5.5", "@types/ws": "^8.5.5",
"@vitest/coverage-v8": "^0.34.1", "@vitest/coverage-v8": "^1.0.1",
"abortcontroller-polyfill": "^1.7.3", "abortcontroller-polyfill": "^1.7.3",
"better-sqlite3": "^8.6.0", "better-sqlite3": "^8.6.0",
"bson-objectid": "^2.0.4", "bson-objectid": "^2.0.4",
@@ -57,7 +57,7 @@
"nanoid": "^5.0.1", "nanoid": "^5.0.1",
"otplib": "^12.0.1", "otplib": "^12.0.1",
"refractor": "^4.8.1", "refractor": "^4.8.1",
"vitest": "^0.34.1", "vitest": "^1.0.1",
"vitest-fetch-mock": "^0.2.2", "vitest-fetch-mock": "^0.2.2",
"ws": "^8.13.0" "ws": "^8.13.0"
} }
@@ -1452,6 +1452,50 @@
"node": ">=6.0.0" "node": ">=6.0.0"
} }
}, },
"node_modules/@babel/helper-string-parser": {
"version": "7.23.4",
"resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz",
"integrity": "sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==",
"dev": true,
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/helper-validator-identifier": {
"version": "7.22.20",
"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz",
"integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==",
"dev": true,
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/parser": {
"version": "7.24.0",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.0.tgz",
"integrity": "sha512-QuP/FxEAzMSjXygs8v4N9dvdXzEHN4W1oF3PxuWAtPo08UdM17u89RDMgjLn/mlc56iM0HlLmVkO/wgR+rDgHg==",
"dev": true,
"bin": {
"parser": "bin/babel-parser.js"
},
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/@babel/types": {
"version": "7.24.0",
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.0.tgz",
"integrity": "sha512-+j7a5c253RfKh8iABBhywc8NSfP5LURe7Uh4qpsh6jc+aLJguvmIUBdjSdEMQv2bENrCR5MfRdjGo7vzS/ob7w==",
"dev": true,
"dependencies": {
"@babel/helper-string-parser": "^7.23.4",
"@babel/helper-validator-identifier": "^7.22.20",
"to-fast-properties": "^2.0.0"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@bcoe/v8-coverage": { "node_modules/@bcoe/v8-coverage": {
"version": "0.2.3", "version": "0.2.3",
"dev": true, "dev": true,
@@ -2194,21 +2238,6 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"node_modules/@types/chai": {
"version": "4.3.14",
"resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.14.tgz",
"integrity": "sha512-Wj71sXE4Q4AkGdG9Tvq1u/fquNz9EdG4LIJMwVVII7ashjD/8cf8fyIfJAjRr6YcsXnSE8cOGQPq1gqeR8z+3w==",
"dev": true
},
"node_modules/@types/chai-subset": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/@types/chai-subset/-/chai-subset-1.3.5.tgz",
"integrity": "sha512-c2mPnw+xHtXDoHmdtcCXGwyLMiauiAyxWMzhGpqHC4nqI/Y5G2XhTampslK2rb59kpcuHon03UH8W6iYUzw88A==",
"dev": true,
"dependencies": {
"@types/chai": "*"
}
},
"node_modules/@types/estree": { "node_modules/@types/estree": {
"version": "1.0.5", "version": "1.0.5",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz",
@@ -2284,38 +2313,40 @@
} }
}, },
"node_modules/@vitest/coverage-v8": { "node_modules/@vitest/coverage-v8": {
"version": "0.34.6", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-0.34.6.tgz", "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-1.3.1.tgz",
"integrity": "sha512-fivy/OK2d/EsJFoEoxHFEnNGTg+MmdZBAVK9Ka4qhXR2K3J0DS08vcGVwzDtXSuUMabLv4KtPcpSKkcMXFDViw==", "integrity": "sha512-UuBnkSJUNE9rdHjDCPyJ4fYuMkoMtnghes1XohYa4At0MS3OQSAo97FrbwSLRshYsXThMZy1+ybD/byK5llyIg==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"@ampproject/remapping": "^2.2.1", "@ampproject/remapping": "^2.2.1",
"@bcoe/v8-coverage": "^0.2.3", "@bcoe/v8-coverage": "^0.2.3",
"istanbul-lib-coverage": "^3.2.0", "debug": "^4.3.4",
"istanbul-lib-coverage": "^3.2.2",
"istanbul-lib-report": "^3.0.1", "istanbul-lib-report": "^3.0.1",
"istanbul-lib-source-maps": "^4.0.1", "istanbul-lib-source-maps": "^4.0.1",
"istanbul-reports": "^3.1.5", "istanbul-reports": "^3.1.6",
"magic-string": "^0.30.1", "magic-string": "^0.30.5",
"magicast": "^0.3.3",
"picocolors": "^1.0.0", "picocolors": "^1.0.0",
"std-env": "^3.3.3", "std-env": "^3.5.0",
"test-exclude": "^6.0.0", "test-exclude": "^6.0.0",
"v8-to-istanbul": "^9.1.0" "v8-to-istanbul": "^9.2.0"
}, },
"funding": { "funding": {
"url": "https://opencollective.com/vitest" "url": "https://opencollective.com/vitest"
}, },
"peerDependencies": { "peerDependencies": {
"vitest": ">=0.32.0 <1" "vitest": "1.3.1"
} }
}, },
"node_modules/@vitest/expect": { "node_modules/@vitest/expect": {
"version": "0.34.6", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-0.34.6.tgz", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-1.3.1.tgz",
"integrity": "sha512-QUzKpUQRc1qC7qdGo7rMK3AkETI7w18gTCUrsNnyjjJKYiuUB9+TQK3QnR1unhCnWRC0AbKv2omLGQDF/mIjOw==", "integrity": "sha512-xofQFwIzfdmLLlHa6ag0dPV8YsnKOCP1KdAeVVh34vSjN2dcUiXYCD9htu/9eM7t8Xln4v03U9HLxLpPlsXdZw==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"@vitest/spy": "0.34.6", "@vitest/spy": "1.3.1",
"@vitest/utils": "0.34.6", "@vitest/utils": "1.3.1",
"chai": "^4.3.10" "chai": "^4.3.10"
}, },
"funding": { "funding": {
@@ -2323,13 +2354,13 @@
} }
}, },
"node_modules/@vitest/runner": { "node_modules/@vitest/runner": {
"version": "0.34.6", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-0.34.6.tgz", "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-1.3.1.tgz",
"integrity": "sha512-1CUQgtJSLF47NnhN+F9X2ycxUP0kLHQ/JWvNHbeBfwW8CzEGgeskzNnHDyv1ieKTltuR6sdIHV+nmR6kPxQqzQ==", "integrity": "sha512-5FzF9c3jG/z5bgCnjr8j9LNq/9OxV2uEBAITOXfoe3rdZJTdO7jzThth7FXv/6b+kdY65tpRQB7WaKhNZwX+Kg==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"@vitest/utils": "0.34.6", "@vitest/utils": "1.3.1",
"p-limit": "^4.0.0", "p-limit": "^5.0.0",
"pathe": "^1.1.1" "pathe": "^1.1.1"
}, },
"funding": { "funding": {
@@ -2337,40 +2368,41 @@
} }
}, },
"node_modules/@vitest/snapshot": { "node_modules/@vitest/snapshot": {
"version": "0.34.6", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-0.34.6.tgz", "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.3.1.tgz",
"integrity": "sha512-B3OZqYn6k4VaN011D+ve+AA4whM4QkcwcrwaKwAbyyvS/NB1hCWjFIBQxAQQSQir9/RtyAAGuq+4RJmbn2dH4w==", "integrity": "sha512-EF++BZbt6RZmOlE3SuTPu/NfwBF6q4ABS37HHXzs2LUVPBLx2QoY/K0fKpRChSo8eLiuxcbCVfqKgx/dplCDuQ==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"magic-string": "^0.30.1", "magic-string": "^0.30.5",
"pathe": "^1.1.1", "pathe": "^1.1.1",
"pretty-format": "^29.5.0" "pretty-format": "^29.7.0"
}, },
"funding": { "funding": {
"url": "https://opencollective.com/vitest" "url": "https://opencollective.com/vitest"
} }
}, },
"node_modules/@vitest/spy": { "node_modules/@vitest/spy": {
"version": "0.34.6", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-0.34.6.tgz", "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-1.3.1.tgz",
"integrity": "sha512-xaCvneSaeBw/cz8ySmF7ZwGvL0lBjfvqc1LpQ/vcdHEvpLn3Ff1vAvjw+CoGn0802l++5L/pxb7whwcWAw+DUQ==", "integrity": "sha512-xAcW+S099ylC9VLU7eZfdT9myV67Nor9w9zhf0mGCYJSO+zM2839tOeROTdikOi/8Qeusffvxb/MyBSOja1Uig==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"tinyspy": "^2.1.1" "tinyspy": "^2.2.0"
}, },
"funding": { "funding": {
"url": "https://opencollective.com/vitest" "url": "https://opencollective.com/vitest"
} }
}, },
"node_modules/@vitest/utils": { "node_modules/@vitest/utils": {
"version": "0.34.6", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-0.34.6.tgz", "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-1.3.1.tgz",
"integrity": "sha512-IG5aDD8S6zlvloDsnzHw0Ut5xczlF+kv2BOTo+iXfPr54Yhi5qbVOgGB1hZaVq4iJ4C/MZ2J0y15IlsV/ZcI0A==", "integrity": "sha512-d3Waie/299qqRyHTm2DjADeTaNdNSVsnwHPWrs20JMpjh6eiVq7ggggweO8rc4arhf6rRkWuHKwvxGvejUXZZQ==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"diff-sequences": "^29.4.3", "diff-sequences": "^29.6.3",
"loupe": "^2.3.6", "estree-walker": "^3.0.3",
"pretty-format": "^29.5.0" "loupe": "^2.3.7",
"pretty-format": "^29.7.0"
}, },
"funding": { "funding": {
"url": "https://opencollective.com/vitest" "url": "https://opencollective.com/vitest"
@@ -2971,6 +3003,15 @@
"@esbuild/win32-x64": "0.20.2" "@esbuild/win32-x64": "0.20.2"
} }
}, },
"node_modules/estree-walker": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz",
"integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==",
"dev": true,
"dependencies": {
"@types/estree": "^1.0.0"
}
},
"node_modules/event-source-polyfill": { "node_modules/event-source-polyfill": {
"version": "1.0.31", "version": "1.0.31",
"dev": true, "dev": true,
@@ -2990,6 +3031,29 @@
"node": ">=12.0.0" "node": ">=12.0.0"
} }
}, },
"node_modules/execa": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz",
"integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==",
"dev": true,
"dependencies": {
"cross-spawn": "^7.0.3",
"get-stream": "^8.0.1",
"human-signals": "^5.0.0",
"is-stream": "^3.0.0",
"merge-stream": "^2.0.0",
"npm-run-path": "^5.1.0",
"onetime": "^6.0.0",
"signal-exit": "^4.1.0",
"strip-final-newline": "^3.0.0"
},
"engines": {
"node": ">=16.17"
},
"funding": {
"url": "https://github.com/sindresorhus/execa?sponsor=1"
}
},
"node_modules/expand-template": { "node_modules/expand-template": {
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@@ -3037,6 +3101,20 @@
"dev": true, "dev": true,
"license": "ISC" "license": "ISC"
}, },
"node_modules/fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
"dev": true,
"hasInstallScript": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/fuzzyjs": { "node_modules/fuzzyjs": {
"version": "5.0.1", "version": "5.0.1",
"resolved": "https://registry.npmjs.org/fuzzyjs/-/fuzzyjs-5.0.1.tgz", "resolved": "https://registry.npmjs.org/fuzzyjs/-/fuzzyjs-5.0.1.tgz",
@@ -3054,6 +3132,18 @@
"node": "*" "node": "*"
} }
}, },
"node_modules/get-stream": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz",
"integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==",
"dev": true,
"engines": {
"node": ">=16"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/github-from-package": { "node_modules/github-from-package": {
"version": "0.0.0", "version": "0.0.0",
"resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
@@ -3192,6 +3282,15 @@
"node": ">= 6" "node": ">= 6"
} }
}, },
"node_modules/human-signals": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz",
"integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==",
"dev": true,
"engines": {
"node": ">=16.17.0"
}
},
"node_modules/iconv-lite": { "node_modules/iconv-lite": {
"version": "0.6.3", "version": "0.6.3",
"dev": true, "dev": true,
@@ -3288,6 +3387,18 @@
"dev": true, "dev": true,
"license": "MIT" "license": "MIT"
}, },
"node_modules/is-stream": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz",
"integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==",
"dev": true,
"engines": {
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/isexe": { "node_modules/isexe": {
"version": "2.0.0", "version": "2.0.0",
"dev": true, "dev": true,
@@ -3348,6 +3459,12 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/js-tokens": {
"version": "8.0.3",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-8.0.3.tgz",
"integrity": "sha512-UfJMcSJc+SEXEl9lH/VLHSZbThQyLpw1vLO1Lb+j4RWDvG3N2f7yj3PVQA3cmkTBNldJ9eFnM+xEXxHIXrYiJw==",
"dev": true
},
"node_modules/jsdom": { "node_modules/jsdom": {
"version": "22.1.0", "version": "22.1.0",
"dev": true, "dev": true,
@@ -3491,10 +3608,14 @@
} }
}, },
"node_modules/local-pkg": { "node_modules/local-pkg": {
"version": "0.4.3", "version": "0.5.0",
"resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-0.4.3.tgz", "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-0.5.0.tgz",
"integrity": "sha512-SFppqq5p42fe2qcZQqqEOiVRXl+WCP1MdT6k7BDEW1j++sp5fIY+/fdRQitvKgB5BrBcmrs5m/L0v2FrU5MY1g==", "integrity": "sha512-ok6z3qlYyCDS4ZEU27HaU6x/xZa9Whf8jD4ptH5UZTQYZVYeb9bnZ3ojVhiJNLiXK1Hfc0GNbLXcmZ5plLDDBg==",
"dev": true, "dev": true,
"dependencies": {
"mlly": "^1.4.2",
"pkg-types": "^1.0.3"
},
"engines": { "engines": {
"node": ">=14" "node": ">=14"
}, },
@@ -3533,6 +3654,17 @@
"node": ">=12" "node": ">=12"
} }
}, },
"node_modules/magicast": {
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/magicast/-/magicast-0.3.3.tgz",
"integrity": "sha512-ZbrP1Qxnpoes8sz47AM0z08U+jW6TyRgZzcWy3Ma3vDhJttwMwAFDMMQFobwdBxByBD46JYmxRzeF7w2+wJEuw==",
"dev": true,
"dependencies": {
"@babel/parser": "^7.23.6",
"@babel/types": "^7.23.6",
"source-map-js": "^1.0.2"
}
},
"node_modules/make-dir": { "node_modules/make-dir": {
"version": "4.0.0", "version": "4.0.0",
"dev": true, "dev": true,
@@ -3547,6 +3679,12 @@
"url": "https://github.com/sponsors/sindresorhus" "url": "https://github.com/sponsors/sindresorhus"
} }
}, },
"node_modules/merge-stream": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
"dev": true
},
"node_modules/mime-db": { "node_modules/mime-db": {
"version": "1.52.0", "version": "1.52.0",
"license": "MIT", "license": "MIT",
@@ -3565,6 +3703,18 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/mimic-fn": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz",
"integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==",
"dev": true,
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/mimic-response": { "node_modules/mimic-response": {
"version": "3.1.0", "version": "3.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
@@ -3723,6 +3873,33 @@
"webidl-conversions": "^3.0.0" "webidl-conversions": "^3.0.0"
} }
}, },
"node_modules/npm-run-path": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz",
"integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==",
"dev": true,
"dependencies": {
"path-key": "^4.0.0"
},
"engines": {
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/npm-run-path/node_modules/path-key": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz",
"integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==",
"dev": true,
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/nth-check": { "node_modules/nth-check": {
"version": "2.1.1", "version": "2.1.1",
"license": "BSD-2-Clause", "license": "BSD-2-Clause",
@@ -3746,6 +3923,21 @@
"wrappy": "1" "wrappy": "1"
} }
}, },
"node_modules/onetime": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz",
"integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==",
"dev": true,
"dependencies": {
"mimic-fn": "^4.0.0"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/otplib": { "node_modules/otplib": {
"version": "12.0.1", "version": "12.0.1",
"dev": true, "dev": true,
@@ -3757,15 +3949,15 @@
} }
}, },
"node_modules/p-limit": { "node_modules/p-limit": {
"version": "4.0.0", "version": "5.0.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz",
"integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"yocto-queue": "^1.0.0" "yocto-queue": "^1.0.0"
}, },
"engines": { "engines": {
"node": "^12.20.0 || ^14.13.1 || >=16.0.0" "node": ">=18"
}, },
"funding": { "funding": {
"url": "https://github.com/sponsors/sindresorhus" "url": "https://github.com/sponsors/sindresorhus"
@@ -4202,6 +4394,18 @@
"dev": true, "dev": true,
"license": "ISC" "license": "ISC"
}, },
"node_modules/signal-exit": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
"integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
"dev": true,
"engines": {
"node": ">=14"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/simple-concat": { "node_modules/simple-concat": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
@@ -4296,6 +4500,18 @@
"safe-buffer": "~5.2.0" "safe-buffer": "~5.2.0"
} }
}, },
"node_modules/strip-final-newline": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz",
"integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==",
"dev": true,
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/strip-json-comments": { "node_modules/strip-json-comments": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
@@ -4306,12 +4522,12 @@
} }
}, },
"node_modules/strip-literal": { "node_modules/strip-literal": {
"version": "1.3.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-1.3.0.tgz", "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-2.0.0.tgz",
"integrity": "sha512-PugKzOsyXpArk0yWmUwqOZecSO0GH0bPoctLcqNDH9J04pVW3lflYE0ujElBGTloevcxF5MofAOZ7C5l2b+wLg==", "integrity": "sha512-f9vHgsCWBq2ugHAkGMiiYY+AYG0D/cbloKKg0nhaaaSNsujdGIpVXCNsrJpCKr5M0f4aI31mr13UjY6GAuXCKA==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"acorn": "^8.10.0" "js-tokens": "^8.0.2"
}, },
"funding": { "funding": {
"url": "https://github.com/sponsors/antfu" "url": "https://github.com/sponsors/antfu"
@@ -4387,9 +4603,9 @@
"license": "MIT" "license": "MIT"
}, },
"node_modules/tinypool": { "node_modules/tinypool": {
"version": "0.7.0", "version": "0.8.2",
"resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.7.0.tgz", "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.8.2.tgz",
"integrity": "sha512-zSYNUlYSMhJ6Zdou4cJwo/p7w5nmAH17GRfU/ui3ctvjXFErXXkruT4MWW6poDeXgCaIBlGLrfU6TbTXxyGMww==", "integrity": "sha512-SUszKYe5wgsxnNOVlBYO6IC+8VGWdVGZWAqUxp3UErNBtptZvWbwyUOyzNL59zigz2rCA92QiL3wvG+JDSdJdQ==",
"dev": true, "dev": true,
"engines": { "engines": {
"node": ">=14.0.0" "node": ">=14.0.0"
@@ -4404,6 +4620,15 @@
"node": ">=14.0.0" "node": ">=14.0.0"
} }
}, },
"node_modules/to-fast-properties": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
"integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==",
"dev": true,
"engines": {
"node": ">=4"
}
},
"node_modules/tough-cookie": { "node_modules/tough-cookie": {
"version": "4.1.3", "version": "4.1.3",
"license": "BSD-3-Clause", "license": "BSD-3-Clause",
@@ -4557,82 +4782,78 @@
} }
}, },
"node_modules/vite-node": { "node_modules/vite-node": {
"version": "0.34.6", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/vite-node/-/vite-node-0.34.6.tgz", "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-1.3.1.tgz",
"integrity": "sha512-nlBMJ9x6n7/Amaz6F3zJ97EBwR2FkzhBRxF5e+jE6LA3yi6Wtc2lyTij1OnDMIr34v5g/tVQtsVAzhT0jc5ygA==", "integrity": "sha512-azbRrqRxlWTJEVbzInZCTchx0X69M/XPTCz4H+TLvlTcR/xH/3hkRqhOakT41fMJCMzXTu4UvegkZiEoJAWvng==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"cac": "^6.7.14", "cac": "^6.7.14",
"debug": "^4.3.4", "debug": "^4.3.4",
"mlly": "^1.4.0",
"pathe": "^1.1.1", "pathe": "^1.1.1",
"picocolors": "^1.0.0", "picocolors": "^1.0.0",
"vite": "^3.0.0 || ^4.0.0 || ^5.0.0-0" "vite": "^5.0.0"
}, },
"bin": { "bin": {
"vite-node": "vite-node.mjs" "vite-node": "vite-node.mjs"
}, },
"engines": { "engines": {
"node": ">=v14.18.0" "node": "^18.0.0 || >=20.0.0"
}, },
"funding": { "funding": {
"url": "https://opencollective.com/vitest" "url": "https://opencollective.com/vitest"
} }
}, },
"node_modules/vitest": { "node_modules/vitest": {
"version": "0.34.6", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/vitest/-/vitest-0.34.6.tgz", "resolved": "https://registry.npmjs.org/vitest/-/vitest-1.3.1.tgz",
"integrity": "sha512-+5CALsOvbNKnS+ZHMXtuUC7nL8/7F1F2DnHGjSsszX8zCjWSSviphCb/NuS9Nzf4Q03KyyDRBAXhF/8lffME4Q==", "integrity": "sha512-/1QJqXs8YbCrfv/GPQ05wAZf2eakUPLPa18vkJAKE7RXOKfVHqMZZ1WlTjiwl6Gcn65M5vpNUB6EFLnEdRdEXQ==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"@types/chai": "^4.3.5", "@vitest/expect": "1.3.1",
"@types/chai-subset": "^1.3.3", "@vitest/runner": "1.3.1",
"@types/node": "*", "@vitest/snapshot": "1.3.1",
"@vitest/expect": "0.34.6", "@vitest/spy": "1.3.1",
"@vitest/runner": "0.34.6", "@vitest/utils": "1.3.1",
"@vitest/snapshot": "0.34.6", "acorn-walk": "^8.3.2",
"@vitest/spy": "0.34.6",
"@vitest/utils": "0.34.6",
"acorn": "^8.9.0",
"acorn-walk": "^8.2.0",
"cac": "^6.7.14",
"chai": "^4.3.10", "chai": "^4.3.10",
"debug": "^4.3.4", "debug": "^4.3.4",
"local-pkg": "^0.4.3", "execa": "^8.0.1",
"magic-string": "^0.30.1", "local-pkg": "^0.5.0",
"magic-string": "^0.30.5",
"pathe": "^1.1.1", "pathe": "^1.1.1",
"picocolors": "^1.0.0", "picocolors": "^1.0.0",
"std-env": "^3.3.3", "std-env": "^3.5.0",
"strip-literal": "^1.0.1", "strip-literal": "^2.0.0",
"tinybench": "^2.5.0", "tinybench": "^2.5.1",
"tinypool": "^0.7.0", "tinypool": "^0.8.2",
"vite": "^3.1.0 || ^4.0.0 || ^5.0.0-0", "vite": "^5.0.0",
"vite-node": "0.34.6", "vite-node": "1.3.1",
"why-is-node-running": "^2.2.2" "why-is-node-running": "^2.2.2"
}, },
"bin": { "bin": {
"vitest": "vitest.mjs" "vitest": "vitest.mjs"
}, },
"engines": { "engines": {
"node": ">=v14.18.0" "node": "^18.0.0 || >=20.0.0"
}, },
"funding": { "funding": {
"url": "https://opencollective.com/vitest" "url": "https://opencollective.com/vitest"
}, },
"peerDependencies": { "peerDependencies": {
"@edge-runtime/vm": "*", "@edge-runtime/vm": "*",
"@vitest/browser": "*", "@types/node": "^18.0.0 || >=20.0.0",
"@vitest/ui": "*", "@vitest/browser": "1.3.1",
"@vitest/ui": "1.3.1",
"happy-dom": "*", "happy-dom": "*",
"jsdom": "*", "jsdom": "*"
"playwright": "*",
"safaridriver": "*",
"webdriverio": "*"
}, },
"peerDependenciesMeta": { "peerDependenciesMeta": {
"@edge-runtime/vm": { "@edge-runtime/vm": {
"optional": true "optional": true
}, },
"@types/node": {
"optional": true
},
"@vitest/browser": { "@vitest/browser": {
"optional": true "optional": true
}, },
@@ -4644,15 +4865,6 @@
}, },
"jsdom": { "jsdom": {
"optional": true "optional": true
},
"playwright": {
"optional": true
},
"safaridriver": {
"optional": true
},
"webdriverio": {
"optional": true
} }
} }
}, },

4
packages/core/package.json
View File

@@ -18,7 +18,7 @@
"@types/spark-md5": "^3.0.2", "@types/spark-md5": "^3.0.2",
"@types/streetwriters__showdown": "npm:@types/showdown@^2.0.6", "@types/streetwriters__showdown": "npm:@types/showdown@^2.0.6",
"@types/ws": "^8.5.5", "@types/ws": "^8.5.5",
"@vitest/coverage-v8": "^0.34.1", "@vitest/coverage-v8": "^1.0.1",
"abortcontroller-polyfill": "^1.7.3", "abortcontroller-polyfill": "^1.7.3",
"better-sqlite3": "^8.6.0", "better-sqlite3": "^8.6.0",
"bson-objectid": "^2.0.4", "bson-objectid": "^2.0.4",
@@ -33,7 +33,7 @@
"nanoid": "^5.0.1", "nanoid": "^5.0.1",
"otplib": "^12.0.1", "otplib": "^12.0.1",
"refractor": "^4.8.1", "refractor": "^4.8.1",
"vitest": "^0.34.1", "vitest": "^1.0.1",
"vitest-fetch-mock": "^0.2.2", "vitest-fetch-mock": "^0.2.2",
"ws": "^8.13.0" "ws": "^8.13.0"
}, },

42
packages/core/src/api/lookup.ts
View File

@@ -100,16 +100,21 @@ export default class Lookup {
trash(query: string): SearchResults<TrashItem> { trash(query: string): SearchResults<TrashItem> {
return { return {
sorted: async (limit?: number) => { sorted: async (limit?: number) => {
const { ids, records } = await this.filterTrash(query, limit); const { ids, items } = await this.filterTrash(query, limit);
return new VirtualizedGrouping<TrashItem>( return new VirtualizedGrouping<TrashItem>(
ids, ids.length,
this.db.options.batchSize, this.db.options.batchSize,
async () => records async (start, end) => {
return {
ids: ids.slice(start, end),
items: items.slice(start, end)
};
}
); );
}, },
items: async (limit?: number) => { items: async (limit?: number) => {
const { records } = await this.filterTrash(query, limit); const { items } = await this.filterTrash(query, limit);
return Object.values(records); return items;
}, },
ids: () => this.filterTrash(query).then(({ ids }) => ids) ids: () => this.filterTrash(query).then(({ ids }) => ids)
}; };
@@ -176,22 +181,23 @@ export default class Lookup {
private async filterTrash(query: string, limit?: number) { private async filterTrash(query: string, limit?: number) {
const items = await this.db.trash.all(); const items = await this.db.trash.all();
const records: Record<string, TrashItem> = {}; const results: Map<string, { rank: number; item: TrashItem }> = new Map();
const results: Map<string, number> = new Map();
for (const item of items) { for (const item of items) {
if (limit && results.size >= limit) break; if (limit && results.size >= limit) break;
const result = match(query, item.title); const result = match(query, item.title);
if (result.match) { if (result.match) {
records[item.id] = item; results.set(item.id, { rank: result.score, item });
results.set(item.id, result.score);
} }
} }
const ids = Array.from(results.entries()) const sorted = Array.from(results.entries()).sort(
.sort((a, b) => a[1] - b[1]) (a, b) => a[1].rank - b[1].rank
.map((a) => a[0]); );
return { ids, records }; return {
ids: sorted.map((a) => a[0]),
items: sorted.map((a) => a[1].item)
};
} }
private toVirtualizedGrouping<T extends Item>( private toVirtualizedGrouping<T extends Item>(
@@ -199,9 +205,15 @@ export default class Lookup {
selector: FilteredSelector<T> selector: FilteredSelector<T>
) { ) {
return new VirtualizedGrouping<T>( return new VirtualizedGrouping<T>(
ids, ids.length,
this.db.options.batchSize, this.db.options.batchSize,
async (ids) => selector.records(ids) async (start, end) => {
const items = await selector.items(ids);
return {
ids: ids.slice(start, end),
items: items.slice(start, end)
};
}
); );
} }

2
packages/core/src/collections/relations.ts
View File

@@ -34,7 +34,7 @@ export class Relations implements ICollection {
} }
async init() { async init() {
await this.buildCache(); // await this.buildCache();
// return this.collection.init(); // return this.collection.init();
} }

30
packages/core/src/collections/trash.ts
View File

@@ -212,18 +212,28 @@ export default class Trash {
} }
async grouped(options: GroupOptions) { async grouped(options: GroupOptions) {
const items = await this.all(); // const items = await this.all();
const ids = groupArray(items, options); // const ids = groupArray(items, options);
const records: Record<string, TrashItem> = {}; // const records: Record<string, TrashItem> = {};
for (const item of items) records[item.id] = item; // for (const item of items) records[item.id] = item;
// const ids = [...this.cache.notebooks,...this.cache.notes]
return new VirtualizedGrouping<TrashItem>( return new VirtualizedGrouping<TrashItem>(
ids, this.cache.notebooks.length + this.cache.notes.length,
this.db.options?.batchSize || 500, this.db.options.batchSize,
async (ids: string[]) => { async (start, end) => {
const items: Record<string, TrashItem> = {}; // const notesRange = end < this.cache.notes.length ? [start, end] : [start, this.cache.notes.length - 1];
for (const id of ids) items[id] = records[id]; // const notebooksRange = start >= this.cache.notes.length ?[start, end] : [
return items; // 0, end
// ]
// TODO:
return { ids: [], items: [] };
// return {
// ids: ids.slice(start,end),
// }
// const items: Record<string, TrashItem> = {};
// for (const id of ids) items[id] = records[id];
// return items;
} }
); );
} }

26
packages/core/src/database/migrations.ts
View File

@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
import { import {
ColumnBuilderCallback,
CreateTableBuilder, CreateTableBuilder,
Kysely, Kysely,
Migration, Migration,
@@ -25,6 +26,9 @@ import {
sql sql
} from "kysely"; } from "kysely";
const COLLATE_NOCASE: ColumnBuilderCallback = (col) =>
col.modifyEnd(sql`collate nocase`);
export class NNMigrationProvider implements MigrationProvider { export class NNMigrationProvider implements MigrationProvider {
async getMigrations(): Promise<Record<string, Migration>> { async getMigrations(): Promise<Record<string, Migration>> {
return { return {
@@ -35,7 +39,7 @@ export class NNMigrationProvider implements MigrationProvider {
// .modifyEnd(sql`without rowid`) // .modifyEnd(sql`without rowid`)
.$call(addBaseColumns) .$call(addBaseColumns)
.$call(addTrashColumns) .$call(addTrashColumns)
.addColumn("title", "text") .addColumn("title", "text", COLLATE_NOCASE)
.addColumn("headline", "text") .addColumn("headline", "text")
.addColumn("contentId", "text") .addColumn("contentId", "text")
.addColumn("pinned", "boolean") .addColumn("pinned", "boolean")
@@ -95,7 +99,7 @@ export class NNMigrationProvider implements MigrationProvider {
.modifyEnd(sql`without rowid`) .modifyEnd(sql`without rowid`)
.$call(addBaseColumns) .$call(addBaseColumns)
.$call(addTrashColumns) .$call(addTrashColumns)
.addColumn("title", "text") .addColumn("title", "text", COLLATE_NOCASE)
.addColumn("description", "text") .addColumn("description", "text")
.addColumn("dateEdited", "integer") .addColumn("dateEdited", "integer")
.addColumn("pinned", "boolean") .addColumn("pinned", "boolean")
@@ -105,14 +109,14 @@ export class NNMigrationProvider implements MigrationProvider {
.createTable("tags") .createTable("tags")
.modifyEnd(sql`without rowid`) .modifyEnd(sql`without rowid`)
.$call(addBaseColumns) .$call(addBaseColumns)
.addColumn("title", "text") .addColumn("title", "text", COLLATE_NOCASE)
.execute(); .execute();
await db.schema await db.schema
.createTable("colors") .createTable("colors")
.modifyEnd(sql`without rowid`) .modifyEnd(sql`without rowid`)
.$call(addBaseColumns) .$call(addBaseColumns)
.addColumn("title", "text") .addColumn("title", "text", COLLATE_NOCASE)
.addColumn("colorCode", "text") .addColumn("colorCode", "text")
.execute(); .execute();
@@ -139,7 +143,7 @@ export class NNMigrationProvider implements MigrationProvider {
.createTable("reminders") .createTable("reminders")
.modifyEnd(sql`without rowid`) .modifyEnd(sql`without rowid`)
.$call(addBaseColumns) .$call(addBaseColumns)
.addColumn("title", "text") .addColumn("title", "text", COLLATE_NOCASE)
.addColumn("description", "text") .addColumn("description", "text")
.addColumn("priority", "text") .addColumn("priority", "text")
.addColumn("date", "integer") .addColumn("date", "integer")
@@ -230,6 +234,18 @@ export class NNMigrationProvider implements MigrationProvider {
.columns(["type"]) .columns(["type"])
.execute(); .execute();
await db.schema
.createIndex("note_deleted")
.on("notes")
.columns(["deleted"])
.execute();
await db.schema
.createIndex("note_date_deleted")
.on("notes")
.columns(["dateDeleted"])
.execute();
await db.schema await db.schema
.createIndex("notebook_type") .createIndex("notebook_type")
.on("notebooks") .on("notebooks")

147
packages/core/src/database/sql-collection.ts
View File

@@ -19,6 +19,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { EVENTS } from "../common"; import { EVENTS } from "../common";
import { import {
GroupHeader,
GroupOptions, GroupOptions,
Item, Item,
MaybeDeletedItem, MaybeDeletedItem,
@@ -34,9 +35,9 @@ import {
isFalse isFalse
} from "."; } from ".";
import { import {
AnyColumn,
AnyColumnWithTable, AnyColumnWithTable,
ExpressionOrFactory, ExpressionOrFactory,
SelectExpression,
SelectQueryBuilder, SelectQueryBuilder,
SqlBool, SqlBool,
sql sql
@@ -353,10 +354,41 @@ export class FilteredSelector<T extends Item> {
} }
async grouped(options: GroupOptions) { async grouped(options: GroupOptions) {
console.time("getting items"); const count = await this.count();
const sortFields = this.sortFields(options, true);
const cursorRowValue = sql.join(sortFields.map((f) => sql.ref(f)));
return new VirtualizedGrouping<T>(
count,
this.batchSize,
async (start, end, cursor) => {
const items = (await this.filter
.$call(this.buildSortExpression(options))
.$if(!cursor, (qb) => qb.offset(start))
.$if(!!cursor, (qb) =>
qb.where(
(eb) => eb.parens(cursorRowValue),
">",
(eb) =>
eb.parens(sql.join(sortFields.map((f) => (cursor as any)[f])))
)
)
.limit(end - start)
.selectAll()
.execute()) as T[];
return {
ids: items.map((i) => i.id),
items
};
},
(items) => groupArray(items as any, options),
() => this.groups(options)
);
}
async groups(options: GroupOptions) {
const fields: Array< const fields: Array<
SelectExpression<DatabaseSchema, keyof DatabaseSchema> | AnyColumnWithTable<DatabaseSchema, keyof DatabaseSchema>
| AnyColumn<DatabaseSchema, keyof DatabaseSchema>
> = ["id", "type", options.sortBy]; > = ["id", "type", options.sortBy];
if (this.type === "notes") fields.push("notes.pinned", "notes.conflicted"); if (this.type === "notes") fields.push("notes.pinned", "notes.conflicted");
else if (this.type === "notebooks") fields.push("notebooks.pinned"); else if (this.type === "notebooks") fields.push("notebooks.pinned");
@@ -372,33 +404,63 @@ export class FilteredSelector<T extends Item> {
"reminders.snoozeUntil" "reminders.snoozeUntil"
); );
} }
return groupArray(
const items = await this.filter await this.filter
.$if(!!this._limit, (eb) => eb.limit(this._limit)) .$call(this.buildSortExpression(options))
.$call(this.buildSortExpression(options)) .select(fields)
.select(fields) .execute(),
.execute(); options
console.timeEnd("getting items");
console.log(items.length);
const ids = groupArray(items, options);
return new VirtualizedGrouping<T>(ids, this.batchSize, (ids) =>
this.records(ids)
); );
} }
async sorted(options: SortOptions) { async sorted(options: SortOptions) {
const items = await this.filter const count = await this.count();
.$if(!!this._limit, (eb) => eb.limit(this._limit))
.$call(this.buildSortExpression(options)) return new VirtualizedGrouping<T>(
.select("id") count,
.execute(); this.batchSize,
const ids = items.map((item) => item.id); async (start, end) => {
return new VirtualizedGrouping<T>(ids, this.batchSize, (ids) => const items = (await this.filter
this.records(ids) .$call(this.buildSortExpression(options))
.offset(start)
.limit(end - start)
.selectAll()
.execute()) as T[];
return {
ids: items.map((i) => i.id),
items
};
}
); );
} }
private buildSortExpression(options: SortOptions) { async *[Symbol.asyncIterator]() {
let lastRow: any | null = null;
while (true) {
const rows = await this.filter
.orderBy("dateCreated asc")
.orderBy("id asc")
.$if(lastRow !== null, (qb) =>
qb.where(
(eb) => eb.refTuple("dateCreated", "id"),
">",
(eb) => eb.tuple(lastRow.dateCreated, lastRow.id)
)
)
.limit(this.batchSize)
.$if(this._fields.length === 0, (eb) => eb.selectAll())
.$if(this._fields.length > 0, (eb) => eb.select(this._fields))
.execute();
if (rows.length === 0) break;
for (const row of rows) {
yield row as T;
}
lastRow = rows[rows.length - 1];
}
}
private buildSortExpression(options: SortOptions, persistent?: boolean) {
return <T>( return <T>(
qb: SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, T> qb: SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, T>
) => { ) => {
@@ -407,34 +469,21 @@ export class FilteredSelector<T extends Item> {
.$if(this.type === "notes" || this.type === "notebooks", (eb) => .$if(this.type === "notes" || this.type === "notebooks", (eb) =>
eb.orderBy("pinned desc") eb.orderBy("pinned desc")
) )
.$if(options.sortBy === "title", (eb) => .orderBy(options.sortBy, options.sortDirection)
eb.orderBy( .$if(!!persistent, (eb) => eb.orderBy("id"));
sql`${sql.raw(options.sortBy)} COLLATE NOCASE ${sql.raw(
options.sortDirection
)}`
)
)
.$if(options.sortBy !== "title", (eb) =>
eb.orderBy(options.sortBy, options.sortDirection)
);
}; };
} }
async *[Symbol.asyncIterator]() { private sortFields(options: SortOptions, persistent?: boolean) {
let index = 0; const fields: Array<
while (true) { | AnyColumnWithTable<DatabaseSchema, keyof DatabaseSchema>
const rows = await this.filter | AnyColumn<DatabaseSchema, keyof DatabaseSchema>
.$if(this._fields.length === 0, (eb) => eb.selectAll()) > = [];
.$if(this._fields.length > 0, (eb) => eb.select(this._fields)) if (this.type === "notes") fields.push("conflicted");
.orderBy("dateCreated asc") if (this.type === "notes" || this.type === "notebooks")
.offset(index) fields.push("pinned");
.limit(this.batchSize) fields.push(options.sortBy);
.execute(); if (persistent) fields.push("id");
if (rows.length === 0) break; return fields;
index += this.batchSize;
for (const row of rows) {
yield row as T;
}
}
} }
} }

168
packages/core/src/utils/__tests__/virtualized-grouping.test.ts
View File

@@ -28,98 +28,98 @@ function createMock() {
Object.fromEntries(ids.map((id) => [id, id])) Object.fromEntries(ids.map((id) => [id, id]))
); );
} }
test("fetch items in batch if not found in cache", async (t) => { // test("fetch items in batch if not found in cache", async (t) => {
const mocked = createMock(); // const mocked = createMock();
const grouping = new VirtualizedGrouping<string>( // const grouping = new VirtualizedGrouping<string>(
["1", "2", "3", "4", "5", "6", "7"], // ["1", "2", "3", "4", "5", "6", "7"],
3, // 3,
mocked // mocked
); // );
t.expect(await grouping.item("4")).toStrictEqual(item("4")); // t.expect(await grouping.item("4")).toStrictEqual(item("4"));
t.expect(mocked).toHaveBeenCalledOnce(); // t.expect(mocked).toHaveBeenCalledOnce();
}); // });
test("do not fetch items in batch if found in cache", async (t) => { // test("do not fetch items in batch if found in cache", async (t) => {
const mocked = createMock(); // const mocked = createMock();
const grouping = new VirtualizedGrouping<string>( // const grouping = new VirtualizedGrouping<string>(
["1", "2", "3", "4", "5", "6", "7"], // ["1", "2", "3", "4", "5", "6", "7"],
3, // 3,
mocked // mocked
); // );
t.expect(await grouping.item("4")).toStrictEqual(item("4")); // t.expect(await grouping.item("4")).toStrictEqual(item("4"));
t.expect(await grouping.item("4")).toStrictEqual(item("4")); // t.expect(await grouping.item("4")).toStrictEqual(item("4"));
t.expect(await grouping.item("4")).toStrictEqual(item("4")); // t.expect(await grouping.item("4")).toStrictEqual(item("4"));
t.expect(await grouping.item("4")).toStrictEqual(item("4")); // t.expect(await grouping.item("4")).toStrictEqual(item("4"));
t.expect(await grouping.item("4")).toStrictEqual(item("4")); // t.expect(await grouping.item("4")).toStrictEqual(item("4"));
t.expect(mocked).toHaveBeenCalledOnce(); // t.expect(mocked).toHaveBeenCalledOnce();
}); // });
test("clear old cached batches", async (t) => { // test("clear old cached batches", async (t) => {
const mocked = createMock(); // const mocked = createMock();
const grouping = new VirtualizedGrouping<string>( // const grouping = new VirtualizedGrouping<string>(
["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], // ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
3, // 3,
mocked // mocked
); // );
t.expect(await grouping.item("1")).toStrictEqual(item("1")); // t.expect(await grouping.item("1")).toStrictEqual(item("1"));
t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); // t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
t.expect(await grouping.item("4")).toStrictEqual(item("4")); // t.expect(await grouping.item("4")).toStrictEqual(item("4"));
t.expect(mocked).toHaveBeenLastCalledWith(["4", "5", "6"]); // t.expect(mocked).toHaveBeenLastCalledWith(["4", "5", "6"]);
t.expect(await grouping.item("7")).toStrictEqual(item("7")); // t.expect(await grouping.item("7")).toStrictEqual(item("7"));
t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]); // t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
t.expect(await grouping.item("1")).toStrictEqual(item("1")); // t.expect(await grouping.item("1")).toStrictEqual(item("1"));
t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); // t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
}); // });
test("clear old cached batches (random access)", async (t) => { // test("clear old cached batches (random access)", async (t) => {
const mocked = createMock(); // const mocked = createMock();
const grouping = new VirtualizedGrouping<string>( // const grouping = new VirtualizedGrouping<string>(
["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], // ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
3, // 3,
mocked // mocked
); // );
t.expect(await grouping.item("1")).toStrictEqual(item("1")); // t.expect(await grouping.item("1")).toStrictEqual(item("1"));
t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); // t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
t.expect(await grouping.item("7")).toStrictEqual(item("7")); // t.expect(await grouping.item("7")).toStrictEqual(item("7"));
t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]); // t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
t.expect(await grouping.item("11")).toStrictEqual(item("11")); // t.expect(await grouping.item("11")).toStrictEqual(item("11"));
t.expect(mocked).toHaveBeenLastCalledWith(["10", "11", "12"]); // t.expect(mocked).toHaveBeenLastCalledWith(["10", "11", "12"]);
t.expect(await grouping.item("1")).toStrictEqual(item("1")); // t.expect(await grouping.item("1")).toStrictEqual(item("1"));
t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); // t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
t.expect(await grouping.item("7")).toStrictEqual(item("7")); // t.expect(await grouping.item("7")).toStrictEqual(item("7"));
t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]); // t.expect(mocked).toHaveBeenLastCalledWith(["7", "8", "9"]);
}); // });
test("reloading ids should clear all cached batches", async (t) => { // test("reloading ids should clear all cached batches", async (t) => {
const mocked = createMock(); // const mocked = createMock();
const grouping = new VirtualizedGrouping<string>( // const grouping = new VirtualizedGrouping<string>(
["1", "3", "4", "5", "7", "6", "50"], // ["1", "3", "4", "5", "7", "6", "50"],
3, // 3,
mocked // mocked
); // );
t.expect(await grouping.item("1")).toStrictEqual(item("1")); // t.expect(await grouping.item("1")).toStrictEqual(item("1"));
t.expect(mocked).toHaveBeenLastCalledWith(["1", "3", "4"]); // t.expect(mocked).toHaveBeenLastCalledWith(["1", "3", "4"]);
grouping.refresh([ // grouping.refresh([
"1", // "1",
"2", // "2",
"3", // "3",
"4", // "4",
"5", // "5",
"6", // "6",
"7", // "7",
"8", // "8",
"9", // "9",
"10", // "10",
"11", // "11",
"12" // "12"
]); // ]);
t.expect(await grouping.item("1")).toStrictEqual(item("1")); // t.expect(await grouping.item("1")).toStrictEqual(item("1"));
t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]); // t.expect(mocked).toHaveBeenLastCalledWith(["1", "2", "3"]);
}); // });

55
packages/core/src/utils/grouping.ts
View File

@@ -96,37 +96,42 @@ export function groupArray(
sortBy: "dateEdited", sortBy: "dateEdited",
sortDirection: "desc" sortDirection: "desc"
} }
): (string | GroupHeader)[] { ): { index: number; group: GroupHeader }[] {
const groups = new Map<string, string[]>([ const groups = new Map<string, number>();
["Conflicted", []], // [
["Pinned", []] // ["Conflicted", 0],
]); // ["Pinned", 1]
// ]
const keySelector = getKeySelector(options); const keySelector = getKeySelector(options);
for (const item of items) { for (let i = 0; i < items.length; ++i) {
const item = items[i];
const groupTitle = keySelector(item); const groupTitle = keySelector(item);
const group = groups.get(groupTitle) || []; const group = groups.get(groupTitle);
group.push(item.id); if (typeof group === "undefined") groups.set(groupTitle, i);
groups.set(groupTitle, group);
} }
const groupIndices: { index: number; group: GroupHeader }[] = [];
return flattenGroups(groups); groups.forEach((index, title) =>
groupIndices.push({ index, group: { id: title, title, type: "header" } })
);
return groupIndices;
// return flattenGroups(groups);
} }
function flattenGroups(groups: Map<string, string[]>) { // function flattenGroups<T extends GroupableItem>(groups: Map<string, T[]>) {
const items: (string | GroupHeader)[] = []; // const items: GroupedItems<T> = [];
groups.forEach((groupItems, groupTitle) => { // groups.forEach((groupItems, groupTitle) => {
if (groupItems.length <= 0) return; // if (groupItems.length <= 0) return;
items.push({ // items.push({
title: groupTitle, // title: groupTitle,
id: groupTitle.toLowerCase(), // id: groupTitle.toLowerCase(),
type: "header" // type: "header"
}); // });
items.push(...groupItems); // items.push(...groupItems);
}); // });
return items; // return items;
} // }
function getFirstCharacter(str: string) { function getFirstCharacter(str: string) {
if (!str) return "-"; if (!str) return "-";
@@ -136,5 +141,5 @@ function getFirstCharacter(str: string) {
} }
function getTitle(item: PartialGroupableItem): string { function getTitle(item: PartialGroupableItem): string {
return item.filename || item.title || "Unknown"; return ("filename" in item ? item.filename : item.title) || "Unknown";
} }

162
packages/core/src/utils/virtualized-grouping.ts
View File

@@ -17,61 +17,78 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
import { GroupHeader, isGroupHeader } from "../types"; import { GroupHeader } from "../types";
type BatchOperator<T> = ( type BatchOperator<T> = (ids: string[], items: T[]) => Promise<unknown[]>;
ids: string[], type Batch<T> = {
items: Record<string, T> items: T[];
) => Promise<Record<string, unknown>>; groups?: { index: number; hidden?: boolean; group: GroupHeader }[];
type Batch<T> = { items: Record<string, T>; data?: Record<string, unknown> }; data?: unknown[];
};
export class VirtualizedGrouping<T> { export class VirtualizedGrouping<T> {
private cache: Map<number, Batch<T>> = new Map(); private cache: Map<number, Batch<T>> = new Map();
private pending: Map<number, Promise<Batch<T>>> = new Map(); private pending: Map<number, Promise<Batch<T>>> = new Map();
groups: GroupHeader[] = []; public ids: number[];
private loadBatchTimeout?: number;
private cacheHits = 0;
constructor( constructor(
public ids: (string | GroupHeader)[], count: number,
private readonly batchSize: number, private readonly batchSize: number,
private readonly fetchItems: (ids: string[]) => Promise<Record<string, T>> private readonly fetchItems: (
start: number,
end: number,
cursor?: T
) => Promise<{ ids: string[]; items: T[] }>,
private readonly groupItems?: (
items: T[]
) => { index: number; hidden?: boolean; group: GroupHeader }[],
readonly groups?: () => Promise<{ index: number; group: GroupHeader }[]>
) { ) {
this.ids = ids; this.ids = new Array(count).fill(0);
this.groups = ids.filter((i) => isGroupHeader(i)) as GroupHeader[];
} }
getKey(index: number) { getKey(index: number) {
const item = this.ids[index];
if (isGroupHeader(item)) return item.id;
return item;
}
get ungrouped() {
return this.ids.filter((i) => !isGroupHeader(i)) as string[];
}
/**
* Get item from cache or request the appropriate batch for caching
* and load it from there.
*/
item(id: string): Promise<T | undefined>;
item(
id: string,
operate: BatchOperator<T>
): Promise<{ item: T; data: unknown } | undefined>;
async item(id: string, operate?: BatchOperator<T>) {
const index = this.ids.indexOf(id);
if (index <= -1) return;
const batchIndex = Math.floor(index / this.batchSize); const batchIndex = Math.floor(index / this.batchSize);
const { items, data } = const batch = this.cache.get(batchIndex);
if (!batch) return `${index}`;
const { items, groups } = batch;
const itemIndexInBatch = index - batchIndex * this.batchSize;
const group = groups?.find(
(f) => f.index === itemIndexInBatch && !f.hidden
);
return group
? group.group.id
: (items[itemIndexInBatch] as any)?.id || `${index}`;
}
item(index: number): Promise<{ item: T; group?: GroupHeader }>;
item(
index: number,
operate: BatchOperator<T>
): Promise<{ item: T; group?: GroupHeader; data: unknown }>;
async item(index: number, operate?: BatchOperator<T>) {
const batchIndex = Math.floor(index / this.batchSize);
if (this.cache.has(batchIndex)) this.cacheHits++;
const { items, groups, data } =
this.cache.get(batchIndex) || (await this.loadBatch(batchIndex, operate)); this.cache.get(batchIndex) || (await this.loadBatch(batchIndex, operate));
return operate ? { item: items[id], data: data?.[id] } : items[id]; const itemIndexInBatch = index - batchIndex * this.batchSize;
const group = groups?.find(
(f) => f.index === itemIndexInBatch && !f.hidden
);
return {
item: items[itemIndexInBatch],
group: group?.group,
data: data?.[itemIndexInBatch]
};
} }
/** /**
* Reload the cache * Reload the cache
*/ */
refresh(ids: (string | GroupHeader)[]) { refresh(ids: number[]) {
this.ids = ids; this.ids = ids;
this.cache.clear(); this.cache.clear();
} }
@@ -80,19 +97,49 @@ export class VirtualizedGrouping<T> {
* *
* @param index * @param index
*/ */
private async load(batchIndex: number, operate?: BatchOperator<T>) { private async load(
batchIndex: number,
operate?: BatchOperator<T>
): Promise<Batch<T>> {
const lastBatchIndex = this.last;
const prev = this.cache.get(lastBatchIndex);
const start = batchIndex * this.batchSize; const start = batchIndex * this.batchSize;
const end = start + this.batchSize; const end = start + this.batchSize;
const batchIds = this.ids // we can use a cursor instead of start/end offsets for batches that are
.slice(start, end) // right next to each other.
.filter((id) => typeof id === "string") as string[]; const cursor =
const items = await this.fetchItems(batchIds); lastBatchIndex + 1 === batchIndex
console.time("operate"); ? prev?.items.at(-1)
: lastBatchIndex - 1 === batchIndex
? prev?.items[0]
: undefined;
const { ids, items } = await this.fetchItems(start, end, cursor);
const groups = this.groupItems?.(items);
if (
prev &&
prev.groups &&
prev.groups.length > 0 &&
groups &&
groups.length > 0
) {
// if user is moving downwards, we hide the first group from the
// current batch, otherwise we hide the last group from the previous
// batch.
const group =
lastBatchIndex < batchIndex
? groups[0] //groups.length - 1]
: prev.groups[prev.groups.length - 1];
if (group.group.title === groups[0].group.title) {
group.hidden = true;
}
}
const batch = { const batch = {
items, items,
data: operate ? await operate(batchIds, items) : undefined groups,
data: operate ? await operate(ids, items) : undefined
}; };
console.timeEnd("operate");
this.cache.set(batchIndex, batch); this.cache.set(batchIndex, batch);
this.clear(); this.clear();
return batch; return batch;
@@ -100,12 +147,18 @@ export class VirtualizedGrouping<T> {
private loadBatch(batch: number, operate?: BatchOperator<T>) { private loadBatch(batch: number, operate?: BatchOperator<T>) {
if (this.pending.has(batch)) return this.pending.get(batch)!; if (this.pending.has(batch)) return this.pending.get(batch)!;
console.time("loading batch"); if (!this.isLastBatch(batch)) clearTimeout(this.loadBatchTimeout);
const promise = this.load(batch, operate); return new Promise<Batch<T>>((resolve, reject) => {
this.pending.set(batch, promise); this.loadBatchTimeout = setTimeout(() => {
return promise.finally(() => { const promise = this.load(batch, operate);
console.timeEnd("loading batch"); this.pending.set(batch, promise);
this.pending.delete(batch); return promise
.then(resolve)
.catch(reject)
.finally(() => {
this.pending.delete(batch);
});
}, 16) as unknown as number;
}); });
} }
@@ -116,4 +169,13 @@ export class VirtualizedGrouping<T> {
if (this.cache.size === 2) break; if (this.cache.size === 2) break;
} }
} }
private get last() {
const keys = Array.from(this.cache.keys());
return keys[keys.length - 1];
}
private isLastBatch(batch: number) {
return Math.floor(this.ids.length / this.batchSize) === batch;
}
} }