From 3179c0ca2aa487debc737b02830a5dd6d9142406 Mon Sep 17 00:00:00 2001 From: Ammar Ahmed Date: Thu, 16 Nov 2023 08:54:37 +0500 Subject: [PATCH] mobile: migrate to sqlite --- apps/mobile/app/app.tsx | 1 + apps/mobile/app/common/database/index.js | 18 +- apps/mobile/app/common/database/logger.js | 4 +- .../app/common/database/sqlite.kysely.ts | 123 + .../common/filesystem/download-attachment.js | 2 +- .../attachments/{actions.js => actions.tsx} | 142 +- ...attachment-item.js => attachment-item.tsx} | 42 +- ...ttachments.js => download-attachments.tsx} | 69 +- .../attachments/{index.js => index.tsx} | 150 +- .../components/container/floating-button.tsx | 2 +- .../app/components/dialog-provider/index.js | 7 +- .../components/dialog-provider/recievers.js | 8 - .../app/components/dialogs/add-topic/index.js | 194 - .../jump-to-section/{index.js => index.tsx} | 125 +- .../app/components/image-preview/index.tsx | 1 - .../headers/{header.js => header.tsx} | 54 +- ...notebook-header.js => notebook-header.tsx} | 33 +- .../{section-header.js => section-header.tsx} | 85 +- .../list-items/note/{index.js => index.tsx} | 204 +- .../note/{wrapper.js => wrapper.tsx} | 109 +- .../notebook/{index.js => index.tsx} | 82 +- .../notebook/{wrapper.js => wrapper.tsx} | 52 +- .../components/list-items/reminder/index.tsx | 8 +- .../{back-fill.js => back-fill.tsx} | 3 +- .../selection-wrapper/{index.js => index.tsx} | 36 +- .../{selection.js => selection.tsx} | 7 +- .../list-items/tag/{index.js => index.tsx} | 34 +- .../app/components/list/{card.js => card.tsx} | 7 +- .../components/list/{empty.js => empty.tsx} | 62 +- .../components/list/{index.js => index.tsx} | 237 +- .../app/components/list/list-item.wrapper.tsx | 269 + .../app/components/note-history/preview.js | 5 + apps/mobile/app/components/premium/pro-tag.js | 5 + .../app/components/properties/color-tags.tsx | 88 +- .../app/components/properties/date-meta.js | 35 +- .../mobile/app/components/properties/index.js | 38 +- .../mobile/app/components/properties/items.js | 1 + .../app/components/properties/notebooks.js | 40 +- apps/mobile/app/components/properties/tags.js | 1 + .../app/components/properties/topics.js | 70 - .../components/sheets/add-notebook/index.js | 560 - .../components/sheets/add-notebook/index.tsx | 173 + .../app/components/sheets/add-to/context.js | 8 +- .../sheets/add-to/{index.js => index.tsx} | 262 +- .../export-notes/{index.js => index.tsx} | 53 +- .../components/sheets/manage-tags/index.tsx | 230 +- .../components/sheets/move-notes/movenote.tsx | 333 +- .../{topic-sheet => notebook-sheet}/index.tsx | 462 +- .../publish-note/{index.js => index.tsx} | 53 +- .../sheets/relations-list/index.tsx | 38 +- .../sheets/reminder-notify/index.tsx | 42 +- .../app/components/sheets/reminder/index.tsx | 8 +- .../components/sheets/restore-data/index.js | 6 +- .../app/components/sheets/sort/index.js | 2 +- .../{color-section.js => color-section.tsx} | 23 +- .../{pinned-section.js => pinned-section.tsx} | 38 +- apps/mobile/app/components/tip/index.tsx | 10 +- apps/mobile/app/components/ui/sheet/index.js | 5 + apps/mobile/app/hooks/use-actions.tsx | 216 +- apps/mobile/app/hooks/use-app-events.tsx | 16 +- .../app/hooks/use-attachment-progress.ts | 5 +- apps/mobile/app/hooks/use-db-item.ts | 131 + .../app/hooks/use-is-compact-mode-enabled.ts | 9 +- apps/mobile/app/hooks/use-notebook.ts | 87 + .../mobile/app/navigation/navigation-stack.js | 4 +- .../app/screens/editor/tiptap/use-editor.ts | 23 +- apps/mobile/app/screens/favorites/index.tsx | 23 +- apps/mobile/app/screens/home/index.tsx | 26 +- apps/mobile/app/screens/notebook/index.tsx | 102 +- apps/mobile/app/screens/notebooks/index.tsx | 44 +- apps/mobile/app/screens/notes/colored.tsx | 15 +- apps/mobile/app/screens/notes/common.ts | 24 +- apps/mobile/app/screens/notes/index.tsx | 109 +- apps/mobile/app/screens/notes/monographs.tsx | 14 +- apps/mobile/app/screens/notes/tagged.tsx | 18 +- apps/mobile/app/screens/notes/topic-notes.tsx | 11 +- apps/mobile/app/screens/reminders/index.tsx | 14 +- apps/mobile/app/screens/tags/index.tsx | 23 +- apps/mobile/app/screens/trash/index.tsx | 29 +- apps/mobile/app/services/backup.js | 4 +- apps/mobile/app/services/notifications.ts | 50 +- apps/mobile/app/services/tip-manager.ts | 2 +- apps/mobile/app/stores/use-favorite-store.ts | 36 +- apps/mobile/app/stores/use-menu-store.ts | 26 +- apps/mobile/app/stores/use-message-store.ts | 4 +- apps/mobile/app/stores/use-notebook-store.ts | 36 +- apps/mobile/app/stores/use-notes-store.ts | 35 +- apps/mobile/app/stores/use-reminder-store.ts | 19 +- apps/mobile/app/stores/use-tag-store.ts | 30 +- apps/mobile/app/stores/use-trash-store.ts | 29 +- apps/mobile/app/utils/events.js | 7 +- apps/mobile/app/utils/functions.js | 82 +- apps/mobile/app/utils/notebooks.ts | 32 + apps/mobile/native/android/gradle.properties | 3 +- apps/mobile/native/babel.config.js | 30 +- apps/mobile/native/globals.js | 2 +- apps/mobile/native/ios/Podfile | 7 + apps/mobile/native/metro.config.js | 9 + apps/mobile/native/package.json | 3 +- apps/mobile/native/polyfills/console-time.js | 50 + apps/mobile/native/webpack.config.js | 9 + apps/mobile/package-lock.json | 70323 ++++++++++++++-- apps/mobile/package.json | 4 +- apps/theme-builder/package-lock.json | 2 + apps/web/package-lock.json | 19 +- packages/common/package-lock.json | 2 + packages/core/src/api/index.ts | 2 - packages/core/src/collections/attachments.ts | 2 +- packages/core/src/collections/relations.ts | 13 + packages/editor/package-lock.json | 2 + 110 files changed, 64666 insertions(+), 11910 deletions(-) create mode 100644 apps/mobile/app/common/database/sqlite.kysely.ts rename apps/mobile/app/components/attachments/{actions.js => actions.tsx} (74%) rename apps/mobile/app/components/attachments/{attachment-item.js => attachment-item.tsx} (80%) rename apps/mobile/app/components/attachments/{download-attachments.js => download-attachments.tsx} (81%) rename apps/mobile/app/components/attachments/{index.js => index.tsx} (71%) delete mode 100644 apps/mobile/app/components/dialogs/add-topic/index.js rename apps/mobile/app/components/dialogs/jump-to-section/{index.js => index.tsx} (69%) rename apps/mobile/app/components/list-items/headers/{header.js => header.tsx} (62%) rename apps/mobile/app/components/list-items/headers/{notebook-header.js => notebook-header.tsx} (89%) rename apps/mobile/app/components/list-items/headers/{section-header.js => section-header.tsx} (69%) rename apps/mobile/app/components/list-items/note/{index.js => index.tsx} (69%) rename apps/mobile/app/components/list-items/note/{wrapper.js => wrapper.tsx} (54%) rename apps/mobile/app/components/list-items/notebook/{index.js => index.tsx} (73%) rename apps/mobile/app/components/list-items/notebook/{wrapper.js => wrapper.tsx} (76%) rename apps/mobile/app/components/list-items/selection-wrapper/{back-fill.js => back-fill.tsx} (93%) rename apps/mobile/app/components/list-items/selection-wrapper/{index.js => index.tsx} (81%) rename apps/mobile/app/components/list-items/selection-wrapper/{selection.js => selection.tsx} (90%) rename apps/mobile/app/components/list-items/tag/{index.js => index.tsx} (83%) rename apps/mobile/app/components/list/{card.js => card.tsx} (94%) rename apps/mobile/app/components/list/{empty.js => empty.tsx} (81%) rename apps/mobile/app/components/list/{index.js => index.tsx} (51%) create mode 100644 apps/mobile/app/components/list/list-item.wrapper.tsx delete mode 100644 apps/mobile/app/components/properties/topics.js delete mode 100644 apps/mobile/app/components/sheets/add-notebook/index.js create mode 100644 apps/mobile/app/components/sheets/add-notebook/index.tsx rename apps/mobile/app/components/sheets/add-to/{index.js => index.tsx} (60%) rename apps/mobile/app/components/sheets/export-notes/{index.js => index.tsx} (90%) rename apps/mobile/app/components/sheets/{topic-sheet => notebook-sheet}/index.tsx (56%) rename apps/mobile/app/components/sheets/publish-note/{index.js => index.tsx} (90%) rename apps/mobile/app/components/side-menu/{color-section.js => color-section.tsx} (90%) rename apps/mobile/app/components/side-menu/{pinned-section.js => pinned-section.tsx} (91%) create mode 100644 apps/mobile/app/hooks/use-db-item.ts create mode 100644 apps/mobile/app/hooks/use-notebook.ts create mode 100644 apps/mobile/app/utils/notebooks.ts create mode 100644 apps/mobile/native/polyfills/console-time.js diff --git a/apps/mobile/app/app.tsx b/apps/mobile/app/app.tsx index 53428ff1c..70cb21005 100644 --- a/apps/mobile/app/app.tsx +++ b/apps/mobile/app/app.tsx @@ -50,6 +50,7 @@ const App = () => { if (appLockMode && appLockMode !== "none") { useUserStore.getState().lockApp(true); } + //@ts-ignore globalThis["IS_MAIN_APP_RUNNING"] = true; init(); setTimeout(async () => { diff --git a/apps/mobile/app/common/database/index.js b/apps/mobile/app/common/database/index.js index 1c950ad0f..a7331e2cd 100644 --- a/apps/mobile/app/common/database/index.js +++ b/apps/mobile/app/common/database/index.js @@ -23,9 +23,10 @@ import { Platform } from "react-native"; import * as Gzip from "react-native-gzip"; import EventSource from "../../utils/sse/even-source-ios"; import AndroidEventSource from "../../utils/sse/event-source"; +import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely"; import filesystem from "../filesystem"; - import Storage from "./storage"; +import { RNSqliteDriver } from "./sqlite.kysely"; database.host( __DEV__ @@ -57,6 +58,21 @@ database.setup({ compressor: { compress: Gzip.deflate, decompress: Gzip.inflate + }, + batchSize: 500, + sqliteOptions: { + dialect: { + createDriver: () => + new RNSqliteDriver({ async: true, dbName: "test.db" }), + createAdapter: () => new SqliteAdapter(), + createIntrospector: (db) => new SqliteIntrospector(db), + createQueryCompiler: () => new SqliteQueryCompiler() + } + // journalMode: "MEMORY", + // synchronous: "normal", + // pageSize: 8192, + // cacheSize: -16000, + // lockingMode: "exclusive" } }); diff --git a/apps/mobile/app/common/database/logger.js b/apps/mobile/app/common/database/logger.js index 429188dec..aed3bfa5e 100644 --- a/apps/mobile/app/common/database/logger.js +++ b/apps/mobile/app/common/database/logger.js @@ -16,14 +16,14 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ -import { initalize } from "@notesnook/core/dist/logger"; import { MMKVLoader } from "react-native-mmkv-storage"; +import { initialize } from "@notesnook/core/dist/logger"; import { KV } from "./storage"; const LoggerStorage = new MMKVLoader() .withInstanceID("notesnook_logs") .initialize(); -initalize(new KV(LoggerStorage)); +initialize(new KV(LoggerStorage)); export { LoggerStorage }; diff --git a/apps/mobile/app/common/database/sqlite.kysely.ts b/apps/mobile/app/common/database/sqlite.kysely.ts new file mode 100644 index 000000000..6c19f2144 --- /dev/null +++ b/apps/mobile/app/common/database/sqlite.kysely.ts @@ -0,0 +1,123 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import type { DatabaseConnection, Driver, QueryResult } from "kysely"; +import { CompiledQuery } from "kysely"; +import { QuickSQLiteConnection, open } from "react-native-quick-sqlite"; + +type Config = { dbName: string; async: boolean; location: string }; + +export class RNSqliteDriver implements Driver { + private connection?: DatabaseConnection; + private connectionMutex = new ConnectionMutex(); + private db: QuickSQLiteConnection; + constructor(private readonly config: Config) { + this.db = open({ + name: config.dbName + }); + } + + async init(): Promise { + this.connection = new RNSqliteConnection(this.db); + } + + async acquireConnection(): Promise { + // SQLite only has one single connection. We use a mutex here to wait + // until the single connection has been released. + await this.connectionMutex.lock(); + return this.connection!; + } + + async beginTransaction(connection: DatabaseConnection): Promise { + await connection.executeQuery(CompiledQuery.raw("begin")); + } + + async commitTransaction(connection: DatabaseConnection): Promise { + await connection.executeQuery(CompiledQuery.raw("commit")); + } + + async rollbackTransaction(connection: DatabaseConnection): Promise { + await connection.executeQuery(CompiledQuery.raw("rollback")); + } + + async releaseConnection(): Promise { + this.connectionMutex.unlock(); + } + + async destroy(): Promise { + this.db.close(); + } +} + +class ConnectionMutex { + private promise?: Promise; + private resolve?: () => void; + + async lock(): Promise { + while (this.promise) { + await this.promise; + } + + this.promise = new Promise((resolve) => { + this.resolve = resolve; + }); + } + + unlock(): void { + const resolve = this.resolve; + + this.promise = undefined; + this.resolve = undefined; + + resolve?.(); + } +} + +class RNSqliteConnection implements DatabaseConnection { + constructor(private readonly db: QuickSQLiteConnection) {} + + streamQuery(): AsyncIterableIterator> { + throw new Error("wasqlite driver doesn't support streaming"); + } + + async executeQuery( + compiledQuery: CompiledQuery + ): Promise> { + const { parameters, sql, query } = compiledQuery; + const mode = + query.kind === "SelectQueryNode" + ? "query" + : query.kind === "RawNode" + ? "raw" + : "exec"; + const result = await this.db.executeAsync(sql, parameters as any[]); + + console.log("SQLITE result:", result?.rows?._array); + if (mode === "query" || !result.insertId) + return { + rows: result.rows?._array || [] + }; + + return { + insertId: BigInt(result.insertId), + numAffectedRows: BigInt(result.rowsAffected), + rows: mode === "raw" ? result.rows?._array || [] : [] + }; + } +} diff --git a/apps/mobile/app/common/filesystem/download-attachment.js b/apps/mobile/app/common/filesystem/download-attachment.js index e603055b8..0f5d731ff 100644 --- a/apps/mobile/app/common/filesystem/download-attachment.js +++ b/apps/mobile/app/common/filesystem/download-attachment.js @@ -189,7 +189,7 @@ export default async function downloadAttachment( ) { await createCacheDir(); - let attachment = db.attachments.attachment(hash); + let attachment = await db.attachments.attachment(hash); if (!attachment) { console.log("attachment not found"); return; diff --git a/apps/mobile/app/components/attachments/actions.js b/apps/mobile/app/components/attachments/actions.tsx similarity index 74% rename from apps/mobile/app/components/attachments/actions.js rename to apps/mobile/app/components/attachments/actions.tsx index 91bba578c..1f695cb44 100644 --- a/apps/mobile/app/components/attachments/actions.js +++ b/apps/mobile/app/components/attachments/actions.tsx @@ -17,9 +17,13 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +import { formatBytes } from "@notesnook/common"; +import { Attachment, Note, VirtualizedGrouping } from "@notesnook/core"; +import { useThemeColors } from "@notesnook/theme"; import Clipboard from "@react-native-clipboard/clipboard"; -import React, { useCallback, useEffect, useState } from "react"; +import React, { RefObject, useEffect, useState } from "react"; import { View } from "react-native"; +import { ActionSheetRef } from "react-native-actions-sheet"; import { ScrollView } from "react-native-gesture-handler"; import { db } from "../../common/database"; import filesystem from "../../common/filesystem"; @@ -27,14 +31,17 @@ import downloadAttachment from "../../common/filesystem/download-attachment"; import { useAttachmentProgress } from "../../hooks/use-attachment-progress"; import picker from "../../screens/editor/tiptap/picker"; import { + ToastManager, eSendEvent, - presentSheet, - ToastManager + presentSheet } from "../../services/event-manager"; import PremiumService from "../../services/premium"; import { useAttachmentStore } from "../../stores/use-attachment-store"; -import { useThemeColors } from "@notesnook/theme"; -import { eCloseAttachmentDialog, eCloseSheet } from "../../utils/events"; +import { + eCloseAttachmentDialog, + eCloseSheet, + eDBItemUpdate +} from "../../utils/events"; import { SIZE } from "../../utils/size"; import { sleep } from "../../utils/time"; import { Dialog } from "../dialog"; @@ -46,28 +53,37 @@ import { Notice } from "../ui/notice"; import { PressableButton } from "../ui/pressable"; import Heading from "../ui/typography/heading"; import Paragraph from "../ui/typography/paragraph"; -import { formatBytes } from "@notesnook/common"; -const Actions = ({ attachment, setAttachments, fwdRef, close }) => { +const Actions = ({ + attachment, + close, + setAttachments, + fwdRef +}: { + attachment: Attachment; + setAttachments: (attachments?: VirtualizedGrouping) => void; + close: () => void; + fwdRef: RefObject; +}) => { const { colors } = useThemeColors(); - const contextId = attachment.metadata.hash; - const [filename, setFilename] = useState(attachment.metadata.filename); + const contextId = attachment.hash; + const [filename, setFilename] = useState(attachment.filename); const [currentProgress] = useAttachmentProgress(attachment); - const [failed, setFailed] = useState(attachment.failed); - const [notes, setNotes] = useState([]); - const [loading, setLoading] = useState({ - name: null - }); + const [failed, setFailed] = useState(attachment.failed); + const [notes, setNotes] = useState([]); + const [loading, setLoading] = useState<{ + name?: string; + }>({}); const actions = [ { name: "Download", onPress: async () => { if (currentProgress) { - await db.fs().cancel(attachment.metadata.hash); - useAttachmentStore.getState().remove(attachment.metadata.hash); + await db.fs().cancel(attachment.hash); + useAttachmentStore.getState().remove(attachment.hash); } - downloadAttachment(attachment.metadata.hash, false); + downloadAttachment(attachment.hash, false); eSendEvent(eCloseSheet, contextId); }, icon: "download" @@ -85,9 +101,9 @@ const Actions = ({ attachment, setAttachments, fwdRef, close }) => { } await picker.pick({ reupload: true, - hash: attachment.metadata.hash, + hash: attachment.hash, context: contextId, - type: attachment.metadata.type + type: attachment.type }); }, icon: "upload" @@ -98,7 +114,7 @@ const Actions = ({ attachment, setAttachments, fwdRef, close }) => { setLoading({ name: "Run file check" }); - let res = await filesystem.checkAttachment(attachment.metadata.hash); + let res = await filesystem.checkAttachment(attachment.hash); if (res.failed) { db.attachments.markAsFailed(attachment.id, res.failed); setFailed(res.failed); @@ -108,8 +124,9 @@ const Actions = ({ attachment, setAttachments, fwdRef, close }) => { context: "local" }); } else { - setFailed(null); - db.attachments.markAsFailed(attachment.id, null); + setFailed(undefined); + db.attachments.markAsFailed(attachment.id); + eSendEvent(eDBItemUpdate, attachment.id); ToastManager.show({ heading: "File check passed", type: "success", @@ -119,7 +136,7 @@ const Actions = ({ attachment, setAttachments, fwdRef, close }) => { setAttachments(); setLoading({ - name: null + name: undefined }); }, icon: "file-check" @@ -128,19 +145,20 @@ const Actions = ({ attachment, setAttachments, fwdRef, close }) => { name: "Rename", onPress: () => { presentDialog({ - context: contextId, + context: contextId as any, input: true, title: "Rename file", paragraph: "Enter a new name for the file", - defaultValue: attachment.metadata.filename, + defaultValue: attachment.filename, positivePress: async (value) => { if (value && value.length > 0) { await db.attachments.add({ - hash: attachment.metadata.hash, + hash: attachment.hash, filename: value }); setFilename(value); setAttachments(); + eSendEvent(eDBItemUpdate, attachment.id); } }, positiveText: "Rename" @@ -151,34 +169,23 @@ const Actions = ({ attachment, setAttachments, fwdRef, close }) => { { name: "Delete", onPress: async () => { - await db.attachments.remove(attachment.metadata.hash, false); + await db.attachments.remove(attachment.hash, false); setAttachments(); + eSendEvent(eDBItemUpdate, attachment.id); close(); }, icon: "delete-outline" } ]; - const getNotes = useCallback(() => { - let allNotes = db.notes.all; - let attachmentNotes = attachment.noteIds?.map((id) => { - let index = allNotes?.findIndex((note) => id === note.id); - if (index !== -1) { - return allNotes[index]; - } else { - return { - type: "notfound", - title: `Note with id ${id} does not exist.`, - id: id - }; - } - }); - return attachmentNotes; - }, [attachment.noteIds]); - useEffect(() => { - setNotes(getNotes()); - }, [attachment, getNotes]); + db.relations + .to(attachment, "note") + .selector.items() + .then((items) => { + setNotes(items); + }); + }, [attachment]); return ( { }} color={colors.secondary.paragraph} > - {attachment.metadata.type} + {attachment.type} { size={SIZE.xs} color={colors.secondary.paragraph} > - {formatBytes(attachment.length)} + {formatBytes(attachment.size)} - {attachment.noteIds ? ( + {notes.length ? ( { size={SIZE.xs} color={colors.secondary.paragraph} > - {attachment.noteIds.length} note - {attachment.noteIds.length > 1 ? "s" : ""} + {notes.length} note + {notes.length > 1 ? "s" : ""} ) : null} { - Clipboard.setString(attachment.metadata.hash); + Clipboard.setString(attachment.hash); ToastManager.show({ type: "success", heading: "Attachment hash copied", @@ -257,7 +264,7 @@ const Actions = ({ attachment, setAttachments, fwdRef, close }) => { size={SIZE.xs} color={colors.secondary.paragraph} > - {attachment.metadata.hash} + {attachment.hash} @@ -286,21 +293,11 @@ const Actions = ({ attachment, setAttachments, fwdRef, close }) => { {notes.map((item) => ( { - if (item.type === "notfound") { - ToastManager.show({ - heading: "Note not found", - message: - "A note with the given id was not found. Maybe you have deleted the note or moved it to trash already.", - type: "error", - context: "local" - }); - return; - } eSendEvent(eCloseSheet, contextId); await sleep(150); eSendEvent(eCloseAttachmentDialog); await sleep(300); - openNote(item, item.type === "trash"); + openNote(item, (item as any).type === "trash"); }} customStyle={{ paddingVertical: 12, @@ -321,17 +318,16 @@ const Actions = ({ attachment, setAttachments, fwdRef, close }) => {