web: use lazy loading to load attachments

This commit is contained in:
Abdullah Atta
2023-11-10 15:33:14 +05:00
parent a77c88bddd
commit 4556e4d230
9 changed files with 379 additions and 317 deletions

View File

@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { showMultiDeleteConfirmation } from "./dialog-controller";
import { confirm, showMultiDeleteConfirmation } from "./dialog-controller";
import { store as noteStore } from "../stores/note-store";
import { store as notebookStore } from "../stores/notebook-store";
import { store as attachmentStore } from "../stores/attachment-store";
@@ -27,12 +27,7 @@ import { showToast } from "../utils/toast";
import Vault from "./vault";
import { TaskManager } from "./task-manager";
import { pluralize } from "@notesnook/common";
type Item = {
id: string;
locked?: boolean;
metadata?: Record<string, unknown>;
};
import { Reminder } from "@notesnook/core";
async function moveNotesToTrash(ids: string[], confirm = true) {
if (confirm && !(await showMultiDeleteConfirmation(ids.length))) return;
@@ -77,27 +72,13 @@ async function moveNotebooksToTrash(ids: string[]) {
showToast("success", `${pluralize(ids.length, "notebook")} moved to trash`);
}
async function deleteTopics(notebookId: string, topics: Item[]) {
await TaskManager.startTask({
type: "status",
id: "deleteTopics",
action: async (report) => {
report({
text: `Deleting ${pluralize(topics.length, "topic")}...`
});
await db.notebooks.topics(notebookId).delete(...topics.map((t) => t.id));
notebookStore.setSelectedNotebook(notebookId);
noteStore.refresh();
}
});
showToast("success", `${pluralize(topics.length, "topic")} deleted`);
}
async function deleteAttachments(attachments: Item[]) {
async function deleteAttachments(ids: string[]) {
if (
!window.confirm(
"Are you sure you want to permanently delete these attachments? This action is IRREVERSIBLE."
)
!(await confirm({
title: "Are you sure?",
message:
"Are you sure you want to permanently delete these attachments? This action is IRREVERSIBLE."
}))
)
return;
@@ -105,24 +86,24 @@ async function deleteAttachments(attachments: Item[]) {
type: "status",
id: "deleteAttachments",
action: async (report) => {
for (let i = 0; i < attachments.length; ++i) {
const attachment = attachments[i];
for (let i = 0; i < ids.length; ++i) {
const id = ids[i];
const attachment = await attachmentStore.get().attachments?.item(id);
if (!attachment) continue;
report({
text: `Deleting ${pluralize(attachments.length, "attachment")}...`,
text: `Deleting ${pluralize(ids.length, "attachment")}...`,
current: i,
total: attachments.length
total: ids.length
});
await attachmentStore.permanentDelete(attachment.metadata?.hash);
await attachmentStore.permanentDelete(attachment);
}
}
});
showToast(
"success",
`${pluralize(attachments.length, "attachment")} deleted`
);
showToast("success", `${pluralize(ids.length, "attachment")} deleted`);
}
async function moveRemindersToTrash(reminders: Item[]) {
async function moveRemindersToTrash(reminders: Reminder[]) {
const isMultiselect = reminders.length > 1;
if (isMultiselect) {
if (!(await showMultiDeleteConfirmation(reminders.length))) return;
@@ -146,6 +127,5 @@ export const Multiselect = {
moveRemindersToTrash,
moveNotebooksToTrash,
moveNotesToTrash,
deleteTopics,
deleteAttachments
};

View File

@@ -59,6 +59,7 @@ import { useEffect, useState } from "react";
import { AppEventManager, AppEvents } from "../../common/app-events";
import { getFormattedDate } from "@notesnook/common";
import { MenuItem } from "@notesnook/ui";
import { Attachment } from "@notesnook/core";
const FILE_ICONS: Record<string, Icon> = {
"image/": FileImage,
@@ -85,7 +86,7 @@ type AttachmentProgressStatus = {
};
type AttachmentProps = {
attachment: any;
attachment: Attachment;
isSelected?: boolean;
onSelected?: () => void;
compact?: boolean;

View File

@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { useEffect, useState, memo, useMemo, useRef } from "react";
import { useEffect, useState, memo, useRef } from "react";
import {
Box,
Button,
@@ -28,11 +28,10 @@ import {
Label,
Text
} from "@theme-ui/components";
import { getTotalSize } from "../common/attachments";
import { useStore, store } from "../stores/attachment-store";
import { store, useStore } from "../stores/attachment-store";
import { formatBytes } from "@notesnook/common";
import Dialog from "../components/dialog";
import { TableVirtuoso } from "react-virtuoso";
import { ItemProps, TableVirtuoso } from "react-virtuoso";
import {
ChevronDown,
ChevronUp,
@@ -52,21 +51,20 @@ import NavigationItem from "../components/navigation-menu/navigation-item";
import { pluralize } from "@notesnook/common";
import { db } from "../common/db";
import { Perform } from "../common/dialog-controller";
import { Multiselect } from "../common/multi-select";
import { CustomScrollbarsVirtualList } from "../components/list-container";
import { Attachment } from "../components/attachment";
import {
isDocument,
isImage,
isVideo
} from "@notesnook/core/dist/utils/filename";
import { alpha } from "@theme-ui/color";
import { ScopedThemeProvider } from "../components/theme-provider";
import {
Attachment as AttachmentType,
VirtualizedGrouping
} from "@notesnook/core";
import usePromise from "../hooks/use-promise";
import { Multiselect } from "../common/multi-select";
type ToolbarAction = {
title: string;
icon: Icon;
onClick: ({ selected }: { selected: any[] }) => void;
onClick: ({ selected }: { selected: string[] }) => void;
};
const TOOLBAR_ACTIONS: ToolbarAction[] = [
@@ -80,9 +78,7 @@ const TOOLBAR_ACTIONS: ToolbarAction[] = [
{
title: "Recheck",
icon: DoubleCheckmark,
onClick: async ({ selected }) => {
await store.recheck(selected.map((a) => a.metadata.hash));
}
onClick: async ({ selected }) => await store.recheck(selected)
},
{
title: "Delete",
@@ -91,53 +87,65 @@ const TOOLBAR_ACTIONS: ToolbarAction[] = [
}
];
const COLUMNS = [
{ id: "filename" as const, title: "Name", width: "65%" },
{ id: "status" as const, width: "24px" },
{ id: "size" as const, title: "Size", width: "15%" },
{ id: "dateUploaded" as const, title: "Date uploaded", width: "20%" }
];
type SortOptions = {
id: "filename" | "size" | "dateUploaded";
direction: "asc" | "desc";
};
type AttachmentsDialogProps = { onClose: Perform };
function AttachmentsDialog({ onClose }: AttachmentsDialogProps) {
const allAttachments = useStore((store) => store.attachments);
const [attachments, setAttachments] = useState<any[]>(allAttachments);
const [attachments, setAttachments] =
useState<VirtualizedGrouping<AttachmentType>>();
const [counts, setCounts] = useState<Record<Route, number>>({
all: 0,
documents: 0,
images: 0,
orphaned: 0,
uploads: 0,
videos: 0
});
const [selected, setSelected] = useState<string[]>([]);
const [sortBy, setSortBy] = useState({ id: "name", direction: "asc" });
const [sortBy, setSortBy] = useState<SortOptions>({
id: "filename",
direction: "asc"
});
const currentRoute = useRef<Route>("all");
const refresh = useStore((store) => store.refresh);
const download = useStore((store) => store.download);
useEffect(() => {
refresh();
}, [refresh]);
useEffect(() => {
setAttachments(filterAttachments(currentRoute.current, allAttachments));
}, [allAttachments]);
(async function () {
setAttachments(
await filterAttachments(currentRoute.current).sorted({
sortBy: sortBy.id,
sortDirection: sortBy.direction
})
);
})();
}, [sortBy, allAttachments]);
useEffect(() => {
setAttachments((a) => {
const attachments = a.slice();
if (sortBy.id === "name") {
attachments.sort(
sortBy.direction === "asc"
? (a, b) => a.metadata.filename.localeCompare(b.metadata.filename)
: (a, b) => b.metadata.filename.localeCompare(a.metadata.filename)
);
} else if (sortBy.id === "size") {
attachments.sort(
sortBy.direction === "asc"
? (a, b) => a.length - b.length
: (a, b) => b.length - a.length
);
} else if (sortBy.id === "dateUploaded") {
attachments.sort(
sortBy.direction === "asc"
? (a, b) => a.dateUploaded - b.dateUploaded
: (a, b) => b.dateUploaded - a.dateUploaded
);
}
return attachments;
});
}, [sortBy]);
(async function () {
setCounts(await getCounts());
})();
}, [allAttachments]);
const totalSize = useMemo(
() => getTotalSize(allAttachments),
[allAttachments]
);
const totalSize = 0;
// useMemo(
// () => getTotalSize(allAttachments),
// [allAttachments]
// );
return (
<Dialog
@@ -154,16 +162,22 @@ function AttachmentsDialog({ onClose }: AttachmentsDialogProps) {
>
<Sidebar
totalSize={totalSize}
onDownloadAll={() => download(allAttachments?.ungrouped || [])}
filter={(query) => {
setAttachments(
db.lookup?.attachments(db.attachments.all || [], query) || []
);
// setAttachments(
// db.lookup?.attachments(db.attachments.all || [], query) || []
// );
}}
counts={getCounts(allAttachments)}
onRouteChange={(route) => {
counts={counts}
onRouteChange={async (route) => {
currentRoute.current = route;
setSelected([]);
setAttachments(filterAttachments(route, allAttachments));
setAttachments(
await filterAttachments(currentRoute.current).sorted({
sortBy: sortBy.id,
sortDirection: sortBy.direction
})
);
}}
/>
<Flex
@@ -192,9 +206,10 @@ function AttachmentsDialog({ onClose }: AttachmentsDialogProps) {
title={tool.title}
onClick={() =>
tool.onClick({
selected: attachments.filter(
(a) => selected.indexOf(a.id) > -1
)
selected
// : attachments.filter(
// (a) => selected.indexOf(a.id) > -1
// )
})
}
disabled={!selected.length}
@@ -218,123 +233,113 @@ function AttachmentsDialog({ onClose }: AttachmentsDialogProps) {
</Text>
</Button> */}
</Flex>
<TableVirtuoso
components={{
Scroller: CustomScrollbarsVirtualList,
TableRow: (props) => {
const attachment = attachments[props["data-item-index"]];
return (
<Attachment
{...props}
key={attachment.id}
attachment={attachment}
isSelected={selected.indexOf(attachment.id) > -1}
onSelected={() => {
setSelected((s) => {
const copy = s.slice();
const index = copy.indexOf(attachment.id);
if (index > -1) copy.splice(index, 1);
else copy.push(attachment.id);
return copy;
});
}}
/>
);
}
}}
style={{ height: "100%" }}
data={attachments}
fixedItemHeight={30}
defaultItemHeight={30}
fixedHeaderContent={() => (
<Box
as="tr"
sx={{
height: 40,
th: { borderBottom: "1px solid var(--separator)" },
bg: "background"
}}
>
<Text
as="th"
variant="body"
{attachments && (
<TableVirtuoso
components={{
Scroller: CustomScrollbarsVirtualList,
TableRow
}}
style={{ height: "100%" }}
computeItemKey={(index) => attachments.getKey(index)}
data={attachments.ungrouped}
context={{
isSelected: (id: string) => selected.indexOf(id) > -1,
select: (id: string) => {
setSelected((s) => {
const copy = s.slice();
const index = copy.indexOf(id);
if (index > -1) copy.splice(index, 1);
else copy.push(id);
return copy;
});
},
attachments
}}
fixedItemHeight={30}
defaultItemHeight={30}
fixedHeaderContent={() => (
<Box
as="tr"
sx={{
width: 24,
textAlign: "left",
fontWeight: "normal",
mb: 2
height: 40,
th: { borderBottom: "1px solid var(--separator)" },
bg: "background"
}}
>
<Label>
<Checkbox
sx={{ width: 18, height: 18 }}
onChange={(e) => {
setSelected(
e.currentTarget.checked
? attachments.map((a) => a.id)
: []
);
}}
/>
</Label>
</Text>
{[
{ id: "name", title: "Name", width: "65%" },
{ id: "status", width: "24px" },
{ id: "size", title: "Size", width: "15%" },
{ id: "dateUploaded", title: "Date uploaded", width: "20%" }
].map((column) =>
!column.title ? (
<th key={column.id} />
) : (
<Box
as="th"
key={column.id}
sx={{
width: column.width,
cursor: "pointer",
px: 1,
mb: 2,
":hover": { bg: "hover" }
}}
onClick={() => {
setSortBy((sortBy) => ({
direction:
sortBy.id === column.id &&
sortBy.direction === "asc"
? "desc"
: "asc",
id: column.id
}));
}}
>
<Flex
<Text
as="th"
variant="body"
sx={{
width: 24,
textAlign: "left",
fontWeight: "normal",
mb: 2
}}
>
<Label>
<Checkbox
sx={{ width: 18, height: 18 }}
onChange={(e) => {
setSelected(
e.currentTarget.checked ? attachments.ungrouped : []
);
}}
/>
</Label>
</Text>
{COLUMNS.map((column) =>
!column.title ? (
<th key={column.id} />
) : (
<Box
as="th"
key={column.id}
sx={{
alignItems: "center",
justifyContent: "space-between"
width: column.width,
cursor: "pointer",
px: 1,
mb: 2,
":hover": { bg: "hover" }
}}
onClick={() => {
setSortBy((sortBy) => ({
direction:
sortBy.id === column.id &&
sortBy.direction === "asc"
? "desc"
: "asc",
id: column.id
}));
}}
>
<Text
variant="body"
sx={{ textAlign: "left", fontWeight: "normal" }}
<Flex
sx={{
alignItems: "center",
justifyContent: "space-between"
}}
>
{column.title}
</Text>
{sortBy.id === column.id ? (
sortBy.direction === "asc" ? (
<ChevronUp size={16} />
) : (
<ChevronDown size={16} />
)
) : null}
</Flex>
</Box>
)
)}
</Box>
)}
itemContent={() => <></>}
/>
<Text
variant="body"
sx={{ textAlign: "left", fontWeight: "normal" }}
>
{column.title}
</Text>
{sortBy.id === column.id ? (
sortBy.direction === "asc" ? (
<ChevronUp size={16} />
) : (
<ChevronDown size={16} />
)
) : null}
</Flex>
</Box>
)
)}
</Box>
)}
itemContent={() => <></>}
/>
)}
</Flex>
</Flex>
</Dialog>
@@ -379,6 +384,7 @@ const routes: { id: Route; icon: Icon; title: string }[] = [
];
type SidebarProps = {
onDownloadAll: () => void;
onRouteChange: (route: Route) => void;
filter: (query: string) => void;
counts: Record<Route, number>;
@@ -386,11 +392,10 @@ type SidebarProps = {
};
const Sidebar = memo(
function Sidebar(props: SidebarProps) {
const { onRouteChange, filter, counts, totalSize } = props;
const { onRouteChange, filter, counts, totalSize, onDownloadAll } = props;
const [route, setRoute] = useState("all");
const downloadStatus = useStore((store) => store.status);
const cancelDownload = useStore((store) => store.cancel);
const download = useStore((store) => store.download);
return (
<ScopedThemeProvider scope="navigationMenu" injectCssVars={false}>
@@ -446,7 +451,7 @@ const Sidebar = memo(
if (downloadStatus) {
await cancelDownload();
} else {
await download(db.attachments.all);
onDownloadAll();
}
}}
>
@@ -482,38 +487,57 @@ const Sidebar = memo(
prev.counts.orphaned === next.counts.orphaned
);
function getCounts(attachments: any[]): Record<Route, number> {
const counts: Record<Route, number> = {
all: 0,
documents: 0,
images: 0,
videos: 0,
uploads: 0,
orphaned: 0
};
for (const attachment of attachments) {
counts.all++;
if (isDocument(attachment.metadata.type)) counts.documents++;
else if (isImage(attachment.metadata.type)) counts.images++;
else if (isVideo(attachment.metadata.type)) counts.videos++;
if (!attachment.dateUploaded) counts.uploads++;
if (!attachment.noteIds.length) counts.orphaned++;
function TableRow(
props: ItemProps<string> & {
context?: {
isSelected: (id: string) => boolean;
select: (id: string) => void;
attachments: VirtualizedGrouping<AttachmentType>;
};
}
return counts;
) {
const { context, item, ...restProps } = props;
const result = usePromise(
() => context?.attachments.item(item),
[item, context]
);
if (result.status !== "fulfilled" || !result.value)
return <div {...restProps} />;
return (
<Attachment
{...restProps}
key={item}
attachment={result.value}
isSelected={context?.isSelected(item)}
onSelected={() => {
context?.select(item);
}}
/>
);
}
function filterAttachments(route: Route, attachments: any[]): any[] {
return route === "all"
? attachments
: route === "images"
? attachments.filter((a) => a.metadata.type.startsWith("image/"))
: route === "videos"
? attachments.filter((a) => a.metadata.type.startsWith("video/"))
: route === "documents"
? attachments.filter((a) => isDocument(a.metadata.type))
: route === "orphaned"
? attachments.filter((a) => !a.noteIds.length)
: attachments.filter((a) => !a.dateUploaded);
async function getCounts(): Promise<Record<Route, number>> {
return {
all: await db.attachments.all.count(),
documents: await db.attachments.documents.count(),
images: await db.attachments.images.count(),
videos: await db.attachments.videos.count(),
uploads: await db.attachments.pending.count(),
orphaned: await db.attachments.orphaned.count()
};
}
function filterAttachments(route: Route) {
return route === "all"
? db.attachments.all
: route === "images"
? db.attachments.images
: route === "videos"
? db.attachments.videos
: route === "documents"
? db.attachments.documents
: route === "orphaned"
? db.attachments.orphaned
: db.attachments.pending;
}

View File

@@ -26,44 +26,45 @@ import { showToast } from "../utils/toast";
import { AttachmentStream } from "../utils/streams/attachment-stream";
import { createZipStream } from "../utils/streams/zip-stream";
import { createWriteStream } from "../utils/stream-saver";
import { Attachment, VirtualizedGrouping } from "@notesnook/core";
let abortController = undefined;
/**
* @extends {BaseStore<AttachmentStore>}
*/
class AttachmentStore extends BaseStore {
attachments = [];
/**
* @type {{current: number, total: number}}
*/
status = undefined;
let abortController: AbortController | undefined = undefined;
class AttachmentStore extends BaseStore<AttachmentStore> {
attachments?: VirtualizedGrouping<Attachment>;
status?: { current: number; total: number };
processing: Record<
string,
{ failed?: string; working?: "delete" | "recheck" }
> = {};
refresh = () => {
this.set((state) => (state.attachments = db.attachments.all));
refresh = async () => {
this.set({
attachments: await db.attachments.all.sorted({
sortBy: "dateCreated",
sortDirection: "desc"
})
});
};
init = () => {
this.refresh();
};
download = async (attachments) => {
download = async (ids: string[]) => {
if (this.get().status)
throw new Error(
"Please wait for the previous download to finish or cancel it."
);
this.set(
(state) => (state.status = { current: 0, total: attachments.length })
);
this.set({ status: { current: 0, total: ids.length } });
abortController = new AbortController();
const attachmentStream = new AttachmentStream(
attachments,
ids,
(id) => this.attachments?.item(id),
abortController.signal,
(current) => {
this.set(
(state) => (state.status = { current, total: attachments.length })
);
this.set({ status: { current, total: ids.length } });
}
);
await attachmentStream
@@ -78,71 +79,74 @@ class AttachmentStore extends BaseStore {
cancel = async () => {
if (abortController) {
await abortController.abort();
abortController.abort();
abortController = undefined;
this.set((state) => (state.status = undefined));
}
};
recheck = async (hashes) => {
const attachments = this.get().attachments;
for (let hash of hashes) {
const index = attachments.findIndex((a) => a.metadata.hash === hash);
recheck = async (ids: string[]) => {
for (const id of ids) {
const attachment = await this.attachments?.item(id);
if (!attachment) continue;
try {
this._changeWorkingStatus(index, "recheck", undefined);
this._changeWorkingStatus(attachment.hash, "recheck");
const { failed, success } = await checkAttachment(hash);
this._changeWorkingStatus(index, false, success ? null : failed);
const { failed, success } = await checkAttachment(attachment.hash);
this._changeWorkingStatus(
attachment.hash,
undefined,
success ? undefined : failed
);
} catch (e) {
console.error(e);
this._changeWorkingStatus(index, false, false);
showToast("error", `Rechecking failed: ${e.message}`);
this._changeWorkingStatus(attachment.hash);
if (e instanceof Error)
showToast("error", `Rechecking failed: ${e.message}`);
}
}
};
rename = async (hash, newName) => {
rename = async (hash: string, newName: string) => {
await db.attachments.add({
hash,
filename: newName
});
this.get().refresh();
await this.get().refresh();
};
permanentDelete = async (hash) => {
const index = this.get().attachments.findIndex(
(a) => a.metadata.hash === hash
);
if (index <= -1) return;
const noteIds = this.get().attachments[index].noteIds.slice();
permanentDelete = async (attachment: Attachment) => {
try {
this._changeWorkingStatus(index, "delete", undefined);
if (await db.attachments.remove(hash, false)) {
this.get().refresh();
this._changeWorkingStatus(attachment.hash, "delete");
if (await db.attachments.remove(attachment.hash, false)) {
await this.get().refresh();
if (noteIds.includes(editorStore.get().session.id)) {
const sessionId = editorStore.get().session.id;
if (
sessionId &&
(await db.relations
.to({ id: attachment.id, type: "attachment" }, "note")
.has(sessionId))
) {
await editorStore.clearSession();
}
}
} catch (e) {
console.error(e);
this._changeWorkingStatus(index, false, false);
showToast("error", `Failed to delete: ${e.message}`);
this._changeWorkingStatus(attachment.hash);
if (e instanceof Error)
showToast("error", `Failed to delete: ${e.message}`);
throw e;
}
};
/**
*
* @param {*} index
* @param {"delete"|"recheck"} workType
* @param {*} failed
*/
_changeWorkingStatus = (index, workType, failed) => {
private _changeWorkingStatus = (
hash: string,
working?: "delete" | "recheck",
failed?: string
) => {
this.set((state) => {
state.attachments[index].failed = failed;
state.attachments[index].working = workType;
state.processing[hash] = { failed, working };
});
};
}

View File

@@ -21,12 +21,14 @@ import { db } from "../../common/db";
import { lazify } from "../lazify";
import { makeUniqueFilename } from "./utils";
import { ZipFile } from "./zip-stream";
import { Attachment } from "@notesnook/core";
export const METADATA_FILENAME = "metadata.json";
const GROUP_ID = "all-attachments";
export class AttachmentStream extends ReadableStream<ZipFile> {
constructor(
attachments: Array<any>,
ids: string[],
resolve: (id: string) => Promise<Attachment | undefined> | undefined,
signal?: AbortSignal,
onProgress?: (current: number) => void
) {
@@ -47,16 +49,12 @@ export class AttachmentStream extends ReadableStream<ZipFile> {
}
onProgress && onProgress(index);
const attachment = attachments[index++];
const attachment = await resolve(ids[index++]);
if (!attachment) return;
await db
.fs()
.downloadFile(
GROUP_ID,
attachment.metadata.hash,
attachment.chunkSize,
attachment.metadata
);
.downloadFile(GROUP_ID, attachment.hash, attachment.chunkSize);
const key = await db.attachments.decryptKey(attachment.key);
if (!key) return;
@@ -67,14 +65,14 @@ export class AttachmentStream extends ReadableStream<ZipFile> {
decryptFile(attachment.metadata.hash, {
key,
iv: attachment.iv,
name: attachment.metadata.filename,
type: attachment.metadata.type,
name: attachment.filename,
type: attachment.mimeType,
isUploaded: !!attachment.dateUploaded
})
);
if (file) {
const filePath: string = attachment.metadata.filename;
const filePath: string = attachment.filename;
controller.enqueue({
path: makeUniqueFilename(filePath, counters),
data: new Uint8Array(await file.arrayBuffer())
@@ -86,7 +84,7 @@ export class AttachmentStream extends ReadableStream<ZipFile> {
console.error(e);
controller.error(e);
} finally {
if (index === attachments.length) {
if (index === ids.length) {
controller.close();
}
}