From add1b8a68aa52f124f019d6218a098f2efceb9ff Mon Sep 17 00:00:00 2001 From: 01zulfi <85733202+01zulfi@users.noreply.github.com> Date: Wed, 18 Feb 2026 10:40:54 +0500 Subject: [PATCH] editor: support audio attachment playback Co-authored-by: Hamish <133548095+Hamster45105@users.noreply.github.com> Signed-off-by: 01zulfi <85733202+01zulfi@users.noreply.github.com> --- apps/web/src/components/editor/picker.ts | 16 +- apps/web/src/components/editor/tiptap.tsx | 2 + .../src/extensions/attachment/attachment.ts | 9 +- .../src/extensions/attachment/component.tsx | 15 +- .../editor/src/extensions/attachment/types.ts | 10 +- packages/editor/src/extensions/audio/audio.ts | 105 ++++++++++++ .../editor/src/extensions/audio/component.tsx | 159 ++++++++++++++++++ packages/editor/src/extensions/audio/index.ts | 20 +++ packages/editor/src/index.ts | 10 +- 9 files changed, 324 insertions(+), 22 deletions(-) create mode 100644 packages/editor/src/extensions/audio/audio.ts create mode 100644 packages/editor/src/extensions/audio/component.tsx create mode 100644 packages/editor/src/extensions/audio/index.ts diff --git a/apps/web/src/components/editor/picker.ts b/apps/web/src/components/editor/picker.ts index bd91d2a37..27b480629 100644 --- a/apps/web/src/components/editor/picker.ts +++ b/apps/web/src/components/editor/picker.ts @@ -87,12 +87,16 @@ export async function attachFiles( : []; } - const documents = files.filter((f) => !f.type.startsWith("image/")); + const documents = files.filter( + (f) => !f.type.startsWith("image/") && !f.type.startsWith("audio/") + ); + const audios = files.filter((f) => f.type.startsWith("audio/")); const attachments: Attachment[] = []; - for (const file of [...images, ...documents]) { - const attachment = !skipSpecialImageHandling && file.type.startsWith("image/") - ? await pickImage(file) - : await pickFile(file); + for (const file of [...images, ...documents, ...audios]) { + const attachment = + !skipSpecialImageHandling && file.type.startsWith("image/") + ? await pickImage(file) + : await pickFile(file); if (!attachment) continue; attachments.push(attachment); } @@ -136,7 +140,7 @@ async function pickFile( const hash = await addAttachment(file, options); return { - type: "file", + type: file.type.startsWith("audio") ? "audio" : "file", filename: file.name, hash, mime: file.type, diff --git a/apps/web/src/components/editor/tiptap.tsx b/apps/web/src/components/editor/tiptap.tsx index d32117318..6f1457cf1 100644 --- a/apps/web/src/components/editor/tiptap.tsx +++ b/apps/web/src/components/editor/tiptap.tsx @@ -728,6 +728,8 @@ function toIEditor(editor: Editor): IEditor { attachFile: (file: Attachment) => file.type === "image" ? editor.commands.insertImage(file) + : file.type === "audio" + ? editor.commands.insertAudio(file) : editor.commands.insertAttachment(file), sendAttachmentProgress: (hash, progress) => editor.commands.updateAttachment( diff --git a/packages/editor/src/extensions/attachment/attachment.ts b/packages/editor/src/extensions/attachment/attachment.ts index b17bd72de..fd14eb88e 100644 --- a/packages/editor/src/extensions/attachment/attachment.ts +++ b/packages/editor/src/extensions/attachment/attachment.ts @@ -85,7 +85,14 @@ export const AttachmentNode = Node.create({ parseHTML() { return [ { - tag: "span[data-hash]" + tag: "span[data-hash]", + getAttrs: (dom) => { + const element = dom as HTMLElement; + if (element.dataset.mime?.startsWith("audio/")) { + return false; + } + return {}; + } } ]; }, diff --git a/packages/editor/src/extensions/attachment/component.tsx b/packages/editor/src/extensions/attachment/component.tsx index 2d0431bce..b542ef920 100644 --- a/packages/editor/src/extensions/attachment/component.tsx +++ b/packages/editor/src/extensions/attachment/component.tsx @@ -25,6 +25,7 @@ import { Icons } from "../../toolbar/icons.js"; import { ReactNodeViewProps } from "../react/index.js"; import { ToolbarGroup } from "../../toolbar/components/toolbar-group.js"; import { DesktopOnly } from "../../components/responsive/index.js"; +import { formatBytes } from "@notesnook/common"; export function AttachmentComponent(props: ReactNodeViewProps) { const { editor, node, selected } = props; @@ -82,7 +83,7 @@ export function AttachmentComponent(props: ReactNodeViewProps) { flexShrink: 0 }} > - {progress ? `${progress}%` : formatBytes(size)} + {progress ? `${progress}%` : formatBytes(size, 1)} {selected && !isDragging && ( @@ -111,15 +112,3 @@ export function AttachmentComponent(props: ReactNodeViewProps) { ); } - -function formatBytes(bytes: number, decimals = 1) { - if (bytes === 0) return "0B"; - - const k = 1024; - const dm = decimals < 0 ? 0 : decimals; - const sizes = ["B", "K", "M", "G", "T", "P", "E", "Z", "Y"]; - - const i = Math.floor(Math.log(bytes) / Math.log(k)); - - return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + sizes[i]; -} diff --git a/packages/editor/src/extensions/attachment/types.ts b/packages/editor/src/extensions/attachment/types.ts index 8d8429b52..e941fb9e0 100644 --- a/packages/editor/src/extensions/attachment/types.ts +++ b/packages/editor/src/extensions/attachment/types.ts @@ -29,6 +29,10 @@ export type FileAttachment = BaseAttachment & { type: "file"; }; +export type AudioAttachment = BaseAttachment & { + type: "audio"; +}; + export type WebClipAttachment = BaseAttachment & { type: "web-clip"; src: string; @@ -50,4 +54,8 @@ export type ImageAlignmentOptions = { align?: "center" | "left" | "right"; }; -export type Attachment = FileAttachment | WebClipAttachment | ImageAttachment; +export type Attachment = + | FileAttachment + | AudioAttachment + | WebClipAttachment + | ImageAttachment; diff --git a/packages/editor/src/extensions/audio/audio.ts b/packages/editor/src/extensions/audio/audio.ts new file mode 100644 index 000000000..3dd952bf0 --- /dev/null +++ b/packages/editor/src/extensions/audio/audio.ts @@ -0,0 +1,105 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import { Node, mergeAttributes } from "@tiptap/core"; +import { hasSameAttributes } from "../../utils/prosemirror.js"; +import { AudioAttachment, getDataAttribute } from "../attachment/index.js"; +import { createNodeView } from "../react/index.js"; +import { AudioComponent } from "./component.js"; + +export interface AudioOptions { + HTMLAttributes: Record; +} + +export type AudioAttributes = AudioAttachment; + +declare module "@tiptap/core" { + interface Commands { + audio: { + insertAudio: (audio: AudioAttachment) => ReturnType; + }; + } +} + +export const AudioNode = Node.create({ + name: "audio", + content: "", + marks: "", + draggable: true, + priority: 51, + + addOptions() { + return { + HTMLAttributes: {} + }; + }, + + group() { + return "block"; + }, + + addAttributes() { + return { + type: { default: "audio", rendered: false }, + progress: { + default: 0, + rendered: false + }, + filename: getDataAttribute("filename"), + size: getDataAttribute("size"), + hash: getDataAttribute("hash"), + mime: getDataAttribute("mime") + }; + }, + + parseHTML() { + return [ + { + tag: "audio[data-hash][data-mime^='audio/']" + } + ]; + }, + + renderHTML({ HTMLAttributes }) { + return [ + "audio", + mergeAttributes(this.options.HTMLAttributes, HTMLAttributes) + ]; + }, + + addCommands() { + return { + insertAudio: + (audio) => + ({ commands }) => { + return commands.insertContent({ + type: this.name, + attrs: audio + }); + } + }; + }, + + addNodeView() { + return createNodeView(AudioComponent, { + shouldUpdate: (prev, next) => !hasSameAttributes(prev.attrs, next.attrs), + forceEnableSelection: true + }); + } +}); diff --git a/packages/editor/src/extensions/audio/component.tsx b/packages/editor/src/extensions/audio/component.tsx new file mode 100644 index 000000000..f0f90765e --- /dev/null +++ b/packages/editor/src/extensions/audio/component.tsx @@ -0,0 +1,159 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import { Box, Text } from "@theme-ui/components"; +import { AudioAttachment } from "../attachment/types.js"; +import { useRef, useState, useEffect } from "react"; +import { Icon } from "@notesnook/ui"; +import { Icons } from "../../toolbar/icons.js"; +import { ReactNodeViewProps } from "../react/index.js"; +import { ToolbarGroup } from "../../toolbar/components/toolbar-group.js"; +import { DesktopOnly } from "../../components/responsive/index.js"; +import { toBlobURL, revokeBloburl } from "../../utils/downloader.js"; +import { formatBytes } from "@notesnook/common"; + +export function AudioComponent(props: ReactNodeViewProps) { + const { editor, node, selected } = props; + const { filename, size, progress, mime, hash } = node.attrs; + const elementRef = useRef(); + const [isDragging, setIsDragging] = useState(false); + const [audioSrc, setAudioSrc] = useState(); + + useEffect(() => { + if (editor.storage?.getAttachmentData && hash) { + editor.storage + .getAttachmentData({ + type: "file", + hash + }) + .then((data: string | undefined) => { + if (data) { + const url = toBlobURL(data, "other", mime, hash); + if (url) { + setAudioSrc(url); + } + } + }) + .catch(console.error); + } + }, [editor.storage, hash, mime]); + + useEffect(() => { + return () => { + if (audioSrc && hash) { + revokeBloburl(hash); + } + }; + }, [audioSrc, hash]); + + return ( + setIsDragging(true)} + onDragEnd={() => setIsDragging(false)} + data-drag-handle + > + + + + {filename} + + + {progress ? `${progress}%` : formatBytes(size, 1)} + + + {audioSrc && ( + + + )} + + {selected && !isDragging && ( + + )} + + + ); +} diff --git a/packages/editor/src/extensions/audio/index.ts b/packages/editor/src/extensions/audio/index.ts new file mode 100644 index 000000000..7079357e4 --- /dev/null +++ b/packages/editor/src/extensions/audio/index.ts @@ -0,0 +1,20 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +export * from "./audio.js"; diff --git a/packages/editor/src/index.ts b/packages/editor/src/index.ts index b8dd1fb2c..122c4bc77 100644 --- a/packages/editor/src/index.ts +++ b/packages/editor/src/index.ts @@ -66,6 +66,7 @@ import { TaskItemNode } from "./extensions/task-item/index.js"; import { TaskListNode } from "./extensions/task-list/index.js"; import TextDirection from "./extensions/text-direction/index.js"; import { WebClipNode, WebClipOptions } from "./extensions/web-clip/index.js"; +import { AudioNode, AudioOptions } from "./extensions/audio/index.js"; import { useEditor } from "./hooks/use-editor.js"; import { usePermissionHandler } from "./hooks/use-permission-handler.js"; import Toolbar from "./toolbar/index.js"; @@ -125,6 +126,7 @@ const CoreExtensions = Object.entries(TiptapCoreExtensions) export type TiptapOptions = EditorOptions & Omit & + Omit & Omit & DateTimeOptions & TiptapStorage & { @@ -312,7 +314,12 @@ const useTiptap = ( ImageNode.configure({ allowBase64: true }), EmbedNode, AttachmentNode.configure({ - types: [AttachmentNode.name, ImageNode.name, WebClipNode.name] + types: [ + AttachmentNode.name, + ImageNode.name, + WebClipNode.name, + AudioNode.name + ] }), OutlineListItem, OutlineList.configure({ keepAttributes: true, keepMarks: true }), @@ -327,6 +334,7 @@ const useTiptap = ( DateTime.configure({ dateFormat, timeFormat, dayFormat }), KeyMap, WebClipNode, + AudioNode, CheckList, CheckListItem.configure({ nested: true