Merge pull request #9364 from streetwriters/editor/audio-playback

editor: support audio attachment playback

packages/editor/src/extensions/attachment/attachment.ts

@@ -24,6 +24,7 @@ import { AttachmentComponent } from "./component.js";
 import { Attachment } from "./types.js";
 import { tiptapKeys } from "@notesnook/common";
 import { hasPermission } from "../../types.js";
+import { AudioNode } from "../audio/audio.js";
 
 export type AttachmentType = "image" | "file" | "camera";
 export interface AttachmentOptions {
@@ -121,13 +122,13 @@ export const AttachmentNode = Node.create<AttachmentOptions>({
             return commands.insertContentAt(
               $from.pos + maybeAttachmentNode.nodeSize,
               {
-                type: this.name,
+                type: mimeToExtension(attachment.mime),
                 attrs: attachment
               }
             );
           }
           return commands.insertContent({
-            type: this.name,
+            type: mimeToExtension(attachment.mime),
             attrs: attachment
           });
         },
@@ -187,6 +188,11 @@ export const AttachmentNode = Node.create<AttachmentOptions>({
   //   },
 });
 
+function mimeToExtension(mime: string): string {
+  if (mime.startsWith("audio/")) return AudioNode.name;
+  return AttachmentNode.name;
+}
+
 export function getDataAttribute(
   name: string,
   def?: unknown | null

packages/editor/src/extensions/attachment/component.tsx

@@ -25,6 +25,7 @@ import { Icons } from "../../toolbar/icons.js";
 import { ReactNodeViewProps } from "../react/index.js";
 import { ToolbarGroup } from "../../toolbar/components/toolbar-group.js";
 import { DesktopOnly } from "../../components/responsive/index.js";
+import { formatBytes } from "@notesnook/common";
 
 export function AttachmentComponent(props: ReactNodeViewProps<FileAttachment>) {
   const { editor, node, selected } = props;
@@ -82,7 +83,7 @@ export function AttachmentComponent(props: ReactNodeViewProps<FileAttachment>) {
           flexShrink: 0
         }}
       >
-        {progress ? `${progress}%` : formatBytes(size)}
+        {progress ? `${progress}%` : formatBytes(size, 1)}
       </Text>
       <DesktopOnly>
         {selected && !isDragging && (
@@ -111,15 +112,3 @@ export function AttachmentComponent(props: ReactNodeViewProps<FileAttachment>) {
     </Box>
   );
 }
-
-function formatBytes(bytes: number, decimals = 1) {
-  if (bytes === 0) return "0B";
-
-  const k = 1024;
-  const dm = decimals < 0 ? 0 : decimals;
-  const sizes = ["B", "K", "M", "G", "T", "P", "E", "Z", "Y"];
-
-  const i = Math.floor(Math.log(bytes) / Math.log(k));
-
-  return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + sizes[i];
-}

packages/editor/src/extensions/attachment/types.ts

@@ -29,6 +29,10 @@ export type FileAttachment = BaseAttachment & {
   type: "file";
 };
 
+export type AudioAttachment = BaseAttachment & {
+  type: "audio";
+};
+
 export type WebClipAttachment = BaseAttachment & {
   type: "web-clip";
   src: string;
@@ -50,4 +54,8 @@ export type ImageAlignmentOptions = {
   align?: "center" | "left" | "right";
 };
 
-export type Attachment = FileAttachment | WebClipAttachment | ImageAttachment;
+export type Attachment =
+  | FileAttachment
+  | AudioAttachment
+  | WebClipAttachment
+  | ImageAttachment;

packages/editor/src/extensions/audio/audio.ts

@@ -0,0 +1,103 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Node, mergeAttributes } from "@tiptap/core";
+import { hasSameAttributes } from "../../utils/prosemirror.js";
+import { AudioAttachment, getDataAttribute } from "../attachment/index.js";
+import { createNodeView } from "../react/index.js";
+import { AudioComponent } from "./component.js";
+
+export interface AudioOptions {
+  HTMLAttributes: Record<string, unknown>;
+}
+
+export type AudioAttributes = AudioAttachment;
+
+declare module "@tiptap/core" {
+  interface Commands<ReturnType> {
+    audio: {
+      insertAudio: (audio: AudioAttachment) => ReturnType;
+    };
+  }
+}
+
+export const AudioNode = Node.create<AudioOptions>({
+  name: "audio",
+  draggable: true,
+  priority: 51,
+
+  addOptions() {
+    return {
+      HTMLAttributes: {}
+    };
+  },
+
+  group() {
+    return "block";
+  },
+
+  addAttributes() {
+    return {
+      type: { default: "audio", rendered: false },
+      progress: {
+        default: 0,
+        rendered: false
+      },
+      filename: getDataAttribute("filename"),
+      size: getDataAttribute("size"),
+      hash: getDataAttribute("hash"),
+      mime: getDataAttribute("mime")
+    };
+  },
+
+  parseHTML() {
+    return [
+      {
+        tag: "audio"
+      }
+    ];
+  },
+
+  renderHTML({ HTMLAttributes }) {
+    return [
+      "audio",
+      mergeAttributes(this.options.HTMLAttributes, HTMLAttributes)
+    ];
+  },
+
+  addCommands() {
+    return {
+      insertAudio:
+        (audio) =>
+        ({ commands }) => {
+          return commands.insertContent({
+            type: this.name,
+            attrs: audio
+          });
+        }
+    };
+  },
+
+  addNodeView() {
+    return createNodeView(AudioComponent, {
+      shouldUpdate: (prev, next) => !hasSameAttributes(prev.attrs, next.attrs),
+      forceEnableSelection: true
+    });
+  }
+});

packages/editor/src/extensions/audio/component.tsx

@@ -0,0 +1,182 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Box, Text } from "@theme-ui/components";
+import { AudioAttachment } from "../attachment/types.js";
+import { useRef, useState, useEffect } from "react";
+import { Icon } from "@notesnook/ui";
+import { Icons } from "../../toolbar/icons.js";
+import { ReactNodeViewProps } from "../react/index.js";
+import { ToolbarGroup } from "../../toolbar/components/toolbar-group.js";
+import { DesktopOnly } from "../../components/responsive/index.js";
+import { toBlobURL, revokeBloburl } from "../../utils/downloader.js";
+import { formatBytes } from "@notesnook/common";
+
+const SAMPLE_AUDIO = toBlobURL(
+  "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YQAAAAA=",
+  "other",
+  "audio/wav",
+  "sample-audio"
+);
+
+export function AudioComponent(props: ReactNodeViewProps<AudioAttachment>) {
+  const { editor, node, selected } = props;
+  const { filename, size, progress, mime, hash } = node.attrs;
+  const elementRef = useRef<HTMLDivElement>();
+  const [isDragging, setIsDragging] = useState(false);
+  const [error, setError] = useState<string>();
+
+  useEffect(() => {
+    return () => {
+      if (hash) {
+        revokeBloburl(hash);
+      }
+    };
+  }, [hash]);
+
+  return (
+    <Box
+      ref={elementRef}
+      contentEditable={false}
+      draggable="false"
+      sx={{
+        display: "flex",
+        flexDirection: "column",
+        position: "relative",
+        userSelect: "none",
+        backgroundColor: "var(--background-secondary)",
+        p: 1,
+        borderRadius: "default",
+        border: "1px solid var(--border)",
+        borderColor: selected ? "accent" : "border",
+        width: "100%",
+        maxWidth: 500,
+        ":hover": {
+          backgroundColor: "var(--hover)"
+        }
+      }}
+      onDragStart={() => setIsDragging(true)}
+      onDragEnd={() => setIsDragging(false)}
+      data-drag-handle
+      data-drag-image
+    >
+      <Box
+        sx={{
+          display: "flex",
+          alignItems: "center",
+          mb: 2
+        }}
+      >
+        <Icon path={Icons.attachment} size={16} />
+        <Text
+          as="span"
+          sx={{
+            ml: "small",
+            fontSize: "body",
+            whiteSpace: "nowrap",
+            textOverflow: "ellipsis",
+            overflow: "hidden",
+            flex: 1
+          }}
+        >
+          {filename}
+        </Text>
+        <Text
+          as="span"
+          sx={{
+            ml: 1,
+            fontSize: "0.65rem",
+            color: "var(--paragraph-secondary)",
+            flexShrink: 0
+          }}
+        >
+          {progress ? `${progress}%` : formatBytes(size, 1)}
+        </Text>
+      </Box>
+      {error ? (
+        <Text variant="error">{error}</Text>
+      ) : (
+        <Box
+          sx={{
+            width: "100%",
+            "& audio": {
+              width: "100%",
+              height: 35
+            }
+          }}
+        >
+          <audio
+            onMouseDown={(e) => e.stopPropagation()}
+            onDragStart={(e) => e.stopPropagation()}
+            onClick={(e) => e.stopPropagation()}
+            controls
+            controlsList="nodownload nofullscreen"
+            src={SAMPLE_AUDIO}
+            onPlay={(e) => {
+              const target = e.currentTarget;
+              if (
+                editor.storage?.getAttachmentData &&
+                hash &&
+                target.src === SAMPLE_AUDIO
+              ) {
+                e.preventDefault();
+                editor.storage
+                  .getAttachmentData({
+                    type: "file",
+                    hash
+                  })
+                  .then((data: string | undefined) => {
+                    if (!data) return;
+                    const url = toBlobURL(data, "other", mime, hash);
+                    if (!url) return;
+                    target.src = url;
+                    return target.play();
+                  })
+                  .catch((e) => setError((e as Error).message));
+              }
+            }}
+          >
+            <Text as="p">Your browser does not support the audio element.</Text>
+          </audio>
+        </Box>
+      )}
+      <DesktopOnly>
+        {selected && !isDragging && (
+          <ToolbarGroup
+            editor={editor}
+            groupId="audioTools"
+            tools={
+              editor.isEditable
+                ? ["removeAttachment", "downloadAttachment"]
+                : ["downloadAttachment"]
+            }
+            sx={{
+              boxShadow: "menu",
+              borderRadius: "default",
+              backgroundColor: "var(--background)",
+              position: "absolute",
+              top: -35,
+              right: 0
+            }}
+          />
+        )}
+      </DesktopOnly>
+    </Box>
+  );
+}

packages/editor/src/extensions/audio/index.ts

@@ -0,0 +1,20 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+export * from "./audio.js";

packages/editor/src/index.ts

@@ -66,6 +66,7 @@ import { TaskItemNode } from "./extensions/task-item/index.js";
 import { TaskListNode } from "./extensions/task-list/index.js";
 import TextDirection from "./extensions/text-direction/index.js";
 import { WebClipNode, WebClipOptions } from "./extensions/web-clip/index.js";
+import { AudioNode, AudioOptions } from "./extensions/audio/index.js";
 import { useEditor } from "./hooks/use-editor.js";
 import { usePermissionHandler } from "./hooks/use-permission-handler.js";
 import Toolbar from "./toolbar/index.js";
@@ -125,6 +126,7 @@ const CoreExtensions = Object.entries(TiptapCoreExtensions)
 
 export type TiptapOptions = EditorOptions &
   Omit<WebClipOptions, "HTMLAttributes"> &
+  Omit<AudioOptions, "HTMLAttributes"> &
   Omit<ImageOptions, "HTMLAttributes"> &
   DateTimeOptions &
   TiptapStorage & {
@@ -312,7 +314,12 @@ const useTiptap = (
         ImageNode.configure({ allowBase64: true }),
         EmbedNode,
         AttachmentNode.configure({
-          types: [AttachmentNode.name, ImageNode.name, WebClipNode.name]
+          types: [
+            AttachmentNode.name,
+            ImageNode.name,
+            WebClipNode.name,
+            AudioNode.name
+          ]
         }),
         OutlineListItem,
         OutlineList.configure({ keepAttributes: true, keepMarks: true }),
@@ -327,6 +334,7 @@ const useTiptap = (
         DateTime.configure({ dateFormat, timeFormat, dayFormat }),
         KeyMap,
         WebClipNode,
+        AudioNode,
         CheckList,
         CheckListItem.configure({
           nested: true

packages/editor/src/toolbar/tools/attachment.tsx

@@ -23,6 +23,7 @@ import { MoreTools } from "../components/more-tools.js";
 import { useToolbarLocation } from "../stores/toolbar-store.js";
 import { findSelectedNode } from "../../utils/prosemirror.js";
 import { Attachment } from "../../extensions/attachment/index.js";
+import { Editor } from "../../types.js";
 
 export function AttachmentSettings(props: ToolProps) {
   const { editor } = props;
@@ -52,9 +53,7 @@ export function DownloadAttachment(props: ToolProps) {
       title={props.title}
       toggled={false}
       onClick={() => {
-        const attachmentNode =
-          findSelectedNode(editor, "attachment") ||
-          findSelectedNode(editor, "image");
+        const attachmentNode = findAttachmentNode(editor);
 
         const attachment = (attachmentNode?.attrs || {}) as Attachment;
         editor.storage.downloadAttachment?.(attachment);
@@ -65,8 +64,7 @@ export function DownloadAttachment(props: ToolProps) {
 
 export function PreviewAttachment(props: ToolProps) {
   const { editor } = props;
-  const attachmentNode =
-    findSelectedNode(editor, "attachment") || findSelectedNode(editor, "image");
+  const attachmentNode = findAttachmentNode(editor);
   const attachment = (attachmentNode?.attrs || {}) as Attachment;
 
   if (!editor.isActive("image") && !canPreviewAttachment(attachment))
@@ -117,3 +115,11 @@ function canPreviewAttachment(attachment: Attachment) {
 
   return previewableFileExtensions.indexOf(extension) > -1;
 }
+
+function findAttachmentNode(editor: Editor) {
+  return (
+    findSelectedNode(editor, "attachment") ||
+    findSelectedNode(editor, "image") ||
+    findSelectedNode(editor, "audio")
+  );
+}

packages/editor/styles/styles.css

@@ -131,8 +131,9 @@
 }
 
 .ProseMirror > div.codeblock-view-content-wrap,
-.ProseMirror > div.taskList-view-content-wrap,
-.ProseMirror > div.math-block.math-node {
+.ProseMirror > div.math-block.math-node,
+.ProseMirror > div.audio-view-content-wrap, 
+.ProseMirror > div.taskList-view-content-wrap {
   margin-top: 1em;
   margin-bottom: 1em;
 }