editor: support audio attachment playback

Co-authored-by: Hamish <133548095+Hamster45105@users.noreply.github.com>
Signed-off-by: 01zulfi <85733202+01zulfi@users.noreply.github.com>

apps/web/src/components/editor/picker.ts

@@ -87,12 +87,16 @@ export async function attachFiles(
           : [];
   }
 
-  const documents = files.filter((f) => !f.type.startsWith("image/"));
+  const documents = files.filter(
+    (f) => !f.type.startsWith("image/") && !f.type.startsWith("audio/")
+  );
+  const audios = files.filter((f) => f.type.startsWith("audio/"));
   const attachments: Attachment[] = [];
-  for (const file of [...images, ...documents]) {
-    const attachment = !skipSpecialImageHandling && file.type.startsWith("image/")
-      ? await pickImage(file)
-      : await pickFile(file);
+  for (const file of [...images, ...documents, ...audios]) {
+    const attachment =
+      !skipSpecialImageHandling && file.type.startsWith("image/")
+        ? await pickImage(file)
+        : await pickFile(file);
     if (!attachment) continue;
     attachments.push(attachment);
   }
@@ -136,7 +140,7 @@ async function pickFile(
 
     const hash = await addAttachment(file, options);
     return {
-      type: "file",
+      type: file.type.startsWith("audio") ? "audio" : "file",
       filename: file.name,
       hash,
       mime: file.type,

apps/web/src/components/editor/tiptap.tsx

@@ -728,6 +728,8 @@ function toIEditor(editor: Editor): IEditor {
     attachFile: (file: Attachment) =>
       file.type === "image"
         ? editor.commands.insertImage(file)
+        : file.type === "audio"
+        ? editor.commands.insertAudio(file)
         : editor.commands.insertAttachment(file),
     sendAttachmentProgress: (hash, progress) =>
       editor.commands.updateAttachment(

packages/editor/src/extensions/attachment/attachment.ts

@@ -85,7 +85,14 @@ export const AttachmentNode = Node.create<AttachmentOptions>({
   parseHTML() {
     return [
       {
-        tag: "span[data-hash]"
+        tag: "span[data-hash]",
+        getAttrs: (dom) => {
+          const element = dom as HTMLElement;
+          if (element.dataset.mime?.startsWith("audio/")) {
+            return false;
+          }
+          return {};
+        }
       }
     ];
   },

packages/editor/src/extensions/attachment/component.tsx

@@ -25,6 +25,7 @@ import { Icons } from "../../toolbar/icons.js";
 import { ReactNodeViewProps } from "../react/index.js";
 import { ToolbarGroup } from "../../toolbar/components/toolbar-group.js";
 import { DesktopOnly } from "../../components/responsive/index.js";
+import { formatBytes } from "@notesnook/common";
 
 export function AttachmentComponent(props: ReactNodeViewProps<FileAttachment>) {
   const { editor, node, selected } = props;
@@ -82,7 +83,7 @@ export function AttachmentComponent(props: ReactNodeViewProps<FileAttachment>) {
           flexShrink: 0
         }}
       >
-        {progress ? `${progress}%` : formatBytes(size)}
+        {progress ? `${progress}%` : formatBytes(size, 1)}
       </Text>
       <DesktopOnly>
         {selected && !isDragging && (
@@ -111,15 +112,3 @@ export function AttachmentComponent(props: ReactNodeViewProps<FileAttachment>) {
     </Box>
   );
 }
-
-function formatBytes(bytes: number, decimals = 1) {
-  if (bytes === 0) return "0B";
-
-  const k = 1024;
-  const dm = decimals < 0 ? 0 : decimals;
-  const sizes = ["B", "K", "M", "G", "T", "P", "E", "Z", "Y"];
-
-  const i = Math.floor(Math.log(bytes) / Math.log(k));
-
-  return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + sizes[i];
-}

packages/editor/src/extensions/attachment/types.ts

@@ -29,6 +29,10 @@ export type FileAttachment = BaseAttachment & {
   type: "file";
 };
 
+export type AudioAttachment = BaseAttachment & {
+  type: "audio";
+};
+
 export type WebClipAttachment = BaseAttachment & {
   type: "web-clip";
   src: string;
@@ -50,4 +54,8 @@ export type ImageAlignmentOptions = {
   align?: "center" | "left" | "right";
 };
 
-export type Attachment = FileAttachment | WebClipAttachment | ImageAttachment;
+export type Attachment =
+  | FileAttachment
+  | AudioAttachment
+  | WebClipAttachment
+  | ImageAttachment;

packages/editor/src/extensions/audio/audio.ts

@@ -0,0 +1,105 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Node, mergeAttributes } from "@tiptap/core";
+import { hasSameAttributes } from "../../utils/prosemirror.js";
+import { AudioAttachment, getDataAttribute } from "../attachment/index.js";
+import { createNodeView } from "../react/index.js";
+import { AudioComponent } from "./component.js";
+
+export interface AudioOptions {
+  HTMLAttributes: Record<string, unknown>;
+}
+
+export type AudioAttributes = AudioAttachment;
+
+declare module "@tiptap/core" {
+  interface Commands<ReturnType> {
+    audio: {
+      insertAudio: (audio: AudioAttachment) => ReturnType;
+    };
+  }
+}
+
+export const AudioNode = Node.create<AudioOptions>({
+  name: "audio",
+  content: "",
+  marks: "",
+  draggable: true,
+  priority: 51,
+
+  addOptions() {
+    return {
+      HTMLAttributes: {}
+    };
+  },
+
+  group() {
+    return "block";
+  },
+
+  addAttributes() {
+    return {
+      type: { default: "audio", rendered: false },
+      progress: {
+        default: 0,
+        rendered: false
+      },
+      filename: getDataAttribute("filename"),
+      size: getDataAttribute("size"),
+      hash: getDataAttribute("hash"),
+      mime: getDataAttribute("mime")
+    };
+  },
+
+  parseHTML() {
+    return [
+      {
+        tag: "audio[data-hash][data-mime^='audio/']"
+      }
+    ];
+  },
+
+  renderHTML({ HTMLAttributes }) {
+    return [
+      "audio",
+      mergeAttributes(this.options.HTMLAttributes, HTMLAttributes)
+    ];
+  },
+
+  addCommands() {
+    return {
+      insertAudio:
+        (audio) =>
+        ({ commands }) => {
+          return commands.insertContent({
+            type: this.name,
+            attrs: audio
+          });
+        }
+    };
+  },
+
+  addNodeView() {
+    return createNodeView(AudioComponent, {
+      shouldUpdate: (prev, next) => !hasSameAttributes(prev.attrs, next.attrs),
+      forceEnableSelection: true
+    });
+  }
+});

packages/editor/src/extensions/audio/component.tsx

@@ -0,0 +1,159 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Box, Text } from "@theme-ui/components";
+import { AudioAttachment } from "../attachment/types.js";
+import { useRef, useState, useEffect } from "react";
+import { Icon } from "@notesnook/ui";
+import { Icons } from "../../toolbar/icons.js";
+import { ReactNodeViewProps } from "../react/index.js";
+import { ToolbarGroup } from "../../toolbar/components/toolbar-group.js";
+import { DesktopOnly } from "../../components/responsive/index.js";
+import { toBlobURL, revokeBloburl } from "../../utils/downloader.js";
+import { formatBytes } from "@notesnook/common";
+
+export function AudioComponent(props: ReactNodeViewProps<AudioAttachment>) {
+  const { editor, node, selected } = props;
+  const { filename, size, progress, mime, hash } = node.attrs;
+  const elementRef = useRef<HTMLDivElement>();
+  const [isDragging, setIsDragging] = useState(false);
+  const [audioSrc, setAudioSrc] = useState<string>();
+
+  useEffect(() => {
+    if (editor.storage?.getAttachmentData && hash) {
+      editor.storage
+        .getAttachmentData({
+          type: "file",
+          hash
+        })
+        .then((data: string | undefined) => {
+          if (data) {
+            const url = toBlobURL(data, "other", mime, hash);
+            if (url) {
+              setAudioSrc(url);
+            }
+          }
+        })
+        .catch(console.error);
+    }
+  }, [editor.storage, hash, mime]);
+
+  useEffect(() => {
+    return () => {
+      if (audioSrc && hash) {
+        revokeBloburl(hash);
+      }
+    };
+  }, [audioSrc, hash]);
+
+  return (
+    <Box
+      ref={elementRef}
+      contentEditable={false}
+      draggable="false"
+      sx={{
+        display: "flex",
+        flexDirection: "column",
+        position: "relative",
+        userSelect: "none",
+        backgroundColor: "var(--background-secondary)",
+        p: 1,
+        borderRadius: "default",
+        border: "1px solid var(--border)",
+        borderColor: selected ? "accent" : "border",
+        width: "100%",
+        maxWidth: 500,
+        ":hover": {
+          backgroundColor: "var(--hover)"
+        }
+      }}
+      onDragStart={() => setIsDragging(true)}
+      onDragEnd={() => setIsDragging(false)}
+      data-drag-handle
+    >
+      <Box
+        sx={{
+          display: "flex",
+          alignItems: "center",
+          mb: audioSrc ? 2 : 0
+        }}
+      >
+        <Icon path={Icons.attachment} size={16} />
+        <Text
+          as="span"
+          sx={{
+            ml: "small",
+            fontSize: "body",
+            whiteSpace: "nowrap",
+            textOverflow: "ellipsis",
+            overflow: "hidden",
+            flex: 1
+          }}
+        >
+          {filename}
+        </Text>
+        <Text
+          as="span"
+          sx={{
+            ml: 1,
+            fontSize: "0.65rem",
+            color: "var(--paragraph-secondary)",
+            flexShrink: 0
+          }}
+        >
+          {progress ? `${progress}%` : formatBytes(size, 1)}
+        </Text>
+      </Box>
+      {audioSrc && (
+        <Box
+          sx={{
+            width: "100%",
+            "& audio": {
+              width: "100%",
+              height: 35
+            }
+          }}
+        >
+          <audio controls preload="metadata" src={audioSrc} />
+        </Box>
+      )}
+      <DesktopOnly>
+        {selected && !isDragging && (
+          <ToolbarGroup
+            editor={editor}
+            groupId="audioTools"
+            tools={
+              editor.isEditable
+                ? ["removeAttachment", "downloadAttachment"]
+                : ["downloadAttachment"]
+            }
+            sx={{
+              boxShadow: "menu",
+              borderRadius: "default",
+              backgroundColor: "var(--background)",
+              position: "absolute",
+              top: -35,
+              right: 0
+            }}
+          />
+        )}
+      </DesktopOnly>
+    </Box>
+  );
+}

packages/editor/src/extensions/audio/index.ts

@@ -0,0 +1,20 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+export * from "./audio.js";

packages/editor/src/index.ts

@@ -66,6 +66,7 @@ import { TaskItemNode } from "./extensions/task-item/index.js";
 import { TaskListNode } from "./extensions/task-list/index.js";
 import TextDirection from "./extensions/text-direction/index.js";
 import { WebClipNode, WebClipOptions } from "./extensions/web-clip/index.js";
+import { AudioNode, AudioOptions } from "./extensions/audio/index.js";
 import { useEditor } from "./hooks/use-editor.js";
 import { usePermissionHandler } from "./hooks/use-permission-handler.js";
 import Toolbar from "./toolbar/index.js";
@@ -125,6 +126,7 @@ const CoreExtensions = Object.entries(TiptapCoreExtensions)
 
 export type TiptapOptions = EditorOptions &
   Omit<WebClipOptions, "HTMLAttributes"> &
+  Omit<AudioOptions, "HTMLAttributes"> &
   Omit<ImageOptions, "HTMLAttributes"> &
   DateTimeOptions &
   TiptapStorage & {
@@ -312,7 +314,12 @@ const useTiptap = (
         ImageNode.configure({ allowBase64: true }),
         EmbedNode,
         AttachmentNode.configure({
-          types: [AttachmentNode.name, ImageNode.name, WebClipNode.name]
+          types: [
+            AttachmentNode.name,
+            ImageNode.name,
+            WebClipNode.name,
+            AudioNode.name
+          ]
         }),
         OutlineListItem,
         OutlineList.configure({ keepAttributes: true, keepMarks: true }),
@@ -327,6 +334,7 @@ const useTiptap = (
         DateTime.configure({ dateFormat, timeFormat, dayFormat }),
         KeyMap,
         WebClipNode,
+        AudioNode,
         CheckList,
         CheckListItem.configure({
           nested: true