feat: support for uploading screenshots (#242)

* feat: support for uploading screenshots

* feat: initiate detection of screen recording permissions

package.json

@@ -13,64 +13,66 @@
     "release-beta": "release-it --preRelease=beta --preReleaseBase=1"
   },
   "dependencies": {
-    "@headlessui/react": "^2.1.10",
+    "@headlessui/react": "^2.2.0",
-    "@tauri-apps/api": "^2.2.0",
+    "@tauri-apps/api": "^2.3.0",
-    "@tauri-apps/plugin-autostart": "~2",
+    "@tauri-apps/plugin-autostart": "~2.2.0",
     "@tauri-apps/plugin-deep-link": "^2.2.0",
     "@tauri-apps/plugin-dialog": "^2.2.0",
     "@tauri-apps/plugin-global-shortcut": "~2.0.0",
-    "@tauri-apps/plugin-http": "~2.0.1",
+    "@tauri-apps/plugin-http": "~2.0.2",
     "@tauri-apps/plugin-os": "^2.2.0",
-    "@tauri-apps/plugin-shell": ">=2.0.0",
+    "@tauri-apps/plugin-shell": "^2.2.0",
-    "@tauri-apps/plugin-updater": "^2.3.0",
+    "@tauri-apps/plugin-updater": "^2.5.1",
-    "@tauri-apps/plugin-websocket": "~2",
+    "@tauri-apps/plugin-websocket": "~2.3.0",
     "@tauri-apps/plugin-window": "2.0.0-alpha.1",
     "ahooks": "^3.8.4",
-    "axios": "^1.7.7",
+    "axios": "^1.8.1",
     "clsx": "^2.1.1",
     "dotenv": "^16.4.7",
     "filesize": "^10.1.6",
-    "i18next": "^23.16.2",
+    "i18next": "^23.16.8",
-    "i18next-browser-languagedetector": "^8.0.3",
+    "i18next-browser-languagedetector": "^8.0.4",
     "lodash-es": "^4.17.21",
     "lucide-react": "^0.461.0",
-    "mermaid": "^11.4.0",
+    "mermaid": "^11.4.1",
     "nanoid": "^5.1.2",
-    "react": "^18.2.0",
+    "react": "^18.3.1",
-    "react-dom": "^18.2.0",
+    "react-dom": "^18.3.1",
-    "react-hotkeys-hook": "^4.5.1",
+    "react-hotkeys-hook": "^4.6.1",
-    "react-i18next": "^15.1.0",
+    "react-i18next": "^15.4.1",
-    "react-markdown": "^9.0.1",
+    "react-markdown": "^9.1.0",
-    "react-router-dom": "^6.27.0",
+    "react-router-dom": "^6.30.0",
     "react-window": "^1.8.11",
-    "rehype-highlight": "^7.0.1",
+    "rehype-highlight": "^7.0.2",
     "rehype-katex": "^7.0.1",
     "remark-breaks": "^4.0.0",
-    "remark-gfm": "^4.0.0",
+    "remark-gfm": "^4.0.1",
     "remark-math": "^6.0.0",
     "tauri-plugin-fs-pro-api": "^2.3.1",
+    "tauri-plugin-macos-permissions-api": "^2.1.1",
+    "tauri-plugin-screenshots-api": "^2.1.0",
     "use-debounce": "^10.0.4",
-    "uuid": "^11.0.3",
+    "uuid": "^11.1.0",
-    "zustand": "^5.0.0"
+    "zustand": "^5.0.3"
   },
   "devDependencies": {
-    "@tauri-apps/cli": "^2.2.7",
+    "@tauri-apps/cli": "^2.3.1",
     "@types/lodash-es": "^4.17.12",
     "@types/markdown-it": "^14.1.2",
-    "@types/node": "^22.8.4",
+    "@types/node": "^22.13.9",
-    "@types/react": "^18.2.15",
+    "@types/react": "^18.3.18",
-    "@types/react-dom": "^18.2.7",
+    "@types/react-dom": "^18.3.5",
     "@types/react-i18next": "^8.1.0",
     "@types/react-katex": "^3.0.4",
     "@types/react-window": "^1.8.8",
-    "@vitejs/plugin-react": "^4.2.1",
+    "@vitejs/plugin-react": "^4.3.4",
     "autoprefixer": "^10.4.20",
     "immer": "^10.1.1",
-    "postcss": "^8.4.47",
+    "postcss": "^8.5.3",
     "release-it": "^18.1.2",
-    "tailwindcss": "^3.4.14",
+    "tailwindcss": "^3.4.17",
     "tsx": "^4.19.3",
-    "typescript": "^5.2.2",
+    "typescript": "^5.8.2",
-    "vite": "^5.3.1"
+    "vite": "^5.4.14"
   }
 }

src-tauri/Cargo.toml

@@ -19,6 +19,7 @@ tauri-build = { version = "2", features = ["default"] }
 [features]
 default = ["desktop"]
 desktop = []
+cargo-clippy = []
 
 [dependencies]
 pizza-common = { git = "https://github.com/infinilabs/pizza-common", branch = "main" }
@@ -38,8 +39,11 @@ tauri-plugin-fs = "2"
 tauri-plugin-updater = "2"
 tauri-plugin-process = "2"
 tauri-plugin-drag = "2"
+tauri-plugin-macos-permissions = "2"
 tauri-plugin-fs-pro = "2"
 
+tauri-plugin-screenshots = "2"
+
 tokio-native-tls = "0.3"  # For wss connections
 tokio = { version = "1", features = ["full"] }
 tokio-tungstenite = { version = "0.20", features = ["rustls-tls-webpki-roots"] }

src-tauri/capabilities/default.json

@@ -64,6 +64,8 @@
       "deny": []
     },
     "dialog:default",
-    "fs-pro:default"
+    "fs-pro:default",
+    "macos-permissions:default",
+    "screenshots:default"
   ]
 }

src-tauri/src/lib.rs

@@ -81,7 +81,9 @@ pub fn run() {
         .plugin(tauri_plugin_deep_link::init())
         .plugin(tauri_plugin_store::Builder::default().build())
         .plugin(tauri_plugin_dialog::init())
-        .plugin(tauri_plugin_fs_pro::init());
+        .plugin(tauri_plugin_fs_pro::init())
+        .plugin(tauri_plugin_macos_permissions::init())
+        .plugin(tauri_plugin_screenshots::init());
 
     // Conditional compilation for macOS
     #[cfg(target_os = "macos")]

src/components/Search/FileList.tsx

@@ -1,4 +1,5 @@
 import { useChatStore } from "@/stores/chatStore";
+import { isImage } from "@/utils";
 import { convertFileSrc } from "@tauri-apps/api/core";
 import { filesize } from "filesize";
 import { X } from "lucide-react";
@@ -14,7 +15,7 @@ const FileList = () => {
   return (
     <div className="flex flex-wrap gap-y-2 -mx-1 text-sm">
       {uploadFiles.map((file) => {
-        const { id, icon, name, extname, size } = file;
+        const { id, path, icon, name, extname, size } = file;
 
         return (
           <div key={id} className="w-1/3 px-1">
@@ -28,7 +29,10 @@ const FileList = () => {
                 <X className="size-[10px] text-white" />
               </div>
 
-              <img src={convertFileSrc(icon)} className="size-[40px]" />
+              <img
+                src={convertFileSrc(isImage(path) ? path : icon)}
+                className="size-[40px]"
+              />
 
               <div className="flex flex-col justify-between overflow-hidden">
                 <div className="truncate text-[#333333] dark:text-[#D8D8D8]">

src/components/Search/InputExtra.tsx

@@ -1,32 +1,67 @@
-import { Plus } from "lucide-react";
+import { ChevronRight, Plus } from "lucide-react";
-import { Menu, MenuButton, MenuItem, MenuItems } from "@headlessui/react";
+import {
+  Menu,
+  MenuButton,
+  MenuItem,
+  MenuItems,
+  Popover,
+  PopoverButton,
+  PopoverPanel,
+} from "@headlessui/react";
 import { open } from "@tauri-apps/plugin-dialog";
-import { find, isNil } from "lodash-es";
+import { castArray, find, isNil } from "lodash-es";
 import { useChatStore } from "@/stores/chatStore";
 import { metadata, icon } from "tauri-plugin-fs-pro-api";
 import { nanoid } from "nanoid";
 import Tooltip from "../Common/Tooltip";
 import { useAppStore } from "@/stores/appStore";
+import { useCreation, useMount, useReactive } from "ahooks";
+import {
+  checkScreenRecordingPermission,
+  requestScreenRecordingPermission,
+} from "tauri-plugin-macos-permissions-api";
+import {
+  getScreenshotableMonitors,
+  getScreenshotableWindows,
+  ScreenshotableMonitor,
+  ScreenshotableWindow,
+  getMonitorScreenshot,
+  getWindowScreenshot,
+} from "tauri-plugin-screenshots-api";
+import { Fragment, MouseEvent } from "react";
+
+interface State {
+  screenRecordingPermission?: boolean;
+  screenshotableMonitors: ScreenshotableMonitor[];
+  screenshotableWindows: ScreenshotableWindow[];
+}
+
+interface MenuItem {
+  label?: string;
+  groupName?: string;
+  groupItems?: MenuItem[];
+  children?: MenuItem[];
+  clickEvent?: (event: MouseEvent) => void;
+}
 
 const InputExtra = () => {
   const uploadFiles = useChatStore((state) => state.uploadFiles);
   const setUploadFiles = useChatStore((state) => state.setUploadFiles);
   const setIsPinned = useAppStore((state) => state.setIsPinned);
 
-  const uploadFile = async () => {
+  const state = useReactive<State>({
-    setIsPinned(true);
+    screenshotableMonitors: [],
-
+    screenshotableWindows: [],
-    const selectedFiles = await open({
-      multiple: true,
   });
 
-    setIsPinned(false);
+  useMount(async () => {
-
+    state.screenRecordingPermission = await checkScreenRecordingPermission();
-    if (isNil(selectedFiles)) return;
+  });
 
+  const handleUploadFiles = async (paths: string | string[]) => {
     const files: typeof uploadFiles = [];
 
-    for await (const path of selectedFiles) {
+    for await (const path of castArray(paths)) {
       if (find(uploadFiles, { path })) continue;
 
       const stat = await metadata(path);
@@ -46,17 +81,74 @@ const InputExtra = () => {
     setUploadFiles([...uploadFiles, ...files]);
   };
 
-  const menuItems = [
+  const menuItems = useCreation<MenuItem[]>(() => {
+    const menuItems: MenuItem[] = [
       {
         label: "上传文件",
-      event: uploadFile,
+        clickEvent: async () => {
+          setIsPinned(true);
+
+          const selectedFiles = await open({
+            multiple: true,
+          });
+
+          setIsPinned(false);
+
+          if (isNil(selectedFiles)) return;
+
+          handleUploadFiles(selectedFiles);
+        },
+      },
+      {
+        label: "截取屏幕截图",
+        clickEvent: async (event) => {
+          if (state.screenRecordingPermission) {
+            state.screenshotableMonitors = await getScreenshotableMonitors();
+            state.screenshotableWindows = await getScreenshotableWindows();
+          } else {
+            event.preventDefault();
+
+            requestScreenRecordingPermission();
+          }
+        },
+        children: [
+          {
+            groupName: "屏幕",
+            groupItems: state.screenshotableMonitors.map((item) => {
+              const { id, name } = item;
+
+              return {
+                label: name,
+                clickEvent: async () => {
+                  const path = await getMonitorScreenshot(id);
+
+                  handleUploadFiles(path);
+                },
+              };
+            }),
+          },
+          {
+            groupName: "窗口",
+            groupItems: state.screenshotableWindows.map((item) => {
+              const { id, name } = item;
+
+              return {
+                label: name,
+                clickEvent: async () => {
+                  const path = await getWindowScreenshot(id);
+
+                  handleUploadFiles(path);
+                },
+              };
+            }),
+          },
+        ],
       },
-    // {
-    //   label: "截取屏幕截图",
-    //   event: () => {},
-    // },
     ];
 
+    return menuItems;
+  }, [state.screenshotableMonitors, state.screenshotableWindows]);
+
   return (
     <Menu>
       <MenuButton>
@@ -66,21 +158,71 @@ const InputExtra = () => {
           </div>
         </Tooltip>
       </MenuButton>
+
       <MenuItems
         anchor="bottom start"
         className="p-1 text-sm bg-white dark:bg-[#202126] rounded-lg shadow-xs border border-gray-200 dark:border-gray-700"
       >
         {menuItems.map((item) => {
-          const { label, event } = item;
+          const { label, children, clickEvent } = item;
 
           return (
             <MenuItem key={label}>
+              {children ? (
+                <Popover>
+                  <PopoverButton
+                    className="flex items-center justify-between gap-2 px-3 py-2 hover:bg-black/5 hover:dark:bg-white/5 rounded-lg cursor-pointer"
+                    onClick={clickEvent}
+                  >
+                    <span>{label}</span>
+
+                    <ChevronRight className="size-4" />
+                  </PopoverButton>
+
+                  <PopoverPanel
+                    transition
+                    anchor="right"
+                    className="p-1 text-sm bg-white dark:bg-[#202126] rounded-lg shadow-xs border border-gray-200 dark:border-gray-700"
+                  >
+                    {children.map((childItem) => {
+                      const { groupName, groupItems } = childItem;
+
+                      return (
+                        <Fragment key={groupName}>
+                          <div
+                            className="px-3 py-1 text-xs text-[#999]"
+                            onClick={(event) => {
+                              event.preventDefault();
+                            }}
+                          >
+                            {groupName}
+                          </div>
+
+                          {groupItems?.map((groupItem) => {
+                            const { label, clickEvent } = groupItem;
+
+                            return (
                               <div
                                 className="px-3 py-2 hover:bg-black/5 hover:dark:bg-white/5 rounded-lg cursor-pointer"
-                onClick={event}
+                                onClick={clickEvent}
                               >
                                 {label}
                               </div>
+                            );
+                          })}
+                        </Fragment>
+                      );
+                    })}
+                  </PopoverPanel>
+                </Popover>
+              ) : (
+                <div
+                  className="px-3 py-2 hover:bg-black/5 hover:dark:bg-white/5 rounded-lg cursor-pointer"
+                  onClick={clickEvent}
+                >
+                  {label}
+                </div>
+              )}
             </MenuItem>
           );
         })}

src/utils/index.ts

@@ -55,7 +55,7 @@ export function useWindowSize() {
 
 export const IsTauri = () => {
   return Boolean(
-    typeof window !== 'undefined' &&
+    typeof window !== "undefined" &&
       window !== undefined &&
       (window as any).__TAURI_INTERNALS__ !== undefined
   );
@@ -86,8 +86,12 @@ export const formatter = {
     const index = Math.floor(Math.log(value) / Math.log(1024));
     const size = (value / Math.pow(1024, index)).toFixed(1);
 
-    return size + (unitArr[index] ?? "B")
+    return size + (unitArr[index] ?? "B");
   },
 };
 
+export const isImage = (value: string) => {
+  const regex = /\.(jpe?g|png|webp|avif|gif|svg|bmp|ico|tiff?|heic|apng)$/i;
 
+  return regex.test(value);
+};