fix: right-click paste for images in clipboard (Issue #8826) (#8845)

* Fix right-click paste command for images (Issue #8826)

* Fix clipboard logic for multiple paste types

* fix: remove unused code

* refactor & robustness

* fix: creating paste event with image files

---------

Co-authored-by: dwelle <5153846+dwelle@users.noreply.github.com>

packages/excalidraw/clipboard.ts

@@ -18,6 +18,8 @@ import { deepCopyElement } from "./element/newElement";
 import { mutateElement } from "./element/mutateElement";
 import { getContainingFrame } from "./frame";
 import { arrayToMap, isMemberOf, isPromiseLike } from "./utils";
+import { createFile, isSupportedImageFileType } from "./data/blob";
+import { ExcalidrawError } from "./errors";
 
 type ElementsClipboard = {
   type: typeof EXPORT_DATA_TYPES.excalidrawClipboard;
@@ -39,7 +41,7 @@ export interface ClipboardData {
 
 type AllowedPasteMimeTypes = typeof ALLOWED_PASTE_MIME_TYPES[number];
 
-type ParsedClipboardEvent =
+type ParsedClipboardEventTextData =
   | { type: "text"; value: string }
   | { type: "mixedContent"; value: PastedMixedContent };
 
@@ -75,7 +77,7 @@ export const createPasteEvent = ({
   types,
   files,
 }: {
-  types?: { [key in AllowedPasteMimeTypes]?: string };
+  types?: { [key in AllowedPasteMimeTypes]?: string | File };
   files?: File[];
 }) => {
   if (!types && !files) {
@@ -88,6 +90,11 @@ export const createPasteEvent = ({
 
   if (types) {
     for (const [type, value] of Object.entries(types)) {
+      if (typeof value !== "string") {
+        files = files || [];
+        files.push(value);
+        continue;
+      }
       try {
         event.clipboardData?.setData(type, value);
         if (event.clipboardData?.getData(type) !== value) {
@@ -217,14 +224,14 @@ function parseHTMLTree(el: ChildNode) {
 const maybeParseHTMLPaste = (
   event: ClipboardEvent,
 ): { type: "mixedContent"; value: PastedMixedContent } | null => {
-  const html = event.clipboardData?.getData("text/html");
+  const html = event.clipboardData?.getData(MIME_TYPES.html);
 
   if (!html) {
     return null;
   }
 
   try {
-    const doc = new DOMParser().parseFromString(html, "text/html");
+    const doc = new DOMParser().parseFromString(html, MIME_TYPES.html);
 
     const content = parseHTMLTree(doc.body);
 
@@ -238,34 +245,44 @@ const maybeParseHTMLPaste = (
   return null;
 };
 
+/**
+ * Reads OS clipboard programmatically. May not work on all browsers.
+ * Will prompt user for permission if not granted.
+ */
 export const readSystemClipboard = async () => {
-  const types: { [key in AllowedPasteMimeTypes]?: string } = {};
+  const types: { [key in AllowedPasteMimeTypes]?: string | File } = {};
-
-  try {
-    if (navigator.clipboard?.readText) {
-      return { "text/plain": await navigator.clipboard?.readText() };
-    }
-  } catch (error: any) {
-    // @ts-ignore
-    if (navigator.clipboard?.read) {
-      console.warn(
-        `navigator.clipboard.readText() failed (${error.message}). Failling back to navigator.clipboard.read()`,
-      );
-    } else {
-      throw error;
-    }
-  }
 
   let clipboardItems: ClipboardItems;
 
   try {
     clipboardItems = await navigator.clipboard?.read();
   } catch (error: any) {
-    if (error.name === "DataError") {
+    try {
-      console.warn(
+      if (navigator.clipboard?.readText) {
-        `navigator.clipboard.read() error, clipboard is probably empty: ${error.message}`,
+        console.warn(
-      );
+          `navigator.clipboard.readText() failed (${error.message}). Failling back to navigator.clipboard.read()`,
-      return types;
+        );
+        const readText = await navigator.clipboard?.readText();
+        if (readText) {
+          return { [MIME_TYPES.text]: readText };
+        }
+      }
+    } catch (error: any) {
+      // @ts-ignore
+      if (navigator.clipboard?.read) {
+        console.warn(
+          `navigator.clipboard.readText() failed (${error.message}). Failling back to navigator.clipboard.read()`,
+        );
+      } else {
+        if (error.name === "DataError") {
+          console.warn(
+            `navigator.clipboard.read() error, clipboard is probably empty: ${error.message}`,
+          );
+          return types;
+        }
+
+        throw error;
+      }
     }
     throw error;
   }
@@ -276,10 +293,20 @@ export const readSystemClipboard = async () => {
         continue;
       }
       try {
-        types[type] = await (await item.getType(type)).text();
+        if (type === MIME_TYPES.text || type === MIME_TYPES.html) {
+          types[type] = await (await item.getType(type)).text();
+        } else if (isSupportedImageFileType(type)) {
+          const imageBlob = await item.getType(type);
+          const file = createFile(imageBlob, type, undefined);
+          types[type] = file;
+        } else {
+          throw new ExcalidrawError(`Unsupported clipboard type: ${type}`);
+        }
       } catch (error: any) {
         console.warn(
-          `Cannot retrieve ${type} from clipboardItem: ${error.message}`,
+          error instanceof ExcalidrawError
+            ? error.message
+            : `Cannot retrieve ${type} from clipboardItem: ${error.message}`,
         );
       }
     }
@@ -296,10 +323,10 @@ export const readSystemClipboard = async () => {
 /**
  * Parses "paste" ClipboardEvent.
  */
-const parseClipboardEvent = async (
+const parseClipboardEventTextData = async (
   event: ClipboardEvent,
   isPlainPaste = false,
-): Promise<ParsedClipboardEvent> => {
+): Promise<ParsedClipboardEventTextData> => {
   try {
     const mixedContent = !isPlainPaste && event && maybeParseHTMLPaste(event);
 
@@ -308,7 +335,7 @@ const parseClipboardEvent = async (
         return {
           type: "text",
           value:
-            event.clipboardData?.getData("text/plain") ||
+            event.clipboardData?.getData(MIME_TYPES.text) ||
             mixedContent.value
               .map((item) => item.value)
               .join("\n")
@@ -319,7 +346,7 @@ const parseClipboardEvent = async (
       return mixedContent;
     }
 
-    const text = event.clipboardData?.getData("text/plain");
+    const text = event.clipboardData?.getData(MIME_TYPES.text);
 
     return { type: "text", value: (text || "").trim() };
   } catch {
@@ -328,13 +355,16 @@ const parseClipboardEvent = async (
 };
 
 /**
- * Attempts to parse clipboard. Prefers system clipboard.
+ * Attempts to parse clipboard event.
  */
 export const parseClipboard = async (
   event: ClipboardEvent,
   isPlainPaste = false,
 ): Promise<ClipboardData> => {
-  const parsedEventData = await parseClipboardEvent(event, isPlainPaste);
+  const parsedEventData = await parseClipboardEventTextData(
+    event,
+    isPlainPaste,
+  );
 
   if (parsedEventData.type === "mixedContent") {
     return {
@@ -423,8 +453,8 @@ export const copyTextToSystemClipboard = async (
   // (2) if fails and we have access to ClipboardEvent, use plain old setData()
   try {
     if (clipboardEvent) {
-      clipboardEvent.clipboardData?.setData("text/plain", text || "");
+      clipboardEvent.clipboardData?.setData(MIME_TYPES.text, text || "");
-      if (clipboardEvent.clipboardData?.getData("text/plain") !== text) {
+      if (clipboardEvent.clipboardData?.getData(MIME_TYPES.text) !== text) {
         throw new Error("Failed to setData on clipboardEvent");
       }
       return;

packages/excalidraw/constants.ts

@@ -214,9 +214,9 @@ export const IMAGE_MIME_TYPES = {
   jfif: "image/jfif",
 } as const;
 
-export const ALLOWED_PASTE_MIME_TYPES = ["text/plain", "text/html"] as const;
-
 export const MIME_TYPES = {
+  text: "text/plain",
+  html: "text/html",
   json: "application/json",
   // excalidraw data
   excalidraw: "application/vnd.excalidraw+json",
@@ -230,6 +230,12 @@ export const MIME_TYPES = {
   ...IMAGE_MIME_TYPES,
 } as const;
 
+export const ALLOWED_PASTE_MIME_TYPES = [
+  MIME_TYPES.text,
+  MIME_TYPES.html,
+  ...Object.values(IMAGE_MIME_TYPES),
+] as const;
+
 export const EXPORT_IMAGE_TYPES = {
   png: "png",
   svg: "svg",

packages/excalidraw/data/blob.ts

@@ -106,11 +106,15 @@ export const isImageFileHandle = (handle: FileSystemHandle | null) => {
   return type === "png" || type === "svg";
 };
 
+export const isSupportedImageFileType = (type: string | null | undefined) => {
+  return !!type && (Object.values(IMAGE_MIME_TYPES) as string[]).includes(type);
+};
+
 export const isSupportedImageFile = (
   blob: Blob | null | undefined,
 ): blob is Blob & { type: ValueOf<typeof IMAGE_MIME_TYPES> } => {
   const { type } = blob || {};
-  return !!type && (Object.values(IMAGE_MIME_TYPES) as string[]).includes(type);
+  return isSupportedImageFileType(type);
 };
 
 export const loadSceneOrLibraryFromBlob = async (