feat: Added Copy/Paste from Google Docs (#7136)

Co-authored-by: dwelle <luzar.david@gmail.com>

src/clipboard.test.ts

@@ -1,26 +1,21 @@
 import { parseClipboard } from "./clipboard";
+import { createPasteEvent } from "./tests/test-utils";
 
 describe("Test parseClipboard", () => {
   it("should parse valid json correctly", async () => {
     let text = "123";
 
-    let clipboardData = await parseClipboard({
-      //@ts-ignore
-      clipboardData: {
-        getData: () => text,
-      },
-    });
+    let clipboardData = await parseClipboard(
+      createPasteEvent({ "text/plain": text }),
+    );
 
     expect(clipboardData.text).toBe(text);
 
     text = "[123]";
 
-    clipboardData = await parseClipboard({
-      //@ts-ignore
-      clipboardData: {
-        getData: () => text,
-      },
-    });
+    clipboardData = await parseClipboard(
+      createPasteEvent({ "text/plain": text }),
+    );
 
     expect(clipboardData.text).toBe(text);
   });

src/clipboard.ts

@@ -18,11 +18,14 @@ type ElementsClipboard = {
   files: BinaryFiles | undefined;
 };
 
+export type PastedMixedContent = { type: "text" | "imageUrl"; value: string }[];
+
 export interface ClipboardData {
   spreadsheet?: Spreadsheet;
   elements?: readonly ExcalidrawElement[];
   files?: BinaryFiles;
   text?: string;
+  mixedContent?: PastedMixedContent;
   errorMessage?: string;
   programmaticAPI?: boolean;
 }
@@ -142,22 +145,74 @@ const parsePotentialSpreadsheet = (
   return null;
 };
 
+/** internal, specific to parsing paste events. Do not reuse. */
+function parseHTMLTree(el: ChildNode) {
+  let result: PastedMixedContent = [];
+  for (const node of el.childNodes) {
+    if (node.nodeType === 3) {
+      const text = node.textContent?.trim();
+      if (text) {
+        result.push({ type: "text", value: text });
+      }
+    } else if (node instanceof HTMLImageElement) {
+      const url = node.getAttribute("src");
+      if (url && url.startsWith("http")) {
+        result.push({ type: "imageUrl", value: url });
+      }
+    } else {
+      result = result.concat(parseHTMLTree(node));
+    }
+  }
+  return result;
+}
+
+const maybeParseHTMLPaste = (event: ClipboardEvent) => {
+  const html = event.clipboardData?.getData("text/html");
+
+  if (!html) {
+    return null;
+  }
+
+  try {
+    const doc = new DOMParser().parseFromString(html, "text/html");
+
+    const content = parseHTMLTree(doc.body);
+
+    if (content.length) {
+      return content;
+    }
+  } catch (error: any) {
+    console.error(`error in parseHTMLFromPaste: ${error.message}`);
+  }
+
+  return null;
+};
+
 /**
  * Retrieves content from system clipboard (either from ClipboardEvent or
  *  via async clipboard API if supported)
  */
-export const getSystemClipboard = async (
+const getSystemClipboard = async (
   event: ClipboardEvent | null,
-): Promise<string> => {
+  isPlainPaste = false,
+): Promise<
+  | { type: "text"; value: string }
+  | { type: "mixedContent"; value: PastedMixedContent }
+> => {
   try {
+    const mixedContent = !isPlainPaste && event && maybeParseHTMLPaste(event);
+    if (mixedContent) {
+      return { type: "mixedContent", value: mixedContent };
+    }
+
     const text = event
       ? event.clipboardData?.getData("text/plain")
       : probablySupportsClipboardReadText &&
         (await navigator.clipboard.readText());
 
-    return (text || "").trim();
+    return { type: "text", value: (text || "").trim() };
   } catch {
-    return "";
+    return { type: "text", value: "" };
   }
 };
 
@@ -168,14 +223,20 @@ export const parseClipboard = async (
   event: ClipboardEvent | null,
   isPlainPaste = false,
 ): Promise<ClipboardData> => {
-  const systemClipboard = await getSystemClipboard(event);
+  const systemClipboard = await getSystemClipboard(event, isPlainPaste);
+
+  if (systemClipboard.type === "mixedContent") {
+    return {
+      mixedContent: systemClipboard.value,
+    };
+  }
 
   // if system clipboard empty, couldn't be resolved, or contains previously
   // copied excalidraw scene as SVG, fall back to previously copied excalidraw
   // elements
   if (
     !systemClipboard ||
-    (!isPlainPaste && systemClipboard.includes(SVG_EXPORT_TAG))
+    (!isPlainPaste && systemClipboard.value.includes(SVG_EXPORT_TAG))
   ) {
     return getAppClipboard();
   }
@@ -183,7 +244,7 @@ export const parseClipboard = async (
   // if system clipboard contains spreadsheet, use it even though it's
   // technically possible it's staler than in-app clipboard
   const spreadsheetResult =
-    !isPlainPaste && parsePotentialSpreadsheet(systemClipboard);
+    !isPlainPaste && parsePotentialSpreadsheet(systemClipboard.value);
 
   if (spreadsheetResult) {
     return spreadsheetResult;
@@ -192,7 +253,7 @@ export const parseClipboard = async (
   const appClipboardData = getAppClipboard();
 
   try {
-    const systemClipboardData = JSON.parse(systemClipboard);
+    const systemClipboardData = JSON.parse(systemClipboard.value);
     const programmaticAPI =
       systemClipboardData.type === EXPORT_DATA_TYPES.excalidrawClipboardWithAPI;
     if (clipboardContainsElements(systemClipboardData)) {
@@ -216,7 +277,7 @@ export const parseClipboard = async (
           ? JSON.stringify(appClipboardData.elements, null, 2)
           : undefined,
       }
-    : { text: systemClipboard };
+    : { text: systemClipboard.value };
 };
 
 export const copyBlobToClipboardAsPng = async (blob: Blob | Promise<Blob>) => {

src/components/App.tsx

@@ -47,7 +47,7 @@ import {
   isEraserActive,
   isHandToolActive,
 } from "../appState";
-import { parseClipboard } from "../clipboard";
+import { PastedMixedContent, parseClipboard } from "../clipboard";
 import {
   APP_NAME,
   CURSOR_TYPE,
@@ -275,6 +275,7 @@ import {
   generateIdFromFile,
   getDataURL,
   getFileFromEvent,
+  ImageURLToFile,
   isImageFileHandle,
   isSupportedImageFile,
   loadSceneOrLibraryFromBlob,
@@ -2183,29 +2184,37 @@ class App extends React.Component<AppProps, AppState> {
         return;
       }
 
+      const { x: sceneX, y: sceneY } = viewportCoordsToSceneCoords(
+        {
+          clientX: this.lastViewportPosition.x,
+          clientY: this.lastViewportPosition.y,
+        },
+        this.state,
+      );
+
       // must be called in the same frame (thus before any awaits) as the paste
       // event else some browsers (FF...) will clear the clipboardData
       // (something something security)
       let file = event?.clipboardData?.files[0];
 
       const data = await parseClipboard(event, isPlainPaste);
-      if (!file && data.text && !isPlainPaste) {
-        const string = data.text.trim();
-        if (string.startsWith("<svg") && string.endsWith("</svg>")) {
-          // ignore SVG validation/normalization which will be done during image
-          // initialization
-          file = SVGStringToFile(string);
+      if (!file && !isPlainPaste) {
+        if (data.mixedContent) {
+          return this.addElementsFromMixedContentPaste(data.mixedContent, {
+            isPlainPaste,
+            sceneX,
+            sceneY,
+          });
+        } else if (data.text) {
+          const string = data.text.trim();
+          if (string.startsWith("<svg") && string.endsWith("</svg>")) {
+            // ignore SVG validation/normalization which will be done during image
+            // initialization
+            file = SVGStringToFile(string);
+          }
         }
       }
 
-      const { x: sceneX, y: sceneY } = viewportCoordsToSceneCoords(
-        {
-          clientX: this.lastViewportPosition.x,
-          clientY: this.lastViewportPosition.y,
-        },
-        this.state,
-      );
-
       // prefer spreadsheet data over image file (MS Office/Libre Office)
       if (isSupportedImageFile(file) && !data.spreadsheet) {
         const imageElement = this.createImageElement({ sceneX, sceneY });
@@ -2259,6 +2268,7 @@ class App extends React.Component<AppProps, AppState> {
         });
       } else if (data.text) {
         const maybeUrl = extractSrc(data.text);
+
         if (
           !isPlainPaste &&
           embeddableURLValidator(maybeUrl, this.props.validateEmbeddable) &&
@@ -2393,6 +2403,85 @@ class App extends React.Component<AppProps, AppState> {
     this.setActiveTool({ type: "selection" });
   };
 
+  // TODO rewrite this to paste both text & images at the same time if
+  // pasted data contains both
+  private async addElementsFromMixedContentPaste(
+    mixedContent: PastedMixedContent,
+    {
+      isPlainPaste,
+      sceneX,
+      sceneY,
+    }: { isPlainPaste: boolean; sceneX: number; sceneY: number },
+  ) {
+    if (
+      !isPlainPaste &&
+      mixedContent.some((node) => node.type === "imageUrl")
+    ) {
+      const imageURLs = mixedContent
+        .filter((node) => node.type === "imageUrl")
+        .map((node) => node.value);
+      const responses = await Promise.all(
+        imageURLs.map(async (url) => {
+          try {
+            return { file: await ImageURLToFile(url) };
+          } catch (error: any) {
+            return { errorMessage: error.message as string };
+          }
+        }),
+      );
+      let y = sceneY;
+      let firstImageYOffsetDone = false;
+      const nextSelectedIds: Record<ExcalidrawElement["id"], true> = {};
+      for (const response of responses) {
+        if (response.file) {
+          const imageElement = this.createImageElement({
+            sceneX,
+            sceneY: y,
+          });
+
+          const initializedImageElement = await this.insertImageElement(
+            imageElement,
+            response.file,
+          );
+          if (initializedImageElement) {
+            // vertically center first image in the batch
+            if (!firstImageYOffsetDone) {
+              firstImageYOffsetDone = true;
+              y -= initializedImageElement.height / 2;
+            }
+            // hack to reset the `y` coord because we vertically center during
+            // insertImageElement
+            mutateElement(initializedImageElement, { y }, false);
+
+            y = imageElement.y + imageElement.height + 25;
+
+            nextSelectedIds[imageElement.id] = true;
+          }
+        }
+      }
+
+      this.setState({
+        selectedElementIds: makeNextSelectedElementIds(
+          nextSelectedIds,
+          this.state,
+        ),
+      });
+
+      const error = responses.find((response) => !!response.errorMessage);
+      if (error && error.errorMessage) {
+        this.setState({ errorMessage: error.errorMessage });
+      }
+    } else {
+      const textNodes = mixedContent.filter((node) => node.type === "text");
+      if (textNodes.length) {
+        this.addTextFromPaste(
+          textNodes.map((node) => node.value).join("\n\n"),
+          isPlainPaste,
+        );
+      }
+    }
+  }
+
   private addTextFromPaste(text: string, isPlainPaste = false) {
     const { x, y } = viewportCoordsToSceneCoords(
       {
@@ -4401,6 +4490,7 @@ class App extends React.Component<AppProps, AppState> {
       setCursor(this.interactiveCanvas, CURSOR_TYPE.AUTO);
     }
   }
+
   private handleCanvasPointerDown = (
     event: React.PointerEvent<HTMLElement>,
   ) => {
@@ -7302,7 +7392,7 @@ class App extends React.Component<AppProps, AppState> {
     this.scene.addNewElement(imageElement);
 
     try {
-      await this.initializeImage({
+      return await this.initializeImage({
         imageFile,
         imageElement,
         showCursorImagePreview,
@@ -7315,6 +7405,7 @@ class App extends React.Component<AppProps, AppState> {
       this.setState({
         errorMessage: error.message || t("errors.imageInsertError"),
       });
+      return null;
     }
   };
 

src/data/blob.ts

@@ -327,6 +327,31 @@ export const SVGStringToFile = (SVGString: string, filename: string = "") => {
   }) as File & { type: typeof MIME_TYPES.svg };
 };
 
+export const ImageURLToFile = async (
+  imageUrl: string,
+  filename: string = "",
+): Promise<File | undefined> => {
+  let response;
+  try {
+    response = await fetch(imageUrl);
+  } catch (error: any) {
+    throw new Error(t("errors.failedToFetchImage"));
+  }
+
+  if (!response.ok) {
+    throw new Error(t("errors.failedToFetchImage"));
+  }
+
+  const blob = await response.blob();
+
+  if (blob.type && isSupportedImageFile(blob)) {
+    const name = filename || blob.name || "";
+    return new File([blob], name, { type: blob.type });
+  }
+
+  throw new Error(t("errors.unsupportedFileType"));
+};
+
 export const getFileFromEvent = async (
   event: React.DragEvent<HTMLDivElement>,
 ) => {

src/element/embeddable.ts

@@ -28,6 +28,7 @@ const embeddedLinkCache = new Map<string, EmbeddedLink>();
 
 const RE_YOUTUBE =
   /^(?:http(?:s)?:\/\/)?(?:www\.)?youtu(?:be\.com|\.be)\/(embed\/|watch\?v=|shorts\/|playlist\?list=|embed\/videoseries\?list=)?([a-zA-Z0-9_-]+)(?:\?t=|&t=|\?start=|&start=)?([a-zA-Z0-9_-]+)?[^\s]*$/;
+
 const RE_VIMEO =
   /^(?:http(?:s)?:\/\/)?(?:(?:w){3}.)?(?:player\.)?vimeo\.com\/(?:video\/)?([^?\s]+)(?:\?.*)?$/;
 const RE_FIGMA = /^https:\/\/(?:www\.)?figma\.com/;

src/locales/en.json

@@ -203,6 +203,7 @@
     "imageInsertError": "Couldn't insert image. Try again later...",
     "fileTooBig": "File is too big. Maximum allowed size is {{maxSize}}.",
     "svgImageInsertError": "Couldn't insert SVG image. The SVG markup looks invalid.",
+    "failedToFetchImage": "Failed to fetch image.",
     "invalidSVGString": "Invalid SVG.",
     "cannotResolveCollabServer": "Couldn't connect to the collab server. Please reload the page and try again.",
     "importLibraryError": "Couldn't load library",

src/tests/clipboard.test.tsx

@@ -35,22 +35,14 @@ vi.mock("../keys.ts", async (importOriginal) => {
   };
 });
 
-const setClipboardText = (text: string) => {
-  Object.assign(navigator, {
-    clipboard: {
-      readText: () => text,
-    },
+const sendPasteEvent = (text: string) => {
+  const clipboardEvent = createPasteEvent({
+    "text/plain": text,
   });
-};
-
-const sendPasteEvent = (text?: string) => {
-  const clipboardEvent = createPasteEvent(
-    text || (() => window.navigator.clipboard.readText()),
-  );
   document.dispatchEvent(clipboardEvent);
 };
 
-const pasteWithCtrlCmdShiftV = (text?: string) => {
+const pasteWithCtrlCmdShiftV = (text: string) => {
   Keyboard.withModifierKeys({ ctrl: true, shift: true }, () => {
     //triggering keydown with an empty clipboard
     Keyboard.keyPress(KEYS.V);
@@ -59,7 +51,7 @@ const pasteWithCtrlCmdShiftV = (text?: string) => {
   });
 };
 
-const pasteWithCtrlCmdV = (text?: string) => {
+const pasteWithCtrlCmdV = (text: string) => {
   Keyboard.withModifierKeys({ ctrl: true }, () => {
     //triggering keydown with an empty clipboard
     Keyboard.keyPress(KEYS.V);
@@ -86,7 +78,6 @@ beforeEach(async () => {
       initialData={{ appState: { zoom: { value: 1 as NormalizedZoomValue } } }}
     />,
   );
-  setClipboardText("");
   Object.assign(document, {
     elementFromPoint: () => GlobalTestState.canvas,
   });
@@ -120,8 +111,7 @@ describe("general paste behavior", () => {
 describe("paste text as single lines", () => {
   it("should create an element for each line when copying with Ctrl/Cmd+V", async () => {
     const text = "sajgfakfn\naaksfnknas\nakefnkasf";
-    setClipboardText(text);
-    pasteWithCtrlCmdV();
+    pasteWithCtrlCmdV(text);
     await waitFor(() => {
       expect(h.elements.length).toEqual(text.split("\n").length);
     });
@@ -129,8 +119,7 @@ describe("paste text as single lines", () => {
 
   it("should ignore empty lines when creating an element for each line", async () => {
     const text = "\n\nsajgfakfn\n\n\naaksfnknas\n\nakefnkasf\n\n\n";
-    setClipboardText(text);
-    pasteWithCtrlCmdV();
+    pasteWithCtrlCmdV(text);
     await waitFor(() => {
       expect(h.elements.length).toEqual(3);
     });
@@ -138,8 +127,7 @@ describe("paste text as single lines", () => {
 
   it("should not create any element if clipboard has only new lines", async () => {
     const text = "\n\n\n\n\n";
-    setClipboardText(text);
-    pasteWithCtrlCmdV();
+    pasteWithCtrlCmdV(text);
     await waitFor(async () => {
       await sleep(50); // elements lenght will always be zero if we don't wait, since paste is async
       expect(h.elements.length).toEqual(0);
@@ -155,8 +143,7 @@ describe("paste text as single lines", () => {
       ) +
       10 / h.app.state.zoom.value;
     mouse.moveTo(100, 100);
-    setClipboardText(text);
-    pasteWithCtrlCmdV();
+    pasteWithCtrlCmdV(text);
     await waitFor(async () => {
       // eslint-disable-next-line @typescript-eslint/no-unused-vars
       const [fx, firstElY] = getElementBounds(h.elements[0]);
@@ -177,8 +164,7 @@ describe("paste text as single lines", () => {
       ) +
       10 / h.app.state.zoom.value;
     mouse.moveTo(100, 100);
-    setClipboardText(text);
-    pasteWithCtrlCmdV();
+    pasteWithCtrlCmdV(text);
     await waitFor(async () => {
       // eslint-disable-next-line @typescript-eslint/no-unused-vars
       const [fx, firstElY] = getElementBounds(h.elements[0]);
@@ -192,16 +178,14 @@ describe("paste text as single lines", () => {
 describe("paste text as a single element", () => {
   it("should create single text element when copying text with Ctrl/Cmd+Shift+V", async () => {
     const text = "sajgfakfn\naaksfnknas\nakefnkasf";
-    setClipboardText(text);
-    pasteWithCtrlCmdShiftV();
+    pasteWithCtrlCmdShiftV(text);
     await waitFor(() => {
       expect(h.elements.length).toEqual(1);
     });
   });
   it("should not create any element when only new lines in clipboard", async () => {
     const text = "\n\n\n\n";
-    setClipboardText(text);
-    pasteWithCtrlCmdShiftV();
+    pasteWithCtrlCmdShiftV(text);
     await waitFor(async () => {
       await sleep(50);
       expect(h.elements.length).toEqual(0);
@@ -243,8 +227,7 @@ describe("Paste bound text container", () => {
       type: "excalidraw/clipboard",
       elements: [container, textElement],
     });
-    setClipboardText(data);
-    pasteWithCtrlCmdShiftV();
+    pasteWithCtrlCmdShiftV(data);
 
     await waitFor(async () => {
       await sleep(1);
@@ -266,8 +249,7 @@ describe("Paste bound text container", () => {
         textElement,
       ],
     });
-    setClipboardText(data);
-    pasteWithCtrlCmdShiftV();
+    pasteWithCtrlCmdShiftV(data);
 
     await waitFor(async () => {
       await sleep(1);

src/tests/flip.test.tsx

@@ -727,7 +727,7 @@ describe("freedraw", () => {
 describe("image", () => {
   const createImage = async () => {
     const sendPasteEvent = (file?: File) => {
-      const clipboardEvent = createPasteEvent("", file ? [file] : []);
+      const clipboardEvent = createPasteEvent({}, file ? [file] : []);
       document.dispatchEvent(clipboardEvent);
     };
 

src/tests/test-utils.ts

@@ -208,10 +208,8 @@ export const assertSelectedElements = (
   expect(selectedElementIds).toEqual(expect.arrayContaining(ids));
 };
 
-export const createPasteEvent = (
-  text:
-    | string
-    | /* getData function */ ((type: string) => string | Promise<string>),
+export const createPasteEvent = <T extends "text/plain" | "text/html">(
+  items: Record<T, string>,
   files?: File[],
 ) => {
   return Object.assign(
@@ -222,11 +220,12 @@ export const createPasteEvent = (
     }),
     {
       clipboardData: {
-        getData: typeof text === "string" ? () => text : text,
+        getData: (type: string) =>
+          (items as Record<string, string>)[type] || "",
         files: files || [],
       },
     },
-  );
+  ) as any as ClipboardEvent;
 };
 
 export const toggleMenu = (container: HTMLElement) => {