diff --git a/src/clipboard.test.ts b/src/clipboard.test.ts index 142038e8..f1717464 100644 --- a/src/clipboard.test.ts +++ b/src/clipboard.test.ts @@ -1,26 +1,21 @@ import { parseClipboard } from "./clipboard"; +import { createPasteEvent } from "./tests/test-utils"; describe("Test parseClipboard", () => { it("should parse valid json correctly", async () => { let text = "123"; - let clipboardData = await parseClipboard({ - //@ts-ignore - clipboardData: { - getData: () => text, - }, - }); + let clipboardData = await parseClipboard( + createPasteEvent({ "text/plain": text }), + ); expect(clipboardData.text).toBe(text); text = "[123]"; - clipboardData = await parseClipboard({ - //@ts-ignore - clipboardData: { - getData: () => text, - }, - }); + clipboardData = await parseClipboard( + createPasteEvent({ "text/plain": text }), + ); expect(clipboardData.text).toBe(text); }); diff --git a/src/clipboard.ts b/src/clipboard.ts index c0deb93a..bc2167a0 100644 --- a/src/clipboard.ts +++ b/src/clipboard.ts @@ -18,11 +18,14 @@ type ElementsClipboard = { files: BinaryFiles | undefined; }; +export type PastedMixedContent = { type: "text" | "imageUrl"; value: string }[]; + export interface ClipboardData { spreadsheet?: Spreadsheet; elements?: readonly ExcalidrawElement[]; files?: BinaryFiles; text?: string; + mixedContent?: PastedMixedContent; errorMessage?: string; programmaticAPI?: boolean; } @@ -142,22 +145,74 @@ const parsePotentialSpreadsheet = ( return null; }; +/** internal, specific to parsing paste events. Do not reuse. */ +function parseHTMLTree(el: ChildNode) { + let result: PastedMixedContent = []; + for (const node of el.childNodes) { + if (node.nodeType === 3) { + const text = node.textContent?.trim(); + if (text) { + result.push({ type: "text", value: text }); + } + } else if (node instanceof HTMLImageElement) { + const url = node.getAttribute("src"); + if (url && url.startsWith("http")) { + result.push({ type: "imageUrl", value: url }); + } + } else { + result = result.concat(parseHTMLTree(node)); + } + } + return result; +} + +const maybeParseHTMLPaste = (event: ClipboardEvent) => { + const html = event.clipboardData?.getData("text/html"); + + if (!html) { + return null; + } + + try { + const doc = new DOMParser().parseFromString(html, "text/html"); + + const content = parseHTMLTree(doc.body); + + if (content.length) { + return content; + } + } catch (error: any) { + console.error(`error in parseHTMLFromPaste: ${error.message}`); + } + + return null; +}; + /** * Retrieves content from system clipboard (either from ClipboardEvent or * via async clipboard API if supported) */ -export const getSystemClipboard = async ( +const getSystemClipboard = async ( event: ClipboardEvent | null, -): Promise => { + isPlainPaste = false, +): Promise< + | { type: "text"; value: string } + | { type: "mixedContent"; value: PastedMixedContent } +> => { try { + const mixedContent = !isPlainPaste && event && maybeParseHTMLPaste(event); + if (mixedContent) { + return { type: "mixedContent", value: mixedContent }; + } + const text = event ? event.clipboardData?.getData("text/plain") : probablySupportsClipboardReadText && (await navigator.clipboard.readText()); - return (text || "").trim(); + return { type: "text", value: (text || "").trim() }; } catch { - return ""; + return { type: "text", value: "" }; } }; @@ -168,14 +223,20 @@ export const parseClipboard = async ( event: ClipboardEvent | null, isPlainPaste = false, ): Promise => { - const systemClipboard = await getSystemClipboard(event); + const systemClipboard = await getSystemClipboard(event, isPlainPaste); + + if (systemClipboard.type === "mixedContent") { + return { + mixedContent: systemClipboard.value, + }; + } // if system clipboard empty, couldn't be resolved, or contains previously // copied excalidraw scene as SVG, fall back to previously copied excalidraw // elements if ( !systemClipboard || - (!isPlainPaste && systemClipboard.includes(SVG_EXPORT_TAG)) + (!isPlainPaste && systemClipboard.value.includes(SVG_EXPORT_TAG)) ) { return getAppClipboard(); } @@ -183,7 +244,7 @@ export const parseClipboard = async ( // if system clipboard contains spreadsheet, use it even though it's // technically possible it's staler than in-app clipboard const spreadsheetResult = - !isPlainPaste && parsePotentialSpreadsheet(systemClipboard); + !isPlainPaste && parsePotentialSpreadsheet(systemClipboard.value); if (spreadsheetResult) { return spreadsheetResult; @@ -192,7 +253,7 @@ export const parseClipboard = async ( const appClipboardData = getAppClipboard(); try { - const systemClipboardData = JSON.parse(systemClipboard); + const systemClipboardData = JSON.parse(systemClipboard.value); const programmaticAPI = systemClipboardData.type === EXPORT_DATA_TYPES.excalidrawClipboardWithAPI; if (clipboardContainsElements(systemClipboardData)) { @@ -216,7 +277,7 @@ export const parseClipboard = async ( ? JSON.stringify(appClipboardData.elements, null, 2) : undefined, } - : { text: systemClipboard }; + : { text: systemClipboard.value }; }; export const copyBlobToClipboardAsPng = async (blob: Blob | Promise) => { diff --git a/src/components/App.tsx b/src/components/App.tsx index c9d58b35..9aa23d62 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -47,7 +47,7 @@ import { isEraserActive, isHandToolActive, } from "../appState"; -import { parseClipboard } from "../clipboard"; +import { PastedMixedContent, parseClipboard } from "../clipboard"; import { APP_NAME, CURSOR_TYPE, @@ -275,6 +275,7 @@ import { generateIdFromFile, getDataURL, getFileFromEvent, + ImageURLToFile, isImageFileHandle, isSupportedImageFile, loadSceneOrLibraryFromBlob, @@ -2183,21 +2184,6 @@ class App extends React.Component { return; } - // must be called in the same frame (thus before any awaits) as the paste - // event else some browsers (FF...) will clear the clipboardData - // (something something security) - let file = event?.clipboardData?.files[0]; - - const data = await parseClipboard(event, isPlainPaste); - if (!file && data.text && !isPlainPaste) { - const string = data.text.trim(); - if (string.startsWith("")) { - // ignore SVG validation/normalization which will be done during image - // initialization - file = SVGStringToFile(string); - } - } - const { x: sceneX, y: sceneY } = viewportCoordsToSceneCoords( { clientX: this.lastViewportPosition.x, @@ -2206,6 +2192,29 @@ class App extends React.Component { this.state, ); + // must be called in the same frame (thus before any awaits) as the paste + // event else some browsers (FF...) will clear the clipboardData + // (something something security) + let file = event?.clipboardData?.files[0]; + + const data = await parseClipboard(event, isPlainPaste); + if (!file && !isPlainPaste) { + if (data.mixedContent) { + return this.addElementsFromMixedContentPaste(data.mixedContent, { + isPlainPaste, + sceneX, + sceneY, + }); + } else if (data.text) { + const string = data.text.trim(); + if (string.startsWith("")) { + // ignore SVG validation/normalization which will be done during image + // initialization + file = SVGStringToFile(string); + } + } + } + // prefer spreadsheet data over image file (MS Office/Libre Office) if (isSupportedImageFile(file) && !data.spreadsheet) { const imageElement = this.createImageElement({ sceneX, sceneY }); @@ -2259,6 +2268,7 @@ class App extends React.Component { }); } else if (data.text) { const maybeUrl = extractSrc(data.text); + if ( !isPlainPaste && embeddableURLValidator(maybeUrl, this.props.validateEmbeddable) && @@ -2393,6 +2403,85 @@ class App extends React.Component { this.setActiveTool({ type: "selection" }); }; + // TODO rewrite this to paste both text & images at the same time if + // pasted data contains both + private async addElementsFromMixedContentPaste( + mixedContent: PastedMixedContent, + { + isPlainPaste, + sceneX, + sceneY, + }: { isPlainPaste: boolean; sceneX: number; sceneY: number }, + ) { + if ( + !isPlainPaste && + mixedContent.some((node) => node.type === "imageUrl") + ) { + const imageURLs = mixedContent + .filter((node) => node.type === "imageUrl") + .map((node) => node.value); + const responses = await Promise.all( + imageURLs.map(async (url) => { + try { + return { file: await ImageURLToFile(url) }; + } catch (error: any) { + return { errorMessage: error.message as string }; + } + }), + ); + let y = sceneY; + let firstImageYOffsetDone = false; + const nextSelectedIds: Record = {}; + for (const response of responses) { + if (response.file) { + const imageElement = this.createImageElement({ + sceneX, + sceneY: y, + }); + + const initializedImageElement = await this.insertImageElement( + imageElement, + response.file, + ); + if (initializedImageElement) { + // vertically center first image in the batch + if (!firstImageYOffsetDone) { + firstImageYOffsetDone = true; + y -= initializedImageElement.height / 2; + } + // hack to reset the `y` coord because we vertically center during + // insertImageElement + mutateElement(initializedImageElement, { y }, false); + + y = imageElement.y + imageElement.height + 25; + + nextSelectedIds[imageElement.id] = true; + } + } + } + + this.setState({ + selectedElementIds: makeNextSelectedElementIds( + nextSelectedIds, + this.state, + ), + }); + + const error = responses.find((response) => !!response.errorMessage); + if (error && error.errorMessage) { + this.setState({ errorMessage: error.errorMessage }); + } + } else { + const textNodes = mixedContent.filter((node) => node.type === "text"); + if (textNodes.length) { + this.addTextFromPaste( + textNodes.map((node) => node.value).join("\n\n"), + isPlainPaste, + ); + } + } + } + private addTextFromPaste(text: string, isPlainPaste = false) { const { x, y } = viewportCoordsToSceneCoords( { @@ -4401,6 +4490,7 @@ class App extends React.Component { setCursor(this.interactiveCanvas, CURSOR_TYPE.AUTO); } } + private handleCanvasPointerDown = ( event: React.PointerEvent, ) => { @@ -7302,7 +7392,7 @@ class App extends React.Component { this.scene.addNewElement(imageElement); try { - await this.initializeImage({ + return await this.initializeImage({ imageFile, imageElement, showCursorImagePreview, @@ -7315,6 +7405,7 @@ class App extends React.Component { this.setState({ errorMessage: error.message || t("errors.imageInsertError"), }); + return null; } }; diff --git a/src/data/blob.ts b/src/data/blob.ts index cb2c0519..81ce340f 100644 --- a/src/data/blob.ts +++ b/src/data/blob.ts @@ -327,6 +327,31 @@ export const SVGStringToFile = (SVGString: string, filename: string = "") => { }) as File & { type: typeof MIME_TYPES.svg }; }; +export const ImageURLToFile = async ( + imageUrl: string, + filename: string = "", +): Promise => { + let response; + try { + response = await fetch(imageUrl); + } catch (error: any) { + throw new Error(t("errors.failedToFetchImage")); + } + + if (!response.ok) { + throw new Error(t("errors.failedToFetchImage")); + } + + const blob = await response.blob(); + + if (blob.type && isSupportedImageFile(blob)) { + const name = filename || blob.name || ""; + return new File([blob], name, { type: blob.type }); + } + + throw new Error(t("errors.unsupportedFileType")); +}; + export const getFileFromEvent = async ( event: React.DragEvent, ) => { diff --git a/src/element/embeddable.ts b/src/element/embeddable.ts index 4aa6f0fd..615fb99a 100644 --- a/src/element/embeddable.ts +++ b/src/element/embeddable.ts @@ -28,6 +28,7 @@ const embeddedLinkCache = new Map(); const RE_YOUTUBE = /^(?:http(?:s)?:\/\/)?(?:www\.)?youtu(?:be\.com|\.be)\/(embed\/|watch\?v=|shorts\/|playlist\?list=|embed\/videoseries\?list=)?([a-zA-Z0-9_-]+)(?:\?t=|&t=|\?start=|&start=)?([a-zA-Z0-9_-]+)?[^\s]*$/; + const RE_VIMEO = /^(?:http(?:s)?:\/\/)?(?:(?:w){3}.)?(?:player\.)?vimeo\.com\/(?:video\/)?([^?\s]+)(?:\?.*)?$/; const RE_FIGMA = /^https:\/\/(?:www\.)?figma\.com/; diff --git a/src/locales/en.json b/src/locales/en.json index f2e6b601..19dbf81f 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -203,6 +203,7 @@ "imageInsertError": "Couldn't insert image. Try again later...", "fileTooBig": "File is too big. Maximum allowed size is {{maxSize}}.", "svgImageInsertError": "Couldn't insert SVG image. The SVG markup looks invalid.", + "failedToFetchImage": "Failed to fetch image.", "invalidSVGString": "Invalid SVG.", "cannotResolveCollabServer": "Couldn't connect to the collab server. Please reload the page and try again.", "importLibraryError": "Couldn't load library", diff --git a/src/tests/clipboard.test.tsx b/src/tests/clipboard.test.tsx index 4a55e068..0ea480ad 100644 --- a/src/tests/clipboard.test.tsx +++ b/src/tests/clipboard.test.tsx @@ -35,22 +35,14 @@ vi.mock("../keys.ts", async (importOriginal) => { }; }); -const setClipboardText = (text: string) => { - Object.assign(navigator, { - clipboard: { - readText: () => text, - }, +const sendPasteEvent = (text: string) => { + const clipboardEvent = createPasteEvent({ + "text/plain": text, }); -}; - -const sendPasteEvent = (text?: string) => { - const clipboardEvent = createPasteEvent( - text || (() => window.navigator.clipboard.readText()), - ); document.dispatchEvent(clipboardEvent); }; -const pasteWithCtrlCmdShiftV = (text?: string) => { +const pasteWithCtrlCmdShiftV = (text: string) => { Keyboard.withModifierKeys({ ctrl: true, shift: true }, () => { //triggering keydown with an empty clipboard Keyboard.keyPress(KEYS.V); @@ -59,7 +51,7 @@ const pasteWithCtrlCmdShiftV = (text?: string) => { }); }; -const pasteWithCtrlCmdV = (text?: string) => { +const pasteWithCtrlCmdV = (text: string) => { Keyboard.withModifierKeys({ ctrl: true }, () => { //triggering keydown with an empty clipboard Keyboard.keyPress(KEYS.V); @@ -86,7 +78,6 @@ beforeEach(async () => { initialData={{ appState: { zoom: { value: 1 as NormalizedZoomValue } } }} />, ); - setClipboardText(""); Object.assign(document, { elementFromPoint: () => GlobalTestState.canvas, }); @@ -120,8 +111,7 @@ describe("general paste behavior", () => { describe("paste text as single lines", () => { it("should create an element for each line when copying with Ctrl/Cmd+V", async () => { const text = "sajgfakfn\naaksfnknas\nakefnkasf"; - setClipboardText(text); - pasteWithCtrlCmdV(); + pasteWithCtrlCmdV(text); await waitFor(() => { expect(h.elements.length).toEqual(text.split("\n").length); }); @@ -129,8 +119,7 @@ describe("paste text as single lines", () => { it("should ignore empty lines when creating an element for each line", async () => { const text = "\n\nsajgfakfn\n\n\naaksfnknas\n\nakefnkasf\n\n\n"; - setClipboardText(text); - pasteWithCtrlCmdV(); + pasteWithCtrlCmdV(text); await waitFor(() => { expect(h.elements.length).toEqual(3); }); @@ -138,8 +127,7 @@ describe("paste text as single lines", () => { it("should not create any element if clipboard has only new lines", async () => { const text = "\n\n\n\n\n"; - setClipboardText(text); - pasteWithCtrlCmdV(); + pasteWithCtrlCmdV(text); await waitFor(async () => { await sleep(50); // elements lenght will always be zero if we don't wait, since paste is async expect(h.elements.length).toEqual(0); @@ -155,8 +143,7 @@ describe("paste text as single lines", () => { ) + 10 / h.app.state.zoom.value; mouse.moveTo(100, 100); - setClipboardText(text); - pasteWithCtrlCmdV(); + pasteWithCtrlCmdV(text); await waitFor(async () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars const [fx, firstElY] = getElementBounds(h.elements[0]); @@ -177,8 +164,7 @@ describe("paste text as single lines", () => { ) + 10 / h.app.state.zoom.value; mouse.moveTo(100, 100); - setClipboardText(text); - pasteWithCtrlCmdV(); + pasteWithCtrlCmdV(text); await waitFor(async () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars const [fx, firstElY] = getElementBounds(h.elements[0]); @@ -192,16 +178,14 @@ describe("paste text as single lines", () => { describe("paste text as a single element", () => { it("should create single text element when copying text with Ctrl/Cmd+Shift+V", async () => { const text = "sajgfakfn\naaksfnknas\nakefnkasf"; - setClipboardText(text); - pasteWithCtrlCmdShiftV(); + pasteWithCtrlCmdShiftV(text); await waitFor(() => { expect(h.elements.length).toEqual(1); }); }); it("should not create any element when only new lines in clipboard", async () => { const text = "\n\n\n\n"; - setClipboardText(text); - pasteWithCtrlCmdShiftV(); + pasteWithCtrlCmdShiftV(text); await waitFor(async () => { await sleep(50); expect(h.elements.length).toEqual(0); @@ -243,8 +227,7 @@ describe("Paste bound text container", () => { type: "excalidraw/clipboard", elements: [container, textElement], }); - setClipboardText(data); - pasteWithCtrlCmdShiftV(); + pasteWithCtrlCmdShiftV(data); await waitFor(async () => { await sleep(1); @@ -266,8 +249,7 @@ describe("Paste bound text container", () => { textElement, ], }); - setClipboardText(data); - pasteWithCtrlCmdShiftV(); + pasteWithCtrlCmdShiftV(data); await waitFor(async () => { await sleep(1); diff --git a/src/tests/flip.test.tsx b/src/tests/flip.test.tsx index b80b25f2..982105e9 100644 --- a/src/tests/flip.test.tsx +++ b/src/tests/flip.test.tsx @@ -727,7 +727,7 @@ describe("freedraw", () => { describe("image", () => { const createImage = async () => { const sendPasteEvent = (file?: File) => { - const clipboardEvent = createPasteEvent("", file ? [file] : []); + const clipboardEvent = createPasteEvent({}, file ? [file] : []); document.dispatchEvent(clipboardEvent); }; diff --git a/src/tests/test-utils.ts b/src/tests/test-utils.ts index fed6b489..0f80182a 100644 --- a/src/tests/test-utils.ts +++ b/src/tests/test-utils.ts @@ -208,10 +208,8 @@ export const assertSelectedElements = ( expect(selectedElementIds).toEqual(expect.arrayContaining(ids)); }; -export const createPasteEvent = ( - text: - | string - | /* getData function */ ((type: string) => string | Promise), +export const createPasteEvent = ( + items: Record, files?: File[], ) => { return Object.assign( @@ -222,11 +220,12 @@ export const createPasteEvent = ( }), { clipboardData: { - getData: typeof text === "string" ? () => text : text, + getData: (type: string) => + (items as Record)[type] || "", files: files || [], }, }, - ); + ) as any as ClipboardEvent; }; export const toggleMenu = (container: HTMLElement) => {