From c7ee46e7f8a738b8003701b9089525c3aefa1204 Mon Sep 17 00:00:00 2001 From: David Luzar <5153846+dwelle@users.noreply.github.com> Date: Thu, 23 Nov 2023 23:07:53 +0100 Subject: [PATCH] feat: wireframe-to-code (#7334) --- src/actions/actionAlign.tsx | 3 +- src/actions/actionDeleteSelected.tsx | 4 +- src/actions/actionDistribute.tsx | 3 +- src/actions/actionDuplicateSelection.tsx | 10 +- src/actions/actionElementLock.ts | 3 +- src/actions/actionFrame.ts | 25 +- src/actions/actionGroup.tsx | 8 +- src/actions/actionStyles.ts | 4 +- src/clipboard.ts | 7 +- src/components/Actions.tsx | 32 +- src/components/App.tsx | 720 ++++++++++++++++-- src/components/LayerUI.tsx | 36 +- src/components/MagicButton.tsx | 38 + src/components/MagicSettings.scss | 9 + src/components/MagicSettings.tsx | 145 ++++ src/components/PasteChartDialog.tsx | 2 +- src/components/PublishLibrary.tsx | 38 +- src/components/TextField.tsx | 19 +- src/components/ToolIcon.scss | 3 +- src/components/canvases/InteractiveCanvas.tsx | 2 - src/components/icons.tsx | 54 ++ src/constants.ts | 32 +- src/css/styles.scss | 2 + src/data/EditorLocalStorage.ts | 51 ++ src/data/ai/types.ts | 300 ++++++++ src/data/index.ts | 14 +- src/data/magic.ts | 104 +++ src/data/restore.ts | 15 +- src/data/transform.ts | 52 +- src/element/ElementCanvasButtons.scss | 14 + src/element/ElementCanvasButtons.tsx | 60 ++ src/element/Hyperlink.scss | 2 +- src/element/Hyperlink.tsx | 3 +- src/element/collision.ts | 37 +- src/element/dragElements.ts | 4 +- src/element/embeddable.ts | 94 ++- src/element/index.ts | 20 +- src/element/newElement.ts | 29 + src/element/resizeElements.ts | 6 +- src/element/textElement.ts | 3 +- src/element/transformHandles.ts | 4 +- src/element/typeChecks.ts | 64 +- src/element/types.ts | 38 +- src/frame.ts | 84 +- src/locales/en.json | 7 +- src/packages/excalidraw/index.tsx | 2 + src/packages/utils.ts | 4 +- src/renderer/renderElement.ts | 14 +- src/renderer/renderScene.ts | 28 +- src/scene/Scene.ts | 31 +- src/scene/Shape.ts | 30 +- src/scene/comparisons.ts | 31 +- src/scene/export.ts | 37 +- src/scene/selection.ts | 6 +- src/scene/types.ts | 2 + src/shapes.tsx | 8 - src/snapping.ts | 16 +- src/tests/helpers/api.ts | 17 +- src/tests/helpers/ui.ts | 11 +- src/tests/queries/toolQueries.ts | 24 +- src/types.ts | 36 +- src/utils.ts | 23 +- src/zindex.ts | 26 +- 63 files changed, 2106 insertions(+), 444 deletions(-) create mode 100644 src/components/MagicButton.tsx create mode 100644 src/components/MagicSettings.scss create mode 100644 src/components/MagicSettings.tsx create mode 100644 src/data/EditorLocalStorage.ts create mode 100644 src/data/ai/types.ts create mode 100644 src/data/magic.ts create mode 100644 src/element/ElementCanvasButtons.scss create mode 100644 src/element/ElementCanvasButtons.tsx diff --git a/src/actions/actionAlign.tsx b/src/actions/actionAlign.tsx index 5697a707..137f68ae 100644 --- a/src/actions/actionAlign.tsx +++ b/src/actions/actionAlign.tsx @@ -9,6 +9,7 @@ import { } from "../components/icons"; import { ToolButton } from "../components/ToolButton"; import { getNonDeletedElements } from "../element"; +import { isFrameLikeElement } from "../element/typeChecks"; import { ExcalidrawElement } from "../element/types"; import { updateFrameMembershipOfSelectedElements } from "../frame"; import { t } from "../i18n"; @@ -28,7 +29,7 @@ const alignActionsPredicate = ( return ( selectedElements.length > 1 && // TODO enable aligning frames when implemented properly - !selectedElements.some((el) => el.type === "frame") + !selectedElements.some((el) => isFrameLikeElement(el)) ); }; diff --git a/src/actions/actionDeleteSelected.tsx b/src/actions/actionDeleteSelected.tsx index 4d7ec6a7..de25ed89 100644 --- a/src/actions/actionDeleteSelected.tsx +++ b/src/actions/actionDeleteSelected.tsx @@ -10,7 +10,7 @@ import { newElementWith } from "../element/mutateElement"; import { getElementsInGroup } from "../groups"; import { LinearElementEditor } from "../element/linearElementEditor"; import { fixBindingsAfterDeletion } from "../element/binding"; -import { isBoundToContainer } from "../element/typeChecks"; +import { isBoundToContainer, isFrameLikeElement } from "../element/typeChecks"; import { updateActiveTool } from "../utils"; import { TrashIcon } from "../components/icons"; @@ -20,7 +20,7 @@ const deleteSelectedElements = ( ) => { const framesToBeDeleted = new Set( getSelectedElements( - elements.filter((el) => el.type === "frame"), + elements.filter((el) => isFrameLikeElement(el)), appState, ).map((el) => el.id), ); diff --git a/src/actions/actionDistribute.tsx b/src/actions/actionDistribute.tsx index d3cdb5c9..bf51bedf 100644 --- a/src/actions/actionDistribute.tsx +++ b/src/actions/actionDistribute.tsx @@ -5,6 +5,7 @@ import { import { ToolButton } from "../components/ToolButton"; import { distributeElements, Distribution } from "../distribute"; import { getNonDeletedElements } from "../element"; +import { isFrameLikeElement } from "../element/typeChecks"; import { ExcalidrawElement } from "../element/types"; import { updateFrameMembershipOfSelectedElements } from "../frame"; import { t } from "../i18n"; @@ -19,7 +20,7 @@ const enableActionGroup = (appState: AppState, app: AppClassProperties) => { return ( selectedElements.length > 1 && // TODO enable distributing frames when implemented properly - !selectedElements.some((el) => el.type === "frame") + !selectedElements.some((el) => isFrameLikeElement(el)) ); }; diff --git a/src/actions/actionDuplicateSelection.tsx b/src/actions/actionDuplicateSelection.tsx index 060a2868..ba079168 100644 --- a/src/actions/actionDuplicateSelection.tsx +++ b/src/actions/actionDuplicateSelection.tsx @@ -20,7 +20,7 @@ import { bindTextToShapeAfterDuplication, getBoundTextElement, } from "../element/textElement"; -import { isBoundToContainer, isFrameElement } from "../element/typeChecks"; +import { isBoundToContainer, isFrameLikeElement } from "../element/typeChecks"; import { normalizeElementOrder } from "../element/sortElements"; import { DuplicateIcon } from "../components/icons"; import { @@ -140,11 +140,11 @@ const duplicateElements = ( } const boundTextElement = getBoundTextElement(element); - const isElementAFrame = isFrameElement(element); + const isElementAFrameLike = isFrameLikeElement(element); if (idsOfElementsToDuplicate.get(element.id)) { // if a group or a container/bound-text or frame, duplicate atomically - if (element.groupIds.length || boundTextElement || isElementAFrame) { + if (element.groupIds.length || boundTextElement || isElementAFrameLike) { const groupId = getSelectedGroupForElement(appState, element); if (groupId) { // TODO: @@ -154,7 +154,7 @@ const duplicateElements = ( sortedElements, groupId, ).flatMap((element) => - isFrameElement(element) + isFrameLikeElement(element) ? [...getFrameChildren(elements, element.id), element] : [element], ); @@ -180,7 +180,7 @@ const duplicateElements = ( ); continue; } - if (isElementAFrame) { + if (isElementAFrameLike) { const elementsInFrame = getFrameChildren(sortedElements, element.id); elementsWithClones.push( diff --git a/src/actions/actionElementLock.ts b/src/actions/actionElementLock.ts index cd539c5a..164240b2 100644 --- a/src/actions/actionElementLock.ts +++ b/src/actions/actionElementLock.ts @@ -1,4 +1,5 @@ import { newElementWith } from "../element/mutateElement"; +import { isFrameLikeElement } from "../element/typeChecks"; import { ExcalidrawElement } from "../element/types"; import { KEYS } from "../keys"; import { arrayToMap } from "../utils"; @@ -51,7 +52,7 @@ export const actionToggleElementLock = register({ selectedElementIds: appState.selectedElementIds, includeBoundTextElement: false, }); - if (selected.length === 1 && selected[0].type !== "frame") { + if (selected.length === 1 && !isFrameLikeElement(selected[0])) { return selected[0].locked ? "labels.elementLock.unlock" : "labels.elementLock.lock"; diff --git a/src/actions/actionFrame.ts b/src/actions/actionFrame.ts index 9e8c16c2..4cddb2ac 100644 --- a/src/actions/actionFrame.ts +++ b/src/actions/actionFrame.ts @@ -7,23 +7,27 @@ import { AppClassProperties, AppState } from "../types"; import { updateActiveTool } from "../utils"; import { setCursorForShape } from "../cursor"; import { register } from "./register"; +import { isFrameLikeElement } from "../element/typeChecks"; const isSingleFrameSelected = (appState: AppState, app: AppClassProperties) => { const selectedElements = app.scene.getSelectedElements(appState); - return selectedElements.length === 1 && selectedElements[0].type === "frame"; + return ( + selectedElements.length === 1 && isFrameLikeElement(selectedElements[0]) + ); }; export const actionSelectAllElementsInFrame = register({ name: "selectAllElementsInFrame", trackEvent: { category: "canvas" }, perform: (elements, appState, _, app) => { - const selectedFrame = app.scene.getSelectedElements(appState)[0]; + const selectedElement = + app.scene.getSelectedElements(appState).at(0) || null; - if (selectedFrame && selectedFrame.type === "frame") { + if (isFrameLikeElement(selectedElement)) { const elementsInFrame = getFrameChildren( getNonDeletedElements(elements), - selectedFrame.id, + selectedElement.id, ).filter((element) => !(element.type === "text" && element.containerId)); return { @@ -54,15 +58,20 @@ export const actionRemoveAllElementsFromFrame = register({ name: "removeAllElementsFromFrame", trackEvent: { category: "history" }, perform: (elements, appState, _, app) => { - const selectedFrame = app.scene.getSelectedElements(appState)[0]; + const selectedElement = + app.scene.getSelectedElements(appState).at(0) || null; - if (selectedFrame && selectedFrame.type === "frame") { + if (isFrameLikeElement(selectedElement)) { return { - elements: removeAllElementsFromFrame(elements, selectedFrame, appState), + elements: removeAllElementsFromFrame( + elements, + selectedElement, + appState, + ), appState: { ...appState, selectedElementIds: { - [selectedFrame.id]: true, + [selectedElement.id]: true, }, }, commitToHistory: true, diff --git a/src/actions/actionGroup.tsx b/src/actions/actionGroup.tsx index 219f1444..e6cb0584 100644 --- a/src/actions/actionGroup.tsx +++ b/src/actions/actionGroup.tsx @@ -22,8 +22,8 @@ import { AppClassProperties, AppState } from "../types"; import { isBoundToContainer } from "../element/typeChecks"; import { getElementsInResizingFrame, - getFrameElements, - groupByFrames, + getFrameLikeElements, + groupByFrameLikes, removeElementsFromFrame, replaceAllElementsInFrame, } from "../frame"; @@ -102,7 +102,7 @@ export const actionGroup = register({ // when it happens, we want to remove elements that are in the frame // and are going to be grouped from the frame (mouthful, I know) if (groupingElementsFromDifferentFrames) { - const frameElementsMap = groupByFrames(selectedElements); + const frameElementsMap = groupByFrameLikes(selectedElements); frameElementsMap.forEach((elementsInFrame, frameId) => { nextElements = removeElementsFromFrame( @@ -219,7 +219,7 @@ export const actionUngroup = register({ .map((element) => element.frameId!), ); - const targetFrames = getFrameElements(elements).filter((frame) => + const targetFrames = getFrameLikeElements(elements).filter((frame) => selectedElementFrameIds.has(frame.id), ); diff --git a/src/actions/actionStyles.ts b/src/actions/actionStyles.ts index 2b656b05..9c6589bb 100644 --- a/src/actions/actionStyles.ts +++ b/src/actions/actionStyles.ts @@ -20,7 +20,7 @@ import { hasBoundTextElement, canApplyRoundnessTypeToElement, getDefaultRoundnessTypeForElement, - isFrameElement, + isFrameLikeElement, isArrowElement, } from "../element/typeChecks"; import { getSelectedElements } from "../scene"; @@ -138,7 +138,7 @@ export const actionPasteStyles = register({ }); } - if (isFrameElement(element)) { + if (isFrameLikeElement(element)) { newElement = newElementWith(newElement, { roundness: null, backgroundColor: "transparent", diff --git a/src/clipboard.ts b/src/clipboard.ts index 32b0edf1..a88402d6 100644 --- a/src/clipboard.ts +++ b/src/clipboard.ts @@ -9,7 +9,10 @@ import { EXPORT_DATA_TYPES, MIME_TYPES, } from "./constants"; -import { isInitializedImageElement } from "./element/typeChecks"; +import { + isFrameLikeElement, + isInitializedImageElement, +} from "./element/typeChecks"; import { deepCopyElement } from "./element/newElement"; import { mutateElement } from "./element/mutateElement"; import { getContainingFrame } from "./frame"; @@ -124,7 +127,7 @@ export const serializeAsClipboardJSON = ({ files: BinaryFiles | null; }) => { const framesToCopy = new Set( - elements.filter((element) => element.type === "frame"), + elements.filter((element) => isFrameLikeElement(element)), ); let foundFile = false; diff --git a/src/components/Actions.tsx b/src/components/Actions.tsx index 6d1d80b1..556dc4af 100644 --- a/src/components/Actions.tsx +++ b/src/components/Actions.tsx @@ -1,7 +1,7 @@ import React, { useState } from "react"; import { ActionManager } from "../actions/manager"; import { getNonDeletedElements } from "../element"; -import { ExcalidrawElement } from "../element/types"; +import { ExcalidrawElement, ExcalidrawElementType } from "../element/types"; import { t } from "../i18n"; import { useDevice } from "../components/App"; import { @@ -36,6 +36,8 @@ import { frameToolIcon, mermaidLogoIcon, laserPointerToolIcon, + OpenAIIcon, + MagicIcon, } from "./icons"; import { KEYS } from "../keys"; @@ -79,7 +81,8 @@ export const SelectedShapeActions = ({ const showLinkIcon = targetElements.length === 1 || isSingleElementBoundContainer; - let commonSelectedType: string | null = targetElements[0]?.type || null; + let commonSelectedType: ExcalidrawElementType | null = + targetElements[0]?.type || null; for (const element of targetElements) { if (element.type !== commonSelectedType) { @@ -94,7 +97,8 @@ export const SelectedShapeActions = ({ {((hasStrokeColor(appState.activeTool.type) && appState.activeTool.type !== "image" && commonSelectedType !== "image" && - commonSelectedType !== "frame") || + commonSelectedType !== "frame" && + commonSelectedType !== "magicframe") || targetElements.some((element) => hasStrokeColor(element.type))) && renderAction("changeStrokeColor")} @@ -331,6 +335,9 @@ export const ShapesSwitcher = ({ > {t("toolBar.laser")} +
+ Generate +
app.setOpenDialog("mermaid")} icon={mermaidLogoIcon} @@ -338,6 +345,25 @@ export const ShapesSwitcher = ({ > {t("toolBar.mermaidToExcalidraw")} + + {app.props.aiEnabled !== false && ( + <> + app.onMagicButtonSelect()} + icon={MagicIcon} + data-testid="toolbar-magicframe" + > + {t("toolBar.magicframe")} + + app.setOpenDialog("magicSettings")} + icon={OpenAIIcon} + data-testid="toolbar-magicSettings" + > + {t("toolBar.magicSettings")} + + + )} diff --git a/src/components/App.tsx b/src/components/App.tsx index 0892a40a..c579d8b1 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -47,12 +47,15 @@ import { isEraserActive, isHandToolActive, } from "../appState"; -import { PastedMixedContent, parseClipboard } from "../clipboard"; +import { + PastedMixedContent, + copyTextToSystemClipboard, + parseClipboard, +} from "../clipboard"; import { APP_NAME, CURSOR_TYPE, DEFAULT_MAX_IMAGE_WIDTH_OR_HEIGHT, - DEFAULT_UI_OPTIONS, DEFAULT_VERTICAL_ALIGN, DRAGGING_THRESHOLD, ELEMENT_READY_TO_ERASE_OPACITY, @@ -86,6 +89,8 @@ import { YOUTUBE_STATES, ZOOM_STEP, POINTER_EVENTS, + TOOL_TYPE, + EDITOR_LS_KEYS, } from "../constants"; import { ExportedElements, exportCanvas, loadFromBlob } from "../data"; import Library, { distributeLibraryItemsOnSquareGrid } from "../data/library"; @@ -139,6 +144,8 @@ import { newFrameElement, newFreeDrawElement, newEmbeddableElement, + newMagicFrameElement, + newIframeElement, } from "../element/newElement"; import { hasBoundTextElement, @@ -146,13 +153,17 @@ import { isBindingElement, isBindingElementType, isBoundToContainer, - isFrameElement, + isFrameLikeElement, isImageElement, isEmbeddableElement, isInitializedImageElement, isLinearElement, isLinearElementType, isUsingAdaptiveRadius, + isFrameElement, + isIframeElement, + isIframeLikeElement, + isMagicFrameElement, } from "../element/typeChecks"; import { ExcalidrawBindableElement, @@ -167,8 +178,11 @@ import { FileId, NonDeletedExcalidrawElement, ExcalidrawTextContainer, - ExcalidrawFrameElement, - ExcalidrawEmbeddableElement, + ExcalidrawFrameLikeElement, + ExcalidrawMagicFrameElement, + ExcalidrawIframeLikeElement, + IframeData, + ExcalidrawIframeElement, } from "../element/types"; import { getCenter, getDistance } from "../gesture"; import { @@ -256,6 +270,7 @@ import { easeOut, } from "../utils"; import { + createSrcDoc, embeddableURLValidator, extractSrc, getEmbedLink, @@ -328,6 +343,7 @@ import { elementOverlapsWithFrame, updateFrameMembershipOfSelectedElements, isElementInFrame, + getFrameLikeTitle, } from "../frame"; import { excludeElementsInFramesFromSelection, @@ -375,6 +391,13 @@ import { setCursorForShape, } from "../cursor"; import { Emitter } from "../emitter"; +import { ElementCanvasButtons } from "../element/ElementCanvasButtons"; +import { MagicCacheData, diagramToHTML } from "../data/magic"; +import { elementsOverlappingBBox, exportToBlob } from "../packages/utils"; +import { COLOR_PALETTE } from "../colors"; +import { ElementCanvasButton } from "./MagicButton"; +import { MagicIcon, copyIcon, fullscreenIcon } from "./icons"; +import { EditorLocalStorage } from "../data/EditorLocalStorage"; const AppContext = React.createContext(null!); const AppPropsContext = React.createContext(null!); @@ -481,11 +504,6 @@ class App extends React.Component { private excalidrawContainerRef = React.createRef(); - public static defaultProps: Partial = { - // needed for tests to pass since we directly render App in many tests - UIOptions: DEFAULT_UI_OPTIONS, - }; - public scene: Scene; public renderer: Renderer; private fonts: Fonts; @@ -692,22 +710,22 @@ class App extends React.Component { } } - private updateEmbeddableRef( - id: ExcalidrawEmbeddableElement["id"], + private cacheEmbeddableRef( + element: ExcalidrawIframeLikeElement, ref: HTMLIFrameElement | null, ) { if (ref) { - this.iFrameRefs.set(id, ref); + this.iFrameRefs.set(element.id, ref); } } private getHTMLIFrameElement( - id: ExcalidrawEmbeddableElement["id"], + element: ExcalidrawIframeLikeElement, ): HTMLIFrameElement | undefined { - return this.iFrameRefs.get(id); + return this.iFrameRefs.get(element.id); } - private handleEmbeddableCenterClick(element: ExcalidrawEmbeddableElement) { + private handleEmbeddableCenterClick(element: ExcalidrawIframeLikeElement) { if ( this.state.activeEmbeddable?.element === element && this.state.activeEmbeddable?.state === "active" @@ -730,7 +748,11 @@ class App extends React.Component { }); }, 100); - const iframe = this.getHTMLIFrameElement(element.id); + if (isIframeElement(element)) { + return; + } + + const iframe = this.getHTMLIFrameElement(element); if (!iframe?.contentWindow) { return; @@ -782,8 +804,8 @@ class App extends React.Component { } } - private isEmbeddableCenter( - el: ExcalidrawEmbeddableElement | null, + private isIframeLikeElementCenter( + el: ExcalidrawIframeLikeElement | null, event: React.PointerEvent | PointerEvent, sceneX: number, sceneY: number, @@ -805,12 +827,12 @@ class App extends React.Component { } private updateEmbeddables = () => { - const embeddableElements = new Map(); + const iframeLikes = new Set(); let updated = false; this.scene.getNonDeletedElements().filter((element) => { if (isEmbeddableElement(element)) { - embeddableElements.set(element.id, true); + iframeLikes.add(element.id); if (element.validated == null) { updated = true; @@ -822,6 +844,8 @@ class App extends React.Component { mutateElement(element, { validated }, false); ShapeCache.delete(element); } + } else if (isIframeElement(element)) { + iframeLikes.add(element.id); } return false; }); @@ -832,7 +856,7 @@ class App extends React.Component { // GC this.iFrameRefs.forEach((ref, id) => { - if (!embeddableElements.has(id)) { + if (!iframeLikes.has(id)) { this.iFrameRefs.delete(id); } }); @@ -846,8 +870,8 @@ class App extends React.Component { const embeddableElements = this.scene .getNonDeletedElements() .filter( - (el): el is NonDeleted => - isEmbeddableElement(el) && !!el.validated, + (el): el is NonDeleted => + (isEmbeddableElement(el) && !!el.validated) || isIframeElement(el), ); return ( @@ -857,7 +881,150 @@ class App extends React.Component { { sceneX: el.x, sceneY: el.y }, this.state, ); - const embedLink = getEmbedLink(toValidURL(el.link || "")); + + let src: IframeData | null; + + if (isIframeElement(el)) { + src = null; + + const data: MagicCacheData = (el.customData?.generationData ?? + this.magicGenerations.get(el.id)) || { + status: "error", + message: "No generation data", + code: "ERR_NO_GENERATION_DATA", + }; + + if (data.status === "done") { + const html = data.html; + src = { + intrinsicSize: { w: el.width, h: el.height }, + type: "document", + srcdoc: () => { + return html; + }, + } as const; + } else if (data.status === "pending") { + src = { + intrinsicSize: { w: el.width, h: el.height }, + type: "document", + srcdoc: () => { + return createSrcDoc(` + +
+ + + +
+
Generating...
+ `); + }, + } as const; + } else { + let message: string; + if (data.code === "ERR_GENERATION_INTERRUPTED") { + message = "Generation was interrupted..."; + } else { + message = data.message || "Generation failed"; + } + src = { + intrinsicSize: { w: el.width, h: el.height }, + type: "document", + srcdoc: () => { + return createSrcDoc(` + +

Error!

+

${message}

+ `); + }, + } as const; + } + } else { + src = getEmbedLink(toValidURL(el.link || "")); + } + + // console.log({ src }); + const isVisible = isElementInViewport( el, normalizedWidth, @@ -929,19 +1096,19 @@ class App extends React.Component { padding: `${el.strokeWidth}px`, }} > - {this.props.renderEmbeddable?.(el, this.state) ?? ( + {(isEmbeddableElement(el) + ? this.props.renderEmbeddable?.(el, this.state) + : null) ?? (