diff --git a/excalidraw-app/App.tsx b/excalidraw-app/App.tsx index a11ea59b32..69d6211293 100644 --- a/excalidraw-app/App.tsx +++ b/excalidraw-app/App.tsx @@ -22,7 +22,6 @@ import { t } from "../packages/excalidraw/i18n"; import { Excalidraw, LiveCollaborationTrigger, - TTDDialog, TTDDialogTrigger, StoreAction, reconcileElements, @@ -121,6 +120,7 @@ import { import { appThemeAtom, useHandleAppTheme } from "./useHandleAppTheme"; import { getPreferredLanguage } from "./app-language/language-detector"; import { useAppLangCode } from "./app-language/language-state"; +import { AIComponents } from "./components/AI"; polyfill(); @@ -846,63 +846,8 @@ const ExcalidrawWrapper = () => { )} - { - try { - const response = await fetch( - `${ - import.meta.env.VITE_APP_AI_BACKEND - }/v1/ai/text-to-diagram/generate`, - { - method: "POST", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - }, - body: JSON.stringify({ prompt: input }), - }, - ); - - const rateLimit = response.headers.has("X-Ratelimit-Limit") - ? parseInt(response.headers.get("X-Ratelimit-Limit") || "0", 10) - : undefined; - - const rateLimitRemaining = response.headers.has( - "X-Ratelimit-Remaining", - ) - ? parseInt( - response.headers.get("X-Ratelimit-Remaining") || "0", - 10, - ) - : undefined; - - const json = await response.json(); - - if (!response.ok) { - if (response.status === 429) { - return { - rateLimit, - rateLimitRemaining, - error: new Error( - "Too many requests today, please try again tomorrow!", - ), - }; - } + {excalidrawAPI && } - throw new Error(json.message || "Generation failed..."); - } - - const generatedResponse = json.generatedResponse; - if (!generatedResponse) { - throw new Error("Generation failed..."); - } - - return { generatedResponse, rateLimit, rateLimitRemaining }; - } catch (err: any) { - throw new Error("Request failed"); - } - }} - /> {isCollaborating && isOffline && (
diff --git a/excalidraw-app/components/AI.tsx b/excalidraw-app/components/AI.tsx new file mode 100644 index 0000000000..f9d3b15513 --- /dev/null +++ b/excalidraw-app/components/AI.tsx @@ -0,0 +1,152 @@ +import type { ExcalidrawImperativeAPI } from "../../packages/excalidraw/types"; +import { + DiagramToCodePlugin, + exportToBlob, + getTextFromElements, + MIME_TYPES, + TTDDialog, +} from "../../packages/excalidraw"; +import { getDataURL } from "../../packages/excalidraw/data/blob"; +import { safelyParseJSON } from "../../packages/excalidraw/utils"; + +export const AIComponents = ({ + excalidrawAPI, +}: { + excalidrawAPI: ExcalidrawImperativeAPI; +}) => { + return ( + <> + { + const appState = excalidrawAPI.getAppState(); + + const blob = await exportToBlob({ + elements: children, + appState: { + ...appState, + exportBackground: true, + viewBackgroundColor: appState.viewBackgroundColor, + }, + exportingFrame: frame, + files: excalidrawAPI.getFiles(), + mimeType: MIME_TYPES.jpg, + }); + + const dataURL = await getDataURL(blob); + + const textFromFrameChildren = getTextFromElements(children); + + const response = await fetch( + `${ + import.meta.env.VITE_APP_AI_BACKEND + }/v1/ai/diagram-to-code/generate`, + { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + texts: textFromFrameChildren, + image: dataURL, + theme: appState.theme, + }), + }, + ); + + if (!response.ok) { + const text = await response.text(); + const error = safelyParseJSON(text); + + if (!error) { + throw new Error(text); + } + + if (error.statusCode === 429) { + return { + html: ` + +
+
Too many requests today,
please try again tomorrow!
+
+
+
You can also try Excalidraw+ to get more requests.
+
+ + `, + }; + } + + throw new Error(error.message || text); + } + + const html = await response.text(); + + return { + html, + }; + }} + /> + + { + try { + const response = await fetch( + `${ + import.meta.env.VITE_APP_AI_BACKEND + }/v1/ai/text-to-diagram/generate`, + { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + }, + body: JSON.stringify({ prompt: input }), + }, + ); + + const rateLimit = response.headers.has("X-Ratelimit-Limit") + ? parseInt(response.headers.get("X-Ratelimit-Limit") || "0", 10) + : undefined; + + const rateLimitRemaining = response.headers.has( + "X-Ratelimit-Remaining", + ) + ? parseInt( + response.headers.get("X-Ratelimit-Remaining") || "0", + 10, + ) + : undefined; + + const json = await response.json(); + + if (!response.ok) { + if (response.status === 429) { + return { + rateLimit, + rateLimitRemaining, + error: new Error( + "Too many requests today, please try again tomorrow!", + ), + }; + } + + throw new Error(json.message || "Generation failed..."); + } + + const generatedResponse = json.generatedResponse; + if (!generatedResponse) { + throw new Error("Generation failed..."); + } + + return { generatedResponse, rateLimit, rateLimitRemaining }; + } catch (err: any) { + throw new Error("Request failed"); + } + }} + /> + + ); +}; diff --git a/packages/excalidraw/actions/actionClipboard.tsx b/packages/excalidraw/actions/actionClipboard.tsx index e4f998d016..4d34bf968c 100644 --- a/packages/excalidraw/actions/actionClipboard.tsx +++ b/packages/excalidraw/actions/actionClipboard.tsx @@ -10,7 +10,7 @@ import { } from "../clipboard"; import { actionDeleteSelected } from "./actionDeleteSelected"; import { exportCanvas, prepareElementsForExport } from "../data/index"; -import { isTextElement } from "../element"; +import { getTextFromElements, isTextElement } from "../element"; import { t } from "../i18n"; import { isFirefox } from "../constants"; import { DuplicateIcon, cutIcon, pngIcon, svgIcon } from "../components/icons"; @@ -239,16 +239,8 @@ export const copyText = register({ includeBoundTextElement: true, }); - const text = selectedElements - .reduce((acc: string[], element) => { - if (isTextElement(element)) { - acc.push(element.text); - } - return acc; - }, []) - .join("\n\n"); try { - copyTextToSystemClipboard(text); + copyTextToSystemClipboard(getTextFromElements(selectedElements)); } catch (e) { throw new Error(t("errors.copyToSystemClipboardFailed")); } diff --git a/packages/excalidraw/components/Actions.tsx b/packages/excalidraw/components/Actions.tsx index c49b4a5f0f..774f19038e 100644 --- a/packages/excalidraw/components/Actions.tsx +++ b/packages/excalidraw/components/Actions.tsx @@ -44,7 +44,6 @@ import { frameToolIcon, mermaidLogoIcon, laserPointerToolIcon, - OpenAIIcon, MagicIcon, } from "./icons"; import { KEYS } from "../keys"; @@ -395,7 +394,7 @@ export const ShapesSwitcher = ({ > {t("toolBar.mermaidToExcalidraw")} - {app.props.aiEnabled !== false && ( + {app.props.aiEnabled !== false && app.plugins.diagramToCode && ( <> app.onMagicframeToolSelect()} @@ -405,20 +404,6 @@ export const ShapesSwitcher = ({ {t("toolBar.magicframe")} AI - { - trackEvent("ai", "open-settings", "d2c"); - app.setOpenDialog({ - name: "settings", - source: "settings", - tab: "diagram-to-code", - }); - }} - icon={OpenAIIcon} - data-testid="toolbar-magicSettings" - > - {t("toolBar.magicSettings")} - )} diff --git a/packages/excalidraw/components/App.tsx b/packages/excalidraw/components/App.tsx index 093f6bfdfe..2aa299bc24 100644 --- a/packages/excalidraw/components/App.tsx +++ b/packages/excalidraw/components/App.tsx @@ -85,7 +85,6 @@ import { ZOOM_STEP, POINTER_EVENTS, TOOL_TYPE, - EDITOR_LS_KEYS, isIOS, supportsResizeObserver, DEFAULT_COLLISION_THRESHOLD, @@ -183,6 +182,7 @@ import type { ExcalidrawIframeElement, ExcalidrawEmbeddableElement, Ordered, + MagicGenerationData, } from "../element/types"; import { getCenter, getDistance } from "../gesture"; import { @@ -253,6 +253,7 @@ import type { UnsubscribeCallback, EmbedsValidationStatus, ElementsPendingErasure, + GenerateDiagramToCode, } from "../types"; import { debounce, @@ -399,13 +400,9 @@ import { } from "../cursor"; import { Emitter } from "../emitter"; import { ElementCanvasButtons } from "../element/ElementCanvasButtons"; -import type { MagicCacheData } from "../data/magic"; -import { diagramToHTML } from "../data/magic"; -import { exportToBlob } from "../../utils/export"; import { COLOR_PALETTE } from "../colors"; import { ElementCanvasButton } from "./MagicButton"; import { MagicIcon, copyIcon, fullscreenIcon } from "./icons"; -import { EditorLocalStorage } from "../data/EditorLocalStorage"; import FollowMode from "./FollowMode/FollowMode"; import { Store, StoreAction } from "../store"; import { AnimationFrameHandler } from "../animation-frame-handler"; @@ -993,7 +990,7 @@ class App extends React.Component { if (isIframeElement(el)) { src = null; - const data: MagicCacheData = (el.customData?.generationData ?? + const data: MagicGenerationData = (el.customData?.generationData ?? this.magicGenerations.get(el.id)) || { status: "error", message: "No generation data", @@ -1543,10 +1540,6 @@ class App extends React.Component { } app={this} isCollaborating={this.props.isCollaborating} - openAIKey={this.OPENAI_KEY} - isOpenAIKeyPersisted={this.OPENAI_KEY_IS_PERSISTED} - onOpenAIAPIKeyChange={this.onOpenAIKeyChange} - onMagicSettingsConfirm={this.onMagicSettingsConfirm} > {this.props.children} @@ -1789,7 +1782,7 @@ class App extends React.Component { private magicGenerations = new Map< ExcalidrawIframeElement["id"], - MagicCacheData + MagicGenerationData >(); private updateMagicGeneration = ({ @@ -1797,7 +1790,7 @@ class App extends React.Component { data, }: { frameElement: ExcalidrawIframeElement; - data: MagicCacheData; + data: MagicGenerationData; }) => { if (data.status === "pending") { // We don't wanna persist pending state to storage. It should be in-app @@ -1820,31 +1813,26 @@ class App extends React.Component { this.triggerRender(); }; - private getTextFromElements(elements: readonly ExcalidrawElement[]) { - const text = elements - .reduce((acc: string[], element) => { - if (isTextElement(element)) { - acc.push(element.text); - } - return acc; - }, []) - .join("\n\n"); - return text; + public plugins: { + diagramToCode?: { + generate: GenerateDiagramToCode; + }; + } = {}; + + public setPlugins(plugins: Partial) { + Object.assign(this.plugins, plugins); } private async onMagicFrameGenerate( magicFrame: ExcalidrawMagicFrameElement, source: "button" | "upstream", ) { - if (!this.OPENAI_KEY) { + const generateDiagramToCode = this.plugins.diagramToCode?.generate; + + if (!generateDiagramToCode) { this.setState({ - openDialog: { - name: "settings", - tab: "diagram-to-code", - source: "generation", - }, + errorMessage: "No diagram to code plugin found", }); - trackEvent("ai", "generate (missing key)", "d2c"); return; } @@ -1883,68 +1871,50 @@ class App extends React.Component { selectedElementIds: { [frameElement.id]: true }, }); - const blob = await exportToBlob({ - elements: this.scene.getNonDeletedElements(), - appState: { - ...this.state, - exportBackground: true, - viewBackgroundColor: this.state.viewBackgroundColor, - }, - exportingFrame: magicFrame, - files: this.files, - }); - - const dataURL = await getDataURL(blob); + trackEvent("ai", "generate (start)", "d2c"); + try { + const { html } = await generateDiagramToCode({ + frame: magicFrame, + children: magicFrameChildren, + }); - const textFromFrameChildren = this.getTextFromElements(magicFrameChildren); + trackEvent("ai", "generate (success)", "d2c"); - trackEvent("ai", "generate (start)", "d2c"); + if (!html.trim()) { + this.updateMagicGeneration({ + frameElement, + data: { + status: "error", + code: "ERR_OAI", + message: "Nothing genereated :(", + }, + }); + return; + } - const result = await diagramToHTML({ - image: dataURL, - apiKey: this.OPENAI_KEY, - text: textFromFrameChildren, - theme: this.state.theme, - }); + const parsedHtml = + html.includes("") && html.includes("") + ? html.slice( + html.indexOf(""), + html.indexOf("") + "".length, + ) + : html; - if (!result.ok) { - trackEvent("ai", "generate (failed)", "d2c"); - console.error(result.error); this.updateMagicGeneration({ frameElement, - data: { - status: "error", - code: "ERR_OAI", - message: result.error?.message || "Unknown error during generation", - }, + data: { status: "done", html: parsedHtml }, }); - return; - } - trackEvent("ai", "generate (success)", "d2c"); - - if (result.choices[0].message.content == null) { + } catch (error: any) { + trackEvent("ai", "generate (failed)", "d2c"); this.updateMagicGeneration({ frameElement, data: { status: "error", code: "ERR_OAI", - message: "Nothing genereated :(", + message: error.message || "Unknown error during generation", }, }); - return; } - - const message = result.choices[0].message.content; - - const html = message.slice( - message.indexOf(""), - message.indexOf("") + "".length, - ); - - this.updateMagicGeneration({ - frameElement, - data: { status: "done", html }, - }); } private onIframeSrcCopy(element: ExcalidrawIframeElement) { @@ -1958,70 +1928,7 @@ class App extends React.Component { } } - private OPENAI_KEY: string | null = EditorLocalStorage.get( - EDITOR_LS_KEYS.OAI_API_KEY, - ); - private OPENAI_KEY_IS_PERSISTED: boolean = - EditorLocalStorage.has(EDITOR_LS_KEYS.OAI_API_KEY) || false; - - private onOpenAIKeyChange = ( - openAIKey: string | null, - shouldPersist: boolean, - ) => { - this.OPENAI_KEY = openAIKey || null; - if (shouldPersist) { - const didPersist = EditorLocalStorage.set( - EDITOR_LS_KEYS.OAI_API_KEY, - openAIKey, - ); - this.OPENAI_KEY_IS_PERSISTED = didPersist; - } else { - this.OPENAI_KEY_IS_PERSISTED = false; - } - }; - - private onMagicSettingsConfirm = ( - apiKey: string, - shouldPersist: boolean, - source: "tool" | "generation" | "settings", - ) => { - this.OPENAI_KEY = apiKey || null; - this.onOpenAIKeyChange(this.OPENAI_KEY, shouldPersist); - - if (source === "settings") { - return; - } - - const selectedElements = this.scene.getSelectedElements({ - selectedElementIds: this.state.selectedElementIds, - }); - - if (apiKey) { - if (selectedElements.length) { - this.onMagicframeToolSelect(); - } else { - this.setActiveTool({ type: "magicframe" }); - } - } else if (!isMagicFrameElement(selectedElements[0])) { - // even if user didn't end up setting api key, let's pick the tool - // so they can draw up a frame and move forward - this.setActiveTool({ type: "magicframe" }); - } - }; - public onMagicframeToolSelect = () => { - if (!this.OPENAI_KEY) { - this.setState({ - openDialog: { - name: "settings", - tab: "diagram-to-code", - source: "tool", - }, - }); - trackEvent("ai", "tool-select (missing key)", "d2c"); - return; - } - const selectedElements = this.scene.getSelectedElements({ selectedElementIds: this.state.selectedElementIds, }); diff --git a/packages/excalidraw/components/DiagramToCodePlugin/DiagramToCodePlugin.tsx b/packages/excalidraw/components/DiagramToCodePlugin/DiagramToCodePlugin.tsx new file mode 100644 index 0000000000..9505999635 --- /dev/null +++ b/packages/excalidraw/components/DiagramToCodePlugin/DiagramToCodePlugin.tsx @@ -0,0 +1,17 @@ +import { useLayoutEffect } from "react"; +import { useApp } from "../App"; +import type { GenerateDiagramToCode } from "../../types"; + +export const DiagramToCodePlugin = (props: { + generate: GenerateDiagramToCode; +}) => { + const app = useApp(); + + useLayoutEffect(() => { + app.setPlugins({ + diagramToCode: { generate: props.generate }, + }); + }, [app, props.generate]); + + return null; +}; diff --git a/packages/excalidraw/components/LayerUI.tsx b/packages/excalidraw/components/LayerUI.tsx index dd3270670b..471f554aec 100644 --- a/packages/excalidraw/components/LayerUI.tsx +++ b/packages/excalidraw/components/LayerUI.tsx @@ -60,7 +60,6 @@ import { mutateElement } from "../element/mutateElement"; import { ShapeCache } from "../scene/ShapeCache"; import Scene from "../scene/Scene"; import { LaserPointerButton } from "./LaserPointerButton"; -import { MagicSettings } from "./MagicSettings"; import { TTDDialog } from "./TTDDialog/TTDDialog"; import { Stats } from "./Stats"; import { actionToggleStats } from "../actions"; @@ -85,14 +84,6 @@ interface LayerUIProps { children?: React.ReactNode; app: AppClassProperties; isCollaborating: boolean; - openAIKey: string | null; - isOpenAIKeyPersisted: boolean; - onOpenAIAPIKeyChange: (apiKey: string, shouldPersist: boolean) => void; - onMagicSettingsConfirm: ( - apiKey: string, - shouldPersist: boolean, - source: "tool" | "generation" | "settings", - ) => void; } const DefaultMainMenu: React.FC<{ @@ -149,10 +140,6 @@ const LayerUI = ({ children, app, isCollaborating, - openAIKey, - isOpenAIKeyPersisted, - onOpenAIAPIKeyChange, - onMagicSettingsConfirm, }: LayerUIProps) => { const device = useDevice(); const tunnels = useInitializeTunnels(); @@ -482,25 +469,6 @@ const LayerUI = ({ }} /> )} - {appState.openDialog?.name === "settings" && ( - { - const source = - appState.openDialog?.name === "settings" - ? appState.openDialog?.source - : "settings"; - setAppState({ openDialog: null }, () => { - onMagicSettingsConfirm(apiKey, shouldPersist, source); - }); - }} - onClose={() => { - setAppState({ openDialog: null }); - }} - /> - )} {renderImageExportDialog()} diff --git a/packages/excalidraw/components/MagicSettings.scss b/packages/excalidraw/components/MagicSettings.scss deleted file mode 100644 index bd07d84003..0000000000 --- a/packages/excalidraw/components/MagicSettings.scss +++ /dev/null @@ -1,18 +0,0 @@ -.excalidraw { - .MagicSettings { - .Island { - height: 100%; - display: flex; - flex-direction: column; - } - } - - .MagicSettings-confirm { - padding: 0.5rem 1rem; - } - - .MagicSettings__confirm { - margin-top: 2rem; - margin-right: auto; - } -} diff --git a/packages/excalidraw/components/MagicSettings.tsx b/packages/excalidraw/components/MagicSettings.tsx deleted file mode 100644 index 855ab109d7..0000000000 --- a/packages/excalidraw/components/MagicSettings.tsx +++ /dev/null @@ -1,160 +0,0 @@ -import { useState } from "react"; -import { Dialog } from "./Dialog"; -import { TextField } from "./TextField"; -import { MagicIcon, OpenAIIcon } from "./icons"; -import { FilledButton } from "./FilledButton"; -import { CheckboxItem } from "./CheckboxItem"; -import { KEYS } from "../keys"; -import { useUIAppState } from "../context/ui-appState"; -import { InlineIcon } from "./InlineIcon"; -import { Paragraph } from "./Paragraph"; - -import "./MagicSettings.scss"; -import TTDDialogTabs from "./TTDDialog/TTDDialogTabs"; -import { TTDDialogTab } from "./TTDDialog/TTDDialogTab"; - -export const MagicSettings = (props: { - openAIKey: string | null; - isPersisted: boolean; - onChange: (key: string, shouldPersist: boolean) => void; - onConfirm: (key: string, shouldPersist: boolean) => void; - onClose: () => void; -}) => { - const [keyInputValue, setKeyInputValue] = useState(props.openAIKey || ""); - const [shouldPersist, setShouldPersist] = useState( - props.isPersisted, - ); - - const appState = useUIAppState(); - - const onConfirm = () => { - props.onConfirm(keyInputValue.trim(), shouldPersist); - }; - - if (appState.openDialog?.name !== "settings") { - return null; - } - - return ( - { - props.onClose(); - props.onConfirm(keyInputValue.trim(), shouldPersist); - }} - title={ -
- Wireframe to Code (AI){" "} -
- Experimental -
-
- } - className="MagicSettings" - autofocus={false} - > - {/*

- AI Settings -

*/} - - {/* - - Text to diagram - - - Wireframe to code - - */} - {/* - TODO - */} - - - For the diagram-to-code feature we use{" "} - - OpenAI. - - - While the OpenAI API is in beta, its use is strictly limited — as - such we require you use your own API key. You can create an{" "} - - OpenAI account - - , add a small credit (5 USD minimum), and{" "} - - generate your own API key - - . - - - Your OpenAI key does not leave the browser, and you can also set - your own limit in your OpenAI account dashboard if needed. - - { - setKeyInputValue(value); - props.onChange(value.trim(), shouldPersist); - }} - selectOnRender - onKeyDown={(event) => event.key === KEYS.ENTER && onConfirm()} - /> - - By default, your API token is not persisted anywhere so you'll need - to insert it again after reload. But, you can persist locally in - your browser below. - - - - Persist API key in browser storage - - - - Once API key is set, you can use the {" "} - tool to wrap your elements in a frame that will then allow you to - turn it into code. This dialog can be accessed using the{" "} - AI Settings . - - - - - -
- ); -}; diff --git a/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx b/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx index 30add91e59..439f92844d 100644 --- a/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx +++ b/packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx @@ -7,10 +7,7 @@ import { isMemberOf } from "../../utils"; const TTDDialogTabs = ( props: { children: ReactNode; - } & ( - | { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" } - | { dialog: "settings"; tab: "text-to-diagram" | "diagram-to-code" } - ), + } & { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" }, ) => { const setAppState = useExcalidrawSetAppState(); @@ -39,13 +36,6 @@ const TTDDialogTabs = ( } } if ( - props.dialog === "settings" && - isMemberOf(["text-to-diagram", "diagram-to-code"], tab) - ) { - setAppState({ - openDialog: { name: props.dialog, tab, source: "settings" }, - }); - } else if ( props.dialog === "ttd" && isMemberOf(["text-to-diagram", "mermaid"], tab) ) { diff --git a/packages/excalidraw/data/magic.ts b/packages/excalidraw/data/magic.ts deleted file mode 100644 index 883a3bdb90..0000000000 --- a/packages/excalidraw/data/magic.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { THEME } from "../constants"; -import type { Theme } from "../element/types"; -import type { DataURL } from "../types"; -import type { OpenAIInput, OpenAIOutput } from "./ai/types"; - -export type MagicCacheData = - | { - status: "pending"; - } - | { status: "done"; html: string } - | { - status: "error"; - message?: string; - code: "ERR_GENERATION_INTERRUPTED" | string; - }; - -const SYSTEM_PROMPT = `You are a skilled front-end developer who builds interactive prototypes from wireframes, and is an expert at CSS Grid and Flex design. -Your role is to transform low-fidelity wireframes into working front-end HTML code. - -YOU MUST FOLLOW FOLLOWING RULES: - -- Use HTML, CSS, JavaScript to build a responsive, accessible, polished prototype -- Leverage Tailwind for styling and layout (import as script ) -- Inline JavaScript when needed -- Fetch dependencies from CDNs when needed (using unpkg or skypack) -- Source images from Unsplash or create applicable placeholders -- Interpret annotations as intended vs literal UI -- Fill gaps using your expertise in UX and business logic -- generate primarily for desktop UI, but make it responsive. -- Use grid and flexbox wherever applicable. -- Convert the wireframe in its entirety, don't omit elements if possible. - -If the wireframes, diagrams, or text is unclear or unreadable, refer to provided text for clarification. - -Your goal is a production-ready prototype that brings the wireframes to life. - -Please output JUST THE HTML file containing your best attempt at implementing the provided wireframes.`; - -export async function diagramToHTML({ - image, - apiKey, - text, - theme = THEME.LIGHT, -}: { - image: DataURL; - apiKey: string; - text: string; - theme?: Theme; -}) { - const body: OpenAIInput.ChatCompletionCreateParamsBase = { - model: "gpt-4-vision-preview", - // 4096 are max output tokens allowed for `gpt-4-vision-preview` currently - max_tokens: 4096, - temperature: 0.1, - messages: [ - { - role: "system", - content: SYSTEM_PROMPT, - }, - { - role: "user", - content: [ - { - type: "image_url", - image_url: { - url: image, - detail: "high", - }, - }, - { - type: "text", - text: `Above is the reference wireframe. Please make a new website based on these and return just the HTML file. Also, please make it for the ${theme} theme. What follows are the wireframe's text annotations (if any)...`, - }, - { - type: "text", - text, - }, - ], - }, - ], - }; - - let result: - | ({ ok: true } & OpenAIOutput.ChatCompletion) - | ({ ok: false } & OpenAIOutput.APIError); - - const resp = await fetch("https://api.openai.com/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify(body), - }); - - if (resp.ok) { - const json: OpenAIOutput.ChatCompletion = await resp.json(); - result = { ...json, ok: true }; - } else { - const json: OpenAIOutput.APIError = await resp.json(); - result = { ...json, ok: false }; - } - - return result; -} diff --git a/packages/excalidraw/element/index.ts b/packages/excalidraw/element/index.ts index 35661608e4..b2f2ec381b 100644 --- a/packages/excalidraw/element/index.ts +++ b/packages/excalidraw/element/index.ts @@ -45,7 +45,7 @@ export { dragNewElement, } from "./dragElements"; export { isTextElement, isExcalidrawElement } from "./typeChecks"; -export { redrawTextBoundingBox } from "./textElement"; +export { redrawTextBoundingBox, getTextFromElements } from "./textElement"; export { getPerfectElementSize, getLockedLinearCursorAlignSize, diff --git a/packages/excalidraw/element/textElement.ts b/packages/excalidraw/element/textElement.ts index db4230e241..c95f636ede 100644 --- a/packages/excalidraw/element/textElement.ts +++ b/packages/excalidraw/element/textElement.ts @@ -945,3 +945,19 @@ export const getMinTextElementWidth = ( ) => { return measureText("", font, lineHeight).width + BOUND_TEXT_PADDING * 2; }; + +/** retrieves text from text elements and concatenates to a single string */ +export const getTextFromElements = ( + elements: readonly ExcalidrawElement[], + separator = "\n\n", +) => { + const text = elements + .reduce((acc: string[], element) => { + if (isTextElement(element)) { + acc.push(element.text); + } + return acc; + }, []) + .join(separator); + return text; +}; diff --git a/packages/excalidraw/element/types.ts b/packages/excalidraw/element/types.ts index 700b7ed6c4..7295dd0a89 100644 --- a/packages/excalidraw/element/types.ts +++ b/packages/excalidraw/element/types.ts @@ -7,7 +7,6 @@ import type { VERTICAL_ALIGN, } from "../constants"; import type { MakeBrand, MarkNonNullable, ValueOf } from "../utility-types"; -import type { MagicCacheData } from "../data/magic"; export type ChartType = "bar" | "line"; export type FillStyle = "hachure" | "cross-hatch" | "solid" | "zigzag"; @@ -96,11 +95,22 @@ export type ExcalidrawEmbeddableElement = _ExcalidrawElementBase & type: "embeddable"; }>; +export type MagicGenerationData = + | { + status: "pending"; + } + | { status: "done"; html: string } + | { + status: "error"; + message?: string; + code: "ERR_GENERATION_INTERRUPTED" | string; + }; + export type ExcalidrawIframeElement = _ExcalidrawElementBase & Readonly<{ type: "iframe"; // TODO move later to AI-specific frame - customData?: { generationData?: MagicCacheData }; + customData?: { generationData?: MagicGenerationData }; }>; export type ExcalidrawIframeLikeElement = diff --git a/packages/excalidraw/frame.ts b/packages/excalidraw/frame.ts index f02ac521d4..cac2328000 100644 --- a/packages/excalidraw/frame.ts +++ b/packages/excalidraw/frame.ts @@ -763,7 +763,7 @@ export const getFrameLikeTitle = ( return element.name === null ? isFrameElement(element) ? `Frame ${frameIdx}` - : `AI Frame $${frameIdx}` + : `AI Frame ${frameIdx}` : element.name; }; diff --git a/packages/excalidraw/index.tsx b/packages/excalidraw/index.tsx index 98dd9d8ebc..3dadb120d4 100644 --- a/packages/excalidraw/index.tsx +++ b/packages/excalidraw/index.tsx @@ -211,6 +211,7 @@ export { hashString, isInvisiblySmallElement, getNonDeletedElements, + getTextFromElements, } from "./element"; export { defaultLang, useI18n, languages } from "./i18n"; export { @@ -284,3 +285,5 @@ export { isElementInsideBBox, elementPartiallyOverlapsWithOrContainsBBox, } from "../utils/withinBounds"; + +export { DiagramToCodePlugin } from "./components/DiagramToCodePlugin/DiagramToCodePlugin"; diff --git a/packages/excalidraw/locales/en.json b/packages/excalidraw/locales/en.json index 345c63ad16..e46d2ad9ca 100644 --- a/packages/excalidraw/locales/en.json +++ b/packages/excalidraw/locales/en.json @@ -267,8 +267,7 @@ "laser": "Laser pointer", "hand": "Hand (panning tool)", "extraTools": "More tools", - "mermaidToExcalidraw": "Mermaid to Excalidraw", - "magicSettings": "AI settings" + "mermaidToExcalidraw": "Mermaid to Excalidraw" }, "element": { "rectangle": "Rectangle", diff --git a/packages/excalidraw/types.ts b/packages/excalidraw/types.ts index 5c2e188514..597acaa9ce 100644 --- a/packages/excalidraw/types.ts +++ b/packages/excalidraw/types.ts @@ -294,14 +294,6 @@ export interface AppState { openDialog: | null | { name: "imageExport" | "help" | "jsonExport" } - | { - name: "settings"; - source: - | "tool" // when magicframe tool is selected - | "generation" // when magicframe generate button is clicked - | "settings"; // when AI settings dialog is explicitly invoked - tab: "text-to-diagram" | "diagram-to-code"; - } | { name: "ttd"; tab: "text-to-diagram" | "mermaid" } | { name: "commandPalette" }; /** @@ -615,6 +607,8 @@ export type AppClassProperties = { insertEmbeddableElement: App["insertEmbeddableElement"]; onMagicframeToolSelect: App["onMagicframeToolSelect"]; getName: App["getName"]; + setPlugins: App["setPlugins"]; + plugins: App["plugins"]; }; export type PointerDownState = Readonly<{ @@ -795,3 +789,8 @@ export type EmbedsValidationStatus = Map< >; export type ElementsPendingErasure = Set; + +export type GenerateDiagramToCode = (props: { + frame: ExcalidrawMagicFrameElement; + children: readonly ExcalidrawElement[]; +}) => MaybePromise<{ html: string }>; diff --git a/packages/excalidraw/utils.ts b/packages/excalidraw/utils.ts index d723a59666..8df5113530 100644 --- a/packages/excalidraw/utils.ts +++ b/packages/excalidraw/utils.ts @@ -1124,3 +1124,11 @@ export const promiseTry = async ( resolve(fn(...args)); }); }; + +export const safelyParseJSON = (json: string): Record | null => { + try { + return JSON.parse(json); + } catch { + return null; + } +};