import React, { useState, useEffect, useRef } from "react"; import { Monitor, MousePointer, Play, Pause, RotateCcw, CheckCircle2, AlertCircle, Eye, Terminal, Sliders, Layers, Crosshair, Sparkles, X, Maximize2, Copy, Download, Square, CheckSquare, ChevronRight, Activity, Video, VideoOff, Cpu, CornerDownLeft, ShieldAlert } from "lucide-react"; interface ComputerUseStudioProps { isOpen: boolean; onClose: () => void; onSendToChat?: (text: string) => void; } interface ActionStep { id: string; stepNumber: number; type: "VISION_SCAN" | "MOVE_CURSOR" | "CLICK" | "TYPE_TEXT" | "KEY_PRESS" | "SCROLL" | "WAIT" | "COMPLETE"; description: string; targetElement?: string; coords?: { x: number; y: number }; textInput?: string; status: "pending" | "running" | "completed" | "failed"; timestamp: string; screenshotUrl?: string; } interface DetectedElement { id: string; label: string; type: "button" | "input" | "menu" | "link" | "image"; coords: { x: number; y: number; w: number; h: number }; } export default function ComputerUseStudio({ isOpen, onClose, onSendToChat }: ComputerUseStudioProps) { const [selectedPreset, setSelectedPreset] = useState<"checkout" | "vscode" | "dashboard" | "custom">("checkout"); const [taskPrompt, setTaskPrompt] = useState("Navigate to customer profile, update email to 'alex.dev@aurastudio.ai', and click Save Settings."); const [isRunning, setIsRunning] = useState(false); const [currentStepIndex, setCurrentStepIndex] = useState(-1); const [cursorPos, setCursorPos] = useState<{ x: number; y: number }>({ x: 50, y: 50 }); const [showClickRipple, setShowClickRipple] = useState(false); const [speed, setSpeed] = useState(1); // 1x speed const [autoApproveSteps, setAutoApproveSteps] = useState(true); const [showVisionOverlay, setShowVisionOverlay] = useState(true); const [capturedStream, setCapturedStream] = useState(null); const [logs, setLogs] = useState([]); const videoRef = useRef(null); // Sample OS Screen Presets with bounding elements const presets = { checkout: { title: "E-Commerce Customer Portal", bgGradient: "from-slate-900 to-indigo-950", elements: [ { id: "e1", label: "Email Address Input", type: "input", coords: { x: 320, y: 180, w: 260, h: 42 } }, { id: "e2", label: "Account Role Selector", type: "menu", coords: { x: 320, y: 240, w: 260, h: 42 } }, { id: "e3", label: "Save Settings Button", type: "button", coords: { x: 320, y: 310, w: 140, h: 40 } }, { id: "e4", label: "Export PDF Invoice", type: "button", coords: { x: 480, y: 310, w: 130, h: 40 } }, { id: "e5", label: "Navigation Profile Tab", type: "link", coords: { x: 80, y: 120, w: 160, h: 36 } }, ] as DetectedElement[] }, vscode: { title: "VS Code IDE — server.ts (Line 42 Debug)", bgGradient: "from-slate-950 via-slate-900 to-slate-950", elements: [ { id: "e1", label: "Line 42 Error Marker", type: "image", coords: { x: 240, y: 220, w: 320, h: 28 } }, { id: "e2", label: "Fix Quick Action Lightbulb", type: "button", coords: { x: 580, y: 220, w: 36, h: 36 } }, { id: "e3", label: "Terminal Console Tab", type: "button", coords: { x: 180, y: 380, w: 110, h: 32 } }, { id: "e4", label: "Run & Debug Icon", type: "button", coords: { x: 20, y: 160, w: 42, h: 42 } }, ] as DetectedElement[] }, dashboard: { title: "Aura Cloud Metrics & Analytics", bgGradient: "from-slate-900 via-purple-950 to-slate-950", elements: [ { id: "e1", label: "Date Range Filter Dropdown", type: "menu", coords: { x: 520, y: 90, w: 160, h: 36 } }, { id: "e2", label: "Generate Live Report Button", type: "button", coords: { x: 700, y: 90, w: 150, h: 36 } }, { id: "e3", label: "Quota Gauge Graph", type: "image", coords: { x: 80, y: 160, w: 380, h: 200 } }, ] as DetectedElement[] }, custom: { title: "Captured Live Screen Stream", bgGradient: "from-slate-900 to-black", elements: [ { id: "e1", label: "Active Window Target", type: "button", coords: { x: 200, y: 150, w: 300, h: 100 } } ] as DetectedElement[] } }; const activePresetData = presets[selectedPreset]; // Generated step plan based on current prompt const stepsPlan: ActionStep[] = [ { id: "s1", stepNumber: 1, type: "VISION_SCAN", description: "Visual Scan & OCR: Analyzing desktop viewport (1920x1080) for interactable UI targets.", status: "pending", timestamp: "00:01" }, { id: "s2", stepNumber: 2, type: "MOVE_CURSOR", description: "Trajectory Navigation: Moving cursor towards 'Email Address Input' at (x: 320, y: 180).", coords: { x: 320, y: 180 }, status: "pending", timestamp: "00:03" }, { id: "s3", stepNumber: 3, type: "CLICK", description: "Mouse Action: Left click to focus email text input element.", targetElement: "Email Address Input", coords: { x: 320, y: 180 }, status: "pending", timestamp: "00:04" }, { id: "s4", stepNumber: 4, type: "TYPE_TEXT", description: "Keyboard Operator: Typing string 'alex.dev@aurastudio.ai' with standard human latency.", textInput: "alex.dev@aurastudio.ai", status: "pending", timestamp: "00:06" }, { id: "s5", stepNumber: 5, type: "MOVE_CURSOR", description: "Trajectory Navigation: Re-positioning cursor to 'Save Settings Button' at (x: 320, y: 310).", coords: { x: 320, y: 310 }, status: "pending", timestamp: "00:08" }, { id: "s6", stepNumber: 6, type: "CLICK", description: "Mouse Action: Primary click 'Save Settings' to commit desktop state change.", targetElement: "Save Settings Button", coords: { x: 320, y: 310 }, status: "pending", timestamp: "00:09" }, { id: "s7", stepNumber: 7, type: "COMPLETE", description: "Execution Finished: Desktop task completed successfully. Form updated and committed.", status: "pending", timestamp: "00:10" } ]; const [steps, setSteps] = useState(stepsPlan); // Start real screen capture const handleStartScreenCapture = async () => { try { const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }); setCapturedStream(stream); setSelectedPreset("custom"); if (videoRef.current) { videoRef.current.srcObject = stream; videoRef.current.play(); } addLog("Live screen display media capture initialized."); } catch (err: any) { console.error("Screen capture error:", err); addLog("Screen capture cancelled or unavailable: " + err.message); } }; const handleStopScreenCapture = () => { if (capturedStream) { capturedStream.getTracks().forEach(track => track.stop()); setCapturedStream(null); setSelectedPreset("checkout"); addLog("Stopped screen display capture."); } }; const addLog = (msg: string) => { setLogs(prev => [`[${new Date().toLocaleTimeString()}] ${msg}`, ...prev.slice(0, 30)]); }; // Automated execution simulator loop useEffect(() => { let timer: NodeJS.Timeout; if (isRunning && currentStepIndex < steps.length - 1) { const nextIdx = currentStepIndex + 1; const step = steps[nextIdx]; // Delay based on speed setting const delay = Math.max(400, 1500 / speed); timer = setTimeout(() => { setCurrentStepIndex(nextIdx); setSteps(prevSteps => prevSteps.map((s, idx) => ({ ...s, status: idx < nextIdx ? "completed" : idx === nextIdx ? "running" : "pending" })) ); addLog(`Executing Step ${step.stepNumber}: ${step.type} - ${step.description}`); if (step.coords) { setCursorPos(step.coords); } if (step.type === "CLICK") { setShowClickRipple(true); setTimeout(() => setShowClickRipple(false), 500); } if (step.type === "COMPLETE") { setIsRunning(false); setSteps(prevSteps => prevSteps.map(s => ({ ...s, status: "completed" }))); addLog("Computer Use Operator finished task successfully!"); } }, delay); } return () => clearTimeout(timer); }, [isRunning, currentStepIndex, speed, steps]); const handleStartOperator = () => { setSteps(stepsPlan.map(s => ({ ...s, status: "pending" }))); setCurrentStepIndex(-1); setIsRunning(true); addLog("Computer Use Operator initiated goal: " + taskPrompt); }; const handleResetOperator = () => { setIsRunning(false); setCurrentStepIndex(-1); setCursorPos({ x: 50, y: 50 }); setSteps(stepsPlan.map(s => ({ ...s, status: "pending" }))); addLog("Reset computer use state."); }; if (!isOpen) return null; return (
e.stopPropagation()} > {/* Header Bar */}

Computer Use OS Operator

Vision Operator

Direct desktop vision, cursor trajectory movement, keyboard typing, and window automation

{/* Main Content Layout */}
{/* Left Column: Screen Viewport & Live Simulation Canvas (7 Cols) */}
{/* Viewport Toolbar */}
Screen Viewport:
{capturedStream ? ( ) : ( )}
{/* Desktop Screen Canvas Display */}
{/* Virtual OS Title Bar */}
{activePresetData.title}
RES: 1920x1080 FPS: 60
{/* Viewport Canvas Body */}
{/* Live Media Stream Video if screen capture active */} {selectedPreset === "custom" && (
{/* Console Log Output */}
Operator Event Stream {logs.length} events
{logs.length === 0 ? ( No events logged yet. Click "Run Operator Goal" to start. ) : ( logs.map((log, i) => ( {log} )) )}
{/* Right Column: Goal Configuration & Action Steps (5 Cols) */}

Autonomous Desktop Goal AI Vision Model