Update app.py

app.py

@@ -12,7 +12,6 @@ from playwright.async_api import async_playwright, Browser, Page
 from groq import AsyncGroq
 
 app = FastAPI()
-
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -21,7 +20,7 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-# ---------- Groq client ----------
+# ---------- Groq ----------
 GROQ_API_KEY = os.environ.get("GROQ_API_KEY")
 if not GROQ_API_KEY:
     raise RuntimeError("GROQ_API_KEY not set.")
@@ -29,18 +28,34 @@ client = AsyncGroq(api_key=GROQ_API_KEY)
 
 # ---------- System prompt ----------
 SYSTEM_PROMPT = """
-You are an autonomous web agent. You have access to a browser via Playwright.
-Your goal is to complete the user's task by issuing a sequence of actions.
+You are an autonomous web agent. You control a real browser via Playwright.
+Your task is given by the user. You must complete it step by step.
+
+You have a memory of the last 5 actions you performed and their outcomes.
+Use this to avoid repeating mistakes (e.g., if a click failed, try a different selector).
 
-Available actions (return a JSON object with exactly one of these):
+Available actions – return a JSON object with exactly one action:
 - {"action": "navigate", "url": "https://..."}
 - {"action": "click", "selector": "css_selector"}
 - {"action": "fill", "selector": "css_selector", "value": "text"}
+- {"action": "select", "selector": "css_selector", "value": "option_text_or_value"}
+- {"action": "scroll", "amount": 300}   (positive = down, negative = up)
+- {"action": "press", "key": "Enter"}   (keyboard key)
+- {"action": "hover", "selector": "..."}
+- {"action": "back"}                    (go back one page)
 - {"action": "wait", "seconds": 2}
-- {"action": "screenshot"}   (captures current page)
-- {"action": "done", "result": "final answer or error message"}
+- {"action": "screenshot"}              (captures current page)
+- {"action": "get_text", "selector": "...", "variable": "myVar"}  (stores text in a variable)
+- {"action": "execute_script", "code": "return document.title"}   (run JS, result stored)
+- {"action": "done", "result": "final answer"}   (finish the task)
+
+Guidelines:
+- If you see a login form, fill it and submit (use click on the submit button).
+- If you need to extract data, use get_text and store it, then continue.
+- If you get stuck, try scrolling to reveal more content.
+- Always prefer using `data-test` or `id` selectors when available.
+- If a page has a table of contents, click on the relevant section link to jump.
 
-You will receive the current page URL and a simplified DOM text (visible text only).
 Think step by step. Only output the JSON action, nothing else.
 """
 
@@ -53,6 +68,8 @@ class SessionState:
         self.steps = 0
         self.last_result = ""
         self.done = False
+        self.action_history = []  # list of {"action": ..., "result": ...}
+        self.variables = {}       # store extracted text
 
 sessions = {}
 
@@ -64,18 +81,40 @@ async def get_screenshot_b64(page):
     screenshot_bytes = await page.screenshot(full_page=False)
     return base64.b64encode(screenshot_bytes).decode("utf-8")
 
-async def ask_llm_for_action(task: str, url: str, page_text: str):
+def build_history_text(session):
+    """Return a string of last 5 actions with results."""
+    history = session.action_history[-5:]
+    if not history:
+        return "No actions taken yet."
+    lines = []
+    for i, entry in enumerate(history):
+        lines.append(f"{i+1}. Action: {entry['action']} → Result: {entry['result']}")
+    return "\n".join(lines)
+
+async def ask_llm_for_action(session, user_instruction=None):
+    """Ask Groq for the next action. If user_instruction is given, it overrides the task."""
+    page = session.page
+    url = page.url
+    page_text = await get_page_text(page)
     if len(page_text) > 4000:
         page_text = page_text[:4000] + "\n... (truncated)"
+
+    history_text = build_history_text(session)
+
     user_prompt = f"""
-Task: {task}
+Task: {session.task}
 
 Current URL: {url}
 
-Page content (visible text):
+Action history (most recent first):
+{history_text}
+
+Current visible text:
 {page_text}
 
-What is your next action? Output a JSON action.
+{user_instruction if user_instruction else "What is your next action?"}
+
+Output a JSON action.
 """
     completion = await client.chat.completions.create(
         model="llama-3.3-70b-versatile",
@@ -97,101 +136,179 @@ What is your next action? Output a JSON action.
             action = {"action": "done", "result": f"Invalid LLM output: {raw}"}
     return action
 
-async def execute_one_action(session: SessionState, user_instruction: str = None):
-    """Execute exactly one action. If user_instruction is given, override the LLM action."""
+async def execute_one_action(session, override=None):
+    """
+    Execute one action.
+    override can be:
+      - None or "auto" → use LLM to decide.
+      - a JSON string → execute that action directly.
+      - a free‑text string → treat as a new instruction, then ask LLM for action.
+    """
     page = session.page
-    # If user gave an instruction, try to parse it as a JSON action; otherwise treat as a new task.
-    if user_instruction and user_instruction.lower() != "auto":
-        # Attempt to parse as a JSON action
+
+    if override and override.lower() != "auto":
+        # Try to parse as JSON
         try:
-            action = json.loads(user_instruction)
-        except:
-            # If not JSON, treat as a new task description – we'll update the session task
-            session.task = user_instruction
-            # Then get LLM action for the new task
-            action = await ask_llm_for_action(
-                session.task,
-                page.url,
-                await get_page_text(page)
-            )
+            action = json.loads(override)
+            # If it's a valid JSON action, we'll execute it directly
+        except json.JSONDecodeError:
+            # Not JSON – treat as new task description
+            session.task = override
+            action = await ask_llm_for_action(session)
     else:
         # Auto – ask LLM
-        action = await ask_llm_for_action(
-            session.task,
-            page.url,
-            await get_page_text(page)
-        )
+        action = await ask_llm_for_action(session)
 
     act = action.get("action")
     result_text = ""
     screenshot_b64 = ""
 
-    if act == "done":
-        session.done = True
-        session.last_result = action.get("result", "Task completed.")
-        result_text = session.last_result
-        screenshot_b64 = await get_screenshot_b64(page)
-        return {"done": True, "result": result_text, "screenshot": screenshot_b64, "action": action}
+    # Execute the action
+    try:
+        if act == "done":
+            session.done = True
+            session.last_result = action.get("result", "Task completed.")
+            result_text = session.last_result
+            screenshot_b64 = await get_screenshot_b64(page)
+            # Record history
+            session.action_history.append({"action": action, "result": result_text})
+            return {"done": True, "result": result_text, "screenshot": screenshot_b64,
+                    "action_taken": action, "steps": session.steps}
+
+        elif act == "navigate":
+            url = action.get("url")
+            if url:
+                await page.goto(url, timeout=30000)
+                result_text = f"Navigated to {url}"
+            else:
+                result_text = "Error: missing 'url'"
+
+        elif act == "click":
+            sel = action.get("selector")
+            if sel:
+                try:
+                    await page.click(sel, timeout=5000)
+                    result_text = f"Clicked {sel}"
+                except Exception as e:
+                    result_text = f"Click failed: {str(e)}"
+            else:
+                result_text = "Error: missing 'selector'"
+
+        elif act == "fill":
+            sel = action.get("selector")
+            val = action.get("value")
+            if sel and val is not None:
+                try:
+                    await page.fill(sel, val, timeout=5000)
+                    result_text = f"Filled {sel} with '{val}'"
+                except Exception as e:
+                    result_text = f"Fill failed: {str(e)}"
+            else:
+                result_text = "Error: missing 'selector' or 'value'"
+
+        elif act == "select":
+            sel = action.get("selector")
+            val = action.get("value")
+            if sel and val:
+                try:
+                    await page.select_option(sel, val)
+                    result_text = f"Selected '{val}' on {sel}"
+                except Exception as e:
+                    result_text = f"Select failed: {str(e)}"
+            else:
+                result_text = "Error: missing 'selector' or 'value'"
+
+        elif act == "scroll":
+            amount = action.get("amount", 300)
+            await page.evaluate(f"window.scrollBy(0, {amount})")
+            result_text = f"Scrolled by {amount}px"
+
+        elif act == "press":
+            key = action.get("key")
+            if key:
+                await page.keyboard.press(key)
+                result_text = f"Pressed '{key}'"
+            else:
+                result_text = "Error: missing 'key'"
+
+        elif act == "hover":
+            sel = action.get("selector")
+            if sel:
+                try:
+                    await page.hover(sel, timeout=5000)
+                    result_text = f"Hovered {sel}"
+                except Exception as e:
+                    result_text = f"Hover failed: {str(e)}"
+            else:
+                result_text = "Error: missing 'selector'"
+
+        elif act == "back":
+            await page.go_back()
+            result_text = "Went back one page"
+
+        elif act == "wait":
+            seconds = action.get("seconds", 1)
+            await asyncio.sleep(seconds)
+            result_text = f"Waited {seconds}s"
+
+        elif act == "screenshot":
+            # just capture later
+            result_text = "Screenshot captured"
+
+        elif act == "get_text":
+            sel = action.get("selector")
+            var = action.get("variable", "last_text")
+            if sel:
+                try:
+                    text = await page.text_content(sel, timeout=5000)
+                    session.variables[var] = text.strip() if text else ""
+                    result_text = f"Stored text from {sel} into '{var}'"
+                except Exception as e:
+                    result_text = f"get_text failed: {str(e)}"
+            else:
+                result_text = "Error: missing 'selector'"
+
+        elif act == "execute_script":
+            code = action.get("code")
+            if code:
+                try:
+                    result = await page.evaluate(code)
+                    session.variables["_last_script"] = result
+                    result_text = f"Script executed, result stored in _last_script"
+                except Exception as e:
+                    result_text = f"Script failed: {str(e)}"
+            else:
+                result_text = "Error: missing 'code'"
 
-    elif act == "navigate":
-        url = action.get("url")
-        if url:
-            await page.goto(url, timeout=30000)
         else:
-            result_text = "Error: missing 'url' for navigate"
-    elif act == "click":
-        sel = action.get("selector")
-        if sel:
-            try:
-                await page.click(sel, timeout=5000)
-            except Exception as e:
-                result_text = f"Click failed: {str(e)}"
-        else:
-            result_text = "Error: missing 'selector' for click"
-    elif act == "fill":
-        sel = action.get("selector")
-        val = action.get("value")
-        if sel and val is not None:
-            try:
-                await page.fill(sel, val, timeout=5000)
-            except Exception as e:
-                result_text = f"Fill failed: {str(e)}"
-        else:
-            result_text = "Error: missing 'selector' or 'value' for fill"
-    elif act == "wait":
-        seconds = action.get("seconds", 1)
-        await asyncio.sleep(seconds)
-    elif act == "screenshot":
-        # just capture, will return later
-        pass
-    else:
-        result_text = f"Unknown action: {act}"
+            result_text = f"Unknown action: {act}"
 
+    except Exception as e:
+        result_text = f"Action execution error: {str(e)}"
+
+    # Record history
+    session.action_history.append({"action": action, "result": result_text})
     session.steps += 1
-    if result_text:
-        session.last_result = result_text
+    session.last_result = result_text
 
     # Capture screenshot after the action
     screenshot_b64 = await get_screenshot_b64(page)
 
     # Ask LLM for the next action (to show the user)
-    next_action = await ask_llm_for_action(
-        session.task,
-        page.url,
-        await get_page_text(page)
-    )
+    next_action = await ask_llm_for_action(session)
 
     return {
         "done": session.done,
-        "result": session.last_result or "Action executed.",
+        "result": result_text,
         "screenshot": screenshot_b64,
         "steps": session.steps,
         "proposed_next_action": next_action,
         "current_url": page.url,
-        "action_taken": action
+        "action_taken": action,
+        "history": session.action_history[-5:]  # send last 5 for display
     }
 
-# ---------- Endpoints ----------
+# ---------- FastAPI Endpoints ----------
 class StartRequest(BaseModel):
     task: str
     headless: bool = True
@@ -219,9 +336,8 @@ async def start_session(req: StartRequest):
         session = SessionState(req.task, browser, page)
         sessions[session_id] = session
 
-        # Get initial screenshot and proposed action
         screenshot_b64 = await get_screenshot_b64(page)
-        next_action = await ask_llm_for_action(req.task, page.url, await get_page_text(page))
+        next_action = await ask_llm_for_action(session)
 
         return StartResponse(
             session_id=session_id,
@@ -235,7 +351,7 @@ async def start_session(req: StartRequest):
 
 class StepRequest(BaseModel):
     session_id: str
-    instruction: str = "auto"   # "auto" or a JSON action string, or a new task description
+    instruction: str = "auto"
 
 class StepResponse(BaseModel):
     done: bool
@@ -245,6 +361,7 @@ class StepResponse(BaseModel):
     proposed_next_action: dict
     current_url: str
     action_taken: dict
+    history: list
 
 @app.post("/step", response_model=StepResponse)
 async def step(req: StepRequest):
@@ -258,12 +375,13 @@ async def step(req: StepRequest):
         result = await execute_one_action(session, req.instruction)
         return StepResponse(
             done=result["done"],
-            result=result.get("result", ""),
+            result=result["result"],
             screenshot=result["screenshot"],
             steps=result["steps"],
-            proposed_next_action=result.get("proposed_next_action", {}),
-            current_url=result.get("current_url", ""),
-            action_taken=result.get("action_taken", {})
+            proposed_next_action=result["proposed_next_action"],
+            current_url=result["current_url"],
+            action_taken=result["action_taken"],
+            history=result["history"]
         )
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
@@ -276,34 +394,36 @@ async def stop_session(session_id: str):
         return {"status": "stopped"}
     return {"status": "not_found"}
 
-# ---------- Serve the interactive HTML ----------
+# ---------- Serve HTML (same as before, but with history display) ----------
 HTML_PAGE = """
 <!DOCTYPE html>
-<html lang="en">
+<html>
 <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Interactive Agent Control</title>
+    <meta charset="UTF-8">
+    <title>Interactive Agent</title>
     <style>
         * { box-sizing: border-box; margin: 0; padding: 0; }
-        body { font-family: system-ui, sans-serif; background: #0b0e14; color: #e8edf5; padding: 20px; }
-        .container { max-width: 1200px; margin: 0 auto; background: #1a1f2a; border-radius: 24px; padding: 30px; }
+        body { font-family: system-ui; background: #0b0e14; color: #e8edf5; padding: 20px; }
+        .container { max-width: 1400px; margin: 0 auto; background: #1a1f2a; border-radius: 24px; padding: 30px; }
         .row { display: flex; flex-wrap: wrap; gap: 30px; }
         .left { flex: 1; min-width: 300px; }
         .right { flex: 2; min-width: 500px; }
         textarea, input[type="text"] { width: 100%; background: #0f141d; border: 1px solid #2f384a; border-radius: 12px; padding: 12px; color: white; }
         .btn { background: #4f7cff; border: none; color: white; padding: 10px 20px; border-radius: 40px; cursor: pointer; margin: 5px; }
         .btn:disabled { opacity: 0.5; }
-        .screenshot-box img { width: 100%; border-radius: 12px; border: 1px solid #2f384a; }
+        .screenshot-box img { width: 100%; border-radius: 12px; border: 1px solid #2f384a; max-height: 500px; object-fit: contain; }
         .status { margin: 10px 0; }
         .action-card { background: #11171f; padding: 15px; border-radius: 12px; margin: 10px 0; }
         pre { background: #0f141d; padding: 10px; border-radius: 8px; overflow: auto; max-height: 200px; }
+        .history-item { background: #0f141d; padding: 5px 10px; border-radius: 6px; margin: 4px 0; font-size: 13px; }
+        .history-item .act { color: #4f7cff; }
+        .history-item .res { color: #8bc34a; }
     </style>
 </head>
 <body>
 <div class="container">
-    <h1>🧠 Step‑by‑Step Agent</h1>
-    <p>Start a session, then see the screenshot and the proposed action. You can <strong>continue</strong> or <strong>override</strong> with a new instruction.</p>
+    <h1>🧠 Interactive Agent</h1>
+    <p>Start a session, see the screenshot, and guide the agent step by step.</p>
     <hr style="border-color:#2f384a; margin:20px 0;">
 
     <div class="row">
@@ -315,6 +435,11 @@ HTML_PAGE = """
             <button class="btn" id="stopBtn" disabled>⏹ Stop</button>
             <div id="status" class="status"></div>
             <div id="info"></div>
+
+            <div id="historyBox" style="margin-top:20px; display:none;">
+                <h4>📜 Action History</h4>
+                <div id="historyList"></div>
+            </div>
         </div>
 
         <div class="right">
@@ -330,12 +455,15 @@ HTML_PAGE = """
                     <button class="btn" id="overrideBtn">✏️ Override</button>
                 </div>
                 <div id="overrideArea" style="display:none; margin-top:10px;">
-                    <input type="text" id="overrideInput" placeholder='e.g. {"action":"click","selector":"#login"}' style="width:70%;" />
+                    <input type="text" id="overrideInput" placeholder='JSON action OR new task description' style="width:70%;" />
                     <button class="btn" id="sendOverrideBtn">Send</button>
-                    <small style="display:block; color:#8892a8;">You can also type a new task description, e.g. "now search for Python"</small>
+                    <small style="display:block; color:#8892a8;">Example JSON: {"action":"click","selector":"#login"}<br>Or free text: "now search for Python"</small>
                 </div>
                 <div id="resultBox" style="margin-top:15px; background:#11171f; padding:15px; border-radius:12px; display:none;">
-                    <strong>✅ Result:</strong> <span id="resultText">—</span>
+                    <strong>✅ Last result:</strong> <span id="resultText">—</span>
+                </div>
+                <div id="actionTakenBox" style="margin-top:10px; background:#11171f; padding:15px; border-radius:12px; display:none;">
+                    <strong>⚡ Action executed:</strong> <pre id="actionTaken" style="margin-top:5px;"></pre>
                 </div>
             </div>
         </div>
@@ -344,8 +472,6 @@ HTML_PAGE = """
 
 <script>
     let sessionId = null;
-    let currentStep = 0;
-
     const statusDiv = document.getElementById('status');
     const infoDiv = document.getElementById('info');
     const screenshotContainer = document.getElementById('screenshotContainer');
@@ -354,6 +480,10 @@ HTML_PAGE = """
     const proposedAction = document.getElementById('proposedAction');
     const resultBox = document.getElementById('resultBox');
     const resultText = document.getElementById('resultText');
+    const actionTakenBox = document.getElementById('actionTakenBox');
+    const actionTaken = document.getElementById('actionTaken');
+    const historyBox = document.getElementById('historyBox');
+    const historyList = document.getElementById('historyList');
 
     const startBtn = document.getElementById('startBtn');
     const stopBtn = document.getElementById('stopBtn');
@@ -377,10 +507,25 @@ HTML_PAGE = """
         proposedAction.textContent = JSON.stringify(action, null, 2);
     }
 
+    function updateHistory(history) {
+        if (!history || history.length === 0) {
+            historyBox.style.display = 'none';
+            return;
+        }
+        historyBox.style.display = 'block';
+        historyList.innerHTML = history.map(h => 
+            `<div class="history-item"><span class="act">${JSON.stringify(h.action)}</span> → <span class="res">${h.result}</span></div>`
+        ).join('');
+    }
+
     function updateUI(data) {
-        currentStep = data.steps || 0;
         showScreenshot(data.screenshot);
         showProposedAction(data.proposed_next_action);
+        if (data.history) updateHistory(data.history);
+        if (data.action_taken) {
+            actionTakenBox.style.display = 'block';
+            actionTaken.textContent = JSON.stringify(data.action_taken, null, 2);
+        }
         if (data.done) {
             resultBox.style.display = 'block';
             resultText.textContent = data.result || 'Task completed.';
@@ -388,7 +533,8 @@ HTML_PAGE = """
             overrideBtn.disabled = true;
             setStatus('✅ Session finished.');
         } else {
-            resultBox.style.display = 'none';
+            resultBox.style.display = 'block';
+            resultText.textContent = data.result || 'Action executed.';
             continueBtn.disabled = false;
             overrideBtn.disabled = false;
             setStatus(`👣 Step ${data.steps} done. Next action proposed.`);
@@ -414,7 +560,6 @@ HTML_PAGE = """
         const task = document.getElementById('taskInput').value.trim();
         if (!task) { setStatus('Please enter a task.', true); return; }
         const headless = document.getElementById('headlessCheck').checked;
-
         startBtn.disabled = true;
         setStatus('Starting session...');
         try {
@@ -473,6 +618,7 @@ HTML_PAGE = """
             setStatus('Session stopped.');
             actionContainer.style.display = 'none';
             screenshotContainer.style.display = 'none';
+            historyBox.style.display = 'none';
         } catch (e) {
             setStatus('Stop error: ' + e.message, true);
         }
@@ -486,7 +632,7 @@ HTML_PAGE = """
 async def serve_html():
     return HTMLResponse(content=HTML_PAGE)
 
-# ---------- Optional: cleanup on exit ----------
+# ---------- Cleanup ----------
 import atexit
 @atexit.register
 async def cleanup():