"""Fara1.5 as released, in Microsoft's own agent loop, on MiniWeb human tasks (EXPERIMENT_PLAN, RQ1; 2026-10-02). Runs in the `fara` venv (github.com/microsoft/fara, playwright 1.51), not the MiniWeb env, and imports only that package and the standard library. Tasks come from `python -m miniweb.evaluation.fara_rq1 prep` ({BASE} placeholders); grading happens later in the MiniWeb env (`python -m miniweb.evaluation.fara_rq1 grade`), from what this saves per task: //session.json {"base", "answer", "status", "rounds", "record": [...], "log": [...], "error"} //fara/ Fara's own trajectory folder (screenshots, messages) Setup mirrors miniwebagent.evaluate.prepare_human: /_admin/session-flags (logged out for a login task), then the start page, the same task wrapper as our agents (miniwebagent.harness.instruction), clipboard allowed (as miniwebagent.harness.enable_clipboard). Fara's eval mode: auto_user_reply (its confirmation questions get a fixed reply instead of halting) and no captcha gate. /bin/python src/miniweb/evaluation/fara_runner.py --tasks /tasks.json --out \\ --bases http://127.0.0.1:8306,http://127.0.0.1:8307 --endpoint http://127.0.0.1:8640/v1 --model fara15-4b """ import argparse import asyncio import json import time import urllib.request from pathlib import Path from urllib.parse import urlsplit from fara.agents.fara.fara15_agent import Fara15Agent, Fara15AgentConfig from fara.core.data_point import SolverStatus, Task from fara.core.run_context import RunContext from fara.environments.playwright import PlaywrightEnvironment def instruction(task_text, start_url): """Same wrapper as miniwebagent.harness.instruction (our agents' task text).""" return f"You are interacting with a web application at {start_url}. Your task: {task_text}" async def admin_get(context, base, path): """GET a privileged /_admin endpoint as this browser's session (its cookies), outside the page.""" host = urlsplit(base).hostname cookies = await context.cookies() header = "; ".join(f"{c['name']}={c['value']}" for c in cookies if (c.get("domain") or "").lstrip(".") in (host, "")) req = urllib.request.Request(base + path, headers={"Cookie": header}) def fetch(): with urllib.request.urlopen(req, timeout=30) as r: return json.loads(r.read().decode()) return await asyncio.to_thread(fetch) async def run_task(t, base, args, out): ep = out / t["task_id"] ep.mkdir(parents=True, exist_ok=True) start = t["start"].replace("{BASE}", base) prompt = t["prompt"].replace("{BASE}", base) flags = base + "/_admin/session-flags" + ("?logout=1" if t["needs_login"] else "") res = {"task_id": t["task_id"], "base": base, "answer": None, "status": None, "rounds": None, "error": None, "record": [], "log": [], "seconds": None} t0 = time.time() env = PlaywrightEnvironment(viewport_width=args.width, viewport_height=args.height, headless=True, browser_channel="chromium", start_page=flags, single_tab_mode=True) agent = None try: await env.initialize() await env._context.grant_permissions(["clipboard-read", "clipboard-write"], origin=base) await env._page.goto(start, wait_until="domcontentloaded") await asyncio.sleep(1.0) agent = Fara15Agent(Fara15AgentConfig( client_config={"model": args.model, "base_url": args.endpoint, "api_key": "not-needed"}, max_rounds=args.max_rounds, identity="fara_qwen35", critical_points="fara-1.5", save_screenshots=True, viewport_width=args.width, viewport_height=args.height, auto_user_reply=True, captcha_timeout_limit=0, raise_on_captcha_timeout=False)) ctx = RunContext.create(environment=env, task=Task(task_id=t["task_id"], instruction=instruction(prompt, start)), output_dir=ep / "fara") await agent.initialize(ctx) answer, _, _ = await asyncio.wait_for(agent.run(ctx), timeout=args.timeout) res["answer"], res["status"] = answer, str(ctx.solver_log.status) res["rounds"] = getattr(getattr(agent, "_state", None), "round", None) except Exception as e: # a timeout or harness error still keeps whatever the session did res["error"] = f"{type(e).__name__}: {str(e)[:300]}" try: res["record"] = (await admin_get(env._context, base, "/_admin/record")).get("entries", []) or [] res["log"] = (await admin_get(env._context, base, "/_admin/log")).get("entries", []) or [] except Exception as e: res["error"] = (res["error"] or "") + f" | backend read: {type(e).__name__}: {str(e)[:200]}" res["seconds"] = round(time.time() - t0, 1) (ep / "session.json").write_text(json.dumps(res, default=str)) try: if agent is not None: await agent.close(ctx) except Exception: pass try: await env.close() except Exception: pass return res async def main(): ap = argparse.ArgumentParser() ap.add_argument("--tasks", required=True) ap.add_argument("--out", required=True) ap.add_argument("--bases", required=True, help="comma list of MiniWeb server bases, e.g. http://127.0.0.1:8306") ap.add_argument("--endpoint", required=True, help="vLLM OpenAI endpoint serving Fara1.5") ap.add_argument("--model", default="fara15-4b") ap.add_argument("--workers", type=int, default=6) ap.add_argument("--max-rounds", type=int, default=40, help="Fara actions per task (our MiniWeb budget: 40)") ap.add_argument("--timeout", type=int, default=2400) ap.add_argument("--width", type=int, default=1440, help="Fara's native viewport (1440x900)") ap.add_argument("--height", type=int, default=900) ap.add_argument("--limit", type=int, default=0) ap.add_argument("--task-ids", default="") args = ap.parse_args() out = Path(args.out) tasks = json.loads(Path(args.tasks).read_text()) if args.task_ids: keep = set(args.task_ids.split(",")) tasks = [t for t in tasks if t["task_id"] in keep] done = {p.parent.name for p in out.glob("*/session.json")} tasks = [t for t in tasks if t["task_id"] not in done] # resume if args.limit: tasks = tasks[:args.limit] bases = args.bases.split(",") queue = asyncio.Queue() for t in tasks: queue.put_nowait(t) print(f"{len(tasks)} tasks, {args.workers} workers on {len(bases)} MiniWeb servers", flush=True) async def worker(i): while not queue.empty(): t = queue.get_nowait() r = await run_task(t, bases[i % len(bases)], args, out) print(f"{t['task_id']}: status={r['status']} rounds={r['rounds']} {r['seconds']}s err={r['error']}", flush=True) await asyncio.gather(*(worker(i) for i in range(args.workers))) if __name__ == "__main__": asyncio.run(main())