"""WebArena-Lite-v2 (154 tasks) through the WebMix browser-use harness. python -m miniwebagent.wa --arm base --out data/webmix/wa/base # 4 instances in parallel python -m miniwebagent.wa --arm base --instances 1 --limit 3 --out ... # smoke test Tasks, per-instance URL maps and the evaluator are ScaleCUA's WebArenaLiteV2 (OpenGVLab/ScaleCUA, src/miniweb/evaluation/WebArenaLiteV2), copied from the host's checkout into data/webmix/wa_lite_v2. Four deployments run on one host, WA_HOST or the URL maps' (instance i: sites at 8081+100(i-1).., homepage 4398+i; map and wikipedia shared); each instance runs its share of the tasks in order, as the benchmark recommends (tasks are not fully independent). Reset the instances first (reset API, one per instance on 8090+i). Each instance worker is its own process: the evaluator's helpers read site URLs from config.envs.webarena.init.env_config at import time, so that module is pointed at the instance before the evaluator is imported. Login: one storage state per site per instance, made with the benchmark's own login flows, loaded into each episode's browser over CDP. Grading: the benchmark's evaluator on the agent's final page (Playwright attached to browser-use's Chromium over CDP) and its done() text as the answer; fuzzy matches use the benchmark's judge (gpt-4o-2024-11-20). """ from __future__ import annotations from helpers.paths import REPO_ROOT import argparse import asyncio import json import os import subprocess import sys import time from pathlib import Path from miniwebagent import harness as H ROOT = REPO_ROOT WA_DIR = ROOT / "data" / "webmix" / "wa_lite_v2" TASK_DIR = WA_DIR / "config" / "envs" / "webarena" / "tasks" SPLIT = ROOT / "data" / "webmix" / "wa" / "split_5050.json" # dev = development evals, test = final (2026-09-29) URL_KEYS = ("SHOPPING_URL", "SHOPPING_ADMIN_URL", "REDDIT_URL", "GITLAB_URL", "MAP_URL", "WIKIPEDIA_URL", "HOMEPAGE_URL") def instance_urls(i): name = "webarena_url.json" if i == 1 else f"webarena_url_{i}.json" return json.loads((WA_DIR / "config" / "envs" / "webarena" / "init" / name).read_text()) def wa_host(): """The WebArena deployments' host: WA_HOST, else the one instance 1's URL map points at (browsing is limited to it).""" from urllib.parse import urlsplit return os.environ.get("WA_HOST") or urlsplit(instance_urls(1)["HOMEPAGE_URL"]).hostname def load_tasks(): tasks = [json.loads(p.read_text()) for p in TASK_DIR.glob("[0-9]*.json")] return sorted(tasks, key=lambda t: t["task_id"]) def localize(cfg, i): """A task config (written for instance 1) with every site URL moved to instance i's.""" if i == 1: return cfg s = json.dumps(cfg) one, mine = instance_urls(1), instance_urls(i) for k in URL_KEYS: a, b = one[k].split("/admin")[0], mine[k].split("/admin")[0] if a != b: s = s.replace(a, b) return json.loads(s) def _point_env_config(i): """Import the benchmark's env_config with instance i's URLs (before its evaluator is imported).""" if str(WA_DIR) not in sys.path: sys.path.insert(0, str(WA_DIR)) cwd = os.getcwd() os.chdir(WA_DIR) # env_config opens its URL file by a path relative to the benchmark root try: import config.envs.webarena.init.env_config as ec finally: os.chdir(cwd) u = instance_urls(i) ec.URL = u ec.REDDIT, ec.SHOPPING, ec.SHOPPING_ADMIN = u["REDDIT_URL"], u["SHOPPING_URL"], u["SHOPPING_ADMIN_URL"] ec.GITLAB, ec.MAP = u["GITLAB_URL"], u["MAP_URL"] ec.WIKIPEDIA, ec.HOMEPAGE = u.get("WIKIPEDIA_URL", ""), u.get("HOMEPAGE_URL", "") return ec # ── login states (the benchmark's auto_login flows, per instance) ───────────── LOGIN_SITES = ("reddit", "shopping_admin", "gitlab", "shopping") def make_login_states(i, out_dir, sites=LOGIN_SITES): """-> {site: storage-state path} for `sites` (default: shopping, shopping_admin, reddit, gitlab) on instance i.""" from playwright.sync_api import sync_playwright ec = _point_env_config(i) acc = ec.ACCOUNTS out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) states = {} with sync_playwright() as p: b = p.chromium.launch(executable_path=H.CHROME, headless=True) for site in sites: for attempt in range(25): # a site just restored (GitLab above all) can take 15+ min to serve try: states[site] = _login(b, ec, acc, site, out_dir) break except Exception as e: print(f"[inst {i}] {site} login attempt {attempt + 1} failed: {type(e).__name__}", flush=True) time.sleep(45) else: raise RuntimeError(f"instance {i}: could not log in to {site}") b.close() return states def _login(b, ec, acc, site, out_dir): """One site's login (the benchmark's auto_login flow) in a fresh context -> its storage-state path.""" ctx = b.new_context() try: page = ctx.new_page() page.set_default_timeout(60000) if site == "reddit": page.goto(f"{ec.REDDIT}/login") page.get_by_label("Username").fill(acc["reddit"]["username"]) page.get_by_label("Password").fill(acc["reddit"]["password"]) page.get_by_role("button", name="Log in").click() elif site == "shopping_admin": page.goto(ec.SHOPPING_ADMIN) page.get_by_placeholder("user name").fill(acc["shopping_admin"]["username"]) page.get_by_placeholder("password").fill(acc["shopping_admin"]["password"]) page.get_by_role("button", name="Sign in").click() elif site == "gitlab": page.goto(f"{ec.GITLAB}/users/sign_in") page.get_by_test_id("username-field").fill(acc["gitlab"]["username"]) page.get_by_test_id("password-field").fill(acc["gitlab"]["password"]) page.get_by_test_id("sign-in-button").click() elif site == "shopping": page.goto(f"{ec.SHOPPING}/customer/account/login/") page.get_by_label("Email", exact=True).fill(acc["shopping"]["username"]) page.get_by_label("Password", exact=True).fill(acc["shopping"]["password"]) page.get_by_role("button", name="Sign In").click() page.wait_for_load_state("load", timeout=60000) time.sleep(1) path = Path(out_dir) / f"{site}_state.json" ctx.storage_state(path=str(path)) return str(path) finally: ctx.close() async def set_cookies(bs, state_paths): cookies = [] for p in state_paths: for c in json.loads(Path(p).read_text()).get("cookies", []): cookies.append({k: c[k] for k in ("name", "value", "domain", "path", "expires", "httpOnly", "secure", "sameSite") if k in c and c[k] is not None and not (k == "expires" and c[k] < 0)}) if cookies: await bs.cdp_client.send.Storage.setCookies(params={"cookies": cookies}) # ── grading (the benchmark's evaluator, on the agent's final page) ──────────── def evaluate_sync(cdp_url, cfg, answer, final_url): """-> (score, {eval_type: score}) by the benchmark's evaluators on the agent's final page.""" from playwright.sync_api import sync_playwright with sync_playwright() as p: browser = p.chromium.connect_over_cdp(cdp_url) pages = [pg for ctx in browser.contexts for pg in ctx.pages] page = next((pg for pg in reversed(pages) if pg.url == final_url), pages[-1] if pages else None) if page is None: page = browser.contexts[0].new_page() return score_parts(cfg, answer, page) def score_parts(cfg, answer, page): import envs.web.webarena.evaluation.vab_evaluators as ve import envs.web.webarena.evaluation.vab_helper_functions as hf # program_html locators are "func:reddit_get_post_url(...)" strings eval()'d in the evaluator module, which # imports only a few helpers: expose them all (else NameError -> the task scores 0) for name in dir(hf): if not name.startswith("_") and not hasattr(ve, name): setattr(ve, name, getattr(hf, name)) webarena_evaluator_router = ve.webarena_evaluator_router action_list = [{}, {"name": "response", "parameters": {"answer": answer or ""}}] comb = webarena_evaluator_router(cfg) parts = {t: float(ev(action_list, cfg, page)) for t, ev in zip(cfg["eval"]["eval_types"], comb.evaluators)} total = 1.0 for v in parts.values(): total *= v return total, parts def use_judge(model): """The fuzzy / unachievable-reason matches through helpers.llm with `model` (the benchmark calls gpt-4o-2024-11-20 through OpenAI; 'gpt-4o-2024-11-20' keeps that). Same prompts, temperature 0.""" import envs.web.webarena.evaluation.vab_helper_functions as hf if model == "gpt-4o-2024-11-20": return from helpers.llm import call_llm def generate(messages, model_=None, temperature=0, max_tokens=768, top_p=1.0, context_length=0, stop_token=None, **kw): system = next((m["content"] for m in messages if m["role"] == "system"), None) user = "\n".join(m["content"] for m in messages if m["role"] == "user") for _ in range(3): out = call_llm(user, system=system, max_tokens=256, temperature=0, model=model) if out: return out return "" hf.generate_from_openai_chat_completion = lambda messages, model=None, **kw: generate(messages, **kw) # ── one instance's worker ───────────────────────────────────────────────────── async def run_task(cfg, args, out_dir, states): from browser_use import Agent t0 = time.time() tid = cfg["task_id"] res = {"task_id": tid, "sites": cfg["sites"], "arm": args.arm, "instance": args.instance, "score": 0.0} ep = out_dir / str(tid) ep.mkdir(parents=True, exist_ok=True) bs = H.make_session(allowed_domains=(wa_host(), "localhost", "127.0.0.1")) try: await bs.start() await H.enable_clipboard(bs) await set_cookies(bs, [states[s] for s in cfg["sites"] if s in states]) await H.goto(bs, cfg["start_url"]) llm = H.make_llm(args.model, base_url=args.vllm, temperature=args.temperature) if args.router == "planner_agent": hist = await asyncio.wait_for(agent_run(bs, cfg, args, ep), timeout=args.timeout) else: agent = Agent(task=H.instruction(cfg["intent"], cfg["start_url"]), llm=llm, browser_session=bs, tools=H.make_tools(), max_failures=3, **H.AGENT_KWARGS) H.use_normalized_coordinates(bs) hist = await asyncio.wait_for(agent.run(max_steps=args.max_steps or H.MAX_STEPS, on_step_start=lambda a: H.enable_clipboard(bs, a)), timeout=args.timeout) answer = hist.final_result() or "" final_url = await bs.get_current_page_url() hist.save_to_file(ep / "history.json") from miniwebagent.eval_viewer import copy_screenshots copy_screenshots(ep) # out of browser-use's /tmp dir, for miniwebagent.eval_viewer res.update(answer=answer, final_url=final_url, steps=len(hist.history), done=hist.is_done(), errors=[e for e in hist.errors() if e][:5]) res["score"], res["parts"] = await asyncio.to_thread(evaluate_sync, bs.cdp_url, cfg, answer, final_url) res["judge"] = args.judge except asyncio.TimeoutError: res["error"] = "timeout" except Exception as e: res["error"] = f"{type(e).__name__}: {str(e)[:300]}" finally: res["duration_s"] = round(time.time() - t0, 1) try: await bs.kill() except Exception: pass (ep / "result.json").write_text(json.dumps(res, indent=1, default=str)) return res async def agent_run(bs, cfg, args, ep): """The planner as a browser-use Agent that delegates each step to a family specialist (miniwebagent.planner_agent).""" from types import SimpleNamespace from miniwebagent.evaluate import sub_model from miniwebagent.planner_agent import run_planner_agent margs = SimpleNamespace(adapters=set(args.adapters.split(",")) - {"none", ""}, fallback=args.fallback, base_name=args.model, reasoning_to_fallback=False) hist, trace, ph = await run_planner_agent(bs, cfg["intent"], cfg["start_url"], lambda f: sub_model(f, margs), args.vllm, args.planner_name, sub_cap=args.sub_cap, max_turns=args.planner_turns, temperature=args.temperature, version=args.agent_version, planner_url=args.planner_vllm) ph.save_to_file(ep / "planner_history.json") (ep / "subtasks.json").write_text(json.dumps({"trace": trace}, indent=1, default=str, ensure_ascii=False)) return hist def worker(args): """Run this instance's share of the tasks, in order.""" _point_env_config(args.instance) use_judge(args.judge) out_dir = Path(args.out) states = make_login_states(args.instance, out_dir / "_auth" / f"instance_{args.instance}") keep = set(json.loads(SPLIT.read_text())[args.split]) if args.split != "all" else None tasks = [t for t in load_tasks() if (not args.task_ids or t["task_id"] in args.task_ids) and (keep is None or t["task_id"] in keep)] tasks = [t for k, t in enumerate(tasks) if k % args.n_instances == args.slot] # this instance's share if args.limit: tasks = tasks[:args.limit] done = set() res_path = out_dir / "results.jsonl" if res_path.exists(): done = {json.loads(line)["task_id"] for line in res_path.read_text().split("\n") if line.strip()} for t in tasks: if t["task_id"] in done: continue # a fresh login per task, as WebArena's own harness does: the saved states are server-side sessions, so one # episode that signs out (2026-09-30: task 40) logged out every later task on that instance need = [s for s in t["sites"] if s in LOGIN_SITES] if need: states.update(make_login_states(args.instance, out_dir / "_auth" / f"instance_{args.instance}", need)) res = asyncio.run(run_task(localize(t, args.instance), args, out_dir, states)) with open(res_path, "a") as f: f.write(json.dumps(res, default=str) + "\n") print(f"[inst {args.instance}] task {t['task_id']} ({'+'.join(t['sites'])}) score {res['score']} " f"{res.get('steps')} steps {res['duration_s']}s {res.get('error') or ''}", flush=True) def main(argv=None): ap = argparse.ArgumentParser(prog="python -m miniwebagent.wa") ap.add_argument("--arm", default="base", help="label for the run (the served model is --model)") ap.add_argument("--model", default="qwen35-4b", help="served model name: the base, or a LoRA adapter's name") ap.add_argument("--vllm", default="http://127.0.0.1:8400/v1") ap.add_argument("--temperature", type=float, default=0.0) ap.add_argument("--timeout", type=int, default=900) ap.add_argument("--max-steps", type=int, default=0, help="single agent: step cap (default harness.MAX_STEPS = 40; review round 1, A3, 2026-10-05)") # instances 1-2 are ours; 3-4 belong to the collaborator's runs (user, 2026-09-29: 50/50 by instance) ap.add_argument("--instances", type=lambda s: [int(x) for x in s.split(",")], default=[1, 2]) ap.add_argument("--task-ids", type=lambda s: {int(x) for x in s.split(",")}, default=None) ap.add_argument("--limit", type=int, default=0) ap.add_argument("--split", default="all", choices=["dev", "test", "all"], help="WebArena-Lite-v2 half (data/webmix/wa/split_5050.json): dev for development runs, test only for " "the final comparison") ap.add_argument("--out", required=True) ap.add_argument("--router", default="none", choices=["none", "planner_agent"], help="planner_agent: the planner agent (miniwebagent.planner_agent) delegating to family specialists; " "none: one agent with --model") ap.add_argument("--planner-name", default="qwen35-4b", help="planner_agent: the planner's served model name") ap.add_argument("--planner-vllm", default=None, help="planner_agent: the planner's own server (default --vllm; e.g. a 9B planner over 4B specialists)") ap.add_argument("--planner-turns", type=int, default=12, help="planner_agent: the planner agent's own turns") ap.add_argument("--agent-version", default="v1", help="planner_agent: version (v1, v2, ...)") ap.add_argument("--adapters", default="drag,io,select,rest", help="planner_agent: family adapters served") ap.add_argument("--fallback", default="pooled_v4", help="planner_agent: a family without its adapter") ap.add_argument("--sub-cap", type=int, default=15, help="planner_agent: steps per delegated step (of the 40)") ap.add_argument("--judge", default="gemini-3.5-flash-lite", help="fuzzy-match judge (user, 2026-09-28: flash-lite for our validation runs; the benchmark's own " "is gpt-4o-2024-11-20)") ap.add_argument("--regrade", action="store_true", help="re-score an existing run's answers with --judge") # internal: one instance worker ap.add_argument("--instance", type=int, default=None) ap.add_argument("--slot", type=int, default=0) ap.add_argument("--n-instances", type=int, default=1) args = ap.parse_args(argv) _load_openai_env() if args.regrade: return regrade(Path(args.out), args.judge) if args.instance is not None: return worker(args) Path(args.out).mkdir(parents=True, exist_ok=True) procs = [] for slot, i in enumerate(args.instances): cmd = [sys.executable, "-m", "miniwebagent.wa", "--instance", str(i), "--slot", str(slot), "--n-instances", str(len(args.instances)), "--arm", args.arm, "--model", args.model, "--vllm", args.vllm, "--temperature", str(args.temperature), "--timeout", str(args.timeout), "--out", args.out, "--judge", args.judge, "--split", args.split, "--router", args.router, "--planner-name", args.planner_name, "--adapters", args.adapters, "--fallback", args.fallback, "--sub-cap", str(args.sub_cap)] cmd += ["--planner-turns", str(args.planner_turns), "--agent-version", args.agent_version] if args.planner_vllm: cmd += ["--planner-vllm", args.planner_vllm] if args.max_steps: cmd += ["--max-steps", str(args.max_steps)] if args.limit: cmd += ["--limit", str(args.limit)] if args.task_ids: cmd += ["--task-ids", ",".join(map(str, sorted(args.task_ids)))] log = open(Path(args.out) / f"instance_{i}.log", "a") procs.append(subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, cwd=str(ROOT))) for p in procs: p.wait() summarize(Path(args.out)) def _load_openai_env(): """The fuzzy-match judge reads OPENAI_API_KEY / OPENAI_BASE_URL (the repo's .env holds the key).""" envf = ROOT / ".env" if "OPENAI_API_KEY" not in os.environ and envf.exists(): for line in envf.read_text().split("\n"): if line.startswith("OPENAI_API_KEY="): os.environ["OPENAI_API_KEY"] = line.split("=", 1)[1].strip().strip('"') os.environ.setdefault("OPENAI_BASE_URL", "https://api.openai.com/v1") def regrade(out_dir, judge): """Re-score a run's fuzzy-match tasks with `judge` from the stored answers (their other evaluator, url_match, is recomputed from the stored final URL). Writes results_.jsonl + summary.""" _point_env_config(1) use_judge(judge) from envs.web.webarena.evaluation.vab_helper_functions import PseudoPage cfgs = {t["task_id"]: t for t in load_tasks()} rows = [json.loads(line) for line in (out_dir / "results.jsonl").read_text().split("\n") if line.strip()] out = [] for r in rows: cfg = localize(cfgs[r["task_id"]], r["instance"]) if "fuzzy_match" in (cfg["eval"].get("reference_answers") or {}) and "error" not in r: r = {**r, "score_before": r["score"], "judge_before": r.get("judge", "gpt-4o-2024-11-20")} r["score"], r["parts"] = score_parts(cfg, r.get("answer") or "", PseudoPage(None, r.get("final_url") or "")) r["judge"] = judge out.append(r) name = f"results_{judge}.jsonl" (out_dir / name).write_text("".join(json.dumps(r, default=str) + "\n" for r in out)) changed = sum(1 for r in out if "score_before" in r and r["score_before"] != r["score"]) print(f"regraded {sum('score_before' in r for r in out)} fuzzy tasks with {judge}; {changed} changed") summarize(out_dir, name) def summarize(out_dir, name="results.jsonl"): rows = [json.loads(line) for line in (out_dir / name).read_text().split("\n") if line.strip()] by = {} for r in rows: s = "+".join(r["sites"]) by.setdefault(s, []).append(r["score"]) total = sum(r["score"] for r in rows) lines = [f"{len(rows)} tasks, success {total:.0f} ({100 * total / max(1, len(rows)):.1f}%)"] lines += [f" {s}: {sum(v):.0f}/{len(v)} ({100 * sum(v) / len(v):.1f}%)" for s, v in sorted(by.items())] (out_dir / ("summary.txt" if name == "results.jsonl" else f"summary_{name[8:-6]}.txt")).write_text("\n".join(lines) + "\n") print("\n".join(lines)) if __name__ == "__main__": main()