MiniWeb-demo / scripts /paper.yaml
sparklabuofu's picture
Update the demo
22de727 verified
Raw History Blame Contribute Delete
19.9 kB
# How every number in the paper is produced (read by scripts/reproduce.py).
#
# Each arm lists the run folders the paper reads (one list per benchmark, one run per seed; the paper reports the mean
# over the list) and the paper's numbers, so `python scripts/reproduce.py table` sets a reproduction next to them. This
# file says how to make each run: the model server, the agent configuration on each benchmark, the grading, and the
# trained adapters. `python scripts/reproduce.py status` checks every run; `python scripts/reproduce.py show <arm>`
# prints its commands. (The figure scripts' own configs, not part of the public repository, list the same runs.)
paper_dir: docs/paper/MiniWeb # LaTeX source; `scripts/reproduce.py figures` copies generated figures/tables into it
# ── model server ─────────────────────────────────────────────────────────────────────────────────────────────────────
# src/miniwebagent/scripts/serve.sh (vLLM, the webmix env). One Qwen3.5-4B server with every adapter an arm needs loaded as a LoRA.
server:
model: Qwen/Qwen3.5-4B
served: qwen35-4b
port: 8400
env: {MAX_LORA_RANK: 64, MAX_LORAS: 8}
# ── benchmarks ──────────────────────────────────────────────────────────────────────────────────────────────────────
# n: tasks per run. MiniWeb needs the app on ports 8300-8320 (the runner starts them); WebArena-Lite needs the four
# WebArena instances and src/miniwebagent/scripts/webarena/ready.sh; live web needs only a browser and the judges' API keys.
benchmarks:
single: {kind: miniweb, n: 103, primitives: final_set, runner: "-m miniwebagent.evaluate --tasks data/webmix/valgen/val_v2.jsonl",
about: "generated single-primitive tasks of the 13 held-out sites (103 of 115 are of well-covered primitives)"}
heldout: {kind: miniweb, n: 63, runner: "-m miniwebagent.evaluate --human",
about: "human tasks of the 13 held-out sites (data/datagen/site_split.json)"}
wa: {kind: webarena, n: 154, runner: "-m miniwebagent.wa --split all --instances 1,2,3,4",
about: "WebArena-Lite (ScaleCUA's 154-task release), benchmark evaluators; fuzzy matches by gemini-3.5-flash-lite"}
wv: {kind: webvoyager, n: 536, runner: "-m miniwebagent.om2w --bench webvoyager --tasks scripts/benchmarks/WebVoyager_reduced_536.jsonl", tasks: scripts/benchmarks/WebVoyager_reduced_536.jsonl,
judge: "-m miniwebagent.om2w_judge {run} --bench webvoyager --workers 4", judged: "{run}_wvjudge",
about: "WebVoyager, normalized set (536 of 643: time-sensitive dates moved, unreachable sites dropped)"}
om2w: {kind: om2w, n: 280, runner: "-m miniwebagent.om2w --bench om2w --tasks scripts/benchmarks/Online_Mind2Web_reduced_280.json", tasks: scripts/benchmarks/Online_Mind2Web_reduced_280.json,
judge: "-m miniwebagent.om2w_judge {run} --bench om2w --workers 4", judged: "{run}_webjudge",
about: "Online-Mind2Web, normalized set (280 of 300), WebJudge (o4-mini)"}
# ── agent configurations ────────────────────────────────────────────────────────────────────────────────────────────
# A single agent acts with one model; the planner (untrained Qwen3.5-4B, agent v6) hands each primitive-level step to
# an executor. {executor} is the served LoRA name (or qwen35-4b for the untrained model). Arguments per benchmark kind:
configs:
single:
miniweb: "--arm {arm_flag} --verifier-only --workers 6" # arm_flag: base, or pooled --pooled-name <lora>
webarena: "--arm {name} --model {executor}"
live: "--arm base --model {executor} --timeout 14400 --max-steps 100"
planner:
miniweb: "--arm agent_planner --agent-version v6 --timeout 2400 --planner-name qwen35-4b --adapters {adapters} --fallback {executor} --verifier-only --workers 6"
webarena: "--router planner_agent --agent-version v6 --timeout 2400 --planner-name qwen35-4b --adapters {adapters} --fallback {executor} --arm {name}"
live: "--arm agent --agent-version v6 --timeout 14400 --max-steps 100 --planner-turns 60 --planner-name qwen35-4b --adapters {adapters} --fallback {executor}"
# ── arms: the paper's agents (names as in tables_config.yaml `runs:` where they appear there) ───────────────────────
arms:
base: {config: single, executor: qwen35-4b, label: "Single agent, untrained",
paper: {single: 55.3, heldout: 37.0, wa: 27.1, wv: 45.5, om2w: 17.9},
runs: {single: [data/webmix/eval/val2_rq2_base, data/webmix/eval/val2_rq2_base_s2, data/webmix/eval/val2_rq2_base_s3],
heldout: [data/webmix/eval/human_base, data/webmix/eval/seeds/base_s2, data/webmix/eval/seeds/base_s3],
wa: [data/webmix/wa/seeds/base_s1, data/webmix/wa/seeds/base_s2, data/webmix/wa/seeds/base_s3],
wv: [data/webmix/om2w/runs/granite/webvoyager100_base_wvjudge],
om2w: [data/webmix/om2w/runs/granite/om2w100_base_webjudge]}}
pooled: {config: single, executor: pooled_v4, label: "Single agent, trained",
paper: {single: 80.3, heldout: 37.0, wa: 20.3, wv: 32.8, om2w: 8.6},
runs: {single: [data/webmix/eval/val2_rq2_pooled, data/webmix/eval/val2_rq2_pooled_s2, data/webmix/eval/val2_rq2_pooled_s3],
heldout: [data/webmix/eval/human_pooled_v4, data/webmix/eval/seeds/pooled_v4_s2, data/webmix/eval/seeds/pooled_v4_s3],
wa: [data/webmix/wa/seeds/single_pooled_s1, data/webmix/wa/seeds/single_pooled_s2, data/webmix/wa/seeds/single_pooled_s3],
wv: [data/webmix/om2w/runs/granite/webvoyager100_single_pooled_wvjudge],
om2w: [data/webmix/om2w/runs/granite/om2w100_single_pooled_webjudge]}}
planner: {config: planner, executor: qwen35-4b, label: "Planner, untrained executor",
paper: {single: 54.0, heldout: 42.3, wa: 30.7, wv: 59.9, om2w: 21.8},
runs: {single: [data/webmix/eval/val2_rq2_planner_base, data/webmix/eval/val2_rq2_planner_base_s2, data/webmix/eval/val2_rq2_planner_base_s3],
heldout: [data/webmix/eval/seeds/pagent_v6_baseexec_s1, data/webmix/eval/seeds/pagent_v6_baseexec_s2, data/webmix/eval/seeds/pagent_v6_baseexec_s3],
wa: [data/webmix/wa/seeds/agent_base_s1, data/webmix/wa/seeds/agent_base_s2, data/webmix/wa/seeds/agent_base_s3],
wv: [data/webmix/om2w/runs/granite/webvoyager100_agent_planner_base_wvjudge],
om2w: [data/webmix/om2w/runs/granite/om2w100_agent_planner_base_webjudge]}}
planner_pooled: {config: planner, executor: pooled_v4, label: "MiniWebAgent (planner + trained executor)",
paper: {single: 71.5, heldout: 54.5, wa: 36.4, wv: 65.5, om2w: 33.9},
runs: {single: [data/webmix/eval/val2_pagent_pooled_s1, data/webmix/eval/val2_pagent_pooled_s2, data/webmix/eval/val2_pagent_pooled_s3],
heldout: [data/webmix/eval/pagent_pooled_s1, data/webmix/eval/pagent_pooled_s2, data/webmix/eval/pagent_pooled_s3],
wa: [data/webmix/wa/seeds/agent_pooled31k_s1, data/webmix/wa/seeds/agent_pooled31k_s2, data/webmix/wa/seeds/agent_pooled31k_s3],
wv: [data/webmix/om2w/runs/granite/webvoyager100_agent_pooled31k_wvjudge],
om2w: [data/webmix/om2w/runs/granite/om2w100_agent_pooled31k_webjudge]}}
taskcentric: {config: planner, executor: taskgen_pooled, label: "Planner + executor trained on task-centric trajectories (Tab. 3)",
paper: {heldout: 45.5, wa: 34.2, wv: 62.1, om2w: 23.2},
runs: {heldout: [data/webmix/eval/pagent_taskgen_s1, data/webmix/eval/pagent_taskgen_s2, data/webmix/eval/pagent_taskgen_s3],
wa: [data/webmix/wa/seeds/agent_taskgen_s1, data/webmix/wa/seeds/agent_taskgen_s2, data/webmix/wa/seeds/agent_taskgen_s3],
wv: [data/webmix/om2w/runs/granite/webvoyager100_agent_taskgen_wvjudge],
om2w: [data/webmix/om2w/runs/granite/om2w100_agent_taskgen_webjudge]}}
agent: {config: planner, executor: pooled_v4, adapters: "drag,io,select,rest,reasoning",
label: "Planner + family specialists (Supp.)", server_loras: [drag=drag_v5b, io=io_v5b, select=select_v5b, rest=rest_v5b, reasoning=reasoning_v4],
paper: {single: 70.9, heldout: 51.3, wa: 34.2, wv: 69.2, om2w: 26.1},
runs: {single: [data/webmix/eval/val2_rq2_planner_v5, data/webmix/eval/val2_rq2_planner_v5_s2, data/webmix/eval/val2_rq2_planner_v5_s3],
heldout: [data/webmix/eval/seeds/pagent_v6_v5b_s1, data/webmix/eval/seeds/pagent_v6_v5b_s2, data/webmix/eval/seeds/pagent_v6_v5b_s3],
wa: [data/webmix/wa/seeds/agent_v5b_s1, data/webmix/wa/seeds/agent_v5b_s2, data/webmix/wa/seeds/agent_v5b_s3],
wv: [data/webmix/om2w/runs/granite/webvoyager100_agent_v5bspec_wvjudge],
om2w: [data/webmix/om2w/runs/granite/om2w100_agent_v5bspec_webjudge]}}
nodone: {config: single, executor: pooled_nodone, label: "Stop-early control: trained without final done steps",
paper: {heldout: 33.9, wa: 22.5},
runs: {heldout: [data/webmix/eval/r1_control_nodone_s1, data/webmix/eval/r1_control_nodone_s2, data/webmix/eval/r1_control_nodone_s3],
wa: [data/webmix/wa/seeds/nodone_s1, data/webmix/wa/seeds/nodone_s2, data/webmix/wa/seeds/nodone_s3]}}
# Protocol deviations, kept as they ran (the paper or its supplement states each):
deviations:
- "WebVoyager, trained single agent (pooled, wv): 40 steps and 1,800 s instead of 100 steps and 14,400 s."
- "WebArena-Lite single-agent arms ran at the default --timeout 900 s; planner arms at 2,400 s."
- "Held-out, untrained single agent run 1 used --workers 4 (runs 2-3: 6)."
- "Online-Mind2Web base started at --timeout 3600 on 300 tasks and resumed at 14,400 s; scored on the 280-task set."
- "Live-web runs ended by model-server timeouts were set aside and rerun (src/miniwebagent/scripts/cluster/retry_timeouts.py); base runs were not affected."
# ── RQ1: ten untrained agents on all 376 tasks (Tab. 2, Fig. 5-6) ───────────────────────────────────────────────────
# Run folder data/webmix/eval/rq1_<served>_all; graded by the task verifiers (results.jsonl, regraded with the flash-lite
# answer tier) and the VLM judge (judge.jsonl: python -m miniweb.evaluation.judge_runs --runs <run> --parallel 4).
rq1:
browser_use: # python -m miniwebagent.evaluate --human --human-split all --arm base --base-name <served> --verifier-only
- {served: qwen35-4b, model: Qwen/Qwen3.5-4B, run: rq1_qwen35_4b_all, workers: 6}
- {served: qwen35-9b, model: Qwen/Qwen3.5-9B, run: rq1_qwen35_9b_all, workers: 12}
- {served: qwen3vl-8b, model: Qwen/Qwen3-VL-8B-Instruct, run: rq1_qwen3vl-8b_all, workers: 12, timeout: 1800,
server_env: {NO_LORA: 1, CHAT_KWARGS: none, EXTRA_ARGS: '--structured-outputs-config {"backend":"xgrammar","disable_any_whitespace":true}'}}
- {served: internvl35-8b, model: OpenGVLab/InternVL3_5-8B, run: rq1_internvl35-8b_all, workers: 12, timeout: 1800,
server_env: {NO_LORA: 1, CHAT_KWARGS: none, EXTRA_ARGS: '--trust-remote-code --structured-outputs-config {"backend":"xgrammar","disable_any_whitespace":true}'}}
own_loop: # python -m miniweb.evaluation.fara_rq1 prep; the agent's released loop; python -m miniweb.evaluation.fara_rq1 grade --arm <served>
- {served: fara15-4b, model: microsoft/Fara1.5-4B, run: rq1_fara15_4b_all, loop: "src/miniweb/evaluation/fara_runner.py (fara venv)"}
- {served: fara-7b, model: microsoft/Fara-7B, run: rq1_fara-7b_all, loop: "ownloop_runner fara7b", server_env: {MM_LIMIT: '{"image": 3}'}}
- {served: uitars15-7b, model: ByteDance-Seed/UI-TARS-1.5-7B, run: rq1_uitars15-7b_all, loop: "ownloop_runner uitars"}
- {served: molmoweb-4b, model: allenai/MolmoWeb-4B, run: rq1_molmoweb-4b_all, loop: "ownloop_runner molmoweb"}
- {served: molmoweb-8b, model: allenai/MolmoWeb-8B, run: rq1_molmoweb-8b_all, loop: "ownloop_runner molmoweb"}
- {served: opencua-7b, model: xlangai/OpenCUA-7B, run: rq1_opencua-7b_all, loop: "ownloop_runner opencua (src/miniwebagent/scripts/cluster/ownloop.sh: tokenizer + vLLM plugin)"}
# ── trained adapters ────────────────────────────────────────────────────────────────────────────────────────────────
# torchrun --standalone --nproc_per_node <gpus> -m miniwebagent.train --out data/webmix/adapters/<name> <args>, with
# WEBMIX_REPLAY_DIR = a folder of the replayed episodes listed in <pool> (scripts/reproduce.py train builds it as symlinks).
# Recipe for all: LoRA rank 64, alpha 128, lr 1e-4, one epoch, a global batch of 8 rows (--grad-accum is global:
# miniwebagent.train divides it by the GPU count, so the GPU count changes speed, not the recipe), no KL anchor (beta 0).
recipe: "--adapter pooled --epochs 1 --lr 1e-4 --rank 64 --alpha 128 --grad-accum 8 --beta 0 --seed 0"
adapters:
pooled_v4: {pool: scripts/pools/pool3k_dirs.txt, gpus: 4, rows: 27646,
about: "MiniWebAgent's executor: the 27.6K-step set (3,128 generated trajectories + 857 chains + 629 recoveries)"}
pooled_nodone: {pool: scripts/pools/pool3k_dirs.txt, gpus: 4, args: "--drop-done", rows: 23288,
about: "review control: the 27.6K-step set without each episode's final done step"}
taskgen_pooled: {rows_dir: data/webmix/taskgen/rows_27k, gpus: 4, rows: 27644,
about: "the task-centric executor of Tab. 3: InSTA's pipeline on the 52 training sites (data.taskgen), 3,795 trajectories"}
# ── data: MiniWebGen and the training set ───────────────────────────────────────────────────────────────────────────
# The paper's pool is fixed by scripts/pools/pool3k_plain_ids.txt (3,128 generated trajectories) and
# pool3k_dirs.txt (the 4,614 replayed episodes: plain + chain + recovery). Regenerating draws a new pool with the same
# recipe; the numbers in the paper come from the listed one.
data:
generate: "python -m miniwebgen run --run-id <id> --macros <primitives> --target <n> --workers 12 --servers 6 --attempts 3 --no-judge"
runs: [kinds2, slice1, slice2, slice3, all46, all46b, scale50a, scale50b, scale50c, scale50d, scale50e, scale50f, scale50g, scale100a]
retired: data/datagen/retired.json # trajectories dropped after review (excluded from replay)
replay: "python -m miniwebgen replay --shard k/4 --memory-model qwen35-4b # k = 0..3, then once more to retry failures"
augment:
- "python -m miniwebgen replay --augment recovery --share 0.35"
- "python -m miniwebgen replay --augment chain --per-page 3"
valset: "python src/miniwebgen/scripts/make_valset.py # the 115 generated single-primitive test tasks of the held-out sites"
taskgen: "python -m miniwebgen.taskgen propose | judge | rows # the task-centric comparison data (attempts: miniwebagent.evaluate --tasks)"
release: "python scripts/build_release.py # the consolidated, split-labelled, anonymized data folder"
# scripts/generate_data.py prints (and with --run runs) all of the above in order, each run with its recorded arguments.
# ── figures and tables, in build order ──────────────────────────────────────────────────────────────────────────────
# Each generator writes into docs/figures/<dir>/ (tracked); `to` is the destination inside paper_dir.
figures:
- {cmd: docs/figures/teaser/make_teaser_v2.py --variant radar, copy: {docs/figures/teaser/teaser_v2_radar.pdf: figures/teaser.pdf}}
- {cmd: docs/figures/sites/make_sites.py, copy: {docs/figures/sites/sites.pdf: figures/sites.pdf}}
- {cmd: docs/figures/method/make_method.py, copy: {docs/figures/method/method.pdf: figures/method.pdf}}
- {cmd: docs/figures/construction/make_construction.py, copy: {docs/figures/construction/construction.pdf: figures/construction.pdf}}
- {cmd: docs/figures/rq1/make_rq1_table.py, copy: {docs/figures/rq1/rq1_table.tex: tables/rq1_table.tex}}
- {cmd: docs/figures/rq1/drag_failures.py, copy: {docs/figures/rq1/drag_failures.pdf: figures/drag_failures.pdf}}
- {cmd: docs/figures/chain/make_chain.py, copy: {docs/figures/chain/chain_length.pdf: figures/chain_length.pdf}}
- {cmd: docs/figures/rq3/make_results_row.py, copy: {docs/figures/rq3/results_row.pdf: figures/results_row.pdf}} # Fig. 7: practice per skill family
- {cmd: docs/figures/rq3/make_rq3_table.py, copy: {docs/figures/rq3/rq3_table.tex: tables/rq3_transfer.tex}} # Tab. 3: the four benchmarks
- {cmd: docs/figures/supp/make_supp.py, copy: {docs/figures/supp/supp_primitives.tex: tables/supp_primitives.tex,
docs/figures/supp/supp_ops.tex: tables/supp_ops.tex,
docs/figures/supp/supp_gallery.tex: tables/supp_gallery.tex,
docs/figures/supp/figures/sites: figures/sites,
docs/figures/supp/figures/annotation: figures/annotation}}
- {cmd: docs/figures/rq1/text_numbers.py} # Sec. 4.1's text: pooled drag CI, group means, Spearman rho
- {cmd: docs/figures/review/round1_numbers.py}
- {cmd: docs/figures/review/compute_budget.py}
- {cmd: docs/figures/review/make_round1_supp.py --out docs/figures/review/supp_results.tex,
copy: {docs/figures/review/supp_results.tex: sections/supp/supp_results.tex}}
- {cmd: docs/figures/mining/mining_stats.py} # Supp. A: the mining stage, from the brainstorm sheet
- {cmd: docs/figures/mining/make_mining_supp.py --out docs/figures/mining/supp_mining.tex,
copy: {docs/figures/mining/supp_mining.tex: sections/supp/supp_mining.tex}}
- {cmd: docs/figures/review/make_graders_supp.py --out docs/figures/review/supp_graders.tex, # Supp. grader validation
copy: {docs/figures/review/supp_graders.tex: sections/supp/supp_graders.tex}}
# Analyses quoted in the text (Sec. 3 grader validity; review round 1 grader audit):
analyses:
grader_agreement: "python -m miniweb.evaluation.grader_agreement --parallel 4 # data/webmix/eval/grader_agreement.json"
grader_audit_sample: "python src/miniweb/scripts/build_grader_audit.py # data/grader_audit/{cases,key.json}"
grader_audit_score: "python src/miniweb/scripts/grader_audit_report.py --url <deployment> # data/grader_audit/report.json"