Spaces:
Running
Running
Download scripts/paper.yaml from sparklabuofu/MiniWeb-demo: direct link, hf CLI and curl.
- Browser
- Download file 19.9 kB
-
https://huggingface.co/spaces/sparklabuofu/MiniWeb-demo/resolve/main/scripts/paper.yaml
- Command line
-
hf download hf://spaces/sparklabuofu/MiniWeb-demo/scripts/paper.yaml
-
curl -L -o paper.yaml https://huggingface.co/spaces/sparklabuofu/MiniWeb-demo/resolve/main/scripts/paper.yaml
19.9 kB
| # How every number in the paper is produced (read by scripts/reproduce.py). | |
| # | |
| # Each arm lists the run folders the paper reads (one list per benchmark, one run per seed; the paper reports the mean | |
| # over the list) and the paper's numbers, so `python scripts/reproduce.py table` sets a reproduction next to them. This | |
| # file says how to make each run: the model server, the agent configuration on each benchmark, the grading, and the | |
| # trained adapters. `python scripts/reproduce.py status` checks every run; `python scripts/reproduce.py show <arm>` | |
| # prints its commands. (The figure scripts' own configs, not part of the public repository, list the same runs.) | |
| paper_dir: docs/paper/MiniWeb # LaTeX source; `scripts/reproduce.py figures` copies generated figures/tables into it | |
| # ββ model server βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # src/miniwebagent/scripts/serve.sh (vLLM, the webmix env). One Qwen3.5-4B server with every adapter an arm needs loaded as a LoRA. | |
| server: | |
| model: Qwen/Qwen3.5-4B | |
| served: qwen35-4b | |
| port: 8400 | |
| env: {MAX_LORA_RANK: 64, MAX_LORAS: 8} | |
| # ββ benchmarks ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # n: tasks per run. MiniWeb needs the app on ports 8300-8320 (the runner starts them); WebArena-Lite needs the four | |
| # WebArena instances and src/miniwebagent/scripts/webarena/ready.sh; live web needs only a browser and the judges' API keys. | |
| benchmarks: | |
| single: {kind: miniweb, n: 103, primitives: final_set, runner: "-m miniwebagent.evaluate --tasks data/webmix/valgen/val_v2.jsonl", | |
| about: "generated single-primitive tasks of the 13 held-out sites (103 of 115 are of well-covered primitives)"} | |
| heldout: {kind: miniweb, n: 63, runner: "-m miniwebagent.evaluate --human", | |
| about: "human tasks of the 13 held-out sites (data/datagen/site_split.json)"} | |
| wa: {kind: webarena, n: 154, runner: "-m miniwebagent.wa --split all --instances 1,2,3,4", | |
| about: "WebArena-Lite (ScaleCUA's 154-task release), benchmark evaluators; fuzzy matches by gemini-3.5-flash-lite"} | |
| wv: {kind: webvoyager, n: 536, runner: "-m miniwebagent.om2w --bench webvoyager --tasks scripts/benchmarks/WebVoyager_reduced_536.jsonl", tasks: scripts/benchmarks/WebVoyager_reduced_536.jsonl, | |
| judge: "-m miniwebagent.om2w_judge {run} --bench webvoyager --workers 4", judged: "{run}_wvjudge", | |
| about: "WebVoyager, normalized set (536 of 643: time-sensitive dates moved, unreachable sites dropped)"} | |
| om2w: {kind: om2w, n: 280, runner: "-m miniwebagent.om2w --bench om2w --tasks scripts/benchmarks/Online_Mind2Web_reduced_280.json", tasks: scripts/benchmarks/Online_Mind2Web_reduced_280.json, | |
| judge: "-m miniwebagent.om2w_judge {run} --bench om2w --workers 4", judged: "{run}_webjudge", | |
| about: "Online-Mind2Web, normalized set (280 of 300), WebJudge (o4-mini)"} | |
| # ββ agent configurations ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # A single agent acts with one model; the planner (untrained Qwen3.5-4B, agent v6) hands each primitive-level step to | |
| # an executor. {executor} is the served LoRA name (or qwen35-4b for the untrained model). Arguments per benchmark kind: | |
| configs: | |
| single: | |
| miniweb: "--arm {arm_flag} --verifier-only --workers 6" # arm_flag: base, or pooled --pooled-name <lora> | |
| webarena: "--arm {name} --model {executor}" | |
| live: "--arm base --model {executor} --timeout 14400 --max-steps 100" | |
| planner: | |
| miniweb: "--arm agent_planner --agent-version v6 --timeout 2400 --planner-name qwen35-4b --adapters {adapters} --fallback {executor} --verifier-only --workers 6" | |
| webarena: "--router planner_agent --agent-version v6 --timeout 2400 --planner-name qwen35-4b --adapters {adapters} --fallback {executor} --arm {name}" | |
| live: "--arm agent --agent-version v6 --timeout 14400 --max-steps 100 --planner-turns 60 --planner-name qwen35-4b --adapters {adapters} --fallback {executor}" | |
| # ββ arms: the paper's agents (names as in tables_config.yaml `runs:` where they appear there) βββββββββββββββββββββββ | |
| arms: | |
| base: {config: single, executor: qwen35-4b, label: "Single agent, untrained", | |
| paper: {single: 55.3, heldout: 37.0, wa: 27.1, wv: 45.5, om2w: 17.9}, | |
| runs: {single: [data/webmix/eval/val2_rq2_base, data/webmix/eval/val2_rq2_base_s2, data/webmix/eval/val2_rq2_base_s3], | |
| heldout: [data/webmix/eval/human_base, data/webmix/eval/seeds/base_s2, data/webmix/eval/seeds/base_s3], | |
| wa: [data/webmix/wa/seeds/base_s1, data/webmix/wa/seeds/base_s2, data/webmix/wa/seeds/base_s3], | |
| wv: [data/webmix/om2w/runs/granite/webvoyager100_base_wvjudge], | |
| om2w: [data/webmix/om2w/runs/granite/om2w100_base_webjudge]}} | |
| pooled: {config: single, executor: pooled_v4, label: "Single agent, trained", | |
| paper: {single: 80.3, heldout: 37.0, wa: 20.3, wv: 32.8, om2w: 8.6}, | |
| runs: {single: [data/webmix/eval/val2_rq2_pooled, data/webmix/eval/val2_rq2_pooled_s2, data/webmix/eval/val2_rq2_pooled_s3], | |
| heldout: [data/webmix/eval/human_pooled_v4, data/webmix/eval/seeds/pooled_v4_s2, data/webmix/eval/seeds/pooled_v4_s3], | |
| wa: [data/webmix/wa/seeds/single_pooled_s1, data/webmix/wa/seeds/single_pooled_s2, data/webmix/wa/seeds/single_pooled_s3], | |
| wv: [data/webmix/om2w/runs/granite/webvoyager100_single_pooled_wvjudge], | |
| om2w: [data/webmix/om2w/runs/granite/om2w100_single_pooled_webjudge]}} | |
| planner: {config: planner, executor: qwen35-4b, label: "Planner, untrained executor", | |
| paper: {single: 54.0, heldout: 42.3, wa: 30.7, wv: 59.9, om2w: 21.8}, | |
| runs: {single: [data/webmix/eval/val2_rq2_planner_base, data/webmix/eval/val2_rq2_planner_base_s2, data/webmix/eval/val2_rq2_planner_base_s3], | |
| heldout: [data/webmix/eval/seeds/pagent_v6_baseexec_s1, data/webmix/eval/seeds/pagent_v6_baseexec_s2, data/webmix/eval/seeds/pagent_v6_baseexec_s3], | |
| wa: [data/webmix/wa/seeds/agent_base_s1, data/webmix/wa/seeds/agent_base_s2, data/webmix/wa/seeds/agent_base_s3], | |
| wv: [data/webmix/om2w/runs/granite/webvoyager100_agent_planner_base_wvjudge], | |
| om2w: [data/webmix/om2w/runs/granite/om2w100_agent_planner_base_webjudge]}} | |
| planner_pooled: {config: planner, executor: pooled_v4, label: "MiniWebAgent (planner + trained executor)", | |
| paper: {single: 71.5, heldout: 54.5, wa: 36.4, wv: 65.5, om2w: 33.9}, | |
| runs: {single: [data/webmix/eval/val2_pagent_pooled_s1, data/webmix/eval/val2_pagent_pooled_s2, data/webmix/eval/val2_pagent_pooled_s3], | |
| heldout: [data/webmix/eval/pagent_pooled_s1, data/webmix/eval/pagent_pooled_s2, data/webmix/eval/pagent_pooled_s3], | |
| wa: [data/webmix/wa/seeds/agent_pooled31k_s1, data/webmix/wa/seeds/agent_pooled31k_s2, data/webmix/wa/seeds/agent_pooled31k_s3], | |
| wv: [data/webmix/om2w/runs/granite/webvoyager100_agent_pooled31k_wvjudge], | |
| om2w: [data/webmix/om2w/runs/granite/om2w100_agent_pooled31k_webjudge]}} | |
| taskcentric: {config: planner, executor: taskgen_pooled, label: "Planner + executor trained on task-centric trajectories (Tab. 3)", | |
| paper: {heldout: 45.5, wa: 34.2, wv: 62.1, om2w: 23.2}, | |
| runs: {heldout: [data/webmix/eval/pagent_taskgen_s1, data/webmix/eval/pagent_taskgen_s2, data/webmix/eval/pagent_taskgen_s3], | |
| wa: [data/webmix/wa/seeds/agent_taskgen_s1, data/webmix/wa/seeds/agent_taskgen_s2, data/webmix/wa/seeds/agent_taskgen_s3], | |
| wv: [data/webmix/om2w/runs/granite/webvoyager100_agent_taskgen_wvjudge], | |
| om2w: [data/webmix/om2w/runs/granite/om2w100_agent_taskgen_webjudge]}} | |
| agent: {config: planner, executor: pooled_v4, adapters: "drag,io,select,rest,reasoning", | |
| label: "Planner + family specialists (Supp.)", server_loras: [drag=drag_v5b, io=io_v5b, select=select_v5b, rest=rest_v5b, reasoning=reasoning_v4], | |
| paper: {single: 70.9, heldout: 51.3, wa: 34.2, wv: 69.2, om2w: 26.1}, | |
| runs: {single: [data/webmix/eval/val2_rq2_planner_v5, data/webmix/eval/val2_rq2_planner_v5_s2, data/webmix/eval/val2_rq2_planner_v5_s3], | |
| heldout: [data/webmix/eval/seeds/pagent_v6_v5b_s1, data/webmix/eval/seeds/pagent_v6_v5b_s2, data/webmix/eval/seeds/pagent_v6_v5b_s3], | |
| wa: [data/webmix/wa/seeds/agent_v5b_s1, data/webmix/wa/seeds/agent_v5b_s2, data/webmix/wa/seeds/agent_v5b_s3], | |
| wv: [data/webmix/om2w/runs/granite/webvoyager100_agent_v5bspec_wvjudge], | |
| om2w: [data/webmix/om2w/runs/granite/om2w100_agent_v5bspec_webjudge]}} | |
| nodone: {config: single, executor: pooled_nodone, label: "Stop-early control: trained without final done steps", | |
| paper: {heldout: 33.9, wa: 22.5}, | |
| runs: {heldout: [data/webmix/eval/r1_control_nodone_s1, data/webmix/eval/r1_control_nodone_s2, data/webmix/eval/r1_control_nodone_s3], | |
| wa: [data/webmix/wa/seeds/nodone_s1, data/webmix/wa/seeds/nodone_s2, data/webmix/wa/seeds/nodone_s3]}} | |
| # Protocol deviations, kept as they ran (the paper or its supplement states each): | |
| deviations: | |
| - "WebVoyager, trained single agent (pooled, wv): 40 steps and 1,800 s instead of 100 steps and 14,400 s." | |
| - "WebArena-Lite single-agent arms ran at the default --timeout 900 s; planner arms at 2,400 s." | |
| - "Held-out, untrained single agent run 1 used --workers 4 (runs 2-3: 6)." | |
| - "Online-Mind2Web base started at --timeout 3600 on 300 tasks and resumed at 14,400 s; scored on the 280-task set." | |
| - "Live-web runs ended by model-server timeouts were set aside and rerun (src/miniwebagent/scripts/cluster/retry_timeouts.py); base runs were not affected." | |
| # ββ RQ1: ten untrained agents on all 376 tasks (Tab. 2, Fig. 5-6) βββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Run folder data/webmix/eval/rq1_<served>_all; graded by the task verifiers (results.jsonl, regraded with the flash-lite | |
| # answer tier) and the VLM judge (judge.jsonl: python -m miniweb.evaluation.judge_runs --runs <run> --parallel 4). | |
| rq1: | |
| browser_use: # python -m miniwebagent.evaluate --human --human-split all --arm base --base-name <served> --verifier-only | |
| - {served: qwen35-4b, model: Qwen/Qwen3.5-4B, run: rq1_qwen35_4b_all, workers: 6} | |
| - {served: qwen35-9b, model: Qwen/Qwen3.5-9B, run: rq1_qwen35_9b_all, workers: 12} | |
| - {served: qwen3vl-8b, model: Qwen/Qwen3-VL-8B-Instruct, run: rq1_qwen3vl-8b_all, workers: 12, timeout: 1800, | |
| server_env: {NO_LORA: 1, CHAT_KWARGS: none, EXTRA_ARGS: '--structured-outputs-config {"backend":"xgrammar","disable_any_whitespace":true}'}} | |
| - {served: internvl35-8b, model: OpenGVLab/InternVL3_5-8B, run: rq1_internvl35-8b_all, workers: 12, timeout: 1800, | |
| server_env: {NO_LORA: 1, CHAT_KWARGS: none, EXTRA_ARGS: '--trust-remote-code --structured-outputs-config {"backend":"xgrammar","disable_any_whitespace":true}'}} | |
| own_loop: # python -m miniweb.evaluation.fara_rq1 prep; the agent's released loop; python -m miniweb.evaluation.fara_rq1 grade --arm <served> | |
| - {served: fara15-4b, model: microsoft/Fara1.5-4B, run: rq1_fara15_4b_all, loop: "src/miniweb/evaluation/fara_runner.py (fara venv)"} | |
| - {served: fara-7b, model: microsoft/Fara-7B, run: rq1_fara-7b_all, loop: "ownloop_runner fara7b", server_env: {MM_LIMIT: '{"image": 3}'}} | |
| - {served: uitars15-7b, model: ByteDance-Seed/UI-TARS-1.5-7B, run: rq1_uitars15-7b_all, loop: "ownloop_runner uitars"} | |
| - {served: molmoweb-4b, model: allenai/MolmoWeb-4B, run: rq1_molmoweb-4b_all, loop: "ownloop_runner molmoweb"} | |
| - {served: molmoweb-8b, model: allenai/MolmoWeb-8B, run: rq1_molmoweb-8b_all, loop: "ownloop_runner molmoweb"} | |
| - {served: opencua-7b, model: xlangai/OpenCUA-7B, run: rq1_opencua-7b_all, loop: "ownloop_runner opencua (src/miniwebagent/scripts/cluster/ownloop.sh: tokenizer + vLLM plugin)"} | |
| # ββ trained adapters ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # torchrun --standalone --nproc_per_node <gpus> -m miniwebagent.train --out data/webmix/adapters/<name> <args>, with | |
| # WEBMIX_REPLAY_DIR = a folder of the replayed episodes listed in <pool> (scripts/reproduce.py train builds it as symlinks). | |
| # Recipe for all: LoRA rank 64, alpha 128, lr 1e-4, one epoch, a global batch of 8 rows (--grad-accum is global: | |
| # miniwebagent.train divides it by the GPU count, so the GPU count changes speed, not the recipe), no KL anchor (beta 0). | |
| recipe: "--adapter pooled --epochs 1 --lr 1e-4 --rank 64 --alpha 128 --grad-accum 8 --beta 0 --seed 0" | |
| adapters: | |
| pooled_v4: {pool: scripts/pools/pool3k_dirs.txt, gpus: 4, rows: 27646, | |
| about: "MiniWebAgent's executor: the 27.6K-step set (3,128 generated trajectories + 857 chains + 629 recoveries)"} | |
| pooled_nodone: {pool: scripts/pools/pool3k_dirs.txt, gpus: 4, args: "--drop-done", rows: 23288, | |
| about: "review control: the 27.6K-step set without each episode's final done step"} | |
| taskgen_pooled: {rows_dir: data/webmix/taskgen/rows_27k, gpus: 4, rows: 27644, | |
| about: "the task-centric executor of Tab. 3: InSTA's pipeline on the 52 training sites (data.taskgen), 3,795 trajectories"} | |
| # ββ data: MiniWebGen and the training set βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # The paper's pool is fixed by scripts/pools/pool3k_plain_ids.txt (3,128 generated trajectories) and | |
| # pool3k_dirs.txt (the 4,614 replayed episodes: plain + chain + recovery). Regenerating draws a new pool with the same | |
| # recipe; the numbers in the paper come from the listed one. | |
| data: | |
| generate: "python -m miniwebgen run --run-id <id> --macros <primitives> --target <n> --workers 12 --servers 6 --attempts 3 --no-judge" | |
| runs: [kinds2, slice1, slice2, slice3, all46, all46b, scale50a, scale50b, scale50c, scale50d, scale50e, scale50f, scale50g, scale100a] | |
| retired: data/datagen/retired.json # trajectories dropped after review (excluded from replay) | |
| replay: "python -m miniwebgen replay --shard k/4 --memory-model qwen35-4b # k = 0..3, then once more to retry failures" | |
| augment: | |
| - "python -m miniwebgen replay --augment recovery --share 0.35" | |
| - "python -m miniwebgen replay --augment chain --per-page 3" | |
| valset: "python src/miniwebgen/scripts/make_valset.py # the 115 generated single-primitive test tasks of the held-out sites" | |
| taskgen: "python -m miniwebgen.taskgen propose | judge | rows # the task-centric comparison data (attempts: miniwebagent.evaluate --tasks)" | |
| release: "python scripts/build_release.py # the consolidated, split-labelled, anonymized data folder" | |
| # scripts/generate_data.py prints (and with --run runs) all of the above in order, each run with its recorded arguments. | |
| # ββ figures and tables, in build order ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Each generator writes into docs/figures/<dir>/ (tracked); `to` is the destination inside paper_dir. | |
| figures: | |
| - {cmd: docs/figures/teaser/make_teaser_v2.py --variant radar, copy: {docs/figures/teaser/teaser_v2_radar.pdf: figures/teaser.pdf}} | |
| - {cmd: docs/figures/sites/make_sites.py, copy: {docs/figures/sites/sites.pdf: figures/sites.pdf}} | |
| - {cmd: docs/figures/method/make_method.py, copy: {docs/figures/method/method.pdf: figures/method.pdf}} | |
| - {cmd: docs/figures/construction/make_construction.py, copy: {docs/figures/construction/construction.pdf: figures/construction.pdf}} | |
| - {cmd: docs/figures/rq1/make_rq1_table.py, copy: {docs/figures/rq1/rq1_table.tex: tables/rq1_table.tex}} | |
| - {cmd: docs/figures/rq1/drag_failures.py, copy: {docs/figures/rq1/drag_failures.pdf: figures/drag_failures.pdf}} | |
| - {cmd: docs/figures/chain/make_chain.py, copy: {docs/figures/chain/chain_length.pdf: figures/chain_length.pdf}} | |
| - {cmd: docs/figures/rq3/make_results_row.py, copy: {docs/figures/rq3/results_row.pdf: figures/results_row.pdf}} # Fig. 7: practice per skill family | |
| - {cmd: docs/figures/rq3/make_rq3_table.py, copy: {docs/figures/rq3/rq3_table.tex: tables/rq3_transfer.tex}} # Tab. 3: the four benchmarks | |
| - {cmd: docs/figures/supp/make_supp.py, copy: {docs/figures/supp/supp_primitives.tex: tables/supp_primitives.tex, | |
| docs/figures/supp/supp_ops.tex: tables/supp_ops.tex, | |
| docs/figures/supp/supp_gallery.tex: tables/supp_gallery.tex, | |
| docs/figures/supp/figures/sites: figures/sites, | |
| docs/figures/supp/figures/annotation: figures/annotation}} | |
| - {cmd: docs/figures/rq1/text_numbers.py} # Sec. 4.1's text: pooled drag CI, group means, Spearman rho | |
| - {cmd: docs/figures/review/round1_numbers.py} | |
| - {cmd: docs/figures/review/compute_budget.py} | |
| - {cmd: docs/figures/review/make_round1_supp.py --out docs/figures/review/supp_results.tex, | |
| copy: {docs/figures/review/supp_results.tex: sections/supp/supp_results.tex}} | |
| - {cmd: docs/figures/mining/mining_stats.py} # Supp. A: the mining stage, from the brainstorm sheet | |
| - {cmd: docs/figures/mining/make_mining_supp.py --out docs/figures/mining/supp_mining.tex, | |
| copy: {docs/figures/mining/supp_mining.tex: sections/supp/supp_mining.tex}} | |
| - {cmd: docs/figures/review/make_graders_supp.py --out docs/figures/review/supp_graders.tex, # Supp. grader validation | |
| copy: {docs/figures/review/supp_graders.tex: sections/supp/supp_graders.tex}} | |
| # Analyses quoted in the text (Sec. 3 grader validity; review round 1 grader audit): | |
| analyses: | |
| grader_agreement: "python -m miniweb.evaluation.grader_agreement --parallel 4 # data/webmix/eval/grader_agreement.json" | |
| grader_audit_sample: "python src/miniweb/scripts/build_grader_audit.py # data/grader_audit/{cases,key.json}" | |
| grader_audit_score: "python src/miniweb/scripts/grader_audit_report.py --url <deployment> # data/grader_audit/report.json" | |