File size: 4,570 Bytes
8086c56 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """Held-out suite B: 18 tasks on sites that appear in NO training source (crawl, rollouts, Mind2Web, NNetNav; checked
against finetune/out/train_domains.json at start). Same runner and services as apps/browser_suite.py.
python apps/browser_suite_b.py [name-filter] REPEATS=3 SUITE_OUT=... for repeated runs
"""
import json, os, sys, time
from urllib.parse import urlparse
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from browser_suite import run # noqa: E402 (sets up the jev env vars)
TASKS = [
# navigation
("sqlite-download", "https://www.sqlite.org/", "Go to the Download page.", lambda u, t, x: "download.html" in u),
("haskell-downloads", "https://www.haskell.org/", "Go to the Downloads page.", lambda u, t, x: "/downloads" in u),
("zig-download", "https://ziglang.org/", "Open the download page.", lambda u, t, x: "/download" in u),
("openbsd-faq", "https://www.openbsd.org/", "Open the FAQ.", lambda u, t, x: "faq" in u.lower()),
("nginx-download", "https://nginx.org/", "Go to the download page.", lambda u, t, x: "download" in u),
("xkcd-archive", "https://xkcd.com/", "Open the archive of all comics.", lambda u, t, x: "/archive" in u),
("elixir-install", "https://elixir-lang.org/", "Open the installation instructions.", lambda u, t, x: "install" in u),
("prometheus-download", "https://prometheus.io/", "Go to the Download page.", lambda u, t, x: "/download" in u),
("ocaml-install", "https://ocaml.org/", "Open the page that explains how to install OCaml.", lambda u, t, x: "install" in u),
# site search
("rosetta-quicksort", "https://rosettacode.org/wiki/Rosetta_Code", "Search Rosetta Code for 'quicksort'.",
lambda u, t, x: "quicksort" in u.lower() or "quicksort" in t.lower()),
("arch-man-grep", "https://man.archlinux.org/", "Look up the manual page for 'grep'.", lambda u, t, x: "grep" in u.lower()),
("debian-man-tar", "https://manpages.debian.org/", "Look up the manual page for 'tar'.", lambda u, t, x: "tar" in urlparse(u).path.lower() + urlparse(u).query.lower()),
("rfc-http-semantics", "https://www.rfc-editor.org/search/rfc_search.php", "Search for RFCs about 'HTTP semantics' and show the results.",
lambda u, t, x: "semantics" in u.lower()),
("nixos-ripgrep", "https://search.nixos.org/packages", "Search for the package 'ripgrep'.", lambda u, t, x: "ripgrep" in u.lower()),
("ocaml-pkg-yojson", "https://ocaml.org/", "Search OCaml packages for 'yojson'.", lambda u, t, x: "yojson" in u.lower()),
("shop-search-jacket", "https://www.scrapingcourse.com/ecommerce/", "Search the shop for 'jacket'.", lambda u, t, x: "s=jacket" in u.lower()),
# listing
("shop-page2", "https://www.scrapingcourse.com/ecommerce/", "Go to page 2 of the products.", lambda u, t, x: "page/2" in u),
("shop-sort-price", "https://www.scrapingcourse.com/ecommerce/", "Sort the products by price, from low to high.",
lambda u, t, x: "orderby=price" in u and "price-desc" not in u),
]
if __name__ == "__main__":
dom = lambda u: urlparse(u).netloc.lower().removeprefix("www.")
try:
seen = set(json.load(open(os.path.join(os.path.dirname(__file__), "..", "finetune", "out", "train_domains.json"))))
leak = sorted({dom(u) for _, u, _, _ in TASKS} & seen)
print("held-out check:", "OK, no suite-B domain appears in training data" if not leak else f"LEAK {leak}", flush=True)
except FileNotFoundError:
print("held-out check skipped (no finetune/out/train_domains.json)", flush=True)
flt = sys.argv[1] if len(sys.argv) > 1 else ""
repeats = int(os.environ.get("REPEATS", "1"))
rows = []
for name, url, goal, check in TASKS:
if flt and flt not in name: continue
for rep in range(repeats):
ok, steps, status, wall, final = run(name, url, goal, check)
rows.append((name, ok, steps, status, wall))
print(f"{'PASS' if ok else 'FAIL'} {name:20s} steps={steps:2d} status={status:9s} {wall:5.1f}s {final[:70]}", flush=True)
n = sum(r[1] for r in rows)
print(f"\n== {n}/{len(rows)} passed ({100*n/len(rows):.0f}%, {repeats} run(s) per task) | median wall {sorted(r[4] for r in rows)[len(rows)//2]:.1f}s")
per = {}
for r in rows: per.setdefault(r[0], []).append(r[1])
print(" per task: " + " ".join(f"{k}={sum(v)}/{len(v)}" for k, v in per.items()))
json.dump([dict(zip(("name", "pass", "steps", "status", "wall"), r)) for r in rows], open(os.environ.get("SUITE_OUT", "/tmp/suite_b.json"), "w"), indent=1)
|