code/apps/browser_suite_b.py

"""Held-out suite B: 18 tasks on sites that appear in NO training source (crawl, rollouts, Mind2Web, NNetNav; checked
against finetune/out/train_domains.json at start).  Same runner and services as apps/browser_suite.py.

    python apps/browser_suite_b.py [name-filter]        REPEATS=3 SUITE_OUT=... for repeated runs
"""
import json, os, sys, time
from urllib.parse import urlparse
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from browser_suite import run  # noqa: E402  (sets up the jev env vars)

TASKS = [
    # navigation
    ("sqlite-download", "https://www.sqlite.org/", "Go to the Download page.", lambda u, t, x: "download.html" in u),
    ("haskell-downloads", "https://www.haskell.org/", "Go to the Downloads page.", lambda u, t, x: "/downloads" in u),
    ("zig-download", "https://ziglang.org/", "Open the download page.", lambda u, t, x: "/download" in u),
    ("openbsd-faq", "https://www.openbsd.org/", "Open the FAQ.", lambda u, t, x: "faq" in u.lower()),
    ("nginx-download", "https://nginx.org/", "Go to the download page.", lambda u, t, x: "download" in u),
    ("xkcd-archive", "https://xkcd.com/", "Open the archive of all comics.", lambda u, t, x: "/archive" in u),
    ("elixir-install", "https://elixir-lang.org/", "Open the installation instructions.", lambda u, t, x: "install" in u),
    ("prometheus-download", "https://prometheus.io/", "Go to the Download page.", lambda u, t, x: "/download" in u),
    ("ocaml-install", "https://ocaml.org/", "Open the page that explains how to install OCaml.", lambda u, t, x: "install" in u),
    # site search
    ("rosetta-quicksort", "https://rosettacode.org/wiki/Rosetta_Code", "Search Rosetta Code for 'quicksort'.",
     lambda u, t, x: "quicksort" in u.lower() or "quicksort" in t.lower()),
    ("arch-man-grep", "https://man.archlinux.org/", "Look up the manual page for 'grep'.", lambda u, t, x: "grep" in u.lower()),
    ("debian-man-tar", "https://manpages.debian.org/", "Look up the manual page for 'tar'.", lambda u, t, x: "tar" in urlparse(u).path.lower() + urlparse(u).query.lower()),
    ("rfc-http-semantics", "https://www.rfc-editor.org/search/rfc_search.php", "Search for RFCs about 'HTTP semantics' and show the results.",
     lambda u, t, x: "semantics" in u.lower()),
    ("nixos-ripgrep", "https://search.nixos.org/packages", "Search for the package 'ripgrep'.", lambda u, t, x: "ripgrep" in u.lower()),
    ("ocaml-pkg-yojson", "https://ocaml.org/", "Search OCaml packages for 'yojson'.", lambda u, t, x: "yojson" in u.lower()),
    ("shop-search-jacket", "https://www.scrapingcourse.com/ecommerce/", "Search the shop for 'jacket'.", lambda u, t, x: "s=jacket" in u.lower()),
    # listing
    ("shop-page2", "https://www.scrapingcourse.com/ecommerce/", "Go to page 2 of the products.", lambda u, t, x: "page/2" in u),
    ("shop-sort-price", "https://www.scrapingcourse.com/ecommerce/", "Sort the products by price, from low to high.",
     lambda u, t, x: "orderby=price" in u and "price-desc" not in u),
]

if __name__ == "__main__":
    dom = lambda u: urlparse(u).netloc.lower().removeprefix("www.")
    try:
        seen = set(json.load(open(os.path.join(os.path.dirname(__file__), "..", "finetune", "out", "train_domains.json"))))
        leak = sorted({dom(u) for _, u, _, _ in TASKS} & seen)
        print("held-out check:", "OK, no suite-B domain appears in training data" if not leak else f"LEAK {leak}", flush=True)
    except FileNotFoundError:
        print("held-out check skipped (no finetune/out/train_domains.json)", flush=True)
    flt = sys.argv[1] if len(sys.argv) > 1 else ""
    repeats = int(os.environ.get("REPEATS", "1"))
    rows = []
    for name, url, goal, check in TASKS:
        if flt and flt not in name: continue
        for rep in range(repeats):
            ok, steps, status, wall, final = run(name, url, goal, check)
            rows.append((name, ok, steps, status, wall))
            print(f"{'PASS' if ok else 'FAIL'}  {name:20s} steps={steps:2d} status={status:9s} {wall:5.1f}s  {final[:70]}", flush=True)
    n = sum(r[1] for r in rows)
    print(f"\n== {n}/{len(rows)} passed ({100*n/len(rows):.0f}%, {repeats} run(s) per task) | median wall {sorted(r[4] for r in rows)[len(rows)//2]:.1f}s")
    per = {}
    for r in rows: per.setdefault(r[0], []).append(r[1])
    print("   per task: " + "  ".join(f"{k}={sum(v)}/{len(v)}" for k, v in per.items()))
    json.dump([dict(zip(("name", "pass", "steps", "status", "wall"), r)) for r in rows], open(os.environ.get("SUITE_OUT", "/tmp/suite_b.json"), "w"), indent=1)