Spaces:
Running
Running
File size: 3,199 Bytes
22de727 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | """The generated single-primitive test tasks of the held-out sites (paper: "103 generated single-primitive tasks from 13
held-out sites"; 115 tasks, 103 of well-covered primitives). MiniWebGen's sampling and phrasing on the held-out sites,
stopped before execution: never executed, never trained on.
python src/miniwebgen/scripts/make_valset.py [--print]
Runs in its own datagen tree (MINIWEB_DATAGEN_DIR=data/webmix/valgen, whose split lists the 13 held-out sites as the sites
to generate on): v2_slider (filter_by_slider, up to 8 tasks) and v2_rest (every other supported macro, up to 6 each),
merged into data/webmix/valgen/val_v2.jsonl. The arguments are those the paper's runs recorded in
data/webmix/valgen/runs/<run>/run.json when present.
"""
import ast
import json
import os
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
VALGEN = ROOT / "data" / "webmix" / "valgen"
COMMON = {"stop_after": "suggest", "mid_chain_share": 0.0, "preset_share": 0.0, "workers": 4, "servers": 2, "ports": [8316, 8317]}
RUNS = {"v2_slider": {"macros": ["filter_by_slider"], "n": 8, "propose_site_cap": 4},
"v2_rest": {"macros": None, "n": 6, "propose_site_cap": 3}} # None: every supported macro but the slider
def run_args(run_id):
saved = VALGEN / "runs" / run_id / "run.json"
if saved.exists():
a = json.loads(saved.read_text())
else:
sys.path.insert(0, str(ROOT / "src"))
from miniwebgen import kinds
a = {**COMMON, **RUNS[run_id], "run_id": run_id}
if a["macros"] is None:
a["macros"] = sorted(set(kinds.supported_macros()) - {"filter_by_slider"})
m = a["macros"]
m = ast.literal_eval(m) if isinstance(m, str) else m
argv = ["-m", "miniwebgen", "run", "--run-id", run_id, "--macros", *m, "--n", str(a["n"]),
"--propose-site-cap", str(a["propose_site_cap"]), "--per-site-cap", str(a.get("per_site_cap", 2)),
"--mid-chain-share", str(a.get("mid_chain_share", 0.0)), "--preset-share", str(a.get("preset_share", 0.0)),
"--stop-after", a.get("stop_after") or "suggest", "--workers", str(a.get("workers", 4)),
"--servers", str(a.get("servers", 2)), "--ports", *[str(p) for p in a.get("ports") or COMMON["ports"]],
"--seed", str(a.get("seed", 0))]
return argv
def main():
env = {**os.environ, "MINIWEB_DATAGEN_DIR": str(VALGEN),
"PYTHONPATH": os.pathsep.join(filter(None, [str(ROOT / "src"), os.environ.get("PYTHONPATH")]))}
for run_id in RUNS:
cmd = [sys.executable, *run_args(run_id)]
print("MINIWEB_DATAGEN_DIR=data/webmix/valgen python", " ".join(cmd[1:]), flush=True)
if "--print" not in sys.argv:
subprocess.run(cmd, cwd=ROOT, env=env, check=True)
if "--print" in sys.argv:
return
rows = [json.loads(line) for run in RUNS for line in (VALGEN / "runs" / run / "tasks.jsonl").read_text().split("\n")
if line.strip()]
(VALGEN / "val_v2.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows))
print(len(rows), "test tasks -> data/webmix/valgen/val_v2.jsonl")
if __name__ == "__main__":
main()
|