PULSE-code / tests /test_figures_f5.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
1.99 kB
"""F5 figure script (data-fraction curve + fraction table) on synthetic aggregated runs."""
import subprocess
import sys
import numpy as np
from conftest import REPO
from pulse import aggregate as A
from pulse.provenance import write_results
from test_pipeline import _fake
def _run(*args):
res = subprocess.run([sys.executable, *map(str, args)], cwd=str(REPO), capture_output=True, text=True)
assert res.returncode == 0, res.stdout + res.stderr
return res.stdout
def test_f5_curve_and_table(tmp_path):
rows = []
for f in (10, 50, 100):
rows += [{"key": f"scratch_f{f:03d}", "label": f"Scratch {f}%", "config": "x"},
{"key": f"s9pre_f{f:03d}", "label": f"S9 {f}%", "config": "x",
"reference": f"scratch_f{f:03d}"}]
spec = {"id": "5", "title": "F5 synthetic", "task": "f5", "split": "s", "folds": list(range(5)),
"seeds": [42, 43], "metrics": [{"key": "macro_f1", "label": "F1", "digits": 3,
"better": "higher"}],
"primary_metric": "macro_f1", "reference_row": None, "rows": rows}
rng = np.random.default_rng(0)
for r in rows:
acc = 0.3 + int(r["key"][-3:]) / 400 + (0.05 if r["key"].startswith("s9pre") else 0)
for k in range(5):
for s in spec["seeds"]:
write_results(A.run_dir(tmp_path / "runs", "5", r["key"], k, s),
_fake("f5", k, s, [f"v{k + 1}s{i}" for i in range(1, 5)], rng, acc=acc))
runs, missing = A.collect(spec, tmp_path / "runs")
A.write_outputs(A.aggregate(spec, runs, missing=missing), tmp_path / "tables")
_run(REPO / "figures" / "f5" / "plot_f5.py", "--table", "5", "--tables", tmp_path / "tables",
"--out", tmp_path / "fig")
assert (tmp_path / "fig" / "f5_data_fraction_tab5.pdf").is_file()
tex = (tmp_path / "tables" / "tab5_fraction.tex").read_text(encoding="utf-8")
assert tex.count(r"\%") == 3 and "/5" in tex