PULSE-code / scripts /build_outputs.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
8.34 kB
#!/usr/bin/env python
"""Every table, number macro, report and figure of the paper from one runs directory.
python scripts/build_outputs.py --runs runs --out out # paper (refuses gaps)
python scripts/build_outputs.py --runs SMOKE --out SMOKE_OUT --allow-incomplete [--latex]
Steps (each one a separate command, so one failure does not hide the others;
the exit code is non-zero if any step failed):
1. ``scripts/aggregate.py`` per registry table -> ``OUT/tables/tab<N>*.tex``, ``tab<N>.json``,
``numbers.tex`` (``--tables`` restricts the list);
2. ``pulse.tasks.t3_report`` (Table 6 per-fold counts), ``pulse.tasks.t5_report``
(Tables 8/9/10/24, R2; ``numbers_t5_<T>.tex``);
3. figures into ``OUT/figures``: Fig. 3 (T1 confusion), the T2 confusion
matrix, the F5 data-fraction curves, Figures 7 and 8 (T5);
4. with ``--f0``: the F0 analysis (``pulse.tasks.f0``, CPU) into ``RUNS/f0`` and its
figures / ``numbers_f0.tex``;
5. with ``--latex``: a LaTeX document that inputs every ``numbers*.tex`` and
``*.tex`` table is compiled with pdflatex (checks that they build).
Tables without any run directory are skipped (with ``--allow-incomplete``) or
reported as failures.
"""
from __future__ import annotations
import argparse
import os
import shutil
import subprocess
import sys
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO))
from pulse.aggregate import load_registry, row_source # noqa: E402
FIGURES = [ # (needs tables, argv after python)
("3", ["figures/t1/fig3_confusion.py", "--table", "3", "--row", "syncfuse",
"--out", "{fig}/fig3_confusion_t1_syncfuse.pdf"]),
("4", ["figures/t2/confusion.py", "--table", "4", "--row", "daf", "--out", "{fig}/t2"]),
("5", ["figures/f5/plot_f5.py", "--table", "5", "--tables", "{tab}", "--out", "{fig}/f5"]),
("5seg", ["figures/f5/plot_f5.py", "--table", "5seg", "--tables", "{tab}", "--out", "{fig}/f5"]),
("8", ["figures/t5/fig_sustained_lift.py", "--report", "{tab}/t5_tab8.json", "--out", "{fig}/t5"]),
]
INCOMPLETE_FLAG = {"figures/t1/fig3_confusion.py", "figures/t2/confusion.py"}
def has_runs(spec, runs_root: Path) -> bool:
for row in spec.get("rows", []):
t, r = row_source(spec, row)
if (runs_root / t / r).is_dir():
return True
return False
def latex_check(tab: Path) -> str:
"""Compile every generated .tex of ``tab``; return '' or the error text."""
exe = shutil.which("pdflatex")
if exe is None:
return "pdflatex not found"
numbers = sorted(p.name for p in tab.glob("numbers*.tex"))
tables = sorted(p.name for p in tab.glob("*.tex") if not p.name.startswith("numbers")
and p.name != "_check.tex")
keys = []
for p in tab.glob("numbers.tex"):
for line in p.read_text(encoding="utf-8").splitlines():
if line.startswith(r"\expandafter\def\csname pulse@"):
keys.append(line.split("pulse@", 1)[1].split(r"\endcsname", 1)[0])
body = [r"\documentclass{article}", r"\usepackage[T1]{fontenc}", r"\usepackage{booktabs}",
r"\usepackage{amsmath}", r"\usepackage{graphicx}",
r"\usepackage[margin=1cm,landscape,a3paper]{geometry}", r"\begin{document}"]
body += [rf"\input{{{n}}}" for n in numbers]
for n in tables:
body += [rf"\section*{{\detokenize{{{n}}}}}", r"\resizebox{\linewidth}{!}{\input{" + n + "}}",
""]
body += [r"\section*{numbers}"] + [rf"\pulsenum{{{k}}}\par" for k in keys[:400]]
body += [r"\end{document}"]
(tab / "_check.tex").write_text("\n".join(body) + "\n", encoding="utf-8", newline="\n")
r = subprocess.run([exe, "-interaction=nonstopmode", "-halt-on-error", "_check.tex"],
cwd=str(tab), capture_output=True, text=True, errors="replace")
log = (tab / "_check.log").read_text(encoding="utf-8", errors="replace") \
if (tab / "_check.log").is_file() else r.stdout
if r.returncode != 0:
return "\n".join(l for l in log.splitlines() if l.startswith("!") or "Error" in l)[:2000] \
or r.stdout[-2000:]
undefined = [l for l in log.splitlines() if "undefined number" in l]
if undefined:
return "undefined \\pulsenum keys: " + "; ".join(undefined[:5])
return ""
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--runs", default="runs")
ap.add_argument("--out", default="out")
ap.add_argument("--registry", default=None)
ap.add_argument("--tables", default=None, help="comma-separated subset (default: every table)")
ap.add_argument("--allow-incomplete", action="store_true")
ap.add_argument("--f0", action="store_true", help="also run the F0 analysis (CPU, needs PULSE_ROOT)")
ap.add_argument("--fig7", action="store_true", help="also draw Figure 7 (needs PULSE_ROOT)")
ap.add_argument("--latex", action="store_true", help="compile the generated .tex with pdflatex")
args = ap.parse_args(argv)
py = os.environ.get("PULSE_PYTHON") or sys.executable
runs, out = Path(args.runs).resolve(), Path(args.out).resolve()
tab, fig = out / "tables", out / "figures"
tab.mkdir(parents=True, exist_ok=True)
fig.mkdir(parents=True, exist_ok=True)
reg = load_registry(args.registry)
want = args.tables.split(",") if args.tables else list(reg)
inc = ["--allow-incomplete"] if args.allow_incomplete else []
reg_flag = ["--registry", args.registry] if args.registry else []
failed, done, skipped = [], [], []
def step(name, argv_):
print(f"+ {name}", flush=True)
r = subprocess.run([py, *argv_], cwd=str(REPO), capture_output=True, text=True, errors="replace")
if r.returncode != 0:
failed.append(name)
print(f" FAILED ({r.returncode}):\n" + "\n".join(
" " + l for l in (r.stderr or r.stdout).strip().splitlines()[-12:]), flush=True)
return False
done.append(name)
return True
present = set()
for t in want:
spec = dict(reg[t], id=t)
if not spec.get("rows"):
continue
if not has_runs(spec, runs):
(skipped if args.allow_incomplete else failed).append(f"aggregate {t} (no runs)")
continue
if step(f"aggregate {t}", ["scripts/aggregate.py", "--table", t, "--runs", str(runs),
"--out", str(tab), *inc, *reg_flag]):
present.add(t)
if "6" in present:
step("t3_report 6", ["-m", "pulse.tasks.t3_report", "--table", "6", "--runs", str(runs),
"--out", str(tab), *inc, *reg_flag])
t5 = [t for t in ("8", "8full") if t in present]
if t5:
step("t5_report " + " ".join(t5), ["-m", "pulse.tasks.t5_report", *sum((["--table", t] for t in t5), []),
"--runs", str(runs), "--out", str(tab), *inc, *reg_flag])
for need, argv_ in FIGURES:
if need not in present:
continue
a = [x.format(fig=fig.as_posix(), tab=tab.as_posix()) for x in argv_]
if a[0] in INCOMPLETE_FLAG:
a += ["--runs", str(runs), *inc, *reg_flag]
(fig / "f5").mkdir(exist_ok=True)
step("figure " + " ".join(argv_[:3]), a)
if args.fig7:
step("figure 7", ["figures/t5/fig_label_derivation.py", "--out", (fig / "t5").as_posix()])
if args.f0:
if step("f0 analysis", ["-m", "pulse.tasks.f0", "--out", str(runs / "f0")]):
step("f0 figures", ["figures/f0/plot_f0.py", "--in", str(runs / "f0"), "--out",
(fig / "f0").as_posix(), "--tables", str(tab)])
if args.latex:
print("+ latex check", flush=True)
err = latex_check(tab)
if err:
failed.append("latex check")
print(" FAILED:\n " + err.replace("\n", "\n "), flush=True)
else:
done.append("latex check")
print(f"\n{len(done)} step(s) ok, {len(failed)} failed, {len(skipped)} skipped")
for f in failed:
print(f" failed: {f}")
for s in skipped:
print(f" skipped: {s}")
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main())