"""Score saved hypotheses: results//.jsonl -> error-rate table. uv run --no-project --with jiwer python score.py [results_dir] [--show N] Columns per test set: WER / CER / CER ignoring spaces (all on canonical spelling, see nimruz/scoring.py), then in brackets the WER without the word-spacing rules. """ from __future__ import annotations import json import sys from pathlib import Path import jiwer from nimruz.scoring import canonical, error_rates def main(): args = [a for a in sys.argv[1:] if not a.startswith("--")] show = int(sys.argv[sys.argv.index("--show") + 1]) if "--show" in sys.argv else 0 root = Path(args[0] if args else "results") table: dict[str, dict[str, dict]] = {} rows_of = {} for f in sorted(root.glob("*/*.jsonl")): rows = [json.loads(l) for l in open(f, encoding="utf-8")] table.setdefault(f.parent.name, {})[f.stem] = error_rates([r["ref"] for r in rows], [r["hyp"] for r in rows]) rows_of[(f.parent.name, f.stem)] = rows sets = sorted({s for m in table.values() for s in m}) print("| model | " + " | ".join(sets) + " |") print("|---|" + "---|" * len(sets)) key = lambda kv: sum(v["wer"] for v in kv[1].values()) / len(kv[1]) for model, res in sorted(table.items(), key=key): cells = [f"{res[s]['wer']:.1f} / {res[s]['cer']:.1f} / {res[s]['ncer']:.1f} [{res[s]['wer_raw']:.1f}]" if s in res else "-" for s in sets] print(f"| {model.replace('__', '/')} | " + " | ".join(cells) + " |") if show: for (model, s), rows in rows_of.items(): print(f"\n## {model} / {s}: worst {show}") per = sorted(((jiwer.wer(canonical(r["ref"]) or "-", canonical(r["hyp"])), canonical(r["ref"]), canonical(r["hyp"])) for r in rows), reverse=True)[:show] for e, a, b in per: print(f"{e:.2f}\n ref: {a}\n hyp: {b}") if __name__ == "__main__": main()