Nava / score.py
xmanii's picture
Add model card, charts and helper code
3ecb728 verified
Raw History Blame Contribute Delete
1.94 kB
"""Score saved hypotheses: results/<model>/<testset>.jsonl -> error-rate table.
uv run --no-project --with jiwer python score.py [results_dir] [--show N]
Columns per test set: WER / CER / CER ignoring spaces (all on canonical spelling, see
nimruz/scoring.py), then in brackets the WER without the word-spacing rules.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
import jiwer
from nimruz.scoring import canonical, error_rates
def main():
args = [a for a in sys.argv[1:] if not a.startswith("--")]
show = int(sys.argv[sys.argv.index("--show") + 1]) if "--show" in sys.argv else 0
root = Path(args[0] if args else "results")
table: dict[str, dict[str, dict]] = {}
rows_of = {}
for f in sorted(root.glob("*/*.jsonl")):
rows = [json.loads(l) for l in open(f, encoding="utf-8")]
table.setdefault(f.parent.name, {})[f.stem] = error_rates([r["ref"] for r in rows], [r["hyp"] for r in rows])
rows_of[(f.parent.name, f.stem)] = rows
sets = sorted({s for m in table.values() for s in m})
print("| model | " + " | ".join(sets) + " |")
print("|---|" + "---|" * len(sets))
key = lambda kv: sum(v["wer"] for v in kv[1].values()) / len(kv[1])
for model, res in sorted(table.items(), key=key):
cells = [f"{res[s]['wer']:.1f} / {res[s]['cer']:.1f} / {res[s]['ncer']:.1f} [{res[s]['wer_raw']:.1f}]" if s in res else "-" for s in sets]
print(f"| {model.replace('__', '/')} | " + " | ".join(cells) + " |")
if show:
for (model, s), rows in rows_of.items():
print(f"\n## {model} / {s}: worst {show}")
per = sorted(((jiwer.wer(canonical(r["ref"]) or "-", canonical(r["hyp"])), canonical(r["ref"]), canonical(r["hyp"])) for r in rows), reverse=True)[:show]
for e, a, b in per:
print(f"{e:.2f}\n ref: {a}\n hyp: {b}")
if __name__ == "__main__":
main()