File size: 1,936 Bytes
3ecb728
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
"""Score saved hypotheses: results/<model>/<testset>.jsonl -> error-rate table.

    uv run --no-project --with jiwer python score.py [results_dir] [--show N]

Columns per test set: WER / CER / CER ignoring spaces (all on canonical spelling, see
nimruz/scoring.py), then in brackets the WER without the word-spacing rules.
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

import jiwer

from nimruz.scoring import canonical, error_rates


def main():
    args = [a for a in sys.argv[1:] if not a.startswith("--")]
    show = int(sys.argv[sys.argv.index("--show") + 1]) if "--show" in sys.argv else 0
    root = Path(args[0] if args else "results")
    table: dict[str, dict[str, dict]] = {}
    rows_of = {}
    for f in sorted(root.glob("*/*.jsonl")):
        rows = [json.loads(l) for l in open(f, encoding="utf-8")]
        table.setdefault(f.parent.name, {})[f.stem] = error_rates([r["ref"] for r in rows], [r["hyp"] for r in rows])
        rows_of[(f.parent.name, f.stem)] = rows
    sets = sorted({s for m in table.values() for s in m})
    print("| model | " + " | ".join(sets) + " |")
    print("|---|" + "---|" * len(sets))
    key = lambda kv: sum(v["wer"] for v in kv[1].values()) / len(kv[1])
    for model, res in sorted(table.items(), key=key):
        cells = [f"{res[s]['wer']:.1f} / {res[s]['cer']:.1f} / {res[s]['ncer']:.1f} [{res[s]['wer_raw']:.1f}]" if s in res else "-" for s in sets]
        print(f"| {model.replace('__', '/')} | " + " | ".join(cells) + " |")
    if show:
        for (model, s), rows in rows_of.items():
            print(f"\n## {model} / {s}: worst {show}")
            per = sorted(((jiwer.wer(canonical(r["ref"]) or "-", canonical(r["hyp"])), canonical(r["ref"]), canonical(r["hyp"])) for r in rows), reverse=True)[:show]
            for e, a, b in per:
                print(f"{e:.2f}\n  ref: {a}\n  hyp: {b}")


if __name__ == "__main__":
    main()