Download eval/summarize.py from itsma77e/yapper-turbo: direct link, hf CLI and curl.
- Browser
- Download file 2.58 kB
-
https://huggingface.co/itsma77e/yapper-turbo/resolve/main/eval/summarize.py
- Command line
-
hf download hf://itsma77e/yapper-turbo/eval/summarize.py
-
curl -L -o summarize.py https://huggingface.co/itsma77e/yapper-turbo/resolve/main/eval/summarize.py
2.58 kB
| """Print the benchmark results as tables, with paired statistics between arms.""" | |
| import json | |
| import random | |
| import statistics | |
| import sys | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent | |
| ARMS = ["turbo", "turbo-crop", "yapper", "yapper-plain"] | |
| def table(summary, languages): | |
| print("lang | arm | WER | CER | wrong language | speed x realtime") | |
| for language in languages: | |
| for arm in ARMS: | |
| v = summary[language][arm] | |
| print(f"{language} | {arm} | {v['wer']:.2%} ({v['word_errors']}/{v['words']}) | {v['cer']:.2%} | {v['wrong_language']} | {v['realtime']:.2f}x") | |
| def paired(rows, key, a, b): | |
| """Bootstrap the WER difference and the speed ratio of a over b, resampling clips.""" | |
| random.seed(20261102) | |
| def stats(sample): | |
| words = sum(r[key][a]["words"] for r in sample) | |
| difference = (sum(r[key][a]["word_errors"] for r in sample) - sum(r[key][b]["word_errors"] for r in sample)) / words | |
| ratio = sum(statistics.median(r[key][b]["seconds"]) for r in sample) / sum(statistics.median(r[key][a]["seconds"]) for r in sample) | |
| return difference, ratio | |
| samples = [stats(random.choices(rows, k=len(rows))) for _ in range(2000)] | |
| interval = lambda values: (sorted(values)[50], sorted(values)[1949]) | |
| point = stats(rows) | |
| low, high = interval([s[0] for s in samples]) | |
| slow, fast = interval([s[1] for s in samples]) | |
| print(f"{a} vs {b}: WER difference {point[0]:+.2%} [{low:+.2%}, {high:+.2%}], speed {point[1]:.2f}x [{slow:.2f}, {fast:.2f}]") | |
| def main(): | |
| for name in sys.argv[1:]: | |
| report = json.loads((ROOT / "results" / f"{name}.json").read_text(encoding="utf-8")) | |
| print(f"\n=== {name}: {report['device']}, {len(report['rows'])} clips, {report['models']}") | |
| for key, summary in report["summary"].items(): | |
| print(f"\n--- language {key}") | |
| table(summary, sorted(summary, key=lambda x: (x != "all", x))) | |
| for subset in ["all", "it", "en"]: | |
| rows = [r for r in report["rows"] if subset == "all" or r["language"] == subset] | |
| print(f"paired, {subset}, {len(rows)} clips") | |
| paired(rows, key, "yapper", "turbo") | |
| paired(rows, key, "yapper", "turbo-crop") | |
| paired(rows, key, "turbo-crop", "turbo") | |
| paired(rows, key, "yapper-plain", "turbo") | |
| durations = [r["duration"] for r in report["rows"]] | |
| print("durations", min(durations), statistics.median(durations), max(durations)) | |
| if __name__ == "__main__": | |
| main() | |