Download eval/benchmark.py from itsma77e/yapper-turbo: direct link, hf CLI and curl.
- Browser
- Download file 8.48 kB
-
https://huggingface.co/itsma77e/yapper-turbo/resolve/main/eval/benchmark.py
- Command line
-
hf download hf://itsma77e/yapper-turbo/eval/benchmark.py
-
curl -L -o benchmark.py https://huggingface.co/itsma77e/yapper-turbo/resolve/main/eval/benchmark.py
8.48 kB
| """Compare Yapper with Whisper large-v3-turbo on the frozen FLEURS set. | |
| Arms use the parameters of Yapper's final transcription pass: | |
| turbo large-v3-turbo with standard faster-whisper | |
| turbo-crop the same weights with Yapper's encoder crop | |
| yapper Yapper weights with the crop, as shipped in the app | |
| yapper-plain Yapper weights with standard faster-whisper | |
| """ | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| import statistics | |
| import time | |
| import unicodedata | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent | |
| ARMS = ["turbo", "turbo-crop", "yapper", "yapper-plain"] | |
| def words(text): | |
| text = unicodedata.normalize("NFKC", text).casefold().replace("’", "'") | |
| return re.findall(r"[^\W_]+(?:'[^\W_]+)*", text, flags=re.UNICODE) | |
| def chars(text): | |
| return [c for c in unicodedata.normalize("NFKC", text).casefold() if unicodedata.category(c)[0] in "LNM"] | |
| def distance(a, b): | |
| previous = list(range(len(b) + 1)) | |
| for i, left in enumerate(a, 1): | |
| current = [i] | |
| for j, right in enumerate(b, 1): | |
| current.append(min(current[-1] + 1, previous[j] + 1, previous[j - 1] + (left != right))) | |
| previous = current | |
| return previous[-1] | |
| def score(reference, hypothesis): | |
| w, c = words(reference), chars(reference) | |
| return dict(word_errors=distance(w, words(hypothesis)), words=len(w), | |
| char_errors=distance(c, chars(hypothesis)), chars=len(c)) | |
| def crop_frames(samples): | |
| """Same bound as FasterBackend.transcribe in the Yapper engine.""" | |
| return min(1500, max(400, ((samples + 15999) // 16000 + 1) * 50)) | |
| class Engine: | |
| def __init__(self, folder, device, threads): | |
| import numpy as np | |
| from faster_whisper import WhisperModel | |
| self.np = np | |
| compute = "int8_float16" if device == "cuda" else "int8" | |
| self.model = WhisperModel(str(folder), device=device, compute_type=compute, | |
| cpu_threads=threads, num_workers=1, local_files_only=True) | |
| self.full_encode = self.model.encode | |
| self.frames = None | |
| self.model.encode = self.encode | |
| def encode(self, features): | |
| if self.frames is None: | |
| return self.full_encode(features) | |
| return self.full_encode(self.np.ascontiguousarray(features[..., :2 * self.frames])) | |
| def run(self, audio, language, cropped): | |
| self.frames = crop_frames(len(audio)) if cropped else None | |
| segments, info = self.model.transcribe( | |
| audio, language=language, task="transcribe", beam_size=3, best_of=1, temperature=0, | |
| condition_on_previous_text=False, vad_filter=False, no_speech_threshold=.6, | |
| log_prob_threshold=-1, compression_ratio_threshold=2.4, | |
| **({"without_timestamps": True} if cropped else {})) | |
| return " ".join(s.text.strip() for s in segments).strip(), info.language | |
| def digest(path): | |
| with open(path, "rb") as handle: | |
| return hashlib.file_digest(handle, "sha256").hexdigest() | |
| def summarize(rows, key): | |
| result = {} | |
| for language in sorted({r["language"] for r in rows}) + ["all"]: | |
| subset = [r for r in rows if language == "all" or r["language"] == language] | |
| audio = sum(r["duration"] for r in subset) | |
| result[language] = {} | |
| for arm in ARMS: | |
| runs = [r[key][arm] for r in subset] | |
| errors, total = sum(x["word_errors"] for x in runs), sum(x["words"] for x in runs) | |
| cer, count = sum(x["char_errors"] for x in runs), sum(x["chars"] for x in runs) | |
| seconds = sum(statistics.median(x["seconds"]) for x in runs) | |
| result[language][arm] = dict(wer=errors / total, word_errors=errors, words=total, cer=cer / count, | |
| seconds=seconds, audio_seconds=audio, realtime=audio / seconds, | |
| wrong_language=sum(x["detected"] != r["language"] for x, r in zip(runs, subset))) | |
| return result | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--yapper", type=Path, required=True) | |
| parser.add_argument("--turbo", type=Path, required=True) | |
| parser.add_argument("--device", choices=["cuda", "cpu"], required=True) | |
| parser.add_argument("--threads", type=int, default=4) | |
| parser.add_argument("--languages", default="") | |
| parser.add_argument("--per-language", type=int, default=0) | |
| parser.add_argument("--repetitions", type=int, default=2) | |
| parser.add_argument("--auto", action="store_true", help="also run with language detection") | |
| parser.add_argument("--cuda-dir", type=Path) | |
| parser.add_argument("--output", type=Path, required=True) | |
| args = parser.parse_args() | |
| if args.cuda_dir: | |
| os.add_dll_directory(str(args.cuda_dir)) | |
| os.environ["PATH"] = str(args.cuda_dir) + os.pathsep + os.environ["PATH"] | |
| if args.output.exists(): | |
| raise FileExistsError(args.output) | |
| manifest_path = ROOT / "fresh-manifest.json" | |
| manifest = json.loads(manifest_path.read_text(encoding="utf-8")) | |
| keep = set(args.languages.split(",")) if args.languages else None | |
| rows, used = [], {} | |
| for row in manifest: | |
| used[row["language"]] = used.get(row["language"], 0) + 1 | |
| if keep and row["language"] not in keep: | |
| continue | |
| if args.per_language and used[row["language"]] > args.per_language: | |
| continue | |
| rows.append(row) | |
| from faster_whisper.audio import decode_audio | |
| import ctranslate2 | |
| import faster_whisper | |
| engines = {"turbo": Engine(args.turbo, args.device, args.threads), "yapper": Engine(args.yapper, args.device, args.threads)} | |
| setup = {"turbo": ("turbo", False), "turbo-crop": ("turbo", True), "yapper": ("yapper", True), "yapper-plain": ("yapper", False)} | |
| import numpy as np | |
| for engine in engines.values(): | |
| engine.run(np.zeros(16000, np.float32), "en", True) | |
| report = dict(complete=False, device=args.device, threads=args.threads, repetitions=args.repetitions, | |
| manifest_sha256=digest(manifest_path), runtime=dict(faster_whisper=faster_whisper.__version__, ctranslate2=ctranslate2.__version__), | |
| models={name: dict(sha256=digest(folder / "model.bin"), bytes=(folder / "model.bin").stat().st_size) | |
| for name, folder in (("turbo", args.turbo), ("yapper", args.yapper))}, rows=[]) | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| def run_arms(audio, language, index, repetitions): | |
| runs = {arm: [] for arm in ARMS} | |
| for repetition in range(repetitions): | |
| shift = (index + repetition) % len(ARMS) | |
| order = ARMS[shift:] + ARMS[:shift] | |
| if repetition % 2: | |
| order.reverse() | |
| for arm in order: | |
| name, cropped = setup[arm] | |
| start = time.perf_counter() | |
| text, detected = engines[name].run(audio, language, cropped) | |
| runs[arm].append(dict(text=text, detected=detected, seconds=time.perf_counter() - start)) | |
| return runs | |
| for index, row in enumerate(rows): | |
| path = ROOT / row["audio"] | |
| if digest(path) != row["audio_sha256"]: | |
| raise ValueError("Audio changed: " + row["id"]) | |
| audio = decode_audio(str(path)) | |
| record = {k: row[k] for k in ("id", "language", "duration", "reference")} | |
| for key, language, repetitions in [("known", row["language"], args.repetitions)] + ([("auto", None, 1)] if args.auto else []): | |
| runs = run_arms(audio, language, index, repetitions) | |
| record[key] = {arm: dict(text=values[0]["text"], detected=values[0]["detected"], | |
| seconds=[v["seconds"] for v in values], **score(row["reference"], values[0]["text"])) | |
| for arm, values in runs.items()} | |
| report["rows"].append(record) | |
| if index % 10 == 9: | |
| print(index + 1, "/", len(rows), flush=True) | |
| report["summary"] = {key: summarize(report["rows"], key) for key in (["known", "auto"] if args.auto else ["known"])} | |
| report["complete"] = True | |
| args.output.write_text(json.dumps(report, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") | |
| all_known = report["summary"]["known"]["all"] | |
| print(json.dumps({arm: {k: round(v, 4) if isinstance(v, float) else v for k, v in all_known[arm].items()} for arm in ARMS}, indent=1)) | |
| if __name__ == "__main__": | |
| main() | |