Download benchmark.py from cfahlgren1/fiato: direct link, hf CLI and curl.
- Browser
- Download file 4.43 kB
-
https://huggingface.co/cfahlgren1/fiato/resolve/main/benchmark.py
- Command line
-
hf download hf://cfahlgren1/fiato/benchmark.py
-
curl -L -o benchmark.py https://huggingface.co/cfahlgren1/fiato/resolve/main/benchmark.py
4.43 kB
| """Check the released model locally and write benchmark.json. | |
| Measures how long the model makes each expression, whether an audio event | |
| classifier hears the expressions in the released samples, and the ONNX speed | |
| on the CPU. | |
| DIZIONE_DATA=release/phonemizer FIATO_FRONTEND=own python benchmark.py release | |
| """ | |
| import json | |
| import statistics | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import onnxruntime as ort | |
| import soundfile as sf | |
| import torch | |
| from scipy.signal import resample_poly | |
| from transformers import ASTFeatureExtractor, ASTForAudioClassification | |
| from fiato.infer import load_model, speak | |
| from fiato.text import EXPRESSIONS, phonemize | |
| THREADS = 4 | |
| EVENT_MODEL_ID = "MIT/ast-finetuned-audioset-10-10-0.4593" | |
| # AudioSet classes that count as hearing each expression, as in generate.py. | |
| EVENT_LABELS = {"sigh": ["Sigh", "Breathing"], "exhale": ["Sigh", "Breathing"], "gasp": ["Gasp", "Breathing"]} | |
| SENTENCES = ( | |
| "Hello! This is fiato.", | |
| "The weather is nice today, and I feel very relaxed.", | |
| "Did you really pay forty two dollars for that on March third?", | |
| "After a long journey across the mountains, the travellers reached the small village, where they were welcomed.", | |
| ) | |
| def expression_lengths(model, frame_seconds: float) -> dict: | |
| """Seconds the model gives each expression, in the middle of a sentence, for both voices.""" | |
| lengths = {} | |
| for tag in EXPRESSIONS: | |
| for voice in model.config.speakers: | |
| _, phones, details = speak(model, f"Well, {tag} I guess we should start over.", voice) | |
| frames = int(details["durations"][0][phones.symbols.index(tag)]) | |
| lengths[f"{tag} {voice}"] = round(frames * frame_seconds, 2) | |
| return lengths | |
| def expression_scores(samples: Path) -> dict: | |
| """Classifier score of each released sample for its own expression. A plain sample is | |
| scored for every expression and its highest score kept, as the level of ordinary speech.""" | |
| features = ASTFeatureExtractor.from_pretrained(EVENT_MODEL_ID) | |
| model = ASTForAudioClassification.from_pretrained(EVENT_MODEL_ID).eval() | |
| label_ids = {tag: [model.config.label2id[label] for label in labels] for tag, labels in EVENT_LABELS.items()} | |
| scores = {} | |
| for path in sorted(samples.glob("*.wav")): | |
| audio, _ = sf.read(path, dtype="float32") | |
| inputs = features(resample_poly(audio, 2, 3).astype(np.float32), sampling_rate=16000, return_tensors="pt") | |
| with torch.inference_mode(): | |
| probabilities = model(**inputs).logits.sigmoid()[0] | |
| name = path.stem.split("-")[1] | |
| tags = [name] if name in label_ids else list(label_ids) | |
| scores[path.stem] = round(max(float(probabilities[label_ids[tag]].max()) for tag in tags), 3) | |
| return scores | |
| def realtime_factor(path: Path, sample_rate: int) -> float: | |
| options = ort.SessionOptions() | |
| options.intra_op_num_threads = THREADS | |
| session = ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"]) | |
| one = lambda value: np.array([value], dtype=np.float32) | |
| factors = [] | |
| for sentence in SENTENCES: | |
| phones = phonemize(sentence) | |
| inputs = { | |
| "tokens": np.array([phones.ids], dtype=np.int64), | |
| "stress": np.array([phones.stress], dtype=np.int64), | |
| "voice": np.array([0], dtype=np.int64), | |
| "speed": one(1.0), | |
| "pitch_shift": one(0.0), | |
| "pitch_range": one(1.0), | |
| } | |
| session.run(None, inputs) # warm up | |
| runs = [] | |
| for _ in range(5): | |
| started = time.perf_counter() | |
| audio, _ = session.run(None, inputs) | |
| runs.append(time.perf_counter() - started) | |
| factors.append(audio.shape[1] / sample_rate / statistics.median(runs)) | |
| return statistics.median(factors) | |
| def main() -> None: | |
| release = Path(sys.argv[1]) | |
| model = load_model(release / "fiato.pt") | |
| config = model.config | |
| report = { | |
| "expression_seconds": expression_lengths(model, config.hop / config.sample_rate), | |
| "expression_scores": expression_scores(release / "samples"), | |
| "onnx_realtime_factor": round(realtime_factor(release / "onnx" / "model.onnx", config.sample_rate), 1), | |
| "threads": THREADS, | |
| } | |
| Path("benchmark.json").write_text(json.dumps(report, indent=2) + "\n") | |
| print(json.dumps(report, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |