fiato / benchmark.py
cfahlgren1's picture
cfahlgren1 HF Staff
Upload benchmark.py with huggingface_hub
627e23e verified
Raw History Blame Contribute Delete
4.43 kB
"""Check the released model locally and write benchmark.json.
Measures how long the model makes each expression, whether an audio event
classifier hears the expressions in the released samples, and the ONNX speed
on the CPU.
DIZIONE_DATA=release/phonemizer FIATO_FRONTEND=own python benchmark.py release
"""
import json
import statistics
import sys
import time
from pathlib import Path
import numpy as np
import onnxruntime as ort
import soundfile as sf
import torch
from scipy.signal import resample_poly
from transformers import ASTFeatureExtractor, ASTForAudioClassification
from fiato.infer import load_model, speak
from fiato.text import EXPRESSIONS, phonemize
THREADS = 4
EVENT_MODEL_ID = "MIT/ast-finetuned-audioset-10-10-0.4593"
# AudioSet classes that count as hearing each expression, as in generate.py.
EVENT_LABELS = {"sigh": ["Sigh", "Breathing"], "exhale": ["Sigh", "Breathing"], "gasp": ["Gasp", "Breathing"]}
SENTENCES = (
"Hello! This is fiato.",
"The weather is nice today, and I feel very relaxed.",
"Did you really pay forty two dollars for that on March third?",
"After a long journey across the mountains, the travellers reached the small village, where they were welcomed.",
)
def expression_lengths(model, frame_seconds: float) -> dict:
"""Seconds the model gives each expression, in the middle of a sentence, for both voices."""
lengths = {}
for tag in EXPRESSIONS:
for voice in model.config.speakers:
_, phones, details = speak(model, f"Well, {tag} I guess we should start over.", voice)
frames = int(details["durations"][0][phones.symbols.index(tag)])
lengths[f"{tag} {voice}"] = round(frames * frame_seconds, 2)
return lengths
def expression_scores(samples: Path) -> dict:
"""Classifier score of each released sample for its own expression. A plain sample is
scored for every expression and its highest score kept, as the level of ordinary speech."""
features = ASTFeatureExtractor.from_pretrained(EVENT_MODEL_ID)
model = ASTForAudioClassification.from_pretrained(EVENT_MODEL_ID).eval()
label_ids = {tag: [model.config.label2id[label] for label in labels] for tag, labels in EVENT_LABELS.items()}
scores = {}
for path in sorted(samples.glob("*.wav")):
audio, _ = sf.read(path, dtype="float32")
inputs = features(resample_poly(audio, 2, 3).astype(np.float32), sampling_rate=16000, return_tensors="pt")
with torch.inference_mode():
probabilities = model(**inputs).logits.sigmoid()[0]
name = path.stem.split("-")[1]
tags = [name] if name in label_ids else list(label_ids)
scores[path.stem] = round(max(float(probabilities[label_ids[tag]].max()) for tag in tags), 3)
return scores
def realtime_factor(path: Path, sample_rate: int) -> float:
options = ort.SessionOptions()
options.intra_op_num_threads = THREADS
session = ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"])
one = lambda value: np.array([value], dtype=np.float32)
factors = []
for sentence in SENTENCES:
phones = phonemize(sentence)
inputs = {
"tokens": np.array([phones.ids], dtype=np.int64),
"stress": np.array([phones.stress], dtype=np.int64),
"voice": np.array([0], dtype=np.int64),
"speed": one(1.0),
"pitch_shift": one(0.0),
"pitch_range": one(1.0),
}
session.run(None, inputs) # warm up
runs = []
for _ in range(5):
started = time.perf_counter()
audio, _ = session.run(None, inputs)
runs.append(time.perf_counter() - started)
factors.append(audio.shape[1] / sample_rate / statistics.median(runs))
return statistics.median(factors)
def main() -> None:
release = Path(sys.argv[1])
model = load_model(release / "fiato.pt")
config = model.config
report = {
"expression_seconds": expression_lengths(model, config.hop / config.sample_rate),
"expression_scores": expression_scores(release / "samples"),
"onnx_realtime_factor": round(realtime_factor(release / "onnx" / "model.onnx", config.sample_rate), 1),
"threads": THREADS,
}
Path("benchmark.json").write_text(json.dumps(report, indent=2) + "\n")
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()