"""Check the released model locally and write benchmark.json. Measures how long the model makes each expression, whether an audio event classifier hears the expressions in the released samples, and the ONNX speed on the CPU. DIZIONE_DATA=release/phonemizer FIATO_FRONTEND=own python benchmark.py release """ import json import statistics import sys import time from pathlib import Path import numpy as np import onnxruntime as ort import soundfile as sf import torch from scipy.signal import resample_poly from transformers import ASTFeatureExtractor, ASTForAudioClassification from fiato.infer import load_model, speak from fiato.text import EXPRESSIONS, phonemize THREADS = 4 EVENT_MODEL_ID = "MIT/ast-finetuned-audioset-10-10-0.4593" # AudioSet classes that count as hearing each expression, as in generate.py. EVENT_LABELS = {"sigh": ["Sigh", "Breathing"], "exhale": ["Sigh", "Breathing"], "gasp": ["Gasp", "Breathing"]} SENTENCES = ( "Hello! This is fiato.", "The weather is nice today, and I feel very relaxed.", "Did you really pay forty two dollars for that on March third?", "After a long journey across the mountains, the travellers reached the small village, where they were welcomed.", ) def expression_lengths(model, frame_seconds: float) -> dict: """Seconds the model gives each expression, in the middle of a sentence, for both voices.""" lengths = {} for tag in EXPRESSIONS: for voice in model.config.speakers: _, phones, details = speak(model, f"Well, {tag} I guess we should start over.", voice) frames = int(details["durations"][0][phones.symbols.index(tag)]) lengths[f"{tag} {voice}"] = round(frames * frame_seconds, 2) return lengths def expression_scores(samples: Path) -> dict: """Classifier score of each released sample for its own expression. A plain sample is scored for every expression and its highest score kept, as the level of ordinary speech.""" features = ASTFeatureExtractor.from_pretrained(EVENT_MODEL_ID) model = ASTForAudioClassification.from_pretrained(EVENT_MODEL_ID).eval() label_ids = {tag: [model.config.label2id[label] for label in labels] for tag, labels in EVENT_LABELS.items()} scores = {} for path in sorted(samples.glob("*.wav")): audio, _ = sf.read(path, dtype="float32") inputs = features(resample_poly(audio, 2, 3).astype(np.float32), sampling_rate=16000, return_tensors="pt") with torch.inference_mode(): probabilities = model(**inputs).logits.sigmoid()[0] name = path.stem.split("-")[1] tags = [name] if name in label_ids else list(label_ids) scores[path.stem] = round(max(float(probabilities[label_ids[tag]].max()) for tag in tags), 3) return scores def realtime_factor(path: Path, sample_rate: int) -> float: options = ort.SessionOptions() options.intra_op_num_threads = THREADS session = ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"]) one = lambda value: np.array([value], dtype=np.float32) factors = [] for sentence in SENTENCES: phones = phonemize(sentence) inputs = { "tokens": np.array([phones.ids], dtype=np.int64), "stress": np.array([phones.stress], dtype=np.int64), "voice": np.array([0], dtype=np.int64), "speed": one(1.0), "pitch_shift": one(0.0), "pitch_range": one(1.0), } session.run(None, inputs) # warm up runs = [] for _ in range(5): started = time.perf_counter() audio, _ = session.run(None, inputs) runs.append(time.perf_counter() - started) factors.append(audio.shape[1] / sample_rate / statistics.median(runs)) return statistics.median(factors) def main() -> None: release = Path(sys.argv[1]) model = load_model(release / "fiato.pt") config = model.config report = { "expression_seconds": expression_lengths(model, config.hop / config.sample_rate), "expression_scores": expression_scores(release / "samples"), "onnx_realtime_factor": round(realtime_factor(release / "onnx" / "model.onnx", config.sample_rate), 1), "threads": THREADS, } Path("benchmark.json").write_text(json.dumps(report, indent=2) + "\n") print(json.dumps(report, indent=2)) if __name__ == "__main__": main()