"""ARIA Audio — TTS (facebook/mms-tts pol/eng) + voice emotion classifier. Exists because neither model family is hosted by any serverless inference provider; the ARIA AGENT app (and its Cloudflare proxy) call this Space. """ import io import threading import wave import numpy as np import soundfile as sf import torch from fastapi import FastAPI, Request, Response from fastapi.responses import JSONResponse app = FastAPI() _lock = threading.Lock() _tts = {} # lang -> (model, tokenizer) [MMS, English fallback] _emo = {} # "fe" / "model" _piper = {} # "pol" -> PiperVoice [Polish FEMALE neural voice] TTS_LANGS = ("pol", "eng") EMOTION_MODEL = "superb/wav2vec2-base-superb-er" # Female Polish neural voice (rhasspy/piper-voices). ARIA is a girl. PIPER_POL = "pl/pl_PL/gosia/medium/pl_PL-gosia-medium" def piper_pol(): """Lazy-load the Polish female Piper voice (gosia).""" with _lock: if "pol" not in _piper: from piper import PiperVoice from huggingface_hub import hf_hub_download onnx = hf_hub_download("rhasspy/piper-voices", PIPER_POL + ".onnx") cfg = hf_hub_download("rhasspy/piper-voices", PIPER_POL + ".onnx.json") _piper["pol"] = PiperVoice.load(onnx, config_path=cfg) return _piper["pol"] def tts_pipeline(lang: str): with _lock: if lang not in _tts: from transformers import AutoTokenizer, VitsModel name = f"facebook/mms-tts-{lang}" _tts[lang] = (VitsModel.from_pretrained(name), AutoTokenizer.from_pretrained(name)) return _tts[lang] def emotion_pipeline(): with _lock: if "model" not in _emo: from transformers import AutoFeatureExtractor, AutoModelForAudioClassification _emo["fe"] = AutoFeatureExtractor.from_pretrained(EMOTION_MODEL) _emo["model"] = AutoModelForAudioClassification.from_pretrained(EMOTION_MODEL) return _emo["fe"], _emo["model"] @app.get("/") def health(): return {"ok": True, "endpoints": ["/tts", "/emotion"]} @app.post("/tts") async def tts(request: Request): body = await request.json() text = str(body.get("text", "")).strip()[:1200] lang = body.get("lang", "pol") if lang not in TTS_LANGS: lang = "pol" if not text: return JSONResponse({"error": "empty text"}, status_code=400) # Polish → Piper FEMALE voice (gosia). If Piper is unavailable we return 503 # (NOT the male MMS), so the app falls back to its on-device female voice. if lang == "pol": try: voice = piper_pol() buf = io.BytesIO() with wave.open(buf, "wb") as wav_file: voice.synthesize(text, wav_file) return Response(buf.getvalue(), media_type="audio/wav") except Exception as e: return JSONResponse({"error": f"piper unavailable: {e}"}, status_code=503) # English → MMS (only a fallback; the app prefers Kokoro for English). model, tokenizer = tts_pipeline(lang) inputs = tokenizer(text, return_tensors="pt") with torch.no_grad(): waveform = model(**inputs).waveform[0].numpy() buf = io.BytesIO() sf.write(buf, waveform, model.config.sampling_rate, format="WAV", subtype="PCM_16") return Response(buf.getvalue(), media_type="audio/wav") @app.post("/emotion") async def emotion(request: Request): raw = await request.body() if len(raw) < 1000: return JSONResponse({"error": "no audio"}, status_code=400) try: data, sr = sf.read(io.BytesIO(raw), dtype="float32") except Exception: return JSONResponse({"error": "bad audio"}, status_code=400) if data.ndim > 1: data = data.mean(axis=1) if sr != 16000 and len(data) > 1: # the app sends 16 kHz; resample defensively idx = np.linspace(0, len(data) - 1, max(1, int(len(data) * 16000 / sr))) data = data[idx.astype(np.int64)] data = data[: 16000 * 30] # cap at 30 s fe, model = emotion_pipeline() inputs = fe(data, sampling_rate=16000, return_tensors="pt") with torch.no_grad(): logits = model(**inputs).logits[0] probs = torch.softmax(logits, dim=-1) labels = model.config.id2label out = [{"label": labels[i], "score": float(p)} for i, p in enumerate(probs)] out.sort(key=lambda x: -x["score"]) return out