aria-audio / app.py
QuerryRaspBerry's picture
Polish TTS β†’ Piper female voice (gosia)
54d3fd1 verified
Raw History Blame Contribute Delete
4.41 kB
"""ARIA Audio β€” TTS (facebook/mms-tts pol/eng) + voice emotion classifier.
Exists because neither model family is hosted by any serverless inference
provider; the ARIA AGENT app (and its Cloudflare proxy) call this Space.
"""
import io
import threading
import wave
import numpy as np
import soundfile as sf
import torch
from fastapi import FastAPI, Request, Response
from fastapi.responses import JSONResponse
app = FastAPI()
_lock = threading.Lock()
_tts = {} # lang -> (model, tokenizer) [MMS, English fallback]
_emo = {} # "fe" / "model"
_piper = {} # "pol" -> PiperVoice [Polish FEMALE neural voice]
TTS_LANGS = ("pol", "eng")
EMOTION_MODEL = "superb/wav2vec2-base-superb-er"
# Female Polish neural voice (rhasspy/piper-voices). ARIA is a girl.
PIPER_POL = "pl/pl_PL/gosia/medium/pl_PL-gosia-medium"
def piper_pol():
"""Lazy-load the Polish female Piper voice (gosia)."""
with _lock:
if "pol" not in _piper:
from piper import PiperVoice
from huggingface_hub import hf_hub_download
onnx = hf_hub_download("rhasspy/piper-voices", PIPER_POL + ".onnx")
cfg = hf_hub_download("rhasspy/piper-voices", PIPER_POL + ".onnx.json")
_piper["pol"] = PiperVoice.load(onnx, config_path=cfg)
return _piper["pol"]
def tts_pipeline(lang: str):
with _lock:
if lang not in _tts:
from transformers import AutoTokenizer, VitsModel
name = f"facebook/mms-tts-{lang}"
_tts[lang] = (VitsModel.from_pretrained(name), AutoTokenizer.from_pretrained(name))
return _tts[lang]
def emotion_pipeline():
with _lock:
if "model" not in _emo:
from transformers import AutoFeatureExtractor, AutoModelForAudioClassification
_emo["fe"] = AutoFeatureExtractor.from_pretrained(EMOTION_MODEL)
_emo["model"] = AutoModelForAudioClassification.from_pretrained(EMOTION_MODEL)
return _emo["fe"], _emo["model"]
@app.get("/")
def health():
return {"ok": True, "endpoints": ["/tts", "/emotion"]}
@app.post("/tts")
async def tts(request: Request):
body = await request.json()
text = str(body.get("text", "")).strip()[:1200]
lang = body.get("lang", "pol")
if lang not in TTS_LANGS:
lang = "pol"
if not text:
return JSONResponse({"error": "empty text"}, status_code=400)
# Polish β†’ Piper FEMALE voice (gosia). If Piper is unavailable we return 503
# (NOT the male MMS), so the app falls back to its on-device female voice.
if lang == "pol":
try:
voice = piper_pol()
buf = io.BytesIO()
with wave.open(buf, "wb") as wav_file:
voice.synthesize(text, wav_file)
return Response(buf.getvalue(), media_type="audio/wav")
except Exception as e:
return JSONResponse({"error": f"piper unavailable: {e}"}, status_code=503)
# English β†’ MMS (only a fallback; the app prefers Kokoro for English).
model, tokenizer = tts_pipeline(lang)
inputs = tokenizer(text, return_tensors="pt")
with torch.no_grad():
waveform = model(**inputs).waveform[0].numpy()
buf = io.BytesIO()
sf.write(buf, waveform, model.config.sampling_rate, format="WAV", subtype="PCM_16")
return Response(buf.getvalue(), media_type="audio/wav")
@app.post("/emotion")
async def emotion(request: Request):
raw = await request.body()
if len(raw) < 1000:
return JSONResponse({"error": "no audio"}, status_code=400)
try:
data, sr = sf.read(io.BytesIO(raw), dtype="float32")
except Exception:
return JSONResponse({"error": "bad audio"}, status_code=400)
if data.ndim > 1:
data = data.mean(axis=1)
if sr != 16000 and len(data) > 1: # the app sends 16 kHz; resample defensively
idx = np.linspace(0, len(data) - 1, max(1, int(len(data) * 16000 / sr)))
data = data[idx.astype(np.int64)]
data = data[: 16000 * 30] # cap at 30 s
fe, model = emotion_pipeline()
inputs = fe(data, sampling_rate=16000, return_tensors="pt")
with torch.no_grad():
logits = model(**inputs).logits[0]
probs = torch.softmax(logits, dim=-1)
labels = model.config.id2label
out = [{"label": labels[i], "score": float(p)} for i, p in enumerate(probs)]
out.sort(key=lambda x: -x["score"])
return out