File size: 4,412 Bytes
eaa9bb7
 
 
 
 
 
 
 
54d3fd1
eaa9bb7
 
 
 
 
 
 
 
 
 
54d3fd1
eaa9bb7
54d3fd1
eaa9bb7
 
 
54d3fd1
 
 
 
 
 
 
 
 
 
 
 
 
 
eaa9bb7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54d3fd1
 
 
 
 
 
 
 
 
 
 
 
 
eaa9bb7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
"""ARIA Audio — TTS (facebook/mms-tts pol/eng) + voice emotion classifier.

Exists because neither model family is hosted by any serverless inference
provider; the ARIA AGENT app (and its Cloudflare proxy) call this Space.
"""

import io
import threading
import wave

import numpy as np
import soundfile as sf
import torch
from fastapi import FastAPI, Request, Response
from fastapi.responses import JSONResponse

app = FastAPI()

_lock = threading.Lock()
_tts = {}      # lang -> (model, tokenizer)   [MMS, English fallback]
_emo = {}      # "fe" / "model"
_piper = {}    # "pol" -> PiperVoice          [Polish FEMALE neural voice]

TTS_LANGS = ("pol", "eng")
EMOTION_MODEL = "superb/wav2vec2-base-superb-er"
# Female Polish neural voice (rhasspy/piper-voices). ARIA is a girl.
PIPER_POL = "pl/pl_PL/gosia/medium/pl_PL-gosia-medium"


def piper_pol():
    """Lazy-load the Polish female Piper voice (gosia)."""
    with _lock:
        if "pol" not in _piper:
            from piper import PiperVoice
            from huggingface_hub import hf_hub_download
            onnx = hf_hub_download("rhasspy/piper-voices", PIPER_POL + ".onnx")
            cfg = hf_hub_download("rhasspy/piper-voices", PIPER_POL + ".onnx.json")
            _piper["pol"] = PiperVoice.load(onnx, config_path=cfg)
        return _piper["pol"]


def tts_pipeline(lang: str):
    with _lock:
        if lang not in _tts:
            from transformers import AutoTokenizer, VitsModel
            name = f"facebook/mms-tts-{lang}"
            _tts[lang] = (VitsModel.from_pretrained(name), AutoTokenizer.from_pretrained(name))
        return _tts[lang]


def emotion_pipeline():
    with _lock:
        if "model" not in _emo:
            from transformers import AutoFeatureExtractor, AutoModelForAudioClassification
            _emo["fe"] = AutoFeatureExtractor.from_pretrained(EMOTION_MODEL)
            _emo["model"] = AutoModelForAudioClassification.from_pretrained(EMOTION_MODEL)
        return _emo["fe"], _emo["model"]


@app.get("/")
def health():
    return {"ok": True, "endpoints": ["/tts", "/emotion"]}


@app.post("/tts")
async def tts(request: Request):
    body = await request.json()
    text = str(body.get("text", "")).strip()[:1200]
    lang = body.get("lang", "pol")
    if lang not in TTS_LANGS:
        lang = "pol"
    if not text:
        return JSONResponse({"error": "empty text"}, status_code=400)

    # Polish → Piper FEMALE voice (gosia). If Piper is unavailable we return 503
    # (NOT the male MMS), so the app falls back to its on-device female voice.
    if lang == "pol":
        try:
            voice = piper_pol()
            buf = io.BytesIO()
            with wave.open(buf, "wb") as wav_file:
                voice.synthesize(text, wav_file)
            return Response(buf.getvalue(), media_type="audio/wav")
        except Exception as e:
            return JSONResponse({"error": f"piper unavailable: {e}"}, status_code=503)

    # English → MMS (only a fallback; the app prefers Kokoro for English).
    model, tokenizer = tts_pipeline(lang)
    inputs = tokenizer(text, return_tensors="pt")
    with torch.no_grad():
        waveform = model(**inputs).waveform[0].numpy()

    buf = io.BytesIO()
    sf.write(buf, waveform, model.config.sampling_rate, format="WAV", subtype="PCM_16")
    return Response(buf.getvalue(), media_type="audio/wav")


@app.post("/emotion")
async def emotion(request: Request):
    raw = await request.body()
    if len(raw) < 1000:
        return JSONResponse({"error": "no audio"}, status_code=400)
    try:
        data, sr = sf.read(io.BytesIO(raw), dtype="float32")
    except Exception:
        return JSONResponse({"error": "bad audio"}, status_code=400)
    if data.ndim > 1:
        data = data.mean(axis=1)
    if sr != 16000 and len(data) > 1:   # the app sends 16 kHz; resample defensively
        idx = np.linspace(0, len(data) - 1, max(1, int(len(data) * 16000 / sr)))
        data = data[idx.astype(np.int64)]
    data = data[: 16000 * 30]           # cap at 30 s

    fe, model = emotion_pipeline()
    inputs = fe(data, sampling_rate=16000, return_tensors="pt")
    with torch.no_grad():
        logits = model(**inputs).logits[0]
    probs = torch.softmax(logits, dim=-1)
    labels = model.config.id2label
    out = [{"label": labels[i], "score": float(p)} for i, p in enumerate(probs)]
    out.sort(key=lambda x: -x["score"])
    return out