#!/usr/bin/env python3 """ VANI Speech API — hosted speech-to-text and text-to-speech Space. PURPOSE: Take the two real capabilities already proven locally in `dashboard_demo/vani_demo_service.py` (speech-to-text, text-to-speech) and make them reachable over HTTPS from the public website, so the website page stops depending on a service running on RP's own Mac at http://127.0.0.1:8901. WHEN TO RUN: as a Hugging Face Space (the platform runs `python app.py` and proxies port 7860), or locally for testing with the same command. WHEN NOT TO RUN: do not run this as the local development service — that role belongs to `dashboard_demo/vani_demo_service.py`, which keeps the local-only extras (macOS `say`, the LM Studio correction lane, the personal dictionary) that deliberately do NOT exist here because they are not portable. PORTABILITY, STATED PLAINLY: * Speech-to-text is faster-whisper (CTranslate2), not the MLX Whisper build used locally. MLX runs on Apple silicon only and would fail on this Linux host. Same model weights family, different runtime. * Text-to-speech is Kokoro (pure PyTorch, CPU), not macOS `say`. `say` is an operating-system binary that exists only on macOS. * Nothing in this file or in requirements.txt needs Apple hardware. """ import io import os import time import tempfile import threading import uuid import numpy as np import soundfile as sf from fastapi import FastAPI, File, HTTPException, UploadFile from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import HTMLResponse, Response from pydantic import BaseModel # -------------------------------------------------------------------------- # Configuration # -------------------------------------------------------------------------- STT_MODEL_SIZE = os.environ.get("VANI_STT_MODEL", "small") TTS_VOICE_DEFAULT = os.environ.get("VANI_TTS_VOICE", "af_heart") TTS_SAMPLE_RATE = 24000 # Kokoro's native output rate PORT = int(os.environ.get("PORT", "7860")) # Which browser origins may call this API. The website is the real consumer; # the localhost entries let the site be previewed locally against the hosted # API without a second deployment. ALLOWED_ORIGINS = [ "https://radhapreetam.com", "https://www.radhapreetam.com", "http://localhost:4321", "http://127.0.0.1:4321", "http://localhost:3000", "http://127.0.0.1:3000", ] _TMP_DIR = tempfile.mkdtemp(prefix="vani_api_") app = FastAPI(title="VANI Speech API", version="1.0.0") app.add_middleware( CORSMiddleware, allow_origins=ALLOWED_ORIGINS, allow_methods=["GET", "POST", "OPTIONS"], allow_headers=["*"], ) # -------------------------------------------------------------------------- # Lazily-loaded engines. # # Both are loaded on first use, not at import, so the Space finishes booting # and starts answering /health immediately. That matters: a Space that spends # two minutes loading models before it binds its port looks dead to every # health check and to any first-time visitor. Load state is reported honestly # through /health so the page can show a real loading indicator instead of # silence. # -------------------------------------------------------------------------- _lock = threading.Lock() _stt = None _tts = None _stt_error = None _tts_error = None def get_stt(): global _stt, _stt_error if _stt is not None or _stt_error is not None: return _stt with _lock: if _stt is None and _stt_error is None: try: from faster_whisper import WhisperModel t0 = time.time() _stt = WhisperModel(STT_MODEL_SIZE, device="cpu", compute_type="int8") print(f"[stt] faster-whisper '{STT_MODEL_SIZE}' loaded in " f"{time.time() - t0:.1f}s (cpu/int8)", flush=True) except Exception as e: # noqa: BLE001 - reported, never swallowed _stt_error = f"{type(e).__name__}: {e}" print(f"[stt] LOAD FAILED: {_stt_error}", flush=True) return _stt def get_tts(): global _tts, _tts_error if _tts is not None or _tts_error is not None: return _tts with _lock: if _tts is None and _tts_error is None: try: from kokoro import KPipeline t0 = time.time() _tts = KPipeline(lang_code="a") # 'a' = American English print(f"[tts] kokoro pipeline loaded in {time.time() - t0:.1f}s (cpu)", flush=True) except Exception as e: # noqa: BLE001 _tts_error = f"{type(e).__name__}: {e}" print(f"[tts] LOAD FAILED: {_tts_error}", flush=True) return _tts def _engine_state(obj, err): if err: return "error" return "ready" if obj is not None else "not_loaded" # -------------------------------------------------------------------------- # API # -------------------------------------------------------------------------- @app.get("/health") def health(): """Cheap, never loads a model. Answers immediately, even during cold start.""" return { "status": "ok", "service": "vani-speech-api", "stt_backend": f"faster-whisper ({STT_MODEL_SIZE}, cpu/int8)", "tts_backend": "kokoro (cpu)", "stt_state": _engine_state(_stt, _stt_error), "tts_state": _engine_state(_tts, _tts_error), "stt_error": _stt_error, "tts_error": _tts_error, "apple_silicon_required": False, } @app.post("/warmup") def warmup(): """Force both engines to load. Lets a caller pay the cold-start cost up front rather than inside a user's first real request.""" t0 = time.time() get_stt() stt_done = time.time() get_tts() return { "stt_state": _engine_state(_stt, _stt_error), "tts_state": _engine_state(_tts, _tts_error), "stt_load_seconds": round(stt_done - t0, 1), "tts_load_seconds": round(time.time() - stt_done, 1), "stt_error": _stt_error, "tts_error": _tts_error, } @app.post("/speech-to-text") async def speech_to_text(audio: UploadFile = File(...)): raw = await audio.read() if not raw: raise HTTPException(400, "Empty audio upload — no bytes received.") model = get_stt() if model is None: raise HTTPException(503, f"Speech-to-text engine unavailable: {_stt_error}") # faster-whisper decodes via PyAV, which reads the browser's webm/opus and # mp4/aac MediaRecorder output directly. The local service needed an # explicit ffmpeg conversion step because it fed bytes to Python's stdlib # `wave` module, which only understands RIFF/PCM. Passing a path here # avoids that whole class of bug. suffix = os.path.splitext(audio.filename or "recording.webm")[1] or ".webm" path = os.path.join(_TMP_DIR, f"in_{uuid.uuid4().hex}{suffix}") with open(path, "wb") as f: f.write(raw) try: t0 = time.time() segments, info = model.transcribe(path, beam_size=1, vad_filter=True) text = "".join(seg.text for seg in segments).strip() elapsed = time.time() - t0 except Exception as e: # noqa: BLE001 # The exception text from the decoder embeds the server-side temp file # path. That is useless to the user and needlessly exposes the server's # filesystem layout to the browser, so it is logged here and kept out of # the response body. print(f"[stt] decode/transcribe failed for {path}: {type(e).__name__}: {e}", flush=True) raise HTTPException( 422, "That audio could not be read. Re-record and try again — hold the " "button down while you speak. If it keeps happening, your browser is " "producing a recording format this server cannot decode.", ) finally: try: os.remove(path) except OSError: pass print(f"[stt] {len(raw)} bytes ({audio.content_type}) -> {text!r} in {elapsed:.2f}s", flush=True) return { "text": text, "language": getattr(info, "language", None), "confidence": round(float(getattr(info, "language_probability", 0.0)), 4), "seconds": round(elapsed, 2), "audio_seconds": round(float(getattr(info, "duration", 0.0)), 2), "note": None if text else "No speech detected in that recording.", } class TTSRequest(BaseModel): text: str voice: str | None = None @app.post("/text-to-speech") def text_to_speech(req: TTSRequest): text = (req.text or "").strip() if not text: raise HTTPException(400, "text must not be empty") if len(text) > 1000: raise HTTPException(400, "text is limited to 1000 characters per request") pipeline = get_tts() if pipeline is None: raise HTTPException(503, f"Text-to-speech engine unavailable: {_tts_error}") voice = req.voice or TTS_VOICE_DEFAULT try: t0 = time.time() chunks = [] for _gs, _ps, audio_chunk in pipeline(text, voice=voice): chunks.append(np.asarray(audio_chunk, dtype=np.float32)) if not chunks: raise HTTPException(500, "Text-to-speech produced no audio for that text.") samples = np.concatenate(chunks) elapsed = time.time() - t0 except HTTPException: raise except Exception as e: # noqa: BLE001 raise HTTPException(500, f"Text-to-speech failed ({type(e).__name__}: {e}). " f"Try a shorter, plainer sentence.") buf = io.BytesIO() sf.write(buf, samples, TTS_SAMPLE_RATE, format="WAV", subtype="PCM_16") wav = buf.getvalue() print(f"[tts] {len(text)} chars -> {len(wav)} bytes wav " f"({len(samples) / TTS_SAMPLE_RATE:.1f}s audio) in {elapsed:.2f}s", flush=True) return Response( content=wav, media_type="audio/wav", headers={ "Content-Disposition": 'inline; filename="vani.wav"', "X-Vani-Seconds": str(round(elapsed, 2)), }, ) # -------------------------------------------------------------------------- # Demo page. # # Interaction model follows Wispr Flow: one large press-to-talk control, no # surrounding chrome competing for attention, and the transcript appearing in # place. Deliberate UX decisions, each one a failure mode seen before: # * The microphone permission is EXPLAINED before it is requested. An # unexplained browser permission prompt is the single most common # first-use dead end. # * A real loading state covers cold start. Hosted CPU inference takes over # a minute on first call and silence reads as broken. # * The text-to-speech box ships with a worked example already in it rather # than a blank field. # * Every error says what to do next, not just that something failed. # -------------------------------------------------------------------------- PAGE = """ VANI — Speech Demo

VANI — Speech Demo

Speech to text, and text to speech. Running on hosted CPU — no software to install.

checking the server…

Speak, and see the words

Before you press the button: your browser will ask for permission to use the microphone. Nothing is recorded until you press and hold. The audio is sent to this server, turned into text, and discarded — it is not stored.
Press and hold, say a sentence, then let go. Try: “Book the badminton court for Sunday morning.”
Your words will appear here.

Type, and hear it spoken

An example is filled in already — press play, or replace it with your own.

""" @app.get("/", response_class=HTMLResponse) def index(): return HTMLResponse(PAGE) # -------------------------------------------------------------------------- # Gradio surface at /ui. # # Mounted so the Space behaves like a normal Gradio Space (the platform's own # expectation for sdk: gradio), and so there is a second, dependency-free way # to exercise the same two engines if the custom page ever misbehaves. # -------------------------------------------------------------------------- def _build_gradio(): import gradio as gr def _stt_fn(filepath): if not filepath: return "Record or upload some audio first." model = get_stt() if model is None: return f"Speech-to-text engine unavailable: {_stt_error}" segments, info = model.transcribe(filepath, beam_size=1, vad_filter=True) return "".join(s.text for s in segments).strip() or "(no speech detected)" def _tts_fn(text): pipeline = get_tts() if pipeline is None: raise gr.Error(f"Text-to-speech engine unavailable: {_tts_error}") chunks = [np.asarray(a, dtype=np.float32) for _g, _p, a in pipeline(text or " ", voice=TTS_VOICE_DEFAULT)] return (TTS_SAMPLE_RATE, np.concatenate(chunks)) with gr.Blocks(title="VANI Speech") as demo: gr.Markdown("### VANI Speech — engine check\nThe main demo is at `/`.") with gr.Tab("Speech to text"): a_in = gr.Audio(sources=["microphone", "upload"], type="filepath", label="Record or upload") t_out = gr.Textbox(label="Transcript") gr.Button("Transcribe").click(_stt_fn, a_in, t_out) with gr.Tab("Text to speech"): t_in = gr.Textbox(label="Text", value="Good evening. This is VANI, reading your words aloud.") a_out = gr.Audio(label="Audio") gr.Button("Speak").click(_tts_fn, t_in, a_out) return demo try: import gradio as gr # noqa: E402 app = gr.mount_gradio_app(app, _build_gradio(), path="/ui") print("[boot] gradio surface mounted at /ui", flush=True) except Exception as _e: # noqa: BLE001 # A missing or incompatible gradio must not take the HTTP API down — the # API is the deliverable, /ui is the convenience. print(f"[boot] gradio surface unavailable ({type(_e).__name__}: {_e}); " f"HTTP API still served", flush=True) if __name__ == "__main__": import uvicorn print(f"[boot] VANI Speech API listening on 0.0.0.0:{PORT}", flush=True) uvicorn.run(app, host="0.0.0.0", port=PORT)