"""Thin streaming API over the PyTorch Breeze inference runtime.""" from __future__ import annotations import argparse import tempfile import threading import uuid from collections.abc import AsyncIterator, Iterator from contextlib import asynccontextmanager from dataclasses import dataclass, replace from pathlib import Path import numpy as np from fastapi import FastAPI, File, Form, HTTPException, UploadFile from fastapi.responses import JSONResponse, StreamingResponse from breeze_infer.runtime import ( load_runtime, resolve_device, set_all_seeds, update_generation_config_for_breeze, ) from breeze_infer.templates import get_template, prepare_inputs from models.fast_streaming import FastBreezeStreamingRuntime, FastStreamingConfig from models.warmup_profile import load_warmup_profile REPO_ROOT = Path(__file__).resolve().parents[1] FAST_CONFIG = REPO_ROOT / "configs" / "fast.json" DEFAULT_CFG_SCALE = 1.0 MAX_NEW_TOKENS = 1500 MAX_SEQ_LEN = 2048 REPETITION_PENALTY = 1.1 OPTIONAL_AUDIO_FILE = File(None) @dataclass(frozen=True) class ApiSettings: model: Path fast_all: bool | None fast_text_encoder: bool fast_backbone_prefill: bool fast_backbone_decode: bool fast_depth_decoder: bool fast_codec: bool _settings: ApiSettings | None = None _request_lock = threading.Lock() def _pcm16(audio: np.ndarray) -> bytes: audio = np.asarray(audio, dtype=np.float32) audio = np.clip(audio, -1.0, 1.0) return (audio * 32767.0).astype(" Path: suffix = Path(upload.filename or "reference.wav").suffix or ".wav" with tempfile.NamedTemporaryFile( prefix="breeze_ref_", suffix=suffix, delete=False ) as temporary: path = Path(temporary.name) try: payload = await upload.read() if not payload: raise HTTPException(status_code=400, detail="Reference audio is empty.") temporary.write(payload) except Exception: path.unlink(missing_ok=True) raise return path def _load_app(app: FastAPI, settings: ApiSettings) -> None: tokenizer, model, audio_tokenizer = load_runtime( settings.model, device=resolve_device(), attn_implementation="eager", ) update_generation_config_for_breeze(model) config = FastStreamingConfig( max_new_tokens=MAX_NEW_TOKENS, max_seq_len=MAX_SEQ_LEN, fast_all=settings.fast_all, fast_text_encoder=settings.fast_text_encoder, fast_backbone_prefill=settings.fast_backbone_prefill, fast_backbone_decode=settings.fast_backbone_decode, fast_depth_decoder=settings.fast_depth_decoder, fast_codec=settings.fast_codec, repetition_penalty=REPETITION_PENALTY, ) runtime = FastBreezeStreamingRuntime( model, audio_tokenizer, config, tokenizer=tokenizer ) if runtime.fast_enabled: profile = load_warmup_profile(FAST_CONFIG) profile = replace(profile, codec_chunk_frames=runtime.codec_chunk_frames) manifest = runtime.warmup_from_profile(profile) print(f"fast warmup: {manifest['total_elapsed_ms']:.2f} ms", flush=True) app.state.tokenizer = tokenizer app.state.model = model app.state.audio_tokenizer = audio_tokenizer app.state.runtime = runtime @asynccontextmanager async def _lifespan(app: FastAPI) -> AsyncIterator[None]: if _settings is None: raise RuntimeError("API settings are not initialized") _load_app(app, _settings) yield app = FastAPI(title="Breeze TTS API", lifespan=_lifespan) @app.get("/health") def health() -> JSONResponse: if not hasattr(app.state, "runtime"): return JSONResponse({"status": "loading"}, status_code=503) return JSONResponse({"status": "ok", "sample_rate": app.state.runtime.sample_rate}) @app.post("/v1/audio/speech") async def speech( text: str = Form(...), instruction: str = Form("Speak clearly and naturally."), cfg_scale: float = Form(DEFAULT_CFG_SCALE), ref_audio: UploadFile | None = OPTIONAL_AUDIO_FILE, ref_text: str = Form(""), seed: int = Form(42), ) -> StreamingResponse: if not _request_lock.acquire(blocking=False): raise HTTPException( status_code=409, detail="An inference request is already running." ) reference_path: Path | None = None try: if not np.isfinite(cfg_scale) or cfg_scale <= 0: raise HTTPException( status_code=400, detail="cfg_scale must be greater than 0." ) ref_text = ref_text.strip() has_reference = ref_audio is not None and bool(ref_audio.filename) if has_reference != bool(ref_text): raise HTTPException( status_code=400, detail="ref_audio and ref_text must be provided together or both omitted.", ) if has_reference: assert ref_audio is not None reference_path = await _save_upload(ref_audio) request_id = f"api-{uuid.uuid4().hex}" request = { "id": request_id, "text": text, "instruction": instruction, "speaker": "S0", } template_name = "tts_instruction" if reference_path is not None: request["ref_audio_path"] = str(reference_path) request["ref_text"] = ref_text template_name = "ref_edit_tata" set_all_seeds(seed) inputs = prepare_inputs( app.state.tokenizer, app.state.audio_tokenizer, app.state.model, [request], get_template(template_name), guidance_scale=cfg_scale, guidance_scale_ref=None, guidance_scale_ins=None, ) except Exception: if reference_path is not None: reference_path.unlink(missing_ok=True) _request_lock.release() raise def body() -> Iterator[bytes]: try: for chunk in app.state.runtime.iter_audio_chunks( inputs, request_id=request_id ): pcm = _pcm16(chunk.audio) if pcm: yield pcm finally: if reference_path is not None: reference_path.unlink(missing_ok=True) _request_lock.release() return StreamingResponse( body(), media_type="audio/pcm", headers={ "X-Sample-Rate": str(app.state.runtime.sample_rate), "X-Sample-Format": "s16le", "Cache-Control": "no-store", }, ) def main() -> None: parser = argparse.ArgumentParser( description="Serve Breeze TTS 2 streaming inference" ) parser.add_argument("model", type=Path) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=7860) parser.add_argument( "--fast-all", action=argparse.BooleanOptionalAction, default=None ) parser.add_argument( "--fast-text-encoder", action=argparse.BooleanOptionalAction, default=False ) parser.add_argument( "--fast-backbone-prefill", action=argparse.BooleanOptionalAction, default=False ) parser.add_argument( "--fast-backbone-decode", action=argparse.BooleanOptionalAction, default=False ) parser.add_argument( "--fast-depth-decoder", action=argparse.BooleanOptionalAction, default=False ) parser.add_argument( "--fast-codec", action=argparse.BooleanOptionalAction, default=False ) args = parser.parse_args() global _settings _settings = ApiSettings( model=args.model, fast_all=args.fast_all, fast_text_encoder=args.fast_text_encoder, fast_backbone_prefill=args.fast_backbone_prefill, fast_backbone_decode=args.fast_backbone_decode, fast_depth_decoder=args.fast_depth_decoder, fast_codec=args.fast_codec, ) import uvicorn uvicorn.run(app, host=args.host, port=args.port, log_level="info") if __name__ == "__main__": main()