Spaces:
Running on Zero
Running on Zero
Download breeze_infer/api.py from Quantumbraid/agent.voice: direct link, hf CLI and curl.
- Browser
- Download file 8.2 kB
-
https://huggingface.co/spaces/Quantumbraid/agent.voice/resolve/main/breeze_infer/api.py
- Command line
-
hf download hf://spaces/Quantumbraid/agent.voice/breeze_infer/api.py
-
curl -L -o api.py https://huggingface.co/spaces/Quantumbraid/agent.voice/resolve/main/breeze_infer/api.py
8.2 kB
| """Thin streaming API over the PyTorch Breeze inference runtime.""" | |
| from __future__ import annotations | |
| import argparse | |
| import tempfile | |
| import threading | |
| import uuid | |
| from collections.abc import AsyncIterator, Iterator | |
| from contextlib import asynccontextmanager | |
| from dataclasses import dataclass, replace | |
| from pathlib import Path | |
| import numpy as np | |
| from fastapi import FastAPI, File, Form, HTTPException, UploadFile | |
| from fastapi.responses import JSONResponse, StreamingResponse | |
| from breeze_infer.runtime import ( | |
| load_runtime, | |
| resolve_device, | |
| set_all_seeds, | |
| update_generation_config_for_breeze, | |
| ) | |
| from breeze_infer.templates import get_template, prepare_inputs | |
| from models.fast_streaming import FastBreezeStreamingRuntime, FastStreamingConfig | |
| from models.warmup_profile import load_warmup_profile | |
| REPO_ROOT = Path(__file__).resolve().parents[1] | |
| FAST_CONFIG = REPO_ROOT / "configs" / "fast.json" | |
| DEFAULT_CFG_SCALE = 1.0 | |
| MAX_NEW_TOKENS = 1500 | |
| MAX_SEQ_LEN = 2048 | |
| REPETITION_PENALTY = 1.1 | |
| OPTIONAL_AUDIO_FILE = File(None) | |
| class ApiSettings: | |
| model: Path | |
| fast_all: bool | None | |
| fast_text_encoder: bool | |
| fast_backbone_prefill: bool | |
| fast_backbone_decode: bool | |
| fast_depth_decoder: bool | |
| fast_codec: bool | |
| _settings: ApiSettings | None = None | |
| _request_lock = threading.Lock() | |
| def _pcm16(audio: np.ndarray) -> bytes: | |
| audio = np.asarray(audio, dtype=np.float32) | |
| audio = np.clip(audio, -1.0, 1.0) | |
| return (audio * 32767.0).astype("<i2", copy=False).tobytes() | |
| async def _save_upload(upload: UploadFile) -> Path: | |
| suffix = Path(upload.filename or "reference.wav").suffix or ".wav" | |
| with tempfile.NamedTemporaryFile( | |
| prefix="breeze_ref_", suffix=suffix, delete=False | |
| ) as temporary: | |
| path = Path(temporary.name) | |
| try: | |
| payload = await upload.read() | |
| if not payload: | |
| raise HTTPException(status_code=400, detail="Reference audio is empty.") | |
| temporary.write(payload) | |
| except Exception: | |
| path.unlink(missing_ok=True) | |
| raise | |
| return path | |
| def _load_app(app: FastAPI, settings: ApiSettings) -> None: | |
| tokenizer, model, audio_tokenizer = load_runtime( | |
| settings.model, | |
| device=resolve_device(), | |
| attn_implementation="eager", | |
| ) | |
| update_generation_config_for_breeze(model) | |
| config = FastStreamingConfig( | |
| max_new_tokens=MAX_NEW_TOKENS, | |
| max_seq_len=MAX_SEQ_LEN, | |
| fast_all=settings.fast_all, | |
| fast_text_encoder=settings.fast_text_encoder, | |
| fast_backbone_prefill=settings.fast_backbone_prefill, | |
| fast_backbone_decode=settings.fast_backbone_decode, | |
| fast_depth_decoder=settings.fast_depth_decoder, | |
| fast_codec=settings.fast_codec, | |
| repetition_penalty=REPETITION_PENALTY, | |
| ) | |
| runtime = FastBreezeStreamingRuntime( | |
| model, audio_tokenizer, config, tokenizer=tokenizer | |
| ) | |
| if runtime.fast_enabled: | |
| profile = load_warmup_profile(FAST_CONFIG) | |
| profile = replace(profile, codec_chunk_frames=runtime.codec_chunk_frames) | |
| manifest = runtime.warmup_from_profile(profile) | |
| print(f"fast warmup: {manifest['total_elapsed_ms']:.2f} ms", flush=True) | |
| app.state.tokenizer = tokenizer | |
| app.state.model = model | |
| app.state.audio_tokenizer = audio_tokenizer | |
| app.state.runtime = runtime | |
| async def _lifespan(app: FastAPI) -> AsyncIterator[None]: | |
| if _settings is None: | |
| raise RuntimeError("API settings are not initialized") | |
| _load_app(app, _settings) | |
| yield | |
| app = FastAPI(title="Breeze TTS API", lifespan=_lifespan) | |
| def health() -> JSONResponse: | |
| if not hasattr(app.state, "runtime"): | |
| return JSONResponse({"status": "loading"}, status_code=503) | |
| return JSONResponse({"status": "ok", "sample_rate": app.state.runtime.sample_rate}) | |
| async def speech( | |
| text: str = Form(...), | |
| instruction: str = Form("Speak clearly and naturally."), | |
| cfg_scale: float = Form(DEFAULT_CFG_SCALE), | |
| ref_audio: UploadFile | None = OPTIONAL_AUDIO_FILE, | |
| ref_text: str = Form(""), | |
| seed: int = Form(42), | |
| ) -> StreamingResponse: | |
| if not _request_lock.acquire(blocking=False): | |
| raise HTTPException( | |
| status_code=409, detail="An inference request is already running." | |
| ) | |
| reference_path: Path | None = None | |
| try: | |
| if not np.isfinite(cfg_scale) or cfg_scale <= 0: | |
| raise HTTPException( | |
| status_code=400, detail="cfg_scale must be greater than 0." | |
| ) | |
| ref_text = ref_text.strip() | |
| has_reference = ref_audio is not None and bool(ref_audio.filename) | |
| if has_reference != bool(ref_text): | |
| raise HTTPException( | |
| status_code=400, | |
| detail="ref_audio and ref_text must be provided together or both omitted.", | |
| ) | |
| if has_reference: | |
| assert ref_audio is not None | |
| reference_path = await _save_upload(ref_audio) | |
| request_id = f"api-{uuid.uuid4().hex}" | |
| request = { | |
| "id": request_id, | |
| "text": text, | |
| "instruction": instruction, | |
| "speaker": "S0", | |
| } | |
| template_name = "tts_instruction" | |
| if reference_path is not None: | |
| request["ref_audio_path"] = str(reference_path) | |
| request["ref_text"] = ref_text | |
| template_name = "ref_edit_tata" | |
| set_all_seeds(seed) | |
| inputs = prepare_inputs( | |
| app.state.tokenizer, | |
| app.state.audio_tokenizer, | |
| app.state.model, | |
| [request], | |
| get_template(template_name), | |
| guidance_scale=cfg_scale, | |
| guidance_scale_ref=None, | |
| guidance_scale_ins=None, | |
| ) | |
| except Exception: | |
| if reference_path is not None: | |
| reference_path.unlink(missing_ok=True) | |
| _request_lock.release() | |
| raise | |
| def body() -> Iterator[bytes]: | |
| try: | |
| for chunk in app.state.runtime.iter_audio_chunks( | |
| inputs, request_id=request_id | |
| ): | |
| pcm = _pcm16(chunk.audio) | |
| if pcm: | |
| yield pcm | |
| finally: | |
| if reference_path is not None: | |
| reference_path.unlink(missing_ok=True) | |
| _request_lock.release() | |
| return StreamingResponse( | |
| body(), | |
| media_type="audio/pcm", | |
| headers={ | |
| "X-Sample-Rate": str(app.state.runtime.sample_rate), | |
| "X-Sample-Format": "s16le", | |
| "Cache-Control": "no-store", | |
| }, | |
| ) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser( | |
| description="Serve Breeze TTS 2 streaming inference" | |
| ) | |
| parser.add_argument("model", type=Path) | |
| parser.add_argument("--host", default="127.0.0.1") | |
| parser.add_argument("--port", type=int, default=7860) | |
| parser.add_argument( | |
| "--fast-all", action=argparse.BooleanOptionalAction, default=None | |
| ) | |
| parser.add_argument( | |
| "--fast-text-encoder", action=argparse.BooleanOptionalAction, default=False | |
| ) | |
| parser.add_argument( | |
| "--fast-backbone-prefill", action=argparse.BooleanOptionalAction, default=False | |
| ) | |
| parser.add_argument( | |
| "--fast-backbone-decode", action=argparse.BooleanOptionalAction, default=False | |
| ) | |
| parser.add_argument( | |
| "--fast-depth-decoder", action=argparse.BooleanOptionalAction, default=False | |
| ) | |
| parser.add_argument( | |
| "--fast-codec", action=argparse.BooleanOptionalAction, default=False | |
| ) | |
| args = parser.parse_args() | |
| global _settings | |
| _settings = ApiSettings( | |
| model=args.model, | |
| fast_all=args.fast_all, | |
| fast_text_encoder=args.fast_text_encoder, | |
| fast_backbone_prefill=args.fast_backbone_prefill, | |
| fast_backbone_decode=args.fast_backbone_decode, | |
| fast_depth_decoder=args.fast_depth_decoder, | |
| fast_codec=args.fast_codec, | |
| ) | |
| import uvicorn | |
| uvicorn.run(app, host=args.host, port=args.port, log_level="info") | |
| if __name__ == "__main__": | |
| main() | |