lfm3_engine.py

"""LFM3-Audio-2B as an optional second engine of the Kokoro Space (for A/B tests).

Off unless the Space variable LFM3_ENABLED=1. When on, the routes below sit
beside the Kokoro ones with the same contract (a client only changes the path):

  POST /lfm3/tts/stream   {"text", "voice", "sample_rate"}  -> raw PCM16 mono
  POST /lfm3/tts/bench    {"text", "voice"}                 -> seconds per piece
  GET  /lfm3/tts/health

The model (about 8 GB in float32 on CPU) loads on the first request, not at
start-up, so Kokoro boots and answers exactly as before; transformers is not
imported until then. One LFM3 synthesis at a time. "speed" is ignored (LFM3 has
no speed control; X-Speed-Ignored says so). Voices: "female", "male" or a
Kokoro id such as "af_heart" (see pieces.parse_voice).

LiquidAI/LFM3-Audio-2B is under the LFM Open License v1.0: commercial use only
below US$10M annual revenue.
"""
import asyncio
import os
import threading
import time
from typing import Callable

import numpy as np
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse

from pieces import parse_voice, system_prompt

MODEL_ID = os.getenv("LFM3_MODEL_ID", "LiquidAI/LFM3-Audio-2B")
MAX_NEW_TOKENS = int(os.getenv("LFM3_MAX_NEW_TOKENS", "256"))  # audio frames per piece
SAMPLE_RATE = 24000  # LFM3 codec output


def enabled() -> bool:
    return os.getenv("LFM3_ENABLED", "").strip().lower() in ("1", "true", "yes")


class Lfm3Engine:
    """Lazy, thread-safe LFM3 text-to-speech."""

    def __init__(self, threads: int) -> None:
        self.threads = threads
        self.device = "cpu"
        self.dtype = "float32"
        self._model = None
        self._processor = None
        self._codec = None
        self._load_lock = threading.Lock()
        self._synth_lock = threading.Lock()

    @property
    def loaded(self) -> bool:
        return self._model is not None

    def _load(self) -> None:
        with self._load_lock:
            if self._model is not None:
                return
            import torch
            from transformers import AutoModelForMultimodalLM, AutoProcessor

            started = time.perf_counter()
            self.device = "cuda" if torch.cuda.is_available() else "cpu"
            dtype = torch.bfloat16 if self.device == "cuda" else torch.float32
            self.dtype = str(dtype).replace("torch.", "")
            model = AutoModelForMultimodalLM.from_pretrained(
                MODEL_ID, dtype=dtype, device_map=self.device, trust_remote_code=True
            ).eval()
            processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True)
            self._codec = processor.audio_tokenizer.to(device=self.device).eval()
            self._processor = processor
            self._model = model  # set last: `loaded` means everything is ready
            print(f"[LFM3] loaded {MODEL_ID} on {self.device} ({self.dtype}) in {time.perf_counter() - started:.1f}s")

    def synth(self, text: str, language: str, gender: str) -> np.ndarray:
        """Float32 mono audio at 24 kHz for one piece of text."""
        import torch

        self._load()
        conversation = [
            {"role": "system", "content": system_prompt(language, gender)},
            {"role": "user", "content": text},
            {"role": "assistant", "content": "<|audio_start|>"},
        ]
        with self._synth_lock, torch.no_grad():
            inputs = self._processor.apply_chat_template(
                conversation,
                continue_final_message=True,
                tokenize=True,
                return_dict=True,
                return_tensors="pt",
            ).to(self.device)
            outputs = self._model.generate(
                **inputs,
                max_new_tokens=MAX_NEW_TOKENS,
                audio_temperature=1.0,
                audio_top_k=128,
                audio_repetition_penalty=1.1,
                audio_repetition_window=4096,
                return_dict_in_generate=True,
            )
            waveform = self._codec.decode(
                outputs.audio_codes[:1, :, : outputs.audio_lengths[0]]
            ).audio_values[0]
        return waveform.float().cpu().numpy().reshape(-1)


def build_router(
    *,
    engine: Lfm3Engine,
    check_key: Callable[[Request], None],
    split: Callable[[str], list],
    to_pcm16: Callable[[np.ndarray, int], bytes],
    max_chars: int,
) -> APIRouter:
    """Routes that share the Kokoro Space's API-key check, text splitting and
    PCM conversion."""
    router = APIRouter(prefix="/lfm3/tts")

    async def read_body(request: Request) -> dict:
        try:
            body = await request.json()
        except Exception:
            raise HTTPException(status_code=400, detail="json body required")
        if not isinstance(body, dict):
            raise HTTPException(status_code=400, detail="json object required")
        return body

    def voice_or_400(body: dict):
        parsed = parse_voice(str(body.get("voice") or "female"))
        if parsed is None:
            raise HTTPException(
                status_code=400, detail="unknown voice: use female, male or an a?_/b?_ Kokoro id"
            )
        return parsed

    @router.get("/health")
    def health():
        return {"ok": True, "engine": "lfm3", "model": MODEL_ID, "loaded": engine.loaded,
                "device": engine.device, "dtype": engine.dtype, "threads": engine.threads}

    @router.post("/stream")
    async def stream(request: Request):
        check_key(request)
        body = await read_body(request)
        text = str(body.get("text") or "").strip()[:max_chars]
        language, gender = voice_or_400(body)
        try:
            rate = int(body.get("sample_rate") or 16000)
        except (TypeError, ValueError):
            raise HTTPException(status_code=400, detail="invalid sample_rate")
        if rate not in (16000, 24000):
            raise HTTPException(status_code=400, detail="sample_rate must be 16000 or 24000")
        if not text:
            return JSONResponse({"detail": "empty text"}, status_code=400)

        pieces = split(text)

        def generate():
            started = time.perf_counter()
            for i, piece in enumerate(pieces):
                pcm = to_pcm16(engine.synth(piece, language, gender), rate)
                if i == 0:
                    print(f"[LFM3 stream] first piece in {time.perf_counter() - started:.2f}s ({len(piece)} chars)")
                yield pcm

        return StreamingResponse(
            generate(),  # sync generator: Starlette runs it in a worker thread
            media_type="application/octet-stream",
            headers={"X-Sample-Rate": str(rate), "X-Pieces": str(len(pieces)),
                     "X-Speed-Ignored": "1", "Cache-Control": "no-store"},
        )

    @router.post("/bench")
    async def bench(request: Request):
        """Seconds spent synthesizing against seconds of audio, per piece
        (rtf below 1 is faster than real time). The first call also loads the
        model, so run it twice and read the second."""
        check_key(request)
        body = await read_body(request)
        text = str(body.get("text") or "").strip()[:max_chars]
        language, gender = voice_or_400(body)
        if not text:
            return JSONResponse({"detail": "empty text"}, status_code=400)

        def run():
            rows = []
            for piece in split(text):
                started = time.perf_counter()
                audio = engine.synth(piece, language, gender)
                took = time.perf_counter() - started
                seconds = audio.size / SAMPLE_RATE
                rows.append({"chars": len(piece), "synth_seconds": round(took, 3),
                             "audio_seconds": round(seconds, 3),
                             "rtf": round(took / seconds, 3) if seconds else None})
            return rows

        rows = await asyncio.to_thread(run)
        return {"device": engine.device, "threads": engine.threads, "pieces": rows,
                "first_piece_seconds": rows[0]["synth_seconds"] if rows else None}

    return router