inference_lfm.py

"""
LFM2.5-Audio-1.5B inference module — end-to-end audio Q&A.

Replaces the 3-model pipeline (Whisper ASR + Qwen Q&A + Qwen TTS) with a single
multimodal model that accepts audio/text input and produces audio+text output.
"""
from __future__ import annotations

import logging
import os
import re
import threading
import time

if os.name == "nt":
    os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")  # Windows typically lacks Triton for torch.compile

import torch
import numpy as np

from runtime_config import GPU_INFERENCE_LOCK

logger = logging.getLogger(__name__)

_processor = None
_model = None
_model_lock = threading.Lock()

HF_REPO = "LiquidAI/LFM2.5-Audio-1.5B"
SAMPLE_RATE = 24000  # Mimi codec native rate (confirmed in official demo + library constants)

# Matches special boundary tokens like <|text_end|>, <|audio_start|> that leak into decoded text
_SPECIAL_TOKEN_RE = re.compile(r"<\|[^|>]+\|>")


def _select_device() -> torch.device:
    return torch.device("cuda" if torch.cuda.is_available() else "cpu")


def _select_dtype() -> torch.dtype:
    if not torch.cuda.is_available():
        return torch.float32
    cap = torch.cuda.get_device_capability()
    return torch.bfloat16 if cap[0] >= 8 else torch.float16


def _move_module(module, device: torch.device, dtype: torch.dtype):
    if not hasattr(module, "to"):
        return module
    try:
        return module.to(device=device, dtype=dtype)
    except TypeError:
        try:
            return module.to(device)
        except Exception:
            return module
    except Exception:
        return module


def _first_parameter_device(module, fallback: torch.device) -> torch.device:
    try:
        return next(module.parameters()).device
    except Exception:
        try:
            return next(module.buffers()).device
        except Exception:
            return fallback


def _module_device(module, fallback: torch.device) -> torch.device:
    try:
        return next(module.parameters()).device
    except Exception:
        return getattr(module, "device", fallback)


def _assemble_waveform(wav_chunks: list) -> np.ndarray | None:
    """Concatenate Mimi output chunks, normalize to peak 0.9, and return float32 array."""
    if not wav_chunks:
        return None
    try:
        waveform = torch.cat(wav_chunks, dim=-1).float().numpy().squeeze()
    except Exception as exc:
        logger.warning("Waveform assembly failed: %s", exc)
        return None

    if waveform.ndim == 0 or waveform.size == 0:
        return None

    peak = float(np.abs(waveform).max())
    logger.info("Waveform peak amplitude: %.6f  (samples: %d)", peak, waveform.size)

    if peak == 0.0:
        logger.warning("Waveform is all zeros — model generated silence")
        return None

    return waveform * (0.9 / peak)


def get_model_status() -> dict:
    """Return current model load state, device, dtype, and GPU memory."""
    status: dict = {
        "model_loaded": _model is not None,
        "repo": HF_REPO,
        "device": None,
        "dtype": None,
        "gpu_name": None,
        "gpu_memory_used_gb": None,
        "gpu_memory_reserved_gb": None,
    }
    if _model is not None:
        device = _module_device(_model, _select_device())
        status["device"] = str(device)
        try:
            status["dtype"] = str(next(_model.parameters()).dtype).replace("torch.", "")
        except Exception:
            pass
        if device.type == "cuda":
            try:
                status["gpu_name"] = torch.cuda.get_device_name(device)
                status["gpu_memory_used_gb"] = round(torch.cuda.memory_allocated(device) / 1e9, 2)
                status["gpu_memory_reserved_gb"] = round(torch.cuda.memory_reserved(device) / 1e9, 2)
            except Exception:
                pass
    return status


def get_lfm_model():
    """Load LFM2.5-Audio-1.5B model. Cached after first call."""
    global _processor, _model
    if _model is None:
        with _model_lock:
            if _model is None:
                from liquid_audio import LFM2AudioModel, LFM2AudioProcessor

                device = _select_device()
                dtype = _select_dtype()
                logger.info("Loading %s on %s (%s)...", HF_REPO, device, dtype)
                _processor = LFM2AudioProcessor.from_pretrained(HF_REPO, device=device).eval()
                try:
                    _model = LFM2AudioModel.from_pretrained(
                        HF_REPO,
                        dtype=dtype if device.type == "cuda" else torch.float32,
                        device=device,
                    ).eval()
                except TypeError:
                    _model = LFM2AudioModel.from_pretrained(HF_REPO).eval()

                moved_processor = _move_module(_processor, device, dtype)
                if moved_processor is not None:
                    _processor = moved_processor
                moved_model = _move_module(_model, device, dtype)
                if moved_model is not None:
                    _model = moved_model
                _model = _model.eval()
                # Force lazy Mimi construction after the processor is on the target device,
                # and fail early if the streaming decoder cannot run there.
                mimi = _processor.mimi.eval()
                if device.type == "cuda":
                    with torch.no_grad(), mimi.streaming(1):
                        mimi.decode(torch.randint(0, 2048, (1, 8, 1), device=device))
                logger.info("LFM2.5-Audio loaded on %s.", _module_device(_model, device))
    return _processor, _model


def warmup_lfm():
    """Run a dummy generation to warm the model after loading.
    Call once at startup after get_lfm_model() to eliminate first-query latency.
    """
    try:
        from liquid_audio import ChatState

        processor, model = get_lfm_model()
        device = _module_device(model, _select_device())
        dtype = next(model.parameters()).dtype

        with GPU_INFERENCE_LOCK, torch.no_grad():
            chat = ChatState(processor, dtype=dtype)
            chat.new_turn("system")
            chat.add_text("Respond with interleaved text and audio.")
            chat.end_turn()
            chat.new_turn("user")
            chat.add_text("Hi")
            chat.end_turn()
            chat.new_turn("assistant")

            for _ in model.generate_interleaved(
                **chat, max_new_tokens=10, audio_temperature=1.0, audio_top_k=4,
            ):
                pass  # Just trigger warmup, discard output

        logger.info("LFM warmup complete.")
    except Exception as exc:
        logger.warning("LFM warmup failed (non-fatal): %s", exc)


def answer_question_audio(
    question_audio_path: str | None = None,
    question_text: str | None = None,
    story_context: str = "",
    max_new_tokens: int = 150,
) -> tuple[str, np.ndarray | None, int]:
    """
    Answer a question about the story using LFM2.5-Audio end-to-end.

    Accepts either audio input (child's voice) or text input.
    Returns (answer_text, audio_waveform_or_None, sample_rate).
    """
    from liquid_audio import ChatState

    processor, model = get_lfm_model()
    device = _module_device(model, _select_device())
    dtype = next(model.parameters()).dtype

    with GPU_INFERENCE_LOCK, torch.no_grad():
        chat = ChatState(processor, dtype=dtype)

        # System prompt: format requirement + brevity constraint
        chat.new_turn("system")
        chat.add_text(
            "Respond with interleaved text and audio. "
            "Give a short, direct answer in 1-2 sentences. Do not repeat the question."
        )
        chat.end_turn()

        # User turn — story context as text prefix, then audio or text question
        chat.new_turn("user")
        if story_context:
            chat.add_text(
                f"Story context:\n{story_context[:2000]}\n\n"
                "Based only on the story above, answer briefly."
            )
        if question_audio_path:
            import librosa
            wav_np, sr = librosa.load(question_audio_path, sr=16000, mono=True)
            wav = torch.from_numpy(wav_np).unsqueeze(0).to(device)
            chat.add_audio(wav, sr)
        elif question_text:
            chat.add_text(question_text)
        else:
            return "Please ask a question!", None, SAMPLE_RATE
        # Closing constraint so the model sees it immediately before generating
        chat.add_text("Answer in 1-2 sentences only.")
        chat.end_turn()

        chat.new_turn("assistant")
        text_out: list[torch.Tensor] = []
        audio_out: list[torch.Tensor] = []
        text_token_count = 0
        t0 = time.perf_counter()

        for t in model.generate_interleaved(
            **chat,
            max_new_tokens=max_new_tokens,
            audio_temperature=1.0,
            audio_top_k=4,
        ):
            if t.numel() == 1:
                text_token_count += 1
                text_out.append(t)
            elif t.numel() == 8:
                audio_out.append(t)

        gen_time = time.perf_counter() - t0
        audio_frame_count = max(0, len(audio_out) - 1)  # last frame is EOS
        logger.info(
            "Generated: %d text tokens, %d audio frames (%.1f sec), %.1f s wall",
            text_token_count, audio_frame_count, audio_frame_count / 12.5, gen_time,
        )

    # Decode text — strip interleaved boundary special tokens that leak through
    answer_text = ""
    if text_out:
        raw_text = "".join(processor.text.decode(t.detach().cpu()) for t in text_out)
        answer_text = _SPECIAL_TOKEN_RE.sub("", raw_text).strip()

    # Decode audio via processor.decode (drops EOS frame)
    waveform = None
    if len(audio_out) > 1:
        audio_codes = torch.stack(audio_out[:-1], dim=1).unsqueeze(0)  # (1, 8, N)
        raw = processor.decode(audio_codes).cpu().float()
        waveform = raw[0].numpy()
        peak = float(np.abs(waveform).max())
        if peak > 0:
            waveform = waveform * (0.9 / peak)
        logger.info("Decoded audio: %.2f sec, peak %.4f", len(waveform) / SAMPLE_RATE, peak)

    return answer_text, waveform, SAMPLE_RATE


def text_to_audio_lfm(
    text: str,
    max_new_tokens: int = 1024,
) -> tuple[np.ndarray | None, int]:
    """Convert text to audio using LFM2.5-Audio in TTS mode.

    Feeds the text as a user message with a "read aloud" system prompt,
    then collects only the audio tokens from generate_interleaved.
    Returns (waveform_or_None, sample_rate).
    """
    from liquid_audio import ChatState

    if not text.strip():
        return None, SAMPLE_RATE

    processor, model = get_lfm_model()
    device = _module_device(model, _select_device())
    dtype = next(model.parameters()).dtype

    with GPU_INFERENCE_LOCK, torch.no_grad():
        chat = ChatState(processor, dtype=dtype)

        chat.new_turn("system")
        chat.add_text("Read the following text aloud clearly and naturally.")
        chat.end_turn()

        chat.new_turn("user")
        chat.add_text(text)
        chat.end_turn()

        chat.new_turn("assistant")
        wav_chunks = []
        audio_frame_count = 0

        mimi = processor.mimi.eval()
        mimi_device = _first_parameter_device(mimi, device)

        with mimi.streaming(1):
            for t in model.generate_interleaved(
                **chat,
                max_new_tokens=max_new_tokens,
                audio_temperature=1.0,
                audio_top_k=4,
            ):
                if t.numel() == 8:
                    if (t == 2048).any():
                        continue
                    audio_frame_count += 1
                    try:
                        wav_chunk = mimi.decode(t[None, :, None].to(device=mimi_device, dtype=torch.long))[0]
                        wav_chunks.append(wav_chunk.cpu())
                    except Exception as exc:
                        logger.warning("TTS decode skipped frame: %s", exc)

        logger.info("TTS: %d audio frames (%.1f sec)", audio_frame_count, audio_frame_count / 12.5)

    return _assemble_waveform(wav_chunks), SAMPLE_RATE