import os import threading import warnings import numpy as np from faster_whisper import WhisperModel from huggingface_hub import InferenceClient from sentence_transformers import SentenceTransformer, CrossEncoder from config import ( CHAT_CHECK_TIMEOUT, CLIP_MODEL, LLM_TIMEOUT, LLM_MODEL, VISUAL_CONCEPTS, WHISPER_CPU_THREADS, WHISPER_FAST, WHISPER_PRECISE, ) warnings.filterwarnings("ignore", message="FP16 is not supported on CPU; using FP32 instead") def _best_torch_device() -> str: try: import torch if torch.cuda.is_available(): return "cuda" if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available(): return "mps" except Exception: pass return "cpu" TORCH_DEVICE = _best_torch_device() print(f"[NINA] Torch device: {TORCH_DEVICE} | Whisper: CPU int8 (x{WHISPER_CPU_THREADS})") print("[NINA] Loading Whisper base (faster-whisper)...") whisper_fast = WhisperModel( WHISPER_FAST, device="cpu", compute_type="int8", cpu_threads=WHISPER_CPU_THREADS ) # Bi-encoder stays on CPU: tiny (sub-100 ms), keeps MPS free for CLIP during parallel cold start. print("[NINA] Loading bi-encoder...") bi_encoder = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2", device="cpu") print(f"[NINA] Loading cross-encoder ({TORCH_DEVICE})...") cross_encoder = CrossEncoder( "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", device=TORCH_DEVICE ) HF_TOKEN = os.getenv("HF_TOKEN") if not HF_TOKEN: _token_file = os.path.expanduser("~/.cache/huggingface/token") if os.path.exists(_token_file): HF_TOKEN = open(_token_file).read().strip() if HF_TOKEN: print("[NINA] HF_TOKEN loaded — LLM calls enabled.") else: print("[NINA] HF_TOKEN not set — LLM calls will fail.") llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT) # Short leash for the one-word checks in chat.py; see CHAT_CHECK_TIMEOUT. check_client = InferenceClient(token=HF_TOKEN, timeout=CHAT_CHECK_TIMEOUT) # ── Lazy: precise Whisper ───────────────────────────────────────────────────── _whisper_precise = None _whisper_precise_lock = threading.Lock() _whisper_precise_failed = False def get_whisper_precise(): """Lazy-load the precise Whisper model. Returns None (never raises) if it can't load — e.g. the model isn't downloadable or the Space is out of memory — so the pipeline degrades to base-model text instead of crashing the request.""" global _whisper_precise, _whisper_precise_failed with _whisper_precise_lock: if _whisper_precise_failed: return None if _whisper_precise is None: try: print(f"[NINA] Loading Whisper {WHISPER_PRECISE} (faster-whisper)...") _whisper_precise = WhisperModel( WHISPER_PRECISE, device="cpu", compute_type="int8", cpu_threads=WHISPER_CPU_THREADS, ) print(f"[NINA] Whisper {WHISPER_PRECISE} ready.") except Exception as e: _whisper_precise_failed = True print(f"[NINA] Whisper {WHISPER_PRECISE} failed to load: {e} " f"— precise re-transcription disabled (using base text).") return None return _whisper_precise # ── Lazy: CLIP ──────────────────────────────────────────────────────────────── _clip_model = None _clip_lock = threading.Lock() _concept_embs = None _concept_lock = threading.Lock() def get_clip() -> SentenceTransformer: global _clip_model with _clip_lock: if _clip_model is None: print(f"[NINA] Loading CLIP ({CLIP_MODEL}, {TORCH_DEVICE})...") _clip_model = SentenceTransformer(CLIP_MODEL, device=TORCH_DEVICE) print("[NINA] CLIP ready.") return _clip_model def get_concept_embs() -> np.ndarray: """Encode the zero-shot vocabulary once, L2-normalised.""" global _concept_embs with _concept_lock: if _concept_embs is None: embs = get_clip().encode( VISUAL_CONCEPTS, convert_to_numpy=True, show_progress_bar=False ) _concept_embs = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9) return _concept_embs # Warm background threading.Thread(target=get_whisper_precise, daemon=True).start() threading.Thread(target=get_clip, daemon=True).start() print(f"[NINA] Ready. ( {WHISPER_PRECISE} + CLIP warming up...)")