NINA / models.py
QuerryRaspBerry's picture
Claude Opus 5 (1M context)
Follow-up z kontekstu szybkim modelem, z limitem na sprawdzenia i z logami
077fd06
Raw History Blame
4.73 kB
import os
import threading
import warnings
import numpy as np
from faster_whisper import WhisperModel
from huggingface_hub import InferenceClient
from sentence_transformers import SentenceTransformer, CrossEncoder
from config import (
CHAT_CHECK_TIMEOUT,
CLIP_MODEL, LLM_TIMEOUT, LLM_MODEL,
VISUAL_CONCEPTS, WHISPER_CPU_THREADS, WHISPER_FAST, WHISPER_PRECISE,
)
warnings.filterwarnings("ignore", message="FP16 is not supported on CPU; using FP32 instead")
def _best_torch_device() -> str:
try:
import torch
if torch.cuda.is_available():
return "cuda"
if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
return "mps"
except Exception:
pass
return "cpu"
TORCH_DEVICE = _best_torch_device()
print(f"[NINA] Torch device: {TORCH_DEVICE} | Whisper: CPU int8 (x{WHISPER_CPU_THREADS})")
print("[NINA] Loading Whisper base (faster-whisper)...")
whisper_fast = WhisperModel(
WHISPER_FAST, device="cpu", compute_type="int8", cpu_threads=WHISPER_CPU_THREADS
)
# Bi-encoder stays on CPU: tiny (sub-100 ms), keeps MPS free for CLIP during parallel cold start.
print("[NINA] Loading bi-encoder...")
bi_encoder = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2", device="cpu")
print(f"[NINA] Loading cross-encoder ({TORCH_DEVICE})...")
cross_encoder = CrossEncoder(
"cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", device=TORCH_DEVICE
)
HF_TOKEN = os.getenv("HF_TOKEN")
if not HF_TOKEN:
_token_file = os.path.expanduser("~/.cache/huggingface/token")
if os.path.exists(_token_file):
HF_TOKEN = open(_token_file).read().strip()
if HF_TOKEN:
print("[NINA] HF_TOKEN loaded β€” LLM calls enabled.")
else:
print("[NINA] HF_TOKEN not set β€” LLM calls will fail.")
llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
# Short leash for the one-word checks in chat.py; see CHAT_CHECK_TIMEOUT.
check_client = InferenceClient(token=HF_TOKEN, timeout=CHAT_CHECK_TIMEOUT)
# ── Lazy: precise Whisper ─────────────────────────────────────────────────────
_whisper_precise = None
_whisper_precise_lock = threading.Lock()
_whisper_precise_failed = False
def get_whisper_precise():
"""Lazy-load the precise Whisper model. Returns None (never raises) if it can't
load β€” e.g. the model isn't downloadable or the Space is out of memory β€” so the
pipeline degrades to base-model text instead of crashing the request."""
global _whisper_precise, _whisper_precise_failed
with _whisper_precise_lock:
if _whisper_precise_failed:
return None
if _whisper_precise is None:
try:
print(f"[NINA] Loading Whisper {WHISPER_PRECISE} (faster-whisper)...")
_whisper_precise = WhisperModel(
WHISPER_PRECISE, device="cpu", compute_type="int8",
cpu_threads=WHISPER_CPU_THREADS,
)
print(f"[NINA] Whisper {WHISPER_PRECISE} ready.")
except Exception as e:
_whisper_precise_failed = True
print(f"[NINA] Whisper {WHISPER_PRECISE} failed to load: {e} "
f"β€” precise re-transcription disabled (using base text).")
return None
return _whisper_precise
# ── Lazy: CLIP ────────────────────────────────────────────────────────────────
_clip_model = None
_clip_lock = threading.Lock()
_concept_embs = None
_concept_lock = threading.Lock()
def get_clip() -> SentenceTransformer:
global _clip_model
with _clip_lock:
if _clip_model is None:
print(f"[NINA] Loading CLIP ({CLIP_MODEL}, {TORCH_DEVICE})...")
_clip_model = SentenceTransformer(CLIP_MODEL, device=TORCH_DEVICE)
print("[NINA] CLIP ready.")
return _clip_model
def get_concept_embs() -> np.ndarray:
"""Encode the zero-shot vocabulary once, L2-normalised."""
global _concept_embs
with _concept_lock:
if _concept_embs is None:
embs = get_clip().encode(
VISUAL_CONCEPTS, convert_to_numpy=True, show_progress_bar=False
)
_concept_embs = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
return _concept_embs
# Warm background
threading.Thread(target=get_whisper_precise, daemon=True).start()
threading.Thread(target=get_clip, daemon=True).start()
print(f"[NINA] Ready. ( {WHISPER_PRECISE} + CLIP warming up...)")