Spaces:
Paused
Paused
File size: 4,725 Bytes
05f095d 077fd06 05f095d 077fd06 05f095d 0695bec 05f095d 0695bec 05f095d 0695bec 05f095d 0695bec 05f095d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | import os
import threading
import warnings
import numpy as np
from faster_whisper import WhisperModel
from huggingface_hub import InferenceClient
from sentence_transformers import SentenceTransformer, CrossEncoder
from config import (
CHAT_CHECK_TIMEOUT,
CLIP_MODEL, LLM_TIMEOUT, LLM_MODEL,
VISUAL_CONCEPTS, WHISPER_CPU_THREADS, WHISPER_FAST, WHISPER_PRECISE,
)
warnings.filterwarnings("ignore", message="FP16 is not supported on CPU; using FP32 instead")
def _best_torch_device() -> str:
try:
import torch
if torch.cuda.is_available():
return "cuda"
if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
return "mps"
except Exception:
pass
return "cpu"
TORCH_DEVICE = _best_torch_device()
print(f"[NINA] Torch device: {TORCH_DEVICE} | Whisper: CPU int8 (x{WHISPER_CPU_THREADS})")
print("[NINA] Loading Whisper base (faster-whisper)...")
whisper_fast = WhisperModel(
WHISPER_FAST, device="cpu", compute_type="int8", cpu_threads=WHISPER_CPU_THREADS
)
# Bi-encoder stays on CPU: tiny (sub-100 ms), keeps MPS free for CLIP during parallel cold start.
print("[NINA] Loading bi-encoder...")
bi_encoder = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2", device="cpu")
print(f"[NINA] Loading cross-encoder ({TORCH_DEVICE})...")
cross_encoder = CrossEncoder(
"cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", device=TORCH_DEVICE
)
HF_TOKEN = os.getenv("HF_TOKEN")
if not HF_TOKEN:
_token_file = os.path.expanduser("~/.cache/huggingface/token")
if os.path.exists(_token_file):
HF_TOKEN = open(_token_file).read().strip()
if HF_TOKEN:
print("[NINA] HF_TOKEN loaded β LLM calls enabled.")
else:
print("[NINA] HF_TOKEN not set β LLM calls will fail.")
llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
# Short leash for the one-word checks in chat.py; see CHAT_CHECK_TIMEOUT.
check_client = InferenceClient(token=HF_TOKEN, timeout=CHAT_CHECK_TIMEOUT)
# ββ Lazy: precise Whisper βββββββββββββββββββββββββββββββββββββββββββββββββββββ
_whisper_precise = None
_whisper_precise_lock = threading.Lock()
_whisper_precise_failed = False
def get_whisper_precise():
"""Lazy-load the precise Whisper model. Returns None (never raises) if it can't
load β e.g. the model isn't downloadable or the Space is out of memory β so the
pipeline degrades to base-model text instead of crashing the request."""
global _whisper_precise, _whisper_precise_failed
with _whisper_precise_lock:
if _whisper_precise_failed:
return None
if _whisper_precise is None:
try:
print(f"[NINA] Loading Whisper {WHISPER_PRECISE} (faster-whisper)...")
_whisper_precise = WhisperModel(
WHISPER_PRECISE, device="cpu", compute_type="int8",
cpu_threads=WHISPER_CPU_THREADS,
)
print(f"[NINA] Whisper {WHISPER_PRECISE} ready.")
except Exception as e:
_whisper_precise_failed = True
print(f"[NINA] Whisper {WHISPER_PRECISE} failed to load: {e} "
f"β precise re-transcription disabled (using base text).")
return None
return _whisper_precise
# ββ Lazy: CLIP ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
_clip_model = None
_clip_lock = threading.Lock()
_concept_embs = None
_concept_lock = threading.Lock()
def get_clip() -> SentenceTransformer:
global _clip_model
with _clip_lock:
if _clip_model is None:
print(f"[NINA] Loading CLIP ({CLIP_MODEL}, {TORCH_DEVICE})...")
_clip_model = SentenceTransformer(CLIP_MODEL, device=TORCH_DEVICE)
print("[NINA] CLIP ready.")
return _clip_model
def get_concept_embs() -> np.ndarray:
"""Encode the zero-shot vocabulary once, L2-normalised."""
global _concept_embs
with _concept_lock:
if _concept_embs is None:
embs = get_clip().encode(
VISUAL_CONCEPTS, convert_to_numpy=True, show_progress_bar=False
)
_concept_embs = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
return _concept_embs
# Warm background
threading.Thread(target=get_whisper_precise, daemon=True).start()
threading.Thread(target=get_clip, daemon=True).start()
print(f"[NINA] Ready. ( {WHISPER_PRECISE} + CLIP warming up...)")
|