File size: 4,725 Bytes
05f095d
 
 
 
 
 
 
 
 
 
077fd06
05f095d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
077fd06
 
05f095d
 
 
 
 
 
0695bec
05f095d
 
0695bec
 
 
 
 
05f095d
0695bec
 
05f095d
0695bec
 
 
 
 
 
 
 
 
 
 
 
05f095d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
import os
import threading
import warnings

import numpy as np
from faster_whisper import WhisperModel
from huggingface_hub import InferenceClient
from sentence_transformers import SentenceTransformer, CrossEncoder

from config import (
    CHAT_CHECK_TIMEOUT,
    CLIP_MODEL, LLM_TIMEOUT, LLM_MODEL,
    VISUAL_CONCEPTS, WHISPER_CPU_THREADS, WHISPER_FAST, WHISPER_PRECISE,
)

warnings.filterwarnings("ignore", message="FP16 is not supported on CPU; using FP32 instead")


def _best_torch_device() -> str:
    try:
        import torch
        if torch.cuda.is_available():
            return "cuda"
        if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
            return "mps"
    except Exception:
        pass
    return "cpu"


TORCH_DEVICE = _best_torch_device()

print(f"[NINA] Torch device: {TORCH_DEVICE}  |  Whisper: CPU int8 (x{WHISPER_CPU_THREADS})")

print("[NINA] Loading Whisper base (faster-whisper)...")
whisper_fast = WhisperModel(
    WHISPER_FAST, device="cpu", compute_type="int8", cpu_threads=WHISPER_CPU_THREADS
)

# Bi-encoder stays on CPU: tiny (sub-100 ms), keeps MPS free for CLIP during parallel cold start.
print("[NINA] Loading bi-encoder...")
bi_encoder = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2", device="cpu")

print(f"[NINA] Loading cross-encoder ({TORCH_DEVICE})...")
cross_encoder = CrossEncoder(
    "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", device=TORCH_DEVICE
)

HF_TOKEN = os.getenv("HF_TOKEN")
if not HF_TOKEN:
    _token_file = os.path.expanduser("~/.cache/huggingface/token")
    if os.path.exists(_token_file):
        HF_TOKEN = open(_token_file).read().strip()

if HF_TOKEN:
    print("[NINA] HF_TOKEN loaded β€” LLM calls enabled.")
else:
    print("[NINA] HF_TOKEN not set β€” LLM calls will fail.")

llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
# Short leash for the one-word checks in chat.py; see CHAT_CHECK_TIMEOUT.
check_client = InferenceClient(token=HF_TOKEN, timeout=CHAT_CHECK_TIMEOUT)


# ── Lazy: precise Whisper ─────────────────────────────────────────────────────

_whisper_precise = None
_whisper_precise_lock = threading.Lock()
_whisper_precise_failed = False


def get_whisper_precise():
    """Lazy-load the precise Whisper model. Returns None (never raises) if it can't
    load β€” e.g. the model isn't downloadable or the Space is out of memory β€” so the
    pipeline degrades to base-model text instead of crashing the request."""
    global _whisper_precise, _whisper_precise_failed
    with _whisper_precise_lock:
        if _whisper_precise_failed:
            return None
        if _whisper_precise is None:
            try:
                print(f"[NINA] Loading Whisper {WHISPER_PRECISE} (faster-whisper)...")
                _whisper_precise = WhisperModel(
                    WHISPER_PRECISE, device="cpu", compute_type="int8",
                    cpu_threads=WHISPER_CPU_THREADS,
                )
                print(f"[NINA] Whisper {WHISPER_PRECISE} ready.")
            except Exception as e:
                _whisper_precise_failed = True
                print(f"[NINA] Whisper {WHISPER_PRECISE} failed to load: {e} "
                      f"β€” precise re-transcription disabled (using base text).")
                return None
    return _whisper_precise


# ── Lazy: CLIP ────────────────────────────────────────────────────────────────

_clip_model = None
_clip_lock = threading.Lock()
_concept_embs = None
_concept_lock = threading.Lock()


def get_clip() -> SentenceTransformer:
    global _clip_model
    with _clip_lock:
        if _clip_model is None:
            print(f"[NINA] Loading CLIP ({CLIP_MODEL}, {TORCH_DEVICE})...")
            _clip_model = SentenceTransformer(CLIP_MODEL, device=TORCH_DEVICE)
            print("[NINA] CLIP ready.")
    return _clip_model


def get_concept_embs() -> np.ndarray:
    """Encode the zero-shot vocabulary once, L2-normalised."""
    global _concept_embs
    with _concept_lock:
        if _concept_embs is None:
            embs = get_clip().encode(
                VISUAL_CONCEPTS, convert_to_numpy=True, show_progress_bar=False
            )
            _concept_embs = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
    return _concept_embs


# Warm background
threading.Thread(target=get_whisper_precise, daemon=True).start()
threading.Thread(target=get_clip, daemon=True).start()
print(f"[NINA] Ready. ( {WHISPER_PRECISE} + CLIP warming up...)")