ir-studio / api /production_models.py
Natnael B. Haile
feat: complete IR Studio with Git LFS tracking for all binary assets and NFCorpus index
fbad523
Raw History Blame Contribute Delete
20.4 kB
import os
import glob
import time
import math
import hashlib
import threading
from typing import List, Dict, Any, Optional, Tuple
import numpy as np
# Try importing faiss
try:
import faiss
FAISS_AVAILABLE = True
except ImportError:
faiss = None
FAISS_AVAILABLE = False
# Try importing ONNX runtime and tokenizers
try:
import onnxruntime as ort
from tokenizers import Tokenizer
ONNX_AVAILABLE = True
except ImportError:
ort = None
Tokenizer = None
ONNX_AVAILABLE = False
def _find_snapshot_file(base_dir_pattern: str, file_rel_path: str) -> Optional[str]:
"""Helper to locate model files in Hugging Face snapshot caches or local model dirs."""
expanded_base = os.path.expanduser(base_dir_pattern)
matched_dirs = glob.glob(expanded_base)
for d in matched_dirs:
candidate = os.path.join(d, file_rel_path)
if os.path.exists(candidate):
return candidate
return None
class ProductionModelsManager:
def __init__(self):
self.faiss_index = None
self.docid_to_idx: Dict[str, int] = {}
self.faiss_loaded = False
self.faiss_index_path: Optional[str] = None
# ONNX sessions & tokenizers
self.bge_session = None
self.bge_tokenizer = None
self.bge_model_path: Optional[str] = None
self.splade_session = None
self.splade_tokenizer = None
self.splade_model_path: Optional[str] = None
# Thread locks for cache & inference safety
self.cache_lock = threading.Lock()
self.max_cache_size = int(os.getenv("MAX_CACHE_ENTRIES", "2000"))
# In-memory caches for instantaneous (<1 ms) retrieval
self.bge_cache: Dict[str, Dict[str, Any]] = {}
self.splade_cache: Dict[str, Dict[str, Any]] = {}
self._init_faiss()
self._init_onnx_models()
def _get_cache_key(self, prefix: str, doc_id: Optional[str], text: str) -> str:
"""Collision-proof cache key generation using doc_id or SHA-256 hash."""
if doc_id:
return f"{prefix}:id_{doc_id}"
text_hash = hashlib.sha256(text.strip().encode("utf-8")).hexdigest()[:24]
return f"{prefix}:h_{text_hash}"
def _store_in_cache(self, cache_dict: Dict[str, Any], key: str, value: Dict[str, Any]):
"""Thread-safe cache write with bounded eviction to prevent memory leaks."""
with self.cache_lock:
if len(cache_dict) >= self.max_cache_size:
# Evict oldest 20% entries
num_to_evict = max(1, self.max_cache_size // 5)
keys_to_remove = list(cache_dict.keys())[:num_to_evict]
for k in keys_to_remove:
cache_dict.pop(k, None)
cache_dict[key] = value
def _init_faiss(self):
# Configurable search paths for precomputed NFCorpus FAISS index
candidate_dirs = [
os.getenv("FAISS_INDEX_DIR", ""),
"/workspaces/pyserini/indexes/nfcorpus.bge-base-en-v1.5",
"./indexes/nfcorpus.bge-base-en-v1.5",
"/app/indexes/nfcorpus.bge-base-en-v1.5",
os.path.expanduser("~/.cache/ir-studio/indexes/nfcorpus.bge-base-en-v1.5"),
]
found_dir = None
for c in candidate_dirs:
if c and os.path.isdir(c):
idx_file = os.path.join(c, "index")
doc_file = os.path.join(c, "docid")
if os.path.exists(idx_file) and os.path.exists(doc_file):
found_dir = c
break
if not found_dir:
print("[ProductionModels] FAISS index directory not found. Precomputed vector lookups disabled.")
return
index_file = os.path.join(found_dir, "index")
docid_file = os.path.join(found_dir, "docid")
if FAISS_AVAILABLE and os.path.exists(index_file) and os.path.exists(docid_file):
try:
self.faiss_index = faiss.read_index(index_file)
with open(docid_file, "r") as f:
self.docid_to_idx = {line.strip(): i for i, line in enumerate(f)}
self.faiss_loaded = True
self.faiss_index_path = index_file
print(f"[ProductionModels] Loaded FAISS index from {index_file} ({len(self.docid_to_idx)} docids)")
except Exception as e:
print(f"[ProductionModels] Error loading FAISS index: {e}")
def _init_onnx_models(self):
if not ONNX_AVAILABLE:
print("[ProductionModels] ONNX Runtime or Tokenizers package not installed. Live neural inference disabled.")
return
# 1. Locate BGE-base ONNX model & tokenizer
bge_model_candidates = [
os.getenv("BGE_MODEL_PATH", ""),
"/home/codespace/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/4d6cd88e18e51a5e020c2c305726d76ada9c03cf/onnx/model_quantized.onnx",
_find_snapshot_file("~/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/*", "onnx/model_quantized.onnx") or "",
_find_snapshot_file("~/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/*", "onnx/model.onnx") or "",
"./models/bge/model_quantized.onnx",
"./models/bge/model.onnx",
"/app/models/bge/model_quantized.onnx",
]
bge_tok_candidates = [
os.getenv("BGE_TOKENIZER_PATH", ""),
"/home/codespace/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/4d6cd88e18e51a5e020c2c305726d76ada9c03cf/tokenizer.json",
_find_snapshot_file("~/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/*", "tokenizer.json") or "",
"./models/bge/tokenizer.json",
"/app/models/bge/tokenizer.json",
]
bge_model_path = next((p for p in bge_model_candidates if p and os.path.exists(p)), None)
bge_tok_path = next((p for p in bge_tok_candidates if p and os.path.exists(p)), None)
# Download from Hugging Face Hub if not found locally
if not bge_model_path or not bge_tok_path:
try:
from huggingface_hub import hf_hub_download
print("[ProductionModels] Attempting to download BGE-base ONNX from Hugging Face Hub...")
bge_model_path = bge_model_path or hf_hub_download(repo_id="Xenova/bge-base-en-v1.5", filename="onnx/model_quantized.onnx")
bge_tok_path = bge_tok_path or hf_hub_download(repo_id="Xenova/bge-base-en-v1.5", filename="tokenizer.json")
except Exception as e:
print(f"[ProductionModels] Hugging Face Hub download failed for BGE: {e}")
if bge_model_path and bge_tok_path:
try:
self.bge_tokenizer = Tokenizer.from_file(bge_tok_path)
self.bge_tokenizer.enable_truncation(max_length=512)
self.bge_tokenizer.no_padding()
num_threads = int(os.getenv("ONNX_NUM_THREADS", "4"))
opts = ort.SessionOptions()
opts.intra_op_num_threads = num_threads
self.bge_session = ort.InferenceSession(bge_model_path, opts)
self.bge_model_path = bge_model_path
print(f"[ProductionModels] Loaded BGE-base ONNX session from {bge_model_path} (threads={num_threads})")
except Exception as e:
print(f"[ProductionModels] Error loading BGE ONNX: {e}")
else:
print("[ProductionModels] BGE-base ONNX model files not found in standard paths.")
# 2. Locate SPLADE ONNX model & tokenizer
splade_model_candidates = [
os.getenv("SPLADE_MODEL_PATH", ""),
"/home/codespace/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/762be6a7206e2f299182705972a65e5c46e62be2/onnx/model.onnx",
_find_snapshot_file("~/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/*", "onnx/model.onnx") or "",
"./models/splade/model.onnx",
"/app/models/splade/model.onnx",
]
splade_tok_candidates = [
os.getenv("SPLADE_TOKENIZER_PATH", ""),
"/home/codespace/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/762be6a7206e2f299182705972a65e5c46e62be2/tokenizer.json",
_find_snapshot_file("~/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/*", "tokenizer.json") or "",
"./models/splade/tokenizer.json",
"/app/models/splade/tokenizer.json",
]
splade_model_path = next((p for p in splade_model_candidates if p and os.path.exists(p)), None)
splade_tok_path = next((p for p in splade_tok_candidates if p and os.path.exists(p)), None)
# Download from Hugging Face Hub if not found locally
if not splade_model_path or not splade_tok_path:
try:
from huggingface_hub import hf_hub_download
print("[ProductionModels] Attempting to download SPLADE ONNX from Hugging Face Hub...")
splade_model_path = splade_model_path or hf_hub_download(repo_id="prithivida/Splade_PP_en_v1", filename="onnx/model.onnx")
splade_tok_path = splade_tok_path or hf_hub_download(repo_id="prithivida/Splade_PP_en_v1", filename="tokenizer.json")
except Exception as e:
print(f"[ProductionModels] Hugging Face Hub download failed for SPLADE: {e}")
if splade_model_path and splade_tok_path:
try:
self.splade_tokenizer = Tokenizer.from_file(splade_tok_path)
self.splade_tokenizer.enable_truncation(max_length=256)
self.splade_tokenizer.no_padding()
num_threads = int(os.getenv("ONNX_NUM_THREADS", "4"))
opts = ort.SessionOptions()
opts.intra_op_num_threads = num_threads
self.splade_session = ort.InferenceSession(splade_model_path, opts)
self.splade_model_path = splade_model_path
print(f"[ProductionModels] Loaded SPLADE-PP ONNX session from {splade_model_path} (threads={num_threads})")
except Exception as e:
print(f"[ProductionModels] Error loading SPLADE ONNX: {e}")
else:
print("[ProductionModels] SPLADE ONNX model files not found in standard paths.")
def get_bge_embedding(
self,
text: str,
doc_id: Optional[str] = None,
is_query: bool = False
) -> Dict[str, Any]:
"""
Retrieves true 768-dim BGE-base vector:
1. From in-memory cache if previously computed (<0.1 ms).
2. From precomputed FAISS index if doc_id exists (<0.5 ms).
3. Live ONNX Runtime inference (~10-15 ms).
4. Deterministic fallback if ONNX is unavailable.
"""
t0 = time.time()
cache_key = self._get_cache_key('q' if is_query else 'd', doc_id, text)
# Check in-memory cache with lock
with self.cache_lock:
if cache_key in self.bge_cache:
res = dict(self.bge_cache[cache_key])
res["latency_ms"] = round((time.time() - t0) * 1000, 2)
return res
# Check precomputed FAISS index for NFCorpus doc IDs
if doc_id and self.faiss_loaded and doc_id in self.docid_to_idx:
try:
idx = self.docid_to_idx[doc_id]
vec = self.faiss_index.reconstruct(idx)
vec_list = [round(float(v), 5) for v in vec]
elapsed = round((time.time() - t0) * 1000, 2)
stats = self._compute_vector_stats(vec_list)
result = {
"vector": vec_list,
"dim": len(vec_list),
"source": "precomputed_pyserini_faiss",
"latency_ms": elapsed,
"doc_id": doc_id,
"stats": stats
}
self._store_in_cache(self.bge_cache, cache_key, result)
return result
except Exception as e:
print(f"[ProductionModels] Error reconstructing vector for {doc_id}: {e}")
# Live ONNX inference
if self.bge_session and self.bge_tokenizer:
try:
query_prefix = "Represent this sentence for searching relevant passages: " if is_query else ""
full_input = query_prefix + text
encoding = self.bge_tokenizer.encode(full_input)
input_ids = np.array([encoding.ids], dtype=np.int64)
attention_mask = np.array([encoding.attention_mask], dtype=np.int64)
token_type_ids = np.array([encoding.type_ids], dtype=np.int64)
outputs = self.bge_session.run(None, {
"input_ids": input_ids,
"attention_mask": attention_mask,
"token_type_ids": token_type_ids
})
# CLS pooling + L2 normalization
cls_vec = outputs[0][0, 0, :]
norm = float(np.linalg.norm(cls_vec))
norm_vec = (cls_vec / norm) if norm > 0 else cls_vec
vec_list = [round(float(v), 5) for v in norm_vec]
elapsed = round((time.time() - t0) * 1000, 2)
stats = self._compute_vector_stats(vec_list)
result = {
"vector": vec_list,
"dim": len(vec_list),
"source": "onnx_runtime_bge_base",
"latency_ms": elapsed,
"doc_id": doc_id,
"stats": stats
}
self._store_in_cache(self.bge_cache, cache_key, result)
return result
except Exception as e:
print(f"[ProductionModels] ONNX BGE inference failed: {e}")
# Deterministic fallback simulation
elapsed = round((time.time() - t0) * 1000, 2)
fallback_vec = self._simulate_fallback_vector(text, 768)
result = {
"vector": fallback_vec,
"dim": 768,
"source": "simulated_fallback",
"latency_ms": elapsed,
"doc_id": doc_id,
"stats": self._compute_vector_stats(fallback_vec)
}
self._store_in_cache(self.bge_cache, cache_key, result)
return result
def get_splade_representation(
self,
text: str,
doc_id: Optional[str] = None
) -> Dict[str, Any]:
"""
Retrieves real SPLADE masked language model lexical expansion weights.
"""
t0 = time.time()
cache_key = self._get_cache_key('splade', doc_id, text)
with self.cache_lock:
if cache_key in self.splade_cache:
res = dict(self.splade_cache[cache_key])
res["latency_ms"] = round((time.time() - t0) * 1000, 2)
return res
# Run ONNX SPLADE inference
if self.splade_session and self.splade_tokenizer:
try:
encoding = self.splade_tokenizer.encode(text)
input_ids = np.array([encoding.ids], dtype=np.int64)
input_mask = np.array([encoding.attention_mask], dtype=np.int64)
segment_ids = np.array([encoding.type_ids], dtype=np.int64)
outputs = self.splade_session.run(None, {
"input_ids": input_ids,
"input_mask": input_mask,
"segment_ids": segment_ids
})
output = outputs[0]
if output.ndim == 3:
# max pooling across seq_len: max(log(1 + relu(logits)))
sparse_vec = np.max(output[0], axis=0)
else:
sparse_vec = output[0]
text_lower = text.lower()
# Get non-zero / positive weights
non_zero_indices = np.where(sparse_vec > 0.01)[0]
sorted_indices = non_zero_indices[np.argsort(sparse_vec[non_zero_indices])[::-1]]
weights_dict: Dict[str, float] = {}
expansions_list: List[Dict[str, Any]] = []
for idx in sorted_indices[:60]: # Top 60 terms
val = float(sparse_vec[idx])
token = self.splade_tokenizer.id_to_token(int(idx))
if not token or token in ['[CLS]', '[SEP]', '[PAD]', '[UNK]']:
continue
clean_token = token.replace('##', '')
is_in_text = clean_token in text_lower
weights_dict[token] = round(val, 3)
expansions_list.append({
"token": token,
"weight": round(val, 3),
"is_expansion": not is_in_text,
"token_id": int(idx)
})
elapsed = round((time.time() - t0) * 1000, 2)
mlm_added = sum(1 for e in expansions_list if e["is_expansion"])
result = {
"weights": weights_dict,
"expansions": expansions_list,
"total_activated_tokens": len(non_zero_indices),
"mlm_expansions_count": mlm_added,
"source": "onnx_runtime_splade_v1",
"latency_ms": elapsed,
"doc_id": doc_id
}
self._store_in_cache(self.splade_cache, cache_key, result)
return result
except Exception as e:
print(f"[ProductionModels] ONNX SPLADE inference failed: {e}")
# Fallback simulation
elapsed = round((time.time() - t0) * 1000, 2)
sim_weights, sim_expansions = self._simulate_fallback_splade(text)
result = {
"weights": sim_weights,
"expansions": sim_expansions,
"total_activated_tokens": len(sim_weights),
"mlm_expansions_count": sum(1 for e in sim_expansions if e["is_expansion"]),
"source": "simulated_fallback",
"latency_ms": elapsed,
"doc_id": doc_id
}
self._store_in_cache(self.splade_cache, cache_key, result)
return result
def _compute_vector_stats(self, vec: List[float]) -> Dict[str, Any]:
arr = np.array(vec, dtype=np.float32)
norm = float(np.linalg.norm(arr))
min_v = float(np.min(arr))
max_v = float(np.max(arr))
mean_v = float(np.mean(arr))
std_v = float(np.std(arr))
# Top 8 highest positive and negative dimensions
top_pos = np.argsort(arr)[::-1][:8]
top_neg = np.argsort(arr)[:8]
return {
"l2_norm": round(norm, 4),
"min_val": round(min_v, 4),
"max_val": round(max_v, 4),
"mean_val": round(mean_v, 4),
"std_val": round(std_v, 4),
"top_positive": [{"dim": int(i), "val": round(float(arr[i]), 4)} for i in top_pos],
"top_negative": [{"dim": int(i), "val": round(float(arr[i]), 4)} for i in top_neg],
"positive_count": int(np.sum(arr > 0)),
"negative_count": int(np.sum(arr < 0)),
}
def _simulate_fallback_vector(self, text: str, dim: int = 768) -> List[float]:
vec = [0.0] * dim
for i, ch in enumerate(text.lower()):
h = (ord(ch) * 31 + i) % dim
vec[h] += 0.1
norm = math.sqrt(sum(v * v for v in vec))
if norm > 0:
vec = [round(v / norm, 5) for v in vec]
return vec
def _simulate_fallback_splade(self, text: str) -> Tuple[Dict[str, float], List[Dict[str, Any]]]:
words = [w.lower() for w in text.split() if len(w) > 3]
weights = {}
expansions = []
for w in set(words[:25]):
weights[w] = round(2.0 + len(w) * 0.2, 3)
expansions.append({"token": w, "weight": weights[w], "is_expansion": False})
return weights, expansions
# Global singleton instance
PRODUCTION_MODELS = ProductionModelsManager()