import os import glob import time import math import hashlib import threading from typing import List, Dict, Any, Optional, Tuple import numpy as np # Try importing faiss try: import faiss FAISS_AVAILABLE = True except ImportError: faiss = None FAISS_AVAILABLE = False # Try importing ONNX runtime and tokenizers try: import onnxruntime as ort from tokenizers import Tokenizer ONNX_AVAILABLE = True except ImportError: ort = None Tokenizer = None ONNX_AVAILABLE = False def _find_snapshot_file(base_dir_pattern: str, file_rel_path: str) -> Optional[str]: """Helper to locate model files in Hugging Face snapshot caches or local model dirs.""" expanded_base = os.path.expanduser(base_dir_pattern) matched_dirs = glob.glob(expanded_base) for d in matched_dirs: candidate = os.path.join(d, file_rel_path) if os.path.exists(candidate): return candidate return None class ProductionModelsManager: def __init__(self): self.faiss_index = None self.docid_to_idx: Dict[str, int] = {} self.faiss_loaded = False self.faiss_index_path: Optional[str] = None # ONNX sessions & tokenizers self.bge_session = None self.bge_tokenizer = None self.bge_model_path: Optional[str] = None self.splade_session = None self.splade_tokenizer = None self.splade_model_path: Optional[str] = None # Thread locks for cache & inference safety self.cache_lock = threading.Lock() self.max_cache_size = int(os.getenv("MAX_CACHE_ENTRIES", "2000")) # In-memory caches for instantaneous (<1 ms) retrieval self.bge_cache: Dict[str, Dict[str, Any]] = {} self.splade_cache: Dict[str, Dict[str, Any]] = {} self._init_faiss() self._init_onnx_models() def _get_cache_key(self, prefix: str, doc_id: Optional[str], text: str) -> str: """Collision-proof cache key generation using doc_id or SHA-256 hash.""" if doc_id: return f"{prefix}:id_{doc_id}" text_hash = hashlib.sha256(text.strip().encode("utf-8")).hexdigest()[:24] return f"{prefix}:h_{text_hash}" def _store_in_cache(self, cache_dict: Dict[str, Any], key: str, value: Dict[str, Any]): """Thread-safe cache write with bounded eviction to prevent memory leaks.""" with self.cache_lock: if len(cache_dict) >= self.max_cache_size: # Evict oldest 20% entries num_to_evict = max(1, self.max_cache_size // 5) keys_to_remove = list(cache_dict.keys())[:num_to_evict] for k in keys_to_remove: cache_dict.pop(k, None) cache_dict[key] = value def _init_faiss(self): # Configurable search paths for precomputed NFCorpus FAISS index candidate_dirs = [ os.getenv("FAISS_INDEX_DIR", ""), "/workspaces/pyserini/indexes/nfcorpus.bge-base-en-v1.5", "./indexes/nfcorpus.bge-base-en-v1.5", "/app/indexes/nfcorpus.bge-base-en-v1.5", os.path.expanduser("~/.cache/ir-studio/indexes/nfcorpus.bge-base-en-v1.5"), ] found_dir = None for c in candidate_dirs: if c and os.path.isdir(c): idx_file = os.path.join(c, "index") doc_file = os.path.join(c, "docid") if os.path.exists(idx_file) and os.path.exists(doc_file): found_dir = c break if not found_dir: print("[ProductionModels] FAISS index directory not found. Precomputed vector lookups disabled.") return index_file = os.path.join(found_dir, "index") docid_file = os.path.join(found_dir, "docid") if FAISS_AVAILABLE and os.path.exists(index_file) and os.path.exists(docid_file): try: self.faiss_index = faiss.read_index(index_file) with open(docid_file, "r") as f: self.docid_to_idx = {line.strip(): i for i, line in enumerate(f)} self.faiss_loaded = True self.faiss_index_path = index_file print(f"[ProductionModels] Loaded FAISS index from {index_file} ({len(self.docid_to_idx)} docids)") except Exception as e: print(f"[ProductionModels] Error loading FAISS index: {e}") def _init_onnx_models(self): if not ONNX_AVAILABLE: print("[ProductionModels] ONNX Runtime or Tokenizers package not installed. Live neural inference disabled.") return # 1. Locate BGE-base ONNX model & tokenizer bge_model_candidates = [ os.getenv("BGE_MODEL_PATH", ""), "/home/codespace/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/4d6cd88e18e51a5e020c2c305726d76ada9c03cf/onnx/model_quantized.onnx", _find_snapshot_file("~/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/*", "onnx/model_quantized.onnx") or "", _find_snapshot_file("~/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/*", "onnx/model.onnx") or "", "./models/bge/model_quantized.onnx", "./models/bge/model.onnx", "/app/models/bge/model_quantized.onnx", ] bge_tok_candidates = [ os.getenv("BGE_TOKENIZER_PATH", ""), "/home/codespace/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/4d6cd88e18e51a5e020c2c305726d76ada9c03cf/tokenizer.json", _find_snapshot_file("~/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/*", "tokenizer.json") or "", "./models/bge/tokenizer.json", "/app/models/bge/tokenizer.json", ] bge_model_path = next((p for p in bge_model_candidates if p and os.path.exists(p)), None) bge_tok_path = next((p for p in bge_tok_candidates if p and os.path.exists(p)), None) # Download from Hugging Face Hub if not found locally if not bge_model_path or not bge_tok_path: try: from huggingface_hub import hf_hub_download print("[ProductionModels] Attempting to download BGE-base ONNX from Hugging Face Hub...") bge_model_path = bge_model_path or hf_hub_download(repo_id="Xenova/bge-base-en-v1.5", filename="onnx/model_quantized.onnx") bge_tok_path = bge_tok_path or hf_hub_download(repo_id="Xenova/bge-base-en-v1.5", filename="tokenizer.json") except Exception as e: print(f"[ProductionModels] Hugging Face Hub download failed for BGE: {e}") if bge_model_path and bge_tok_path: try: self.bge_tokenizer = Tokenizer.from_file(bge_tok_path) self.bge_tokenizer.enable_truncation(max_length=512) self.bge_tokenizer.no_padding() num_threads = int(os.getenv("ONNX_NUM_THREADS", "4")) opts = ort.SessionOptions() opts.intra_op_num_threads = num_threads self.bge_session = ort.InferenceSession(bge_model_path, opts) self.bge_model_path = bge_model_path print(f"[ProductionModels] Loaded BGE-base ONNX session from {bge_model_path} (threads={num_threads})") except Exception as e: print(f"[ProductionModels] Error loading BGE ONNX: {e}") else: print("[ProductionModels] BGE-base ONNX model files not found in standard paths.") # 2. Locate SPLADE ONNX model & tokenizer splade_model_candidates = [ os.getenv("SPLADE_MODEL_PATH", ""), "/home/codespace/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/762be6a7206e2f299182705972a65e5c46e62be2/onnx/model.onnx", _find_snapshot_file("~/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/*", "onnx/model.onnx") or "", "./models/splade/model.onnx", "/app/models/splade/model.onnx", ] splade_tok_candidates = [ os.getenv("SPLADE_TOKENIZER_PATH", ""), "/home/codespace/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/762be6a7206e2f299182705972a65e5c46e62be2/tokenizer.json", _find_snapshot_file("~/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/*", "tokenizer.json") or "", "./models/splade/tokenizer.json", "/app/models/splade/tokenizer.json", ] splade_model_path = next((p for p in splade_model_candidates if p and os.path.exists(p)), None) splade_tok_path = next((p for p in splade_tok_candidates if p and os.path.exists(p)), None) # Download from Hugging Face Hub if not found locally if not splade_model_path or not splade_tok_path: try: from huggingface_hub import hf_hub_download print("[ProductionModels] Attempting to download SPLADE ONNX from Hugging Face Hub...") splade_model_path = splade_model_path or hf_hub_download(repo_id="prithivida/Splade_PP_en_v1", filename="onnx/model.onnx") splade_tok_path = splade_tok_path or hf_hub_download(repo_id="prithivida/Splade_PP_en_v1", filename="tokenizer.json") except Exception as e: print(f"[ProductionModels] Hugging Face Hub download failed for SPLADE: {e}") if splade_model_path and splade_tok_path: try: self.splade_tokenizer = Tokenizer.from_file(splade_tok_path) self.splade_tokenizer.enable_truncation(max_length=256) self.splade_tokenizer.no_padding() num_threads = int(os.getenv("ONNX_NUM_THREADS", "4")) opts = ort.SessionOptions() opts.intra_op_num_threads = num_threads self.splade_session = ort.InferenceSession(splade_model_path, opts) self.splade_model_path = splade_model_path print(f"[ProductionModels] Loaded SPLADE-PP ONNX session from {splade_model_path} (threads={num_threads})") except Exception as e: print(f"[ProductionModels] Error loading SPLADE ONNX: {e}") else: print("[ProductionModels] SPLADE ONNX model files not found in standard paths.") def get_bge_embedding( self, text: str, doc_id: Optional[str] = None, is_query: bool = False ) -> Dict[str, Any]: """ Retrieves true 768-dim BGE-base vector: 1. From in-memory cache if previously computed (<0.1 ms). 2. From precomputed FAISS index if doc_id exists (<0.5 ms). 3. Live ONNX Runtime inference (~10-15 ms). 4. Deterministic fallback if ONNX is unavailable. """ t0 = time.time() cache_key = self._get_cache_key('q' if is_query else 'd', doc_id, text) # Check in-memory cache with lock with self.cache_lock: if cache_key in self.bge_cache: res = dict(self.bge_cache[cache_key]) res["latency_ms"] = round((time.time() - t0) * 1000, 2) return res # Check precomputed FAISS index for NFCorpus doc IDs if doc_id and self.faiss_loaded and doc_id in self.docid_to_idx: try: idx = self.docid_to_idx[doc_id] vec = self.faiss_index.reconstruct(idx) vec_list = [round(float(v), 5) for v in vec] elapsed = round((time.time() - t0) * 1000, 2) stats = self._compute_vector_stats(vec_list) result = { "vector": vec_list, "dim": len(vec_list), "source": "precomputed_pyserini_faiss", "latency_ms": elapsed, "doc_id": doc_id, "stats": stats } self._store_in_cache(self.bge_cache, cache_key, result) return result except Exception as e: print(f"[ProductionModels] Error reconstructing vector for {doc_id}: {e}") # Live ONNX inference if self.bge_session and self.bge_tokenizer: try: query_prefix = "Represent this sentence for searching relevant passages: " if is_query else "" full_input = query_prefix + text encoding = self.bge_tokenizer.encode(full_input) input_ids = np.array([encoding.ids], dtype=np.int64) attention_mask = np.array([encoding.attention_mask], dtype=np.int64) token_type_ids = np.array([encoding.type_ids], dtype=np.int64) outputs = self.bge_session.run(None, { "input_ids": input_ids, "attention_mask": attention_mask, "token_type_ids": token_type_ids }) # CLS pooling + L2 normalization cls_vec = outputs[0][0, 0, :] norm = float(np.linalg.norm(cls_vec)) norm_vec = (cls_vec / norm) if norm > 0 else cls_vec vec_list = [round(float(v), 5) for v in norm_vec] elapsed = round((time.time() - t0) * 1000, 2) stats = self._compute_vector_stats(vec_list) result = { "vector": vec_list, "dim": len(vec_list), "source": "onnx_runtime_bge_base", "latency_ms": elapsed, "doc_id": doc_id, "stats": stats } self._store_in_cache(self.bge_cache, cache_key, result) return result except Exception as e: print(f"[ProductionModels] ONNX BGE inference failed: {e}") # Deterministic fallback simulation elapsed = round((time.time() - t0) * 1000, 2) fallback_vec = self._simulate_fallback_vector(text, 768) result = { "vector": fallback_vec, "dim": 768, "source": "simulated_fallback", "latency_ms": elapsed, "doc_id": doc_id, "stats": self._compute_vector_stats(fallback_vec) } self._store_in_cache(self.bge_cache, cache_key, result) return result def get_splade_representation( self, text: str, doc_id: Optional[str] = None ) -> Dict[str, Any]: """ Retrieves real SPLADE masked language model lexical expansion weights. """ t0 = time.time() cache_key = self._get_cache_key('splade', doc_id, text) with self.cache_lock: if cache_key in self.splade_cache: res = dict(self.splade_cache[cache_key]) res["latency_ms"] = round((time.time() - t0) * 1000, 2) return res # Run ONNX SPLADE inference if self.splade_session and self.splade_tokenizer: try: encoding = self.splade_tokenizer.encode(text) input_ids = np.array([encoding.ids], dtype=np.int64) input_mask = np.array([encoding.attention_mask], dtype=np.int64) segment_ids = np.array([encoding.type_ids], dtype=np.int64) outputs = self.splade_session.run(None, { "input_ids": input_ids, "input_mask": input_mask, "segment_ids": segment_ids }) output = outputs[0] if output.ndim == 3: # max pooling across seq_len: max(log(1 + relu(logits))) sparse_vec = np.max(output[0], axis=0) else: sparse_vec = output[0] text_lower = text.lower() # Get non-zero / positive weights non_zero_indices = np.where(sparse_vec > 0.01)[0] sorted_indices = non_zero_indices[np.argsort(sparse_vec[non_zero_indices])[::-1]] weights_dict: Dict[str, float] = {} expansions_list: List[Dict[str, Any]] = [] for idx in sorted_indices[:60]: # Top 60 terms val = float(sparse_vec[idx]) token = self.splade_tokenizer.id_to_token(int(idx)) if not token or token in ['[CLS]', '[SEP]', '[PAD]', '[UNK]']: continue clean_token = token.replace('##', '') is_in_text = clean_token in text_lower weights_dict[token] = round(val, 3) expansions_list.append({ "token": token, "weight": round(val, 3), "is_expansion": not is_in_text, "token_id": int(idx) }) elapsed = round((time.time() - t0) * 1000, 2) mlm_added = sum(1 for e in expansions_list if e["is_expansion"]) result = { "weights": weights_dict, "expansions": expansions_list, "total_activated_tokens": len(non_zero_indices), "mlm_expansions_count": mlm_added, "source": "onnx_runtime_splade_v1", "latency_ms": elapsed, "doc_id": doc_id } self._store_in_cache(self.splade_cache, cache_key, result) return result except Exception as e: print(f"[ProductionModels] ONNX SPLADE inference failed: {e}") # Fallback simulation elapsed = round((time.time() - t0) * 1000, 2) sim_weights, sim_expansions = self._simulate_fallback_splade(text) result = { "weights": sim_weights, "expansions": sim_expansions, "total_activated_tokens": len(sim_weights), "mlm_expansions_count": sum(1 for e in sim_expansions if e["is_expansion"]), "source": "simulated_fallback", "latency_ms": elapsed, "doc_id": doc_id } self._store_in_cache(self.splade_cache, cache_key, result) return result def _compute_vector_stats(self, vec: List[float]) -> Dict[str, Any]: arr = np.array(vec, dtype=np.float32) norm = float(np.linalg.norm(arr)) min_v = float(np.min(arr)) max_v = float(np.max(arr)) mean_v = float(np.mean(arr)) std_v = float(np.std(arr)) # Top 8 highest positive and negative dimensions top_pos = np.argsort(arr)[::-1][:8] top_neg = np.argsort(arr)[:8] return { "l2_norm": round(norm, 4), "min_val": round(min_v, 4), "max_val": round(max_v, 4), "mean_val": round(mean_v, 4), "std_val": round(std_v, 4), "top_positive": [{"dim": int(i), "val": round(float(arr[i]), 4)} for i in top_pos], "top_negative": [{"dim": int(i), "val": round(float(arr[i]), 4)} for i in top_neg], "positive_count": int(np.sum(arr > 0)), "negative_count": int(np.sum(arr < 0)), } def _simulate_fallback_vector(self, text: str, dim: int = 768) -> List[float]: vec = [0.0] * dim for i, ch in enumerate(text.lower()): h = (ord(ch) * 31 + i) % dim vec[h] += 0.1 norm = math.sqrt(sum(v * v for v in vec)) if norm > 0: vec = [round(v / norm, 5) for v in vec] return vec def _simulate_fallback_splade(self, text: str) -> Tuple[Dict[str, float], List[Dict[str, Any]]]: words = [w.lower() for w in text.split() if len(w) > 3] weights = {} expansions = [] for w in set(words[:25]): weights[w] = round(2.0 + len(w) * 0.2, 3) expansions.append({"token": w, "weight": weights[w], "is_expansion": False}) return weights, expansions # Global singleton instance PRODUCTION_MODELS = ProductionModelsManager()