Natnael B. Haile
feat: complete IR Studio with Git LFS tracking for all binary assets and NFCorpus index
fbad523 Download api/production_models.py from NathyB/ir-studio: direct link, hf CLI and curl.
- Browser
- Download file 20.4 kB
-
https://huggingface.co/spaces/NathyB/ir-studio/resolve/main/api/production_models.py
- Command line
-
hf download hf://spaces/NathyB/ir-studio/api/production_models.py
-
curl -L -o production_models.py https://huggingface.co/spaces/NathyB/ir-studio/resolve/main/api/production_models.py
20.4 kB
| import os | |
| import glob | |
| import time | |
| import math | |
| import hashlib | |
| import threading | |
| from typing import List, Dict, Any, Optional, Tuple | |
| import numpy as np | |
| # Try importing faiss | |
| try: | |
| import faiss | |
| FAISS_AVAILABLE = True | |
| except ImportError: | |
| faiss = None | |
| FAISS_AVAILABLE = False | |
| # Try importing ONNX runtime and tokenizers | |
| try: | |
| import onnxruntime as ort | |
| from tokenizers import Tokenizer | |
| ONNX_AVAILABLE = True | |
| except ImportError: | |
| ort = None | |
| Tokenizer = None | |
| ONNX_AVAILABLE = False | |
| def _find_snapshot_file(base_dir_pattern: str, file_rel_path: str) -> Optional[str]: | |
| """Helper to locate model files in Hugging Face snapshot caches or local model dirs.""" | |
| expanded_base = os.path.expanduser(base_dir_pattern) | |
| matched_dirs = glob.glob(expanded_base) | |
| for d in matched_dirs: | |
| candidate = os.path.join(d, file_rel_path) | |
| if os.path.exists(candidate): | |
| return candidate | |
| return None | |
| class ProductionModelsManager: | |
| def __init__(self): | |
| self.faiss_index = None | |
| self.docid_to_idx: Dict[str, int] = {} | |
| self.faiss_loaded = False | |
| self.faiss_index_path: Optional[str] = None | |
| # ONNX sessions & tokenizers | |
| self.bge_session = None | |
| self.bge_tokenizer = None | |
| self.bge_model_path: Optional[str] = None | |
| self.splade_session = None | |
| self.splade_tokenizer = None | |
| self.splade_model_path: Optional[str] = None | |
| # Thread locks for cache & inference safety | |
| self.cache_lock = threading.Lock() | |
| self.max_cache_size = int(os.getenv("MAX_CACHE_ENTRIES", "2000")) | |
| # In-memory caches for instantaneous (<1 ms) retrieval | |
| self.bge_cache: Dict[str, Dict[str, Any]] = {} | |
| self.splade_cache: Dict[str, Dict[str, Any]] = {} | |
| self._init_faiss() | |
| self._init_onnx_models() | |
| def _get_cache_key(self, prefix: str, doc_id: Optional[str], text: str) -> str: | |
| """Collision-proof cache key generation using doc_id or SHA-256 hash.""" | |
| if doc_id: | |
| return f"{prefix}:id_{doc_id}" | |
| text_hash = hashlib.sha256(text.strip().encode("utf-8")).hexdigest()[:24] | |
| return f"{prefix}:h_{text_hash}" | |
| def _store_in_cache(self, cache_dict: Dict[str, Any], key: str, value: Dict[str, Any]): | |
| """Thread-safe cache write with bounded eviction to prevent memory leaks.""" | |
| with self.cache_lock: | |
| if len(cache_dict) >= self.max_cache_size: | |
| # Evict oldest 20% entries | |
| num_to_evict = max(1, self.max_cache_size // 5) | |
| keys_to_remove = list(cache_dict.keys())[:num_to_evict] | |
| for k in keys_to_remove: | |
| cache_dict.pop(k, None) | |
| cache_dict[key] = value | |
| def _init_faiss(self): | |
| # Configurable search paths for precomputed NFCorpus FAISS index | |
| candidate_dirs = [ | |
| os.getenv("FAISS_INDEX_DIR", ""), | |
| "/workspaces/pyserini/indexes/nfcorpus.bge-base-en-v1.5", | |
| "./indexes/nfcorpus.bge-base-en-v1.5", | |
| "/app/indexes/nfcorpus.bge-base-en-v1.5", | |
| os.path.expanduser("~/.cache/ir-studio/indexes/nfcorpus.bge-base-en-v1.5"), | |
| ] | |
| found_dir = None | |
| for c in candidate_dirs: | |
| if c and os.path.isdir(c): | |
| idx_file = os.path.join(c, "index") | |
| doc_file = os.path.join(c, "docid") | |
| if os.path.exists(idx_file) and os.path.exists(doc_file): | |
| found_dir = c | |
| break | |
| if not found_dir: | |
| print("[ProductionModels] FAISS index directory not found. Precomputed vector lookups disabled.") | |
| return | |
| index_file = os.path.join(found_dir, "index") | |
| docid_file = os.path.join(found_dir, "docid") | |
| if FAISS_AVAILABLE and os.path.exists(index_file) and os.path.exists(docid_file): | |
| try: | |
| self.faiss_index = faiss.read_index(index_file) | |
| with open(docid_file, "r") as f: | |
| self.docid_to_idx = {line.strip(): i for i, line in enumerate(f)} | |
| self.faiss_loaded = True | |
| self.faiss_index_path = index_file | |
| print(f"[ProductionModels] Loaded FAISS index from {index_file} ({len(self.docid_to_idx)} docids)") | |
| except Exception as e: | |
| print(f"[ProductionModels] Error loading FAISS index: {e}") | |
| def _init_onnx_models(self): | |
| if not ONNX_AVAILABLE: | |
| print("[ProductionModels] ONNX Runtime or Tokenizers package not installed. Live neural inference disabled.") | |
| return | |
| # 1. Locate BGE-base ONNX model & tokenizer | |
| bge_model_candidates = [ | |
| os.getenv("BGE_MODEL_PATH", ""), | |
| "/home/codespace/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/4d6cd88e18e51a5e020c2c305726d76ada9c03cf/onnx/model_quantized.onnx", | |
| _find_snapshot_file("~/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/*", "onnx/model_quantized.onnx") or "", | |
| _find_snapshot_file("~/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/*", "onnx/model.onnx") or "", | |
| "./models/bge/model_quantized.onnx", | |
| "./models/bge/model.onnx", | |
| "/app/models/bge/model_quantized.onnx", | |
| ] | |
| bge_tok_candidates = [ | |
| os.getenv("BGE_TOKENIZER_PATH", ""), | |
| "/home/codespace/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/4d6cd88e18e51a5e020c2c305726d76ada9c03cf/tokenizer.json", | |
| _find_snapshot_file("~/.cache/huggingface/hub/models--Xenova--bge-base-en-v1.5/snapshots/*", "tokenizer.json") or "", | |
| "./models/bge/tokenizer.json", | |
| "/app/models/bge/tokenizer.json", | |
| ] | |
| bge_model_path = next((p for p in bge_model_candidates if p and os.path.exists(p)), None) | |
| bge_tok_path = next((p for p in bge_tok_candidates if p and os.path.exists(p)), None) | |
| # Download from Hugging Face Hub if not found locally | |
| if not bge_model_path or not bge_tok_path: | |
| try: | |
| from huggingface_hub import hf_hub_download | |
| print("[ProductionModels] Attempting to download BGE-base ONNX from Hugging Face Hub...") | |
| bge_model_path = bge_model_path or hf_hub_download(repo_id="Xenova/bge-base-en-v1.5", filename="onnx/model_quantized.onnx") | |
| bge_tok_path = bge_tok_path or hf_hub_download(repo_id="Xenova/bge-base-en-v1.5", filename="tokenizer.json") | |
| except Exception as e: | |
| print(f"[ProductionModels] Hugging Face Hub download failed for BGE: {e}") | |
| if bge_model_path and bge_tok_path: | |
| try: | |
| self.bge_tokenizer = Tokenizer.from_file(bge_tok_path) | |
| self.bge_tokenizer.enable_truncation(max_length=512) | |
| self.bge_tokenizer.no_padding() | |
| num_threads = int(os.getenv("ONNX_NUM_THREADS", "4")) | |
| opts = ort.SessionOptions() | |
| opts.intra_op_num_threads = num_threads | |
| self.bge_session = ort.InferenceSession(bge_model_path, opts) | |
| self.bge_model_path = bge_model_path | |
| print(f"[ProductionModels] Loaded BGE-base ONNX session from {bge_model_path} (threads={num_threads})") | |
| except Exception as e: | |
| print(f"[ProductionModels] Error loading BGE ONNX: {e}") | |
| else: | |
| print("[ProductionModels] BGE-base ONNX model files not found in standard paths.") | |
| # 2. Locate SPLADE ONNX model & tokenizer | |
| splade_model_candidates = [ | |
| os.getenv("SPLADE_MODEL_PATH", ""), | |
| "/home/codespace/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/762be6a7206e2f299182705972a65e5c46e62be2/onnx/model.onnx", | |
| _find_snapshot_file("~/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/*", "onnx/model.onnx") or "", | |
| "./models/splade/model.onnx", | |
| "/app/models/splade/model.onnx", | |
| ] | |
| splade_tok_candidates = [ | |
| os.getenv("SPLADE_TOKENIZER_PATH", ""), | |
| "/home/codespace/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/762be6a7206e2f299182705972a65e5c46e62be2/tokenizer.json", | |
| _find_snapshot_file("~/.cache/huggingface/hub/models--prithivida--Splade_PP_en_v1/snapshots/*", "tokenizer.json") or "", | |
| "./models/splade/tokenizer.json", | |
| "/app/models/splade/tokenizer.json", | |
| ] | |
| splade_model_path = next((p for p in splade_model_candidates if p and os.path.exists(p)), None) | |
| splade_tok_path = next((p for p in splade_tok_candidates if p and os.path.exists(p)), None) | |
| # Download from Hugging Face Hub if not found locally | |
| if not splade_model_path or not splade_tok_path: | |
| try: | |
| from huggingface_hub import hf_hub_download | |
| print("[ProductionModels] Attempting to download SPLADE ONNX from Hugging Face Hub...") | |
| splade_model_path = splade_model_path or hf_hub_download(repo_id="prithivida/Splade_PP_en_v1", filename="onnx/model.onnx") | |
| splade_tok_path = splade_tok_path or hf_hub_download(repo_id="prithivida/Splade_PP_en_v1", filename="tokenizer.json") | |
| except Exception as e: | |
| print(f"[ProductionModels] Hugging Face Hub download failed for SPLADE: {e}") | |
| if splade_model_path and splade_tok_path: | |
| try: | |
| self.splade_tokenizer = Tokenizer.from_file(splade_tok_path) | |
| self.splade_tokenizer.enable_truncation(max_length=256) | |
| self.splade_tokenizer.no_padding() | |
| num_threads = int(os.getenv("ONNX_NUM_THREADS", "4")) | |
| opts = ort.SessionOptions() | |
| opts.intra_op_num_threads = num_threads | |
| self.splade_session = ort.InferenceSession(splade_model_path, opts) | |
| self.splade_model_path = splade_model_path | |
| print(f"[ProductionModels] Loaded SPLADE-PP ONNX session from {splade_model_path} (threads={num_threads})") | |
| except Exception as e: | |
| print(f"[ProductionModels] Error loading SPLADE ONNX: {e}") | |
| else: | |
| print("[ProductionModels] SPLADE ONNX model files not found in standard paths.") | |
| def get_bge_embedding( | |
| self, | |
| text: str, | |
| doc_id: Optional[str] = None, | |
| is_query: bool = False | |
| ) -> Dict[str, Any]: | |
| """ | |
| Retrieves true 768-dim BGE-base vector: | |
| 1. From in-memory cache if previously computed (<0.1 ms). | |
| 2. From precomputed FAISS index if doc_id exists (<0.5 ms). | |
| 3. Live ONNX Runtime inference (~10-15 ms). | |
| 4. Deterministic fallback if ONNX is unavailable. | |
| """ | |
| t0 = time.time() | |
| cache_key = self._get_cache_key('q' if is_query else 'd', doc_id, text) | |
| # Check in-memory cache with lock | |
| with self.cache_lock: | |
| if cache_key in self.bge_cache: | |
| res = dict(self.bge_cache[cache_key]) | |
| res["latency_ms"] = round((time.time() - t0) * 1000, 2) | |
| return res | |
| # Check precomputed FAISS index for NFCorpus doc IDs | |
| if doc_id and self.faiss_loaded and doc_id in self.docid_to_idx: | |
| try: | |
| idx = self.docid_to_idx[doc_id] | |
| vec = self.faiss_index.reconstruct(idx) | |
| vec_list = [round(float(v), 5) for v in vec] | |
| elapsed = round((time.time() - t0) * 1000, 2) | |
| stats = self._compute_vector_stats(vec_list) | |
| result = { | |
| "vector": vec_list, | |
| "dim": len(vec_list), | |
| "source": "precomputed_pyserini_faiss", | |
| "latency_ms": elapsed, | |
| "doc_id": doc_id, | |
| "stats": stats | |
| } | |
| self._store_in_cache(self.bge_cache, cache_key, result) | |
| return result | |
| except Exception as e: | |
| print(f"[ProductionModels] Error reconstructing vector for {doc_id}: {e}") | |
| # Live ONNX inference | |
| if self.bge_session and self.bge_tokenizer: | |
| try: | |
| query_prefix = "Represent this sentence for searching relevant passages: " if is_query else "" | |
| full_input = query_prefix + text | |
| encoding = self.bge_tokenizer.encode(full_input) | |
| input_ids = np.array([encoding.ids], dtype=np.int64) | |
| attention_mask = np.array([encoding.attention_mask], dtype=np.int64) | |
| token_type_ids = np.array([encoding.type_ids], dtype=np.int64) | |
| outputs = self.bge_session.run(None, { | |
| "input_ids": input_ids, | |
| "attention_mask": attention_mask, | |
| "token_type_ids": token_type_ids | |
| }) | |
| # CLS pooling + L2 normalization | |
| cls_vec = outputs[0][0, 0, :] | |
| norm = float(np.linalg.norm(cls_vec)) | |
| norm_vec = (cls_vec / norm) if norm > 0 else cls_vec | |
| vec_list = [round(float(v), 5) for v in norm_vec] | |
| elapsed = round((time.time() - t0) * 1000, 2) | |
| stats = self._compute_vector_stats(vec_list) | |
| result = { | |
| "vector": vec_list, | |
| "dim": len(vec_list), | |
| "source": "onnx_runtime_bge_base", | |
| "latency_ms": elapsed, | |
| "doc_id": doc_id, | |
| "stats": stats | |
| } | |
| self._store_in_cache(self.bge_cache, cache_key, result) | |
| return result | |
| except Exception as e: | |
| print(f"[ProductionModels] ONNX BGE inference failed: {e}") | |
| # Deterministic fallback simulation | |
| elapsed = round((time.time() - t0) * 1000, 2) | |
| fallback_vec = self._simulate_fallback_vector(text, 768) | |
| result = { | |
| "vector": fallback_vec, | |
| "dim": 768, | |
| "source": "simulated_fallback", | |
| "latency_ms": elapsed, | |
| "doc_id": doc_id, | |
| "stats": self._compute_vector_stats(fallback_vec) | |
| } | |
| self._store_in_cache(self.bge_cache, cache_key, result) | |
| return result | |
| def get_splade_representation( | |
| self, | |
| text: str, | |
| doc_id: Optional[str] = None | |
| ) -> Dict[str, Any]: | |
| """ | |
| Retrieves real SPLADE masked language model lexical expansion weights. | |
| """ | |
| t0 = time.time() | |
| cache_key = self._get_cache_key('splade', doc_id, text) | |
| with self.cache_lock: | |
| if cache_key in self.splade_cache: | |
| res = dict(self.splade_cache[cache_key]) | |
| res["latency_ms"] = round((time.time() - t0) * 1000, 2) | |
| return res | |
| # Run ONNX SPLADE inference | |
| if self.splade_session and self.splade_tokenizer: | |
| try: | |
| encoding = self.splade_tokenizer.encode(text) | |
| input_ids = np.array([encoding.ids], dtype=np.int64) | |
| input_mask = np.array([encoding.attention_mask], dtype=np.int64) | |
| segment_ids = np.array([encoding.type_ids], dtype=np.int64) | |
| outputs = self.splade_session.run(None, { | |
| "input_ids": input_ids, | |
| "input_mask": input_mask, | |
| "segment_ids": segment_ids | |
| }) | |
| output = outputs[0] | |
| if output.ndim == 3: | |
| # max pooling across seq_len: max(log(1 + relu(logits))) | |
| sparse_vec = np.max(output[0], axis=0) | |
| else: | |
| sparse_vec = output[0] | |
| text_lower = text.lower() | |
| # Get non-zero / positive weights | |
| non_zero_indices = np.where(sparse_vec > 0.01)[0] | |
| sorted_indices = non_zero_indices[np.argsort(sparse_vec[non_zero_indices])[::-1]] | |
| weights_dict: Dict[str, float] = {} | |
| expansions_list: List[Dict[str, Any]] = [] | |
| for idx in sorted_indices[:60]: # Top 60 terms | |
| val = float(sparse_vec[idx]) | |
| token = self.splade_tokenizer.id_to_token(int(idx)) | |
| if not token or token in ['[CLS]', '[SEP]', '[PAD]', '[UNK]']: | |
| continue | |
| clean_token = token.replace('##', '') | |
| is_in_text = clean_token in text_lower | |
| weights_dict[token] = round(val, 3) | |
| expansions_list.append({ | |
| "token": token, | |
| "weight": round(val, 3), | |
| "is_expansion": not is_in_text, | |
| "token_id": int(idx) | |
| }) | |
| elapsed = round((time.time() - t0) * 1000, 2) | |
| mlm_added = sum(1 for e in expansions_list if e["is_expansion"]) | |
| result = { | |
| "weights": weights_dict, | |
| "expansions": expansions_list, | |
| "total_activated_tokens": len(non_zero_indices), | |
| "mlm_expansions_count": mlm_added, | |
| "source": "onnx_runtime_splade_v1", | |
| "latency_ms": elapsed, | |
| "doc_id": doc_id | |
| } | |
| self._store_in_cache(self.splade_cache, cache_key, result) | |
| return result | |
| except Exception as e: | |
| print(f"[ProductionModels] ONNX SPLADE inference failed: {e}") | |
| # Fallback simulation | |
| elapsed = round((time.time() - t0) * 1000, 2) | |
| sim_weights, sim_expansions = self._simulate_fallback_splade(text) | |
| result = { | |
| "weights": sim_weights, | |
| "expansions": sim_expansions, | |
| "total_activated_tokens": len(sim_weights), | |
| "mlm_expansions_count": sum(1 for e in sim_expansions if e["is_expansion"]), | |
| "source": "simulated_fallback", | |
| "latency_ms": elapsed, | |
| "doc_id": doc_id | |
| } | |
| self._store_in_cache(self.splade_cache, cache_key, result) | |
| return result | |
| def _compute_vector_stats(self, vec: List[float]) -> Dict[str, Any]: | |
| arr = np.array(vec, dtype=np.float32) | |
| norm = float(np.linalg.norm(arr)) | |
| min_v = float(np.min(arr)) | |
| max_v = float(np.max(arr)) | |
| mean_v = float(np.mean(arr)) | |
| std_v = float(np.std(arr)) | |
| # Top 8 highest positive and negative dimensions | |
| top_pos = np.argsort(arr)[::-1][:8] | |
| top_neg = np.argsort(arr)[:8] | |
| return { | |
| "l2_norm": round(norm, 4), | |
| "min_val": round(min_v, 4), | |
| "max_val": round(max_v, 4), | |
| "mean_val": round(mean_v, 4), | |
| "std_val": round(std_v, 4), | |
| "top_positive": [{"dim": int(i), "val": round(float(arr[i]), 4)} for i in top_pos], | |
| "top_negative": [{"dim": int(i), "val": round(float(arr[i]), 4)} for i in top_neg], | |
| "positive_count": int(np.sum(arr > 0)), | |
| "negative_count": int(np.sum(arr < 0)), | |
| } | |
| def _simulate_fallback_vector(self, text: str, dim: int = 768) -> List[float]: | |
| vec = [0.0] * dim | |
| for i, ch in enumerate(text.lower()): | |
| h = (ord(ch) * 31 + i) % dim | |
| vec[h] += 0.1 | |
| norm = math.sqrt(sum(v * v for v in vec)) | |
| if norm > 0: | |
| vec = [round(v / norm, 5) for v in vec] | |
| return vec | |
| def _simulate_fallback_splade(self, text: str) -> Tuple[Dict[str, float], List[Dict[str, Any]]]: | |
| words = [w.lower() for w in text.split() if len(w) > 3] | |
| weights = {} | |
| expansions = [] | |
| for w in set(words[:25]): | |
| weights[w] = round(2.0 + len(w) * 0.2, 3) | |
| expansions.append({"token": w, "weight": weights[w], "is_expansion": False}) | |
| return weights, expansions | |
| # Global singleton instance | |
| PRODUCTION_MODELS = ProductionModelsManager() | |