"""Real local GPU inference for semantic routing on a free ZeroGPU Space.""" # spaces must be imported before torch and model placement for CUDA emulation. import spaces import torch from transformers import AutoModel, AutoTokenizer MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2" tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) device = "cuda" if torch.cuda.is_available() else "cpu" model = AutoModel.from_pretrained(MODEL_ID).eval().to(device) @spaces.GPU(duration=30) def embed_texts(texts): """Encode evidence/query on GPU, with masked pooling and unit vectors.""" results = [] with torch.inference_mode(): for offset in range(0, len(texts), 32): encoded = tokenizer([text[:2000] for text in texts[offset:offset + 32]], padding=True, truncation=True, max_length=256, return_tensors="pt").to(device) hidden = model(**encoded).last_hidden_state mask = encoded["attention_mask"].unsqueeze(-1).to(hidden.dtype) pooled = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1) vectors = torch.nn.functional.normalize(pooled, p=2, dim=1) results.extend(vectors.cpu().tolist()) return results