lecturelens-demo / space_app /gpu_embeddings.py
Phoenix21's picture
Deploy LectureLens runtime from GitHub a72af33744c8
b39b72f verified
Raw History Blame Contribute Delete
1.23 kB
"""Real local GPU inference for semantic routing on a free ZeroGPU Space."""
# spaces must be imported before torch and model placement for CUDA emulation.
import spaces
import torch
from transformers import AutoModel, AutoTokenizer
MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2"
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
device = "cuda" if torch.cuda.is_available() else "cpu"
model = AutoModel.from_pretrained(MODEL_ID).eval().to(device)
@spaces.GPU(duration=30)
def embed_texts(texts):
"""Encode evidence/query on GPU, with masked pooling and unit vectors."""
results = []
with torch.inference_mode():
for offset in range(0, len(texts), 32):
encoded = tokenizer([text[:2000] for text in texts[offset:offset + 32]],
padding=True, truncation=True, max_length=256, return_tensors="pt").to(device)
hidden = model(**encoded).last_hidden_state
mask = encoded["attention_mask"].unsqueeze(-1).to(hidden.dtype)
pooled = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1)
vectors = torch.nn.functional.normalize(pooled, p=2, dim=1)
results.extend(vectors.cpu().tolist())
return results