Spaces:
Configuration error
Configuration error
File size: 1,449 Bytes
afa96cc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 |
from sentence_transformers import SentenceTransformer
# Load the model once when this file is first imported
# This happens only one time — not every time we call the functions
model = SentenceTransformer("all-MiniLM-L6-v2")
def embed_chunks(chunks):
"""
Converts all chunk texts into vectors.
Input : chunks → list of chunk dicts from chunk_text()
Output : same list but each chunk now has an "embedding" field added
"""
print("Embedding chunks... this may take a minute.")
# Extract just the text from each chunk into a simple list
texts = [chunk["text"] for chunk in chunks]
# Convert all texts to vectors in one batch (faster than one by one)
# show_progress_bar=True shows a progress bar so we know it is working
embeddings = model.encode(texts, show_progress_bar=True)
# Attach each embedding back to its chunk dict
for i, chunk in enumerate(chunks):
chunk["embedding"] = embeddings[i]
print(f"Done! Each chunk is now a vector of {len(embeddings[0])} numbers.")
return chunks
def embed_query(query):
"""
Converts a user question into a vector.
Input : query → a plain string question from the user
Output : a single vector (list of 384 numbers)
We use this at search time to find which chunks are most
similar to what the user is asking.
"""
return model.encode([query])[0] # encode returns a list, [0] gets first item
|