File size: 1,094 Bytes
2db7cb0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 | import json
from tqdm import tqdm
from sentence_transformers import SentenceTransformer
from chromadb import PersistentClient
model = SentenceTransformer("BAAI/bge-small-en-v1.5")
client = PersistentClient(path="embeddings/")
collection = client.get_or_create_collection(name="rag_docs")
def embed_and_store(input_path="processed/chunks.json"):
with open(input_path, "r") as f:
chunks = json.load(f)
documents, metadatas, ids = [], [], []
for i, chunk in enumerate(tqdm(chunks)):
documents.append("passage: " + chunk["text"])
metadatas.append(chunk["metadata"])
ids.append(f"chunk_{i}")
client.delete_collection(name="rag_docs")
collection = client.get_or_create_collection(name="rag_docs")
embeddings = model.encode(
documents,
batch_size=32,
normalize_embeddings=True,
show_progress_bar=True
)
collection.add(
documents=documents,
metadatas=metadatas,
ids=ids,
embeddings=embeddings.tolist()
)
if __name__ == "__main__":
embed_and_store() |