Spaces:
Runtime error
Runtime error
Download scripts/setup_vectorstore.py from MaryamKarimi080/SCR_Course_ChatBot: direct link, hf CLI and curl.
- Browser
- Download file 1.11 kB
-
https://huggingface.co/spaces/MaryamKarimi080/SCR_Course_ChatBot/resolve/4e3dac959605e93ccd7ca91c0d320f8d3fc0778c/scripts/setup_vectorstore.py
- Command line
-
hf download hf://spaces/MaryamKarimi080/SCR_Course_ChatBot@4e3dac959605e93ccd7ca91c0d320f8d3fc0778c/scripts/setup_vectorstore.py
-
curl -L -o setup_vectorstore.py https://huggingface.co/spaces/MaryamKarimi080/SCR_Course_ChatBot/resolve/4e3dac959605e93ccd7ca91c0d320f8d3fc0778c/scripts/setup_vectorstore.py
1.11 kB
| from langchain_community.vectorstores import Chroma | |
| from langchain_openai import OpenAIEmbeddings | |
| import pickle | |
| import os | |
| CHUNKS_PATH = "E:/courses/LangChain Project/main root/output/chunks.pkl" | |
| DB_DIR = "E:/courses/LangChain Project/main root/db" | |
| BATCH_SIZE = 100 # You can tune this depending on average token size per chunk | |
| if not os.path.exists(CHUNKS_PATH): | |
| raise FileNotFoundError("Run chunk_and_embed.py first") | |
| with open(CHUNKS_PATH, "rb") as f: | |
| chunks = pickle.load(f) | |
| embedding = OpenAIEmbeddings(model="text-embedding-3-small") | |
| # Create or load the vectorstore | |
| vectorstore = Chroma(persist_directory=DB_DIR, embedding_function=embedding) | |
| print(f"🧠 Embedding and adding {len(chunks)} chunks in batches of {BATCH_SIZE}...") | |
| # Add documents in batches to avoid hitting token limits | |
| for i in range(0, len(chunks), BATCH_SIZE): | |
| batch = chunks[i:i + BATCH_SIZE] | |
| vectorstore.add_documents(batch) | |
| print(f"✅ Added batch {i // BATCH_SIZE + 1} of {len(chunks) // BATCH_SIZE + 1}") | |
| # vectorstore.persist() | |
| print(f"✅ Vectorstore saved to {DB_DIR}") | |