Spaces:
Runtime error
Runtime error
Download src/utils/Safe-Batching.py from aiassembled/agentic-diet-engine: direct link, hf CLI and curl.
- Browser
- Download file 1.38 kB
-
https://huggingface.co/spaces/aiassembled/agentic-diet-engine/resolve/main/src/utils/Safe-Batching.py
- Command line
-
hf download hf://spaces/aiassembled/agentic-diet-engine/src/utils/Safe-Batching.py
-
curl -L -o Safe-Batching.py https://huggingface.co/spaces/aiassembled/agentic-diet-engine/resolve/main/src/utils/Safe-Batching.py
1.38 kB
| import os | |
| import time | |
| from dotenv import load_dotenv | |
| from langchain_google_genai import GoogleGenerativeAIEmbeddings | |
| from langchain_community.document_loaders import PyPDFLoader | |
| from langchain_text_splitters import RecursiveCharacterTextSplitter | |
| from langchain_chroma import Chroma | |
| load_dotenv() | |
| CHROMA_PATH = "./chroma_db" | |
| embeddings = GoogleGenerativeAIEmbeddings(model="models/gemini-embedding-001") | |
| # 1. Load and Split as before | |
| loader = PyPDFLoader("data/cookbook.pdf") | |
| docs = loader.load() | |
| text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=150) | |
| splits = text_splitter.split_documents(docs) | |
| print(f" Total chunks to index: {len(splits)}") | |
| # 2. Manual Batching (The 100 RPM Bypass) | |
| # We initialize an empty vector store first | |
| vectorstore = Chroma(persist_directory=CHROMA_PATH, embedding_function=embeddings) | |
| batch_size = 50 # Stay well under the 100 limit | |
| for i in range(0, len(splits), batch_size): | |
| batch = splits[i : i + batch_size] | |
| print(f" Indexing batch {i//batch_size + 1} ({len(batch)} chunks)...") | |
| vectorstore.add_documents(batch) | |
| # Architect's Cooldown: Wait 60 seconds after 2 batches (100 requests) | |
| if (i + batch_size) < len(splits): | |
| print(" Rate limit cooldown... waiting 65 seconds.") | |
| time.sleep(65) | |
| print(f" Final verification: {vectorstore._collection.count()} items in DB.") |