File size: 1,376 Bytes
e8eed08
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
import os
import time
from dotenv import load_dotenv
from langchain_google_genai import GoogleGenerativeAIEmbeddings
from langchain_community.document_loaders import PyPDFLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_chroma import Chroma

load_dotenv()

CHROMA_PATH = "./chroma_db"
embeddings = GoogleGenerativeAIEmbeddings(model="models/gemini-embedding-001")

# 1. Load and Split as before
loader = PyPDFLoader("data/cookbook.pdf")
docs = loader.load()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=150)
splits = text_splitter.split_documents(docs)
print(f" Total chunks to index: {len(splits)}")

# 2. Manual Batching (The 100 RPM Bypass)
# We initialize an empty vector store first
vectorstore = Chroma(persist_directory=CHROMA_PATH, embedding_function=embeddings)

batch_size = 50  # Stay well under the 100 limit
for i in range(0, len(splits), batch_size):
    batch = splits[i : i + batch_size]
    print(f" Indexing batch {i//batch_size + 1} ({len(batch)} chunks)...")
    
    vectorstore.add_documents(batch)
    
    # Architect's Cooldown: Wait 60 seconds after 2 batches (100 requests)
    if (i + batch_size) < len(splits):
        print(" Rate limit cooldown... waiting 65 seconds.")
        time.sleep(65)

print(f" Final verification: {vectorstore._collection.count()} items in DB.")