import os import json from langchain_core.documents import Document from langchain_text_splitters import RecursiveCharacterTextSplitter from langchain_openai import OpenAIEmbeddings from langchain_pinecone import Pinecone as LangchainPinecone from dotenv import load_dotenv # Load .env file load_dotenv() # ----------- 1. Load JSON Document ---------------- with open("Pinecone_KnowledgeBaseDocuments.json", "r", encoding="utf-8") as f: data = json.load(f) # Convert to LangChain Document objects docs = [ Document(page_content=entry["text"], metadata={"source": entry["url"]}) for entry in data if entry.get("text", "").strip() # Skip entries with no text ] # ----------- 2. Split Documents into Chunks ---------------- text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100) all_splits = text_splitter.split_documents(docs) # Print a few chunks to verify for i, doc in enumerate(all_splits[:3]): print(f"\nChunk {i+1}") print(f"Page Content Length: {len(doc.page_content)}") print(f"Metadata: {doc.metadata}") print(f"\nāœ… Total chunks created: {len(all_splits)}") # ----------- 3. Initialize OpenAI Embeddings ---------------- embedding_model = OpenAIEmbeddings(model="text-embedding-3-large") # ----------- 4. Initialize Pinecone Vector Store ---------------- pinecone_api_key = os.getenv("PINECONE_API_KEY") pinecone_env = os.getenv("PINECONE_ENVIRONMENT") # Example: "gcp-starter" pinecone_index_name = os.getenv("PINECONE_INDEX_NAME") vector_store = LangchainPinecone( index_name=pinecone_index_name, embedding=embedding_model, pinecone_api_key=pinecone_api_key ) # ----------- 5. Add Chunks to Pinecone in Batches ---------------- BATCH_SIZE = 50 for i in range(0, len(all_splits), BATCH_SIZE): batch = all_splits[i:i + BATCH_SIZE] print(f"šŸš€ Uploading batch {i // BATCH_SIZE + 1}, size: {len(batch)} chunks...") vector_store.add_documents(documents=batch) print(f"āœ… Batch {i // BATCH_SIZE + 1} uploaded.") # ----------- 6. Confirmation ---------------- print(f"\nšŸŽ‰ Successfully added {len(all_splits)} chunks to Pinecone!")