Spaces:
Sleeping
Sleeping
File size: 2,134 Bytes
ee98f9e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | import os
import json
from langchain_core.documents import Document
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings
from langchain_pinecone import Pinecone as LangchainPinecone
from dotenv import load_dotenv
# Load .env file
load_dotenv()
# ----------- 1. Load JSON Document ----------------
with open("Pinecone_KnowledgeBaseDocuments.json", "r", encoding="utf-8") as f:
data = json.load(f)
# Convert to LangChain Document objects
docs = [
Document(page_content=entry["text"], metadata={"source": entry["url"]})
for entry in data if entry.get("text", "").strip() # Skip entries with no text
]
# ----------- 2. Split Documents into Chunks ----------------
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
all_splits = text_splitter.split_documents(docs)
# Print a few chunks to verify
for i, doc in enumerate(all_splits[:3]):
print(f"\nChunk {i+1}")
print(f"Page Content Length: {len(doc.page_content)}")
print(f"Metadata: {doc.metadata}")
print(f"\nโ
Total chunks created: {len(all_splits)}")
# ----------- 3. Initialize OpenAI Embeddings ----------------
embedding_model = OpenAIEmbeddings(model="text-embedding-3-large")
# ----------- 4. Initialize Pinecone Vector Store ----------------
pinecone_api_key = os.getenv("PINECONE_API_KEY")
pinecone_env = os.getenv("PINECONE_ENVIRONMENT") # Example: "gcp-starter"
pinecone_index_name = os.getenv("PINECONE_INDEX_NAME")
vector_store = LangchainPinecone(
index_name=pinecone_index_name,
embedding=embedding_model,
pinecone_api_key=pinecone_api_key
)
# ----------- 5. Add Chunks to Pinecone in Batches ----------------
BATCH_SIZE = 50
for i in range(0, len(all_splits), BATCH_SIZE):
batch = all_splits[i:i + BATCH_SIZE]
print(f"๐ Uploading batch {i // BATCH_SIZE + 1}, size: {len(batch)} chunks...")
vector_store.add_documents(documents=batch)
print(f"โ
Batch {i // BATCH_SIZE + 1} uploaded.")
# ----------- 6. Confirmation ----------------
print(f"\n๐ Successfully added {len(all_splits)} chunks to Pinecone!")
|