File size: 2,134 Bytes
ee98f9e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
import os
import json
from langchain_core.documents import Document
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings
from langchain_pinecone import Pinecone as LangchainPinecone
from dotenv import load_dotenv

# Load .env file
load_dotenv()

# ----------- 1. Load JSON Document ----------------
with open("Pinecone_KnowledgeBaseDocuments.json", "r", encoding="utf-8") as f:
    data = json.load(f)

# Convert to LangChain Document objects
docs = [
    Document(page_content=entry["text"], metadata={"source": entry["url"]})
    for entry in data if entry.get("text", "").strip()  # Skip entries with no text
]

# ----------- 2. Split Documents into Chunks ----------------
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
all_splits = text_splitter.split_documents(docs)

# Print a few chunks to verify
for i, doc in enumerate(all_splits[:3]):
    print(f"\nChunk {i+1}")
    print(f"Page Content Length: {len(doc.page_content)}")
    print(f"Metadata: {doc.metadata}")

print(f"\n✅ Total chunks created: {len(all_splits)}")

# ----------- 3. Initialize OpenAI Embeddings ----------------
embedding_model = OpenAIEmbeddings(model="text-embedding-3-large")

# ----------- 4. Initialize Pinecone Vector Store ----------------
pinecone_api_key = os.getenv("PINECONE_API_KEY")
pinecone_env = os.getenv("PINECONE_ENVIRONMENT")  # Example: "gcp-starter"
pinecone_index_name = os.getenv("PINECONE_INDEX_NAME")

vector_store = LangchainPinecone(
    index_name=pinecone_index_name,
    embedding=embedding_model,
    pinecone_api_key=pinecone_api_key
)

# ----------- 5. Add Chunks to Pinecone in Batches ----------------
BATCH_SIZE = 50

for i in range(0, len(all_splits), BATCH_SIZE):
    batch = all_splits[i:i + BATCH_SIZE]
    print(f"🚀 Uploading batch {i // BATCH_SIZE + 1}, size: {len(batch)} chunks...")
    vector_store.add_documents(documents=batch)
    print(f"✅ Batch {i // BATCH_SIZE + 1} uploaded.")

# ----------- 6. Confirmation ----------------
print(f"\n🎉 Successfully added {len(all_splits)} chunks to Pinecone!")