Spaces:
Sleeping
Sleeping
Download testing_pinecone.py from german-chatbot/README: direct link, hf CLI and curl.
- Browser
- Download file 2.13 kB
-
https://huggingface.co/spaces/german-chatbot/README/resolve/main/testing_pinecone.py
- Command line
-
hf download hf://spaces/german-chatbot/README/testing_pinecone.py
-
curl -L -o testing_pinecone.py https://huggingface.co/spaces/german-chatbot/README/resolve/main/testing_pinecone.py
2.13 kB
| import os | |
| import json | |
| from langchain_core.documents import Document | |
| from langchain_text_splitters import RecursiveCharacterTextSplitter | |
| from langchain_openai import OpenAIEmbeddings | |
| from langchain_pinecone import Pinecone as LangchainPinecone | |
| from dotenv import load_dotenv | |
| # Load .env file | |
| load_dotenv() | |
| # ----------- 1. Load JSON Document ---------------- | |
| with open("Pinecone_KnowledgeBaseDocuments.json", "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| # Convert to LangChain Document objects | |
| docs = [ | |
| Document(page_content=entry["text"], metadata={"source": entry["url"]}) | |
| for entry in data if entry.get("text", "").strip() # Skip entries with no text | |
| ] | |
| # ----------- 2. Split Documents into Chunks ---------------- | |
| text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100) | |
| all_splits = text_splitter.split_documents(docs) | |
| # Print a few chunks to verify | |
| for i, doc in enumerate(all_splits[:3]): | |
| print(f"\nChunk {i+1}") | |
| print(f"Page Content Length: {len(doc.page_content)}") | |
| print(f"Metadata: {doc.metadata}") | |
| print(f"\nโ Total chunks created: {len(all_splits)}") | |
| # ----------- 3. Initialize OpenAI Embeddings ---------------- | |
| embedding_model = OpenAIEmbeddings(model="text-embedding-3-large") | |
| # ----------- 4. Initialize Pinecone Vector Store ---------------- | |
| pinecone_api_key = os.getenv("PINECONE_API_KEY") | |
| pinecone_env = os.getenv("PINECONE_ENVIRONMENT") # Example: "gcp-starter" | |
| pinecone_index_name = os.getenv("PINECONE_INDEX_NAME") | |
| vector_store = LangchainPinecone( | |
| index_name=pinecone_index_name, | |
| embedding=embedding_model, | |
| pinecone_api_key=pinecone_api_key | |
| ) | |
| # ----------- 5. Add Chunks to Pinecone in Batches ---------------- | |
| BATCH_SIZE = 50 | |
| for i in range(0, len(all_splits), BATCH_SIZE): | |
| batch = all_splits[i:i + BATCH_SIZE] | |
| print(f"๐ Uploading batch {i // BATCH_SIZE + 1}, size: {len(batch)} chunks...") | |
| vector_store.add_documents(documents=batch) | |
| print(f"โ Batch {i // BATCH_SIZE + 1} uploaded.") | |
| # ----------- 6. Confirmation ---------------- | |
| print(f"\n๐ Successfully added {len(all_splits)} chunks to Pinecone!") | |