ahmad-agilekode commited on
Commit
ee98f9e
·
verified ·
1 Parent(s): bf024e4

Upload 9 files

Browse files
Dockerfile.txt ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Use an official Python runtime as a parent image
2
+ FROM python:3.9-slim
3
+
4
+ # Set the working directory in the container
5
+ WORKDIR /app
6
+
7
+ # Copy the requirements file into the container
8
+ COPY requirements.txt .
9
+
10
+ # Install dependencies
11
+ RUN pip install --no-cache-dir -r requirements.txt
12
+
13
+ # Copy the rest of the application code into the container
14
+ COPY . .
15
+
16
+ # Expose the port Gradio will use
17
+ EXPOSE 7860
18
+
19
+ # Command to run the Gradio app
20
+ CMD ["python", "app.py"]
Documentation.docx ADDED
Binary file (34.8 kB). View file
 
Pinecone_DocumentIngestion.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #Ingest scraped documents into Pinecone
2
+ import bs4
3
+ from langchain import hub
4
+ from langchain_community.document_loaders import WebBaseLoader
5
+ from langchain_core.documents import Document
6
+ from langchain_text_splitters import RecursiveCharacterTextSplitter
7
+ from typing_extensions import List, TypedDict
8
+ import json
9
+
10
+ # Load JSON file
11
+ with open("all_scraped.json", "r", encoding="utf-8") as f:
12
+ data = json.load(f)
13
+
14
+ # Convert to list of LangChain Document objects with metadata
15
+ docs = [
16
+ Document(page_content=entry["text"], metadata={"source": entry["url"]})
17
+ for entry in data if entry["text"].strip() # Skip empty texts
18
+ ]
19
+
20
+ # Initialize text splitter
21
+ text_splitter = RecursiveCharacterTextSplitter(
22
+ chunk_size=150,
23
+ chunk_overlap=35
24
+ )
25
+
26
+ # Split the documents into chunks, keeping metadata
27
+ all_splits = text_splitter.split_documents(docs)
28
+
29
+ # Print out a few chunks to confirm
30
+ for i, chunk in enumerate(all_splits[:3]):
31
+ print(f"Chunk {i+1}")
32
+ print(f"Source URL: {chunk.metadata['source']}")
33
+ print(f"Content (excerpt):\n{chunk.page_content[:300]}...\n")
34
+
35
+ # Index chunks
36
+ _ = vector_store.add_documents(documents=all_splits)
Pinecone_KnowledgeBaseDocuments.json ADDED
The diff for this file is too large to render. See raw diff
 
PostUserTesting_forUpwork.xlsx ADDED
Binary file (31 kB). View file
 
requirements.txt ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pydantic>=2.0
2
+ gradio==3.39.0
3
+ gradio_client>=0.15.0
4
+ python-dotenv
5
+ supabase
6
+ pinecone-client
7
+ langchain
8
+ langchain-openai
9
+ langchain-pinecone
10
+ langchain-community
11
+ langchain_community
12
+ langchain-core
13
+ langchain-text-splitters
14
+ langgraph
15
+ typing-extensions
16
+
supabase_integration.py ADDED
@@ -0,0 +1,212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import uuid
3
+ from datetime import datetime
4
+ from dotenv import load_dotenv
5
+ load_dotenv()
6
+
7
+ from supabase import create_client
8
+ from langchain_core.runnables import RunnableConfig
9
+ from typing import TypedDict, List, Optional, Annotated
10
+ from langchain_core.messages import BaseMessage, AIMessage, HumanMessage, SystemMessage
11
+ from langgraph.graph.message import add_messages
12
+ from langgraph.graph import StateGraph, END
13
+ from langchain_openai import OpenAIEmbeddings
14
+ from langchain.chat_models import init_chat_model
15
+ from langchain_pinecone import Pinecone as LangchainPinecone
16
+ from langchain_core.chat_history import BaseChatMessageHistory
17
+ from langchain_core.messages import messages_from_dict, messages_to_dict
18
+
19
+ # --- Supabase Setup ---
20
+ SUPABASE_URL = os.getenv("SUPABASE_URL")
21
+ SUPABASE_KEY = os.getenv("SUPABASE_KEY")
22
+ supabase = create_client(SUPABASE_URL, SUPABASE_KEY)
23
+
24
+ # --- Custom State ---
25
+ class CustomState(TypedDict):
26
+ messages: Annotated[List[BaseMessage], add_messages]
27
+ user_id: str
28
+ chat_history: Optional[List[List[str]]]
29
+
30
+ # --- AI Model ---
31
+ if not os.environ.get("OPENAI_API_KEY"):
32
+ os.environ["OPENAI_API_KEY"] = input("Enter API key for OpenAI: ")
33
+ llm = init_chat_model("gpt-4o", model_provider="openai", temperature=0.0)
34
+
35
+ # --- Vector Store ---
36
+ pinecone_api_key = os.getenv("PINECONE_API_KEY", "").strip()
37
+ pinecone_index_name = os.getenv("PINECONE_INDEX_NAME", "").strip()
38
+ embeddings = OpenAIEmbeddings(model="text-embedding-3-large")
39
+ vector_store = LangchainPinecone(
40
+ index_name=pinecone_index_name,
41
+ embedding=embeddings,
42
+ pinecone_api_key=pinecone_api_key
43
+ )
44
+
45
+ # --- Supabase Chat Memory ---
46
+ class SupabaseChatMessageHistory(BaseChatMessageHistory):
47
+ def __init__(self, user_id: str, supabase_client):
48
+ self.user_id = user_id
49
+ self.supabase = supabase_client
50
+
51
+ @property
52
+ def messages(self):
53
+ response = self.supabase.table("chat_memory").select("*").eq("user_id", self.user_id).execute()
54
+ data = response.data[0] if response.data else {}
55
+ return messages_from_dict(data.get("messages", []))
56
+
57
+ def add_message(self, message):
58
+ all_messages = messages_to_dict(self.messages + [message])
59
+ self.supabase.table("chat_memory").upsert({
60
+ "user_id": self.user_id,
61
+ "messages": all_messages
62
+ }).execute()
63
+
64
+ def add_user_message(self, content: str):
65
+ self.add_message(HumanMessage(content=content))
66
+
67
+ def add_ai_message(self, content: str):
68
+ self.add_message(AIMessage(content=content))
69
+
70
+ def clear(self):
71
+ self.supabase.table("chat_memory").delete().eq("user_id", self.user_id).execute()
72
+
73
+ class MemoryStore:
74
+ def get_memory(self, user_id: str):
75
+ return SupabaseChatMessageHistory(user_id, supabase)
76
+
77
+ store = MemoryStore()
78
+
79
+ # --- Logging ---
80
+ def log_interaction_cloud(user_id: str, question: str, answer: str):
81
+ payload = {
82
+ "user_id": user_id,
83
+ "question": question,
84
+ "answer": answer,
85
+ "timestamp": datetime.utcnow().isoformat(),
86
+ }
87
+ try:
88
+ res = supabase.table("logs").insert(payload).execute()
89
+ print("[Supabase Log Success]:", res.data)
90
+ except Exception as e:
91
+ print("[Supabase Log Error]:", e)
92
+
93
+ # --- Retrieval ---
94
+ def retrieve(query: str):
95
+ retrieved_docs = vector_store.similarity_search(query, k=6)
96
+ serialized = "\n\n".join(
97
+ (f"Source: {doc.metadata}\n" f"Content: {doc.page_content}")
98
+ for doc in retrieved_docs
99
+ )
100
+ return serialized, retrieved_docs
101
+
102
+ # --- System Message ---
103
+ SYSTEM_MESSAGE_TEXT = (
104
+ "You are Einbürgerungs-Helfer, a highly skilled, friendly, and helpful administrative assistant for naturalization matters in Germany."
105
+ " You only support the following languages: German, English, Turkish, and Arabic."
106
+ " You MUST always respond in the language of the user's last message."
107
+ " If a user sends a message in an unsupported language, politely inform them with the following message:"
108
+ " 'Bitte stellen Sie Ihre Frage auf Deutsch, Englisch, Türkisch oder Arabisch.'"
109
+
110
+ " Your sole purpose is to assist individuals who are eligible for German naturalization but have not yet applied."
111
+ " You only answer questions directly related to the current naturalization process in Germany."
112
+ " You do not assist with completing applications or submitting them."
113
+ " Your answers must be based solely on the provided context. Do NOT generate answers from your own knowledge."
114
+ " Your responses must always be accurate, concise, and easy to understand."
115
+ " Never hallucinate, never go beyond the topic of naturalization, and never share information that is false or misleading."
116
+
117
+ " If you do not have enough knowledge to answer a question based on the provided context, respond with:"
118
+ " 'I do not have enough knowledge to answer your question accurately. Please email your question to SBD-Umfrage@polsoz.fu-berlin.de, and we will respond as soon as possible.'"
119
+
120
+ " If a user uses toxic, offensive, or inappropriate language, kindly ask them to rephrase their question"
121
+ " and explain that such behavior is not acceptable."
122
+ )
123
+
124
+ def generate_user_id():
125
+ return "user_" + uuid.uuid4().hex[:6]
126
+
127
+ def query_or_respond(state: CustomState) -> CustomState:
128
+ user_id = state.get("user_id") or generate_user_id()
129
+ human_messages = [m for m in state["messages"] if m.type == "human"]
130
+ latest_user_msg = human_messages[-1]
131
+
132
+ memory = store.get_memory(user_id)
133
+ history = memory.messages[-5:]
134
+
135
+ retrieved_text, _ = retrieve(latest_user_msg.content)
136
+ context = SystemMessage(SYSTEM_MESSAGE_TEXT + "\n\nRetrieved Context:\n" + retrieved_text)
137
+
138
+ prompt = [context] + history + [latest_user_msg]
139
+ response = llm.invoke(prompt)
140
+
141
+ memory.add_user_message(latest_user_msg.content)
142
+ memory.add_ai_message(response.content)
143
+
144
+ log_interaction_cloud(user_id, latest_user_msg.content, response.content)
145
+
146
+ return {**state, "messages": [response], "user_id": user_id}
147
+
148
+ # --- Graph ---
149
+ graph_builder = StateGraph(CustomState)
150
+ graph_builder.add_node("query_or_respond", query_or_respond)
151
+ graph_builder.set_entry_point("query_or_respond")
152
+ graph_builder.add_edge("query_or_respond", END)
153
+ graph = graph_builder.compile()
154
+
155
+ # --- Gradio UI ---
156
+ import gradio as gr
157
+
158
+ def clear_history(user_id_state):
159
+ if user_id_state:
160
+ store.get_memory(user_id_state).clear()
161
+ return [["", "Ich bin hier, um Ihnen bei allen Fragen zur Einbürgerung in Deutschland zu helfen..."]], user_id_state, "", user_id_state
162
+
163
+ def respond(user_message, chatbot, user_id_input, chat_state, user_id_state):
164
+ if not user_message:
165
+ return chatbot, chat_state, user_id_input, "", user_id_state
166
+
167
+ user_id = user_id_state or user_id_input or generate_user_id()
168
+
169
+ state: CustomState = {
170
+ "messages": [HumanMessage(content=user_message)],
171
+ "user_id": user_id,
172
+ "chat_history": None
173
+ }
174
+
175
+ config = RunnableConfig(configurable={"session_id": user_id})
176
+ final_state = graph.invoke(state, config=config)
177
+
178
+ bot_response = final_state["messages"][-1].content
179
+ chatbot.append([user_message, bot_response])
180
+
181
+ return chatbot, chatbot, user_id, "", user_id
182
+
183
+ description = """
184
+ ### Was ist **Einbürgerungs-Helfer**?
185
+ **Deutsch:** Einbürgerungs-Helfer hilft dir, den Weg zur deutschen Staatsangehörigkeit zu verstehen – einfach, mehrsprachig, datenschutzfreundlich.
186
+ """
187
+
188
+ with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue")) as iface:
189
+ gr.Markdown(f"# Einbürgerungs-Helfer\n{description}")
190
+ with gr.Row():
191
+ user_id_input = gr.Textbox(label="Benutzer-ID (User ID)", placeholder="z.B. user_123", lines=1)
192
+ clear_btn = gr.Button("🗑️ Verlauf löschen")
193
+
194
+ user_id_state = gr.State(value="")
195
+ chat_state = gr.State([])
196
+
197
+ chatbot = gr.Chatbot(
198
+ label="Chat",
199
+ show_copy_button=True,
200
+ value=[[None, "Ich bin hier, um Ihnen bei allen Fragen zur Einbürgerung in Deutschland zu helfen.\nLassen Sie uns loslegen – wie kann ich Ihnen heute helfen?"]]
201
+ )
202
+
203
+ msg_input = gr.Textbox(placeholder="Frage eingeben...", label="Deine Nachricht", lines=1, value="")
204
+
205
+ with gr.Row():
206
+ submit_btn = gr.Button("Senden", variant="primary")
207
+ submit_btn.click(fn=respond, inputs=[msg_input, chatbot, user_id_input, chat_state, user_id_state], outputs=[chatbot, chat_state, user_id_input, msg_input, user_id_state])
208
+ msg_input.submit(fn=respond, inputs=[msg_input, chatbot, user_id_input, chat_state, user_id_state], outputs=[chatbot, chat_state, user_id_input, msg_input, user_id_state])
209
+
210
+ clear_btn.click(fn=clear_history, inputs=[user_id_state], outputs=[chatbot, user_id_input, msg_input, user_id_state])
211
+
212
+ iface.launch(server_name="0.0.0.0", server_port=int(os.getenv("PORT", 7860)), share=True)
testing_app.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import uuid
3
+ from dotenv import load_dotenv
4
+ load_dotenv()
5
+
6
+ from langchain_core.runnables import RunnableConfig
7
+ from typing import TypedDict, List, Optional, Annotated
8
+ from langchain_core.messages import BaseMessage, AIMessage, HumanMessage, SystemMessage
9
+ from langgraph.graph.message import add_messages
10
+ from langgraph.graph import StateGraph, END
11
+ from langchain_core.tools import tool
12
+ from langchain_openai import OpenAIEmbeddings
13
+ from langchain.chat_models import init_chat_model
14
+ from langchain_pinecone import Pinecone as LangchainPinecone
15
+ from langchain_core.chat_history import InMemoryChatMessageHistory
16
+
17
+ class CustomState(TypedDict):
18
+ messages: Annotated[List[BaseMessage], add_messages]
19
+ user_id: str
20
+ chat_history: Optional[List[List[str]]]
21
+
22
+ if not os.environ.get("OPENAI_API_KEY"):
23
+ os.environ["OPENAI_API_KEY"] = input("Enter API key for OpenAI: ")
24
+ llm = init_chat_model("gpt-4o", model_provider="openai", temperature=0.0)
25
+
26
+ pinecone_api_key = os.getenv("PINECONE_API_KEY", "").strip()
27
+ pinecone_index_name = os.getenv("PINECONE_INDEX_NAME", "").strip()
28
+ embeddings = OpenAIEmbeddings(model="text-embedding-3-large")
29
+ vector_store = LangchainPinecone(
30
+ index_name=pinecone_index_name,
31
+ embedding=embeddings,
32
+ pinecone_api_key=pinecone_api_key
33
+ )
34
+
35
+ class MemoryStore:
36
+ def get_memory(self, user_id: str):
37
+ return InMemoryChatMessageHistory()
38
+
39
+ store = MemoryStore()
40
+
41
+ def retrieve(query: str):
42
+ """Retrieve information related to the last query."""
43
+ retrieved_docs = vector_store.similarity_search(query, k=6)
44
+ print("\n🔎 Retrieved Documents:")
45
+ for i, doc in enumerate(retrieved_docs, 1):
46
+ print(f"\n--- Document {i} ---")
47
+ print(f"Source: {doc.metadata}")
48
+ print(f"Content (excerpt):\n{doc.page_content}...")
49
+
50
+ serialized = "\n\n".join(
51
+ (f"Source: {doc.metadata}\n" f"Content: {doc.page_content}")
52
+ for doc in retrieved_docs
53
+ )
54
+ return serialized, retrieved_docs
55
+
56
+ SYSTEM_MESSAGE_TEXT = (
57
+ "You are Einbürgerungs-Helfer, a highly skilled, friendly, and helpful administrative assistant for naturalization matters in Germany."
58
+ " You only support the following languages: German, English, Turkish, and Arabic."
59
+ " You MUST always respond in the language of the user's last message."
60
+ " If a user sends a message in an unsupported language, politely inform them with the following message:"
61
+ " 'Bitte stellen Sie Ihre Frage auf Deutsch, Englisch, Türkisch oder Arabisch.'"
62
+
63
+ " Your sole purpose is to assist individuals who are eligible for German naturalization but have not yet applied."
64
+ " You only answer questions directly related to the current naturalization process in Germany."
65
+ " You do not assist with completing applications or submitting them."
66
+ " Your answers must be based solely on the provided context. Do NOT generate answers from your own knowledge."
67
+ " Your responses must always be accurate, concise, and easy to understand."
68
+ " Never hallucinate, never go beyond the topic of naturalization, and never share information that is false or misleading."
69
+
70
+ " If you do not have enough knowledge to answer a question based on the provided context, respond with:"
71
+ " 'I do not have enough knowledge to answer your question accurately. Please email your question to SBD-Umfrage@polsoz.fu-berlin.de, and we will respond as soon as possible.'"
72
+
73
+ " If a user uses toxic, offensive, or inappropriate language, kindly ask them to rephrase their question"
74
+ " and explain that such behavior is not acceptable."
75
+ )
76
+
77
+
78
+ def generate_user_id():
79
+ return "user_" + uuid.uuid4().hex[:6]
80
+
81
+ def query_or_respond(state: CustomState) -> CustomState:
82
+ user_id = state.get("user_id") or generate_user_id()
83
+ human_messages = [m for m in state["messages"] if m.type == "human"]
84
+ latest_user_msg = human_messages[-1]
85
+
86
+ memory = store.get_memory(user_id)
87
+ history = memory.messages[-5:] if hasattr(memory, 'messages') else []
88
+
89
+ retrieved_text, _ = retrieve(latest_user_msg.content)
90
+ context = SystemMessage(SYSTEM_MESSAGE_TEXT + "\n\nRetrieved Context:\n" + retrieved_text)
91
+
92
+ prompt = [context] + history + [latest_user_msg]
93
+ response = llm.invoke(prompt)
94
+
95
+ memory.add_user_message(latest_user_msg.content)
96
+ memory.add_ai_message(response.content)
97
+
98
+ return {**state, "messages": [response], "user_id": user_id}
99
+
100
+ graph_builder = StateGraph(CustomState)
101
+ graph_builder.add_node("query_or_respond", query_or_respond)
102
+ graph_builder.set_entry_point("query_or_respond")
103
+ graph_builder.add_edge("query_or_respond", END)
104
+ graph = graph_builder.compile()
105
+
106
+ import gradio as gr
107
+
108
+ def clear_history(user_id_state):
109
+ return [["", "Ich bin hier, um Ihnen bei allen Fragen zur Einbürgerung in Deutschland zu helfen..."]], user_id_state, "", user_id_state
110
+
111
+ def respond(user_message, chatbot, user_id_input, chat_state, user_id_state):
112
+ if not user_message:
113
+ return chatbot, chat_state, user_id_input, "", user_id_state
114
+
115
+ user_id = user_id_state or user_id_input or generate_user_id()
116
+
117
+ state: CustomState = {
118
+ "messages": [HumanMessage(content=user_message)],
119
+ "user_id": user_id,
120
+ "chat_history": None
121
+ }
122
+
123
+ config = RunnableConfig(configurable={"session_id": user_id})
124
+ final_state = graph.invoke(state, config=config)
125
+
126
+ bot_response = final_state["messages"][-1].content
127
+ chatbot.append([user_message, bot_response])
128
+
129
+ return chatbot, chatbot, user_id, "", user_id
130
+
131
+ description = """
132
+ ### Was ist **Einbürgerungs-Helfer**?
133
+ **Deutsch:** Einbürgerungs-Helfer hilft dir, den Weg zur deutschen Staatsangehörigkeit zu verstehen – einfach, mehrsprachig, datenschutzfreundlich.
134
+ """
135
+
136
+ with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue")) as iface:
137
+ gr.Markdown(f"# Einbürgerungs-Helfer\n{description}")
138
+ with gr.Row():
139
+ user_id_input = gr.Textbox(label="Benutzer-ID (User ID)", placeholder="z.B. user_123", lines=1)
140
+ clear_btn = gr.Button("🗑️ Verlauf löschen")
141
+
142
+ user_id_state = gr.State(value="")
143
+ chat_state = gr.State([])
144
+
145
+ chatbot = gr.Chatbot(
146
+ label="Chat",
147
+ show_copy_button=True,
148
+ value=[[None, "Ich bin hier, um Ihnen bei allen Fragen zur Einbürgerung in Deutschland zu helfen.\nLassen Sie uns loslegen – wie kann ich Ihnen heute helfen?"]]
149
+ )
150
+
151
+ msg_input = gr.Textbox(placeholder="Frage eingeben...", label="Deine Nachricht", lines=1, value="")
152
+
153
+ with gr.Row():
154
+ submit_btn = gr.Button("Senden", variant="primary")
155
+ submit_btn.click(fn=respond, inputs=[msg_input, chatbot, user_id_input, chat_state, user_id_state], outputs=[chatbot, chat_state, user_id_input, msg_input, user_id_state])
156
+ msg_input.submit(fn=respond, inputs=[msg_input, chatbot, user_id_input, chat_state, user_id_state], outputs=[chatbot, chat_state, user_id_input, msg_input, user_id_state])
157
+
158
+ clear_btn.click(fn=clear_history, inputs=[user_id_state], outputs=[chatbot, user_id_input, msg_input, user_id_state])
159
+
160
+ iface.launch(server_name="0.0.0.0", server_port=int(os.getenv("PORT", 7860)), share=True)
testing_pinecone.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ from langchain_core.documents import Document
4
+ from langchain_text_splitters import RecursiveCharacterTextSplitter
5
+ from langchain_openai import OpenAIEmbeddings
6
+ from langchain_pinecone import Pinecone as LangchainPinecone
7
+ from dotenv import load_dotenv
8
+
9
+ # Load .env file
10
+ load_dotenv()
11
+
12
+ # ----------- 1. Load JSON Document ----------------
13
+ with open("Pinecone_KnowledgeBaseDocuments.json", "r", encoding="utf-8") as f:
14
+ data = json.load(f)
15
+
16
+ # Convert to LangChain Document objects
17
+ docs = [
18
+ Document(page_content=entry["text"], metadata={"source": entry["url"]})
19
+ for entry in data if entry.get("text", "").strip() # Skip entries with no text
20
+ ]
21
+
22
+ # ----------- 2. Split Documents into Chunks ----------------
23
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
24
+ all_splits = text_splitter.split_documents(docs)
25
+
26
+ # Print a few chunks to verify
27
+ for i, doc in enumerate(all_splits[:3]):
28
+ print(f"\nChunk {i+1}")
29
+ print(f"Page Content Length: {len(doc.page_content)}")
30
+ print(f"Metadata: {doc.metadata}")
31
+
32
+ print(f"\n✅ Total chunks created: {len(all_splits)}")
33
+
34
+ # ----------- 3. Initialize OpenAI Embeddings ----------------
35
+ embedding_model = OpenAIEmbeddings(model="text-embedding-3-large")
36
+
37
+ # ----------- 4. Initialize Pinecone Vector Store ----------------
38
+ pinecone_api_key = os.getenv("PINECONE_API_KEY")
39
+ pinecone_env = os.getenv("PINECONE_ENVIRONMENT") # Example: "gcp-starter"
40
+ pinecone_index_name = os.getenv("PINECONE_INDEX_NAME")
41
+
42
+ vector_store = LangchainPinecone(
43
+ index_name=pinecone_index_name,
44
+ embedding=embedding_model,
45
+ pinecone_api_key=pinecone_api_key
46
+ )
47
+
48
+ # ----------- 5. Add Chunks to Pinecone in Batches ----------------
49
+ BATCH_SIZE = 50
50
+
51
+ for i in range(0, len(all_splits), BATCH_SIZE):
52
+ batch = all_splits[i:i + BATCH_SIZE]
53
+ print(f"🚀 Uploading batch {i // BATCH_SIZE + 1}, size: {len(batch)} chunks...")
54
+ vector_store.add_documents(documents=batch)
55
+ print(f"✅ Batch {i // BATCH_SIZE + 1} uploaded.")
56
+
57
+ # ----------- 6. Confirmation ----------------
58
+ print(f"\n🎉 Successfully added {len(all_splits)} chunks to Pinecone!")