seraj-b2 commited on
Commit
d3f7d23
·
1 Parent(s): 9c8bf34

graph rag added

Browse files
__pycache__/agent.cpython-314.pyc CHANGED
Binary files a/__pycache__/agent.cpython-314.pyc and b/__pycache__/agent.cpython-314.pyc differ
 
__pycache__/graph_store.cpython-314.pyc ADDED
Binary file (11 kB). View file
 
__pycache__/main.cpython-314.pyc CHANGED
Binary files a/__pycache__/main.cpython-314.pyc and b/__pycache__/main.cpython-314.pyc differ
 
__pycache__/test_api.cpython-314-pytest-9.0.3.pyc CHANGED
Binary files a/__pycache__/test_api.cpython-314-pytest-9.0.3.pyc and b/__pycache__/test_api.cpython-314-pytest-9.0.3.pyc differ
 
agent.py CHANGED
@@ -1,25 +1,68 @@
1
  import logging
 
 
2
  from google import genai
3
  from google.genai import types
4
 
5
  logger = logging.getLogger("rag_backend")
6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  def run_agent_rag(
8
  user_message: str,
9
  embedding_model,
10
  vector_store,
 
11
  chat_history: list = None
12
  ):
13
  """
14
  Executes an Agentic RAG loop using the Gemini API (via the google-genai SDK).
15
- It defines a custom search tool over the Qdrant vector database, allows the model to
16
- dynamically decide when to call the search tool, handles query reformulation/multi-step searches,
17
- and returns the final answers alongside search traces and retrieved sources.
18
 
19
  Args:
20
  user_message: The prompt/query from the user.
21
  embedding_model: The SentenceTransformer embedding model to vectorize search queries.
22
  vector_store: The QdrantVectorStore instance to search documents.
 
23
  chat_history: Optional history of the conversation formatted as a list of dicts.
24
  """
25
  # Trace variables to return for API response
@@ -29,9 +72,9 @@ def run_agent_rag(
29
  # 1. Define the Vector Search Tool
30
  def query_knowledge_base(query: str, top_k: int = 3) -> str:
31
  """
32
- Searches the internal knowledge database for relevant document chunks.
33
  Use this tool when the user asks questions that require retrieving facts
34
- or searching internal documents.
35
 
36
  Args:
37
  query: The search query to run against the vector database.
@@ -57,7 +100,7 @@ def run_agent_rag(
57
 
58
  if not search_results:
59
  tool_calls_log.append(call_details)
60
- return "No matching documents found in the database."
61
 
62
  formatted_chunks = []
63
  for i, hit in enumerate(search_results):
@@ -77,6 +120,35 @@ def run_agent_rag(
77
  tool_calls_log.append(call_details)
78
  return "\n\n".join(formatted_chunks)
79
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  # 2. Initialize Gemini Client (picks up GEMINI_API_KEY from environment)
81
  client = genai.Client()
82
  model_name = "gemini-2.5-flash"
@@ -98,20 +170,28 @@ def run_agent_rag(
98
  parts=[types.Part.from_text(text=user_message)]
99
  ))
100
 
 
 
 
 
 
101
  # Generate content configuration with system instruction and tools
102
  config = types.GenerateContentConfig(
103
- tools=[query_knowledge_base],
104
  automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
105
  system_instruction=(
106
- "You are a helpful, precise Agentic RAG assistant. You have access to the query_knowledge_base tool to search the "
107
- "internal vector database. Follow these guidelines:\n"
 
108
  "1. If a query requires factual information from the database, call the tool query_knowledge_base first.\n"
109
- "2. Inspect the retrieved context. If it does not answer the question or you need more information, you "
110
- "may call the tool again with a reformulated or more specific query.\n"
111
- "3. If the user's request is simple chitchat or doesn't require database lookups (e.g. 'hello', 'how are you'), "
112
- "do not call the tool; answer directly.\n"
113
- "4. Make sure your final answer is fully grounded in the retrieved documents. State clearly if some information "
114
- "was not found or could not be verified in the database."
 
 
115
  )
116
  )
117
 
@@ -138,10 +218,15 @@ def run_agent_rag(
138
  for call in response.function_calls:
139
  func_args = call.args if call.args else {}
140
  try:
141
- result = query_knowledge_base(**func_args)
 
 
 
 
 
142
  except Exception as e:
143
  logger.error(f"Error executing agent tool '{call.name}': {e}")
144
- result = f"Error executing tool query_knowledge_base: {str(e)}"
145
 
146
  function_responses.append(
147
  types.Part.from_function_response(
 
1
  import logging
2
+ from typing import List, Dict, Any, Optional
3
+ from pydantic import BaseModel, Field
4
  from google import genai
5
  from google.genai import types
6
 
7
  logger = logging.getLogger("rag_backend")
8
 
9
+ class EntityModel(BaseModel):
10
+ name: str = Field(..., description="The name of the entity, e.g. 'Albert Einstein', 'Google', 'Gravity'. Keep it concise, singular, and normalized.")
11
+ type: str = Field(..., description="The category/type of entity, e.g., Person, Organization, Concept, Location, Product.")
12
+ description: str = Field(..., description="A short sentence summarizing what or who this entity is based on the text.")
13
+
14
+ class RelationshipModel(BaseModel):
15
+ source: str = Field(..., description="The name of the source entity.")
16
+ target: str = Field(..., description="The name of the target entity.")
17
+ type: str = Field(..., description="The type of connection, e.g., WORKS_AT, INVENTED, IS_A, PART_OF, ASSOCIATED_WITH. Uppercase snake_case preferred.")
18
+ description: str = Field(..., description="A short clause explaining the connection (e.g. 'discovered general relativity').")
19
+
20
+ class GraphExtractionModel(BaseModel):
21
+ entities: List[EntityModel]
22
+ relationships: List[RelationshipModel]
23
+
24
+ def extract_graph_elements(chunk_text: str) -> GraphExtractionModel:
25
+ """
26
+ Uses Gemini structured outputs to extract key entities and relationships from a text chunk.
27
+ """
28
+ try:
29
+ client = genai.Client()
30
+ response = client.models.generate_content(
31
+ model="gemini-2.5-flash",
32
+ contents=f"Analyze the following text chunk and extract key entities and their semantic relationships:\n\n{chunk_text}",
33
+ config=types.GenerateContentConfig(
34
+ response_mime_type="application/json",
35
+ response_schema=GraphExtractionModel,
36
+ system_instruction=(
37
+ "You are a precise information extraction engine. Extract only entities and relations that are "
38
+ "explicitly supported by the text chunk. Do not extrapolate. Keep entity names normalized (e.g. use "
39
+ "singular nouns, expand acronyms if clear)."
40
+ )
41
+ )
42
+ )
43
+ return GraphExtractionModel.model_validate_json(response.text)
44
+ except Exception as e:
45
+ logger.error(f"Failed to extract graph elements from text: {e}")
46
+ return GraphExtractionModel(entities=[], relationships=[])
47
+
48
  def run_agent_rag(
49
  user_message: str,
50
  embedding_model,
51
  vector_store,
52
+ graph_store=None,
53
  chat_history: list = None
54
  ):
55
  """
56
  Executes an Agentic RAG loop using the Gemini API (via the google-genai SDK).
57
+ It defines custom search tools over Qdrant (vector) and Neo4j (graph) databases,
58
+ allowing the model to dynamically decide when to call which tool, handles query
59
+ reformulation/multi-step searches, and returns final answers alongside traces.
60
 
61
  Args:
62
  user_message: The prompt/query from the user.
63
  embedding_model: The SentenceTransformer embedding model to vectorize search queries.
64
  vector_store: The QdrantVectorStore instance to search documents.
65
+ graph_store: The Neo4jGraphStore instance to query knowledge graph relationships.
66
  chat_history: Optional history of the conversation formatted as a list of dicts.
67
  """
68
  # Trace variables to return for API response
 
72
  # 1. Define the Vector Search Tool
73
  def query_knowledge_base(query: str, top_k: int = 3) -> str:
74
  """
75
+ Searches the internal vector database for relevant document chunks.
76
  Use this tool when the user asks questions that require retrieving facts
77
+ or searching general text documents.
78
 
79
  Args:
80
  query: The search query to run against the vector database.
 
100
 
101
  if not search_results:
102
  tool_calls_log.append(call_details)
103
+ return "No matching documents found in the vector database."
104
 
105
  formatted_chunks = []
106
  for i, hit in enumerate(search_results):
 
120
  tool_calls_log.append(call_details)
121
  return "\n\n".join(formatted_chunks)
122
 
123
+ # 1b. Define the Graph Search Tool
124
+ def query_graph_database(entity_names: List[str]) -> str:
125
+ """
126
+ Queries the internal Neo4j Knowledge Graph for relationships, properties, and
127
+ connections surrounding specific key entities/concepts. Use this tool when
128
+ you need to answer complex relationship queries, find paths between topics,
129
+ or see how concepts connect to each other.
130
+
131
+ Args:
132
+ entity_names: A list of entity names/concepts to look up in the graph database.
133
+ """
134
+ logger.info(f"Agent tool executing: query_graph_database with entities={entity_names}")
135
+ if not graph_store or not graph_store.is_connected:
136
+ return "Graph database connection is not available. Please fall back to query_knowledge_base (vector search)."
137
+
138
+ try:
139
+ result = graph_store.query_subgraph(entity_names)
140
+ call_details = {
141
+ "query": f"Graph lookup for: {entity_names}",
142
+ "top_k": len(entity_names),
143
+ "results_count": 1 if result else 0,
144
+ "results": [{"text": result, "score": 1.0, "metadata": {"entities": entity_names}}]
145
+ }
146
+ tool_calls_log.append(call_details)
147
+ return result
148
+ except Exception as e:
149
+ logger.error(f"Error querying Neo4j in agent tool: {e}")
150
+ return f"Error querying graph database: {str(e)}"
151
+
152
  # 2. Initialize Gemini Client (picks up GEMINI_API_KEY from environment)
153
  client = genai.Client()
154
  model_name = "gemini-2.5-flash"
 
170
  parts=[types.Part.from_text(text=user_message)]
171
  ))
172
 
173
+ # Register available tools
174
+ tools = [query_knowledge_base]
175
+ if graph_store and graph_store.is_connected:
176
+ tools.append(query_graph_database)
177
+
178
  # Generate content configuration with system instruction and tools
179
  config = types.GenerateContentConfig(
180
+ tools=tools,
181
  automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
182
  system_instruction=(
183
+ "You are a helpful, precise Agentic RAG assistant. You have access to a vector search tool (query_knowledge_base) "
184
+ + ("and a graph search tool (query_graph_database). " if len(tools) > 1 else "") +
185
+ "Follow these guidelines:\n"
186
  "1. If a query requires factual information from the database, call the tool query_knowledge_base first.\n"
187
+ + ("2. If the user asks about connections, relationships, paths, or structured associations between specific concepts, "
188
+ "call query_graph_database with the list of concept/entity names.\n" if len(tools) > 1 else "") +
189
+ "3. Inspect the retrieved context. If it does not answer the question or you need more information, you "
190
+ "may call the tools again with a reformulated or more specific query. You can use both tools in a multi-step sequence.\n"
191
+ "4. If the user's request is simple chitchat or doesn't require database lookups (e.g. 'hello', 'how are you'), "
192
+ "do not call the tools; answer directly.\n"
193
+ "5. Make sure your final answer is fully grounded in the retrieved documents or relationships. State clearly if "
194
+ "some information was not found or could not be verified in the databases."
195
  )
196
  )
197
 
 
218
  for call in response.function_calls:
219
  func_args = call.args if call.args else {}
220
  try:
221
+ if call.name == "query_knowledge_base":
222
+ result = query_knowledge_base(**func_args)
223
+ elif call.name == "query_graph_database":
224
+ result = query_graph_database(**func_args)
225
+ else:
226
+ result = f"Unknown tool: {call.name}"
227
  except Exception as e:
228
  logger.error(f"Error executing agent tool '{call.name}': {e}")
229
+ result = f"Error executing tool {call.name}: {str(e)}"
230
 
231
  function_responses.append(
232
  types.Part.from_function_response(
graph_store.py ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import logging
3
+ from typing import List, Dict, Any
4
+ from neo4j import GraphDatabase
5
+
6
+ logger = logging.getLogger("rag_backend")
7
+
8
+ class Neo4jGraphStore:
9
+ def __init__(self):
10
+ self.uri = os.getenv("NEO4J_URI", "bolt://localhost:7687")
11
+ self.username = os.getenv("NEO4J_USERNAME", "neo4j")
12
+ self.password = os.getenv("NEO4J_PASSWORD", "password")
13
+ self.driver = None
14
+ self.is_connected = False
15
+
16
+ if not os.getenv("NEO4J_URI") and not os.getenv("NEO4J_PASSWORD"):
17
+ logger.warning("NEO4J_URI and NEO4J_PASSWORD not found in environment variables. Graph features will be disabled or fallback to default bolt://localhost:7687.")
18
+
19
+ try:
20
+ logger.info(f"Connecting to Neo4j at {self.uri}...")
21
+ self.driver = GraphDatabase.driver(self.uri, auth=(self.username, self.password))
22
+ # Verify connection with a lightweight query
23
+ with self.driver.session() as session:
24
+ session.run("RETURN 1")
25
+ self.is_connected = True
26
+ logger.info("Connected to Neo4j successfully.")
27
+ self._ensure_constraints()
28
+ except Exception as e:
29
+ logger.error(f"Failed to connect to Neo4j: {e}. GraphRAG feature will run in fallback (degraded) mode.")
30
+ self.driver = None
31
+ self.is_connected = False
32
+
33
+ def _ensure_constraints(self):
34
+ """Creates unique constraints to prevent duplicate entities and chunks."""
35
+ if not self.driver:
36
+ return
37
+ try:
38
+ with self.driver.session() as session:
39
+ session.run("CREATE CONSTRAINT unique_entity_name IF NOT EXISTS FOR (e:Entity) REQUIRE e.name IS UNIQUE")
40
+ session.run("CREATE CONSTRAINT unique_chunk_id IF NOT EXISTS FOR (c:Chunk) REQUIRE c.id IS UNIQUE")
41
+ logger.info("Neo4j database constraints verified/created.")
42
+ except Exception as e:
43
+ logger.error(f"Failed to create Neo4j constraints: {e}")
44
+
45
+ def add_graph_data(
46
+ self,
47
+ chunk_id: str,
48
+ chunk_text: str,
49
+ source: str,
50
+ entities: List[Dict[str, Any]],
51
+ relationships: List[Dict[str, Any]]
52
+ ) -> bool:
53
+ """
54
+ Inserts entities, relationships, and links them to the source chunk in Neo4j.
55
+ """
56
+ if not self.driver:
57
+ logger.warning("Neo4j driver not initialized. Skipping graph ingestion.")
58
+ return False
59
+
60
+ query = """
61
+ // 1. Create Chunk
62
+ MERGE (c:Chunk {id: $chunk_id})
63
+ SET c.text = $chunk_text, c.source = $source
64
+
65
+ // 2. Create Entities & Link to Chunk
66
+ WITH c
67
+ UNWIND $entities AS ent
68
+ MERGE (e:Entity {name: ent.name})
69
+ ON CREATE SET e.type = ent.type, e.description = ent.description
70
+ MERGE (c)-[:MENTIONS]->(e)
71
+
72
+ // 3. Create Relationships between Entities
73
+ WITH c
74
+ UNWIND $relationships AS rel
75
+ MERGE (source:Entity {name: rel.source})
76
+ MERGE (target:Entity {name: rel.target})
77
+ MERGE (source)-[r:CONNECTS_TO {relationship_type: rel.type}]->(target)
78
+ ON CREATE SET r.description = rel.description
79
+ """
80
+ try:
81
+ with self.driver.session() as session:
82
+ session.run(
83
+ query,
84
+ chunk_id=chunk_id,
85
+ chunk_text=chunk_text,
86
+ source=source,
87
+ entities=entities,
88
+ relationships=relationships
89
+ )
90
+ return True
91
+ except Exception as e:
92
+ logger.error(f"Error adding data to Neo4j: {e}")
93
+ return False
94
+
95
+ def query_subgraph(self, entity_names: List[str]) -> str:
96
+ """
97
+ Retrieves 1-hop and 2-hop connected entities for context injection.
98
+ """
99
+ if not self.driver:
100
+ return "Graph database connection is not available."
101
+
102
+ if not entity_names:
103
+ return "No entities provided for graph query."
104
+
105
+ # Case-insensitive or direct match? Since they are extracted, exact matches on normalized names are best.
106
+ query = """
107
+ MATCH (e:Entity) WHERE e.name IN $names
108
+ MATCH (e)-[r:CONNECTS_TO]-(neighbor:Entity)
109
+ RETURN e.name AS source, type(r) AS rel, r.relationship_type AS rel_type, r.description AS desc, neighbor.name AS target
110
+ LIMIT 30
111
+ """
112
+ try:
113
+ with self.driver.session() as session:
114
+ result = session.run(query, names=entity_names)
115
+ records = list(result)
116
+ if not records:
117
+ return f"No graph relationships found for entities: {', '.join(entity_names)}"
118
+
119
+ lines = []
120
+ for record in records:
121
+ rel_label = record['rel_type'] or record['rel']
122
+ desc = f" ({record['desc']})" if record['desc'] else ""
123
+ lines.append(f"({record['source']}) -[{rel_label}{desc}]-> ({record['target']})")
124
+ return "\n".join(lines)
125
+ except Exception as e:
126
+ logger.error(f"Error querying Neo4j subgraph: {e}")
127
+ return f"Error querying graph database: {str(e)}"
128
+
129
+ def clear(self) -> bool:
130
+ """Clears all nodes and relationships from the Neo4j database."""
131
+ if not self.driver:
132
+ return False
133
+ try:
134
+ with self.driver.session() as session:
135
+ session.run("MATCH (n) DETACH DELETE n")
136
+ logger.info("Neo4j database has been reset successfully.")
137
+ return True
138
+ except Exception as e:
139
+ logger.error(f"Error clearing Neo4j database: {e}")
140
+ return False
141
+
142
+ def get_status(self) -> Dict[str, Any]:
143
+ """Returns connection status and basic database metrics."""
144
+ if not self.driver:
145
+ return {
146
+ "connection_type": "Disconnected",
147
+ "uri": self.uri,
148
+ "status": "offline",
149
+ "nodes_count": 0,
150
+ "relationships_count": 0
151
+ }
152
+
153
+ try:
154
+ with self.driver.session() as session:
155
+ nodes_res = session.run("MATCH (n) RETURN count(n) AS c")
156
+ nodes_count = nodes_res.single()["c"]
157
+
158
+ rels_res = session.run("MATCH ()-[r]->() RETURN count(r) AS c")
159
+ rels_count = rels_res.single()["c"]
160
+
161
+ return {
162
+ "connection_type": f"Neo4j ({self.uri})",
163
+ "status": "online",
164
+ "nodes_count": nodes_count,
165
+ "relationships_count": rels_count
166
+ }
167
+ except Exception as e:
168
+ logger.error(f"Error getting Neo4j status: {e}")
169
+ return {
170
+ "connection_type": f"Neo4j ({self.uri})",
171
+ "status": "error",
172
+ "error": str(e),
173
+ "nodes_count": 0,
174
+ "relationships_count": 0
175
+ }
176
+
177
+ def close(self):
178
+ """Closes the Neo4j driver connection."""
179
+ if self.driver:
180
+ self.driver.close()
181
+ logger.info("Neo4j connection closed.")
main.py CHANGED
@@ -10,8 +10,10 @@ from langchain_text_splitters import RecursiveCharacterTextSplitter
10
  from pypdf import PdfReader
11
 
12
  import os
 
13
  from vector_store import QdrantVectorStore
14
- from agent import run_agent_rag
 
15
 
16
  # Configure logging
17
  logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
@@ -44,6 +46,10 @@ logger.info("Initializing Qdrant Vector Store...")
44
  vector_store = QdrantVectorStore()
45
  logger.info("Qdrant Vector Store initialized successfully.")
46
 
 
 
 
 
47
 
48
  # Pydantic Schemas for Request/Response Validation
49
  class DocumentInput(BaseModel):
@@ -88,6 +94,26 @@ class AgentResponse(BaseModel):
88
  tool_calls: List[dict] = Field(..., description="A log of all tool calls made during reasoning.")
89
  sources: List[dict] = Field(..., description="A list of retrieved document chunks used as sources.")
90
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
 
92
  # API Endpoints
93
  @app.get("/", status_code=status.HTTP_200_OK)
@@ -109,7 +135,8 @@ def health():
109
  """
110
  return {
111
  "status": "healthy",
112
- "vector_store": vector_store.get_status()
 
113
  }
114
 
115
  @app.post("/documents", response_model=IngestionResponse, status_code=status.HTTP_201_CREATED)
@@ -164,6 +191,29 @@ def ingest_documents(payload: IngestionRequest):
164
  metadatas=all_metadatas
165
  )
166
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
  return IngestionResponse(
168
  status="success",
169
  message=f"Successfully indexed {len(payload.documents)} documents into {chunks_inserted} chunks.",
@@ -314,6 +364,29 @@ async def upload_document(
314
  metadatas=all_metadatas
315
  )
316
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  return IngestionResponse(
318
  status="success",
319
  message=f"Successfully indexed document '{filename}' into {chunks_inserted} chunks.",
@@ -390,6 +463,7 @@ def agent_chat(payload: AgentRequest):
390
  user_message=payload.prompt,
391
  embedding_model=embedding_model,
392
  vector_store=vector_store,
 
393
  chat_history=chat_history_dicts
394
  )
395
  return AgentResponse(
@@ -414,15 +488,126 @@ def get_documents_status():
414
  @app.delete("/documents", status_code=status.HTTP_200_OK)
415
  def clear_documents():
416
  """
417
- Resets/clears the vector database collection.
418
  """
419
- success = vector_store.clear()
420
- if not success:
 
 
 
 
 
 
421
  raise HTTPException(
422
  status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
423
- detail="Failed to reset the vector database collection."
424
  )
425
  return {
426
  "status": "success",
427
- "message": "Vector database has been reset successfully."
428
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  from pypdf import PdfReader
11
 
12
  import os
13
+ import uuid
14
  from vector_store import QdrantVectorStore
15
+ from graph_store import Neo4jGraphStore
16
+ from agent import run_agent_rag, extract_graph_elements
17
 
18
  # Configure logging
19
  logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 
46
  vector_store = QdrantVectorStore()
47
  logger.info("Qdrant Vector Store initialized successfully.")
48
 
49
+ logger.info("Initializing Neo4j Graph Store...")
50
+ graph_store = Neo4jGraphStore()
51
+ logger.info("Neo4j Graph Store initialized successfully.")
52
+
53
 
54
  # Pydantic Schemas for Request/Response Validation
55
  class DocumentInput(BaseModel):
 
94
  tool_calls: List[dict] = Field(..., description="A log of all tool calls made during reasoning.")
95
  sources: List[dict] = Field(..., description="A list of retrieved document chunks used as sources.")
96
 
97
+ class GraphRetrieveRequest(BaseModel):
98
+ query: str = Field(..., description="The query to extract entities from and search in the graph.")
99
+ top_k_entities: int = Field(default=5, ge=1, le=20, description="The maximum number of entities to extract and query.")
100
+
101
+ class GraphRetrieveResponse(BaseModel):
102
+ query: str
103
+ extracted_entities: List[str]
104
+ relationships: str
105
+
106
+ class GraphQueryRequest(BaseModel):
107
+ query: str = Field(..., description="The user question to answer using the graph context.")
108
+
109
+ class GraphQueryResponse(BaseModel):
110
+ query: str
111
+ answer: str
112
+ context: str
113
+
114
+ class EntityListModel(BaseModel):
115
+ entities: List[str] = Field(..., description="List of normalized entity names/nouns found in the query.")
116
+
117
 
118
  # API Endpoints
119
  @app.get("/", status_code=status.HTTP_200_OK)
 
135
  """
136
  return {
137
  "status": "healthy",
138
+ "vector_store": vector_store.get_status(),
139
+ "graph_store": graph_store.get_status()
140
  }
141
 
142
  @app.post("/documents", response_model=IngestionResponse, status_code=status.HTTP_201_CREATED)
 
191
  metadatas=all_metadatas
192
  )
193
 
194
+ # Dual indexing to Neo4j
195
+ if graph_store.is_connected:
196
+ logger.info("Extracting and indexing graph relationships into Neo4j...")
197
+ for idx, chunk in enumerate(all_chunks):
198
+ try:
199
+ chunk_uuid = str(uuid.uuid4())
200
+ meta = all_metadatas[idx]
201
+ source = meta.get("source", f"doc_{meta.get('doc_index', 0)}")
202
+
203
+ graph_data = extract_graph_elements(chunk)
204
+ entities_list = [ent.model_dump() for ent in graph_data.entities]
205
+ relationships_list = [rel.model_dump() for rel in graph_data.relationships]
206
+
207
+ graph_store.add_graph_data(
208
+ chunk_id=chunk_uuid,
209
+ chunk_text=chunk,
210
+ source=source,
211
+ entities=entities_list,
212
+ relationships=relationships_list
213
+ )
214
+ except Exception as ex:
215
+ logger.error(f"Failed to index chunk {idx} in Neo4j: {ex}")
216
+
217
  return IngestionResponse(
218
  status="success",
219
  message=f"Successfully indexed {len(payload.documents)} documents into {chunks_inserted} chunks.",
 
364
  metadatas=all_metadatas
365
  )
366
 
367
+ # Dual indexing to Neo4j
368
+ if graph_store.is_connected:
369
+ logger.info("Extracting and indexing graph relationships from uploaded file into Neo4j...")
370
+ for idx, chunk in enumerate(chunks):
371
+ try:
372
+ chunk_uuid = str(uuid.uuid4())
373
+ meta = all_metadatas[idx]
374
+ source = meta.get("source", filename)
375
+
376
+ graph_data = extract_graph_elements(chunk)
377
+ entities_list = [ent.model_dump() for ent in graph_data.entities]
378
+ relationships_list = [rel.model_dump() for rel in graph_data.relationships]
379
+
380
+ graph_store.add_graph_data(
381
+ chunk_id=chunk_uuid,
382
+ chunk_text=chunk,
383
+ source=source,
384
+ entities=entities_list,
385
+ relationships=relationships_list
386
+ )
387
+ except Exception as ex:
388
+ logger.error(f"Failed to index uploaded chunk {idx} in Neo4j: {ex}")
389
+
390
  return IngestionResponse(
391
  status="success",
392
  message=f"Successfully indexed document '{filename}' into {chunks_inserted} chunks.",
 
463
  user_message=payload.prompt,
464
  embedding_model=embedding_model,
465
  vector_store=vector_store,
466
+ graph_store=graph_store,
467
  chat_history=chat_history_dicts
468
  )
469
  return AgentResponse(
 
488
  @app.delete("/documents", status_code=status.HTTP_200_OK)
489
  def clear_documents():
490
  """
491
+ Resets/clears the vector database and the graph database collections.
492
  """
493
+ vector_success = vector_store.clear()
494
+
495
+ # Clear Neo4j
496
+ graph_success = True
497
+ if graph_store.is_connected:
498
+ graph_success = graph_store.clear()
499
+
500
+ if not vector_success or not graph_success:
501
  raise HTTPException(
502
  status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
503
+ detail="Failed to reset the vector database or graph database collections."
504
  )
505
  return {
506
  "status": "success",
507
+ "message": "Vector and Graph databases have been reset successfully."
508
  }
509
+
510
+ def extract_query_entities(query_text: str) -> List[str]:
511
+ if not os.getenv("GEMINI_API_KEY"):
512
+ return []
513
+ from google import genai
514
+ from google.genai import types
515
+ client = genai.Client()
516
+ try:
517
+ response = client.models.generate_content(
518
+ model="gemini-2.5-flash",
519
+ contents=f"Extract the main named entities, concepts, or terms from this search query:\n\n{query_text}",
520
+ config=types.GenerateContentConfig(
521
+ response_mime_type="application/json",
522
+ response_schema=EntityListModel,
523
+ system_instruction="Extract only the key noun entities/concepts that would be useful for searching a knowledge graph."
524
+ )
525
+ )
526
+ obj = EntityListModel.model_validate_json(response.text)
527
+ return obj.entities
528
+ except Exception as e:
529
+ logger.error(f"Failed to extract query entities: {e}")
530
+ return []
531
+
532
+ @app.post("/retrieve/graph", response_model=GraphRetrieveResponse, status_code=status.HTTP_200_OK)
533
+ def retrieve_graph_context(payload: GraphRetrieveRequest):
534
+ """
535
+ Extracts key entities from the search query and retrieves their
536
+ 1-hop and 2-hop relationships from the Neo4j Graph Database.
537
+ """
538
+ if not graph_store.is_connected:
539
+ raise HTTPException(
540
+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
541
+ detail="Neo4j Graph Database is not connected."
542
+ )
543
+ try:
544
+ entities = extract_query_entities(payload.query)
545
+ entities = entities[:payload.top_k_entities]
546
+ relationships = graph_store.query_subgraph(entities)
547
+ return GraphRetrieveResponse(
548
+ query=payload.query,
549
+ extracted_entities=entities,
550
+ relationships=relationships
551
+ )
552
+ except Exception as e:
553
+ logger.error(f"Error during graph retrieval: {e}")
554
+ raise HTTPException(
555
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
556
+ detail=f"An error occurred during graph retrieval: {str(e)}"
557
+ )
558
+
559
+ @app.post("/query/graph", response_model=GraphQueryResponse, status_code=status.HTTP_200_OK)
560
+ def query_graph_rag(payload: GraphQueryRequest):
561
+ """
562
+ Performs GraphRAG querying: retrieves relevant entity relationships from
563
+ Neo4j and prompts Gemini to answer the query grounded in that context.
564
+ """
565
+ if not graph_store.is_connected:
566
+ raise HTTPException(
567
+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
568
+ detail="Neo4j Graph Database is not connected."
569
+ )
570
+ if not os.getenv("GEMINI_API_KEY"):
571
+ raise HTTPException(
572
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
573
+ detail="GEMINI_API_KEY environment variable is not configured."
574
+ )
575
+
576
+ try:
577
+ entities = extract_query_entities(payload.query)
578
+ relationships = graph_store.query_subgraph(entities)
579
+
580
+ from google import genai
581
+ from google.genai import types
582
+ client = genai.Client()
583
+
584
+ prompt = f"""
585
+ Answer the user query based ONLY on the provided knowledge graph relationships.
586
+ If the information is not present, state clearly that it was not found in the graph.
587
+
588
+ Knowledge Graph Relationships:
589
+ {relationships}
590
+
591
+ User Query:
592
+ {payload.query}
593
+ """
594
+
595
+ response = client.models.generate_content(
596
+ model="gemini-2.5-flash",
597
+ contents=prompt,
598
+ config=types.GenerateContentConfig(
599
+ system_instruction="You are a precise GraphRAG assistant. Answer questions using only the relationships provided."
600
+ )
601
+ )
602
+
603
+ return GraphQueryResponse(
604
+ query=payload.query,
605
+ answer=response.text,
606
+ context=relationships
607
+ )
608
+ except Exception as e:
609
+ logger.error(f"Error during graph query: {e}")
610
+ raise HTTPException(
611
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
612
+ detail=f"An error occurred during graph query: {str(e)}"
613
+ )
requirements.txt CHANGED
@@ -11,3 +11,4 @@ pytest>=7.0.0
11
  pypdf>=4.0.0
12
  python-multipart>=0.0.9
13
  google-genai>=0.1.1
 
 
11
  pypdf>=4.0.0
12
  python-multipart>=0.0.9
13
  google-genai>=0.1.1
14
+ neo4j>=5.0.0
test_api.py CHANGED
@@ -10,7 +10,41 @@ if "QDRANT_URL" in os.environ:
10
  if "QDRANT_API_KEY" in os.environ:
11
  del os.environ["QDRANT_API_KEY"]
12
 
13
- from main import app
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
 
15
  client = TestClient(app)
16
 
@@ -260,11 +294,79 @@ def test_agent_chat_endpoint_success():
260
  mock_run.assert_called_once()
261
  called_args, called_kwargs = mock_run.call_args
262
  assert called_kwargs["user_message"] == "explain antigravity"
 
263
  assert called_kwargs["chat_history"] == [
264
  {"role": "user", "content": "hi"},
265
  {"role": "model", "content": "hello"}
266
  ]
267
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
268
  if __name__ == "__main__":
269
  import sys
270
  import pytest
 
10
  if "QDRANT_API_KEY" in os.environ:
11
  del os.environ["QDRANT_API_KEY"]
12
 
13
+ from unittest.mock import MagicMock, patch
14
+
15
+ # Create a mock graph store globally for testing
16
+ mock_graph_store = MagicMock()
17
+ mock_graph_store.is_connected = True
18
+ mock_graph_store.get_status.return_value = {
19
+ "connection_type": "Mock Neo4j",
20
+ "status": "online",
21
+ "nodes_count": 5,
22
+ "relationships_count": 4
23
+ }
24
+ mock_graph_store.query_subgraph.return_value = "(Alice) -[WORKS_AT]-> (BobCorp)"
25
+ mock_graph_store.clear.return_value = True
26
+
27
+ with patch("graph_store.Neo4jGraphStore", return_value=mock_graph_store):
28
+ from main import app
29
+
30
+ # Mock Gemini extraction functions globally for test stability
31
+ from agent import GraphExtractionModel, EntityModel, RelationshipModel
32
+
33
+ mock_extraction_result = GraphExtractionModel(
34
+ entities=[
35
+ EntityModel(name="Alice", type="Person", description="A software engineer."),
36
+ EntityModel(name="BobCorp", type="Organization", description="A software company.")
37
+ ],
38
+ relationships=[
39
+ RelationshipModel(source="Alice", target="BobCorp", type="WORKS_AT", description="works at")
40
+ ]
41
+ )
42
+
43
+ patcher_extract_graph = patch("main.extract_graph_elements", return_value=mock_extraction_result)
44
+ patcher_extract_graph.start()
45
+
46
+ patcher_extract_entities = patch("main.extract_query_entities", return_value=["Alice", "BobCorp"])
47
+ patcher_extract_entities.start()
48
 
49
  client = TestClient(app)
50
 
 
294
  mock_run.assert_called_once()
295
  called_args, called_kwargs = mock_run.call_args
296
  assert called_kwargs["user_message"] == "explain antigravity"
297
+ assert called_kwargs["graph_store"] == mock_graph_store
298
  assert called_kwargs["chat_history"] == [
299
  {"role": "user", "content": "hi"},
300
  {"role": "model", "content": "hello"}
301
  ]
302
 
303
+ def test_health_endpoint_includes_graph():
304
+ response = client.get("/health")
305
+ assert response.status_code == 200
306
+ data = response.json()
307
+ assert "graph_store" in data
308
+ assert data["graph_store"]["connection_type"] == "Mock Neo4j"
309
+ assert data["graph_store"]["status"] == "online"
310
+
311
+ def test_documents_ingest_calls_graph_store():
312
+ mock_graph_store.reset_mock()
313
+ # Ingest document
314
+ payload = {
315
+ "documents": [
316
+ {
317
+ "text": "Alice works at BobCorp.",
318
+ "metadata": {"source": "test_doc"}
319
+ }
320
+ ],
321
+ "chunk_size": 100,
322
+ "chunk_overlap": 10
323
+ }
324
+ response = client.post("/documents", json=payload)
325
+ assert response.status_code == 201
326
+
327
+ # Verify graph store add_graph_data was called
328
+ mock_graph_store.add_graph_data.assert_called_once()
329
+ called_args, called_kwargs = mock_graph_store.add_graph_data.call_args
330
+ assert called_kwargs["source"] == "test_doc"
331
+ assert called_kwargs["chunk_text"] == "Alice works at BobCorp."
332
+ assert called_kwargs["entities"][0]["name"] == "Alice"
333
+ assert called_kwargs["relationships"][0]["type"] == "WORKS_AT"
334
+
335
+ def test_retrieve_graph_endpoint():
336
+ response = client.post(
337
+ "/retrieve/graph",
338
+ json={"query": "Alice BobCorp", "top_k_entities": 2}
339
+ )
340
+ assert response.status_code == 200
341
+ data = response.json()
342
+ assert data["query"] == "Alice BobCorp"
343
+ assert data["extracted_entities"] == ["Alice", "BobCorp"]
344
+ assert "WORKS_AT" in data["relationships"]
345
+
346
+ def test_query_graph_endpoint():
347
+ mock_genai_client = MagicMock()
348
+ mock_response = MagicMock()
349
+ mock_response.text = "Based on the graph, Alice works at BobCorp."
350
+ mock_genai_client.models.generate_content.return_value = mock_response
351
+
352
+ with patch("google.genai.Client", return_value=mock_genai_client):
353
+ with patch.dict(os.environ, {"GEMINI_API_KEY": "mock-key"}):
354
+ response = client.post(
355
+ "/query/graph",
356
+ json={"query": "Where does Alice work?"}
357
+ )
358
+ assert response.status_code == 200
359
+ data = response.json()
360
+ assert data["query"] == "Where does Alice work?"
361
+ assert "Alice works at BobCorp" in data["answer"]
362
+ assert "WORKS_AT" in data["context"]
363
+
364
+ def test_clear_documents_clears_graph():
365
+ mock_graph_store.reset_mock()
366
+ response = client.delete("/documents")
367
+ assert response.status_code == 200
368
+ mock_graph_store.clear.assert_called_once()
369
+
370
  if __name__ == "__main__":
371
  import sys
372
  import pytest