GraphRAG / graph_processor.py
dipenvekaria's picture
Upload 6 files
7b43396 verified
Raw History Blame
18 kB
import os
import uuid
from datetime import datetime
from dotenv import load_dotenv
from neo4j import GraphDatabase
from openai import OpenAI # Use OpenAI API directly
from utils import extract_pdf_text
import logging
import re
import json
load_dotenv()
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class GraphDBProcessor:
def __init__(self):
self.openai_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) # Initialize OpenAI client
try:
self.driver = GraphDatabase.driver(
os.getenv("NEO4J_URI"),
auth=(os.getenv("NEO4J_USERNAME"), os.getenv("NEO4J_PASSWORD"))
)
self._ensure_graph_schema()
except Exception as e:
logger.error(f"Failed to initialize Neo4j driver: {str(e)}")
raise
def _ensure_graph_schema(self):
try:
with self.driver.session() as session:
session.run("""
CREATE CONSTRAINT IF NOT EXISTS FOR (e:Entity) REQUIRE e.id IS UNIQUE
""")
session.run("""
CREATE CONSTRAINT IF NOT EXISTS FOR (d:Document) REQUIRE d.id IS UNIQUE
""")
except Exception as e:
logger.error(f"Error creating schema constraints: {str(e)}")
raise
def _extract_entities_and_relationships(self, text):
try:
prompt = f"""Extract key entities (e.g., people, organizations, locations) and their relationships from the following text. Return a JSON list of entities and relationships in this format:
{{
"entities": [{{"id": "unique_id", "name": "entity_name", "type": "entity_type"}}],
"relationships": [{{"source": "source_id", "target": "target_id", "type": "relationship_type"}}]
}}
Entities should include people (type: "Person"), organizations (type: "Organization"), and locations (type: "Location"). Relationships should describe connections like WORKS_FOR (e.g., a person works for an organization), HEADQUARTERED_IN (e.g., an organization is located in a city), FOUNDED (e.g., a person founded an organization), or COLLABORATED_WITH (e.g., two people collaborated). Ensure relationship types are in uppercase (e.g., WORKS_FOR, FOUNDED). Ensure IDs are unique, lowercase, and use underscores instead of spaces (e.g., "alice_johnson"). If no entities or relationships are found, return empty lists. Pay special attention to identifying founding relationships (e.g., "Founded by Alice Johnson" should result in a FOUNDED relationship).
Examples:
Text: "Alice Johnson is the CEO of TechCorp, which is headquartered in San Francisco. TechCorp was founded by Alice Johnson."
Output:
{{
"entities": [
{{"id": "alice_johnson", "name": "Alice Johnson", "type": "Person"}},
{{"id": "techcorp", "name": "TechCorp", "type": "Organization"}},
{{"id": "san_francisco", "name": "San Francisco", "type": "Location"}}
],
"relationships": [
{{"source": "alice_johnson", "target": "techcorp", "type": "WORKS_FOR"}},
{{"source": "alice_johnson", "target": "techcorp", "type": "FOUNDED"}},
{{"source": "techcorp", "target": "san_francisco", "type": "HEADQUARTERED_IN"}}
]
}}
Text: "TechCorp was founded in 2015."
Output:
{{
"entities": [
{{"id": "techcorp", "name": "TechCorp", "type": "Organization"}}
],
"relationships": []
}}
Text: {text}
Output: """
logger.info(f"Input text to LLM (first 5000 chars): {text[:5000]}")
# Call OpenAI API
response = self.openai_client.chat.completions.create(
model="gpt-4o-mini",
messages=[
{"role": "system", "content": "You are a helpful assistant that extracts entities and relationships from text in JSON format."},
{"role": "user", "content": prompt}
],
temperature=0,
max_tokens=1000
)
result = response.choices[0].message.content.strip()
logger.info(f"Raw LLM output for entity extraction: {result}")
# Clean up Markdown code block markers
cleaned_result = re.sub(r'```json\s*|\s*```', '', result).strip()
logger.info(f"Cleaned LLM output for JSON parsing: {cleaned_result}")
# Validate JSON
parsed_result = json.loads(cleaned_result)
if not isinstance(parsed_result, dict):
logger.error("LLM output is not a dictionary")
return {"entities": [], "relationships": []}
if "entities" not in parsed_result or "relationships" not in parsed_result:
logger.error("LLM output missing 'entities' or 'relationships' keys")
return {"entities": [], "relationships": []}
# Validate entities
entities = parsed_result["entities"]
if not isinstance(entities, list):
logger.error(f"Entities is not a list: {entities}")
return {"entities": [], "relationships": []}
for entity in entities:
if not isinstance(entity, dict):
logger.error(f"Entity is not a dict: {entity}")
return {"entities": [], "relationships": []}
if not all(key in entity for key in ["id", "name", "type"]):
logger.error(f"Entity missing required keys: {entity}")
return {"entities": [], "relationships": []}
if entity["type"] not in ["Person", "Organization", "Location"]:
logger.warning(f"Unexpected entity type: {entity['type']}")
# Validate relationships
relationships = parsed_result["relationships"]
if not isinstance(relationships, list):
logger.error(f"Relationships is not a list: {relationships}")
return {"entities": [], "relationships": []}
for rel in relationships:
if not isinstance(rel, dict):
logger.error(f"Relationship is not a dict: {rel}")
return {"entities": [], "relationships": []}
if not all(key in rel for key in ["source", "target", "type"]):
logger.error(f"Relationship missing required keys: {rel}")
return {"entities": [], "relationships": []}
entity_ids = {e["id"] for e in entities}
if rel["source"] not in entity_ids or rel["target"] not in entity_ids:
logger.error(f"Relationship references unknown entity IDs: {rel}")
return {"entities": [], "relationships": []}
rel["type"] = rel["type"].upper()
logger.info(f"Extracted {len(entities)} entities and {len(relationships)} relationships")
return parsed_result
except json.JSONDecodeError as e:
logger.error(f"JSON parsing error: {str(e)}")
return {"entities": [], "relationships": []}
except Exception as e:
logger.error(f"Error extracting entities: {str(e)}")
return {"entities": [], "relationships": []}
def process_and_store_pdf(self, file_obj):
try:
text = extract_pdf_text(file_obj.name)
filename = os.path.basename(file_obj.name)
doc_id = str(uuid.uuid4())
graph_data = self._extract_entities_and_relationships(text)
entities = graph_data["entities"]
relationships = graph_data["relationships"]
if not entities:
logger.warning(f"No entities extracted for {filename}")
with self.driver.session() as session:
logger.info(f"Creating Document node with id: {doc_id}")
session.run("""
MERGE (d:Document {id: $doc_id})
SET d.filename = $filename, d.conversion_date = $date
""", doc_id=doc_id, filename=filename, date=datetime.now().isoformat())
for entity in entities:
logger.info(f"Creating Entity node: {entity}")
session.run("""
MERGE (e:Entity {id: $id})
SET e.name = $name, e.type = $type
MERGE (d:Document {id: $doc_id})
MERGE (e)-[:MENTIONED_IN]->(d)
""", id=entity["id"], name=entity["name"], type=entity["type"], doc_id=doc_id)
for rel in relationships:
logger.info(f"Creating RELATED relationship: {rel}")
session.run("""
MATCH (source:Entity {id: $source})
MATCH (target:Entity {id: $target})
MERGE (source)-[r:RELATED {type: $type}]->(target)
""", source=rel["source"], target=rel["target"], type=rel["type"])
result = session.run("""
MATCH (e:Entity)-[r:RELATED]->(t:Entity)
WHERE e.id IN $entity_ids
RETURN e, r, t
""", entity_ids=[e["id"] for e in entities])
created_relationships = [record for record in result]
logger.info(f"Verified {len(created_relationships)} RELATED relationships for {filename}")
result = session.run("""
MATCH (e:Entity)-[r:MENTIONED_IN]->(d:Document)
WHERE d.id = $doc_id
RETURN e, r, d
""", doc_id=doc_id)
mentioned_relationships = [record for record in result]
logger.info(f"Verified {len(mentioned_relationships)} MENTIONED_IN relationships for {filename}")
logger.info(f"Processed PDF {filename} in Neo4j with {len(entities)} entities and {len(relationships)} relationships")
return filename
except Exception as e:
logger.error(f"Error processing PDF {file_obj.name}: {str(e)}")
raise
def query(self, question):
try:
# Define the prompt for Cypher query generation
prompt = f"""Convert the following question into a valid Cypher query to retrieve relevant entities and relationships from a Neo4j database. The database has Entity nodes (with id, name, type) and Document nodes (with id, filename), connected by MENTIONED_IN relationships, and Entity nodes connected by RELATED relationships with a 'type' property (e.g., WORKS_FOR, HEADQUARTERED_IN). Return a single Cypher query without using UNION. If a relationship is queried, assign it to a variable (e.g., [r:RELATED]) and use it in the RETURN clause (e.g., type(r) AS relationshipType). Ensure all variables in the RETURN clause are defined in the MATCH clause. For entity names in the question, use a CONTAINS condition to match partial names (e.g., for 'Alice', use name CONTAINS 'Alice' to match 'Alice Johnson'). For questions asking about the relationship between two entities, focus on the direct relationship between them and return only the relationship type. Return only the Cypher query text, without any Markdown, code blocks, or prefixes like '```cypher'.
Examples:
Question: "What is the relationship between Alice and TechCorp?"
Cypher Query:
MATCH (e:Entity)-[r:RELATED]->(t:Entity {{name: 'TechCorp'}})
WHERE e.name CONTAINS 'Alice'
RETURN type(r) AS relationshipType
Question: "Who works for TechCorp?"
Cypher Query:
MATCH (e:Entity)-[r:RELATED {{type: 'WORKS_FOR'}}]->(t:Entity {{name: 'TechCorp'}})
RETURN e.name AS entityName
Question: {question}
Cypher Query: """
# Call OpenAI API to generate the Cypher query
response = self.openai_client.chat.completions.create(
model="gpt-4o-mini",
messages=[
{"role": "system",
"content": "You are a helpful assistant that generates Cypher queries for a Neo4j database."},
{"role": "user", "content": prompt}
],
temperature=0,
max_tokens=150
)
cypher_query = response.choices[0].message.content.strip()
# Clean up any Markdown or code block markers
cypher_query = re.sub(r'```cypher\s*|\s*```', '', cypher_query).strip()
logger.info(f"Generated Cypher query: {cypher_query}")
# Basic query validation
if not cypher_query.startswith(('MATCH', 'WITH', 'MERGE', 'CREATE', 'RETURN')):
raise ValueError(f"Invalid Cypher query: {cypher_query}")
# Validate variables in RETURN clause
return_clause = cypher_query[cypher_query.upper().find("RETURN"):].lower()
return_vars = set()
# Improved parsing of RETURN clause
return_parts = return_clause.split(',')
for part in return_parts:
part = part.strip()
if not part:
continue
# Remove 'return' keyword if present
part = part.replace('return', '').strip()
if ' as ' in part:
var = part.split(' as ')[0].strip()
else:
var = part.strip()
# Handle expressions like type(r)
if var.startswith('type('):
var = var[5:var.find(')')]
# Handle dot notation like e.name
elif '.' in var:
var = var.split('.')[0]
if var and var not in {'type', 'as'}:
return_vars.add(var)
match_clause = cypher_query[:cypher_query.upper().find("RETURN")].lower()
defined_vars = set()
# Extract variables from MATCH and WHERE clauses
for match in re.finditer(r'\b([a-zA-Z_]\w*)\b(?=:|\s*{|\s*-|\s*\))', match_clause):
defined_vars.add(match.group(1))
# Extract additional variables from WHERE clause or other patterns
for match in re.finditer(r'\b([a-zA-Z_]\w*)\b', match_clause):
var = match.group(1)
if var not in {'match', 'optional', 'related', 'mentioned_in', 'entity', 'document', 'name', 'where',
'contains'}:
defined_vars.add(var)
undefined_vars = return_vars - defined_vars
if undefined_vars:
raise ValueError(f"Undefined variables in RETURN clause: {undefined_vars}")
with self.driver.session() as session:
result = session.run(cypher_query)
records = [record.data() for record in result]
context = "\n".join([str(record) for record in records])
if not context:
return {"answer": "No relevant information found in graph.", "chunks": []}
# Generate a human-readable answer using OpenAI API
answer_prompt = f"""Based solely on the provided graph data, provide a concise, accurate answer to the question. Do not use external knowledge. If the graph data contains multiple relationships, list them all in a human-readable format (e.g., 'Alice Johnson founded TechCorp (FOUNDED) and works for TechCorp (WORKS_FOR).').
Question: {question}
Graph Data: {context}
Answer: """
answer_response = self.openai_client.chat.completions.create(
model="gpt-4o-mini",
messages=[
{"role": "system",
"content": "You are a helpful assistant that answers questions based on graph data."},
{"role": "user", "content": answer_prompt}
],
temperature=0,
max_tokens=100
)
answer = answer_response.choices[0].message.content.strip()
return {"answer": answer, "chunks": records}
except Exception as e:
logger.error(f"Error querying graph: {str(e)}")
return {"answer": f"Graph query failed: {str(e)}", "chunks": []}
def get_processed_files(self):
try:
with self.driver.session() as session:
total_nodes = session.run("MATCH (d:Document) RETURN count(d) AS count").single()["count"]
result = session.run("""
MATCH (d:Document)
WHERE d.filename IS NOT NULL
RETURN d.filename AS filename
""")
files = {record["filename"] for record in result}
logger.info(f"Found {len(files)} Document nodes with filename property out of {total_nodes} total Document nodes.")
if not files:
logger.warning("No Document nodes with filename property found in Neo4j. Check database for inconsistent nodes.")
return files
except Exception as e:
logger.error(f"Error fetching processed files from Neo4j: {str(e)}")
return set()
def delete_file_points(self, filename):
try:
with self.driver.session() as session:
session.run("""
MATCH (d:Document {filename: $filename})
OPTIONAL MATCH (d)<-[:MENTIONED_IN]-(e:Entity)
OPTIONAL MATCH (e)-[r:RELATED]-()
DELETE d, r
WITH e
WHERE NOT (e)-[:MENTIONED_IN]->()
DELETE e
""", filename=filename)
logger.info(f"Deleted graph data for file: {filename}")
return f"Deleted graph data for file: {filename}"
except Exception as e:
logger.error(f"Error deleting file {filename}: {str(e)}")
return f"Error deleting file: {str(e)}"