Commit ·
083c974
1
Parent(s): 9b209cc
Update generate_knowledge_graph.py
Browse files- generate_knowledge_graph.py +23 -11
generate_knowledge_graph.py
CHANGED
|
@@ -15,14 +15,17 @@ api_key = os.getenv("OPENAI_API_KEY")
|
|
| 15 |
llm = ChatOpenAI(temperature=0, model_name="gpt-4o")
|
| 16 |
graph_transformer = LLMGraphTransformer(llm=llm)
|
| 17 |
|
|
|
|
| 18 |
async def extract_graph_data(text):
|
| 19 |
documents = [Document(page_content=text)]
|
| 20 |
graph_documents = await graph_transformer.aconvert_to_graph_documents(documents)
|
| 21 |
return graph_documents
|
| 22 |
|
|
|
|
| 23 |
def visualize_graph(graph_documents):
|
| 24 |
-
net = Network(height="600px", width="100%", directed=True, notebook=False,
|
| 25 |
-
|
|
|
|
| 26 |
# Collect all nodes and relationships across all graph documents in the list
|
| 27 |
all_nodes = []
|
| 28 |
all_relationships = []
|
|
@@ -32,7 +35,7 @@ def visualize_graph(graph_documents):
|
|
| 32 |
|
| 33 |
# Build node dictionary to deduplicate nodes by ID
|
| 34 |
node_dict = {node.id: node for node in all_nodes}
|
| 35 |
-
|
| 36 |
# Dynamically add any nodes referenced in relationships but missing from nodes list
|
| 37 |
for rel in all_relationships:
|
| 38 |
if rel.source.id not in node_dict:
|
|
@@ -51,7 +54,8 @@ def visualize_graph(graph_documents):
|
|
| 51 |
for node_id in valid_node_ids:
|
| 52 |
node = node_dict[node_id]
|
| 53 |
try:
|
| 54 |
-
net.add_node(node.id, label=node.id,
|
|
|
|
| 55 |
except:
|
| 56 |
continue
|
| 57 |
|
|
@@ -61,7 +65,8 @@ def visualize_graph(graph_documents):
|
|
| 61 |
edge_key = (rel.source.id, rel.target.id, rel.type.lower())
|
| 62 |
if edge_key not in edges_added:
|
| 63 |
try:
|
| 64 |
-
net.add_edge(rel.source.id, rel.target.id,
|
|
|
|
| 65 |
edges_added.add(edge_key)
|
| 66 |
except:
|
| 67 |
continue
|
|
@@ -75,6 +80,7 @@ def generate_knowledge_graph(text):
|
|
| 75 |
net = visualize_graph(graph_documents)
|
| 76 |
return net, graph_documents
|
| 77 |
|
|
|
|
| 78 |
def answer_question_with_graph(question, graph_documents, k_relations=7):
|
| 79 |
all_relationships = []
|
| 80 |
for doc in graph_documents:
|
|
@@ -86,13 +92,15 @@ def answer_question_with_graph(question, graph_documents, k_relations=7):
|
|
| 86 |
rel_docs = []
|
| 87 |
for i, rel in enumerate(all_relationships):
|
| 88 |
text_rep = f"The entity '{rel.source.id}' has a '{rel.type}' relationship with the entity '{rel.target.id}'."
|
| 89 |
-
rel_docs.append(Document(page_content=text_rep,
|
|
|
|
| 90 |
|
| 91 |
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
|
| 92 |
vectorstore = FAISS.from_documents(rel_docs, embeddings)
|
| 93 |
retrieved_docs = vectorstore.similarity_search(question, k=k_relations)
|
| 94 |
|
| 95 |
-
used_relationships = [
|
|
|
|
| 96 |
context = "\n".join([doc.page_content for doc in retrieved_docs])
|
| 97 |
|
| 98 |
prompt = PromptTemplate(
|
|
@@ -103,22 +111,26 @@ def answer_question_with_graph(question, graph_documents, k_relations=7):
|
|
| 103 |
chain = prompt | llm
|
| 104 |
answer = chain.invoke({"context": context, "question": question}).content
|
| 105 |
|
| 106 |
-
net = Network(height="450px", width="100%", directed=True,
|
|
|
|
| 107 |
|
| 108 |
nodes_added = set()
|
| 109 |
for rel in used_relationships:
|
| 110 |
if rel.source.id not in nodes_added:
|
| 111 |
-
net.add_node(rel.source.id, label=rel.source.id,
|
|
|
|
| 112 |
nodes_added.add(rel.source.id)
|
| 113 |
if rel.target.id not in nodes_added:
|
| 114 |
-
net.add_node(rel.target.id, label=rel.target.id,
|
|
|
|
| 115 |
nodes_added.add(rel.target.id)
|
| 116 |
try:
|
| 117 |
net.add_edge(rel.source.id, rel.target.id, label=rel.type)
|
| 118 |
except:
|
| 119 |
pass
|
| 120 |
|
| 121 |
-
net.set_options(
|
|
|
|
| 122 |
net.save_graph("filtered_graph.html")
|
| 123 |
|
| 124 |
return answer, net
|
|
|
|
| 15 |
llm = ChatOpenAI(temperature=0, model_name="gpt-4o")
|
| 16 |
graph_transformer = LLMGraphTransformer(llm=llm)
|
| 17 |
|
| 18 |
+
|
| 19 |
async def extract_graph_data(text):
|
| 20 |
documents = [Document(page_content=text)]
|
| 21 |
graph_documents = await graph_transformer.aconvert_to_graph_documents(documents)
|
| 22 |
return graph_documents
|
| 23 |
|
| 24 |
+
|
| 25 |
def visualize_graph(graph_documents):
|
| 26 |
+
net = Network(height="600px", width="100%", directed=True, notebook=False,
|
| 27 |
+
bgcolor="#222222", font_color="white", filter_menu=True, cdn_resources='remote')
|
| 28 |
+
|
| 29 |
# Collect all nodes and relationships across all graph documents in the list
|
| 30 |
all_nodes = []
|
| 31 |
all_relationships = []
|
|
|
|
| 35 |
|
| 36 |
# Build node dictionary to deduplicate nodes by ID
|
| 37 |
node_dict = {node.id: node for node in all_nodes}
|
| 38 |
+
|
| 39 |
# Dynamically add any nodes referenced in relationships but missing from nodes list
|
| 40 |
for rel in all_relationships:
|
| 41 |
if rel.source.id not in node_dict:
|
|
|
|
| 54 |
for node_id in valid_node_ids:
|
| 55 |
node = node_dict[node_id]
|
| 56 |
try:
|
| 57 |
+
net.add_node(node.id, label=node.id,
|
| 58 |
+
title=node.type, group=node.type)
|
| 59 |
except:
|
| 60 |
continue
|
| 61 |
|
|
|
|
| 65 |
edge_key = (rel.source.id, rel.target.id, rel.type.lower())
|
| 66 |
if edge_key not in edges_added:
|
| 67 |
try:
|
| 68 |
+
net.add_edge(rel.source.id, rel.target.id,
|
| 69 |
+
label=rel.type.lower())
|
| 70 |
edges_added.add(edge_key)
|
| 71 |
except:
|
| 72 |
continue
|
|
|
|
| 80 |
net = visualize_graph(graph_documents)
|
| 81 |
return net, graph_documents
|
| 82 |
|
| 83 |
+
|
| 84 |
def answer_question_with_graph(question, graph_documents, k_relations=7):
|
| 85 |
all_relationships = []
|
| 86 |
for doc in graph_documents:
|
|
|
|
| 92 |
rel_docs = []
|
| 93 |
for i, rel in enumerate(all_relationships):
|
| 94 |
text_rep = f"The entity '{rel.source.id}' has a '{rel.type}' relationship with the entity '{rel.target.id}'."
|
| 95 |
+
rel_docs.append(Document(page_content=text_rep,
|
| 96 |
+
metadata={"rel_index": i}))
|
| 97 |
|
| 98 |
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
|
| 99 |
vectorstore = FAISS.from_documents(rel_docs, embeddings)
|
| 100 |
retrieved_docs = vectorstore.similarity_search(question, k=k_relations)
|
| 101 |
|
| 102 |
+
used_relationships = [
|
| 103 |
+
all_relationships[doc.metadata["rel_index"]] for doc in retrieved_docs]
|
| 104 |
context = "\n".join([doc.page_content for doc in retrieved_docs])
|
| 105 |
|
| 106 |
prompt = PromptTemplate(
|
|
|
|
| 111 |
chain = prompt | llm
|
| 112 |
answer = chain.invoke({"context": context, "question": question}).content
|
| 113 |
|
| 114 |
+
net = Network(height="450px", width="100%", directed=True,
|
| 115 |
+
bgcolor="#222222", font_color="white")
|
| 116 |
|
| 117 |
nodes_added = set()
|
| 118 |
for rel in used_relationships:
|
| 119 |
if rel.source.id not in nodes_added:
|
| 120 |
+
net.add_node(rel.source.id, label=rel.source.id,
|
| 121 |
+
title=f"Label : {rel.source.id}\nGroup : {rel.source.type}", group=rel.source.type)
|
| 122 |
nodes_added.add(rel.source.id)
|
| 123 |
if rel.target.id not in nodes_added:
|
| 124 |
+
net.add_node(rel.target.id, label=rel.target.id,
|
| 125 |
+
title=f"Label : {rel.target.id}\nGroup : {rel.target.type}", group=rel.target.type)
|
| 126 |
nodes_added.add(rel.target.id)
|
| 127 |
try:
|
| 128 |
net.add_edge(rel.source.id, rel.target.id, label=rel.type)
|
| 129 |
except:
|
| 130 |
pass
|
| 131 |
|
| 132 |
+
net.set_options(
|
| 133 |
+
'{"physics": {"forceAtlas2Based": {"gravitationalConstant": -50}}}')
|
| 134 |
net.save_graph("filtered_graph.html")
|
| 135 |
|
| 136 |
return answer, net
|