yusufcalisir's picture
deploy: Hugging Face space upload
eff6797
Raw History Blame Contribute Delete
63.1 kB
"""Graph engine.
Builds and queries the entity relationship graph. The graph represents
connections between entities (customers, merchants, devices, etc.)
across institutions. Graph operations include neighbor expansion,
connected component detection, and serialization to React Flow format
for interactive visualization.
"""
from __future__ import annotations
import hashlib
import logging
import re
import threading
from collections import defaultdict, deque
from datetime import UTC, datetime, timedelta
from typing import Any
from app.config import get_settings
from app.domain.enums import EntityType, RelationshipType, RiskLevel
from app.domain.investigation_entities import Entity, Relationship
from app.domain.value_objects_investigation import GraphSubgraph
from app.infrastructure.redis_store import RedisStore
logger = logging.getLogger(__name__)
def _entity_to_dict(e: Entity) -> dict[str, Any]:
return {
"id": e.id,
"entity_type": e.entity_type.value,
"privacy_id": e.privacy_id,
"bank_id": e.bank_id,
"display_label": e.display_label,
"attributes": e.attributes,
"risk_level": e.risk_level.value,
"alert_count": e.alert_count,
"first_seen": e.first_seen.isoformat(),
"last_seen": e.last_seen.isoformat(),
}
def _dict_to_entity(d: dict[str, Any]) -> Entity:
from datetime import datetime
d_copy = d.copy()
d_copy["entity_type"] = EntityType(d_copy["entity_type"])
d_copy["risk_level"] = RiskLevel(d_copy["risk_level"])
d_copy["first_seen"] = datetime.fromisoformat(d_copy["first_seen"])
d_copy["last_seen"] = datetime.fromisoformat(d_copy["last_seen"])
return Entity(**d_copy)
def _relationship_to_dict(r: Relationship) -> dict[str, Any]:
return {
"id": r.id,
"source_entity_id": r.source_entity_id,
"target_entity_id": r.target_entity_id,
"relationship_type": r.relationship_type.value,
"confidence": r.confidence,
"evidence": r.evidence,
"created_at": r.created_at.isoformat(),
}
def _dict_to_relationship(d: dict[str, Any]) -> Relationship:
from datetime import datetime
d_copy = d.copy()
d_copy["relationship_type"] = RelationshipType(d_copy["relationship_type"])
d_copy["created_at"] = datetime.fromisoformat(d_copy["created_at"])
return Relationship(**d_copy)
# Color mapping for React Flow node types
_NODE_COLORS: dict[str, str] = {
EntityType.CUSTOMER: "#6366f1",
EntityType.MERCHANT: "#f59e0b",
EntityType.DEVICE: "#14b8a6",
EntityType.CARD: "#ec4899",
EntityType.EMAIL: "#8b5cf6",
EntityType.PHONE: "#06b6d4",
EntityType.IP_ADDRESS: "#f43f5e",
}
_EDGE_STYLES: dict[str, dict] = {
RelationshipType.OWNS: {"stroke": "#6366f1", "strokeWidth": 2},
RelationshipType.USES: {"stroke": "#14b8a6", "strokeWidth": 1.5},
RelationshipType.TRANSACTS_WITH: {"stroke": "#f59e0b", "strokeWidth": 2},
RelationshipType.SHARES_DEVICE: {"stroke": "#ec4899", "strokeDasharray": "5,5"},
RelationshipType.SHARES_IP: {"stroke": "#f43f5e", "strokeDasharray": "5,5"},
RelationshipType.LINKED_ALERT: {"stroke": "#ef4444", "strokeWidth": 2.5},
RelationshipType.SAME_ENTITY: {"stroke": "#a855f7", "strokeWidth": 3},
}
RISK_LEVEL_TO_SCORE: dict[str, float] = {
RiskLevel.CRITICAL.value: 1.0,
RiskLevel.HIGH.value: 0.8,
RiskLevel.MEDIUM.value: 0.5,
RiskLevel.LOW.value: 0.2,
RiskLevel.MINIMAL.value: 0.1,
}
_MUTATING_CYPHER_RE = re.compile(
r"\b(CREATE|MERGE|DELETE|SET|REMOVE|DROP|DETACH)\b",
re.IGNORECASE,
)
def _neo4j_node_to_entity(node) -> Entity:
import json
from datetime import datetime
props = dict(node)
if isinstance(props.get("first_seen"), str):
props["first_seen"] = datetime.fromisoformat(props["first_seen"])
if isinstance(props.get("last_seen"), str):
props["last_seen"] = datetime.fromisoformat(props["last_seen"])
props["entity_type"] = EntityType(props["entity_type"])
props["risk_level"] = RiskLevel(props["risk_level"])
if isinstance(props.get("attributes"), str):
props["attributes"] = json.loads(props["attributes"])
return Entity(**props)
def _neo4j_rel_to_relationship(rel) -> Relationship:
from datetime import datetime
props = dict(rel)
if isinstance(props.get("created_at"), str):
props["created_at"] = datetime.fromisoformat(props["created_at"])
props["relationship_type"] = RelationshipType(props["relationship_type"])
return Relationship(**props)
class GraphEngine:
"""Builds and queries the entity relationship graph.
The graph is backed by a graph database (Neo4j, Memgraph, etc.)
if enabled, or falls back to Redis / in-memory adjacency structures.
"""
def __init__(self) -> None:
self._entities = RedisStore("entity")
self._relationships = RedisStore("relationship")
self._adjacency: defaultdict[str, set[tuple[str, str]]] = defaultdict(set)
self._lock = threading.RLock()
settings = get_settings()
self.db_type = settings.graph_db_type
self.driver = None
if self.db_type in ("neo4j", "memgraph"):
try:
import importlib
neo4j_mod = importlib.import_module("neo4j")
self.driver = neo4j_mod.GraphDatabase.driver(
settings.neo4j_uri, auth=(settings.neo4j_user, settings.neo4j_password)
)
logger.info("Connected to graph database of type: %s", self.db_type)
except Exception as e:
logger.error(
"Failed to connect to graph database of type %s: %s. Falling back to redis/in-memory.",
self.db_type,
e,
)
self.db_type = "redis"
def _build_adjacency_list(self) -> None:
self._adjacency = defaultdict(set)
raw_relationships = self._relationships.list_values()
for r_dict in raw_relationships:
r = _dict_to_relationship(r_dict)
self._adjacency[r.source_entity_id].add((r.target_entity_id, r.id))
self._adjacency[r.target_entity_id].add((r.source_entity_id, r.id))
def register_entity(self, entity: Entity) -> None:
"""Register an entity in the graph."""
with self._lock:
self._entities.set(entity.id, _entity_to_dict(entity))
if self.db_type in ("neo4j", "memgraph") and self.driver:
import json
with self.driver.session() as session:
session.run(
"MERGE (e:Entity {id: $id}) "
"SET e.entity_type = $entity_type, "
" e.privacy_id = $privacy_id, "
" e.bank_id = $bank_id, "
" e.display_label = $display_label, "
" e.risk_level = $risk_level, "
" e.alert_count = $alert_count, "
" e.first_seen = $first_seen, "
" e.last_seen = $last_seen, "
" e.attributes = $attributes",
id=entity.id,
entity_type=entity.entity_type.value,
privacy_id=entity.privacy_id,
bank_id=entity.bank_id,
display_label=entity.display_label,
risk_level=entity.risk_level.value,
alert_count=entity.alert_count,
first_seen=entity.first_seen.isoformat(),
last_seen=entity.last_seen.isoformat(),
attributes=json.dumps(entity.attributes),
)
def register_entities(self, entities: list[Entity]) -> None:
with self._lock:
for entity in entities:
self.register_entity(entity)
def add_relationship(self, relationship: Relationship) -> None:
"""Add a relationship (edge) to the graph."""
with self._lock:
self._relationships.set(relationship.id, _relationship_to_dict(relationship))
# Incrementally update in-memory adjacency
self._adjacency[relationship.source_entity_id].add(
(relationship.target_entity_id, relationship.id)
)
self._adjacency[relationship.target_entity_id].add(
(relationship.source_entity_id, relationship.id)
)
if self.db_type in ("neo4j", "memgraph") and self.driver:
rel_type = relationship.relationship_type.value
query = (
f"MATCH (s:Entity {{id: $source_id}}) "
f"MATCH (t:Entity {{id: $target_id}}) "
f"MERGE (s)-[r:{rel_type} {{id: $id}}]->(t) "
"SET r.relationship_type = $relationship_type, "
" r.confidence = $confidence, "
" r.evidence = $evidence, "
" r.created_at = $created_at, "
" r.source_entity_id = $source_id, "
" r.target_entity_id = $target_id"
)
with self.driver.session() as session:
session.run(
query,
id=relationship.id,
source_id=relationship.source_entity_id,
target_id=relationship.target_entity_id,
relationship_type=relationship.relationship_type.value,
confidence=relationship.confidence,
evidence=relationship.evidence,
created_at=relationship.created_at.isoformat(),
)
def find_neighbors(
self,
entity_id: str,
depth: int = 1,
relationship_types: set[RelationshipType] | None = None,
as_of: datetime | None = None,
) -> list[Entity]:
"""BFS traversal to find neighbors up to a given depth.
Args:
entity_id: Root entity ID to find neighbors for.
depth: Maximum traversal depth hops.
relationship_types: Optional set of allowed RelationshipTypes.
as_of: Optional temporal cutoff. Relationships created after as_of are excluded.
"""
with self._lock:
if self.db_type in ("neo4j", "memgraph") and self.driver:
rel_filter = ""
if relationship_types:
rel_types_str = "|".join(rt.value for rt in relationship_types)
rel_filter = f":{rel_types_str}"
# Exclude self-node (n.id <> $entity_id) to prevent cyclic return of root entity
query = (
f"MATCH (s:Entity {{id: $entity_id}})-[r{rel_filter}*1..{depth}]-(n:Entity) "
"WHERE n.id <> $entity_id "
)
params: dict[str, Any] = {"entity_id": entity_id}
if as_of is not None:
query += " AND all(rel IN relationships(r) WHERE rel.created_at <= $as_of) "
params["as_of"] = as_of.isoformat()
query += "RETURN DISTINCT n"
neighbors = []
with self.driver.session() as session:
result = session.run(query, **params)
for record in result:
neighbors.append(_neo4j_node_to_entity(record["n"]))
return neighbors
self._build_adjacency_list()
visited: set[str] = {entity_id}
queue: deque[tuple[str, int]] = deque([(entity_id, 0)])
neighbors = []
while queue:
current_id, current_depth = queue.popleft()
if current_depth >= depth:
continue
for neighbor_id, rel_id in self._adjacency.get(current_id, set()):
if neighbor_id in visited:
continue
rel_val = self._relationships.get(rel_id)
rel = _dict_to_relationship(rel_val) if rel_val else None
if not rel:
continue
# Filter by temporal boundary if specified
if as_of is not None:
rel_time = rel.created_at
if rel_time.tzinfo is None:
rel_time = rel_time.replace(tzinfo=UTC)
cutoff = as_of if as_of.tzinfo is not None else as_of.replace(tzinfo=UTC)
if rel_time > cutoff:
continue
# Filter by relationship type if specified
if relationship_types and rel.relationship_type not in relationship_types:
continue
visited.add(neighbor_id)
entity_val = self._entities.get(neighbor_id)
if entity_val:
entity = _dict_to_entity(entity_val)
neighbors.append(entity)
queue.append((neighbor_id, current_depth + 1))
return neighbors
def detect_clusters(self, min_size: int = 3, as_of: datetime | None = None) -> list[list[str]]:
"""Find connected components (clusters) of at least min_size."""
with self._lock:
if self.db_type in ("neo4j", "memgraph") and self.driver:
adjacency = defaultdict(set)
all_nodes = set()
query = (
"MATCH (n:Entity) "
"OPTIONAL MATCH (n)-[r]->(m:Entity) "
)
params: dict[str, Any] = {}
if as_of is not None:
query += "WHERE r IS NULL OR r.created_at <= $as_of "
params["as_of"] = as_of.isoformat()
query += "RETURN n.id as source, m.id as target"
with self.driver.session() as session:
result = session.run(query, **params)
for record in result:
s = record["source"]
t = record["target"]
all_nodes.add(s)
if t:
all_nodes.add(t)
adjacency[s].add(t)
adjacency[t].add(s)
visited = set()
clusters = []
for node_id in all_nodes:
if node_id in visited:
continue
component = []
queue = deque([node_id])
while queue:
curr = queue.popleft()
if curr in visited:
continue
visited.add(curr)
component.append(curr)
for neighbor in adjacency.get(curr, set()):
if neighbor not in visited:
queue.append(neighbor)
if len(component) >= min_size:
clusters.append(component)
clusters.sort(key=len, reverse=True)
logger.info("Found %d clusters (min_size=%d) in graph DB", len(clusters), min_size)
return clusters
self._build_adjacency_list()
fallback_visited: set[str] = set()
fallback_clusters: list[list[str]] = []
# If as_of is specified, construct a temporally filtered adjacency view
if as_of is not None:
adj: defaultdict[str, set[str]] = defaultdict(set)
raw_rels = [_dict_to_relationship(v) for v in self._relationships.list_values()]
for rel in raw_rels:
rel_time = rel.created_at
if rel_time.tzinfo is None:
rel_time = rel_time.replace(tzinfo=UTC)
cutoff = as_of if as_of.tzinfo is not None else as_of.replace(tzinfo=UTC)
if rel_time <= cutoff:
adj[rel.source_entity_id].add(rel.target_entity_id)
adj[rel.target_entity_id].add(rel.source_entity_id)
else:
adj = defaultdict(set)
for u, nbrs in self._adjacency.items():
for v, _ in nbrs:
adj[u].add(v)
raw_entities = [_dict_to_entity(v) for v in self._entities.list_values()]
for entity in raw_entities:
entity_id = entity.id
if entity_id in fallback_visited:
continue
fallback_component: list[str] = []
fallback_queue: deque[str] = deque([entity_id])
while fallback_queue:
current = fallback_queue.popleft()
if current in fallback_visited:
continue
fallback_visited.add(current)
fallback_component.append(current)
for neighbor_id in adj.get(current, set()):
if neighbor_id not in fallback_visited:
fallback_queue.append(neighbor_id)
if len(fallback_component) >= min_size:
fallback_clusters.append(fallback_component)
fallback_clusters.sort(key=len, reverse=True)
logger.info("Found %d clusters (min_size=%d)", len(fallback_clusters), min_size)
return fallback_clusters
def get_subgraph(
self,
center_entity_id: str,
radius: int = 2,
max_nodes: int = 100,
as_of: datetime | None = None,
) -> GraphSubgraph:
"""Extract a subgraph centered on an entity with bounded node budget."""
with self._lock:
if self.db_type in ("neo4j", "memgraph") and self.driver:
nodes_dict = {}
rels_dict = {}
query = (
"MATCH (s:Entity {id: $center_id}) "
"OPTIONAL MATCH p = (s)-[r*1..$radius]-(n:Entity) "
)
params: dict[str, Any] = {
"center_id": center_entity_id,
"radius": radius,
"max_nodes": max_nodes,
}
if as_of is not None:
query += "WHERE all(rel IN relationships(p) WHERE rel.created_at <= $as_of) "
params["as_of"] = as_of.isoformat()
query += "RETURN s, collect(p)[..$max_nodes] as paths"
with self.driver.session() as session:
result = session.run(query, **params)
record = result.single()
if not record:
return GraphSubgraph(center_entity_id=center_entity_id, depth=radius)
s_node = record["s"]
s_entity = _neo4j_node_to_entity(s_node)
nodes_dict[s_entity.id] = s_entity
paths = record["paths"] or []
for path in paths:
for node in path.nodes:
entity = _neo4j_node_to_entity(node)
nodes_dict[entity.id] = entity
for rel in path.relationships:
relationship = _neo4j_rel_to_relationship(rel)
rels_dict[relationship.id] = relationship
node_ids = list(nodes_dict.keys())[:max_nodes]
# Build React Flow nodes
nodes = []
for i, nid in enumerate(node_ids):
entity = nodes_dict[nid]
import math
if nid == center_entity_id:
x: float = 400.0
y: float = 300.0
else:
angle = 2 * math.pi * (i - 1) / max(1, len(node_ids) - 1)
layer = 1
for depth_check in range(radius):
if i > len(node_ids) * (depth_check + 1) / (radius + 1):
layer = depth_check + 2
r = 150 * layer
x = 400 + r * math.cos(angle)
y = 300 + r * math.sin(angle)
color = _NODE_COLORS.get(entity.entity_type, "#6366f1")
is_high_risk = entity.risk_level in (RiskLevel.HIGH, RiskLevel.CRITICAL)
nodes.append(
{
"id": nid,
"type": "default",
"position": {"x": round(x), "y": round(y)},
"data": {
"label": entity.display_label,
"entityType": entity.entity_type.value,
"bankId": entity.bank_id,
"riskLevel": entity.risk_level.value,
"alertCount": entity.alert_count,
"isCenter": nid == center_entity_id,
},
"style": {
"background": color if not is_high_risk else "#ef4444",
"color": "#ffffff",
"border": f"2px solid {'#ef4444' if is_high_risk else color}",
"borderRadius": "8px",
"padding": "8px 12px",
"fontSize": "11px",
"fontWeight": "600",
},
}
)
# Build React Flow edges
edges = []
node_id_set = set(node_ids)
for rel in rels_dict.values():
if rel.source_entity_id in node_id_set and rel.target_entity_id in node_id_set:
style = _EDGE_STYLES.get(rel.relationship_type, {})
edges.append(
{
"id": rel.id,
"source": rel.source_entity_id,
"target": rel.target_entity_id,
"label": rel.relationship_type.value.replace("_", " "),
"type": "smoothstep",
"animated": rel.relationship_type == RelationshipType.LINKED_ALERT,
"style": style,
"data": {
"confidence": rel.confidence,
"relationshipType": rel.relationship_type.value,
},
}
)
# Detect clusters within subgraph using local adjacency
sub_adj: defaultdict[str, set[tuple[str, str]]] = defaultdict(set)
for rel in rels_dict.values():
sub_adj[rel.source_entity_id].add((rel.target_entity_id, rel.id))
sub_adj[rel.target_entity_id].add((rel.source_entity_id, rel.id))
clusters = self._detect_subgraph_clusters(node_id_set, adjacency=sub_adj)
return GraphSubgraph(
nodes=nodes,
edges=edges,
clusters=clusters,
center_entity_id=center_entity_id,
depth=radius,
)
self._build_adjacency_list()
center_entity_val = self._entities.get(center_entity_id)
if not center_entity_val:
return GraphSubgraph(center_entity_id=center_entity_id, depth=radius)
# Collect nodes via BFS bounded by max_nodes and as_of cutoff
fb_visited: set[str] = {center_entity_id}
fb_queue: deque[tuple[str, int]] = deque([(center_entity_id, 0)])
fb_node_ids: list[str] = [center_entity_id]
bounded_max_nodes = max(1, min(max_nodes, 200))
while fb_queue and len(fb_node_ids) < bounded_max_nodes:
current_id, current_depth = fb_queue.popleft()
if current_depth >= radius:
continue
for neighbor_id, rel_id in self._adjacency.get(current_id, set()):
if as_of is not None:
rel_val = self._relationships.get(rel_id)
if rel_val:
rel_obj = _dict_to_relationship(rel_val)
rel_time = rel_obj.created_at
if rel_time.tzinfo is None:
rel_time = rel_time.replace(tzinfo=UTC)
cutoff = as_of if as_of.tzinfo is not None else as_of.replace(tzinfo=UTC)
if rel_time > cutoff:
continue
if neighbor_id not in fb_visited:
fb_visited.add(neighbor_id)
fb_node_ids.append(neighbor_id)
fb_queue.append((neighbor_id, current_depth + 1))
if len(fb_node_ids) >= bounded_max_nodes:
break
# Build React Flow nodes
nodes = []
for i, nid in enumerate(fb_node_ids):
entity_val = self._entities.get(nid)
if not entity_val:
continue
entity = _dict_to_entity(entity_val)
import math
if nid == center_entity_id:
x = 400.0
y = 300.0
else:
angle = 2 * math.pi * (i - 1) / max(1, len(fb_node_ids) - 1)
layer = 1
for depth_check in range(radius):
if i > len(fb_node_ids) * (depth_check + 1) / (radius + 1):
layer = depth_check + 2
r = 150 * layer
x = 400 + r * math.cos(angle)
y = 300 + r * math.sin(angle)
color = _NODE_COLORS.get(entity.entity_type, "#6366f1")
is_high_risk = entity.risk_level in (RiskLevel.HIGH, RiskLevel.CRITICAL)
nodes.append(
{
"id": nid,
"type": "default",
"position": {"x": round(x), "y": round(y)},
"data": {
"label": entity.display_label,
"entityType": entity.entity_type.value,
"bankId": entity.bank_id,
"riskLevel": entity.risk_level.value,
"alertCount": entity.alert_count,
"isCenter": nid == center_entity_id,
},
"style": {
"background": color if not is_high_risk else "#ef4444",
"color": "#ffffff",
"border": f"2px solid {'#ef4444' if is_high_risk else color}",
"borderRadius": "8px",
"padding": "8px 12px",
"fontSize": "11px",
"fontWeight": "600",
},
}
)
# Build React Flow edges
edges = []
node_id_set = set(fb_node_ids)
raw_relationships = [_dict_to_relationship(v) for v in self._relationships.list_values()]
sub_adj_fb: defaultdict[str, set[tuple[str, str]]] = defaultdict(set)
for rel in raw_relationships:
if as_of is not None:
rel_time = rel.created_at
if rel_time.tzinfo is None:
rel_time = rel_time.replace(tzinfo=UTC)
cutoff = as_of if as_of.tzinfo is not None else as_of.replace(tzinfo=UTC)
if rel_time > cutoff:
continue
if rel.source_entity_id in node_id_set and rel.target_entity_id in node_id_set:
sub_adj_fb[rel.source_entity_id].add((rel.target_entity_id, rel.id))
sub_adj_fb[rel.target_entity_id].add((rel.source_entity_id, rel.id))
style = _EDGE_STYLES.get(rel.relationship_type, {})
edges.append(
{
"id": rel.id,
"source": rel.source_entity_id,
"target": rel.target_entity_id,
"label": rel.relationship_type.value.replace("_", " "),
"type": "smoothstep",
"animated": rel.relationship_type == RelationshipType.LINKED_ALERT,
"style": style,
"data": {
"confidence": rel.confidence,
"relationshipType": rel.relationship_type.value,
},
}
)
# Detect clusters within subgraph using local adjacency
clusters = self._detect_subgraph_clusters(node_id_set, adjacency=sub_adj_fb)
return GraphSubgraph(
nodes=nodes,
edges=edges,
clusters=clusters,
center_entity_id=center_entity_id,
depth=radius,
)
def search_nodes(
self,
query: str,
entity_type: EntityType | None = None,
limit: int = 20,
) -> list[Entity]:
"""Search entities by display label or privacy ID prefix."""
if self.db_type in ("neo4j", "memgraph") and self.driver:
et_str = entity_type.value if entity_type else None
query_lower = query.lower()
results = []
with self.driver.session() as session:
res = session.run(
"MATCH (n:Entity) "
"WHERE ($entity_type IS NULL OR n.entity_type = $entity_type) "
" AND (toLower(n.display_label) CONTAINS $query OR toLower(n.privacy_id) CONTAINS $query) "
"RETURN n LIMIT $limit",
entity_type=et_str,
query=query_lower,
limit=limit,
)
for r in res:
results.append(_neo4j_node_to_entity(r["n"]))
return results
query_lower = query.lower()
results = []
raw_entities = [_dict_to_entity(v) for v in self._entities.list_values()]
for entity in raw_entities:
if entity_type and entity.entity_type != entity_type:
continue
if (
query_lower in entity.display_label.lower()
or query_lower in entity.privacy_id.lower()
):
results.append(entity)
if len(results) >= limit:
break
return results
@property
def node_count(self) -> int:
if self.db_type in ("neo4j", "memgraph") and self.driver:
with self.driver.session() as session:
res = session.run("MATCH (n:Entity) RETURN count(n) as count")
return res.single()["count"]
return len(self._entities.list_values())
@property
def edge_count(self) -> int:
if self.db_type in ("neo4j", "memgraph") and self.driver:
with self.driver.session() as session:
res = session.run("MATCH ()-[r]->() RETURN count(r) as count")
return res.single()["count"]
return len(self._relationships.list_values())
def get_stats(self) -> dict:
"""Graph statistics for the dashboard."""
if self.db_type in ("neo4j", "memgraph") and self.driver:
with self.driver.session() as session:
res_type = session.run(
"MATCH (n:Entity) RETURN n.entity_type as type, count(n) as count"
)
type_counts = {r["type"]: r["count"] for r in res_type}
res_risk = session.run(
"MATCH (n:Entity) RETURN n.risk_level as risk, count(n) as count"
)
risk_counts = {r["risk"]: r["count"] for r in res_risk}
total_nodes = sum(type_counts.values())
total_edges = self.edge_count
cluster_count = len(self.detect_clusters(min_size=3))
return {
"total_nodes": total_nodes,
"total_edges": total_edges,
"nodes_by_type": type_counts,
"nodes_by_risk": risk_counts,
"cluster_count": cluster_count,
"database_backend": self.db_type.capitalize(),
}
fb_type_counts: dict[str, int] = defaultdict(int)
fb_risk_counts: dict[str, int] = defaultdict(int)
raw_entities = [_dict_to_entity(v) for v in self._entities.list_values()]
for entity in raw_entities:
fb_type_counts[entity.entity_type.value] += 1
fb_risk_counts[entity.risk_level.value] += 1
return {
"total_nodes": self.node_count,
"total_edges": self.edge_count,
"nodes_by_type": dict(fb_type_counts),
"nodes_by_risk": dict(fb_risk_counts),
"cluster_count": len(self.detect_clusters(min_size=3)),
"database_backend": "Redis (in-memory)",
}
def detect_cyclic_mule_rings(
self,
min_length: int = 3,
max_length: int = 7,
bank_id: str | None = None,
as_of: datetime | None = None,
) -> list[dict[str, Any]]:
"""Detect closed transaction cycles (mule rings) with L in [min_length, max_length].
Applies canonical rotation deduplication: every cycle C is rotated to start
with its lexicographically minimum node ID. A deterministic SHA-256 ring ID is assigned.
"""
with self._lock:
# 1. If Neo4j driver is active, query Neo4j
if self.db_type in ("neo4j", "memgraph") and self.driver:
try:
query = (
f"MATCH path = (start:Entity)-[r*{min_length}..{max_length}]->(start) "
)
params: dict[str, Any] = {}
if as_of is not None:
query += "WHERE all(rel IN relationships(path) WHERE rel.created_at <= $as_of) "
params["as_of"] = as_of.isoformat()
query += (
"RETURN [n IN nodes(path) | n.id] AS cycle_ids, "
" [n IN nodes(path) | n.bank_id] AS cycle_banks, "
" [n IN nodes(path) | n.risk_level] AS cycle_risks, "
" [rel IN relationships(path) | rel.evidence] AS cycle_evidences "
"LIMIT 200"
)
with self.driver.session() as session:
result = session.run(query, **params)
raw_rings = []
seen_ring_ids: set[str] = set()
for record in result:
node_ids = record["cycle_ids"]
if len(node_ids) > 1 and node_ids[0] == node_ids[-1]:
node_ids = node_ids[:-1]
if len(node_ids) < min_length or len(node_ids) > max_length:
continue
min_idx = node_ids.index(min(node_ids))
canon_nodes = node_ids[min_idx:] + node_ids[:min_idx]
ring_id = hashlib.sha256(":".join(canon_nodes).encode("utf-8")).hexdigest()[:16]
if ring_id in seen_ring_ids:
continue
seen_ring_ids.add(ring_id)
banks = [b for b in record["cycle_banks"] if b]
banks_involved = sorted(list(set(banks)))
if bank_id and bank_id not in banks_involved:
continue
is_cross_bank = len(banks_involved) >= 2
total_vol = 0.0
for ev in record["cycle_evidences"]:
if isinstance(ev, dict):
total_vol += float(ev.get("amount") or ev.get("volume") or 0.0)
risk_scores = [RISK_LEVEL_TO_SCORE.get(r, 0.5) for r in record["cycle_risks"]]
base_risk = sum(risk_scores) / max(len(risk_scores), 1)
cross_boost = 0.2 if is_cross_bank else 0.0
len_factor = 0.1 if len(canon_nodes) <= 4 else 0.05
risk_score = round(min(1.0, max(0.1, base_risk + cross_boost + len_factor)), 4)
raw_rings.append(
{
"ring_id": ring_id,
"length": len(canon_nodes),
"entity_ids": canon_nodes,
"banks_involved": banks_involved,
"is_cross_bank": is_cross_bank,
"risk_score": risk_score,
"total_volume": round(total_vol, 2),
"detected_at": datetime.now(UTC),
}
)
return raw_rings
except Exception as e:
logger.warning("Neo4j ring detection query failed (%s), falling back to in-memory DFS", e)
# In-memory directed DFS cycle finder
rel_dicts = self._relationships.list_values()
directed_adj: defaultdict[str, list[tuple[str, float]]] = defaultdict(list)
cutoff_dt = as_of if as_of is not None else None
if cutoff_dt is not None and cutoff_dt.tzinfo is None:
cutoff_dt = cutoff_dt.replace(tzinfo=UTC)
for r_val in rel_dicts:
r = _dict_to_relationship(r_val)
if cutoff_dt is not None:
r_time = r.created_at
if r_time.tzinfo is None:
r_time = r_time.replace(tzinfo=UTC)
if r_time > cutoff_dt:
continue
u = r.source_entity_id
v = r.target_entity_id
vol = 0.0
if isinstance(r.evidence, dict):
vol = float(r.evidence.get("amount") or r.evidence.get("volume") or 0.0)
directed_adj[u].append((v, vol))
all_nodes = sorted(
list(
set(self._entities.list_keys())
| set(directed_adj.keys())
| {target for targets in directed_adj.values() for target, _ in targets}
)
)
mem_seen_ring_ids: set[str] = set()
rings: list[dict[str, Any]] = []
for s in all_nodes:
stack: list[tuple[str, list[str], float]] = [(s, [s], 0.0)]
while stack:
curr, path, path_vol = stack.pop()
if len(path) > max_length:
continue
for nxt, vol in directed_adj.get(curr, []):
if nxt == s:
# Closed cycle back to start node
if len(path) >= min_length:
ring_id = hashlib.sha256(":".join(path).encode("utf-8")).hexdigest()[:16]
if ring_id not in mem_seen_ring_ids:
mem_seen_ring_ids.add(ring_id)
node_banks: list[str] = []
node_risk_scores: list[float] = []
for nid in path:
ent_val = self._entities.get(nid)
if ent_val:
ent = _dict_to_entity(ent_val)
if ent.bank_id:
node_banks.append(ent.bank_id)
node_risk_scores.append(RISK_LEVEL_TO_SCORE.get(ent.risk_level.value, 0.5))
else:
node_risk_scores.append(0.5)
banks_involved = sorted(list(set(node_banks)))
if bank_id and bank_id not in banks_involved:
continue
is_cross_bank = len(banks_involved) >= 2
base_risk = sum(node_risk_scores) / max(len(node_risk_scores), 1)
cross_boost = 0.2 if is_cross_bank else 0.0
len_factor = 0.1 if len(path) <= 4 else 0.05
risk_score = round(min(1.0, max(0.1, base_risk + cross_boost + len_factor)), 4)
rings.append(
{
"ring_id": ring_id,
"length": len(path),
"entity_ids": list(path),
"banks_involved": banks_involved,
"is_cross_bank": is_cross_bank,
"risk_score": risk_score,
"total_volume": round(path_vol + vol, 2),
"detected_at": datetime.now(UTC),
}
)
elif nxt > s and nxt not in path:
stack.append((nxt, path + [nxt], path_vol + vol))
return sorted(rings, key=lambda x: x["risk_score"], reverse=True)
def detect_smurfing_patterns(
self,
window_hours: int = 24,
min_fan: int = 3,
max_depth: int = 3,
bank_id: str | None = None,
as_of: datetime | None = None,
) -> list[dict[str, Any]]:
"""Detect multi-hop financial smurfing patterns (fan-in, fan-out, layering)."""
with self._lock:
now = as_of if as_of is not None else datetime.now(UTC)
if now.tzinfo is None:
now = now.replace(tzinfo=UTC)
cutoff_time = now - timedelta(hours=window_hours)
rel_dicts = self._relationships.list_values()
in_edges: defaultdict[str, list[tuple[str, float]]] = defaultdict(list)
out_edges: defaultdict[str, list[tuple[str, float]]] = defaultdict(list)
for r_val in rel_dicts:
r = _dict_to_relationship(r_val)
r_time = r.created_at
if r_time.tzinfo is None:
r_time = r_time.replace(tzinfo=UTC)
if r_time < cutoff_time or r_time > now:
continue
u = r.source_entity_id
v = r.target_entity_id
vol = 0.0
if isinstance(r.evidence, dict):
vol = float(r.evidence.get("amount") or r.evidence.get("volume") or 0.0)
out_edges[u].append((v, vol))
in_edges[v].append((u, vol))
patterns: list[dict[str, Any]] = []
seen_pattern_ids: set[str] = set()
all_node_ids = set(self._entities.list_keys()) | set(in_edges.keys()) | set(out_edges.keys())
for node_id in sorted(all_node_ids):
node_val = self._entities.get(node_id)
node_ent = _dict_to_entity(node_val) if node_val else None
node_risk_val = node_ent.risk_level.value if node_ent else RiskLevel.MEDIUM.value
base_risk = RISK_LEVEL_TO_SCORE.get(node_risk_val, 0.5)
# 1. Fan-In Check (Aggregation Mules)
in_sources = list({src for src, _ in in_edges.get(node_id, []) if src != node_id})
if len(in_sources) >= min_fan:
spokes = sorted(in_sources)
pattern_key = f"fan_in:{node_id}:{':'.join(spokes)}"
pattern_id = hashlib.sha256(pattern_key.encode("utf-8")).hexdigest()[:16]
if pattern_id not in seen_pattern_ids:
seen_pattern_ids.add(pattern_id)
banks = set()
if node_ent and node_ent.bank_id:
banks.add(node_ent.bank_id)
for spk in spokes:
spk_val = self._entities.get(spk)
if spk_val:
spk_ent = _dict_to_entity(spk_val)
if spk_ent.bank_id:
banks.add(spk_ent.bank_id)
banks_involved = sorted(list(banks))
if not bank_id or bank_id in banks_involved:
is_cross_bank = len(banks_involved) >= 2
total_vol = sum(v for src, v in in_edges[node_id] if src in spokes)
fan_degree = len(spokes)
risk_score = round(
min(
1.0,
base_risk
+ min(0.3, (fan_degree - min_fan + 1) * 0.05)
+ (0.15 if is_cross_bank else 0.0),
),
4,
)
patterns.append(
{
"pattern_id": pattern_id,
"pattern_type": "fan_in",
"hub_entity_id": node_id,
"spoke_entity_ids": spokes,
"fan_degree": fan_degree,
"total_volume": round(
total_vol if total_vol > 0 else float(fan_degree * 1000.0), 2
),
"depth": 1,
"banks_involved": banks_involved,
"is_cross_bank": is_cross_bank,
"risk_score": risk_score,
"detected_at": datetime.now(UTC),
}
)
# 2. Fan-Out Check (Dispersion Mules)
out_targets = list({tgt for tgt, _ in out_edges.get(node_id, []) if tgt != node_id})
if len(out_targets) >= min_fan:
spokes = sorted(out_targets)
pattern_key = f"fan_out:{node_id}:{':'.join(spokes)}"
pattern_id = hashlib.sha256(pattern_key.encode("utf-8")).hexdigest()[:16]
if pattern_id not in seen_pattern_ids:
seen_pattern_ids.add(pattern_id)
banks = set()
if node_ent and node_ent.bank_id:
banks.add(node_ent.bank_id)
for spk in spokes:
spk_val = self._entities.get(spk)
if spk_val:
spk_ent = _dict_to_entity(spk_val)
if spk_ent.bank_id:
banks.add(spk_ent.bank_id)
banks_involved = sorted(list(banks))
if not bank_id or bank_id in banks_involved:
is_cross_bank = len(banks_involved) >= 2
total_vol = sum(v for tgt, v in out_edges[node_id] if tgt in spokes)
fan_degree = len(spokes)
risk_score = round(
min(
1.0,
base_risk
+ min(0.3, (fan_degree - min_fan + 1) * 0.05)
+ (0.15 if is_cross_bank else 0.0),
),
4,
)
patterns.append(
{
"pattern_id": pattern_id,
"pattern_type": "fan_out",
"hub_entity_id": node_id,
"spoke_entity_ids": spokes,
"fan_degree": fan_degree,
"total_volume": round(
total_vol if total_vol > 0 else float(fan_degree * 1000.0), 2
),
"depth": 1,
"banks_involved": banks_involved,
"is_cross_bank": is_cross_bank,
"risk_score": risk_score,
"detected_at": datetime.now(UTC),
}
)
# 3. Multi-hop Layering Check (Transit Mule Hub with In-Flow and Out-Flow)
if len(in_sources) >= 2 and len(out_targets) >= 2:
all_spokes = sorted(list(set(in_sources) | set(out_targets)))
pattern_key = f"layering:{node_id}:{':'.join(all_spokes)}"
pattern_id = hashlib.sha256(pattern_key.encode("utf-8")).hexdigest()[:16]
if pattern_id not in seen_pattern_ids:
seen_pattern_ids.add(pattern_id)
banks = set()
if node_ent and node_ent.bank_id:
banks.add(node_ent.bank_id)
for spk in all_spokes:
spk_val = self._entities.get(spk)
if spk_val:
spk_ent = _dict_to_entity(spk_val)
if spk_ent.bank_id:
banks.add(spk_ent.bank_id)
banks_involved = sorted(list(banks))
if not bank_id or bank_id in banks_involved:
is_cross_bank = len(banks_involved) >= 2
fan_degree = len(in_sources) + len(out_targets)
in_vol = sum(v for src, v in in_edges[node_id] if src in in_sources)
out_vol = sum(v for tgt, v in out_edges[node_id] if tgt in out_targets)
total_vol = in_vol + out_vol
risk_score = round(min(1.0, base_risk + 0.2 + (0.15 if is_cross_bank else 0.0)), 4)
patterns.append(
{
"pattern_id": pattern_id,
"pattern_type": "multi_hop_layering",
"hub_entity_id": node_id,
"spoke_entity_ids": all_spokes,
"fan_degree": fan_degree,
"total_volume": round(
total_vol if total_vol > 0 else float(fan_degree * 1000.0), 2
),
"depth": 2,
"banks_involved": banks_involved,
"is_cross_bank": is_cross_bank,
"risk_score": risk_score,
"detected_at": datetime.now(UTC),
}
)
return sorted(patterns, key=lambda x: x["risk_score"], reverse=True)
def execute_cypher(
self,
query: str,
params: dict[str, Any] | None = None,
read_only: bool = True,
) -> list[dict[str, Any]]:
"""Safely execute a parameterized Cypher query.
If read_only=True, rejects any query containing mutating keywords
(CREATE, MERGE, DELETE, SET, REMOVE, DROP, DETACH).
"""
if read_only:
match = _MUTATING_CYPHER_RE.search(query)
if match:
raise ValueError(
f"Cypher execution rejected: query contains mutating keyword '{match.group(0)}' while read_only=True"
)
with self._lock:
if self.db_type in ("neo4j", "memgraph") and self.driver:
with self.driver.session() as session:
result = session.run(query, **(params or {}))
return [record.data() for record in result]
# In-memory Redis fallback emulation
logger.info("Executing Cypher query in in-memory fallback mode: %s", query[:80])
upper_q = query.upper()
if "RETURN N.ENTITY_TYPE" in upper_q or "COUNT(N)" in upper_q:
counts: dict[str, int] = defaultdict(int)
for val in self._entities.list_values():
e = _dict_to_entity(val)
counts[e.entity_type.value] += 1
return [{"type": k, "count": v} for k, v in counts.items()]
if "MATCH (N:ENTITY)" in upper_q and "RETURN N" in upper_q:
return [{"n": _entity_to_dict(_dict_to_entity(v))} for v in self._entities.list_values()]
return []
# ── Private helpers ────────────────────────
def _detect_subgraph_clusters(
self,
node_ids: set[str],
adjacency: dict[str, set[tuple[str, str]]] | None = None,
) -> list[list[str]]:
"""Find clusters within a subset of nodes."""
visited: set[str] = set()
clusters: list[list[str]] = []
adj = adjacency if adjacency is not None else self._adjacency
for nid in node_ids:
if nid in visited:
continue
component: list[str] = []
queue: deque[str] = deque([nid])
while queue:
current = queue.popleft()
if current in visited or current not in node_ids:
continue
visited.add(current)
component.append(current)
for neighbor_id, _ in adj.get(current, set()):
if neighbor_id not in visited and neighbor_id in node_ids:
queue.append(neighbor_id)
if len(component) >= 2:
clusters.append(component)
return clusters
# ── MinHash LSH Fuzzy Entity Resolution ──────────────────────
def find_fuzzy_matches(
self,
entity_id: str,
similarity_threshold: float = 0.40,
num_hashes: int = 64,
num_bands: int = 16,
rows_per_band: int = 4,
cross_bank_only: bool = True,
) -> list[dict[str, Any]]:
"""Finds candidate matching entities in the graph using MinHash LSH.
Extracts display labels and text attributes from the target entity,
computes MinHash signatures, and queries across all registered entities.
"""
from app.domain.minhash_lsh import (
compute_minhash_signature,
estimate_jaccard_similarity,
partition_into_lsh_bands,
)
with self._lock:
target_data = self._entities.get(entity_id)
if not target_data:
return []
target_ent = _dict_to_entity(target_data)
target_text = target_ent.display_label or target_ent.id
target_sig = compute_minhash_signature(target_text, num_hashes=num_hashes)
target_bands = set(
partition_into_lsh_bands(
target_sig, num_bands=num_bands, rows_per_band=rows_per_band
)
)
candidates: list[dict[str, Any]] = []
for val in self._entities.list_values():
cand_ent = _dict_to_entity(val)
if cand_ent.id == entity_id:
continue
if cross_bank_only and cand_ent.bank_id == target_ent.bank_id:
continue
cand_text = cand_ent.display_label or cand_ent.id
cand_sig = compute_minhash_signature(cand_text, num_hashes=num_hashes)
cand_bands = partition_into_lsh_bands(
cand_sig, num_bands=num_bands, rows_per_band=rows_per_band
)
collided_bands = sum(1 for b in cand_bands if b in target_bands)
if collided_bands > 0:
sim = estimate_jaccard_similarity(target_sig, cand_sig)
if sim >= similarity_threshold:
candidates.append(
{
"entity_id": cand_ent.id,
"bank_id": cand_ent.bank_id,
"display_label": cand_ent.display_label,
"entity_type": cand_ent.entity_type.value,
"estimated_jaccard": sim,
"collided_bands": collided_bands,
"total_bands": num_bands,
}
)
candidates.sort(key=lambda x: x["estimated_jaccard"], reverse=True)
return candidates
def link_fuzzy_entities(
self,
similarity_threshold: float = 0.50,
num_hashes: int = 64,
num_bands: int = 16,
rows_per_band: int = 4,
auto_add_relationship: bool = False,
) -> list[dict[str, Any]]:
"""Scans the graph for cross-institution entities with high fuzzy similarity.
Returns detected duplicate / mule candidate pairs. If auto_add_relationship=True,
creates RelationshipType.SAME_ENTITY edges between matched entities.
"""
from app.domain.minhash_lsh import (
compute_minhash_signature,
estimate_jaccard_similarity,
partition_into_lsh_bands,
)
with self._lock:
entities = [_dict_to_entity(v) for v in self._entities.list_values()]
if len(entities) < 2:
return []
# Index all entities by LSH bands
band_index: dict[str, list[tuple[Entity, list[int]]]] = defaultdict(list)
for ent in entities:
text = ent.display_label or ent.id
sig = compute_minhash_signature(text, num_hashes=num_hashes)
bands = partition_into_lsh_bands(
sig, num_bands=num_bands, rows_per_band=rows_per_band
)
for b in bands:
band_index[b].append((ent, sig))
# Candidate pair tracking
candidate_pairs: dict[
tuple[str, str], tuple[Entity, Entity, list[int], list[int], int]
] = {}
for entries in band_index.values():
for i in range(len(entries)):
for j in range(i + 1, len(entries)):
e1, s1 = entries[i]
e2, s2 = entries[j]
if e1.bank_id != e2.bank_id:
pair_key = (min(e1.id, e2.id), max(e1.id, e2.id))
if pair_key not in candidate_pairs:
candidate_pairs[pair_key] = (e1, e2, s1, s2, 1)
else:
prior = candidate_pairs[pair_key]
candidate_pairs[pair_key] = (
prior[0],
prior[1],
prior[2],
prior[3],
prior[4] + 1,
)
matches: list[dict[str, Any]] = []
for (id1, id2), (e1, e2, s1, s2, band_cnt) in candidate_pairs.items():
sim = estimate_jaccard_similarity(s1, s2)
if sim >= similarity_threshold:
match_info = {
"source_entity_id": id1,
"source_bank_id": e1.bank_id,
"target_entity_id": id2,
"target_bank_id": e2.bank_id,
"estimated_jaccard": sim,
"collided_bands": band_cnt,
"total_bands": num_bands,
}
matches.append(match_info)
if auto_add_relationship:
rel = Relationship(
id=f"fuzzy_same_{id1[:8]}_{id2[:8]}",
source_entity_id=id1,
target_entity_id=id2,
relationship_type=RelationshipType.SAME_ENTITY,
confidence=sim,
evidence=[
f"MinHash LSH fuzzy match J={sim:.4f} across {band_cnt}/{num_bands} bands"
],
)
self.add_relationship(rel)
matches.sort(key=lambda x: x["estimated_jaccard"], reverse=True)
return matches