from __future__ import annotations import os, copy import pandas as pd from pathlib import Path from typing import Any from collections import deque import requests import yaml from node_relation import extract_relationships, relationships_to_dataframe, enrich_relationships_with_properties from graph.DFS import build_graph, extract_clusters, cluster_with_relations_df, deduplicate, bfs_all_paths DEFAULT_SKIP = {"sample_id", "crdc_id", "comment", "uuid", "created", "updated"} class NoAliasDumper(yaml.SafeDumper): def ignore_aliases(self, data): return True def remove_yaml_anchors(data: Any) -> Any: return copy.deepcopy(data) def load_yaml_from_url(url: str) -> dict[str, Any]: resp = requests.get(url, timeout=30) resp.raise_for_status() data = yaml.safe_load(resp.text) if not isinstance(data, dict): raise ValueError(f"Expected YAML object from {url}, got {type(data).__name__}") return data def _get_nodes(node_model: dict[str, Any]) -> dict[str, Any]: nodes = node_model.get("Nodes") or node_model.get("nodes") or {} if not isinstance(nodes, dict): raise ValueError("Node model does not contain a valid Nodes mapping.") return nodes def _get_prop_defs(prop_model: dict[str, Any]) -> dict[str, Any]: props = prop_model.get("PropDefinitions") or prop_model.get("properties") or {} if not isinstance(props, dict): raise ValueError("Property model does not contain a valid PropDefinitions mapping.") return props def _normalize_node_spec(node_name: str, node_spec: dict[str, Any], prop_defs: dict[str, Any]) -> dict[str, Any]: props = node_spec.get("Props") or node_spec.get("props") or [] if not isinstance(props, list): raise ValueError(f"Node '{node_name}' Props must be a list.") merged_props: dict[str, Any] = {} for prop_name in props: if prop_name not in prop_defs: continue spec = prop_defs[prop_name] if not isinstance(spec, dict): continue # Keep the full property spec, but normalize common keys so the # generated YAML is easy to consume downstream. merged_props[prop_name] = { "Desc": spec.get("Desc", spec.get("description", "")), "Src": spec.get("Src", spec.get("source", "")), "Type": spec.get("Type", spec.get("type")), "Enum": spec.get("Enum", spec.get("enum", [])) or [], "Req": spec.get("Req", spec.get("required")), "Tags": spec.get("Tags", spec.get("tags", {})) or {}, } # Preserve optional/source flags when present for extra_key in ("Key", "Auto", "Deprecated", "Term"): if extra_key in spec: merged_props[prop_name][extra_key] = spec[extra_key] exclude_like = [p for p in DEFAULT_SKIP if p in merged_props] out: dict[str, Any] = { "node": node_name, "description": node_spec.get("Desc", node_spec.get("description", "")), "tags": node_spec.get("Tags", node_spec.get("tags", {})) or {}, "props": props, "exclude_like": exclude_like, "properties": merged_props, } # Keep the raw node spec too, so nothing is lost. out["node_spec"] = node_spec return out # def build_relation_schemas(): # node_url = os.getenv("NODE_MODEL_URL", "") # prop_url = os.getenv("PROP_MODEL_URL", "") # node_model = load_yaml_from_url(node_url) # prop_model = load_yaml_from_url(prop_url) # nodes = _get_nodes(node_model) # prop_defs = _get_prop_defs(prop_model) # node_schemas = {} # try: # for node_name, node_spec in nodes.items(): # if not isinstance(node_spec, dict): # continue # merged = _normalize_node_spec(node_name, node_spec, prop_defs) # merged = remove_yaml_anchors(merged) # node_schemas[node_name] = merged # relationships = extract_relationships(node_model) # relationships = enrich_relationships_with_properties(relationships, node_schemas) # rel_df = relationships_to_dataframe(relationships) # return node_schemas, rel_df, "" # except Exception as e: # return {}, None, f"Error on building relation \n {str(e)}" def filter_clusters_by_nodes(clusters, node_list): node_set = set(node_list) filtered = [] for cluster in clusters: if all(node in node_set for node in cluster): filtered.append(cluster) return filtered def build_join_paths_df(paths, rel_df): rows = [] for path_id, path in enumerate(paths): for i in range(len(path) - 1): parent = path[i] child = path[i + 1] rel = rel_df[ (rel_df["parent"] == parent) & (rel_df["child"] == child) ] for _, r in rel.iterrows(): rows.append({ "path_id": path_id, "step": i, "parent": parent, "child": child, "type": r["type"], "parent_prop": r["parent_prop"], "child_prop": r["child_prop"], "path": " → ".join(path) }) return pd.DataFrame(rows) def build_relation_schemas(node_list_state): import pandas as pd node_url = os.getenv("NODE_MODEL_URL", "") prop_url = os.getenv("PROP_MODEL_URL", "") try: node_model = load_yaml_from_url(node_url) prop_model = load_yaml_from_url(prop_url) nodes = _get_nodes(node_model) prop_defs = _get_prop_defs(prop_model) rows = [] node_schemas = {} # ✅ Build schemas (same as before) for node_name, node_spec in nodes.items(): if not isinstance(node_spec, dict): continue merged = _normalize_node_spec(node_name, node_spec, prop_defs) merged = remove_yaml_anchors(merged) node_schemas[node_name] = merged # ✅ Process relationships directly → rows relationships = node_model.get("Relationships", {}) # avoid circular dependencies for rel_name, rel_spec in relationships.items(): default_mul = rel_spec.get("Mul", "many_to_one") for end in rel_spec.get("Ends", []): src = end.get("Src") dst = end.get("Dst") if src == dst: continue if not src or not dst: continue mul = (end.get("Mul") or default_mul).lower() # --- resolve direction --- if mul == "many_to_one": parent, child, rel_type = dst, src, "one_to_many" elif mul == "one_to_many": parent, child, rel_type = src, dst, "one_to_many" elif mul == "one_to_one": parent, child, rel_type = dst, src, "one_to_one" elif mul == "many_to_many": parent, child, rel_type = src, dst, "many_to_many" else: parent, child, rel_type = dst, src, mul parent_props = node_schemas.get(parent, {}).get("properties", {}) child_props = node_schemas.get(child, {}).get("properties", {}) matches = [] # ✅ simple inline matching (no extra function) for p in parent_props: for c in child_props: if p == c or (p in c or c in p): # optional filter → only id-like if "id" in p or "id" in c: matches.append((p, c)) # ✅ write rows directly if not matches: rows.append({ "relation": rel_name, "parent": parent, "child": child, "type": rel_type, "parent_prop": None, "child_prop": None, }) else: for p, c in matches: rows.append({ "relation": rel_name, "parent": parent, "child": child, "type": rel_type, "parent_prop": f"{parent}.{p}", "child_prop": f"{child}.{c}", }) df = pd.DataFrame(rows).sort_values( by=["parent", "child"] ).reset_index(drop=True) node_tree_df = df.copy() # build node tree graph = build_graph(df) # paths = bfs_layers(graph) paths = bfs_all_paths(graph) # valid_paths = [p for p in paths if 2 <= len(p) <= 4] print(node_list_state) for i in range(len(paths)): print(paths[i]) print(len(paths)) valid_paths = filter_paths_by_nodes(paths, node_list_state) # valid_paths = valid_paths[:10] print(len(valid_paths)) for i in range(len(valid_paths)): print(valid_paths[i]) join_df = build_join_paths_df(valid_paths, df) return node_schemas, df, "", node_tree_df, join_df except Exception as e: return {}, None, f"Error on building relation\n{str(e)}" def filter_paths_by_nodes(paths, node_list): node_set = set(node_list) return [ p for p in paths if set(p).issubset(node_set) ] def find_paths(graph, start, target): queue = deque([(start, [start])]) paths = [] while queue: node, path = queue.popleft() if node == target: paths.append(path) continue for neighbor in graph.get(node, []): if neighbor not in path: # avoid cycles queue.append((neighbor, path + [neighbor])) return paths