from __future__ import annotations import gradio as gr import pandas as pd from dataclasses import dataclass from typing import Any, Dict import os, requests, json import pandas as pd from feature.doc_alignment import DocAlignmentModel from feature.base import FeatureBase from schema import NodeSchema from feature.categorical import CategoricalFeatureAnalyzer from feature.substring import SubstringFeatureAnalyzer from feature.cluster import ClusteringFeatureAnalyzer from feature.bio_term_overlap import BioTermFeatureAnalyzer from render.status import build_status_md @dataclass(frozen=True) class RelationshipResult: A: str B: str feature_type: str strength: float classification: str support: float predictive_strength: float determinism: float stability: float doc_alignment: float heldout_accuracy: float baseline_accuracy: float row_count: int total_rows: int train_rows: int test_rows: int suffix_match: float = 0.0 substring_match: float = 0.0 evidence: list[dict[str, Any]] = None a_to_b_mapping: str | None = None @classmethod def from_dict(cls, row: dict[str, Any]) -> "RelationshipResult": return cls(**row) class PairwiseRelationshipEvaluator(FeatureBase): def __init__(self, node_schema: NodeSchema, weights: dict[str, float] | None = None): self.node_schema = node_schema self.doc_model = DocAlignmentModel().fit(node_schema) self.categorical = CategoricalFeatureAnalyzer( node_schema=self.node_schema, doc_model=self.doc_model, all_weights=weights.get("categorical", None) ) self.substring = SubstringFeatureAnalyzer( node_schema=self.node_schema, doc_model=self.doc_model, all_weights=weights.get("substring", None) ) self.cluster = ClusteringFeatureAnalyzer( node_schema=self.node_schema, doc_model=self.doc_model, all_weights=weights.get("cluster", None) ) self.bio_term = BioTermFeatureAnalyzer( node_schema=self.node_schema, doc_model=self.doc_model, all_weights=weights.get("bio-term", None) ) def get_model_columns(self, df: pd.DataFrame) -> list[str]: schema_columns = list(getattr(self.node_schema, "properties", {}).keys()) return [ c for c in schema_columns if c in df.columns and not self.should_skip(c, self.node_schema) ] def evaluate(self, df: pd.DataFrame) -> pd.DataFrame: return self.evaluate_all_pairs(df) def get_bio_score(self, a: str, b: str) -> float: return self._bio_cache.get((a, b), 0.0) def evaluate_all_pairs(self, df: pd.DataFrame) -> pd.DataFrame: results: list[dict[str, Any]] = [] columns = self.get_model_columns(df) for a in columns: for b in columns: if a == b: continue cat_result = self.categorical.analyze(df, a, b) if cat_result is not None: results.append(cat_result) string_result = self.substring.analyze(df, a, b) if string_result is not None: results.append(string_result) cluster_result = self.cluster.analyze(df, a, b) if cluster_result is not None: results.append(cluster_result) results_df = pd.DataFrame(results) if not results_df.empty: sort_cols = [c for c in ["strength", "predictive_strength", "support"] if c in results_df.columns] if sort_cols: results_df = results_df.sort_values(sort_cols, ascending=False).reset_index(drop=True) return results_df def build_evaluator(node_schema: NodeSchema) -> PairwiseRelationshipEvaluator: return PairwiseRelationshipEvaluator(node_schema=node_schema) def run_intra_node_analysis( weights_state, schema_state, data_state, selected_node ): try: if not schema_state: raise gr.Error("Load schema first.") if not selected_node: raise gr.Error("Select a node.") schema = next((s for s in schema_state if s.name == selected_node), None) if schema is None: raise gr.Error(f"Schema for node '{selected_node}' was not found.") df = data_state.get(selected_node) if df is None or df.empty: return {}, {}, f"No data available for node `{selected_node}`.", pd.DataFrame(), # if schema.name == "case": # print(json.dumps(weights_state, indent=2, default=str)) engine = PairwiseRelationshipEvaluator(schema, weights_state) results = engine.evaluate_all_pairs(df) summary_md = build_status_md(schema.name, None, results) features_dfs = dict(tuple(results.groupby('feature_type'))) # Optional: if you want a markdown report table in the UI # report_html = df_to_html2(results) results_by_node = {selected_node: results.copy()} relationship_by_node = {selected_node: results.copy()} return results_by_node, relationship_by_node, summary_md, features_dfs except Exception as e: return {}, {}, f"
Error: {e}
", pd.DataFrame(),