Data-Generator-MVP / analyze /intra_node.py
rana22
refactor
f3c80e5
Raw History Blame Contribute Delete
5.42 kB
from __future__ import annotations
import gradio as gr
import pandas as pd
from dataclasses import dataclass
from typing import Any, Dict
import os, requests, json
import pandas as pd
from feature.doc_alignment import DocAlignmentModel
from feature.base import FeatureBase
from schema import NodeSchema
from feature.categorical import CategoricalFeatureAnalyzer
from feature.substring import SubstringFeatureAnalyzer
from feature.cluster import ClusteringFeatureAnalyzer
from feature.bio_term_overlap import BioTermFeatureAnalyzer
from render.status import build_status_md
@dataclass(frozen=True)
class RelationshipResult:
A: str
B: str
feature_type: str
strength: float
classification: str
support: float
predictive_strength: float
determinism: float
stability: float
doc_alignment: float
heldout_accuracy: float
baseline_accuracy: float
row_count: int
total_rows: int
train_rows: int
test_rows: int
suffix_match: float = 0.0
substring_match: float = 0.0
evidence: list[dict[str, Any]] = None
a_to_b_mapping: str | None = None
@classmethod
def from_dict(cls, row: dict[str, Any]) -> "RelationshipResult":
return cls(**row)
class PairwiseRelationshipEvaluator(FeatureBase):
def __init__(self, node_schema: NodeSchema, weights: dict[str, float] | None = None):
self.node_schema = node_schema
self.doc_model = DocAlignmentModel().fit(node_schema)
self.categorical = CategoricalFeatureAnalyzer(
node_schema=self.node_schema,
doc_model=self.doc_model,
all_weights=weights.get("categorical", None)
)
self.substring = SubstringFeatureAnalyzer(
node_schema=self.node_schema,
doc_model=self.doc_model,
all_weights=weights.get("substring", None)
)
self.cluster = ClusteringFeatureAnalyzer(
node_schema=self.node_schema,
doc_model=self.doc_model,
all_weights=weights.get("cluster", None)
)
self.bio_term = BioTermFeatureAnalyzer(
node_schema=self.node_schema,
doc_model=self.doc_model,
all_weights=weights.get("bio-term", None)
)
def get_model_columns(self, df: pd.DataFrame) -> list[str]:
schema_columns = list(getattr(self.node_schema, "properties", {}).keys())
return [
c for c in schema_columns if c in df.columns and not self.should_skip(c, self.node_schema)
]
def evaluate(self, df: pd.DataFrame) -> pd.DataFrame:
return self.evaluate_all_pairs(df)
def get_bio_score(self, a: str, b: str) -> float:
return self._bio_cache.get((a, b), 0.0)
def evaluate_all_pairs(self, df: pd.DataFrame) -> pd.DataFrame:
results: list[dict[str, Any]] = []
columns = self.get_model_columns(df)
for a in columns:
for b in columns:
if a == b:
continue
cat_result = self.categorical.analyze(df, a, b)
if cat_result is not None:
results.append(cat_result)
string_result = self.substring.analyze(df, a, b)
if string_result is not None:
results.append(string_result)
cluster_result = self.cluster.analyze(df, a, b)
if cluster_result is not None:
results.append(cluster_result)
results_df = pd.DataFrame(results)
if not results_df.empty:
sort_cols = [c for c in ["strength", "predictive_strength", "support"] if c in results_df.columns]
if sort_cols:
results_df = results_df.sort_values(sort_cols, ascending=False).reset_index(drop=True)
return results_df
def build_evaluator(node_schema: NodeSchema) -> PairwiseRelationshipEvaluator:
return PairwiseRelationshipEvaluator(node_schema=node_schema)
def run_intra_node_analysis(
weights_state,
schema_state,
data_state,
selected_node
):
try:
if not schema_state:
raise gr.Error("Load schema first.")
if not selected_node:
raise gr.Error("Select a node.")
schema = next((s for s in schema_state if s.name == selected_node), None)
if schema is None:
raise gr.Error(f"Schema for node '{selected_node}' was not found.")
df = data_state.get(selected_node)
if df is None or df.empty:
return {}, {}, f"No data available for node `{selected_node}`.", pd.DataFrame(),
# if schema.name == "case":
# print(json.dumps(weights_state, indent=2, default=str))
engine = PairwiseRelationshipEvaluator(schema, weights_state)
results = engine.evaluate_all_pairs(df)
summary_md = build_status_md(schema.name, None, results)
features_dfs = dict(tuple(results.groupby('feature_type')))
# Optional: if you want a markdown report table in the UI
# report_html = df_to_html2(results)
results_by_node = {selected_node: results.copy()}
relationship_by_node = {selected_node: results.copy()}
return results_by_node, relationship_by_node, summary_md, features_dfs
except Exception as e:
return {}, {}, f"<div style='color:red;font-weight:700'>Error: {e}</div>", pd.DataFrame(),