Spaces:
Sleeping
Sleeping
Download analyze/intra_node.py from arana22/Data-Generator-MVP: direct link, hf CLI and curl.
- Browser
- Download file 5.42 kB
-
https://huggingface.co/spaces/arana22/Data-Generator-MVP/resolve/main/analyze/intra_node.py
- Command line
-
hf download hf://spaces/arana22/Data-Generator-MVP/analyze/intra_node.py
-
curl -L -o intra_node.py https://huggingface.co/spaces/arana22/Data-Generator-MVP/resolve/main/analyze/intra_node.py
5.42 kB
| from __future__ import annotations | |
| import gradio as gr | |
| import pandas as pd | |
| from dataclasses import dataclass | |
| from typing import Any, Dict | |
| import os, requests, json | |
| import pandas as pd | |
| from feature.doc_alignment import DocAlignmentModel | |
| from feature.base import FeatureBase | |
| from schema import NodeSchema | |
| from feature.categorical import CategoricalFeatureAnalyzer | |
| from feature.substring import SubstringFeatureAnalyzer | |
| from feature.cluster import ClusteringFeatureAnalyzer | |
| from feature.bio_term_overlap import BioTermFeatureAnalyzer | |
| from render.status import build_status_md | |
| class RelationshipResult: | |
| A: str | |
| B: str | |
| feature_type: str | |
| strength: float | |
| classification: str | |
| support: float | |
| predictive_strength: float | |
| determinism: float | |
| stability: float | |
| doc_alignment: float | |
| heldout_accuracy: float | |
| baseline_accuracy: float | |
| row_count: int | |
| total_rows: int | |
| train_rows: int | |
| test_rows: int | |
| suffix_match: float = 0.0 | |
| substring_match: float = 0.0 | |
| evidence: list[dict[str, Any]] = None | |
| a_to_b_mapping: str | None = None | |
| def from_dict(cls, row: dict[str, Any]) -> "RelationshipResult": | |
| return cls(**row) | |
| class PairwiseRelationshipEvaluator(FeatureBase): | |
| def __init__(self, node_schema: NodeSchema, weights: dict[str, float] | None = None): | |
| self.node_schema = node_schema | |
| self.doc_model = DocAlignmentModel().fit(node_schema) | |
| self.categorical = CategoricalFeatureAnalyzer( | |
| node_schema=self.node_schema, | |
| doc_model=self.doc_model, | |
| all_weights=weights.get("categorical", None) | |
| ) | |
| self.substring = SubstringFeatureAnalyzer( | |
| node_schema=self.node_schema, | |
| doc_model=self.doc_model, | |
| all_weights=weights.get("substring", None) | |
| ) | |
| self.cluster = ClusteringFeatureAnalyzer( | |
| node_schema=self.node_schema, | |
| doc_model=self.doc_model, | |
| all_weights=weights.get("cluster", None) | |
| ) | |
| self.bio_term = BioTermFeatureAnalyzer( | |
| node_schema=self.node_schema, | |
| doc_model=self.doc_model, | |
| all_weights=weights.get("bio-term", None) | |
| ) | |
| def get_model_columns(self, df: pd.DataFrame) -> list[str]: | |
| schema_columns = list(getattr(self.node_schema, "properties", {}).keys()) | |
| return [ | |
| c for c in schema_columns if c in df.columns and not self.should_skip(c, self.node_schema) | |
| ] | |
| def evaluate(self, df: pd.DataFrame) -> pd.DataFrame: | |
| return self.evaluate_all_pairs(df) | |
| def get_bio_score(self, a: str, b: str) -> float: | |
| return self._bio_cache.get((a, b), 0.0) | |
| def evaluate_all_pairs(self, df: pd.DataFrame) -> pd.DataFrame: | |
| results: list[dict[str, Any]] = [] | |
| columns = self.get_model_columns(df) | |
| for a in columns: | |
| for b in columns: | |
| if a == b: | |
| continue | |
| cat_result = self.categorical.analyze(df, a, b) | |
| if cat_result is not None: | |
| results.append(cat_result) | |
| string_result = self.substring.analyze(df, a, b) | |
| if string_result is not None: | |
| results.append(string_result) | |
| cluster_result = self.cluster.analyze(df, a, b) | |
| if cluster_result is not None: | |
| results.append(cluster_result) | |
| results_df = pd.DataFrame(results) | |
| if not results_df.empty: | |
| sort_cols = [c for c in ["strength", "predictive_strength", "support"] if c in results_df.columns] | |
| if sort_cols: | |
| results_df = results_df.sort_values(sort_cols, ascending=False).reset_index(drop=True) | |
| return results_df | |
| def build_evaluator(node_schema: NodeSchema) -> PairwiseRelationshipEvaluator: | |
| return PairwiseRelationshipEvaluator(node_schema=node_schema) | |
| def run_intra_node_analysis( | |
| weights_state, | |
| schema_state, | |
| data_state, | |
| selected_node | |
| ): | |
| try: | |
| if not schema_state: | |
| raise gr.Error("Load schema first.") | |
| if not selected_node: | |
| raise gr.Error("Select a node.") | |
| schema = next((s for s in schema_state if s.name == selected_node), None) | |
| if schema is None: | |
| raise gr.Error(f"Schema for node '{selected_node}' was not found.") | |
| df = data_state.get(selected_node) | |
| if df is None or df.empty: | |
| return {}, {}, f"No data available for node `{selected_node}`.", pd.DataFrame(), | |
| # if schema.name == "case": | |
| # print(json.dumps(weights_state, indent=2, default=str)) | |
| engine = PairwiseRelationshipEvaluator(schema, weights_state) | |
| results = engine.evaluate_all_pairs(df) | |
| summary_md = build_status_md(schema.name, None, results) | |
| features_dfs = dict(tuple(results.groupby('feature_type'))) | |
| # Optional: if you want a markdown report table in the UI | |
| # report_html = df_to_html2(results) | |
| results_by_node = {selected_node: results.copy()} | |
| relationship_by_node = {selected_node: results.copy()} | |
| return results_by_node, relationship_by_node, summary_md, features_dfs | |
| except Exception as e: | |
| return {}, {}, f"<div style='color:red;font-weight:700'>Error: {e}</div>", pd.DataFrame(), | |