Spaces:
Sleeping
Sleeping
| import re | |
| import pandas as pd | |
| import numpy as np | |
| from collections import Counter | |
| FAULT_ABSENCE_DIMS = ["overclaiming_score", "uncertainty_score", "bias_framing_score"] | |
| PRESENCE_DIMS = ["faithfulness_score", "methods_limitations_score"] | |
| AUTOMATED = ["summac_conv", "bertscore_f1"] | |
| _MARKUP_RE = re.compile(r"\[/?(?:HTML|INST)\]|</?(?:html|body|div|s|p)\s*>|<\|.*?\|>|</?s>", re.I) | |
| _REFUSAL_RE = re.compile(r"\b(as an ai|i cannot|i'm unable|i am unable|cannot provide|i can't)\b", re.I) | |
| def _repetition_ratio(text): | |
| words = text.split() | |
| if len(words) < 10: | |
| return 0.0 | |
| return 1 - (len(Counter(words)) / len(words)) | |
| def screen_validity(df, text_col="generated_output", min_words=20, rep_threshold=0.85): | |
| valids, reasons = [], [] | |
| for _, r in df.iterrows(): | |
| txt = r.get(text_col) | |
| has = r["has_output"] if "has_output" in df.columns else pd.notna(txt) | |
| if (not has) or pd.isna(txt) or not str(txt).strip(): | |
| valids.append(False); reasons.append("missing_or_empty"); continue | |
| s = str(txt).strip() | |
| wc = len(s.split()) | |
| if _MARKUP_RE.search(s) and wc < 40: | |
| valids.append(False); reasons.append("markup_fragment"); continue | |
| if wc < min_words: | |
| valids.append(False); reasons.append("too_short"); continue | |
| if wc > 30 and _repetition_ratio(s) >= rep_threshold: | |
| valids.append(False); reasons.append("degenerate_repetition"); continue | |
| if _REFUSAL_RE.search(s) and wc < 60: | |
| valids.append(False); reasons.append("refusal"); continue | |
| valids.append(True); reasons.append("valid") | |
| df = df.copy() | |
| df["is_valid"] = valids | |
| df["degeneracy_type"] = reasons | |
| return df | |
| def degeneracy_summary(df): | |
| present = df[df["has_output"]] if "has_output" in df.columns else df | |
| return (present.groupby(["task", "model", "degeneracy_type"]).size() | |
| .unstack(fill_value=0)) | |
| def build_scorecard(df): | |
| rows = [] | |
| for (task, model), g in df.groupby(["task", "model"]): | |
| present = g[g["has_output"]] if "has_output" in g.columns else g | |
| valid = present[present["is_valid"]] | |
| rec = {"task": task, "model": model, | |
| "n_total": len(g), | |
| "n_missing": int((~g["has_output"]).sum()) if "has_output" in g.columns else 0, | |
| "n_degenerate": int((~present["is_valid"]).sum()), | |
| "n_valid": len(valid)} | |
| for col in PRESENCE_DIMS + AUTOMATED: | |
| if col in present.columns and present[col].notna().any(): | |
| rec[col + "_mean"] = round(present[col].mean(), 3) | |
| for col in FAULT_ABSENCE_DIMS: | |
| if col in present.columns and present[col].notna().any(): | |
| rec[col + "_gated"] = round(valid[col].mean(), 3) if len(valid) else np.nan | |
| rec[col + "_naive"] = round(present[col].mean(), 3) | |
| rows.append(rec) | |
| return pd.DataFrame(rows) | |
| if __name__ == "__main__": | |
| df = pd.read_csv("long_scored_dataset.csv") | |
| df = screen_validity(df) | |
| print("Degeneracy type counts by model x task:") | |
| print(degeneracy_summary(df).to_string()) | |
| print("\nInvalid outputs found:", int((~df["is_valid"]).sum()), "of", len(df)) | |
| print("\nExamples flagged degenerate:") | |
| bad = df[~df["is_valid"] & df["has_output"]][["output_id", "degeneracy_type", "generated_output"]] | |
| for _, r in bad.head(6).iterrows(): | |
| print(f" [{r.degeneracy_type}] {r.output_id}: {str(r.generated_output)[:70]!r}") |