Spaces:
Sleeping
Sleeping
File size: 3,514 Bytes
b27de80 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | import re
import pandas as pd
import numpy as np
from collections import Counter
FAULT_ABSENCE_DIMS = ["overclaiming_score", "uncertainty_score", "bias_framing_score"]
PRESENCE_DIMS = ["faithfulness_score", "methods_limitations_score"]
AUTOMATED = ["summac_conv", "bertscore_f1"]
_MARKUP_RE = re.compile(r"\[/?(?:HTML|INST)\]|</?(?:html|body|div|s|p)\s*>|<\|.*?\|>|</?s>", re.I)
_REFUSAL_RE = re.compile(r"\b(as an ai|i cannot|i'm unable|i am unable|cannot provide|i can't)\b", re.I)
def _repetition_ratio(text):
words = text.split()
if len(words) < 10:
return 0.0
return 1 - (len(Counter(words)) / len(words))
def screen_validity(df, text_col="generated_output", min_words=20, rep_threshold=0.85):
valids, reasons = [], []
for _, r in df.iterrows():
txt = r.get(text_col)
has = r["has_output"] if "has_output" in df.columns else pd.notna(txt)
if (not has) or pd.isna(txt) or not str(txt).strip():
valids.append(False); reasons.append("missing_or_empty"); continue
s = str(txt).strip()
wc = len(s.split())
if _MARKUP_RE.search(s) and wc < 40:
valids.append(False); reasons.append("markup_fragment"); continue
if wc < min_words:
valids.append(False); reasons.append("too_short"); continue
if wc > 30 and _repetition_ratio(s) >= rep_threshold:
valids.append(False); reasons.append("degenerate_repetition"); continue
if _REFUSAL_RE.search(s) and wc < 60:
valids.append(False); reasons.append("refusal"); continue
valids.append(True); reasons.append("valid")
df = df.copy()
df["is_valid"] = valids
df["degeneracy_type"] = reasons
return df
def degeneracy_summary(df):
present = df[df["has_output"]] if "has_output" in df.columns else df
return (present.groupby(["task", "model", "degeneracy_type"]).size()
.unstack(fill_value=0))
def build_scorecard(df):
rows = []
for (task, model), g in df.groupby(["task", "model"]):
present = g[g["has_output"]] if "has_output" in g.columns else g
valid = present[present["is_valid"]]
rec = {"task": task, "model": model,
"n_total": len(g),
"n_missing": int((~g["has_output"]).sum()) if "has_output" in g.columns else 0,
"n_degenerate": int((~present["is_valid"]).sum()),
"n_valid": len(valid)}
for col in PRESENCE_DIMS + AUTOMATED:
if col in present.columns and present[col].notna().any():
rec[col + "_mean"] = round(present[col].mean(), 3)
for col in FAULT_ABSENCE_DIMS:
if col in present.columns and present[col].notna().any():
rec[col + "_gated"] = round(valid[col].mean(), 3) if len(valid) else np.nan
rec[col + "_naive"] = round(present[col].mean(), 3)
rows.append(rec)
return pd.DataFrame(rows)
if __name__ == "__main__":
df = pd.read_csv("long_scored_dataset.csv")
df = screen_validity(df)
print("Degeneracy type counts by model x task:")
print(degeneracy_summary(df).to_string())
print("\nInvalid outputs found:", int((~df["is_valid"]).sum()), "of", len(df))
print("\nExamples flagged degenerate:")
bad = df[~df["is_valid"] & df["has_output"]][["output_id", "degeneracy_type", "generated_output"]]
for _, r in bad.head(6).iterrows():
print(f" [{r.degeneracy_type}] {r.output_id}: {str(r.generated_output)[:70]!r}") |