import re
import pandas as pd
import numpy as np
from collections import Counter

FAULT_ABSENCE_DIMS = ["overclaiming_score", "uncertainty_score", "bias_framing_score"]
PRESENCE_DIMS = ["faithfulness_score", "methods_limitations_score"]
AUTOMATED = ["summac_conv", "bertscore_f1"]

_MARKUP_RE  = re.compile(r"\[/?(?:HTML|INST)\]|</?(?:html|body|div|s|p)\s*>|<\|.*?\|>|</?s>", re.I)
_REFUSAL_RE = re.compile(r"\b(as an ai|i cannot|i'm unable|i am unable|cannot provide|i can't)\b", re.I)


def _repetition_ratio(text):
    words = text.split()
    if len(words) < 10:
        return 0.0
    return 1 - (len(Counter(words)) / len(words))


def screen_validity(df, text_col="generated_output", min_words=20, rep_threshold=0.85):
    valids, reasons = [], []
    for _, r in df.iterrows():
        txt = r.get(text_col)
        has = r["has_output"] if "has_output" in df.columns else pd.notna(txt)
        if (not has) or pd.isna(txt) or not str(txt).strip():
            valids.append(False); reasons.append("missing_or_empty"); continue
        s = str(txt).strip()
        wc = len(s.split())
        if _MARKUP_RE.search(s) and wc < 40:
            valids.append(False); reasons.append("markup_fragment"); continue
        if wc < min_words:
            valids.append(False); reasons.append("too_short"); continue
        if wc > 30 and _repetition_ratio(s) >= rep_threshold:
            valids.append(False); reasons.append("degenerate_repetition"); continue
        if _REFUSAL_RE.search(s) and wc < 60:
            valids.append(False); reasons.append("refusal"); continue
        valids.append(True); reasons.append("valid")
    df = df.copy()
    df["is_valid"] = valids
    df["degeneracy_type"] = reasons
    return df


def degeneracy_summary(df):
    present = df[df["has_output"]] if "has_output" in df.columns else df
    return (present.groupby(["task", "model", "degeneracy_type"]).size()
                   .unstack(fill_value=0))


def build_scorecard(df):
    rows = []
    for (task, model), g in df.groupby(["task", "model"]):
        present = g[g["has_output"]] if "has_output" in g.columns else g
        valid = present[present["is_valid"]]
        rec = {"task": task, "model": model,
               "n_total": len(g),
               "n_missing": int((~g["has_output"]).sum()) if "has_output" in g.columns else 0,
               "n_degenerate": int((~present["is_valid"]).sum()),
               "n_valid": len(valid)}
        for col in PRESENCE_DIMS + AUTOMATED:
            if col in present.columns and present[col].notna().any():
                rec[col + "_mean"] = round(present[col].mean(), 3)
        for col in FAULT_ABSENCE_DIMS:
            if col in present.columns and present[col].notna().any():
                rec[col + "_gated"] = round(valid[col].mean(), 3) if len(valid) else np.nan
                rec[col + "_naive"] = round(present[col].mean(), 3)
        rows.append(rec)
    return pd.DataFrame(rows)


if __name__ == "__main__":
    df = pd.read_csv("long_scored_dataset.csv")
    df = screen_validity(df)

    print("Degeneracy type counts by model x task:")
    print(degeneracy_summary(df).to_string())
    print("\nInvalid outputs found:", int((~df["is_valid"]).sum()), "of", len(df))
    print("\nExamples flagged degenerate:")
    bad = df[~df["is_valid"] & df["has_output"]][["output_id", "degeneracy_type", "generated_output"]]
    for _, r in bad.head(6).iterrows():
        print(f"  [{r.degeneracy_type}] {r.output_id}: {str(r.generated_output)[:70]!r}")