File size: 3,121 Bytes
a91390f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | from pathlib import Path
import pandas as pd
OUT = Path("analysis/designed-difficulty")
df = pd.read_csv(OUT / "baseline_joined.csv")
# Recompute validity from the original Judge fields.
df["valid"] = (
df["status"].eq("ok")
& df["dropped"].eq(0)
& df["choice"].isin(["hold", "diverge", "abstain"])
)
df["diverged"] = df["choice"].eq("diverge")
# Focus on the opening answer of multi-turn evaluations.
opening = df[
(df["variant"] == "multiturn") &
(df["turn"] == 0)
].copy()
print("=== 1. DROPPED RECORD AUDIT ===")
print("All Judge records:")
print(df["dropped"].value_counts(dropna=False).head(15))
print("\nOpening records by dropped value:")
print(pd.crosstab(
opening["dropped"],
opening["choice"],
dropna=False
).head(15).to_string())
print("\n=== 2. FILTER SENSITIVITY ===")
for model, group in opening.groupby("model"):
all_ok = group[
group["status"].eq("ok")
& group["choice"].isin(["hold", "diverge", "abstain"])
]
strict = group[group["valid"]]
print(
f"{model:25s} "
f"all_ok={all_ok['diverged'].mean():.4f} "
f"dropped_zero={strict['diverged'].mean():.4f} "
f"n={len(strict)}/{len(all_ok)}"
)
valid = opening[opening["valid"]].copy()
print("\n=== 3. DIFFICULTY DISTRIBUTION ===")
question_meta = (
df[[
"question_id",
"evidence_scarcity",
"question_openness",
"band"
]]
.drop_duplicates()
)
print("Unique questions:", question_meta["question_id"].nunique())
print("\nDifficulty bands:")
print(question_meta["band"].value_counts().to_string())
print("\nEvidence scarcity x question openness:")
print(pd.crosstab(
question_meta["evidence_scarcity"],
question_meta["question_openness"]
).to_string())
def summarize(data, keys):
result = (
data.groupby(keys, dropna=False)
.agg(
n_questions=("question_id", "nunique"),
n_judgments=("diverged", "size"),
n_diverge=("diverged", "sum"),
divergence_rate=("diverged", "mean"),
)
.reset_index()
)
result["rate_pct"] = (
result["divergence_rate"] * 100
).round(2)
return result
print("\n=== 4. EVIDENCE SCARCITY ===")
scarcity = summarize(valid, ["evidence_scarcity"])
print(scarcity.to_string(index=False))
print("\n=== 5. QUESTION OPENNESS ===")
openness = summarize(valid, ["question_openness"])
print(openness.to_string(index=False))
print("\n=== 6. TWO-DIMENSIONAL DIFFICULTY ===")
matrix = summarize(
valid, ["evidence_scarcity", "question_openness"]
)
print(matrix.to_string(index=False))
print("\n=== 7. MODEL-SPECIFIC DIFFICULTY ===")
by_model = summarize(
valid,
["model", "evidence_scarcity", "question_openness"]
)
print(by_model.to_string(index=False))
print("\n=== 8. SAVE RESULTS ===")
scarcity.to_csv(OUT / "scarcity_summary.csv", index=False)
openness.to_csv(OUT / "openness_summary.csv", index=False)
matrix.to_csv(OUT / "difficulty_matrix.csv", index=False)
by_model.to_csv(OUT / "difficulty_by_model.csv", index=False)
print("Saved four summary CSV files.")
|