xaheli's picture
download
raw
16.3 kB
"""Aggregate the raw model outputs into the metrics that carry the argument.
The three headline numbers:
1. CAUSAL SELECTION quality (precision / recall of the true parent set) for the LM
versus two statistical baselines that are given *more* information than the LM.
2. INTERVENTIONAL ESTIMATION error -- how far the LM's causal effect estimate is from
the exact structural effect, broken down per variable.
3. WHAT THE MODEL IS ACTUALLY TRACKING -- the correlation between the LM's predicted
effect and (a) the true causal effect, versus (b) the marginal correlation. If (b)
wins, the model is re-ranking the correlation table, not reasoning about mechanism.
"""
from __future__ import annotations
import glob
import json
import os
from typing import Dict, List
import numpy as np
from baselines import (
corr_baseline,
oracle,
partial_corr_baseline,
per_variable_scores,
score_selection,
)
from dgp import CANDIDATE_NAMES, TRUE_PARENTS, make_instances
SEL_K = 3 # reference k for the statistical baselines (top-3 by score)
RESULTS_DIR = "results"
def _pearson(a: List[float], b: List[float]) -> float:
if len(a) < 3:
return float("nan")
a, b = np.asarray(a, float), np.asarray(b, float)
if np.std(a) < 1e-12 or np.std(b) < 1e-12:
return float("nan")
return float(np.corrcoef(a, b)[0, 1])
def analyse_model(path: str, instances) -> Dict:
blob = json.load(open(path, encoding="utf8"))
res = blob["results"]
by_seed = {i.spec.seed: i for i in instances}
# ---------------- SELECT ----------------
sel_rows = []
for row in res["select"]:
inst = by_seed[row["instance"]]
sel_rows.append(
{
"seed": row["instance"],
"pred": row["pred"],
"truth": inst.true_parents,
"n_pred": len(row["pred"]),
}
)
def agg_set_metric(keyfn):
precs, recs, f1s, jacs, sizes = [], [], [], [], []
for r in sel_rows:
s = score_selection(keyfn(r), r["truth"])
precs.append(s["precision"])
recs.append(s["recall"])
f1s.append(s["f1"])
jacs.append(s["jaccard"])
sizes.append(len(keyfn(r)))
return {
"precision": float(np.mean(precs)),
"recall": float(np.mean(recs)),
"f1": float(np.mean(f1s)),
"jaccard": float(np.mean(jacs)),
"mean_n_selected": float(np.mean(sizes)),
}
lm_sel = agg_set_metric(lambda r: r["pred"])
corr_sel = agg_set_metric(
lambda r: corr_baseline(by_seed[r["seed"]], SEL_K)
)
pcorr_sel = agg_set_metric(
lambda r: partial_corr_baseline(by_seed[r["seed"]], SEL_K)
)
oracle_sel = agg_set_metric(lambda r: oracle(by_seed[r["seed"]], SEL_K))
# per-variable: how often is each variable called a cause?
per_var_lm = {n: [] for n in CANDIDATE_NAMES}
per_var_corr = {n: [] for n in CANDIDATE_NAMES}
for r in sel_rows:
inst = by_seed[r["seed"]]
pv = per_variable_scores(r["pred"], inst.true_parents, CANDIDATE_NAMES)
for n, v in pv.items():
per_var_lm[n].append(v["called_cause"])
pvc = per_variable_scores(
corr_baseline(inst, SEL_K), inst.true_parents, CANDIDATE_NAMES
)
for n, v in pvc.items():
per_var_corr[n].append(v["called_cause"])
# ---------------- INTERVENE ----------------
per_var_eff = {n: {"pred": [], "true": [], "marg": []} for n in CANDIDATE_NAMES}
parse_fail = 0
for row in res["intervene"]:
inst = by_seed[row["instance"]]
v = row["variable"]
if row["effect"] is None:
parse_fail += 1
continue
per_var_eff[v]["pred"].append(float(row["effect"]))
per_var_eff[v]["true"].append(float(inst.do_effects[v]))
per_var_eff[v]["marg"].append(abs(float(inst.marginal_corr[v])))
eff_summary = {}
all_pred, all_true, all_marg = [], [], []
for n in CANDIDATE_NAMES:
p = per_var_eff[n]["pred"]
t = per_var_eff[n]["true"]
m = per_var_eff[n]["marg"]
if not p:
continue
true_mean = float(np.mean(t))
pred_mean = float(np.mean(p))
# scale of the true effect distribution, used to normalise the error
scale = max(0.1, abs(true_mean))
eff_summary[n] = {
"true_effect": true_mean,
"pred_effect_mean": pred_mean,
"pred_effect_std": float(np.std(p)),
"mean_abs_error": float(np.mean(np.abs(np.array(p) - np.array(t)))),
"normalised_error": float(np.mean(np.abs(np.array(p) - np.array(t))) / scale),
"sign_correct_rate": float(
np.mean([np.sign(a) == np.sign(b) for a, b in zip(p, t)])
),
"is_parent": n in TRUE_PARENTS,
"mean_abs_marginal_corr": float(np.mean(m)),
}
all_pred += p
all_true += t
all_marg += m
# ---------------- WHAT IS IT TRACKING? ----------------
r_with_true = _pearson(all_pred, all_true)
r_with_corr = _pearson(all_pred, all_marg)
# ---------------- DECOUPLING: causal answer vs predictive answer ----------------
dec = {"same_set_rate": [], "lm_causal_f1": [], "lm_predictive_f1": []}
for r in sel_rows:
inst = by_seed[r["seed"]]
prow = next(
(x for x in res["predict"] if x["instance"] == r["seed"]), None
)
if prow is None:
continue
dec["same_set_rate"].append(
float(set(r["pred"]) == set(prow["pred"]))
)
dec["lm_causal_f1"].append(score_selection(r["pred"], inst.true_parents)["f1"])
dec["lm_predictive_f1"].append(
score_selection(prow["pred"], inst.true_parents)["f1"]
)
return {
"label": blob["label"],
"model": blob["model"],
"select": {
"lm": lm_sel,
"corr_baseline": corr_sel,
"partial_corr_baseline": pcorr_sel,
"oracle": oracle_sel,
"per_var_called_rate_lm": {n: float(np.mean(v)) for n, v in per_var_lm.items()},
"per_var_called_rate_corr": {n: float(np.mean(v)) for n, v in per_var_corr.items()},
},
"intervene": eff_summary,
"intervene_parse_failures": parse_fail,
"tracking": {
"r_pred_vs_true_effect": r_with_true,
"r_pred_vs_abs_marginal_corr": r_with_corr,
},
"decoupling": {
"causal_answer_equals_predictive_answer_rate": float(np.mean(dec["same_set_rate"])),
"mean_f1_causal_question": float(np.mean(dec["lm_causal_f1"])),
"mean_f1_predictive_question": float(np.mean(dec["lm_predictive_f1"])),
},
}
def analyse_trial_condition(path: str, instances) -> Dict:
"""Second condition: the model additionally sees real randomised-trial measurements."""
from dgp import trial_table
blob = json.load(open(path, encoding="utf8"))
res = blob["results"]
by_seed = {i.spec.seed: i for i in instances}
# ---- selection with trial evidence ----
sel_f1, sel_prec, sel_rec = [], [], []
for row in res.get("select_trial", []):
inst = by_seed[row["instance"]]
s = score_selection(row["pred"], inst.true_parents)
sel_f1.append(s["f1"])
sel_prec.append(s["precision"])
sel_rec.append(s["recall"])
# ---- intervention with trial evidence ----
# Split the (model, variable, instance) triples by whether a trial exists for that
# variable in that instance, and check which evidence source the model tracked.
with_trial = {"pred": [], "trial": [], "true": [], "corr": []}
without_trial = {"pred": [], "true": [], "corr": []}
parse_fail = 0
for row in res.get("intervene_trial", []):
inst = by_seed[row["instance"]]
v = row["variable"]
if row["effect"] is None:
parse_fail += 1
continue
pred = float(row["effect"])
corr = abs(float(inst.marginal_corr[v]))
tt = {t["variable"]: t for t in trial_table(inst)}
if v in tt:
with_trial["pred"].append(pred)
with_trial["trial"].append(float(tt[v]["measured_effect"]))
with_trial["true"].append(float(inst.do_effects[v]))
with_trial["corr"].append(corr)
else:
without_trial["pred"].append(pred)
without_trial["true"].append(float(inst.do_effects[v]))
without_trial["corr"].append(corr)
def block(d):
out = {"n": len(d["pred"])}
if not d["pred"]:
return out
p = np.asarray(d["pred"])
out["mean_pred"] = float(p.mean())
if "corr" in d:
out["r_pred_vs_abs_corr"] = _pearson(list(p), d["corr"])
out["mean_abs_dev_from_abs_corr"] = float(np.mean(np.abs(p - np.asarray(d["corr"]))))
if "true" in d:
out["r_pred_vs_true_effect"] = _pearson(list(p), d["true"])
out["mean_abs_dev_from_true_effect"] = float(
np.mean(np.abs(p - np.asarray(d["true"])))
)
if "trial" in d:
out["r_pred_vs_trial_measurement"] = _pearson(list(p), d["trial"])
out["mean_abs_dev_from_trial_measurement"] = float(
np.mean(np.abs(p - np.asarray(d["trial"])))
)
# did the model actually *use* its own trial measurement for this variable?
dev = np.abs(p - np.asarray(d["trial"]))
out["frac_matching_own_trial_measurement"] = float(np.mean(dev < 0.10))
out["frac_matching_own_trial_measurement_strict"] = float(np.mean(dev < 0.03))
return out
return {
"label": blob["label"],
"select_trial": {
"precision": float(np.mean(sel_prec)) if sel_prec else float("nan"),
"recall": float(np.mean(sel_rec)) if sel_rec else float("nan"),
"f1": float(np.mean(sel_f1)) if sel_f1 else float("nan"),
"n": len(sel_f1),
},
"intervene_trial": {
"with_trial_evidence": block(with_trial),
"without_trial_evidence": block(without_trial),
},
"intervene_trial_parse_failures": parse_fail,
}
def main():
instances = make_instances(12)
out = {}
for path in sorted(glob.glob(os.path.join(RESULTS_DIR, "raw_*.json"))):
label = os.path.basename(path)[4:-5]
try:
out[label] = analyse_model(path, instances)
print(f"analysed {label}")
except Exception as e:
print(f"!! failed {label}: {e}")
with open(os.path.join(RESULTS_DIR, "summary.json"), "w", encoding="utf8") as f:
json.dump(out, f, indent=1)
# -------- human readable report --------
lines = []
A = lines.append
A("=" * 100)
A("BLIND SPOTS OF FRONTIER MODELS -- RESULTS")
A("=" * 100)
A("")
A("TASK 1: CAUSAL SELECTION (which signals directly cause demand?)")
A(f" ground truth parents per instance: {len(TRUE_PARENTS)} of {len(CANDIDATE_NAMES)} candidates")
A(f" {'model':26s} {'method':26s} {'prec':>7s} {'rec':>7s} {'F1':>7s} {'Jacc':>7s} {'|sel|':>7s}")
for label, r in out.items():
for key, nm in [
("lm", "language model"),
("corr_baseline", "|corr| top-3"),
("partial_corr_baseline", "partial-corr top-3"),
("oracle", "oracle (ground truth)"),
]:
s = r["select"][key]
A(
f" {label:26s} {nm:26s} {s['precision']:7.3f} {s['recall']:7.3f} "
f"{s['f1']:7.3f} {s['jaccard']:7.3f} {s['mean_n_selected']:7.2f}"
)
A("")
A("TASK 2: INTERVENTIONAL ESTIMATION (effect of a +1 SD intervention)")
A(f" {'model':26s} {'variable':16s} {'true':>8s} {'pred':>8s} {'MAE':>8s} {'|r|':>7s} {'sign ok':>8s} parent")
for label, r in out.items():
for v, e in r["intervene"].items():
A(
f" {label:26s} {v:16s} {e['true_effect']:8.3f} {e['pred_effect_mean']:8.3f} "
f"{e['mean_abs_error']:8.3f} {e['mean_abs_marginal_corr']:7.3f} "
f"{e['sign_correct_rate']:8.2f} {'yes' if e['is_parent'] else 'NO'}"
)
A("")
A("TASK 3: WHAT IS THE MODEL TRACKING?")
A(" correlation between the model's predicted causal effect and ...")
A(f" {'model':26s} {'true causal effect':>20s} {'|marginal corr|':>18s}")
for label, r in out.items():
t = r["tracking"]
A(f" {label:26s} {t['r_pred_vs_true_effect']:20.3f} {t['r_pred_vs_abs_marginal_corr']:18.3f}")
A("")
A("TASK 4: CAUSAL vs PREDICTIVE DECOUPLING")
A(f" {'model':26s} {'same set':>10s} {'F1 causal Q':>12s} {'F1 predict Q':>13s}")
for label, r in out.items():
d = r["decoupling"]
A(
f" {label:26s} {d['causal_answer_equals_predictive_answer_rate']:10.3f} "
f"{d['mean_f1_causal_question']:12.3f} {d['mean_f1_predictive_question']:13.3f}"
)
report = "\n".join(lines)
print(report)
with open(os.path.join(RESULTS_DIR, "report.txt"), "w", encoding="utf8") as f:
f.write(report + "\n")
# ---------------- randomised-trial condition ----------------
trial = {}
for path in sorted(glob.glob(os.path.join(RESULTS_DIR, "raw_*.json"))):
label = os.path.basename(path)[4:-5]
try:
trial[label] = analyse_trial_condition(path, instances)
except Exception as e:
print(f"!! trial analysis failed {label}: {e}")
with open(os.path.join(RESULTS_DIR, "summary_trial.json"), "w", encoding="utf8") as f:
json.dump(trial, f, indent=1)
T = []
B = T.append
B("")
B("=" * 100)
B("CONDITION 2: RANDOMISED-TRIAL EVIDENCE ADDED")
B("=" * 100)
B("")
B("Causal selection, with a real randomised trial supplied for 3 of the 6 signals:")
B(f" {'model':26s} {'F1 (obs only)':>15s} {'F1 (+trial)':>12s} {'precision':>10s} {'recall':>8s}")
for label, r in trial.items():
base = out.get(label, {}).get("select", {}).get("lm", {}).get("f1", float("nan"))
t = r["select_trial"]
B(f" {label:26s} {base:15.3f} {t['f1']:12.3f} {t['precision']:10.3f} {t['recall']:8.3f}")
B("")
B("Interventional estimation, split by whether a trial measurement existed:")
B(f" {'model':26s} {'subset':>22s} {'r(pred,corr)':>13s} {'r(pred,true)':>13s} {'r(pred,trial)':>14s}")
for label, r in trial.items():
for key, nm in [
("with_trial_evidence", "signals WITH a trial"),
("without_trial_evidence", "signals WITHOUT a trial"),
]:
b = r["intervene_trial"][key]
if not b.get("n"):
continue
B(
f" {label:26s} {nm:>22s} {b.get('r_pred_vs_abs_corr', float('nan')):13.3f} "
f"{b.get('r_pred_vs_true_effect', float('nan')):13.3f} "
f"{b.get('r_pred_vs_trial_measurement', float('nan')):14.3f}"
)
B("")
B("")
B("Trial utilisation (did the model reproduce ITS OWN trial measurement for that variable?):")
B(f" {'model':26s} {'subset':>22s} {'frac |dev|<0.10':>16s} {'frac |dev|<0.03':>16s} {'mean |dev|':>12s}")
for label, r in trial.items():
for key, nm in [
("with_trial_evidence", "signals WITH a trial"),
("without_trial_evidence", "signals WITHOUT a trial"),
]:
b = r["intervene_trial"][key]
if not b.get("n"):
continue
B(
f" {label:26s} {nm:>22s} "
f"{b.get('frac_matching_own_trial_measurement', float('nan')):16.3f} "
f"{b.get('frac_matching_own_trial_measurement_strict', float('nan')):16.3f} "
f"{b.get('mean_abs_dev_from_trial_measurement', float('nan')):12.3f}"
)
B("")
rep2 = "\n".join(T)
print(rep2)
with open(os.path.join(RESULTS_DIR, "report.txt"), "a", encoding="utf8") as f:
f.write(rep2 + "\n")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
16.3 kB
·
Xet hash:
aba6c0c41ee34c3884d7e8dd775e40ec146f4f3ddb1de5c8a564ef3aa52e32f5

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.