Buckets:
| """Aggregate the raw model outputs into the metrics that carry the argument. | |
| The three headline numbers: | |
| 1. CAUSAL SELECTION quality (precision / recall of the true parent set) for the LM | |
| versus two statistical baselines that are given *more* information than the LM. | |
| 2. INTERVENTIONAL ESTIMATION error -- how far the LM's causal effect estimate is from | |
| the exact structural effect, broken down per variable. | |
| 3. WHAT THE MODEL IS ACTUALLY TRACKING -- the correlation between the LM's predicted | |
| effect and (a) the true causal effect, versus (b) the marginal correlation. If (b) | |
| wins, the model is re-ranking the correlation table, not reasoning about mechanism. | |
| """ | |
| from __future__ import annotations | |
| import glob | |
| import json | |
| import os | |
| from typing import Dict, List | |
| import numpy as np | |
| from baselines import ( | |
| corr_baseline, | |
| oracle, | |
| partial_corr_baseline, | |
| per_variable_scores, | |
| score_selection, | |
| ) | |
| from dgp import CANDIDATE_NAMES, TRUE_PARENTS, make_instances | |
| SEL_K = 3 # reference k for the statistical baselines (top-3 by score) | |
| RESULTS_DIR = "results" | |
| def _pearson(a: List[float], b: List[float]) -> float: | |
| if len(a) < 3: | |
| return float("nan") | |
| a, b = np.asarray(a, float), np.asarray(b, float) | |
| if np.std(a) < 1e-12 or np.std(b) < 1e-12: | |
| return float("nan") | |
| return float(np.corrcoef(a, b)[0, 1]) | |
| def analyse_model(path: str, instances) -> Dict: | |
| blob = json.load(open(path, encoding="utf8")) | |
| res = blob["results"] | |
| by_seed = {i.spec.seed: i for i in instances} | |
| # ---------------- SELECT ---------------- | |
| sel_rows = [] | |
| for row in res["select"]: | |
| inst = by_seed[row["instance"]] | |
| sel_rows.append( | |
| { | |
| "seed": row["instance"], | |
| "pred": row["pred"], | |
| "truth": inst.true_parents, | |
| "n_pred": len(row["pred"]), | |
| } | |
| ) | |
| def agg_set_metric(keyfn): | |
| precs, recs, f1s, jacs, sizes = [], [], [], [], [] | |
| for r in sel_rows: | |
| s = score_selection(keyfn(r), r["truth"]) | |
| precs.append(s["precision"]) | |
| recs.append(s["recall"]) | |
| f1s.append(s["f1"]) | |
| jacs.append(s["jaccard"]) | |
| sizes.append(len(keyfn(r))) | |
| return { | |
| "precision": float(np.mean(precs)), | |
| "recall": float(np.mean(recs)), | |
| "f1": float(np.mean(f1s)), | |
| "jaccard": float(np.mean(jacs)), | |
| "mean_n_selected": float(np.mean(sizes)), | |
| } | |
| lm_sel = agg_set_metric(lambda r: r["pred"]) | |
| corr_sel = agg_set_metric( | |
| lambda r: corr_baseline(by_seed[r["seed"]], SEL_K) | |
| ) | |
| pcorr_sel = agg_set_metric( | |
| lambda r: partial_corr_baseline(by_seed[r["seed"]], SEL_K) | |
| ) | |
| oracle_sel = agg_set_metric(lambda r: oracle(by_seed[r["seed"]], SEL_K)) | |
| # per-variable: how often is each variable called a cause? | |
| per_var_lm = {n: [] for n in CANDIDATE_NAMES} | |
| per_var_corr = {n: [] for n in CANDIDATE_NAMES} | |
| for r in sel_rows: | |
| inst = by_seed[r["seed"]] | |
| pv = per_variable_scores(r["pred"], inst.true_parents, CANDIDATE_NAMES) | |
| for n, v in pv.items(): | |
| per_var_lm[n].append(v["called_cause"]) | |
| pvc = per_variable_scores( | |
| corr_baseline(inst, SEL_K), inst.true_parents, CANDIDATE_NAMES | |
| ) | |
| for n, v in pvc.items(): | |
| per_var_corr[n].append(v["called_cause"]) | |
| # ---------------- INTERVENE ---------------- | |
| per_var_eff = {n: {"pred": [], "true": [], "marg": []} for n in CANDIDATE_NAMES} | |
| parse_fail = 0 | |
| for row in res["intervene"]: | |
| inst = by_seed[row["instance"]] | |
| v = row["variable"] | |
| if row["effect"] is None: | |
| parse_fail += 1 | |
| continue | |
| per_var_eff[v]["pred"].append(float(row["effect"])) | |
| per_var_eff[v]["true"].append(float(inst.do_effects[v])) | |
| per_var_eff[v]["marg"].append(abs(float(inst.marginal_corr[v]))) | |
| eff_summary = {} | |
| all_pred, all_true, all_marg = [], [], [] | |
| for n in CANDIDATE_NAMES: | |
| p = per_var_eff[n]["pred"] | |
| t = per_var_eff[n]["true"] | |
| m = per_var_eff[n]["marg"] | |
| if not p: | |
| continue | |
| true_mean = float(np.mean(t)) | |
| pred_mean = float(np.mean(p)) | |
| # scale of the true effect distribution, used to normalise the error | |
| scale = max(0.1, abs(true_mean)) | |
| eff_summary[n] = { | |
| "true_effect": true_mean, | |
| "pred_effect_mean": pred_mean, | |
| "pred_effect_std": float(np.std(p)), | |
| "mean_abs_error": float(np.mean(np.abs(np.array(p) - np.array(t)))), | |
| "normalised_error": float(np.mean(np.abs(np.array(p) - np.array(t))) / scale), | |
| "sign_correct_rate": float( | |
| np.mean([np.sign(a) == np.sign(b) for a, b in zip(p, t)]) | |
| ), | |
| "is_parent": n in TRUE_PARENTS, | |
| "mean_abs_marginal_corr": float(np.mean(m)), | |
| } | |
| all_pred += p | |
| all_true += t | |
| all_marg += m | |
| # ---------------- WHAT IS IT TRACKING? ---------------- | |
| r_with_true = _pearson(all_pred, all_true) | |
| r_with_corr = _pearson(all_pred, all_marg) | |
| # ---------------- DECOUPLING: causal answer vs predictive answer ---------------- | |
| dec = {"same_set_rate": [], "lm_causal_f1": [], "lm_predictive_f1": []} | |
| for r in sel_rows: | |
| inst = by_seed[r["seed"]] | |
| prow = next( | |
| (x for x in res["predict"] if x["instance"] == r["seed"]), None | |
| ) | |
| if prow is None: | |
| continue | |
| dec["same_set_rate"].append( | |
| float(set(r["pred"]) == set(prow["pred"])) | |
| ) | |
| dec["lm_causal_f1"].append(score_selection(r["pred"], inst.true_parents)["f1"]) | |
| dec["lm_predictive_f1"].append( | |
| score_selection(prow["pred"], inst.true_parents)["f1"] | |
| ) | |
| return { | |
| "label": blob["label"], | |
| "model": blob["model"], | |
| "select": { | |
| "lm": lm_sel, | |
| "corr_baseline": corr_sel, | |
| "partial_corr_baseline": pcorr_sel, | |
| "oracle": oracle_sel, | |
| "per_var_called_rate_lm": {n: float(np.mean(v)) for n, v in per_var_lm.items()}, | |
| "per_var_called_rate_corr": {n: float(np.mean(v)) for n, v in per_var_corr.items()}, | |
| }, | |
| "intervene": eff_summary, | |
| "intervene_parse_failures": parse_fail, | |
| "tracking": { | |
| "r_pred_vs_true_effect": r_with_true, | |
| "r_pred_vs_abs_marginal_corr": r_with_corr, | |
| }, | |
| "decoupling": { | |
| "causal_answer_equals_predictive_answer_rate": float(np.mean(dec["same_set_rate"])), | |
| "mean_f1_causal_question": float(np.mean(dec["lm_causal_f1"])), | |
| "mean_f1_predictive_question": float(np.mean(dec["lm_predictive_f1"])), | |
| }, | |
| } | |
| def analyse_trial_condition(path: str, instances) -> Dict: | |
| """Second condition: the model additionally sees real randomised-trial measurements.""" | |
| from dgp import trial_table | |
| blob = json.load(open(path, encoding="utf8")) | |
| res = blob["results"] | |
| by_seed = {i.spec.seed: i for i in instances} | |
| # ---- selection with trial evidence ---- | |
| sel_f1, sel_prec, sel_rec = [], [], [] | |
| for row in res.get("select_trial", []): | |
| inst = by_seed[row["instance"]] | |
| s = score_selection(row["pred"], inst.true_parents) | |
| sel_f1.append(s["f1"]) | |
| sel_prec.append(s["precision"]) | |
| sel_rec.append(s["recall"]) | |
| # ---- intervention with trial evidence ---- | |
| # Split the (model, variable, instance) triples by whether a trial exists for that | |
| # variable in that instance, and check which evidence source the model tracked. | |
| with_trial = {"pred": [], "trial": [], "true": [], "corr": []} | |
| without_trial = {"pred": [], "true": [], "corr": []} | |
| parse_fail = 0 | |
| for row in res.get("intervene_trial", []): | |
| inst = by_seed[row["instance"]] | |
| v = row["variable"] | |
| if row["effect"] is None: | |
| parse_fail += 1 | |
| continue | |
| pred = float(row["effect"]) | |
| corr = abs(float(inst.marginal_corr[v])) | |
| tt = {t["variable"]: t for t in trial_table(inst)} | |
| if v in tt: | |
| with_trial["pred"].append(pred) | |
| with_trial["trial"].append(float(tt[v]["measured_effect"])) | |
| with_trial["true"].append(float(inst.do_effects[v])) | |
| with_trial["corr"].append(corr) | |
| else: | |
| without_trial["pred"].append(pred) | |
| without_trial["true"].append(float(inst.do_effects[v])) | |
| without_trial["corr"].append(corr) | |
| def block(d): | |
| out = {"n": len(d["pred"])} | |
| if not d["pred"]: | |
| return out | |
| p = np.asarray(d["pred"]) | |
| out["mean_pred"] = float(p.mean()) | |
| if "corr" in d: | |
| out["r_pred_vs_abs_corr"] = _pearson(list(p), d["corr"]) | |
| out["mean_abs_dev_from_abs_corr"] = float(np.mean(np.abs(p - np.asarray(d["corr"])))) | |
| if "true" in d: | |
| out["r_pred_vs_true_effect"] = _pearson(list(p), d["true"]) | |
| out["mean_abs_dev_from_true_effect"] = float( | |
| np.mean(np.abs(p - np.asarray(d["true"]))) | |
| ) | |
| if "trial" in d: | |
| out["r_pred_vs_trial_measurement"] = _pearson(list(p), d["trial"]) | |
| out["mean_abs_dev_from_trial_measurement"] = float( | |
| np.mean(np.abs(p - np.asarray(d["trial"]))) | |
| ) | |
| # did the model actually *use* its own trial measurement for this variable? | |
| dev = np.abs(p - np.asarray(d["trial"])) | |
| out["frac_matching_own_trial_measurement"] = float(np.mean(dev < 0.10)) | |
| out["frac_matching_own_trial_measurement_strict"] = float(np.mean(dev < 0.03)) | |
| return out | |
| return { | |
| "label": blob["label"], | |
| "select_trial": { | |
| "precision": float(np.mean(sel_prec)) if sel_prec else float("nan"), | |
| "recall": float(np.mean(sel_rec)) if sel_rec else float("nan"), | |
| "f1": float(np.mean(sel_f1)) if sel_f1 else float("nan"), | |
| "n": len(sel_f1), | |
| }, | |
| "intervene_trial": { | |
| "with_trial_evidence": block(with_trial), | |
| "without_trial_evidence": block(without_trial), | |
| }, | |
| "intervene_trial_parse_failures": parse_fail, | |
| } | |
| def main(): | |
| instances = make_instances(12) | |
| out = {} | |
| for path in sorted(glob.glob(os.path.join(RESULTS_DIR, "raw_*.json"))): | |
| label = os.path.basename(path)[4:-5] | |
| try: | |
| out[label] = analyse_model(path, instances) | |
| print(f"analysed {label}") | |
| except Exception as e: | |
| print(f"!! failed {label}: {e}") | |
| with open(os.path.join(RESULTS_DIR, "summary.json"), "w", encoding="utf8") as f: | |
| json.dump(out, f, indent=1) | |
| # -------- human readable report -------- | |
| lines = [] | |
| A = lines.append | |
| A("=" * 100) | |
| A("BLIND SPOTS OF FRONTIER MODELS -- RESULTS") | |
| A("=" * 100) | |
| A("") | |
| A("TASK 1: CAUSAL SELECTION (which signals directly cause demand?)") | |
| A(f" ground truth parents per instance: {len(TRUE_PARENTS)} of {len(CANDIDATE_NAMES)} candidates") | |
| A(f" {'model':26s} {'method':26s} {'prec':>7s} {'rec':>7s} {'F1':>7s} {'Jacc':>7s} {'|sel|':>7s}") | |
| for label, r in out.items(): | |
| for key, nm in [ | |
| ("lm", "language model"), | |
| ("corr_baseline", "|corr| top-3"), | |
| ("partial_corr_baseline", "partial-corr top-3"), | |
| ("oracle", "oracle (ground truth)"), | |
| ]: | |
| s = r["select"][key] | |
| A( | |
| f" {label:26s} {nm:26s} {s['precision']:7.3f} {s['recall']:7.3f} " | |
| f"{s['f1']:7.3f} {s['jaccard']:7.3f} {s['mean_n_selected']:7.2f}" | |
| ) | |
| A("") | |
| A("TASK 2: INTERVENTIONAL ESTIMATION (effect of a +1 SD intervention)") | |
| A(f" {'model':26s} {'variable':16s} {'true':>8s} {'pred':>8s} {'MAE':>8s} {'|r|':>7s} {'sign ok':>8s} parent") | |
| for label, r in out.items(): | |
| for v, e in r["intervene"].items(): | |
| A( | |
| f" {label:26s} {v:16s} {e['true_effect']:8.3f} {e['pred_effect_mean']:8.3f} " | |
| f"{e['mean_abs_error']:8.3f} {e['mean_abs_marginal_corr']:7.3f} " | |
| f"{e['sign_correct_rate']:8.2f} {'yes' if e['is_parent'] else 'NO'}" | |
| ) | |
| A("") | |
| A("TASK 3: WHAT IS THE MODEL TRACKING?") | |
| A(" correlation between the model's predicted causal effect and ...") | |
| A(f" {'model':26s} {'true causal effect':>20s} {'|marginal corr|':>18s}") | |
| for label, r in out.items(): | |
| t = r["tracking"] | |
| A(f" {label:26s} {t['r_pred_vs_true_effect']:20.3f} {t['r_pred_vs_abs_marginal_corr']:18.3f}") | |
| A("") | |
| A("TASK 4: CAUSAL vs PREDICTIVE DECOUPLING") | |
| A(f" {'model':26s} {'same set':>10s} {'F1 causal Q':>12s} {'F1 predict Q':>13s}") | |
| for label, r in out.items(): | |
| d = r["decoupling"] | |
| A( | |
| f" {label:26s} {d['causal_answer_equals_predictive_answer_rate']:10.3f} " | |
| f"{d['mean_f1_causal_question']:12.3f} {d['mean_f1_predictive_question']:13.3f}" | |
| ) | |
| report = "\n".join(lines) | |
| print(report) | |
| with open(os.path.join(RESULTS_DIR, "report.txt"), "w", encoding="utf8") as f: | |
| f.write(report + "\n") | |
| # ---------------- randomised-trial condition ---------------- | |
| trial = {} | |
| for path in sorted(glob.glob(os.path.join(RESULTS_DIR, "raw_*.json"))): | |
| label = os.path.basename(path)[4:-5] | |
| try: | |
| trial[label] = analyse_trial_condition(path, instances) | |
| except Exception as e: | |
| print(f"!! trial analysis failed {label}: {e}") | |
| with open(os.path.join(RESULTS_DIR, "summary_trial.json"), "w", encoding="utf8") as f: | |
| json.dump(trial, f, indent=1) | |
| T = [] | |
| B = T.append | |
| B("") | |
| B("=" * 100) | |
| B("CONDITION 2: RANDOMISED-TRIAL EVIDENCE ADDED") | |
| B("=" * 100) | |
| B("") | |
| B("Causal selection, with a real randomised trial supplied for 3 of the 6 signals:") | |
| B(f" {'model':26s} {'F1 (obs only)':>15s} {'F1 (+trial)':>12s} {'precision':>10s} {'recall':>8s}") | |
| for label, r in trial.items(): | |
| base = out.get(label, {}).get("select", {}).get("lm", {}).get("f1", float("nan")) | |
| t = r["select_trial"] | |
| B(f" {label:26s} {base:15.3f} {t['f1']:12.3f} {t['precision']:10.3f} {t['recall']:8.3f}") | |
| B("") | |
| B("Interventional estimation, split by whether a trial measurement existed:") | |
| B(f" {'model':26s} {'subset':>22s} {'r(pred,corr)':>13s} {'r(pred,true)':>13s} {'r(pred,trial)':>14s}") | |
| for label, r in trial.items(): | |
| for key, nm in [ | |
| ("with_trial_evidence", "signals WITH a trial"), | |
| ("without_trial_evidence", "signals WITHOUT a trial"), | |
| ]: | |
| b = r["intervene_trial"][key] | |
| if not b.get("n"): | |
| continue | |
| B( | |
| f" {label:26s} {nm:>22s} {b.get('r_pred_vs_abs_corr', float('nan')):13.3f} " | |
| f"{b.get('r_pred_vs_true_effect', float('nan')):13.3f} " | |
| f"{b.get('r_pred_vs_trial_measurement', float('nan')):14.3f}" | |
| ) | |
| B("") | |
| B("") | |
| B("Trial utilisation (did the model reproduce ITS OWN trial measurement for that variable?):") | |
| B(f" {'model':26s} {'subset':>22s} {'frac |dev|<0.10':>16s} {'frac |dev|<0.03':>16s} {'mean |dev|':>12s}") | |
| for label, r in trial.items(): | |
| for key, nm in [ | |
| ("with_trial_evidence", "signals WITH a trial"), | |
| ("without_trial_evidence", "signals WITHOUT a trial"), | |
| ]: | |
| b = r["intervene_trial"][key] | |
| if not b.get("n"): | |
| continue | |
| B( | |
| f" {label:26s} {nm:>22s} " | |
| f"{b.get('frac_matching_own_trial_measurement', float('nan')):16.3f} " | |
| f"{b.get('frac_matching_own_trial_measurement_strict', float('nan')):16.3f} " | |
| f"{b.get('mean_abs_dev_from_trial_measurement', float('nan')):12.3f}" | |
| ) | |
| B("") | |
| rep2 = "\n".join(T) | |
| print(rep2) | |
| with open(os.path.join(RESULTS_DIR, "report.txt"), "a", encoding="utf8") as f: | |
| f.write(rep2 + "\n") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 16.3 kB
- Xet hash:
- aba6c0c41ee34c3884d7e8dd775e40ec146f4f3ddb1de5c8a564ef3aa52e32f5
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.