xaheli/causal-blind-spots / code /make_figures.py
xaheli's picture
download
raw
5.85 kB
"""Summary figures for the HF bucket."""
from __future__ import annotations
import glob
import json
import os
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
from dgp import CANDIDATE_NAMES, TRUE_PARENTS, make_instances
RESULTS = "results"
FIGS = "hf_bucket/figures"
def fig1_evidence_vs_truth(instances):
"""The disconnect: marginal correlation vs true causal effect."""
fig, axes = plt.subplots(1, 2, figsize=(13, 5.2))
names = CANDIDATE_NAMES
corr = [float(np.mean([abs(i.marginal_corr[n]) for i in instances])) for n in names]
eff = [float(np.mean([i.do_effects[n] for i in instances])) for n in names]
colors = ["#1f77b4" if n in TRUE_PARENTS else "#d62728" for n in names]
ax = axes[0]
x = np.arange(len(names))
ax.bar(x - 0.2, corr, 0.4, label="|marginal correlation| (what a correlation model sees)", color="#7fb3d5")
ax.bar(x + 0.2, eff, 0.4, label="true causal effect (what a mechanism model needs)", color=colors)
ax.set_xticks(x)
ax.set_xticklabels(names, rotation=20, ha="right")
ax.set_ylabel("value")
ax.set_title("Observational evidence disagrees with causal truth")
ax.legend(fontsize=9)
ax.grid(axis="y", alpha=0.3)
ax = axes[1]
ax.scatter(corr, eff, c=colors, s=130, zorder=3)
for n, cx, ey in zip(names, corr, eff):
ax.annotate(n, (cx, ey), textcoords="offset points", xytext=(7, 6), fontsize=9)
ax.set_xlabel("|marginal correlation| with target")
ax.set_ylabel("true causal effect on target")
ax.set_title("No monotone relation between the two\n(blue = true parent, red = not a parent)")
ax.grid(alpha=0.3)
fig.tight_layout()
fig.savefig(os.path.join(FIGS, "fig1_evidence_vs_truth.png"), dpi=170)
plt.close(fig)
def fig2_model_tracking(summary, summary_trial):
"""Predicted causal effect vs truth and vs correlation, per model."""
models = list(summary.keys())
fig, axes = plt.subplots(1, 2, figsize=(13, 5.2))
ax = axes[0]
w = 0.28
x = np.arange(len(models))
r_true = [summary[m]["tracking"]["r_pred_vs_true_effect"] for m in models]
r_corr = [summary[m]["tracking"]["r_pred_vs_abs_marginal_corr"] for m in models]
ax.bar(x - w / 2, r_corr, w, label="corr(model's effect estimate, |marginal corr|)", color="#e07b39")
ax.bar(x + w / 2, r_true, w, label="corr(model's effect estimate, TRUE causal effect)", color="#1f77b4")
ax.set_xticks(x)
ax.set_xticklabels(models, rotation=18, ha="right", fontsize=8)
ax.set_ylabel("Pearson r")
ax.set_ylim(-0.2, 1.05)
ax.set_title("What the model's 'causal effect' estimate tracks")
ax.legend(fontsize=8.5)
ax.grid(axis="y", alpha=0.3)
ax = axes[1]
labels = ["observation only", "+ randomised trial"]
for i, (src, title) in enumerate([(summary, "observational"), (summary_trial, "trial")]):
vals, labs = [], []
for m in models:
if src is summary:
vals.append(src[m]["select"]["lm"]["f1"])
else:
vals.append(src[m]["select_trial"]["f1"])
labs.append(m)
ax.bar(np.arange(len(labs)) + i * 0.4, vals, 0.38, label=labels[i])
ax.set_xticks(np.arange(len(models)) + 0.2)
ax.set_xticklabels(models, rotation=18, ha="right", fontsize=8)
ax.set_ylabel("F1 against ground-truth causal parents")
ax.set_title("Causal selection quality, and the effect of real interventional data")
ax.legend(fontsize=8.5)
ax.grid(axis="y", alpha=0.3)
fig.tight_layout()
fig.savefig(os.path.join(FIGS, "fig2_model_tracking.png"), dpi=170)
plt.close(fig)
def fig3_per_variable(summary):
"""Per-variable: how often is each signal called a cause, LM vs correlation baseline."""
models = list(summary.keys())
fig, axes = plt.subplots(1, 2, figsize=(13, 5.2))
ax = axes[0]
x = np.arange(len(CANDIDATE_NAMES))
w = 0.8 / max(1, len(models))
for i, m in enumerate(models):
v = [summary[m]["select"]["per_var_called_rate_lm"][n] for n in CANDIDATE_NAMES]
ax.bar(x + i * w - 0.4 + w / 2, v, w, label=m)
ax.set_xticks(x)
ax.set_xticklabels(CANDIDATE_NAMES, rotation=20, ha="right")
ax.set_ylabel("fraction of instances called a cause")
ax.set_title("How often each signal is called a cause\n(orange = true parent)")
for xi, n in enumerate(CANDIDATE_NAMES):
if n in TRUE_PARENTS:
ax.axvspan(xi - 0.5, xi + 0.5, color="#f4a261", alpha=0.13, zorder=0)
ax.legend(fontsize=7.5)
ax.grid(axis="y", alpha=0.3)
ax = axes[1]
for m in models:
e = summary[m]["intervene"]
xs = [e[n]["true_effect"] for n in CANDIDATE_NAMES]
ys = [e[n]["pred_effect_mean"] for n in CANDIDATE_NAMES]
ax.scatter(xs, ys, s=90, label=m, alpha=0.85)
lim = 2.6
ax.plot([0, lim], [0, lim], "k--", lw=1, label="perfect (effect = truth)")
ax.set_xlabel("true causal effect")
ax.set_ylabel("model's estimated causal effect")
ax.set_title("Estimated vs true intervention effect")
ax.legend(fontsize=8)
ax.grid(alpha=0.3)
fig.tight_layout()
fig.savefig(os.path.join(FIGS, "fig3_per_variable.png"), dpi=170)
plt.close(fig)
def main():
os.makedirs(FIGS, exist_ok=True)
instances = make_instances(12)
with open(os.path.join(RESULTS, "summary.json"), encoding="utf8") as f:
summary = json.load(f)
st_path = os.path.join(RESULTS, "summary_trial.json")
summary_trial = json.load(open(st_path, encoding="utf8")) if os.path.exists(st_path) else {}
fig1_evidence_vs_truth(instances)
print("fig1 ok")
if summary:
fig2_model_tracking(summary, summary_trial)
print("fig2 ok")
fig3_per_variable(summary)
print("fig3 ok")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
5.85 kB
·
Xet hash:
6d5398b41870ada34762622eac37644e6e8b6131628b459659b002228f0cd09b

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.