Buckets:
| """Summary figures for the HF bucket.""" | |
| from __future__ import annotations | |
| import glob | |
| import json | |
| import os | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| from dgp import CANDIDATE_NAMES, TRUE_PARENTS, make_instances | |
| RESULTS = "results" | |
| FIGS = "hf_bucket/figures" | |
| def fig1_evidence_vs_truth(instances): | |
| """The disconnect: marginal correlation vs true causal effect.""" | |
| fig, axes = plt.subplots(1, 2, figsize=(13, 5.2)) | |
| names = CANDIDATE_NAMES | |
| corr = [float(np.mean([abs(i.marginal_corr[n]) for i in instances])) for n in names] | |
| eff = [float(np.mean([i.do_effects[n] for i in instances])) for n in names] | |
| colors = ["#1f77b4" if n in TRUE_PARENTS else "#d62728" for n in names] | |
| ax = axes[0] | |
| x = np.arange(len(names)) | |
| ax.bar(x - 0.2, corr, 0.4, label="|marginal correlation| (what a correlation model sees)", color="#7fb3d5") | |
| ax.bar(x + 0.2, eff, 0.4, label="true causal effect (what a mechanism model needs)", color=colors) | |
| ax.set_xticks(x) | |
| ax.set_xticklabels(names, rotation=20, ha="right") | |
| ax.set_ylabel("value") | |
| ax.set_title("Observational evidence disagrees with causal truth") | |
| ax.legend(fontsize=9) | |
| ax.grid(axis="y", alpha=0.3) | |
| ax = axes[1] | |
| ax.scatter(corr, eff, c=colors, s=130, zorder=3) | |
| for n, cx, ey in zip(names, corr, eff): | |
| ax.annotate(n, (cx, ey), textcoords="offset points", xytext=(7, 6), fontsize=9) | |
| ax.set_xlabel("|marginal correlation| with target") | |
| ax.set_ylabel("true causal effect on target") | |
| ax.set_title("No monotone relation between the two\n(blue = true parent, red = not a parent)") | |
| ax.grid(alpha=0.3) | |
| fig.tight_layout() | |
| fig.savefig(os.path.join(FIGS, "fig1_evidence_vs_truth.png"), dpi=170) | |
| plt.close(fig) | |
| def fig2_model_tracking(summary, summary_trial): | |
| """Predicted causal effect vs truth and vs correlation, per model.""" | |
| models = list(summary.keys()) | |
| fig, axes = plt.subplots(1, 2, figsize=(13, 5.2)) | |
| ax = axes[0] | |
| w = 0.28 | |
| x = np.arange(len(models)) | |
| r_true = [summary[m]["tracking"]["r_pred_vs_true_effect"] for m in models] | |
| r_corr = [summary[m]["tracking"]["r_pred_vs_abs_marginal_corr"] for m in models] | |
| ax.bar(x - w / 2, r_corr, w, label="corr(model's effect estimate, |marginal corr|)", color="#e07b39") | |
| ax.bar(x + w / 2, r_true, w, label="corr(model's effect estimate, TRUE causal effect)", color="#1f77b4") | |
| ax.set_xticks(x) | |
| ax.set_xticklabels(models, rotation=18, ha="right", fontsize=8) | |
| ax.set_ylabel("Pearson r") | |
| ax.set_ylim(-0.2, 1.05) | |
| ax.set_title("What the model's 'causal effect' estimate tracks") | |
| ax.legend(fontsize=8.5) | |
| ax.grid(axis="y", alpha=0.3) | |
| ax = axes[1] | |
| labels = ["observation only", "+ randomised trial"] | |
| for i, (src, title) in enumerate([(summary, "observational"), (summary_trial, "trial")]): | |
| vals, labs = [], [] | |
| for m in models: | |
| if src is summary: | |
| vals.append(src[m]["select"]["lm"]["f1"]) | |
| else: | |
| vals.append(src[m]["select_trial"]["f1"]) | |
| labs.append(m) | |
| ax.bar(np.arange(len(labs)) + i * 0.4, vals, 0.38, label=labels[i]) | |
| ax.set_xticks(np.arange(len(models)) + 0.2) | |
| ax.set_xticklabels(models, rotation=18, ha="right", fontsize=8) | |
| ax.set_ylabel("F1 against ground-truth causal parents") | |
| ax.set_title("Causal selection quality, and the effect of real interventional data") | |
| ax.legend(fontsize=8.5) | |
| ax.grid(axis="y", alpha=0.3) | |
| fig.tight_layout() | |
| fig.savefig(os.path.join(FIGS, "fig2_model_tracking.png"), dpi=170) | |
| plt.close(fig) | |
| def fig3_per_variable(summary): | |
| """Per-variable: how often is each signal called a cause, LM vs correlation baseline.""" | |
| models = list(summary.keys()) | |
| fig, axes = plt.subplots(1, 2, figsize=(13, 5.2)) | |
| ax = axes[0] | |
| x = np.arange(len(CANDIDATE_NAMES)) | |
| w = 0.8 / max(1, len(models)) | |
| for i, m in enumerate(models): | |
| v = [summary[m]["select"]["per_var_called_rate_lm"][n] for n in CANDIDATE_NAMES] | |
| ax.bar(x + i * w - 0.4 + w / 2, v, w, label=m) | |
| ax.set_xticks(x) | |
| ax.set_xticklabels(CANDIDATE_NAMES, rotation=20, ha="right") | |
| ax.set_ylabel("fraction of instances called a cause") | |
| ax.set_title("How often each signal is called a cause\n(orange = true parent)") | |
| for xi, n in enumerate(CANDIDATE_NAMES): | |
| if n in TRUE_PARENTS: | |
| ax.axvspan(xi - 0.5, xi + 0.5, color="#f4a261", alpha=0.13, zorder=0) | |
| ax.legend(fontsize=7.5) | |
| ax.grid(axis="y", alpha=0.3) | |
| ax = axes[1] | |
| for m in models: | |
| e = summary[m]["intervene"] | |
| xs = [e[n]["true_effect"] for n in CANDIDATE_NAMES] | |
| ys = [e[n]["pred_effect_mean"] for n in CANDIDATE_NAMES] | |
| ax.scatter(xs, ys, s=90, label=m, alpha=0.85) | |
| lim = 2.6 | |
| ax.plot([0, lim], [0, lim], "k--", lw=1, label="perfect (effect = truth)") | |
| ax.set_xlabel("true causal effect") | |
| ax.set_ylabel("model's estimated causal effect") | |
| ax.set_title("Estimated vs true intervention effect") | |
| ax.legend(fontsize=8) | |
| ax.grid(alpha=0.3) | |
| fig.tight_layout() | |
| fig.savefig(os.path.join(FIGS, "fig3_per_variable.png"), dpi=170) | |
| plt.close(fig) | |
| def main(): | |
| os.makedirs(FIGS, exist_ok=True) | |
| instances = make_instances(12) | |
| with open(os.path.join(RESULTS, "summary.json"), encoding="utf8") as f: | |
| summary = json.load(f) | |
| st_path = os.path.join(RESULTS, "summary_trial.json") | |
| summary_trial = json.load(open(st_path, encoding="utf8")) if os.path.exists(st_path) else {} | |
| fig1_evidence_vs_truth(instances) | |
| print("fig1 ok") | |
| if summary: | |
| fig2_model_tracking(summary, summary_trial) | |
| print("fig2 ok") | |
| fig3_per_variable(summary) | |
| print("fig3 ok") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 5.85 kB
- Xet hash:
- 6d5398b41870ada34762622eac37644e6e8b6131628b459659b002228f0cd09b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.