{
  "adversarial": {
    "scenarios": {
      "collider_bias": {
        "ate_error": 0.0535808801651001,
        "ate_sigma": 0.24587032198905945,
        "baseline_backdoor_adjustment_error": 0.07940781116485596,
        "baseline_g_computation_error": 0.0839529037475586,
        "baseline_linear_regression_error": 1.0498626232147217,
        "baseline_naive_correlation_error": 1.0376276969909668,
        "baseline_neural_regression_error": 2.398400902748108,
        "baseline_propensity_weighting_error": 0.4118959903717041,
        "causal_slope": 1.1000001448268764,
        "counterfactual_error": 0.07461073994636536,
        "edge_auroc": 0.20000000298023224,
        "edge_f1": 0.0,
        "edge_precision": 0.0,
        "edge_recall": 0.0,
        "error_to_sigma_ratio": 0.2179233334533328,
        "identified": 1.0,
        "ite_error": 0.10452093183994293,
        "marginal_slope": 1.6864241361618042,
        "n_predicted_edges": 4.0,
        "n_true_edges": 5.0,
        "n_variables": 4.0,
        "normalised_ate_error": 0.02874200515882636,
        "normalised_ate_sigma": 0.13189044377839107,
        "normalised_counterfactual_error": 0.04002290118853031,
        "normalised_ite_error": 0.05606741085648537,
        "reversed_edges": 4.0,
        "shd": 5.0,
        "true_ate": 1.9693048000335693
      },
      "confounding": {
        "ate_error": 2.2281384468078613,
        "ate_sigma": 0.1943705826997757,
        "baseline_backdoor_adjustment_error": 2.565906524658203,
        "baseline_g_computation_error": 2.878518581390381,
        "baseline_linear_regression_error": 2.565906524658203,
        "baseline_naive_correlation_error": 2.6589088439941406,
        "baseline_neural_regression_error": 0.04033350944519043,
        "baseline_propensity_weighting_error": 2.565906524658203,
        "causal_slope": 0.8999999460761252,
        "counterfactual_error": 1.5746076107025146,
        "edge_auroc": 0.75,
        "edge_f1": 0.4,
        "edge_precision": 0.3333333333333333,
        "edge_recall": 0.5,
        "error_to_sigma_ratio": 11.463352200005689,
        "identified": 0.0,
        "ite_error": 2.229747772216797,
        "marginal_slope": 1.8474929332733154,
        "n_predicted_edges": 3.0,
        "n_true_edges": 2.0,
        "n_variables": 3.0,
        "normalised_ate_error": 0.8004391092629032,
        "normalised_ate_sigma": 0.06982591961734422,
        "normalised_counterfactual_error": 0.5656639133690219,
        "normalised_ite_error": 0.8010172247886658,
        "reversed_edges": 1.0,
        "shd": 2.0,
        "true_ate": 2.4372920989990234
      },
      "distribution_shift": {
        "ate_error": 4.67472767829895,
        "ate_sigma": 3.688892126083374,
        "baseline_backdoor_adjustment_error": 17.955249071121216,
        "baseline_g_computation_error": 13.376930952072144,
        "baseline_linear_regression_error": 28.42704129219055,
        "baseline_naive_correlation_error": 30.10206913948059,
        "baseline_neural_regression_error": 3.2862675189971924,
        "baseline_propensity_weighting_error": 23.800013780593872,
        "causal_slope": 1.3033428528651472,
        "counterfactual_error": 1.8358383178710938,
        "edge_auroc": 0.5555555820465088,
        "edge_f1": 0.6666666666666666,
        "edge_precision": 0.6666666666666666,
        "edge_recall": 0.6666666666666666,
        "error_to_sigma_ratio": 1.2672443428868365,
        "identified": 1.0,
        "ite_error": 5.052611351013184,
        "marginal_slope": 16.117734909057617,
        "n_predicted_edges": 3.0,
        "n_true_edges": 3.0,
        "n_variables": 3.0,
        "normalised_ate_error": 1.0795293952085245,
        "normalised_ate_sigma": 0.8518715441643328,
        "normalised_counterfactual_error": 0.4239479955575024,
        "normalised_ite_error": 1.1667935848236084,
        "reversed_edges": 1.0,
        "shd": 1.0,
        "true_ate": 2.500958204269409
      },
      "hidden_variables": {
        "ate_error": 2.5442872047424316,
        "ate_sigma": 0.5252423882484436,
        "baseline_backdoor_adjustment_error": 2.57559871673584,
        "baseline_g_computation_error": 2.9756970405578613,
        "baseline_linear_regression_error": 2.57559871673584,
        "baseline_naive_correlation_error": 2.7625694274902344,
        "baseline_neural_regression_error": 2.9572925567626953,
        "baseline_propensity_weighting_error": 2.57559871673584,
        "causal_slope": 0.9974367782446439,
        "counterfactual_error": 1.7296648025512695,
        "edge_auroc": NaN,
        "edge_f1": 0.0,
        "edge_precision": 0.0,
        "edge_recall": 0.0,
        "error_to_sigma_ratio": 4.844024895300272,
        "identified": 0.0,
        "ite_error": 2.5377490520477295,
        "marginal_slope": 2.007148265838623,
        "n_predicted_edges": 1.0,
        "n_true_edges": 0.0,
        "n_variables": 2.0,
        "normalised_ate_error": 0.9220334736203645,
        "normalised_ate_sigma": 0.19034449523885227,
        "normalised_counterfactual_error": 0.6268195049374066,
        "normalised_ite_error": 0.9196640849113464,
        "reversed_edges": 0.0,
        "shd": 1.0,
        "true_ate": 2.5442872047424316
      },
      "nonlinear": {
        "ate_error": 0.173872709274292,
        "ate_sigma": 0.09913791716098785,
        "baseline_backdoor_adjustment_error": 0.6513547897338867,
        "baseline_g_computation_error": 0.9216175079345703,
        "baseline_linear_regression_error": 0.6513547897338867,
        "baseline_naive_correlation_error": 1.2258453369140625,
        "baseline_neural_regression_error": 2.005765378475189,
        "baseline_propensity_weighting_error": 0.6513547897338867,
        "causal_slope": 0.9333865277512824,
        "counterfactual_error": 0.20367702841758728,
        "edge_auroc": 0.875,
        "edge_f1": 0.5,
        "edge_precision": 0.5,
        "edge_recall": 0.5,
        "error_to_sigma_ratio": 1.753846704202429,
        "identified": 1.0,
        "ite_error": 0.2109552025794983,
        "marginal_slope": 1.1372480392456055,
        "n_predicted_edges": 2.0,
        "n_true_edges": 2.0,
        "n_variables": 3.0,
        "normalised_ate_error": 0.12467042747883068,
        "normalised_ate_sigma": 0.07108399336162348,
        "normalised_counterfactual_error": 0.1460407576693404,
        "normalised_ite_error": 0.15125936269760132,
        "reversed_edges": 1.0,
        "shd": 1.0,
        "true_ate": 2.9822487831115723
      },
      "outside_range": {
        "ate_error": 0.2608070373535156,
        "ate_sigma": 5.753205299377441,
        "baseline_backdoor_adjustment_error": 29.526872634887695,
        "baseline_g_computation_error": 133.46777725219727,
        "baseline_linear_regression_error": 14.822834014892578,
        "baseline_naive_correlation_error": 24.001174926757812,
        "baseline_neural_regression_error": 1.5242118835449219,
        "baseline_propensity_weighting_error": 43.52757549285889,
        "causal_slope": 6.159307069261119,
        "counterfactual_error": 7.563058853149414,
        "edge_auroc": 0.8888888955116272,
        "edge_f1": 0.8,
        "edge_precision": 1.0,
        "edge_recall": 0.6666666666666666,
        "error_to_sigma_ratio": 0.04533247533887548,
        "identified": 1.0,
        "ite_error": 4.7379150390625,
        "marginal_slope": 4.536205291748047,
        "n_predicted_edges": 2.0,
        "n_true_edges": 3.0,
        "n_variables": 3.0,
        "normalised_ate_error": 0.045549851192149216,
        "normalised_ate_sigma": 1.0047951463415306,
        "normalised_counterfactual_error": 1.3208853902644846,
        "normalised_ite_error": 0.8274750709533691,
        "reversed_edges": 0.0,
        "shd": 1.0,
        "true_ate": 56.24929428100586
      },
      "selection_bias": {
        "ate_error": 0.13124418258666992,
        "ate_sigma": 0.30816394090652466,
        "baseline_backdoor_adjustment_error": 0.32472920417785645,
        "baseline_g_computation_error": 0.5110855102539062,
        "baseline_linear_regression_error": 0.32472920417785645,
        "baseline_naive_correlation_error": 0.28928041458129883,
        "baseline_neural_regression_error": 0.6534314155578613,
        "baseline_propensity_weighting_error": 0.32472920417785645,
        "causal_slope": 1.200000014507461,
        "counterfactual_error": 0.06795314699411392,
        "edge_auroc": 0.2222222238779068,
        "edge_f1": 0.0,
        "edge_precision": 0.0,
        "edge_recall": 0.0,
        "error_to_sigma_ratio": 0.4258907846277842,
        "identified": 0.0,
        "ite_error": 0.11888636648654938,
        "marginal_slope": 1.068271279335022,
        "n_predicted_edges": 1.0,
        "n_true_edges": 3.0,
        "n_variables": 3.0,
        "normalised_ate_error": 0.14681131025663335,
        "normalised_ate_sigma": 0.3447158650895488,
        "normalised_counterfactual_error": 0.07601320187795306,
        "normalised_ite_error": 0.13298770785331726,
        "reversed_edges": 1.0,
        "shd": 3.0,
        "true_ate": 2.9581568241119385
      },
      "simpsons_paradox": {
        "ate_error": 0.7794182300567627,
        "ate_sigma": 0.27058735489845276,
        "baseline_backdoor_adjustment_error": 0.01840519905090332,
        "baseline_g_computation_error": 0.07550394535064697,
        "baseline_linear_regression_error": 2.463747024536133,
        "baseline_naive_correlation_error": 2.397135704755783,
        "baseline_neural_regression_error": 0.013787031173706055,
        "baseline_propensity_weighting_error": 1.3367897272109985,
        "causal_slope": -1.0000001452680656,
        "counterfactual_error": 0.47993287444114685,
        "edge_auroc": 1.0,
        "edge_f1": 1.0,
        "edge_precision": 1.0,
        "edge_recall": 1.0,
        "error_to_sigma_ratio": 2.88046804829171,
        "identified": 1.0,
        "ite_error": 1.018937110900879,
        "marginal_slope": 0.2009255588054657,
        "n_predicted_edges": 3.0,
        "n_true_edges": 3.0,
        "n_variables": 3.0,
        "normalised_ate_error": 1.0139955220160821,
        "normalised_ate_sigma": 0.3520245685826795,
        "normalised_counterfactual_error": 0.6243756776335451,
        "normalised_ite_error": 1.3256011009216309,
        "reversed_edges": 0.0,
        "shd": 0.0,
        "true_ate": -2.0515401363372803
      }
    },
    "summary": {
      "mean_normalised_ate_error": 0.5202214121818542,
      "mean_sigma_identified": 0.48233312368392944,
      "mean_sigma_not_identified": 0.20162875950336456,
      "overconfident_scenarios": [
        "confounding",
        "hidden_variables"
      ],
      "uncertainty_ordering_correct": false,
      "worst_error_to_sigma_ratio": 11.463352200005689
    }
  },
  "baselines": {
    "backdoor_adjustment": {
      "normalised_ate_error": 0.4930418133735657
    },
    "g_computation": {
      "normalised_ate_error": 0.43038827180862427
    },
    "linear_regression": {
      "normalised_ate_error": 0.5331624746322632
    },
    "naive_correlation": {
      "normalised_ate_error": 0.5140460729598999
    },
    "neural_regression": {
      "normalised_ate_error": 1.0974092483520508
    },
    "propensity_weighting": {
      "normalised_ate_error": 0.5330392122268677
    },
    "proposed_model": {
      "normalised_ate_error": 0.423109233379364
    }
  },
  "baselines_unseen": {
    "backdoor_adjustment": {
      "normalised_ate_error": 0.4284541606903076
    },
    "g_computation": {
      "normalised_ate_error": 0.4658964276313782
    },
    "linear_regression": {
      "normalised_ate_error": 0.46599188446998596
    },
    "naive_correlation": {
      "normalised_ate_error": 0.503264307975769
    },
    "neural_regression": {
      "normalised_ate_error": 1.071964979171753
    },
    "propensity_weighting": {
      "normalised_ate_error": 0.47844383120536804
    },
    "proposed_model": {
      "normalised_ate_error": 0.500436007976532
    }
  },
  "configuration": {
    "counterfactual_rows": 64,
    "edge_threshold": 0.5,
    "held_out_tasks": 256,
    "parameters": 26119098,
    "replicas": 64,
    "seed": 0,
    "unseen_structure_tasks": 128
  },
  "held_out": {
    "ate_bias": -0.0419006422162056,
    "ate_error": 0.4225336015224457,
    "ate_mae": 0.4225336015224457,
    "ate_normalised_mae": 0.423109233379364,
    "ate_normalised_rmse": 0.733408510684967,
    "ate_rmse": 0.7471932172775269,
    "ate_sigma": 0.5646735429763794,
    "calibration_coverage_1sigma": 0.70703125,
    "calibration_coverage_2sigma": 0.91015625,
    "calibration_mean_abs_z": 0.8785681128501892,
    "calibration_mean_sigma": 0.5783447623252869,
    "calibration_uncertainty_error_correlation": 0.4662990868091583,
    "counterfactual_error": 0.28894758224487305,
    "edge_auroc": 0.9193468689918518,
    "edge_f1": 0.7056852579116821,
    "edge_precision": 0.6512736082077026,
    "edge_recall": 0.8075383901596069,
    "identified": 0.765625,
    "ite_error": 0.5081815719604492,
    "n_predicted_edges": 21.75390625,
    "n_true_edges": 14.80078125,
    "n_variables": 11.65625,
    "normalised_ate_error": 0.423109233379364,
    "normalised_ate_sigma": 0.5783447623252869,
    "normalised_counterfactual_error": 0.28902265429496765,
    "normalised_ite_error": 0.5149435997009277,
    "reversed_edges": 1.86328125,
    "shd": 11.30859375,
    "sigma_identification_gap": 0.3878597021102905,
    "sigma_identified": 0.4874401092529297,
    "sigma_not_identified": 0.8752998113632202
  },
  "identification": {
    "held_out": {
      "fraction_backdoor": 0.265625,
      "fraction_identified": 0.765625,
      "n_tasks": 256.0
    },
    "unseen_structures": {
      "fraction_backdoor": 0.3046875,
      "fraction_identified": 0.796875,
      "n_tasks": 128.0
    }
  },
  "real_data": {
    "ihdp": {
      "ate_error": 1.7996239438652992,
      "baseline_linear_adjustment_error": 0.04008984565734863,
      "baseline_naive_difference_error": 0.0023185014724731445,
      "baseline_t_learner_ate_error": 0.025522828102111816,
      "baseline_t_learner_pehe": 0.2680015563964844,
      "description": "747 infants, 25 covariates, known response surfaces",
      "ite_mae": 1.805983543395996,
      "n_samples": 747.0,
      "n_variables": 27.0,
      "pehe": 1.8435747623443604,
      "predicted_ate": 0.042645715177059174,
      "true_ate": 1.8422696590423584
    },
    "sachs": {
      "baseline_correlation_auroc": 0.7375078797340393,
      "description": "853 single-cell measurements of 11 phosphoproteins (log-standardised)",
      "edge_auroc": 0.6540164351463318,
      "edge_f1": 0.2790697674418605,
      "edge_precision": 0.23076923076923078,
      "edge_recall": 0.35294117647058826,
      "n_predicted_edges": 26.0,
      "n_samples": 853.0,
      "n_true_edges": 17.0,
      "reversed_edges": 4.0,
      "shd": 27.0,
      "variants": {
        "log_standardised": {
          "baseline_correlation_auroc": 0.7375078797340393,
          "description": "853 single-cell measurements of 11 phosphoproteins (log-standardised)",
          "edge_auroc": 0.6540164351463318,
          "edge_f1": 0.2790697674418605,
          "edge_precision": 0.23076923076923078,
          "edge_recall": 0.35294117647058826,
          "n_predicted_edges": 26.0,
          "n_samples": 853.0,
          "n_true_edges": 17.0,
          "reversed_edges": 4.0,
          "shd": 27.0
        },
        "standardised": {
          "baseline_correlation_auroc": 0.7305502891540527,
          "description": "853 single-cell measurements of 11 phosphoproteins (standardised)",
          "edge_auroc": 0.6521189212799072,
          "edge_f1": 0.0,
          "edge_precision": 0.0,
          "edge_recall": 0.0,
          "n_predicted_edges": 5.0,
          "n_samples": 853.0,
          "n_true_edges": 17.0,
          "reversed_edges": 4.0,
          "shd": 18.0
        }
      }
    }
  },
  "training": {
    "promoted_stage": "unseen_structures",
    "promoted_step": 22000,
    "validation_score": 1.0260702967643738
  },
  "unseen_structures": {
    "ate_bias": -0.13809093832969666,
    "ate_error": 0.490360826253891,
    "ate_mae": 0.490360826253891,
    "ate_normalised_mae": 0.500436007976532,
    "ate_normalised_rmse": 1.0647010803222656,
    "ate_rmse": 1.0703402757644653,
    "ate_sigma": 0.6500532627105713,
    "calibration_coverage_1sigma": 0.78125,
    "calibration_coverage_2sigma": 0.9375,
    "calibration_mean_abs_z": 0.6925615072250366,
    "calibration_mean_sigma": 0.6671552062034607,
    "calibration_uncertainty_error_correlation": 0.6199653148651123,
    "counterfactual_error": 0.3348301649093628,
    "edge_auroc": 0.9375064969062805,
    "edge_f1": 0.6931359171867371,
    "edge_precision": 0.6278237104415894,
    "edge_recall": 0.8394230008125305,
    "identified": 0.796875,
    "ite_error": 0.5735531449317932,
    "n_predicted_edges": 68.078125,
    "n_true_edges": 38.578125,
    "n_variables": 24.0390625,
    "normalised_ate_error": 0.500436007976532,
    "normalised_ate_sigma": 0.6671552062034607,
    "normalised_counterfactual_error": 0.3433118462562561,
    "normalised_ite_error": 0.5847880244255066,
    "reversed_edges": 5.734375,
    "shd": 43.640625,
    "sigma_identification_gap": 0.8151648640632629,
    "sigma_identified": 0.5015749335289001,
    "sigma_not_identified": 1.316739797592163
  }
}