{
 "summary": {
  "split": "test",
  "grouping": "one randomised A/B test = one group",
  "groups": {
   "ab_test/validation": {
    "n_groups": 1689,
    "n_ads": 7568,
    "hash": "baf058e3121e0061"
   },
   "ab_test/test": {
    "n_groups": 1945,
    "n_ads": 8654,
    "hash": "d6360136df563ce6"
   }
  },
  "pairacc": 0.5885,
  "pairacc_ci95": [
   0.5767,
   0.6
  ],
  "pairacc_confw": 0.6274,
  "top1": 0.345,
  "top1_random": 0.2423,
  "top1_decisive_tests": {
   "top1": 0.4861,
   "ci95": [
    0.4241,
    0.5387
   ],
   "random": 0.2661,
   "n_tests": 323,
   "conf": 0.9
  },
  "validation_pairacc": 0.6008,
  "same_groups_baselines_pairacc": {
   "random": 0.5029,
   "tfidf_ridge": 0.5722,
   "lgbm_meta": 0.5389
  },
  "pairwise_calibration": {
   "n_pairs": 16200,
   "logloss": 0.6583,
   "logloss_constant": 0.6931,
   "ece15": 0.0153,
   "mean_p_preferred_wins": 0.5963,
   "observed_preferred_wins": 0.5956
  }
 },
 "report": {
  "snapshot": "upworthy_brx-pipe-0.2.0",
  "dataset": "upworthy",
  "rows": {
   "validation": 7568,
   "test": 8654
  },
  "groups": {
   "ab_test/validation": {
    "n_groups": 1689,
    "n_ads": 7568,
    "hash": "baf058e3121e0061"
   },
   "ab_test/test": {
    "n_groups": 1945,
    "n_ads": 8654,
    "hash": "d6360136df563ce6"
   }
  },
  "results": {
   "ab_test": {
    "propensity_one_lite": {
     "validation": {
      "n_groups": 1689,
      "n_ads": 7568,
      "pairacc_confw": 0.6347,
      "pairacc_confw_ci95": [
       0.6228,
       0.6469
      ],
      "pairacc": 0.6008,
      "pairacc_ci95": [
       0.5884,
       0.6124
      ],
      "pairacc_random": 0.5,
      "pairacc_lift": 0.2016,
      "ndcg3": 0.9134,
      "ndcg3_ci95": [
       0.9091,
       0.9173
      ],
      "ndcg3_random": 0.8711,
      "ndcg3_lift": 0.3279,
      "top1": 0.3508,
      "top1_ci95": [
       0.3271,
       0.3739
      ],
      "top1_random": 0.2418,
      "top1_lift": 0.1438,
      "spearman": 0.242,
      "spearman_ci95": [
       0.2149,
       0.2675
      ],
      "spearman_random": 0.0,
      "spearman_global": 0.4202,
      "calibration": {
       "logloss": 0.6829,
       "kl": 0.3148,
       "logloss_constant_base": 0.6921,
       "brier": 0.1253,
       "brier_constant_base": 0.1298,
       "ece15": 0.0221,
       "p_mean": 0.477,
       "t_mean": 0.477,
       "reliability": [
        {
         "p_mean": 0.3557,
         "t_mean": 0.2969,
         "n": 505
        },
        {
         "p_mean": 0.3925,
         "t_mean": 0.3691,
         "n": 505
        },
        {
         "p_mean": 0.4112,
         "t_mean": 0.4231,
         "n": 505
        },
        {
         "p_mean": 0.4264,
         "t_mean": 0.4424,
         "n": 505
        },
        {
         "p_mean": 0.4381,
         "t_mean": 0.46,
         "n": 505
        },
        {
         "p_mean": 0.4503,
         "t_mean": 0.4508,
         "n": 505
        },
        {
         "p_mean": 0.4618,
         "t_mean": 0.4869,
         "n": 505
        },
        {
         "p_mean": 0.4733,
         "t_mean": 0.4816,
         "n": 505
        },
        {
         "p_mean": 0.4853,
         "t_mean": 0.502,
         "n": 504
        },
        {
         "p_mean": 0.4981,
         "t_mean": 0.5234,
         "n": 504
        },
        {
         "p_mean": 0.5117,
         "t_mean": 0.534,
         "n": 504
        },
        {
         "p_mean": 0.5266,
         "t_mean": 0.5444,
         "n": 504
        },
        {
         "p_mean": 0.5444,
         "t_mean": 0.5295,
         "n": 504
        },
        {
         "p_mean": 0.5664,
         "t_mean": 0.5411,
         "n": 504
        },
        {
         "p_mean": 0.6144,
         "t_mean": 0.5709,
         "n": 504
        }
       ]
      },
      "top1_decisive_tests": {
       "top1": 0.479,
       "ci95": [
        0.4231,
        0.535
       ],
       "random": 0.2553,
       "n_tests": 286,
       "conf": 0.9
      }
     },
     "test": {
      "n_groups": 1945,
      "n_ads": 8654,
      "pairacc_confw": 0.6274,
      "pairacc_confw_ci95": [
       0.6159,
       0.64
      ],
      "pairacc": 0.5885,
      "pairacc_ci95": [
       0.5767,
       0.6
      ],
      "pairacc_random": 0.5,
      "pairacc_lift": 0.1771,
      "ndcg3": 0.9114,
      "ndcg3_ci95": [
       0.9077,
       0.9151
      ],
      "ndcg3_random": 0.8728,
      "ndcg3_lift": 0.3031,
      "top1": 0.345,
      "top1_ci95": [
       0.3254,
       0.365
      ],
      "top1_random": 0.2423,
      "top1_lift": 0.1356,
      "spearman": 0.2189,
      "spearman_ci95": [
       0.1948,
       0.2429
      ],
      "spearman_random": 0.0,
      "spearman_global": 0.3979,
      "calibration": {
       "logloss": 0.683,
       "kl": 0.3233,
       "logloss_constant_base": 0.6922,
       "brier": 0.1286,
       "brier_constant_base": 0.1331,
       "ece15": 0.0178,
       "p_mean": 0.4824,
       "t_mean": 0.4783,
       "reliability": [
        {
         "p_mean": 0.3607,
         "t_mean": 0.3042,
         "n": 577
        },
        {
         "p_mean": 0.3955,
         "t_mean": 0.3809,
         "n": 577
        },
        {
         "p_mean": 0.4147,
         "t_mean": 0.4071,
         "n": 577
        },
        {
         "p_mean": 0.4296,
         "t_mean": 0.4361,
         "n": 577
        },
        {
         "p_mean": 0.4424,
         "t_mean": 0.4532,
         "n": 577
        },
        {
         "p_mean": 0.4547,
         "t_mean": 0.4815,
         "n": 577
        },
        {
         "p_mean": 0.467,
         "t_mean": 0.4829,
         "n": 577
        },
        {
         "p_mean": 0.4793,
         "t_mean": 0.504,
         "n": 577
        },
        {
         "p_mean": 0.4911,
         "t_mean": 0.5028,
         "n": 577
        },
        {
         "p_mean": 0.5038,
         "t_mean": 0.4994,
         "n": 577
        },
        {
         "p_mean": 0.5173,
         "t_mean": 0.506,
         "n": 577
        },
        {
         "p_mean": 0.5322,
         "t_mean": 0.5383,
         "n": 577
        },
        {
         "p_mean": 0.5506,
         "t_mean": 0.5492,
         "n": 577
        },
        {
         "p_mean": 0.5743,
         "t_mean": 0.5534,
         "n": 577
        },
        {
         "p_mean": 0.6233,
         "t_mean": 0.5758,
         "n": 576
        }
       ]
      },
      "top1_decisive_tests": {
       "top1": 0.4861,
       "ci95": [
        0.4241,
        0.5387
       ],
       "random": 0.2661,
       "n_tests": 323,
       "conf": 0.9
      }
     }
    }
   }
  },
  "paired": {
   "ab_test": {
    "primary": "propensity_one_lite"
   }
  }
 }
}