{
"rows": [
{
"samples": 64,
"model": "additive_full",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.09743589743589744,
"top3": 0.36923076923076925,
"nll": 2.7052516394680084,
"confidence": 0.2678716639535774,
"top3_mass": 0.5446903242724688
},
{
"samples": 64,
"model": "additive_full",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.4375,
"nll": 2.6498329787463515,
"confidence": 0.3139366236646697,
"top3_mass": 0.6062274244402662
},
{
"samples": 64,
"model": "additive_full",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.125,
"top3": 0.35416666666666663,
"nll": 2.7929448974879656,
"confidence": 0.19396763506594805,
"top3_mass": 0.4453588031762193
},
{
"samples": 64,
"model": "additive_full",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.35416666666666663,
"nll": 2.6746201539675853,
"confidence": 0.2854937346369333,
"top3_mass": 0.5671318027034806
},
{
"samples": 64,
"model": "additive_full",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.09803921568627451,
"top3": 0.33333333333333326,
"nll": 2.7037051813052995,
"confidence": 0.27748766253539503,
"top3_mass": 0.5591401525053532
},
{
"samples": 64,
"model": "additive_full",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.30186823992133727,
"top3": 0.6224188790560472,
"nll": 2.189344434424096,
"confidence": 0.2838412004888323,
"top3_mass": 0.566064286224066
},
{
"samples": 64,
"model": "additive_full",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.3490196078431372,
"top3": 0.6509803921568628,
"nll": 2.051049710373143,
"confidence": 0.30972475129987465,
"top3_mass": 0.5912781286604109
},
{
"samples": 64,
"model": "additive_full",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.2352941176470588,
"top3": 0.5215686274509804,
"nll": 2.4919428671027486,
"confidence": 0.2406240820147914,
"top3_mass": 0.5258750504253592
},
{
"samples": 64,
"model": "additive_full",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.3294117647058823,
"top3": 0.6745098039215686,
"nll": 2.1003701481929236,
"confidence": 0.287126521566589,
"top3_mass": 0.5661468685692526
},
{
"samples": 64,
"model": "additive_full",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.2936507936507936,
"top3": 0.6428571428571428,
"nll": 2.113118233189516,
"confidence": 0.2980566881048511,
"top3_mass": 0.5811343926104937
},
{
"samples": 64,
"model": "additive_full",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "additive_full",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.36923076923076925
},
{
"samples": 64,
"model": "additive_full",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.12307692307692308,
"accuracy": 0.2916666666666667
},
{
"samples": 64,
"model": "additive_full",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.020512820512820513,
"accuracy": 0.75
},
{
"samples": 64,
"model": "additive_full",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.0,
"accuracy": null
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.041025641025641026,
"top3": 0.22564102564102564,
"nll": 14.795769929002507,
"confidence": 0.6919640705994234,
"top3_mass": 0.9133101428228636
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.041666666666666664,
"top3": 0.22916666666666666,
"nll": 14.248002393844894,
"confidence": 0.7031485279108601,
"top3_mass": 0.9223906812501139
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.0625,
"top3": 0.16666666666666666,
"nll": 17.996434724255273,
"confidence": 0.6640754970720933,
"top3_mass": 0.8954982491515744
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.041666666666666664,
"top3": 0.20833333333333331,
"nll": 13.638990954862098,
"confidence": 0.681304875436822,
"top3_mass": 0.9183208569810671
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.0196078431372549,
"top3": 0.2941176470588235,
"nll": 13.387658601045098,
"confidence": 0.7177177754262424,
"top3_mass": 0.9168119226683555
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.17010816125860373,
"top3": 0.4475674615284014,
"nll": 10.015167775516005,
"confidence": 0.7065046685261072,
"top3_mass": 0.9289292644422132
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.1843137254901961,
"top3": 0.4511204481792717,
"nll": 9.07373271646867,
"confidence": 0.7072057686074378,
"top3_mass": 0.9265707354876807
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.16862745098039214,
"top3": 0.392156862745098,
"nll": 13.763151722510091,
"confidence": 0.7277271153800143,
"top3_mass": 0.9449059110960412
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.1843137254901961,
"top3": 0.47099874432531635,
"nll": 8.850456574072084,
"confidence": 0.6821135871196693,
"top3_mass": 0.9142404511341209
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.14285714285714285,
"top3": 0.4763321995464852,
"nll": 8.353784402269092,
"confidence": 0.7090015784076789,
"top3_mass": 0.9300127064273054
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.22564102564102564
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.7076923076923077,
"accuracy": 0.021739130434782608
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.517948717948718,
"accuracy": 0.0297029702970297
},
{
"samples": 64,
"model": "additive_diag",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.3282051282051282,
"accuracy": 0.046875
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.23589743589743586,
"top3": 0.6717948717948717,
"nll": 4.429629893875045,
"confidence": 0.6135165953585808,
"top3_mass": 0.9156864830483303
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.3333333333333333,
"top3": 0.75,
"nll": 2.62978894501559,
"confidence": 0.6377474097758559,
"top3_mass": 0.9290275218234602
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.125,
"top3": 0.3958333333333333,
"nll": 9.789497640639205,
"confidence": 0.5621522447279443,
"top3_mass": 0.8904230991650923
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.22916666666666666,
"top3": 0.75,
"nll": 3.1261938221750647,
"confidence": 0.6229632451766559,
"top3_mass": 0.9216198732834624
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.2549019607843137,
"top3": 0.784313725490196,
"nll": 2.305779798623542,
"confidence": 0.6301630766717916,
"top3_mass": 0.9213231464581906
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.5417895771878072,
"top3": 0.775811209439528,
"nll": 2.911979581604918,
"confidence": 0.6532837292329028,
"top3_mass": 0.9268126120284738
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.6392156862745098,
"top3": 0.8509803921568626,
"nll": 1.7004297719310166,
"confidence": 0.6851936552144112,
"top3_mass": 0.9394369865480234
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.37254901960784315,
"top3": 0.5686274509803921,
"nll": 6.128270478418061,
"confidence": 0.5933642366495517,
"top3_mass": 0.9086387886403716
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.596078431372549,
"top3": 0.8352941176470587,
"nll": 1.8773649793574834,
"confidence": 0.6660819512682848,
"top3_mass": 0.9275621238428065
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.5595238095238094,
"top3": 0.8492063492063491,
"nll": 1.9303039718929735,
"confidence": 0.6686761612346782,
"top3_mass": 0.931669691190482
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.6717948717948717
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.6871794871794872,
"accuracy": 0.1865671641791045
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.3128205128205128,
"accuracy": 0.18032786885245902
},
{
"samples": 64,
"model": "factorized_neural",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.1076923076923077,
"accuracy": 0.14285714285714285
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.0071876271876271875,
"top3": 0.07907203907203907,
"nll": 7.578354969750123,
"confidence": 0.4605407143336895,
"top3_mass": 0.6802111846242919
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.005291005291005291,
"top3": 0.06828703703703703,
"nll": 9.111893686770863,
"confidence": 0.5198190196387541,
"top3_mass": 0.7386307777989312
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.0103505291005291,
"top3": 0.09609788359788358,
"nll": 4.928531879241698,
"confidence": 0.31957205655963095,
"top3_mass": 0.5702970321868619
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.005291005291005291,
"top3": 0.06554232804232804,
"nll": 8.621220204043993,
"confidence": 0.5222552226923262,
"top3_mass": 0.7280353083872384
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.00778089013383131,
"top3": 0.08593215063803297,
"nll": 7.647455336050183,
"confidence": 0.4793415087904961,
"top3_mass": 0.6836657121535568
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.4750667228543335,
"top3": 0.6997331085826659,
"nll": 2.762500293635357,
"confidence": 0.4982190413250134,
"top3_mass": 0.7554462606931474
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.5647058823529412,
"top3": 0.7647058823529411,
"nll": 2.3266545418766453,
"confidence": 0.545085683844873,
"top3_mass": 0.7886435789208275
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.3103641456582633,
"top3": 0.520112044817927,
"nll": 3.8074366481933835,
"confidence": 0.40782955431617907,
"top3_mass": 0.7059474674281697
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.5411764705882353,
"top3": 0.7725490196078431,
"nll": 2.3963065146833906,
"confidence": 0.5137204837938891,
"top3_mass": 0.7591793480596293
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.484126984126984,
"top3": 0.7420634920634921,
"nll": 2.5167118409326816,
"confidence": 0.5265740314643992,
"top3_mass": 0.7681642720267111
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.07907203907203907
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.39487179487179486,
"accuracy": 0.0
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.26666666666666666,
"accuracy": 0.0
},
{
"samples": 64,
"model": "joint_neural",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.14871794871794872,
"accuracy": 0.0
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.05128205128205128,
"top3": 0.29029935581659716,
"nll": 42.91473218195748,
"confidence": 0.906467353850722,
"top3_mass": 0.9977711548307759
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.041666666666666664,
"top3": 0.34423747947454847,
"nll": 39.99626509140771,
"confidence": 0.8968154467193972,
"top3_mass": 0.9979913420655012
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.0625,
"top3": 0.213439039408867,
"nll": 58.1222590329112,
"confidence": 0.8962163235731074,
"top3_mass": 0.9992409522973286
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.041666666666666664,
"top3": 0.23719724958949095,
"nll": 39.058661431923205,
"confidence": 0.9227196348307279,
"top3_mass": 0.9988752574498261
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.0588235294117647,
"top3": 0.36185163720660674,
"nll": 34.977801348668386,
"confidence": 0.9099032657838357,
"top3_mass": 0.9951414255881132
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.3126843657817109,
"top3": 0.6100818111803769,
"nll": 22.194156930407917,
"confidence": 0.9123757336666849,
"top3_mass": 0.9977169128173773
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.3647058823529412,
"top3": 0.6696802859074664,
"nll": 17.189823366960677,
"confidence": 0.9074550331620637,
"top3_mass": 0.9977407473992306
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.19215686274509802,
"top3": 0.41045590650053115,
"nll": 39.614162217680416,
"confidence": 0.9252652435448174,
"top3_mass": 0.998643637399928
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.3333333333333333,
"top3": 0.6372259248526997,
"nll": 17.145857676203747,
"confidence": 0.9005945668270137,
"top3_mass": 0.9970133701438374
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.36111111111111105,
"top3": 0.7243089764641488,
"nll": 14.739077645672282,
"confidence": 0.9162334763407756,
"top3_mass": 0.9974669556540026
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.2902993558165972
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.9846153846153847,
"accuracy": 0.052083333333333336
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.882051282051282,
"accuracy": 0.046511627906976744
},
{
"samples": 64,
"model": "factorized_nb",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.6871794871794872,
"accuracy": 0.05970149253731343
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.2358974358974359,
"top3": 0.5948717948717948,
"nll": 4.057728967050472,
"confidence": 0.609057438246369,
"top3_mass": 0.9063522434424671
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.29166666666666663,
"top3": 0.6041666666666666,
"nll": 3.323668952015748,
"confidence": 0.65075730991364,
"top3_mass": 0.9294590404260523
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.16666666666666666,
"top3": 0.5208333333333333,
"nll": 6.681514735094668,
"confidence": 0.5304536465344599,
"top3_mass": 0.8694363322324119
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.20833333333333331,
"top3": 0.6041666666666666,
"nll": 3.3364354155288103,
"confidence": 0.6365700501971762,
"top3_mass": 0.9144966934555481
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.2745098039215686,
"top3": 0.6470588235294117,
"nll": 2.9580280715325316,
"confidence": 0.6178963164523277,
"top3_mass": 0.9116836921138913
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.5073746312684365,
"top3": 0.7826941986234021,
"nll": 2.1541319823271787,
"confidence": 0.6156095729157138,
"top3_mass": 0.9100986736970845
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.6039215686274509,
"top3": 0.8588235294117645,
"nll": 1.407347051037576,
"confidence": 0.6385095395089564,
"top3_mass": 0.9179138049541038
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.3176470588235294,
"top3": 0.592156862745098,
"nll": 4.339448128156216,
"confidence": 0.590129363358726,
"top3_mass": 0.9020575026056967
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.5725490196078431,
"top3": 0.8392156862745097,
"nll": 1.4354558951812746,
"confidence": 0.61505317302631,
"top3_mass": 0.9092340207076758
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.5357142857142857,
"top3": 0.8412698412698412,
"nll": 1.4257071510361057,
"confidence": 0.6187835567552102,
"top3_mass": 0.9112023509592403
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.5948717948717949
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.717948717948718,
"accuracy": 0.17857142857142858
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.29743589743589743,
"accuracy": 0.1206896551724138
},
{
"samples": 64,
"model": "factorized_lda",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.07692307692307693,
"accuracy": 0.0
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.03076923076923077,
"top3": 0.22564102564102564,
"nll": 6.021488392317806,
"confidence": 0.43642609704021906,
"top3_mass": 0.697166444210037
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.041666666666666664,
"top3": 0.2708333333333333,
"nll": 6.981712020464046,
"confidence": 0.46465075636875747,
"top3_mass": 0.7144575314745634
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.0625,
"top3": 0.20833333333333331,
"nll": 4.326893731431082,
"confidence": 0.36421276794761265,
"top3_mass": 0.6397597231587657
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.020833333333333332,
"top3": 0.1875,
"nll": 6.51669265554186,
"confidence": 0.45743183592393977,
"top3_mass": 0.71817466765056
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.0,
"top3": 0.2352941176470588,
"nll": 6.246586528333271,
"confidence": 0.45805709080995777,
"top3_mass": 0.7151498892417754
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.4464110127826942,
"top3": 0.6450344149459193,
"nll": 2.4638715185475624,
"confidence": 0.5093889701399161,
"top3_mass": 0.7685113931493037
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.5333333333333333,
"top3": 0.7294117647058822,
"nll": 2.2561270365241994,
"confidence": 0.5613217412696829,
"top3_mass": 0.8030932390650225
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.2784313725490196,
"top3": 0.4862745098039215,
"nll": 3.0244756629296106,
"confidence": 0.4354934453428863,
"top3_mass": 0.7195433739681395
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.5019607843137255,
"top3": 0.6980392156862744,
"nll": 2.2425292486936304,
"confidence": 0.5176323105985691,
"top3_mass": 0.770024768433547
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.47222222222222215,
"top3": 0.6666666666666665,
"nll": 2.330788442798707,
"confidence": 0.5232717096962479,
"top3_mass": 0.7815374387255202
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.22564102564102564
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.31794871794871793,
"accuracy": 0.0
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.20512820512820512,
"accuracy": 0.0
},
{
"samples": 64,
"model": "factorized_kernel",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.08717948717948718,
"accuracy": 0.0
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.07179487179487179,
"top3": 0.2512820512820513,
"nll": 3.8936374068228163,
"confidence": 0.16161467209769148,
"top3_mass": 0.3986772747416128
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.375,
"nll": 3.757296078944873,
"confidence": 0.17685296555217445,
"top3_mass": 0.4278089705034639
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.0625,
"top3": 0.1875,
"nll": 3.761080894386952,
"confidence": 0.14334166000284088,
"top3_mass": 0.3611179240629979
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.25,
"nll": 3.9339779034302538,
"confidence": 0.16105217758860563,
"top3_mass": 0.3986744969431073
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.0588235294117647,
"top3": 0.196078431372549,
"nll": 4.108750200899399,
"confidence": 0.16500028447364773,
"top3_mass": 0.40661179965010164
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.30383480825958703,
"top3": 0.5457227138643068,
"nll": 2.5416625881982657,
"confidence": 0.17462620939738682,
"top3_mass": 0.416827363526773
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.3529411764705882,
"top3": 0.5921568627450979,
"nll": 2.4167808083696514,
"confidence": 0.18345919742981434,
"top3_mass": 0.4293698144865865
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.2235294117647059,
"top3": 0.44313725490196076,
"nll": 2.7517157811999766,
"confidence": 0.16118413846904117,
"top3_mass": 0.39381336731682276
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.3372549019607843,
"top3": 0.580392156862745,
"nll": 2.4466151952188997,
"confidence": 0.1737226534834257,
"top3_mass": 0.4165671640430074
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.30158730158730157,
"top3": 0.5674603174603174,
"nll": 2.5516561391927057,
"confidence": 0.18020447485976449,
"top3_mass": 0.427686867126555
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.2512820512820513
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.0,
"accuracy": null
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.0,
"accuracy": null
},
{
"samples": 64,
"model": "factorized_ridge",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.0,
"accuracy": null
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.0,
"top3": 0.00761904761904762,
"nll": 5.197826003203163,
"confidence": 0.3723606374759144,
"top3_mass": 0.5871707605841656
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.0,
"top3": 0.004166666666666667,
"nll": 5.890948796150754,
"confidence": 0.44726399817334184,
"top3_mass": 0.6611013832597284
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.0,
"top3": 0.007142857142857143,
"nll": 4.011263751449254,
"confidence": 0.24176444752032095,
"top3_mass": 0.4763038968589209
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.0,
"top3": 0.004166666666666667,
"nll": 5.604214330938199,
"confidence": 0.41917961534023485,
"top3_mass": 0.6255005652718985
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.0,
"top3": 0.014565826330532213,
"nll": 5.279756479505546,
"confidence": 0.38071249761129816,
"top3_mass": 0.5858591712777643
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.4021632251720747,
"top3": 0.6602612726506532,
"nll": 2.324701055360709,
"confidence": 0.4046705309138936,
"top3_mass": 0.6473552229616311
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.4666666666666666,
"top3": 0.7176470588235293,
"nll": 2.1618157990318405,
"confidence": 0.44299805414768795,
"top3_mass": 0.676329430501364
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.27843137254901956,
"top3": 0.5195518207282913,
"nll": 2.710196057618621,
"confidence": 0.33910861645915413,
"top3_mass": 0.6034486108740916
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.4666666666666666,
"top3": 0.7176470588235293,
"nll": 2.1815517829104705,
"confidence": 0.4120192880318859,
"top3_mass": 0.6480524842915467
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.3968253968253968,
"top3": 0.6865079365079364,
"nll": 2.244294623864297,
"confidence": 0.4247928987561676,
"top3_mass": 0.6617598321702111
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.007619047619047619
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.31794871794871793,
"accuracy": 0.0
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.1794871794871795,
"accuracy": 0.0
},
{
"samples": 64,
"model": "joint_gaussian",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.08717948717948718,
"accuracy": 0.0
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.08717948717948718,
"top3": 0.31794871794871793,
"nll": 3.301105921648819,
"confidence": 0.27400361906566534,
"top3_mass": 0.588542218281554
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.10416666666666666,
"top3": 0.4583333333333333,
"nll": 3.298263140192685,
"confidence": 0.2824954017366445,
"top3_mass": 0.6190831790309694
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.22916666666666666,
"nll": 3.0187125972523914,
"confidence": 0.2411529184064826,
"top3_mass": 0.5148486139319096
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.29166666666666663,
"nll": 3.4491611800644035,
"confidence": 0.29396594688447286,
"top3_mass": 0.6174606890194885
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.0784313725490196,
"top3": 0.2941176470588235,
"nll": 3.4302173074718536,
"confidence": 0.27814158628392033,
"top3_mass": 0.6019390868578307
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.38544739429695174,
"top3": 0.6774827925270404,
"nll": 2.182165054708315,
"confidence": 0.3035966808262512,
"top3_mass": 0.6150424322806329
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.4862745098039215,
"top3": 0.7568627450980392,
"nll": 1.944572313843207,
"confidence": 0.32109399843753383,
"top3_mass": 0.6286204629780792
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.26274509803921564,
"top3": 0.5411764705882353,
"nll": 2.5402656701642035,
"confidence": 0.280167009618971,
"top3_mass": 0.5955899176079885
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.4274509803921568,
"top3": 0.7137254901960783,
"nll": 2.050025153801209,
"confidence": 0.3036822726429944,
"top3_mass": 0.6147433983329477
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.3650793650793651,
"top3": 0.6984126984126983,
"nll": 2.193935557528407,
"confidence": 0.30951304691240134,
"top3_mass": 0.621289444464503
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.31794871794871793
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.06153846153846154,
"accuracy": 0.0
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.015384615384615385,
"accuracy": 0.0
},
{
"samples": 64,
"model": "factorized_logistic",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.0,
"accuracy": null
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.05128205128205128,
"top3": 0.36410256410256414,
"nll": 4.819105922086376,
"confidence": 0.5452796687377128,
"top3_mass": 0.8428033475434669
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.041666666666666664,
"top3": 0.375,
"nll": 4.8606523929318834,
"confidence": 0.5514425838686916,
"top3_mass": 0.8578318995890505
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.3125,
"nll": 4.9037881563551435,
"confidence": 0.5713728799058602,
"top3_mass": 0.8449862658843946
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.041666666666666664,
"top3": 0.35416666666666663,
"nll": 4.929448997451223,
"confidence": 0.5299653099187817,
"top3_mass": 0.8403715217167804
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.0392156862745098,
"top3": 0.4117647058823529,
"nll": 4.596450128694255,
"confidence": 0.5293344758151761,
"top3_mass": 0.8288930938401023
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.48672566371681414,
"top3": 0.7236971484759095,
"nll": 2.259576138741716,
"confidence": 0.5526288192581503,
"top3_mass": 0.8417840726549156
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.5999999999999999,
"top3": 0.807843137254902,
"nll": 1.7954573279197075,
"confidence": 0.5716253524234861,
"top3_mass": 0.8516330957637923
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.30196078431372547,
"top3": 0.5333333333333333,
"nll": 3.4299281865664377,
"confidence": 0.5520046453530802,
"top3_mass": 0.8412753194086539
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.5098039215686274,
"top3": 0.776470588235294,
"nll": 1.9204013487611018,
"confidence": 0.5360379957394399,
"top3_mass": 0.8315762847942413
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.5357142857142857,
"top3": 0.7777777777777777,
"nll": 1.8881479483026862,
"confidence": 0.5508260747577671,
"top3_mass": 0.8426619182481898
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.3641025641025641
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.5743589743589743,
"accuracy": 0.03571428571428571
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.23076923076923078,
"accuracy": 0.0
},
{
"samples": 64,
"model": "factorized_classifier",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.03076923076923077,
"accuracy": 0.0
},
{
"samples": 128,
"model": "additive_full",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.12307692307692308,
"top3": 0.4615384615384615,
"nll": 2.786559313564322,
"confidence": 0.3992859131791558,
"top3_mass": 0.7043455811342321
},
{
"samples": 128,
"model": "additive_full",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.10416666666666666,
"top3": 0.5625,
"nll": 2.7766416573568677,
"confidence": 0.44945750644682164,
"top3_mass": 0.7540644800921116
},
{
"samples": 128,
"model": "additive_full",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.14583333333333331,
"top3": 0.4375,
"nll": 2.9223865295824876,
"confidence": 0.31458668891905717,
"top3_mass": 0.6277173215655086
},
{
"samples": 128,
"model": "additive_full",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.10416666666666666,
"top3": 0.3958333333333333,
"nll": 2.7165645306496597,
"confidence": 0.4195958154315265,
"top3_mass": 0.7248188657213583
},
{
"samples": 128,
"model": "additive_full",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.1372549019607843,
"top3": 0.45098039215686275,
"nll": 2.7339336411915696,
"confidence": 0.4126673049345083,
"top3_mass": 0.7104030644506719
},
{
"samples": 128,
"model": "additive_full",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.3746312684365781,
"top3": 0.7059980334316617,
"nll": 1.8892189567509576,
"confidence": 0.384255624706463,
"top3_mass": 0.7062429178122351
},
{
"samples": 128,
"model": "additive_full",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.43529411764705883,
"top3": 0.7607843137254902,
"nll": 1.7079116620086672,
"confidence": 0.41032935945571364,
"top3_mass": 0.7260527032579823
},
{
"samples": 128,
"model": "additive_full",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.29803921568627456,
"top3": 0.5568627450980391,
"nll": 2.273655096425015,
"confidence": 0.3498772823771228,
"top3_mass": 0.6775196892154636
},
{
"samples": 128,
"model": "additive_full",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.39999999999999997,
"top3": 0.7725490196078431,
"nll": 1.778413562099199,
"confidence": 0.38854849684618464,
"top3_mass": 0.7054937405287097
},
{
"samples": 128,
"model": "additive_full",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.365079365079365,
"top3": 0.7341269841269842,
"nll": 1.7957964177771406,
"confidence": 0.3883151189020734,
"top3_mass": 0.7160205694424342
},
{
"samples": 128,
"model": "additive_full",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "additive_full",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.46153846153846156
},
{
"samples": 128,
"model": "additive_full",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.2717948717948718,
"accuracy": 0.16981132075471697
},
{
"samples": 128,
"model": "additive_full",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.12307692307692308,
"accuracy": 0.375
},
{
"samples": 128,
"model": "additive_full",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.035897435897435895,
"accuracy": 0.42857142857142855
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.09743589743589744,
"top3": 0.2774725274725275,
"nll": 18.666995513635634,
"confidence": 0.7759177838492293,
"top3_mass": 0.9632808879997385
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.10416666666666666,
"top3": 0.25148809523809523,
"nll": 16.413653621240517,
"confidence": 0.7942905874941398,
"top3_mass": 0.9616314385766398
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.20833333333333331,
"nll": 28.036218133866175,
"confidence": 0.753586047246348,
"top3_mass": 0.9597221815089552
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.29241071428571425,
"nll": 15.589850625751916,
"confidence": 0.7952216130457916,
"top3_mass": 0.9671585185716636
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.1176470588235294,
"top3": 0.35294117647058815,
"nll": 14.86583237015168,
"confidence": 0.7614755285658488,
"top3_mass": 0.9645331471451097
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.18977384464110125,
"top3": 0.5059905255968729,
"nll": 11.73109733245523,
"confidence": 0.7608781022497457,
"top3_mass": 0.9597249545090075
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.21568627450980388,
"top3": 0.5264512701632377,
"nll": 9.985076979146122,
"confidence": 0.7571324592030254,
"top3_mass": 0.9567159224282744
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.15294117647058825,
"top3": 0.39999999999999997,
"nll": 18.002840662297956,
"confidence": 0.7979183527642478,
"top3_mass": 0.9717485509104878
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.18823529411764706,
"top3": 0.5497005698831257,
"nll": 9.938153503803218,
"confidence": 0.7290551254484098,
"top3_mass": 0.9529962986214127
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.20238095238095236,
"top3": 0.5483081163499882,
"nll": 8.965785099479799,
"confidence": 0.7593889521706041,
"top3_mass": 0.9574118328802219
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.2774725274725275
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.8666666666666667,
"accuracy": 0.09467455621301775
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.6461538461538462,
"accuracy": 0.11904761904761904
},
{
"samples": 128,
"model": "additive_diag",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.4307692307692308,
"accuracy": 0.11904761904761904
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.31794871794871793,
"top3": 0.7282051282051283,
"nll": 4.7858047996485595,
"confidence": 0.6848218064035226,
"top3_mass": 0.9499950626094817
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.47916666666666663,
"top3": 0.8333333333333333,
"nll": 2.6448228254841553,
"confidence": 0.7042986171516052,
"top3_mass": 0.9621928436152363
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.16666666666666666,
"top3": 0.5208333333333333,
"nll": 11.30456105275099,
"confidence": 0.6320063069952321,
"top3_mass": 0.9365293179610614
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.2708333333333333,
"top3": 0.7291666666666666,
"nll": 3.1382140066801463,
"confidence": 0.7157076264068023,
"top3_mass": 0.9483172185710682
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.3529411764705882,
"top3": 0.8235294117647058,
"nll": 2.216220342265395,
"confidence": 0.6871303886688668,
"top3_mass": 0.9527675874857915
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.5968534906588003,
"top3": 0.8456243854473942,
"nll": 2.578519101938684,
"confidence": 0.7385156122596523,
"top3_mass": 0.9622960755638232
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.7137254901960783,
"top3": 0.9137254901960784,
"nll": 1.1724433646372425,
"confidence": 0.7752087518784624,
"top3_mass": 0.9697567410887675
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.3803921568627451,
"top3": 0.6509803921568628,
"nll": 6.204719292900048,
"confidence": 0.6644762071030531,
"top3_mass": 0.946533334730309
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.6666666666666666,
"top3": 0.9176470588235294,
"nll": 1.4361091710284244,
"confidence": 0.7522713994879371,
"top3_mass": 0.9665755392963411
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.626984126984127,
"top3": 0.9007936507936507,
"nll": 1.4879746443943844,
"confidence": 0.7623869295966507,
"top3_mass": 0.9663665753729239
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.7282051282051282
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.8307692307692308,
"accuracy": 0.345679012345679
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.4717948717948718,
"accuracy": 0.2826086956521739
},
{
"samples": 128,
"model": "factorized_neural",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.18461538461538463,
"accuracy": 0.16666666666666666
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.007081807081807082,
"top3": 0.06968864468864469,
"nll": 18.437054959514278,
"confidence": 0.6247180691136638,
"top3_mass": 0.8143641270618712
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.002976190476190476,
"top3": 0.0536954365079365,
"nll": 23.06105667603866,
"confidence": 0.6988595432867891,
"top3_mass": 0.8611707153093504
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.012896825396825396,
"top3": 0.12152777777777776,
"nll": 10.041144850462839,
"confidence": 0.43107548123738904,
"top3_mass": 0.704445215887433
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.006448412698412698,
"top3": 0.04340277777777777,
"nll": 21.676251961205224,
"confidence": 0.718297746928185,
"top3_mass": 0.8710358578673391
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.006069094304388421,
"top3": 0.060690943043884206,
"nll": 18.938430386183544,
"confidence": 0.6491147146559015,
"top3_mass": 0.8204258608232746
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.5832279814580699,
"top3": 0.8049936788874842,
"nll": 2.4396589444730514,
"confidence": 0.648068051250821,
"top3_mass": 0.8701887761740914
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.7019607843137253,
"top3": 0.8862745098039215,
"nll": 1.5172525306583875,
"confidence": 0.6947924428713661,
"top3_mass": 0.8962351249257164
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.36918767507002803,
"top3": 0.6222689075630252,
"nll": 4.549346656652398,
"confidence": 0.5638677634904052,
"top3_mass": 0.8330196009097595
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.6666666666666666,
"top3": 0.8705882352941177,
"nll": 1.7741868946303019,
"confidence": 0.6618197305905147,
"top3_mass": 0.8741011124252351
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.5952380952380952,
"top3": 0.8412698412698412,
"nll": 1.911637681087763,
"confidence": 0.6720746992033813,
"top3_mass": 0.8774851056054348
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.06968864468864469
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.6461538461538462,
"accuracy": 0.0
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.4512820512820513,
"accuracy": 0.0
},
{
"samples": 128,
"model": "joint_neural",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.27692307692307694,
"accuracy": 0.0
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.09743589743589744,
"top3": 0.284331186055324,
"nll": 48.13746092912729,
"confidence": 0.9065427759403374,
"top3_mass": 0.9981097824964309
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.125,
"top3": 0.3428520114942529,
"nll": 45.738489030930296,
"confidence": 0.933956161721246,
"top3_mass": 0.997760486858071
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.041666666666666664,
"top3": 0.19116892446633826,
"nll": 63.03011092291135,
"confidence": 0.9050554136295755,
"top3_mass": 0.9980967245885937
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.23868534482758622,
"nll": 45.19546472036351,
"confidence": 0.908694873527863,
"top3_mass": 0.9993339675231241
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.1372549019607843,
"top3": 0.3598956824108954,
"nll": 39.1476426827054,
"confidence": 0.8801163090625279,
"top3_mass": 0.9972986469853755
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.2772861356932153,
"top3": 0.6474405549016473,
"nll": 21.442833017263453,
"confidence": 0.9294887562809013,
"top3_mass": 0.9988629733857499
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.3137254901960784,
"top3": 0.7102482372259248,
"nll": 15.606794335052992,
"confidence": 0.9253803872789539,
"top3_mass": 0.9985347339373325
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.14901960784313725,
"top3": 0.4464309861875784,
"nll": 41.07799988263615,
"confidence": 0.9321181285361739,
"top3_mass": 0.9992559969678647
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.2980392156862745,
"top3": 0.6857577513764126,
"nll": 15.767704451304635,
"confidence": 0.9283679901300479,
"top3_mass": 0.9987350836737942
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.34920634920634913,
"top3": 0.7485143482680428,
"nll": 13.222119118902842,
"confidence": 0.9321194687367346,
"top3_mass": 0.9989268302208926
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.2843311860553239
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.9897435897435898,
"accuracy": 0.09844559585492228
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.8666666666666667,
"accuracy": 0.07692307692307693
},
{
"samples": 128,
"model": "factorized_nb",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.7128205128205128,
"accuracy": 0.07194244604316546
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.28205128205128205,
"top3": 0.6564102564102565,
"nll": 4.37381785501415,
"confidence": 0.6226594081166319,
"top3_mass": 0.9254941614556081
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.3958333333333333,
"top3": 0.7083333333333333,
"nll": 2.9813366645107,
"confidence": 0.6450391110420814,
"top3_mass": 0.9378408733687948
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.125,
"top3": 0.5625,
"nll": 8.834442313864965,
"confidence": 0.5695354127026289,
"top3_mass": 0.9132466781087576
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.25,
"top3": 0.6666666666666666,
"nll": 3.1956736940163446,
"confidence": 0.6666437977427184,
"top3_mass": 0.9358711599917441
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.35294117647058815,
"top3": 0.6862745098039215,
"nll": 2.5949951657439776,
"confidence": 0.6101981402224831,
"top3_mass": 0.9156341830062225
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.5801376597836774,
"top3": 0.8505408062930186,
"nll": 1.98662723118646,
"confidence": 0.6437062979717997,
"top3_mass": 0.9282823339096133
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.6745098039215686,
"top3": 0.9294117647058823,
"nll": 0.9734109039953873,
"confidence": 0.6733018560167467,
"top3_mass": 0.9387235535803247
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.3647058823529412,
"top3": 0.6509803921568628,
"nll": 4.820057628972093,
"confidence": 0.589226194798963,
"top3_mass": 0.9091790786826228
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.6431372549019607,
"top3": 0.9098039215686274,
"nll": 1.0843282878890155,
"confidence": 0.6550976981148339,
"top3_mass": 0.92876277078777
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.6388888888888888,
"top3": 0.9126984126984127,
"nll": 1.0577845428500907,
"confidence": 0.6573600756350465,
"top3_mass": 0.9365613325719991
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.6564102564102564
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.7230769230769231,
"accuracy": 0.3120567375886525
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.35384615384615387,
"accuracy": 0.3188405797101449
},
{
"samples": 128,
"model": "factorized_lda",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.06666666666666667,
"accuracy": 0.15384615384615385
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.03076923076923077,
"top3": 0.24615384615384617,
"nll": 5.316300051864069,
"confidence": 0.4089796567394144,
"top3_mass": 0.6577881603905575
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.020833333333333332,
"top3": 0.2708333333333333,
"nll": 6.187049752340277,
"confidence": 0.4493025956520194,
"top3_mass": 0.6918674618903067
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.041666666666666664,
"top3": 0.20833333333333331,
"nll": 3.461922371491744,
"confidence": 0.30143660896758695,
"top3_mass": 0.5669963007745659
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.041666666666666664,
"top3": 0.20833333333333331,
"nll": 5.870456106672641,
"confidence": 0.46915596501383805,
"top3_mass": 0.7021096855030582
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.0196078431372549,
"top3": 0.2941176470588235,
"nll": 5.720508922534696,
"confidence": 0.4156091161133427,
"top3_mass": 0.669450309099962
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.5722713864306784,
"top3": 0.7787610619469026,
"nll": 1.7035370484779366,
"confidence": 0.49888559422638545,
"top3_mass": 0.7573332211820869
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.7254901960784315,
"top3": 0.8745098039215686,
"nll": 1.2902660282580372,
"confidence": 0.5657241634206073,
"top3_mass": 0.8019281941633623
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.3333333333333333,
"top3": 0.5686274509803921,
"nll": 2.591686258672908,
"confidence": 0.4063719853839861,
"top3_mass": 0.6929345364305184
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.6196078431372549,
"top3": 0.8431372549019607,
"nll": 1.4628537930179126,
"confidence": 0.5130087715445121,
"top3_mass": 0.7636207627245573
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.611111111111111,
"top3": 0.8293650793650793,
"nll": 1.4665540790281857,
"confidence": 0.5105749787270324,
"top3_mass": 0.771010298198336
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.24615384615384617
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.3128205128205128,
"accuracy": 0.0
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.1641025641025641,
"accuracy": 0.0
},
{
"samples": 128,
"model": "factorized_kernel",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.09743589743589744,
"accuracy": 0.0
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.1076923076923077,
"top3": 0.36410256410256414,
"nll": 3.620308522945126,
"confidence": 0.17084406487121953,
"top3_mass": 0.4103784885167152
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.125,
"top3": 0.47916666666666663,
"nll": 3.33964135933288,
"confidence": 0.1833077082564009,
"top3_mass": 0.43335286071445334
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.14583333333333331,
"top3": 0.35416666666666663,
"nll": 3.716761296733412,
"confidence": 0.15931156467431082,
"top3_mass": 0.3779365525325371
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.375,
"nll": 3.577939539157799,
"confidence": 0.16847233985142074,
"top3_mass": 0.4108801891075339
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.0784313725490196,
"top3": 0.2549019607843137,
"nll": 3.8335634628145705,
"confidence": 0.17219990659500875,
"top3_mass": 0.41881694799494673
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.3215339233038348,
"top3": 0.5899705014749261,
"nll": 2.3604777513523505,
"confidence": 0.1764189173031722,
"top3_mass": 0.40782343230321827
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.36078431372549025,
"top3": 0.6313725490196078,
"nll": 2.208778546676431,
"confidence": 0.1884269554799133,
"top3_mass": 0.42213499987960895
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.23137254901960783,
"top3": 0.48235294117647054,
"nll": 2.6689342104759657,
"confidence": 0.16022324157550283,
"top3_mass": 0.38220544811571566
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.37254901960784315,
"top3": 0.6352941176470588,
"nll": 2.261208907201252,
"confidence": 0.17567131515171852,
"top3_mass": 0.4074165967155741
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.3214285714285714,
"top3": 0.611111111111111,
"nll": 2.3023049552188883,
"confidence": 0.18141290985915393,
"top3_mass": 0.4196761279805311
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.3641025641025641
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.0,
"accuracy": null
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.0,
"accuracy": null
},
{
"samples": 128,
"model": "factorized_ridge",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.0,
"accuracy": null
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.0007326007326007326,
"top3": 0.02808302808302808,
"nll": 8.683776289280036,
"confidence": 0.47664317064361944,
"top3_mass": 0.6904258429615099
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.0,
"top3": 0.01450892857142857,
"nll": 10.463109312827804,
"confidence": 0.5501762141451358,
"top3_mass": 0.7462547130434303
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.002976190476190476,
"top3": 0.04637896825396824,
"nll": 5.468643657090163,
"confidence": 0.32543171680576677,
"top3_mass": 0.6021420340670871
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.0,
"top3": 0.01450892857142857,
"nll": 9.804143820197483,
"confidence": 0.534624723675783,
"top3_mass": 0.7310632699145432
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.0,
"top3": 0.03641456582633053,
"nll": 8.98065353890324,
"confidence": 0.4951813892828406,
"top3_mass": 0.6827246776527751
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.4966989745750808,
"top3": 0.7548461862621154,
"nll": 1.9412243688145847,
"confidence": 0.5113309621708919,
"top3_mass": 0.7733886222881894
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.596078431372549,
"top3": 0.8352941176470587,
"nll": 1.611282662679589,
"confidence": 0.5552280968559002,
"top3_mass": 0.8047972955323657
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.357422969187675,
"top3": 0.5752100840336134,
"nll": 2.618356821210056,
"confidence": 0.4406606310118416,
"top3_mass": 0.7250116019798449
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.5568627450980392,
"top3": 0.8117647058823528,
"nll": 1.7282351983874105,
"confidence": 0.526111620463152,
"top3_mass": 0.778812249009826
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.4761904761904761,
"top3": 0.7976190476190476,
"nll": 1.8054251551737437,
"confidence": 0.5234662686643619,
"top3_mass": 0.7850707797776559
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.02808302808302808
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.39487179487179486,
"accuracy": 0.0
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.3076923076923077,
"accuracy": 0.0
},
{
"samples": 128,
"model": "joint_gaussian",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.16923076923076924,
"accuracy": 0.0
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.1076923076923077,
"top3": 0.4307692307692308,
"nll": 3.7817498590116956,
"confidence": 0.4645334023468606,
"top3_mass": 0.7936232085317145
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.125,
"top3": 0.5833333333333333,
"nll": 3.7440981101843676,
"confidence": 0.47671897075699377,
"top3_mass": 0.8155404893983167
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.10416666666666666,
"top3": 0.3958333333333333,
"nll": 3.2766895816392716,
"confidence": 0.42915523409441214,
"top3_mass": 0.739998888027352
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.35416666666666663,
"nll": 4.004430273633833,
"confidence": 0.48397430952104453,
"top3_mass": 0.8185530107447356
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.1176470588235294,
"top3": 0.392156862745098,
"nll": 4.082956081672977,
"confidence": 0.46806440721098386,
"top3_mass": 0.8000017849314692
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.43461160275319566,
"top3": 0.7482792527040314,
"nll": 1.8028433744980925,
"confidence": 0.4914587870628711,
"top3_mass": 0.8162383481935278
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.5294117647058822,
"top3": 0.8235294117647058,
"nll": 1.4503317429687543,
"confidence": 0.5167145841671837,
"top3_mass": 0.8326853433917029
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.2901960784313725,
"top3": 0.5882352941176471,
"nll": 2.4678163469415884,
"confidence": 0.45086069155978015,
"top3_mass": 0.7924621706356182
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.4666666666666667,
"top3": 0.8117647058823529,
"nll": 1.579481327012168,
"confidence": 0.49522601921199116,
"top3_mass": 0.8183147905705261
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.45238095238095233,
"top3": 0.7698412698412698,
"nll": 1.712683375195713,
"confidence": 0.5031716517202631,
"top3_mass": 0.8215536207951063
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.4307692307692308
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.40512820512820513,
"accuracy": 0.11392405063291139
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.14871794871794872,
"accuracy": 0.0
},
{
"samples": 128,
"model": "factorized_logistic",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.020512820512820513,
"accuracy": 0.0
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.12307692307692308,
"top3": 0.4512820512820513,
"nll": 3.929254435005725,
"confidence": 0.5129046884375597,
"top3_mass": 0.8397141138738512
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.10416666666666666,
"top3": 0.5208333333333333,
"nll": 3.865407385126544,
"confidence": 0.5290677572448904,
"top3_mass": 0.859553154583891
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.125,
"top3": 0.375,
"nll": 4.136536066087745,
"confidence": 0.5199122362266275,
"top3_mass": 0.8330711460726914
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.41666666666666663,
"nll": 3.9905205938832484,
"confidence": 0.5131103216833921,
"top3_mass": 0.8462324391318675
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.17647058823529407,
"top3": 0.4901960784313726,
"nll": 3.736594914930089,
"confidence": 0.4909035121154595,
"top3_mass": 0.8211593861873603
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.6617502458210422,
"top3": 0.8426745329400196,
"nll": 1.5429868239575844,
"confidence": 0.5849239206921237,
"top3_mass": 0.8717717671284061
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.807843137254902,
"top3": 0.9176470588235294,
"nll": 1.1046133442102997,
"confidence": 0.6249101609228995,
"top3_mass": 0.8877857118457841
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.4117647058823529,
"top3": 0.6745098039215686,
"nll": 2.6039072740177707,
"confidence": 0.556564704730889,
"top3_mass": 0.8604317161659223
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "seen",
"day": 11,
"records": 85,
"top1": 0.7176470588235293,
"top3": 0.8823529411764706,
"nll": 1.2577026897055186,
"confidence": 0.5761943098766678,
"top3_mass": 0.867468027192817
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "seen",
"day": 12,
"records": 84,
"top1": 0.7103174603174601,
"top3": 0.8968253968253967,
"nll": 1.2017089541341197,
"confidence": 0.5819920142207278,
"top3_mass": 0.8713972067160143
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "novel_level",
"records_across_seeds": 0,
"top3": null
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "supported",
"records_across_seeds": 195,
"top3": 0.4512820512820513
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "selective_unseen",
"threshold": 0.5,
"coverage": 0.5230769230769231,
"accuracy": 0.1568627450980392
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "selective_unseen",
"threshold": 0.7,
"coverage": 0.11282051282051282,
"accuracy": 0.09090909090909091
},
{
"samples": 128,
"model": "factorized_classifier",
"stratum": "selective_unseen",
"threshold": 0.9,
"coverage": 0.005128205128205128,
"accuracy": 0.0
}
],
"comparisons": [
{
"samples": 64,
"candidate": "additive_full",
"control": "additive_diag",
"gain": 0.14522058823529407,
"fixed_day_simultaneous_interval": [
0.03768344056372547,
0.2637875306372556
],
"day_cluster_simultaneous_interval": [
0.03921568627450983,
0.2083333333333333
],
"day_gains": [
0.2083333333333333,
0.1874999999999999,
0.14583333333333337,
0.03921568627450983
]
},
{
"samples": 64,
"candidate": "additive_full",
"control": "factorized_neural",
"gain": -0.3002450980392156,
"fixed_day_simultaneous_interval": [
-0.46017233455882345,
-0.13357843137254893
],
"day_cluster_simultaneous_interval": [
-0.45098039215686264,
-0.04166666666666674
],
"day_gains": [
-0.31250000000000006,
-0.04166666666666674,
-0.3958333333333333,
-0.45098039215686264
]
},
{
"samples": 64,
"candidate": "additive_full",
"control": "joint_neural",
"gain": 0.2908268168378462,
"fixed_day_simultaneous_interval": [
0.15122954963235294,
0.4490091547035483
],
"day_cluster_simultaneous_interval": [
0.24740118269530031,
0.3692129629629629
],
"day_gains": [
0.3692129629629629,
0.258068783068783,
0.28862433862433867,
0.24740118269530031
]
},
{
"samples": 64,
"candidate": "additive_full",
"control": "factorized_nb",
"gain": 0.08061031524678836,
"fixed_day_simultaneous_interval": [
-0.11089530491614752,
0.2728356448991244
],
"day_cluster_simultaneous_interval": [
-0.02851830387327342,
0.14072762725779958
],
"day_gains": [
0.09326252052545153,
0.14072762725779958,
0.11696941707717573,
-0.02851830387327342
]
},
{
"samples": 64,
"candidate": "additive_full",
"control": "factorized_lda",
"gain": -0.22426470588235292,
"fixed_day_simultaneous_interval": [
-0.3673406862745098,
-0.0854775582107839
],
"day_cluster_simultaneous_interval": [
-0.31372549019607837,
-0.16666666666666663
],
"day_gains": [
-0.16666666666666669,
-0.16666666666666669,
-0.24999999999999994,
-0.31372549019607837
]
},
{
"samples": 64,
"candidate": "additive_full",
"control": "factorized_kernel",
"gain": 0.14430147058823528,
"fixed_day_simultaneous_interval": [
-0.01976122089460783,
0.3167915134803943
],
"day_cluster_simultaneous_interval": [
0.0980392156862745,
0.1666666666666667
],
"day_gains": [
0.16666666666666663,
0.14583333333333326,
0.1666666666666667,
0.0980392156862745
]
},
{
"samples": 64,
"candidate": "additive_full",
"control": "factorized_ridge",
"gain": 0.11764705882352938,
"fixed_day_simultaneous_interval": [
0.02420343137254899,
0.21614602481617665
],
"day_cluster_simultaneous_interval": [
0.062499999999999944,
0.16666666666666657
],
"day_gains": [
0.062499999999999944,
0.16666666666666657,
0.10416666666666669,
0.1372549019607843
]
},
{
"samples": 64,
"candidate": "additive_full",
"control": "joint_gaussian",
"gain": 0.362281162464986,
"fixed_day_simultaneous_interval": [
0.22231461998862043,
0.509790796787465
],
"day_cluster_simultaneous_interval": [
0.3187675070028011,
0.4333333333333333
],
"day_gains": [
0.4333333333333333,
0.34702380952380946,
0.35000000000000003,
0.3187675070028011
]
},
{
"samples": 64,
"candidate": "additive_full",
"control": "factorized_logistic",
"gain": 0.0514705882352941,
"fixed_day_simultaneous_interval": [
-0.05177734375000001,
0.1554842601102943
],
"day_cluster_simultaneous_interval": [
-0.020833333333333426,
0.12499999999999994
],
"day_gains": [
-0.020833333333333426,
0.12499999999999994,
0.06250000000000006,
0.039215686274509776
]
},
{
"samples": 64,
"candidate": "additive_full",
"control": "factorized_classifier",
"gain": 0.006433823529411797,
"fixed_day_simultaneous_interval": [
-0.17769646139705883,
0.1962325750612754
],
"day_cluster_simultaneous_interval": [
-0.0784313725490196,
0.0625
],
"day_gains": [
0.0625,
0.041666666666666574,
5.551115123125783e-17,
-0.0784313725490196
]
},
{
"samples": 64,
"candidate": "factorized_neural",
"control": "additive_full",
"gain": 0.3002450980392156,
"fixed_day_simultaneous_interval": [
0.13357843137254893,
0.46017233455882417
],
"day_cluster_simultaneous_interval": [
0.04166666666666674,
0.45098039215686264
],
"day_gains": [
0.31250000000000006,
0.04166666666666674,
0.3958333333333333,
0.45098039215686264
]
},
{
"samples": 64,
"candidate": "factorized_neural",
"control": "additive_diag",
"gain": 0.44546568627450966,
"fixed_day_simultaneous_interval": [
0.30882161458333335,
0.5879296875000006
],
"day_cluster_simultaneous_interval": [
0.22916666666666663,
0.5416666666666667
],
"day_gains": [
0.5208333333333334,
0.22916666666666663,
0.5416666666666667,
0.4901960784313725
]
},
{
"samples": 64,
"candidate": "factorized_neural",
"control": "joint_neural",
"gain": 0.5910719148770618,
"fixed_day_simultaneous_interval": [
0.4668121908919818,
0.7175138627499611
],
"day_cluster_simultaneous_interval": [
0.29973544973544974,
0.698381574852163
],
"day_gains": [
0.681712962962963,
0.29973544973544974,
0.6844576719576719,
0.698381574852163
]
},
{
"samples": 64,
"candidate": "factorized_neural",
"control": "factorized_nb",
"gain": 0.380855413286004,
"fixed_day_simultaneous_interval": [
0.25141999125215814,
0.5084680903274564
],
"day_cluster_simultaneous_interval": [
0.18239429392446632,
0.512802750410509
],
"day_gains": [
0.4057625205254516,
0.18239429392446632,
0.512802750410509,
0.4224620882835892
]
},
{
"samples": 64,
"candidate": "factorized_neural",
"control": "factorized_lda",
"gain": 0.0759803921568627,
"fixed_day_simultaneous_interval": [
-0.04978726256127447,
0.21599264705882348
],
"day_cluster_simultaneous_interval": [
-0.12499999999999994,
0.14583333333333337
],
"day_gains": [
0.14583333333333337,
-0.12499999999999994,
0.14583333333333337,
0.13725490196078427
]
},
{
"samples": 64,
"candidate": "factorized_neural",
"control": "factorized_kernel",
"gain": 0.4445465686274509,
"fixed_day_simultaneous_interval": [
0.30882238051470584,
0.5882356770833336
],
"day_cluster_simultaneous_interval": [
0.1875,
0.5625
],
"day_gains": [
0.4791666666666667,
0.1875,
0.5625,
0.5490196078431371
]
},
{
"samples": 64,
"candidate": "factorized_neural",
"control": "factorized_ridge",
"gain": 0.417892156862745,
"fixed_day_simultaneous_interval": [
0.2650103400735294,
0.5674019607843137
],
"day_cluster_simultaneous_interval": [
0.20833333333333331,
0.588235294117647
],
"day_gains": [
0.375,
0.20833333333333331,
0.5,
0.588235294117647
]
},
{
"samples": 64,
"candidate": "factorized_neural",
"control": "joint_gaussian",
"gain": 0.6625262605042016,
"fixed_day_simultaneous_interval": [
0.5315407338716737,
0.7929958557641814
],
"day_cluster_simultaneous_interval": [
0.3886904761904762,
0.7697478991596638
],
"day_gains": [
0.7458333333333333,
0.3886904761904762,
0.7458333333333333,
0.7697478991596638
]
},
{
"samples": 64,
"candidate": "factorized_neural",
"control": "factorized_logistic",
"gain": 0.3517156862745097,
"fixed_day_simultaneous_interval": [
0.21231617647058817,
0.49893018535539446
],
"day_cluster_simultaneous_interval": [
0.16666666666666669,
0.4901960784313724
],
"day_gains": [
0.29166666666666663,
0.16666666666666669,
0.45833333333333337,
0.4901960784313724
]
},
{
"samples": 64,
"candidate": "factorized_neural",
"control": "factorized_classifier",
"gain": 0.3066789215686274,
"fixed_day_simultaneous_interval": [
0.14246285232843134,
0.4664523973651962
],
"day_cluster_simultaneous_interval": [
0.08333333333333331,
0.39583333333333337
],
"day_gains": [
0.37500000000000006,
0.08333333333333331,
0.39583333333333337,
0.37254901960784303
]
},
{
"samples": 128,
"candidate": "additive_full",
"control": "additive_diag",
"gain": 0.1854101015406162,
"fixed_day_simultaneous_interval": [
0.030144487482493015,
0.34463663395045546
],
"day_cluster_simultaneous_interval": [
0.0980392156862745,
0.31101190476190466
],
"day_gains": [
0.31101190476190466,
0.22916666666666669,
0.10342261904761901,
0.0980392156862745
]
},
{
"samples": 128,
"candidate": "additive_full",
"control": "factorized_neural",
"gain": -0.2650122549019607,
"fixed_day_simultaneous_interval": [
-0.4399509803921568,
-0.10569738051470483
],
"day_cluster_simultaneous_interval": [
-0.37254901960784303,
-0.08333333333333326
],
"day_gains": [
-0.27083333333333337,
-0.08333333333333326,
-0.3333333333333333,
-0.37254901960784303
]
},
{
"samples": 128,
"candidate": "additive_full",
"control": "joint_neural",
"gain": 0.3918741975957049,
"fixed_day_simultaneous_interval": [
0.23187639736884333,
0.5575254478418984
],
"day_cluster_simultaneous_interval": [
0.3159722222222222,
0.5088045634920634
],
"day_gains": [
0.5088045634920634,
0.3159722222222222,
0.3524305555555555,
0.39028944911297847
]
},
{
"samples": 128,
"candidate": "additive_full",
"control": "factorized_nb",
"gain": 0.17855294057278082,
"fixed_day_simultaneous_interval": [
0.0031240892774135414,
0.36346698434896646
],
"day_cluster_simultaneous_interval": [
0.09108470974596727,
0.24633107553366174
],
"day_gains": [
0.21964798850574702,
0.24633107553366174,
0.1571479885057471,
0.09108470974596727
]
},
{
"samples": 128,
"candidate": "additive_full",
"control": "factorized_lda",
"gain": -0.19424019607843146,
"fixed_day_simultaneous_interval": [
-0.3379296875,
-0.054533930759803596
],
"day_cluster_simultaneous_interval": [
-0.2708333333333333,
-0.125
],
"day_gains": [
-0.14583333333333348,
-0.125,
-0.2708333333333333,
-0.23529411764705888
]
},
{
"samples": 128,
"candidate": "additive_full",
"control": "factorized_kernel",
"gain": 0.2162990196078431,
"fixed_day_simultaneous_interval": [
0.0372233072916667,
0.4010416666666667
],
"day_cluster_simultaneous_interval": [
0.15686274509803916,
0.2916666666666666
],
"day_gains": [
0.2916666666666666,
0.22916666666666669,
0.1875,
0.15686274509803916
]
},
{
"samples": 128,
"candidate": "additive_full",
"control": "factorized_ridge",
"gain": 0.09589460784313725,
"fixed_day_simultaneous_interval": [
-0.050247012867647045,
0.24448606004902032
],
"day_cluster_simultaneous_interval": [
0.02083333333333337,
0.196078431372549
],
"day_gains": [
0.0833333333333332,
0.08333333333333337,
0.02083333333333337,
0.196078431372549
]
},
{
"samples": 128,
"candidate": "additive_full",
"control": "joint_gaussian",
"gain": 0.43375058356676,
"fixed_day_simultaneous_interval": [
0.2698694257976628,
0.5984603867588718
],
"day_cluster_simultaneous_interval": [
0.38132440476190477,
0.5479910714285713
],
"day_gains": [
0.5479910714285713,
0.39112103174603174,
0.38132440476190477,
0.4145658263305322
]
},
{
"samples": 128,
"candidate": "additive_full",
"control": "factorized_logistic",
"gain": 0.030330882352941124,
"fixed_day_simultaneous_interval": [
-0.08961416207107836,
0.1551778875612747
],
"day_cluster_simultaneous_interval": [
-0.02083333333333348,
0.05882352941176472
],
"day_gains": [
-0.02083333333333348,
0.041666666666666685,
0.041666666666666685,
0.05882352941176472
]
},
{
"samples": 128,
"candidate": "additive_full",
"control": "factorized_classifier",
"gain": 0.011029411764705843,
"fixed_day_simultaneous_interval": [
-0.168811274509804,
0.19975643382353078
],
"day_cluster_simultaneous_interval": [
-0.03921568627450989,
0.0625
],
"day_gains": [
0.04166666666666663,
0.0625,
-0.020833333333333315,
-0.03921568627450989
]
},
{
"samples": 128,
"candidate": "factorized_neural",
"control": "additive_full",
"gain": 0.2650122549019607,
"fixed_day_simultaneous_interval": [
0.1056973805147059,
0.4399509803921568
],
"day_cluster_simultaneous_interval": [
0.08333333333333326,
0.37254901960784303
],
"day_gains": [
0.27083333333333337,
0.08333333333333326,
0.3333333333333333,
0.37254901960784303
]
},
{
"samples": 128,
"candidate": "factorized_neural",
"control": "additive_diag",
"gain": 0.4504223564425769,
"fixed_day_simultaneous_interval": [
0.2874636456144958,
0.6043749042585786
],
"day_cluster_simultaneous_interval": [
0.31249999999999994,
0.581845238095238
],
"day_gains": [
0.581845238095238,
0.31249999999999994,
0.43675595238095233,
0.47058823529411753
]
},
{
"samples": 128,
"candidate": "factorized_neural",
"control": "joint_neural",
"gain": 0.6568864524976656,
"fixed_day_simultaneous_interval": [
0.502825503736651,
0.7944206344647236
],
"day_cluster_simultaneous_interval": [
0.39930555555555547,
0.7796378968253967
],
"day_gains": [
0.7796378968253967,
0.39930555555555547,
0.6857638888888888,
0.7628384687208215
]
},
{
"samples": 128,
"candidate": "factorized_neural",
"control": "factorized_nb",
"gain": 0.4435651954747415,
"fixed_day_simultaneous_interval": [
0.292112954691575,
0.6013496142328166
],
"day_cluster_simultaneous_interval": [
0.329664408866995,
0.49048132183908044
],
"day_gains": [
0.4904813218390804,
0.329664408866995,
0.49048132183908044,
0.4636337293538103
]
},
{
"samples": 128,
"candidate": "factorized_neural",
"control": "factorized_lda",
"gain": 0.07077205882352922,
"fixed_day_simultaneous_interval": [
-0.021293083639705945,
0.1786151960784313
],
"day_cluster_simultaneous_interval": [
-0.04166666666666674,
0.13725490196078416
],
"day_gains": [
0.12499999999999989,
-0.04166666666666674,
0.0625,
0.13725490196078416
]
},
{
"samples": 128,
"candidate": "factorized_neural",
"control": "factorized_kernel",
"gain": 0.48131127450980377,
"fixed_day_simultaneous_interval": [
0.31969956341911765,
0.6472133501838246
],
"day_cluster_simultaneous_interval": [
0.31249999999999994,
0.5625
],
"day_gains": [
0.5625,
0.31249999999999994,
0.5208333333333333,
0.5294117647058822
]
},
{
"samples": 128,
"candidate": "factorized_neural",
"control": "factorized_ridge",
"gain": 0.3609068627450979,
"fixed_day_simultaneous_interval": [
0.17601045496323528,
0.540900926776961
],
"day_cluster_simultaneous_interval": [
0.16666666666666663,
0.568627450980392
],
"day_gains": [
0.3541666666666666,
0.16666666666666663,
0.3541666666666667,
0.568627450980392
]
},
{
"samples": 128,
"candidate": "factorized_neural",
"control": "joint_gaussian",
"gain": 0.6987628384687207,
"fixed_day_simultaneous_interval": [
0.5531442964555613,
0.8327207683203491
],
"day_cluster_simultaneous_interval": [
0.474454365079365,
0.8188244047619047
],
"day_gains": [
0.8188244047619047,
0.474454365079365,
0.714657738095238,
0.7871148459383752
]
},
{
"samples": 128,
"candidate": "factorized_neural",
"control": "factorized_logistic",
"gain": 0.2953431372549018,
"fixed_day_simultaneous_interval": [
0.13771426930147046,
0.46201018688725526
],
"day_cluster_simultaneous_interval": [
0.12499999999999994,
0.43137254901960775
],
"day_gains": [
0.2499999999999999,
0.12499999999999994,
0.375,
0.43137254901960775
]
},
{
"samples": 128,
"candidate": "factorized_neural",
"control": "factorized_classifier",
"gain": 0.2760416666666665,
"fixed_day_simultaneous_interval": [
0.1361824065563726,
0.42156977634804027
],
"day_cluster_simultaneous_interval": [
0.14583333333333326,
0.33333333333333315
],
"day_gains": [
0.3125,
0.14583333333333326,
0.3125,
0.33333333333333315
]
}
],
"gates": [
{
"samples": 64,
"candidate": "additive_full",
"top3": 0.36979166666666663,
"substantial_gain": false,
"strongest_available": false,
"learning_specific": null,
"promotion": false,
"strongest_observed_model": "factorized_neural",
"strongest_top3": 0.6700367647058822
},
{
"samples": 64,
"candidate": "factorized_neural",
"top3": 0.6700367647058822,
"substantial_gain": false,
"strongest_available": false,
"learning_specific": true,
"promotion": false,
"strongest_observed_model": "factorized_neural",
"strongest_top3": 0.6700367647058822
},
{
"samples": 128,
"candidate": "additive_full",
"top3": 0.461703431372549,
"substantial_gain": false,
"strongest_available": false,
"learning_specific": null,
"promotion": false,
"strongest_observed_model": "factorized_neural",
"strongest_top3": 0.7267156862745097
},
{
"samples": 128,
"candidate": "factorized_neural",
"top3": 0.7267156862745097,
"substantial_gain": false,
"strongest_available": false,
"learning_specific": true,
"promotion": false,
"strongest_observed_model": "factorized_neural",
"strongest_top3": 0.7267156862745097
}
],
"all_thirteen_days_audited": true,
"complete_source_cohort": false,
"valid_source_days": 8,
"all_four_target_days_retained": true,
"secondary_incomplete_cohort_evaluation": true,
"confirmation_days": [
9,
10,
11,
12
],
"one_physical_platform": true,
"draws": 20000,
"comparison_family": 40,
"original_dynamic_mixture_source_gate_remains_failed": true
}
{
"rows": [
{
"samples": 64,
"model": "additive_full",
"stratum": "unseen",
"day": null,
"records": 65,
"top1": 0.09743589743589744,
"top3": 0.36923076923076925,
"nll": 2.7052516394680084,
"confidence": 0.2678716639535774,
"top3_mass": 0.5446903242724688
},
{
"samples": 64,
"model": "additive_full",
"stratum": "unseen",
"day": 9,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.4375,
"nll": 2.6498329787463515,
"confidence": 0.3139366236646697,
"top3_mass": 0.6062274244402662
},
{
"samples": 64,
"model": "additive_full",
"stratum": "unseen",
"day": 10,
"records": 16,
"top1": 0.125,
"top3": 0.35416666666666663,
"nll": 2.7929448974879656,
"confidence": 0.19396763506594805,
"top3_mass": 0.4453588031762193
},
{
"samples": 64,
"model": "additive_full",
"stratum": "unseen",
"day": 11,
"records": 16,
"top1": 0.08333333333333333,
"top3": 0.35416666666666663,
"nll": 2.6746201539675853,
"confidence": 0.2854937346369333,
"top3_mass": 0.5671318027034806
},
{
"samples": 64,
"model": "additive_full",
"stratum": "unseen",
"day": 12,
"records": 17,
"top1": 0.09803921568627451,
"top3": 0.33333333333333326,
"nll": 2.7037051813052995,
"confidence": 0.27748766253539503,
"top3_mass": 0.5591401525053532
},
{
"samples": 64,
"model": "additive_full",
"stratum": "seen",
"day": null,
"records": 339,
"top1": 0.30186823992133727,
"top3": 0.6224188790560472,
"nll": 2.189344434424096,
"confidence": 0.2838412004888323,
"top3_mass": 0.566064286224066
},
{
"samples": 64,
"model": "additive_full",
"stratum": "seen",
"day": 9,
"records": 85,
"top1": 0.3490196078431372,
"top3": 0.6509803921568628,
"nll": 2.051049710373143,
"confidence": 0.30972475129987465,
"top3_mass": 0.5912781286604109
},
{
"samples": 64,
"model": "additive_full",
"stratum": "seen",
"day": 10,
"records": 85,
"top1": 0.2352941176470588,
"top3": 0.5215686274509804,
"nll": 2.4919428671027486,
"confidence": 0.2406240820147914,
"top3_mass": 0.5258750504253592
},
{
"samples": 64,