diff --git "a/evaluation.json" "b/evaluation.json" --- "a/evaluation.json" +++ "b/evaluation.json" @@ -1,8443 +1,5604 @@ { - "schema": "sev-behavioral-round-report-v1", - "created_at": "2026-09-25T16:37:28.519871+00:00", - "script_sha256": "5beee8c302040233e2950ebe2d3f8944e84d6202f08b95943d679690bcdad8d8", - "question": "operator_origin", - "suite": { - "path": "/Users/mac/Sev-release-20260925/evals/sev/behavioral-research-v1", - "manifest_sha256": "372d8709cce5fc50262afc5107cd4b3637614ff2b2a7e84a8ccfa4293a4b21c5", - "partition_sha256": { - "calibration": "eebb5ddeb322b16bbb312ddb6927b1d71226b72e4b01d323c698d1e2a3c4f38d", - "development": "bd76cba90825cfb2699ff8a4419c99f7b395541ed1cfe51c4a69305451f659e5" - } - }, - "prediction_files": { - "parent": { - "calibration": { - "path": "/Users/mac/Sev/runs/sev-r2-behavioral-4b-parents-v1/00-kev-4b@485ace8703592fcf405488b262449990824cfed1/calibration/rows.json", - "sha256": "e307a8170f6c1c159e252c8e6c55c88f51e37a784e0a4579bd5625c6742952cf" - }, - "development": { - "path": "/Users/mac/Sev/runs/sev-r2-behavioral-4b-parents-v1/00-kev-4b@485ace8703592fcf405488b262449990824cfed1/development/rows.json", - "sha256": "9ca8eeeb877ea120e5f4c5d54fa72f9bf31f200bbf460e14e6ec85c223162d56" - } - }, - "candidate": { - "calibration": { - "path": "/Users/mac/Sev/runs/sev-r2-research-4b-v1/00-trial-0/calibration/rows.json", - "sha256": "00909b4d6b838c534d8d403d84a578fea1ddaac5e30a2b488c2d1c138c0d05ab" - }, - "development": { - "path": "/Users/mac/Sev/runs/sev-r2-research-4b-v1/00-trial-0/development/rows.json", - "sha256": "994e39a231bc64037f392e606c5fbd3c455f79e68f7b34ba44517238b269b26a" - } - } - }, - "calibration_fits": { + "scope": "One preregistered SwarmTraces continuation; manual source panel is primary, parser-derived development is secondary.", + "models": { "parent": { - "temperature": 4.0, - "split": "calibration", - "view": "full", - "windows": 182, - "episodes": 96, - "method": "kev.metrics.fit_temperature; macro task=episode; 121-point log grid 0.25..4", - "weighting": "equal episodes, then equal windows within each episode", - "experiment_fit_relationship": "Diagnostic refit; may differ from the experiment's ordinary row/task-weighted fit." - }, - "candidate": { - "temperature": 1.9999999999999998, - "split": "calibration", - "view": "full", - "windows": 182, - "episodes": 96, - "method": "kev.metrics.fit_temperature; macro task=episode; 121-point log grid 0.25..4", - "weighting": "equal episodes, then equal windows within each episode", - "experiment_fit_relationship": "Diagnostic refit; may differ from the experiment's ordinary row/task-weighted fit." - } - }, - "primary_panel": "development/full/iid/all", - "primary_metrics": [ - "acc", - "nll" - ], - "secondary_panels": [ - "development/full/heldout_task/all", - "development/full/all/all", - "loss views and transaction-count panels" - ], - "reporting_corrections": [ - "Earlier reporter metadata named combined development as primary. The preregistered primary is full-view IID development accuracy/NLL. All panel calculations remain unchanged; the earlier parent report is preserved." - ], - "test_read": false, - "panels": { - "calibration": { - "full/all/all": { - "windows": 182, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 36, - "2": 13, - "3": 24, - "4": 26, - "5": 41, - "6": 38, - "7": 4 + "temperature": 1.3660402567543952, + "domains": { + "excytin": { + "raw": { + "n": 398, + "accuracy": 1.0, + "nll": 0.002029807853842489, + "brier": 0.00013267567616055285, + "correct": 398 + }, + "global_calibrated": { + "n": 398, + "accuracy": 1.0, + "nll": 0.00820502390654013, + "brier": 0.0006264068412257906, + "correct": 398 + } }, - "raw": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.7039948885030416, - "brier": 0.9366734114222882, - "ece": 0.3836640270561081 - }, - "candidate": { - "acc": 0.3342013888888889, - "nll": 1.0988623728398248, - "brier": 0.6670543370614523, - "ece": 0.06554797243603253 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0008680555555555802, - "ci95": [ - -0.0078125, - 0.010416666666666685 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3255208333333333, - 0.34375 - ] - }, - "nll": { - "candidate_minus_parent": -0.6051325156632168, - "ci95": [ - -0.6803591005740874, - -0.530793390037944 - ], - "parent_ci95": [ - 1.6258755139905854, - 1.7835939337351452 - ], - "candidate_ci95": [ - 1.0872959592876683, - 1.1104023477998943 - ] - }, - "brier": { - "candidate_minus_parent": -0.2696190743608359, - "ci95": [ - -0.2966854808852652, - -0.24314883056422498 - ], - "parent_ci95": [ - 0.9086695403765366, - 0.9664303635041863 - ], - "candidate_ci95": [ - 0.6585256055156145, - 0.6756684634800705 - ] - }, - "ece": { - "candidate_minus_parent": -0.31811605462007553, - "ci95": [ - -0.3467234011742797, - -0.2776120036720403 - ], - "parent_ci95": [ - 0.3658444671289674, - 0.4154924939910463 - ], - "candidate_ci95": [ - 0.055802842793600815, - 0.10411501636343842 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + "guide": { + "raw": { + "n": 231, + "accuracy": 0.5497835497835498, + "nll": 0.7756690534124361, + "brier": 0.479220229400239, + "macro_f1": 0.4478062632890602, + "class_order": [ + "BenignPositive", + "FalsePositive", + "TruePositive" + ], + "per_class": { + "BenignPositive": { + "precision": 0.2987012987012987, + "recall": 0.3898305084745763, + "f1-score": 0.3382352941176471, + "support": 59.0 + }, + "FalsePositive": { + "precision": 0.6956521739130435, + "recall": 0.7111111111111111, + "f1-score": 0.7032967032967034, + "support": 135.0 + }, + "TruePositive": { + "precision": 0.5, + "recall": 0.21621621621621623, + "f1-score": 0.3018867924528302, + "support": 37.0 + } + }, + "confusion": [ + [ + 23, + 33, + 3 + ], + [ + 34, + 96, + 5 + ], + [ + 20, + 9, + 8 + ] + ], + "equal_organization_accuracy": 0.6445652173913043, + "organization_results": [ + { + "group": "0d1b7eba9c4d5a5852ef2805e5f1c1018c827e36bfb922fefa4d0a92d76f322c", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "108482877637fd1bbe809b2faddd12c9affef276cdb170667c0234a670019cd4", + "n": 20, + "accuracy": 0.4, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "13a6e1ec428dca68e3cc729ac008e19870e5caa60b3f2d8d013eeb5a4f25aa71", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "17515941d0c83a2c0515166a1503db5e3d24c63b53c0398a9c4f3345d3ef5eda", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "22e3ca4ff2df3a13f6db322d247b7b7f8742181d456be68cb84fb4dfbda7b993", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5667ea75698bc671a6165fb7604bf6d7b7874ba2ef04005f82edf2811b406b53", + "n": 20, + "accuracy": 0.6, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "5cd724f6f8c622cf83c9ee56eeb3e956e4c0428f4bb398608581d51b9f3bd4cf", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5fd9d56568dd7e69c4871b023d3df17203f2eb6d9aaf7a808371a391407f9026", + "n": 20, + "accuracy": 0.05, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "66288757ecaf36bf368cd8d4c43f454658be762a12957c52eae438ca3e727ad8", + "n": 8, + "accuracy": 0.375, + "true_positive_n": 6, + "true_positive_correct": 1 + }, + { + "group": "6d53f455fa642b810781930d63db6b35443da067812f1bf329a37de938179aed", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "79707d8a1f7240a758d67ba5212ab21ef286647a4b71ba71c892f88341f3f4e6", + "n": 20, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 1 + }, + { + "group": "7b4cb8f8c017bd94da14e9460c49cdf44fa47f6eab9726918b70ebb7b2695ee1", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "7f55e216b81a67b2fcebf627b534d1d18d9d734f31cb41477725aaa041f6d42b", + "n": 20, + "accuracy": 0.55, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "974cf500c91fb4e80273489c70f89803fe8eaf2fc9d4e68a6aac899d0e580551", + "n": 20, + "accuracy": 0.15, + "true_positive_n": 18, + "true_positive_correct": 1 + }, + { + "group": "97dbafe0dc6d23395bb2008b78ccf190cf0710c18154cbbfc664b3d388086baf", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a756d4fa124cf854b444faed605f6fee522c5418d960b8cd60a8a06c44ef56b0", + "n": 5, + "accuracy": 0.6, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "adcffe87f88661c654325a95ccbc252777e27ff83f947ff85ea77cf167ad57c0", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b4edbf01a09fe17e2fc4eba076213094ab09b40f0ff73c9d3e5c9592f0f59353", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "bfe24bf96507627d9c2e0eb183e0cb607aa50a5195abded30b612c252216e7b0", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cbdba8017701da737b2f23ee11598f766b99054550311935a5730acb5f7e57d9", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cc3696bd6f2c8a2d65d8376338102471db2565326ab124811ce53c428e54425e", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 2 + }, + { + "group": "d592c0767849012441fe14ab7451f4371421ba152f26c3587463adcf58b911e5", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "f562314a5da425b50c820fa25a6756b09f54b042ff1a3a4b580647dd4c20aa7d", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + } + ], + "correct": 127 + }, + "global_calibrated": { + "n": 231, + "accuracy": 0.5497835497835498, + "nll": 0.7508521265465798, + "brier": 0.4566593103734014, + "macro_f1": 0.4478062632890602, + "class_order": [ + "BenignPositive", + "FalsePositive", + "TruePositive" + ], + "per_class": { + "BenignPositive": { + "precision": 0.2987012987012987, + "recall": 0.3898305084745763, + "f1-score": 0.3382352941176471, + "support": 59.0 + }, + "FalsePositive": { + "precision": 0.6956521739130435, + "recall": 0.7111111111111111, + "f1-score": 0.7032967032967034, + "support": 135.0 + }, + "TruePositive": { + "precision": 0.5, + "recall": 0.21621621621621623, + "f1-score": 0.3018867924528302, + "support": 37.0 + } + }, + "confusion": [ + [ + 23, + 33, + 3 + ], + [ + 34, + 96, + 5 + ], + [ + 20, + 9, + 8 + ] + ], + "equal_organization_accuracy": 0.6445652173913043, + "organization_results": [ + { + "group": "0d1b7eba9c4d5a5852ef2805e5f1c1018c827e36bfb922fefa4d0a92d76f322c", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "108482877637fd1bbe809b2faddd12c9affef276cdb170667c0234a670019cd4", + "n": 20, + "accuracy": 0.4, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "13a6e1ec428dca68e3cc729ac008e19870e5caa60b3f2d8d013eeb5a4f25aa71", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "17515941d0c83a2c0515166a1503db5e3d24c63b53c0398a9c4f3345d3ef5eda", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "22e3ca4ff2df3a13f6db322d247b7b7f8742181d456be68cb84fb4dfbda7b993", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5667ea75698bc671a6165fb7604bf6d7b7874ba2ef04005f82edf2811b406b53", + "n": 20, + "accuracy": 0.6, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "5cd724f6f8c622cf83c9ee56eeb3e956e4c0428f4bb398608581d51b9f3bd4cf", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5fd9d56568dd7e69c4871b023d3df17203f2eb6d9aaf7a808371a391407f9026", + "n": 20, + "accuracy": 0.05, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "66288757ecaf36bf368cd8d4c43f454658be762a12957c52eae438ca3e727ad8", + "n": 8, + "accuracy": 0.375, + "true_positive_n": 6, + "true_positive_correct": 1 + }, + { + "group": "6d53f455fa642b810781930d63db6b35443da067812f1bf329a37de938179aed", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "79707d8a1f7240a758d67ba5212ab21ef286647a4b71ba71c892f88341f3f4e6", + "n": 20, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 1 + }, + { + "group": "7b4cb8f8c017bd94da14e9460c49cdf44fa47f6eab9726918b70ebb7b2695ee1", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "7f55e216b81a67b2fcebf627b534d1d18d9d734f31cb41477725aaa041f6d42b", + "n": 20, + "accuracy": 0.55, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "974cf500c91fb4e80273489c70f89803fe8eaf2fc9d4e68a6aac899d0e580551", + "n": 20, + "accuracy": 0.15, + "true_positive_n": 18, + "true_positive_correct": 1 + }, + { + "group": "97dbafe0dc6d23395bb2008b78ccf190cf0710c18154cbbfc664b3d388086baf", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a756d4fa124cf854b444faed605f6fee522c5418d960b8cd60a8a06c44ef56b0", + "n": 5, + "accuracy": 0.6, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "adcffe87f88661c654325a95ccbc252777e27ff83f947ff85ea77cf167ad57c0", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b4edbf01a09fe17e2fc4eba076213094ab09b40f0ff73c9d3e5c9592f0f59353", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "bfe24bf96507627d9c2e0eb183e0cb607aa50a5195abded30b612c252216e7b0", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cbdba8017701da737b2f23ee11598f766b99054550311935a5730acb5f7e57d9", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cc3696bd6f2c8a2d65d8376338102471db2565326ab124811ce53c428e54425e", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 2 + }, + { + "group": "d592c0767849012441fe14ab7451f4371421ba152f26c3587463adcf58b911e5", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "f562314a5da425b50c820fa25a6756b09f54b042ff1a3a4b580647dd4c20aa7d", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + } + ], + "correct": 127 } }, - "calibrated": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.1516241769020001, - "brier": 0.7011284844520596, - "ece": 0.1126117849166719 - }, - "candidate": { - "acc": 0.3342013888888889, - "nll": 1.094225810223573, - "brier": 0.6635708549658472, - "ece": 0.04216089485176933 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0008680555555555802, - "ci95": [ - -0.0078125, - 0.010416666666666685 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3255208333333333, - 0.34375 - ] - }, - "nll": { - "candidate_minus_parent": -0.05739836667842724, - "ci95": [ - -0.06962603452379744, - -0.04488653163402823 - ], - "parent_ci95": [ - 1.1374408274590326, - 1.166659736233086 - ], - "candidate_ci95": [ - 1.0877924378749446, - 1.1001794102062266 - ] - }, - "brier": { - "candidate_minus_parent": -0.03755762948621233, - "ci95": [ - -0.04590448711935209, - -0.02933488075617265 - ], - "parent_ci95": [ - 0.6916327369871181, - 0.7113507865114055 - ], - "candidate_ci95": [ - 0.6589866278321939, - 0.6679576727482305 - ] - }, - "ece": { - "candidate_minus_parent": -0.07045089006490257, - "ci95": [ - -0.10204058462593617, - -0.03478029248966403 - ], - "parent_ci95": [ - 0.10663147728974164, - 0.13809590546759193 - ], - "candidate_ci95": [ - 0.02485568463820158, - 0.07566025500259763 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + "native": { + "raw": { + "n": 410, + "accuracy": 0.9975609756097561, + "nll": 0.012999284410721951, + "brier": 0.004191458920366933, + "correct": 409 + }, + "global_calibrated": { + "n": 410, + "accuracy": 0.9975609756097561, + "nll": 0.030200841306362223, + "brier": 0.006697687496008216, + "correct": 409 } - } - }, - "full/all/transactions_1_2": { - "windows": 49, - "episodes": 45, - "latent_groups": 26, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 36, - "2": 13 }, - "raw": { - "parent": { - "acc": 0.3111111111111111, - "nll": 1.6603323212661212, - "brier": 0.8907029573817137, - "ece": 0.37799735259416234 - }, - "candidate": { - "acc": 0.3888888888888889, - "nll": 1.1006948393144855, - "brier": 0.6675216739911485, - "ece": 0.02197163304242618 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.07777777777777778, - "ci95": [ - -0.12229166666666667, - 0.26746313102665914 - ], - "parent_ci95": [ - 0.20512820512820512, - 0.425 - ], - "candidate_ci95": [ - 0.28888888888888886, - 0.48984374999999974 - ] - }, - "nll": { - "candidate_minus_parent": -0.5596374819516357, - "ci95": [ - -0.8551553776743378, - -0.2835412394003477 - ], - "parent_ci95": [ - 1.3960780027038617, - 1.9415518588113363 - ], - "candidate_ci95": [ - 1.07961907447766, - 1.1190005382885808 - ] - }, - "brier": { - "candidate_minus_parent": -0.2231812833905652, - "ci95": [ - -0.36429527240524406, - -0.07974528756374481 - ], - "parent_ci95": [ - 0.7571302612447247, - 1.0218668962200836 - ], - "candidate_ci95": [ - 0.6529239682286327, - 0.6800289575226035 - ] - }, - "ece": { - "candidate_minus_parent": -0.3560257195517362, - "ci95": [ - -0.4212366104265594, - -0.19688232190049682 - ], - "parent_ci95": [ - 0.27558167651761906, - 0.4812973135389953 - ], - "candidate_ci95": [ - 0.005189460689758951, - 0.13286403298320185 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 26, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + "guide_detector": { + "raw": { + "n": 1064, + "accuracy": 0.5855263157894737, + "nll": 0.8173590431098086, + "brier": 0.47423653600178534, + "macro_f1": 0.5415438082741008, + "class_order": [ + "BenignPositive", + "FalsePositive", + "TruePositive" + ], + "per_class": { + "BenignPositive": { + "precision": 0.4096153846153846, + "recall": 0.7580071174377224, + "f1-score": 0.5318352059925093, + "support": 281.0 + }, + "FalsePositive": { + "precision": 0.7873831775700935, + "recall": 0.6686507936507936, + "f1-score": 0.723175965665236, + "support": 504.0 + }, + "TruePositive": { + "precision": 0.6293103448275862, + "recall": 0.2616487455197133, + "f1-score": 0.369620253164557, + "support": 279.0 + } + }, + "confusion": [ + [ + 213, + 60, + 8 + ], + [ + 132, + 337, + 35 + ], + [ + 175, + 31, + 73 + ] + ], + "equal_organization_accuracy": 0.649889118396736, + "organization_results": [ + { + "group": "0228d088db03da24994da8d12bb80e2925bf424387c12a0cab440661f61e53db", + "n": 2, + "accuracy": 0.0, + "true_positive_n": 2, + "true_positive_correct": 0 + }, + { + "group": "03b2cff809e7692ca15f889834563f866b356cc60a41023c8611dd2b60bcf51a", + "n": 2, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "04eb81bd28a99e8d254c7a0a9c07234f55af1f9e4fe167269f504109495c29a5", + "n": 4, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "06646577f2484818ed5044762571197f87d3e79272459df5462e193f43ddd530", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "06f295d348152073a6bf3ba5485ca147a0c14bb94baf88391d465d7ea7e6eca9", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "08b767258688d27f07153aa89f0a2c2c6e8e426289043614eb48c2c881591192", + "n": 17, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "09890e550c1a9a583cf9aaa2a997c1bbc61162a5d36c8a891bbbc1a837dffc36", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "0b8649cde9e63fe45d17a7941ef90f01d1b3068c464c013d3ff7e21047164b9c", + "n": 5, + "accuracy": 0.8, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "10f18e39bc3c8e8713158ad66e666edf9118b8b83fb7eb21e5e39476a75a3125", + "n": 12, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "14919c2e5da07951c425b98441bab1107d02cbcb0496a9148b184b1fea1eb4d2", + "n": 3, + "accuracy": 0.6666666666666666, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "14a068f7500ef993a7f2e4cfcd5e194c57217338de4e72cf848d710c20392e91", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 18, + "true_positive_correct": 0 + }, + { + "group": "16cb7259665afd83462953cc90bf025352bab6f7210a59e5d86d49561f03fcbe", + "n": 3, + "accuracy": 0.3333333333333333, + "true_positive_n": 2, + "true_positive_correct": 0 + }, + { + "group": "16fe8b7b9a0987bf861cee078b8763e8f75ec46733a6a59b61ffa8e2bd6e49f7", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "17cd31c2cd2d2ccbabdfab902aee6e4e759167d126e1bcaf4d638c23b93272ac", + "n": 17, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "18aa7b3932a87bfdfc7e3f032e91b38999f82e037146754bd8bba044870170ac", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "191618c5651b68d639db0f90f67315430ae08a9467ebc761ca10a54847034de5", + "n": 20, + "accuracy": 0.75, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "1e7cf0ee593015038b9cf7686c7db7604b7de8b52d0a8b556038b7795a59dd5a", + "n": 19, + "accuracy": 0.7894736842105263, + "true_positive_n": 4, + "true_positive_correct": 0 + }, + { + "group": "23f95dce880a8574d254a19ac7f94f1284a556dfa61126a92844ba979083ede1", + "n": 2, + "accuracy": 0.5, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "2413f2cb2e9a29595c0278b868a30250d3573d487f99e35b3fe3836713d79e96", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "280f0d43bcdf087751b7a2b90739860bc02c2cccea845f81fae7555cdd9d7745", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "2af6f963a1301ac7cbf959fb0f63442e6b1b28be622e0d74c6b94c1e459c8bf0", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "2d4fd6af7767d8fbcfeddff36a76b5c993a49d5e45a515806ec30b1b53afa2ce", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3229724a48ff16d1ea5cff92fcbc1a29fb636760780576606a1f931cba64c7e6", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "33593514e9eb3b4a2f11fbff9a900e304d7f7088413e1a6ed36210de11e1a7cd", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "357e4fcd5f01346ea07a7b6805f5e0e94f11cd39d870ebde997c06b3ca23403b", + "n": 20, + "accuracy": 0.4, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "36be30d9fffc6077ec297a0ccde6690eb34709163c58b4b5df24236dd44c0e91", + "n": 18, + "accuracy": 0.9444444444444444, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3788764c156c1d8107b5e65fc226f169b6cfeb28e92341b3776e0323894d1182", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "397381ff8e300cf10164a627a5a6e614e68a7fb2a4dbac6daa5c67e331248cb3", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3a3a67375cabf27ee46ffe17a8e3122e2bc0d77c22a4bc5fd3caf21af2fdf827", + "n": 9, + "accuracy": 0.8888888888888888, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "3ba2e76ba41ace83ac5fc70feb81f7bd790ebcc01cef912db9a520e0659301e6", + "n": 20, + "accuracy": 0.3, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "3c519ebea078526e0c7724d67b71d0e9d928d62adbb68de9cc3ab45caeab0ba6", + "n": 20, + "accuracy": 0.35, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "42a20c36367ded75633c56b5468efe4dab76b5242f7fc0fafe5de10ae8b9a078", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "4f4986e7532d0ba637710cb0010206dd22720f9b35cc163fd2f833fc93c25334", + "n": 14, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "52ed406f643d4adbab570be68c48860e0e9f239fe00b7f1bd0c35e00868fc879", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "557c22b05ccf851b94b1fc4571234d237e761ded29b151b3a3adcacd17f9ba00", + "n": 5, + "accuracy": 0.6, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "58f3e1e6f3eca7efe52c2f97b888733f6a541ca293e36d6e1a1f22c2332d05fb", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5d11eb324b736d8f810b14b708ea1982028cb405fb0fb2e3f20dab9ab5bb3fe1", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5d2fc2acbf5e2afa16d5ad87f1cce4b784cdd43ca2da54c3eea33521e52a5a81", + "n": 20, + "accuracy": 0.45, + "true_positive_n": 10, + "true_positive_correct": 0 + }, + { + "group": "5f21eec69194bcd2e80220b2701002f2da10e7cd2b04625674cdf69feb57dedd", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "61c9e0872a43ed30a90ab06f7a3e8b85d33f54355b964a7fd1ffa361531c5626", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "61e7fb88134413ce324703852a01f81dda0cb96d4ae07a44cf845c9b1de8baa0", + "n": 20, + "accuracy": 0.25, + "true_positive_n": 13, + "true_positive_correct": 0 + }, + { + "group": "6201b69130be0b440362fc95b4cc2ef6c4c856543d9beb9584d43899e72d0283", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "629dffdc159a2dbb566dab56a6f17419a63aa2b0815020338d686277ff90f623", + "n": 20, + "accuracy": 0.35, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "63b98074727eecae10e68ab106f9c5f2ec98bd2347e0c3eaa72d5557c1ae1b64", + "n": 7, + "accuracy": 0.42857142857142855, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "63deca8572354a01b85154c34b64122772c89023b59c7931064bb347d8389e7c", + "n": 14, + "accuracy": 0.5714285714285714, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "659e4d7e826303db51376a34ed1ee1187305d8b4fcd2ba86de0f1df79cbe9d09", + "n": 20, + "accuracy": 0.95, + "true_positive_n": 20, + "true_positive_correct": 19 + }, + { + "group": "681a13deadc6006e0d4d996f2b006e35b1d71fdeaaebfe98db36b84ce9e1bb82", + "n": 9, + "accuracy": 0.8888888888888888, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6972efb6c1c8c5719fc446f6b4e9bbf812c8274c04ddce38ae95cf5b06e65e10", + "n": 12, + "accuracy": 0.9166666666666666, + "true_positive_n": 5, + "true_positive_correct": 5 + }, + { + "group": "6d2a8e5968c904fbeceb057dc6d374d5cd1ba38019b8d31da27cc658c12aa476", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6d5ea32830050e4ac2840eed05d6b46ec9d7a78f65fbf85d0742d83cc628fa9c", + "n": 20, + "accuracy": 0.35, + "true_positive_n": 7, + "true_positive_correct": 0 + }, + { + "group": "6d74b56adb5791668e05d491f03f083060be7b184f47660a3b9cc14d7a32b511", + "n": 13, + "accuracy": 1.0, + "true_positive_n": 13, + "true_positive_correct": 13 + }, + { + "group": "6e73fd09bd5175004650b33517e33671c77ebb649384e14ffd95309ac3e567a7", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6f07feecb02e317c2db269ed9ec22266aa9195e11612cf7362bcdd1f3c9a7d3c", + "n": 20, + "accuracy": 0.9, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6fc0443881957355817ec4b23a2d3027d6ce1ba949d0c313ac5134c5a33d7752", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "725e8bad3bf121140d3f1e8bb8625fac12766a1a7ebea88a432b39f8af246111", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "745320acbe1a562a24246846ea4e7c92fe29683dd9e75636a6183142147be543", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "77af40ba5530d0abf3b141de72a4c48f4d594f8bc50515d71d71bb0ef57ba861", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "798e40bc3bfcfea7bb0935e395494ac4836e1d49f294b2ca803f7e234a478493", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 2 + }, + { + "group": "79e0a2063b1c1395f9032d7bcca4a7987b25c134851f2b7087535b66973f8767", + "n": 3, + "accuracy": 0.6666666666666666, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "7c2c1e933c21cfd9f0534226991df5542463f0d2c40138378024cbbabc1f205f", + "n": 20, + "accuracy": 0.55, + "true_positive_n": 7, + "true_positive_correct": 0 + }, + { + "group": "7d092ad55211811544dc5993b69d906906fce9350e187508760b25a549bbc9ad", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 5, + "true_positive_correct": 0 + }, + { + "group": "8269675fe1c1fcb6500b8262056237a7eb277d8d7822e63cc1b335c9b3587400", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 20, + "true_positive_correct": 0 + }, + { + "group": "8433ad4a51679bb9fc21396e090e351720fb331d25292de538c1e1a8ca524bab", + "n": 8, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8d353f585c30ae854c944e56bcab8cfff771273fefcde8d929e6faf7cb75488f", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8da5a2b78094becb52357364f9f1ea558c4052e869e9636c912fa88cce650666", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8e11912bbdeb4ce0b56ca82a5e0ad37a9159b78dd620607541dc93cb9ac8e02e", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8f019450f830cbc42ded96cce0b7b4c1ee38439a9e198406ffb38ba5e0524fd7", + "n": 4, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "90994dcfeb71c7a87a4784556f5bf0bdb0b542c15a5f6b92da96e8020c4f1bf1", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "94d54be3b552337f2fc772ce8ff09d2b668ec497130ded8b947506a120acfb4e", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 14, + "true_positive_correct": 14 + }, + { + "group": "97d7c3f3b10a157cdb2791ff320a99a7d1856f8bbdd7ee020ad9ddbe7ec14b8e", + "n": 13, + "accuracy": 0.0, + "true_positive_n": 13, + "true_positive_correct": 0 + }, + { + "group": "97f8ab62aee5d5d53245a565b836e892a9a7cda46748be23097421abb76ee39a", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 6, + "true_positive_correct": 0 + }, + { + "group": "99a049ff4aba4bc20df5f424b24efb9bab9118c192fd17f29b31ec910e497def", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "9b54c928a43ef6ff8b5b53cfeaad5e554106ed4d6718d08bc4f7440e968414af", + "n": 20, + "accuracy": 0.5, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "9bcc91cc05b92cb30cead1b81acb706497144e4815da75642e99d3fd7ebfd154", + "n": 11, + "accuracy": 0.9090909090909091, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "9d2f045465f59eb2bcb8a37818341ce3635a125bced7bfbcb7bf9acdd80c1a71", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 20, + "true_positive_correct": 0 + }, + { + "group": "9e08a63fef975cba0f7fe19b9931144a9480e43b5b78497f97cfe4079c916045", + "n": 10, + "accuracy": 0.6, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "9e38ecba2d710c7a9e3c2bef6abd472773b0edae845c5af26f317a4f5a2a3bc2", + "n": 19, + "accuracy": 0.10526315789473684, + "true_positive_n": 7, + "true_positive_correct": 0 + }, + { + "group": "a04b59b050c050568baa9abccd205e73547f1e5111afaabdbcbe5949c3511ec6", + "n": 20, + "accuracy": 0.25, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "a1e75cda474e69786231632ed64953c19d60658191ea63553964e1d27015a7ce", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a52745698cb48cb1a28ac9bf212d724174c320e4648c9bececd83422b3364c1d", + "n": 19, + "accuracy": 0.10526315789473684, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a5a3b2d9f1ca0af015ed0584c90b28830aa11e86e8311c7f8f32f26b6d4f2768", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "a68f2269c6412f5885bdd26d3023dae3411476f5dd5b2e103355d4e1b0a41bb5", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a7b13e8d802fa5d584ad83d610714c858aab70e0c1f005324a2a91e7a369aed1", + "n": 16, + "accuracy": 0.0625, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a94db659229b95e557d461f6de9071da0762515528aa3a77d5cb870fcf0f914b", + "n": 20, + "accuracy": 0.75, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "aab21c537c79f9eee1a053d0d9f7fbea9eb1451e166156d59dd6c464104a7992", + "n": 10, + "accuracy": 0.7, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "abd52b07ebdf5d9235ca1239d9c3acbe23eb7cabdbc661e0dc93653479a095f0", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "aefb87718e7fc3725adba69a4f789e9de7d638ec0a0d1f12fcff8a8735fb622c", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b095632bbe63fbdc009ed98e30a50899f6a8edc165e3be8a924a7f0f600be4c0", + "n": 20, + "accuracy": 0.4, + "true_positive_n": 18, + "true_positive_correct": 6 + }, + { + "group": "b1b47d1978d292c9b80319bd2b531d2f2af206e3bcabf9aa0c46472112638ba9", + "n": 6, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b7fa0bdd60d2f2321a27ad4ef1c84b2258bb60cb55372ea8aa56643d6d7a6d2b", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 15, + "true_positive_correct": 0 + }, + { + "group": "b8b4a7b5769be3d4db661d88d40336ade544fcac1e5bd2ec6606c09f83e5263c", + "n": 7, + "accuracy": 0.7142857142857143, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "b9063c091cb8deaab4ea7fc341aebc169033dfb16792a7f0f253ce89ce556510", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "bc9c57f088052889f45c0b709137c494d747464872fca72f16cfdde538281d64", + "n": 15, + "accuracy": 0.8666666666666667, + "true_positive_n": 6, + "true_positive_correct": 4 + }, + { + "group": "bd4aa774324b8affeb489e7b796b6f32a9a2f46442394a8f5042c1967eac05dc", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "c0371b16a587352143c383ca9799634e07feb94a2df75ef8830adb86efe153c1", + "n": 7, + "accuracy": 0.5714285714285714, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "c045d0efa4ee5ab2be1d352625f16f6fb9e5cfbe90e02574363e83cd27bc5109", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "c98451abd4304660daabdb977564d36fdad533b4458c467bd2b97d80a651955f", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cb85febf11725dacf437b764d64bf097c9c2bab08e74ac2eeadb496ccd51b5e8", + "n": 16, + "accuracy": 0.875, + "true_positive_n": 3, + "true_positive_correct": 1 + }, + { + "group": "cfe165abb396d68223741b950f77ec55bd59d4c200bd310d7e9a956548c1590e", + "n": 8, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d530b979376d67460a1a3dcafcc95c91e1a51ce2d97124dc2050c2030ae223b7", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 12, + "true_positive_correct": 0 + }, + { + "group": "d5496d95b9bcd56d2d382354f44af6098e30a6b132c6bd2ccbac429ee48d7d3b", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d6e9a943c91ab8d80d913c89d053cf89f2fe4bcd7df1963f3fd5a70bf56f3deb", + "n": 12, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d794d72182a0c6f8a3009e01f6d607875a60752bf7f05920823cc55729bcbf3c", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d82b9f42323202feb6db7ed45f9990070e72e41705aeb5aa64d60a3fd7549f17", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "dfc85bf0674a0000b0dfbcf621419502a19c4fffabf786b4a5e74a69776a1f9d", + "n": 5, + "accuracy": 0.4, + "true_positive_n": 3, + "true_positive_correct": 1 + }, + { + "group": "e35b0e3bc4a28a3bcf49fd9fe5c6e3c6ece59e921ecabf1904c9459cb7b84ad1", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "e6e40718557ace0b1764c2b7ce7021ff2b42e61db43cd59c7a54132f69807b2d", + "n": 7, + "accuracy": 0.5714285714285714, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "f5b769ca29423ada31602cdd07bba0ab057d937256622788fa0872f6fe095865", + "n": 4, + "accuracy": 0.75, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "f72e0a3b72341d46d7eb5ab932fcce07cf34702afa40d217c5e997bef74764be", + "n": 15, + "accuracy": 0.9333333333333333, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "f9476bc6e284b58a4d168d14b188b2ca1636ebe4a0adfcb0c40060bed41781c8", + "n": 3, + "accuracy": 0.3333333333333333, + "true_positive_n": 2, + "true_positive_correct": 0 + }, + { + "group": "f97f37793bfaa1b487125bd7b1f34bff0fed50146a5606962c763220f17bb6de", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fb39d2600c554fe3ac1eddb65704bc34c6102ca0fbb4a77744ac525869d35be4", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fcd64091f029e53dd4a770300ecf446dbf5bca8cc267148f28b198584caeae46", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fe703c5eb241d3542fcc57e11688b3641c4f1f0b9506a720b42a9b9031382154", + "n": 19, + "accuracy": 0.8947368421052632, + "true_positive_n": 2, + "true_positive_correct": 0 + } + ], + "correct": 623 + }, + "global_calibrated": { + "n": 1064, + "accuracy": 0.5855263157894737, + "nll": 0.7976013493654481, + "brier": 0.46891907838977726, + "macro_f1": 0.5415438082741008, + "class_order": [ + "BenignPositive", + "FalsePositive", + "TruePositive" + ], + "per_class": { + "BenignPositive": { + "precision": 0.4096153846153846, + "recall": 0.7580071174377224, + "f1-score": 0.5318352059925093, + "support": 281.0 + }, + "FalsePositive": { + "precision": 0.7873831775700935, + "recall": 0.6686507936507936, + "f1-score": 0.723175965665236, + "support": 504.0 + }, + "TruePositive": { + "precision": 0.6293103448275862, + "recall": 0.2616487455197133, + "f1-score": 0.369620253164557, + "support": 279.0 + } + }, + "confusion": [ + [ + 213, + 60, + 8 + ], + [ + 132, + 337, + 35 + ], + [ + 175, + 31, + 73 + ] + ], + "equal_organization_accuracy": 0.649889118396736, + "organization_results": [ + { + "group": "0228d088db03da24994da8d12bb80e2925bf424387c12a0cab440661f61e53db", + "n": 2, + "accuracy": 0.0, + "true_positive_n": 2, + "true_positive_correct": 0 + }, + { + "group": "03b2cff809e7692ca15f889834563f866b356cc60a41023c8611dd2b60bcf51a", + "n": 2, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "04eb81bd28a99e8d254c7a0a9c07234f55af1f9e4fe167269f504109495c29a5", + "n": 4, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "06646577f2484818ed5044762571197f87d3e79272459df5462e193f43ddd530", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "06f295d348152073a6bf3ba5485ca147a0c14bb94baf88391d465d7ea7e6eca9", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "08b767258688d27f07153aa89f0a2c2c6e8e426289043614eb48c2c881591192", + "n": 17, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "09890e550c1a9a583cf9aaa2a997c1bbc61162a5d36c8a891bbbc1a837dffc36", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "0b8649cde9e63fe45d17a7941ef90f01d1b3068c464c013d3ff7e21047164b9c", + "n": 5, + "accuracy": 0.8, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "10f18e39bc3c8e8713158ad66e666edf9118b8b83fb7eb21e5e39476a75a3125", + "n": 12, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "14919c2e5da07951c425b98441bab1107d02cbcb0496a9148b184b1fea1eb4d2", + "n": 3, + "accuracy": 0.6666666666666666, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "14a068f7500ef993a7f2e4cfcd5e194c57217338de4e72cf848d710c20392e91", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 18, + "true_positive_correct": 0 + }, + { + "group": "16cb7259665afd83462953cc90bf025352bab6f7210a59e5d86d49561f03fcbe", + "n": 3, + "accuracy": 0.3333333333333333, + "true_positive_n": 2, + "true_positive_correct": 0 + }, + { + "group": "16fe8b7b9a0987bf861cee078b8763e8f75ec46733a6a59b61ffa8e2bd6e49f7", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "17cd31c2cd2d2ccbabdfab902aee6e4e759167d126e1bcaf4d638c23b93272ac", + "n": 17, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "18aa7b3932a87bfdfc7e3f032e91b38999f82e037146754bd8bba044870170ac", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "191618c5651b68d639db0f90f67315430ae08a9467ebc761ca10a54847034de5", + "n": 20, + "accuracy": 0.75, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "1e7cf0ee593015038b9cf7686c7db7604b7de8b52d0a8b556038b7795a59dd5a", + "n": 19, + "accuracy": 0.7894736842105263, + "true_positive_n": 4, + "true_positive_correct": 0 + }, + { + "group": "23f95dce880a8574d254a19ac7f94f1284a556dfa61126a92844ba979083ede1", + "n": 2, + "accuracy": 0.5, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "2413f2cb2e9a29595c0278b868a30250d3573d487f99e35b3fe3836713d79e96", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "280f0d43bcdf087751b7a2b90739860bc02c2cccea845f81fae7555cdd9d7745", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "2af6f963a1301ac7cbf959fb0f63442e6b1b28be622e0d74c6b94c1e459c8bf0", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "2d4fd6af7767d8fbcfeddff36a76b5c993a49d5e45a515806ec30b1b53afa2ce", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3229724a48ff16d1ea5cff92fcbc1a29fb636760780576606a1f931cba64c7e6", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "33593514e9eb3b4a2f11fbff9a900e304d7f7088413e1a6ed36210de11e1a7cd", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "357e4fcd5f01346ea07a7b6805f5e0e94f11cd39d870ebde997c06b3ca23403b", + "n": 20, + "accuracy": 0.4, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "36be30d9fffc6077ec297a0ccde6690eb34709163c58b4b5df24236dd44c0e91", + "n": 18, + "accuracy": 0.9444444444444444, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3788764c156c1d8107b5e65fc226f169b6cfeb28e92341b3776e0323894d1182", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "397381ff8e300cf10164a627a5a6e614e68a7fb2a4dbac6daa5c67e331248cb3", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3a3a67375cabf27ee46ffe17a8e3122e2bc0d77c22a4bc5fd3caf21af2fdf827", + "n": 9, + "accuracy": 0.8888888888888888, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "3ba2e76ba41ace83ac5fc70feb81f7bd790ebcc01cef912db9a520e0659301e6", + "n": 20, + "accuracy": 0.3, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "3c519ebea078526e0c7724d67b71d0e9d928d62adbb68de9cc3ab45caeab0ba6", + "n": 20, + "accuracy": 0.35, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "42a20c36367ded75633c56b5468efe4dab76b5242f7fc0fafe5de10ae8b9a078", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "4f4986e7532d0ba637710cb0010206dd22720f9b35cc163fd2f833fc93c25334", + "n": 14, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "52ed406f643d4adbab570be68c48860e0e9f239fe00b7f1bd0c35e00868fc879", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "557c22b05ccf851b94b1fc4571234d237e761ded29b151b3a3adcacd17f9ba00", + "n": 5, + "accuracy": 0.6, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "58f3e1e6f3eca7efe52c2f97b888733f6a541ca293e36d6e1a1f22c2332d05fb", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5d11eb324b736d8f810b14b708ea1982028cb405fb0fb2e3f20dab9ab5bb3fe1", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5d2fc2acbf5e2afa16d5ad87f1cce4b784cdd43ca2da54c3eea33521e52a5a81", + "n": 20, + "accuracy": 0.45, + "true_positive_n": 10, + "true_positive_correct": 0 + }, + { + "group": "5f21eec69194bcd2e80220b2701002f2da10e7cd2b04625674cdf69feb57dedd", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "61c9e0872a43ed30a90ab06f7a3e8b85d33f54355b964a7fd1ffa361531c5626", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "61e7fb88134413ce324703852a01f81dda0cb96d4ae07a44cf845c9b1de8baa0", + "n": 20, + "accuracy": 0.25, + "true_positive_n": 13, + "true_positive_correct": 0 + }, + { + "group": "6201b69130be0b440362fc95b4cc2ef6c4c856543d9beb9584d43899e72d0283", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "629dffdc159a2dbb566dab56a6f17419a63aa2b0815020338d686277ff90f623", + "n": 20, + "accuracy": 0.35, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "63b98074727eecae10e68ab106f9c5f2ec98bd2347e0c3eaa72d5557c1ae1b64", + "n": 7, + "accuracy": 0.42857142857142855, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "63deca8572354a01b85154c34b64122772c89023b59c7931064bb347d8389e7c", + "n": 14, + "accuracy": 0.5714285714285714, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "659e4d7e826303db51376a34ed1ee1187305d8b4fcd2ba86de0f1df79cbe9d09", + "n": 20, + "accuracy": 0.95, + "true_positive_n": 20, + "true_positive_correct": 19 + }, + { + "group": "681a13deadc6006e0d4d996f2b006e35b1d71fdeaaebfe98db36b84ce9e1bb82", + "n": 9, + "accuracy": 0.8888888888888888, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6972efb6c1c8c5719fc446f6b4e9bbf812c8274c04ddce38ae95cf5b06e65e10", + "n": 12, + "accuracy": 0.9166666666666666, + "true_positive_n": 5, + "true_positive_correct": 5 + }, + { + "group": "6d2a8e5968c904fbeceb057dc6d374d5cd1ba38019b8d31da27cc658c12aa476", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6d5ea32830050e4ac2840eed05d6b46ec9d7a78f65fbf85d0742d83cc628fa9c", + "n": 20, + "accuracy": 0.35, + "true_positive_n": 7, + "true_positive_correct": 0 + }, + { + "group": "6d74b56adb5791668e05d491f03f083060be7b184f47660a3b9cc14d7a32b511", + "n": 13, + "accuracy": 1.0, + "true_positive_n": 13, + "true_positive_correct": 13 + }, + { + "group": "6e73fd09bd5175004650b33517e33671c77ebb649384e14ffd95309ac3e567a7", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6f07feecb02e317c2db269ed9ec22266aa9195e11612cf7362bcdd1f3c9a7d3c", + "n": 20, + "accuracy": 0.9, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6fc0443881957355817ec4b23a2d3027d6ce1ba949d0c313ac5134c5a33d7752", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "725e8bad3bf121140d3f1e8bb8625fac12766a1a7ebea88a432b39f8af246111", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "745320acbe1a562a24246846ea4e7c92fe29683dd9e75636a6183142147be543", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "77af40ba5530d0abf3b141de72a4c48f4d594f8bc50515d71d71bb0ef57ba861", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "798e40bc3bfcfea7bb0935e395494ac4836e1d49f294b2ca803f7e234a478493", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 2 + }, + { + "group": "79e0a2063b1c1395f9032d7bcca4a7987b25c134851f2b7087535b66973f8767", + "n": 3, + "accuracy": 0.6666666666666666, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "7c2c1e933c21cfd9f0534226991df5542463f0d2c40138378024cbbabc1f205f", + "n": 20, + "accuracy": 0.55, + "true_positive_n": 7, + "true_positive_correct": 0 + }, + { + "group": "7d092ad55211811544dc5993b69d906906fce9350e187508760b25a549bbc9ad", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 5, + "true_positive_correct": 0 + }, + { + "group": "8269675fe1c1fcb6500b8262056237a7eb277d8d7822e63cc1b335c9b3587400", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 20, + "true_positive_correct": 0 + }, + { + "group": "8433ad4a51679bb9fc21396e090e351720fb331d25292de538c1e1a8ca524bab", + "n": 8, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8d353f585c30ae854c944e56bcab8cfff771273fefcde8d929e6faf7cb75488f", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8da5a2b78094becb52357364f9f1ea558c4052e869e9636c912fa88cce650666", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8e11912bbdeb4ce0b56ca82a5e0ad37a9159b78dd620607541dc93cb9ac8e02e", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8f019450f830cbc42ded96cce0b7b4c1ee38439a9e198406ffb38ba5e0524fd7", + "n": 4, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "90994dcfeb71c7a87a4784556f5bf0bdb0b542c15a5f6b92da96e8020c4f1bf1", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "94d54be3b552337f2fc772ce8ff09d2b668ec497130ded8b947506a120acfb4e", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 14, + "true_positive_correct": 14 + }, + { + "group": "97d7c3f3b10a157cdb2791ff320a99a7d1856f8bbdd7ee020ad9ddbe7ec14b8e", + "n": 13, + "accuracy": 0.0, + "true_positive_n": 13, + "true_positive_correct": 0 + }, + { + "group": "97f8ab62aee5d5d53245a565b836e892a9a7cda46748be23097421abb76ee39a", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 6, + "true_positive_correct": 0 + }, + { + "group": "99a049ff4aba4bc20df5f424b24efb9bab9118c192fd17f29b31ec910e497def", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "9b54c928a43ef6ff8b5b53cfeaad5e554106ed4d6718d08bc4f7440e968414af", + "n": 20, + "accuracy": 0.5, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "9bcc91cc05b92cb30cead1b81acb706497144e4815da75642e99d3fd7ebfd154", + "n": 11, + "accuracy": 0.9090909090909091, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "9d2f045465f59eb2bcb8a37818341ce3635a125bced7bfbcb7bf9acdd80c1a71", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 20, + "true_positive_correct": 0 + }, + { + "group": "9e08a63fef975cba0f7fe19b9931144a9480e43b5b78497f97cfe4079c916045", + "n": 10, + "accuracy": 0.6, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "9e38ecba2d710c7a9e3c2bef6abd472773b0edae845c5af26f317a4f5a2a3bc2", + "n": 19, + "accuracy": 0.10526315789473684, + "true_positive_n": 7, + "true_positive_correct": 0 + }, + { + "group": "a04b59b050c050568baa9abccd205e73547f1e5111afaabdbcbe5949c3511ec6", + "n": 20, + "accuracy": 0.25, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "a1e75cda474e69786231632ed64953c19d60658191ea63553964e1d27015a7ce", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a52745698cb48cb1a28ac9bf212d724174c320e4648c9bececd83422b3364c1d", + "n": 19, + "accuracy": 0.10526315789473684, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a5a3b2d9f1ca0af015ed0584c90b28830aa11e86e8311c7f8f32f26b6d4f2768", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "a68f2269c6412f5885bdd26d3023dae3411476f5dd5b2e103355d4e1b0a41bb5", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a7b13e8d802fa5d584ad83d610714c858aab70e0c1f005324a2a91e7a369aed1", + "n": 16, + "accuracy": 0.0625, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a94db659229b95e557d461f6de9071da0762515528aa3a77d5cb870fcf0f914b", + "n": 20, + "accuracy": 0.75, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "aab21c537c79f9eee1a053d0d9f7fbea9eb1451e166156d59dd6c464104a7992", + "n": 10, + "accuracy": 0.7, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "abd52b07ebdf5d9235ca1239d9c3acbe23eb7cabdbc661e0dc93653479a095f0", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "aefb87718e7fc3725adba69a4f789e9de7d638ec0a0d1f12fcff8a8735fb622c", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b095632bbe63fbdc009ed98e30a50899f6a8edc165e3be8a924a7f0f600be4c0", + "n": 20, + "accuracy": 0.4, + "true_positive_n": 18, + "true_positive_correct": 6 + }, + { + "group": "b1b47d1978d292c9b80319bd2b531d2f2af206e3bcabf9aa0c46472112638ba9", + "n": 6, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b7fa0bdd60d2f2321a27ad4ef1c84b2258bb60cb55372ea8aa56643d6d7a6d2b", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 15, + "true_positive_correct": 0 + }, + { + "group": "b8b4a7b5769be3d4db661d88d40336ade544fcac1e5bd2ec6606c09f83e5263c", + "n": 7, + "accuracy": 0.7142857142857143, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "b9063c091cb8deaab4ea7fc341aebc169033dfb16792a7f0f253ce89ce556510", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "bc9c57f088052889f45c0b709137c494d747464872fca72f16cfdde538281d64", + "n": 15, + "accuracy": 0.8666666666666667, + "true_positive_n": 6, + "true_positive_correct": 4 + }, + { + "group": "bd4aa774324b8affeb489e7b796b6f32a9a2f46442394a8f5042c1967eac05dc", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "c0371b16a587352143c383ca9799634e07feb94a2df75ef8830adb86efe153c1", + "n": 7, + "accuracy": 0.5714285714285714, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "c045d0efa4ee5ab2be1d352625f16f6fb9e5cfbe90e02574363e83cd27bc5109", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "c98451abd4304660daabdb977564d36fdad533b4458c467bd2b97d80a651955f", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cb85febf11725dacf437b764d64bf097c9c2bab08e74ac2eeadb496ccd51b5e8", + "n": 16, + "accuracy": 0.875, + "true_positive_n": 3, + "true_positive_correct": 1 + }, + { + "group": "cfe165abb396d68223741b950f77ec55bd59d4c200bd310d7e9a956548c1590e", + "n": 8, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d530b979376d67460a1a3dcafcc95c91e1a51ce2d97124dc2050c2030ae223b7", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 12, + "true_positive_correct": 0 + }, + { + "group": "d5496d95b9bcd56d2d382354f44af6098e30a6b132c6bd2ccbac429ee48d7d3b", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d6e9a943c91ab8d80d913c89d053cf89f2fe4bcd7df1963f3fd5a70bf56f3deb", + "n": 12, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d794d72182a0c6f8a3009e01f6d607875a60752bf7f05920823cc55729bcbf3c", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d82b9f42323202feb6db7ed45f9990070e72e41705aeb5aa64d60a3fd7549f17", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "dfc85bf0674a0000b0dfbcf621419502a19c4fffabf786b4a5e74a69776a1f9d", + "n": 5, + "accuracy": 0.4, + "true_positive_n": 3, + "true_positive_correct": 1 + }, + { + "group": "e35b0e3bc4a28a3bcf49fd9fe5c6e3c6ece59e921ecabf1904c9459cb7b84ad1", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "e6e40718557ace0b1764c2b7ce7021ff2b42e61db43cd59c7a54132f69807b2d", + "n": 7, + "accuracy": 0.5714285714285714, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "f5b769ca29423ada31602cdd07bba0ab057d937256622788fa0872f6fe095865", + "n": 4, + "accuracy": 0.75, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "f72e0a3b72341d46d7eb5ab932fcce07cf34702afa40d217c5e997bef74764be", + "n": 15, + "accuracy": 0.9333333333333333, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "f9476bc6e284b58a4d168d14b188b2ca1636ebe4a0adfcb0c40060bed41781c8", + "n": 3, + "accuracy": 0.3333333333333333, + "true_positive_n": 2, + "true_positive_correct": 0 + }, + { + "group": "f97f37793bfaa1b487125bd7b1f34bff0fed50146a5606962c763220f17bb6de", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fb39d2600c554fe3ac1eddb65704bc34c6102ca0fbb4a77744ac525869d35be4", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fcd64091f029e53dd4a770300ecf446dbf5bca8cc267148f28b198584caeae46", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fe703c5eb241d3542fcc57e11688b3641c4f1f0b9506a720b42a9b9031382154", + "n": 19, + "accuracy": 0.8947368421052632, + "true_positive_n": 2, + "true_positive_correct": 0 + } + ], + "correct": 623 } }, - "calibrated": { - "parent": { - "acc": 0.3111111111111111, - "nll": 1.1519141587962758, - "brier": 0.6972118196346561, - "ece": 0.12505951037027957 - }, - "candidate": { - "acc": 0.3888888888888889, - "nll": 1.0987834540411583, - "brier": 0.6666360545746582, - "ece": 0.03892731596092908 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.07777777777777778, - "ci95": [ - -0.12229166666666667, - 0.26746313102665914 - ], - "parent_ci95": [ - 0.20512820512820512, - 0.425 - ], - "candidate_ci95": [ - 0.28888888888888886, - 0.48984374999999974 - ] - }, - "nll": { - "candidate_minus_parent": -0.0531307047551175, - "ci95": [ - -0.13131384096372054, - 0.02030876595375617 - ], - "parent_ci95": [ - 1.0836231878965614, - 1.2232119820149001 - ], - "candidate_ci95": [ - 1.087965251457569, - 1.10803281863799 - ] - }, - "brier": { - "candidate_minus_parent": -0.030575765059997817, - "ci95": [ - -0.08053924500142515, - 0.016278720227622917 - ], - "parent_ci95": [ - 0.6536499526341443, - 0.7418442945165746 - ], - "candidate_ci95": [ - 0.6592384181572574, - 0.6728962291231819 - ] - }, - "ece": { - "candidate_minus_parent": -0.0861321944093505, - "ci95": [ - -0.1587048303302882, - 0.024643558296109955 - ], - "parent_ci95": [ - 0.02664428234558302, - 0.22892663757370982 - ], - "candidate_ci95": [ - 0.002359617988633989, - 0.14107288255595657 - ] + "swarm_static": { + "raw": { + "n": 354, + "accuracy": 0.9152542372881356, + "nll": 0.23980562862108395, + "brier": 0.11952531272784916, + "correct": 324 + }, + "global_calibrated": { + "n": 354, + "accuracy": 0.9152542372881356, + "nll": 0.23046829278993344, + "brier": 0.11775650997940981, + "correct": 324 + }, + "families": { + "binding": { + "raw": { + "n": 164, + "accuracy": 0.8292682926829268, + "nll": 0.4973668964039057, + "brier": 0.24305378453073645, + "correct": 136 + }, + "global_calibrated": { + "n": 164, + "accuracy": 0.8292682926829268, + "nll": 0.47756760916281, + "brier": 0.24123129045340497, + "correct": 136 + } + }, + "direct": { + "raw": { + "n": 134, + "accuracy": 1.0, + "nll": 0.0006701158955142708, + "brier": 3.3165775411476116e-05, + "correct": 134 + }, + "global_calibrated": { + "n": 134, + "accuracy": 1.0, + "nll": 0.0034830280523404688, + "brier": 0.0001921529648151749, + "correct": 134 + } + }, + "flow": { + "raw": { + "n": 56, + "accuracy": 0.9642857142857143, + "nll": 0.05773617842186206, + "brier": 0.04369099694129809, + "correct": 54 + }, + "global_calibrated": { + "n": 56, + "accuracy": 0.9642857142857143, + "nll": 0.04996360689146422, + "brier": 0.037466507161918496, + "correct": 54 + } } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 26, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" } - } - }, - "full/all/transactions_3_plus": { - "windows": 133, - "episodes": 93, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 24, - "4": 26, - "5": 41, - "6": 38, - "7": 4 }, - "raw": { - "parent": { - "acc": 0.3118279569892473, - "nll": 1.8007600408262734, - "brier": 0.9912122366031354, - "ece": 0.41777594278222113 - }, - "candidate": { - "acc": 0.34050179211469533, - "nll": 1.0897724637391926, - "brier": 0.6608081864876919, - "ece": 0.06945969138416776 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.02867383512544802, - "ci95": [ - -0.00347222222222221, - 0.06666666666666671 - ], - "parent_ci95": [ - 0.28888888888888886, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3263157894736842, - 0.35555555555555557 - ] - }, - "nll": { - "candidate_minus_parent": -0.7109875770870808, - "ci95": [ - -0.8249378106097072, - -0.5928019108611865 - ], - "parent_ci95": [ - 1.6800381011917274, - 1.9200895008447456 - ], - "candidate_ci95": [ - 1.0736534993392457, - 1.1068153124221158 - ] - }, - "brier": { - "candidate_minus_parent": -0.3304040501154435, - "ci95": [ - -0.3793116868541628, - -0.28256738524317954 - ], - "parent_ci95": [ - 0.9404112325282276, - 1.0398186656005743 - ], - "candidate_ci95": [ - 0.6494179878255043, - 0.6727965288233754 - ] - }, - "ece": { - "candidate_minus_parent": -0.3483162513980534, - "ci95": [ - -0.40399898613809715, - -0.29044166032303326 - ], - "parent_ci95": [ - 0.38626474035396524, - 0.473126989756557 - ], - "candidate_ci95": [ - 0.05996149153641762, - 0.11740475732652438 - ] + "manual": { + "raw": { + "n": 83, + "accuracy": 0.6987951807228916, + "nll": 1.2611921240040693, + "brier": 0.5153410721147468, + "correct": 58 + }, + "global_calibrated": { + "n": 83, + "accuracy": 0.6987951807228916, + "nll": 1.0302111739617255, + "brier": 0.48339882880272644, + "correct": 58 + }, + "families": { + "binding": { + "raw": { + "n": 40, + "accuracy": 0.625, + "nll": 1.3389280289714236, + "brier": 0.6296514707199694, + "correct": 25 + }, + "global_calibrated": { + "n": 40, + "accuracy": 0.625, + "nll": 1.1010460949559158, + "brier": 0.5735795406676166, + "correct": 25 + } + }, + "flow": { + "raw": { + "n": 40, + "accuracy": 0.75, + "nll": 1.2780391309680283, + "brier": 0.4396812526809053, + "correct": 30 + }, + "global_calibrated": { + "n": 40, + "accuracy": 0.75, + "nll": 1.0365517358773104, + "brier": 0.4294728191968137, + "correct": 30 + } + }, + "prerequisite": { + "raw": { + "n": 3, + "accuracy": 1.0, + "nll": 8.663158656148827e-05, + "brier": 1.649633063680738e-08, + "correct": 3 + }, + "global_calibrated": { + "n": 3, + "accuracy": 1.0, + "nll": 0.0012047351647270432, + "brier": 2.7986830271292114e-06, + "correct": 3 + } } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" } }, - "calibrated": { - "parent": { - "acc": 0.3118279569892473, - "nll": 1.171633543085405, - "brier": 0.7154649932664467, - "ece": 0.14465601993335808 - }, - "candidate": { - "acc": 0.34050179211469533, - "nll": 1.088970247788087, - "brier": 0.6599466665496634, - "ece": 0.055631274925033 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.02867383512544802, - "ci95": [ - -0.00347222222222221, - 0.06666666666666671 - ], - "parent_ci95": [ - 0.28888888888888886, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3263157894736842, - 0.35555555555555557 - ] - }, - "nll": { - "candidate_minus_parent": -0.08266329529731808, - "ci95": [ - -0.10527160530287753, - -0.060813425309605006 - ], - "parent_ci95": [ - 1.1497915115579906, - 1.1953183319652614 - ], - "candidate_ci95": [ - 1.0798257200325903, - 1.0975630581184335 - ] - }, - "brier": { - "candidate_minus_parent": -0.05551832671678336, - "ci95": [ - -0.07142755322765607, - -0.040168620031008025 - ], - "parent_ci95": [ - 0.7003199621121581, - 0.7316456618117463 - ], - "candidate_ci95": [ - 0.6534735207307115, - 0.6661281632812587 - ] - }, - "ece": { - "candidate_minus_parent": -0.08902474500832508, - "ci95": [ - -0.15777008194974856, - -0.032028872805473535 - ], - "parent_ci95": [ - 0.11917374104868377, - 0.19726621817887172 - ], - "candidate_ci95": [ - 0.02223594121005041, - 0.1014834181087044 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + "sysmon": { + "raw": { + "n": 64, + "accuracy": 0.96875, + "nll": 0.08465266759433944, + "brier": 0.044922383555011054, + "correct": 62 + }, + "global_calibrated": { + "n": 64, + "accuracy": 0.96875, + "nll": 0.10719175042876139, + "brier": 0.048442258840311225, + "correct": 62 + } + }, + "general": { + "raw": { + "n": 105, + "accuracy": 0.819047619047619, + "nll": 0.5016458133964976, + "brier": 0.2771177005204226, + "correct": 86 + }, + "global_calibrated": { + "n": 105, + "accuracy": 0.819047619047619, + "nll": 0.4265669988063534, + "brier": 0.24785251439683742, + "correct": 86 } } - }, - "full/iid/all": { - "windows": 182, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 36, - "2": 13, - "3": 24, - "4": 26, - "5": 41, - "6": 38, - "7": 4 + } + }, + "candidate": { + "temperature": 1.6245047927124707, + "domains": { + "excytin": { + "raw": { + "n": 398, + "accuracy": 1.0, + "nll": 0.0009678808724190956, + "brier": 4.8256567178328636e-05, + "correct": 398 + }, + "global_calibrated": { + "n": 398, + "accuracy": 1.0, + "nll": 0.009470865385844369, + "brier": 0.0006565384935901168, + "correct": 398 + } }, - "raw": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.7039948885030416, - "brier": 0.9366734114222882, - "ece": 0.3836640270561081 - }, - "candidate": { - "acc": 0.3342013888888889, - "nll": 1.0988623728398248, - "brier": 0.6670543370614523, - "ece": 0.06554797243603253 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0008680555555555802, - "ci95": [ - -0.0078125, - 0.010416666666666685 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3255208333333333, - 0.34375 - ] - }, - "nll": { - "candidate_minus_parent": -0.6051325156632168, - "ci95": [ - -0.6803591005740874, - -0.530793390037944 - ], - "parent_ci95": [ - 1.6258755139905854, - 1.7835939337351452 - ], - "candidate_ci95": [ - 1.0872959592876683, - 1.1104023477998943 - ] - }, - "brier": { - "candidate_minus_parent": -0.2696190743608359, - "ci95": [ - -0.2966854808852652, - -0.24314883056422498 - ], - "parent_ci95": [ - 0.9086695403765366, - 0.9664303635041863 - ], - "candidate_ci95": [ - 0.6585256055156145, - 0.6756684634800705 - ] - }, - "ece": { - "candidate_minus_parent": -0.31811605462007553, - "ci95": [ - -0.3467234011742797, - -0.2776120036720403 - ], - "parent_ci95": [ - 0.3658444671289674, - 0.4154924939910463 - ], - "candidate_ci95": [ - 0.055802842793600815, - 0.10411501636343842 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + "guide": { + "raw": { + "n": 231, + "accuracy": 0.5627705627705628, + "nll": 0.8563265185203435, + "brier": 0.5255142828500304, + "macro_f1": 0.4613423970886119, + "class_order": [ + "BenignPositive", + "FalsePositive", + "TruePositive" + ], + "per_class": { + "BenignPositive": { + "precision": 0.3333333333333333, + "recall": 0.4915254237288136, + "f1-score": 0.3972602739726027, + "support": 59.0 + }, + "FalsePositive": { + "precision": 0.7175572519083969, + "recall": 0.6962962962962963, + "f1-score": 0.706766917293233, + "support": 135.0 + }, + "TruePositive": { + "precision": 0.5384615384615384, + "recall": 0.1891891891891892, + "f1-score": 0.28, + "support": 37.0 + } + }, + "confusion": [ + [ + 29, + 27, + 3 + ], + [ + 38, + 94, + 3 + ], + [ + 20, + 10, + 7 + ] + ], + "equal_organization_accuracy": 0.6510869565217392, + "organization_results": [ + { + "group": "0d1b7eba9c4d5a5852ef2805e5f1c1018c827e36bfb922fefa4d0a92d76f322c", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "108482877637fd1bbe809b2faddd12c9affef276cdb170667c0234a670019cd4", + "n": 20, + "accuracy": 0.45, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "13a6e1ec428dca68e3cc729ac008e19870e5caa60b3f2d8d013eeb5a4f25aa71", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "17515941d0c83a2c0515166a1503db5e3d24c63b53c0398a9c4f3345d3ef5eda", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "22e3ca4ff2df3a13f6db322d247b7b7f8742181d456be68cb84fb4dfbda7b993", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5667ea75698bc671a6165fb7604bf6d7b7874ba2ef04005f82edf2811b406b53", + "n": 20, + "accuracy": 0.65, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "5cd724f6f8c622cf83c9ee56eeb3e956e4c0428f4bb398608581d51b9f3bd4cf", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5fd9d56568dd7e69c4871b023d3df17203f2eb6d9aaf7a808371a391407f9026", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "66288757ecaf36bf368cd8d4c43f454658be762a12957c52eae438ca3e727ad8", + "n": 8, + "accuracy": 0.375, + "true_positive_n": 6, + "true_positive_correct": 1 + }, + { + "group": "6d53f455fa642b810781930d63db6b35443da067812f1bf329a37de938179aed", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "79707d8a1f7240a758d67ba5212ab21ef286647a4b71ba71c892f88341f3f4e6", + "n": 20, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 1 + }, + { + "group": "7b4cb8f8c017bd94da14e9460c49cdf44fa47f6eab9726918b70ebb7b2695ee1", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "7f55e216b81a67b2fcebf627b534d1d18d9d734f31cb41477725aaa041f6d42b", + "n": 20, + "accuracy": 0.6, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "974cf500c91fb4e80273489c70f89803fe8eaf2fc9d4e68a6aac899d0e580551", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 18, + "true_positive_correct": 0 + }, + { + "group": "97dbafe0dc6d23395bb2008b78ccf190cf0710c18154cbbfc664b3d388086baf", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a756d4fa124cf854b444faed605f6fee522c5418d960b8cd60a8a06c44ef56b0", + "n": 5, + "accuracy": 0.6, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "adcffe87f88661c654325a95ccbc252777e27ff83f947ff85ea77cf167ad57c0", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b4edbf01a09fe17e2fc4eba076213094ab09b40f0ff73c9d3e5c9592f0f59353", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "bfe24bf96507627d9c2e0eb183e0cb607aa50a5195abded30b612c252216e7b0", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cbdba8017701da737b2f23ee11598f766b99054550311935a5730acb5f7e57d9", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cc3696bd6f2c8a2d65d8376338102471db2565326ab124811ce53c428e54425e", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 2 + }, + { + "group": "d592c0767849012441fe14ab7451f4371421ba152f26c3587463adcf58b911e5", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "f562314a5da425b50c820fa25a6756b09f54b042ff1a3a4b580647dd4c20aa7d", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + } + ], + "correct": 130 + }, + "global_calibrated": { + "n": 231, + "accuracy": 0.5627705627705628, + "nll": 0.7793526965750157, + "brier": 0.4718856823661165, + "macro_f1": 0.4613423970886119, + "class_order": [ + "BenignPositive", + "FalsePositive", + "TruePositive" + ], + "per_class": { + "BenignPositive": { + "precision": 0.3333333333333333, + "recall": 0.4915254237288136, + "f1-score": 0.3972602739726027, + "support": 59.0 + }, + "FalsePositive": { + "precision": 0.7175572519083969, + "recall": 0.6962962962962963, + "f1-score": 0.706766917293233, + "support": 135.0 + }, + "TruePositive": { + "precision": 0.5384615384615384, + "recall": 0.1891891891891892, + "f1-score": 0.28, + "support": 37.0 + } + }, + "confusion": [ + [ + 29, + 27, + 3 + ], + [ + 38, + 94, + 3 + ], + [ + 20, + 10, + 7 + ] + ], + "equal_organization_accuracy": 0.6510869565217392, + "organization_results": [ + { + "group": "0d1b7eba9c4d5a5852ef2805e5f1c1018c827e36bfb922fefa4d0a92d76f322c", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "108482877637fd1bbe809b2faddd12c9affef276cdb170667c0234a670019cd4", + "n": 20, + "accuracy": 0.45, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "13a6e1ec428dca68e3cc729ac008e19870e5caa60b3f2d8d013eeb5a4f25aa71", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "17515941d0c83a2c0515166a1503db5e3d24c63b53c0398a9c4f3345d3ef5eda", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "22e3ca4ff2df3a13f6db322d247b7b7f8742181d456be68cb84fb4dfbda7b993", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5667ea75698bc671a6165fb7604bf6d7b7874ba2ef04005f82edf2811b406b53", + "n": 20, + "accuracy": 0.65, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "5cd724f6f8c622cf83c9ee56eeb3e956e4c0428f4bb398608581d51b9f3bd4cf", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5fd9d56568dd7e69c4871b023d3df17203f2eb6d9aaf7a808371a391407f9026", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "66288757ecaf36bf368cd8d4c43f454658be762a12957c52eae438ca3e727ad8", + "n": 8, + "accuracy": 0.375, + "true_positive_n": 6, + "true_positive_correct": 1 + }, + { + "group": "6d53f455fa642b810781930d63db6b35443da067812f1bf329a37de938179aed", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "79707d8a1f7240a758d67ba5212ab21ef286647a4b71ba71c892f88341f3f4e6", + "n": 20, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 1 + }, + { + "group": "7b4cb8f8c017bd94da14e9460c49cdf44fa47f6eab9726918b70ebb7b2695ee1", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "7f55e216b81a67b2fcebf627b534d1d18d9d734f31cb41477725aaa041f6d42b", + "n": 20, + "accuracy": 0.6, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "974cf500c91fb4e80273489c70f89803fe8eaf2fc9d4e68a6aac899d0e580551", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 18, + "true_positive_correct": 0 + }, + { + "group": "97dbafe0dc6d23395bb2008b78ccf190cf0710c18154cbbfc664b3d388086baf", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a756d4fa124cf854b444faed605f6fee522c5418d960b8cd60a8a06c44ef56b0", + "n": 5, + "accuracy": 0.6, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "adcffe87f88661c654325a95ccbc252777e27ff83f947ff85ea77cf167ad57c0", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b4edbf01a09fe17e2fc4eba076213094ab09b40f0ff73c9d3e5c9592f0f59353", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "bfe24bf96507627d9c2e0eb183e0cb607aa50a5195abded30b612c252216e7b0", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cbdba8017701da737b2f23ee11598f766b99054550311935a5730acb5f7e57d9", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cc3696bd6f2c8a2d65d8376338102471db2565326ab124811ce53c428e54425e", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 2 + }, + { + "group": "d592c0767849012441fe14ab7451f4371421ba152f26c3587463adcf58b911e5", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "f562314a5da425b50c820fa25a6756b09f54b042ff1a3a4b580647dd4c20aa7d", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + } + ], + "correct": 130 } }, - "calibrated": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.1516241769020001, - "brier": 0.7011284844520596, - "ece": 0.1126117849166719 - }, - "candidate": { - "acc": 0.3342013888888889, - "nll": 1.094225810223573, - "brier": 0.6635708549658472, - "ece": 0.04216089485176933 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0008680555555555802, - "ci95": [ - -0.0078125, - 0.010416666666666685 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3255208333333333, - 0.34375 - ] - }, - "nll": { - "candidate_minus_parent": -0.05739836667842724, - "ci95": [ - -0.06962603452379744, - -0.04488653163402823 - ], - "parent_ci95": [ - 1.1374408274590326, - 1.166659736233086 - ], - "candidate_ci95": [ - 1.0877924378749446, - 1.1001794102062266 - ] - }, - "brier": { - "candidate_minus_parent": -0.03755762948621233, - "ci95": [ - -0.04590448711935209, - -0.02933488075617265 - ], - "parent_ci95": [ - 0.6916327369871181, - 0.7113507865114055 - ], - "candidate_ci95": [ - 0.6589866278321939, - 0.6679576727482305 - ] - }, - "ece": { - "candidate_minus_parent": -0.07045089006490257, - "ci95": [ - -0.10204058462593617, - -0.03478029248966403 - ], - "parent_ci95": [ - 0.10663147728974164, - 0.13809590546759193 - ], - "candidate_ci95": [ - 0.02485568463820158, - 0.07566025500259763 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + "native": { + "raw": { + "n": 410, + "accuracy": 0.9975609756097561, + "nll": 0.008457485545022507, + "brier": 0.0035344210430228854, + "correct": 409 + }, + "global_calibrated": { + "n": 410, + "accuracy": 0.9975609756097561, + "nll": 0.028769043116345983, + "brier": 0.006284804103393485, + "correct": 409 } - } - }, - "full/iid/transactions_1_2": { - "windows": 49, - "episodes": 45, - "latent_groups": 26, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 36, - "2": 13 }, - "raw": { - "parent": { - "acc": 0.3111111111111111, - "nll": 1.6603323212661212, - "brier": 0.8907029573817137, - "ece": 0.37799735259416234 - }, - "candidate": { - "acc": 0.3888888888888889, - "nll": 1.1006948393144855, - "brier": 0.6675216739911485, - "ece": 0.02197163304242618 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.07777777777777778, - "ci95": [ - -0.12229166666666667, - 0.26746313102665914 - ], - "parent_ci95": [ - 0.20512820512820512, - 0.425 - ], - "candidate_ci95": [ - 0.28888888888888886, - 0.48984374999999974 - ] - }, - "nll": { - "candidate_minus_parent": -0.5596374819516357, - "ci95": [ - -0.8551553776743378, - -0.2835412394003477 - ], - "parent_ci95": [ - 1.3960780027038617, - 1.9415518588113363 - ], - "candidate_ci95": [ - 1.07961907447766, - 1.1190005382885808 - ] - }, - "brier": { - "candidate_minus_parent": -0.2231812833905652, - "ci95": [ - -0.36429527240524406, - -0.07974528756374481 - ], - "parent_ci95": [ - 0.7571302612447247, - 1.0218668962200836 - ], - "candidate_ci95": [ - 0.6529239682286327, - 0.6800289575226035 - ] - }, - "ece": { - "candidate_minus_parent": -0.3560257195517362, - "ci95": [ - -0.4212366104265594, - -0.19688232190049682 - ], - "parent_ci95": [ - 0.27558167651761906, - 0.4812973135389953 - ], - "candidate_ci95": [ - 0.005189460689758951, - 0.13286403298320185 - ] + "guide_detector": { + "raw": { + "n": 1064, + "accuracy": 0.599624060150376, + "nll": 0.8276776090882748, + "brier": 0.48019952548397726, + "macro_f1": 0.5681608349992006, + "class_order": [ + "BenignPositive", + "FalsePositive", + "TruePositive" + ], + "per_class": { + "BenignPositive": { + "precision": 0.44772727272727275, + "recall": 0.701067615658363, + "f1-score": 0.5464632454923717, + "support": 281.0 + }, + "FalsePositive": { + "precision": 0.7606263982102909, + "recall": 0.6746031746031746, + "f1-score": 0.7150368033648791, + "support": 504.0 + }, + "TruePositive": { + "precision": 0.5706214689265536, + "recall": 0.36200716845878134, + "f1-score": 0.44298245614035087, + "support": 279.0 + } + }, + "confusion": [ + [ + 197, + 55, + 29 + ], + [ + 117, + 340, + 47 + ], + [ + 126, + 52, + 101 + ] + ], + "equal_organization_accuracy": 0.6743365093994473, + "organization_results": [ + { + "group": "0228d088db03da24994da8d12bb80e2925bf424387c12a0cab440661f61e53db", + "n": 2, + "accuracy": 0.5, + "true_positive_n": 2, + "true_positive_correct": 1 + }, + { + "group": "03b2cff809e7692ca15f889834563f866b356cc60a41023c8611dd2b60bcf51a", + "n": 2, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "04eb81bd28a99e8d254c7a0a9c07234f55af1f9e4fe167269f504109495c29a5", + "n": 4, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "06646577f2484818ed5044762571197f87d3e79272459df5462e193f43ddd530", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "06f295d348152073a6bf3ba5485ca147a0c14bb94baf88391d465d7ea7e6eca9", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "08b767258688d27f07153aa89f0a2c2c6e8e426289043614eb48c2c881591192", + "n": 17, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "09890e550c1a9a583cf9aaa2a997c1bbc61162a5d36c8a891bbbc1a837dffc36", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "0b8649cde9e63fe45d17a7941ef90f01d1b3068c464c013d3ff7e21047164b9c", + "n": 5, + "accuracy": 0.8, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "10f18e39bc3c8e8713158ad66e666edf9118b8b83fb7eb21e5e39476a75a3125", + "n": 12, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "14919c2e5da07951c425b98441bab1107d02cbcb0496a9148b184b1fea1eb4d2", + "n": 3, + "accuracy": 0.6666666666666666, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "14a068f7500ef993a7f2e4cfcd5e194c57217338de4e72cf848d710c20392e91", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 18, + "true_positive_correct": 0 + }, + { + "group": "16cb7259665afd83462953cc90bf025352bab6f7210a59e5d86d49561f03fcbe", + "n": 3, + "accuracy": 0.3333333333333333, + "true_positive_n": 2, + "true_positive_correct": 0 + }, + { + "group": "16fe8b7b9a0987bf861cee078b8763e8f75ec46733a6a59b61ffa8e2bd6e49f7", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "17cd31c2cd2d2ccbabdfab902aee6e4e759167d126e1bcaf4d638c23b93272ac", + "n": 17, + "accuracy": 0.9411764705882353, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "18aa7b3932a87bfdfc7e3f032e91b38999f82e037146754bd8bba044870170ac", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "191618c5651b68d639db0f90f67315430ae08a9467ebc761ca10a54847034de5", + "n": 20, + "accuracy": 0.65, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "1e7cf0ee593015038b9cf7686c7db7604b7de8b52d0a8b556038b7795a59dd5a", + "n": 19, + "accuracy": 0.9473684210526315, + "true_positive_n": 4, + "true_positive_correct": 4 + }, + { + "group": "23f95dce880a8574d254a19ac7f94f1284a556dfa61126a92844ba979083ede1", + "n": 2, + "accuracy": 0.5, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "2413f2cb2e9a29595c0278b868a30250d3573d487f99e35b3fe3836713d79e96", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "280f0d43bcdf087751b7a2b90739860bc02c2cccea845f81fae7555cdd9d7745", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "2af6f963a1301ac7cbf959fb0f63442e6b1b28be622e0d74c6b94c1e459c8bf0", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "2d4fd6af7767d8fbcfeddff36a76b5c993a49d5e45a515806ec30b1b53afa2ce", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3229724a48ff16d1ea5cff92fcbc1a29fb636760780576606a1f931cba64c7e6", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "33593514e9eb3b4a2f11fbff9a900e304d7f7088413e1a6ed36210de11e1a7cd", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "357e4fcd5f01346ea07a7b6805f5e0e94f11cd39d870ebde997c06b3ca23403b", + "n": 20, + "accuracy": 0.3, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "36be30d9fffc6077ec297a0ccde6690eb34709163c58b4b5df24236dd44c0e91", + "n": 18, + "accuracy": 0.9444444444444444, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3788764c156c1d8107b5e65fc226f169b6cfeb28e92341b3776e0323894d1182", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "397381ff8e300cf10164a627a5a6e614e68a7fb2a4dbac6daa5c67e331248cb3", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3a3a67375cabf27ee46ffe17a8e3122e2bc0d77c22a4bc5fd3caf21af2fdf827", + "n": 9, + "accuracy": 0.8888888888888888, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "3ba2e76ba41ace83ac5fc70feb81f7bd790ebcc01cef912db9a520e0659301e6", + "n": 20, + "accuracy": 0.25, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "3c519ebea078526e0c7724d67b71d0e9d928d62adbb68de9cc3ab45caeab0ba6", + "n": 20, + "accuracy": 0.35, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "42a20c36367ded75633c56b5468efe4dab76b5242f7fc0fafe5de10ae8b9a078", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "4f4986e7532d0ba637710cb0010206dd22720f9b35cc163fd2f833fc93c25334", + "n": 14, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "52ed406f643d4adbab570be68c48860e0e9f239fe00b7f1bd0c35e00868fc879", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "557c22b05ccf851b94b1fc4571234d237e761ded29b151b3a3adcacd17f9ba00", + "n": 5, + "accuracy": 0.6, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "58f3e1e6f3eca7efe52c2f97b888733f6a541ca293e36d6e1a1f22c2332d05fb", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5d11eb324b736d8f810b14b708ea1982028cb405fb0fb2e3f20dab9ab5bb3fe1", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5d2fc2acbf5e2afa16d5ad87f1cce4b784cdd43ca2da54c3eea33521e52a5a81", + "n": 20, + "accuracy": 0.45, + "true_positive_n": 10, + "true_positive_correct": 0 + }, + { + "group": "5f21eec69194bcd2e80220b2701002f2da10e7cd2b04625674cdf69feb57dedd", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "61c9e0872a43ed30a90ab06f7a3e8b85d33f54355b964a7fd1ffa361531c5626", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "61e7fb88134413ce324703852a01f81dda0cb96d4ae07a44cf845c9b1de8baa0", + "n": 20, + "accuracy": 0.7, + "true_positive_n": 13, + "true_positive_correct": 9 + }, + { + "group": "6201b69130be0b440362fc95b4cc2ef6c4c856543d9beb9584d43899e72d0283", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "629dffdc159a2dbb566dab56a6f17419a63aa2b0815020338d686277ff90f623", + "n": 20, + "accuracy": 0.4, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "63b98074727eecae10e68ab106f9c5f2ec98bd2347e0c3eaa72d5557c1ae1b64", + "n": 7, + "accuracy": 0.5714285714285714, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "63deca8572354a01b85154c34b64122772c89023b59c7931064bb347d8389e7c", + "n": 14, + "accuracy": 0.5714285714285714, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "659e4d7e826303db51376a34ed1ee1187305d8b4fcd2ba86de0f1df79cbe9d09", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 20, + "true_positive_correct": 20 + }, + { + "group": "681a13deadc6006e0d4d996f2b006e35b1d71fdeaaebfe98db36b84ce9e1bb82", + "n": 9, + "accuracy": 0.8888888888888888, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6972efb6c1c8c5719fc446f6b4e9bbf812c8274c04ddce38ae95cf5b06e65e10", + "n": 12, + "accuracy": 0.9166666666666666, + "true_positive_n": 5, + "true_positive_correct": 5 + }, + { + "group": "6d2a8e5968c904fbeceb057dc6d374d5cd1ba38019b8d31da27cc658c12aa476", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6d5ea32830050e4ac2840eed05d6b46ec9d7a78f65fbf85d0742d83cc628fa9c", + "n": 20, + "accuracy": 0.4, + "true_positive_n": 7, + "true_positive_correct": 0 + }, + { + "group": "6d74b56adb5791668e05d491f03f083060be7b184f47660a3b9cc14d7a32b511", + "n": 13, + "accuracy": 1.0, + "true_positive_n": 13, + "true_positive_correct": 13 + }, + { + "group": "6e73fd09bd5175004650b33517e33671c77ebb649384e14ffd95309ac3e567a7", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6f07feecb02e317c2db269ed9ec22266aa9195e11612cf7362bcdd1f3c9a7d3c", + "n": 20, + "accuracy": 0.95, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6fc0443881957355817ec4b23a2d3027d6ce1ba949d0c313ac5134c5a33d7752", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "725e8bad3bf121140d3f1e8bb8625fac12766a1a7ebea88a432b39f8af246111", + "n": 20, + "accuracy": 0.95, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "745320acbe1a562a24246846ea4e7c92fe29683dd9e75636a6183142147be543", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "77af40ba5530d0abf3b141de72a4c48f4d594f8bc50515d71d71bb0ef57ba861", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "798e40bc3bfcfea7bb0935e395494ac4836e1d49f294b2ca803f7e234a478493", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 2 + }, + { + "group": "79e0a2063b1c1395f9032d7bcca4a7987b25c134851f2b7087535b66973f8767", + "n": 3, + "accuracy": 0.3333333333333333, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "7c2c1e933c21cfd9f0534226991df5542463f0d2c40138378024cbbabc1f205f", + "n": 20, + "accuracy": 0.55, + "true_positive_n": 7, + "true_positive_correct": 1 + }, + { + "group": "7d092ad55211811544dc5993b69d906906fce9350e187508760b25a549bbc9ad", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 5, + "true_positive_correct": 0 + }, + { + "group": "8269675fe1c1fcb6500b8262056237a7eb277d8d7822e63cc1b335c9b3587400", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 20, + "true_positive_correct": 0 + }, + { + "group": "8433ad4a51679bb9fc21396e090e351720fb331d25292de538c1e1a8ca524bab", + "n": 8, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8d353f585c30ae854c944e56bcab8cfff771273fefcde8d929e6faf7cb75488f", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8da5a2b78094becb52357364f9f1ea558c4052e869e9636c912fa88cce650666", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8e11912bbdeb4ce0b56ca82a5e0ad37a9159b78dd620607541dc93cb9ac8e02e", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8f019450f830cbc42ded96cce0b7b4c1ee38439a9e198406ffb38ba5e0524fd7", + "n": 4, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "90994dcfeb71c7a87a4784556f5bf0bdb0b542c15a5f6b92da96e8020c4f1bf1", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "94d54be3b552337f2fc772ce8ff09d2b668ec497130ded8b947506a120acfb4e", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 14, + "true_positive_correct": 14 + }, + { + "group": "97d7c3f3b10a157cdb2791ff320a99a7d1856f8bbdd7ee020ad9ddbe7ec14b8e", + "n": 13, + "accuracy": 0.15384615384615385, + "true_positive_n": 13, + "true_positive_correct": 2 + }, + { + "group": "97f8ab62aee5d5d53245a565b836e892a9a7cda46748be23097421abb76ee39a", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 6, + "true_positive_correct": 0 + }, + { + "group": "99a049ff4aba4bc20df5f424b24efb9bab9118c192fd17f29b31ec910e497def", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "9b54c928a43ef6ff8b5b53cfeaad5e554106ed4d6718d08bc4f7440e968414af", + "n": 20, + "accuracy": 0.5, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "9bcc91cc05b92cb30cead1b81acb706497144e4815da75642e99d3fd7ebfd154", + "n": 11, + "accuracy": 0.9090909090909091, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "9d2f045465f59eb2bcb8a37818341ce3635a125bced7bfbcb7bf9acdd80c1a71", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 20, + "true_positive_correct": 0 + }, + { + "group": "9e08a63fef975cba0f7fe19b9931144a9480e43b5b78497f97cfe4079c916045", + "n": 10, + "accuracy": 0.5, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "9e38ecba2d710c7a9e3c2bef6abd472773b0edae845c5af26f317a4f5a2a3bc2", + "n": 19, + "accuracy": 0.42105263157894735, + "true_positive_n": 7, + "true_positive_correct": 6 + }, + { + "group": "a04b59b050c050568baa9abccd205e73547f1e5111afaabdbcbe5949c3511ec6", + "n": 20, + "accuracy": 0.05, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "a1e75cda474e69786231632ed64953c19d60658191ea63553964e1d27015a7ce", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a52745698cb48cb1a28ac9bf212d724174c320e4648c9bececd83422b3364c1d", + "n": 19, + "accuracy": 0.10526315789473684, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a5a3b2d9f1ca0af015ed0584c90b28830aa11e86e8311c7f8f32f26b6d4f2768", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "a68f2269c6412f5885bdd26d3023dae3411476f5dd5b2e103355d4e1b0a41bb5", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a7b13e8d802fa5d584ad83d610714c858aab70e0c1f005324a2a91e7a369aed1", + "n": 16, + "accuracy": 0.0625, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a94db659229b95e557d461f6de9071da0762515528aa3a77d5cb870fcf0f914b", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "aab21c537c79f9eee1a053d0d9f7fbea9eb1451e166156d59dd6c464104a7992", + "n": 10, + "accuracy": 0.5, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "abd52b07ebdf5d9235ca1239d9c3acbe23eb7cabdbc661e0dc93653479a095f0", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "aefb87718e7fc3725adba69a4f789e9de7d638ec0a0d1f12fcff8a8735fb622c", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b095632bbe63fbdc009ed98e30a50899f6a8edc165e3be8a924a7f0f600be4c0", + "n": 20, + "accuracy": 0.45, + "true_positive_n": 18, + "true_positive_correct": 7 + }, + { + "group": "b1b47d1978d292c9b80319bd2b531d2f2af206e3bcabf9aa0c46472112638ba9", + "n": 6, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b7fa0bdd60d2f2321a27ad4ef1c84b2258bb60cb55372ea8aa56643d6d7a6d2b", + "n": 20, + "accuracy": 0.05, + "true_positive_n": 15, + "true_positive_correct": 0 + }, + { + "group": "b8b4a7b5769be3d4db661d88d40336ade544fcac1e5bd2ec6606c09f83e5263c", + "n": 7, + "accuracy": 0.7142857142857143, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "b9063c091cb8deaab4ea7fc341aebc169033dfb16792a7f0f253ce89ce556510", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "bc9c57f088052889f45c0b709137c494d747464872fca72f16cfdde538281d64", + "n": 15, + "accuracy": 1.0, + "true_positive_n": 6, + "true_positive_correct": 6 + }, + { + "group": "bd4aa774324b8affeb489e7b796b6f32a9a2f46442394a8f5042c1967eac05dc", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "c0371b16a587352143c383ca9799634e07feb94a2df75ef8830adb86efe153c1", + "n": 7, + "accuracy": 0.5714285714285714, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "c045d0efa4ee5ab2be1d352625f16f6fb9e5cfbe90e02574363e83cd27bc5109", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "c98451abd4304660daabdb977564d36fdad533b4458c467bd2b97d80a651955f", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cb85febf11725dacf437b764d64bf097c9c2bab08e74ac2eeadb496ccd51b5e8", + "n": 16, + "accuracy": 0.875, + "true_positive_n": 3, + "true_positive_correct": 1 + }, + { + "group": "cfe165abb396d68223741b950f77ec55bd59d4c200bd310d7e9a956548c1590e", + "n": 8, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d530b979376d67460a1a3dcafcc95c91e1a51ce2d97124dc2050c2030ae223b7", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 12, + "true_positive_correct": 0 + }, + { + "group": "d5496d95b9bcd56d2d382354f44af6098e30a6b132c6bd2ccbac429ee48d7d3b", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d6e9a943c91ab8d80d913c89d053cf89f2fe4bcd7df1963f3fd5a70bf56f3deb", + "n": 12, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d794d72182a0c6f8a3009e01f6d607875a60752bf7f05920823cc55729bcbf3c", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d82b9f42323202feb6db7ed45f9990070e72e41705aeb5aa64d60a3fd7549f17", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "dfc85bf0674a0000b0dfbcf621419502a19c4fffabf786b4a5e74a69776a1f9d", + "n": 5, + "accuracy": 0.4, + "true_positive_n": 3, + "true_positive_correct": 1 + }, + { + "group": "e35b0e3bc4a28a3bcf49fd9fe5c6e3c6ece59e921ecabf1904c9459cb7b84ad1", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "e6e40718557ace0b1764c2b7ce7021ff2b42e61db43cd59c7a54132f69807b2d", + "n": 7, + "accuracy": 0.5714285714285714, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "f5b769ca29423ada31602cdd07bba0ab057d937256622788fa0872f6fe095865", + "n": 4, + "accuracy": 0.75, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "f72e0a3b72341d46d7eb5ab932fcce07cf34702afa40d217c5e997bef74764be", + "n": 15, + "accuracy": 0.6666666666666666, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "f9476bc6e284b58a4d168d14b188b2ca1636ebe4a0adfcb0c40060bed41781c8", + "n": 3, + "accuracy": 0.3333333333333333, + "true_positive_n": 2, + "true_positive_correct": 0 + }, + { + "group": "f97f37793bfaa1b487125bd7b1f34bff0fed50146a5606962c763220f17bb6de", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fb39d2600c554fe3ac1eddb65704bc34c6102ca0fbb4a77744ac525869d35be4", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fcd64091f029e53dd4a770300ecf446dbf5bca8cc267148f28b198584caeae46", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fe703c5eb241d3542fcc57e11688b3641c4f1f0b9506a720b42a9b9031382154", + "n": 19, + "accuracy": 0.7368421052631579, + "true_positive_n": 2, + "true_positive_correct": 0 + } + ], + "correct": 638 + }, + "global_calibrated": { + "n": 1064, + "accuracy": 0.599624060150376, + "nll": 0.785071927880167, + "brier": 0.46075730837789786, + "macro_f1": 0.5681608349992006, + "class_order": [ + "BenignPositive", + "FalsePositive", + "TruePositive" + ], + "per_class": { + "BenignPositive": { + "precision": 0.44772727272727275, + "recall": 0.701067615658363, + "f1-score": 0.5464632454923717, + "support": 281.0 + }, + "FalsePositive": { + "precision": 0.7606263982102909, + "recall": 0.6746031746031746, + "f1-score": 0.7150368033648791, + "support": 504.0 + }, + "TruePositive": { + "precision": 0.5706214689265536, + "recall": 0.36200716845878134, + "f1-score": 0.44298245614035087, + "support": 279.0 + } + }, + "confusion": [ + [ + 197, + 55, + 29 + ], + [ + 117, + 340, + 47 + ], + [ + 126, + 52, + 101 + ] + ], + "equal_organization_accuracy": 0.6743365093994473, + "organization_results": [ + { + "group": "0228d088db03da24994da8d12bb80e2925bf424387c12a0cab440661f61e53db", + "n": 2, + "accuracy": 0.5, + "true_positive_n": 2, + "true_positive_correct": 1 + }, + { + "group": "03b2cff809e7692ca15f889834563f866b356cc60a41023c8611dd2b60bcf51a", + "n": 2, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "04eb81bd28a99e8d254c7a0a9c07234f55af1f9e4fe167269f504109495c29a5", + "n": 4, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "06646577f2484818ed5044762571197f87d3e79272459df5462e193f43ddd530", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "06f295d348152073a6bf3ba5485ca147a0c14bb94baf88391d465d7ea7e6eca9", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "08b767258688d27f07153aa89f0a2c2c6e8e426289043614eb48c2c881591192", + "n": 17, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "09890e550c1a9a583cf9aaa2a997c1bbc61162a5d36c8a891bbbc1a837dffc36", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "0b8649cde9e63fe45d17a7941ef90f01d1b3068c464c013d3ff7e21047164b9c", + "n": 5, + "accuracy": 0.8, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "10f18e39bc3c8e8713158ad66e666edf9118b8b83fb7eb21e5e39476a75a3125", + "n": 12, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "14919c2e5da07951c425b98441bab1107d02cbcb0496a9148b184b1fea1eb4d2", + "n": 3, + "accuracy": 0.6666666666666666, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "14a068f7500ef993a7f2e4cfcd5e194c57217338de4e72cf848d710c20392e91", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 18, + "true_positive_correct": 0 + }, + { + "group": "16cb7259665afd83462953cc90bf025352bab6f7210a59e5d86d49561f03fcbe", + "n": 3, + "accuracy": 0.3333333333333333, + "true_positive_n": 2, + "true_positive_correct": 0 + }, + { + "group": "16fe8b7b9a0987bf861cee078b8763e8f75ec46733a6a59b61ffa8e2bd6e49f7", + "n": 5, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "17cd31c2cd2d2ccbabdfab902aee6e4e759167d126e1bcaf4d638c23b93272ac", + "n": 17, + "accuracy": 0.9411764705882353, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "18aa7b3932a87bfdfc7e3f032e91b38999f82e037146754bd8bba044870170ac", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "191618c5651b68d639db0f90f67315430ae08a9467ebc761ca10a54847034de5", + "n": 20, + "accuracy": 0.65, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "1e7cf0ee593015038b9cf7686c7db7604b7de8b52d0a8b556038b7795a59dd5a", + "n": 19, + "accuracy": 0.9473684210526315, + "true_positive_n": 4, + "true_positive_correct": 4 + }, + { + "group": "23f95dce880a8574d254a19ac7f94f1284a556dfa61126a92844ba979083ede1", + "n": 2, + "accuracy": 0.5, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "2413f2cb2e9a29595c0278b868a30250d3573d487f99e35b3fe3836713d79e96", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "280f0d43bcdf087751b7a2b90739860bc02c2cccea845f81fae7555cdd9d7745", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "2af6f963a1301ac7cbf959fb0f63442e6b1b28be622e0d74c6b94c1e459c8bf0", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "2d4fd6af7767d8fbcfeddff36a76b5c993a49d5e45a515806ec30b1b53afa2ce", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3229724a48ff16d1ea5cff92fcbc1a29fb636760780576606a1f931cba64c7e6", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "33593514e9eb3b4a2f11fbff9a900e304d7f7088413e1a6ed36210de11e1a7cd", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "357e4fcd5f01346ea07a7b6805f5e0e94f11cd39d870ebde997c06b3ca23403b", + "n": 20, + "accuracy": 0.3, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "36be30d9fffc6077ec297a0ccde6690eb34709163c58b4b5df24236dd44c0e91", + "n": 18, + "accuracy": 0.9444444444444444, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3788764c156c1d8107b5e65fc226f169b6cfeb28e92341b3776e0323894d1182", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "397381ff8e300cf10164a627a5a6e614e68a7fb2a4dbac6daa5c67e331248cb3", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "3a3a67375cabf27ee46ffe17a8e3122e2bc0d77c22a4bc5fd3caf21af2fdf827", + "n": 9, + "accuracy": 0.8888888888888888, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "3ba2e76ba41ace83ac5fc70feb81f7bd790ebcc01cef912db9a520e0659301e6", + "n": 20, + "accuracy": 0.25, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "3c519ebea078526e0c7724d67b71d0e9d928d62adbb68de9cc3ab45caeab0ba6", + "n": 20, + "accuracy": 0.35, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "42a20c36367ded75633c56b5468efe4dab76b5242f7fc0fafe5de10ae8b9a078", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "4f4986e7532d0ba637710cb0010206dd22720f9b35cc163fd2f833fc93c25334", + "n": 14, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "52ed406f643d4adbab570be68c48860e0e9f239fe00b7f1bd0c35e00868fc879", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "557c22b05ccf851b94b1fc4571234d237e761ded29b151b3a3adcacd17f9ba00", + "n": 5, + "accuracy": 0.6, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "58f3e1e6f3eca7efe52c2f97b888733f6a541ca293e36d6e1a1f22c2332d05fb", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5d11eb324b736d8f810b14b708ea1982028cb405fb0fb2e3f20dab9ab5bb3fe1", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "5d2fc2acbf5e2afa16d5ad87f1cce4b784cdd43ca2da54c3eea33521e52a5a81", + "n": 20, + "accuracy": 0.45, + "true_positive_n": 10, + "true_positive_correct": 0 + }, + { + "group": "5f21eec69194bcd2e80220b2701002f2da10e7cd2b04625674cdf69feb57dedd", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "61c9e0872a43ed30a90ab06f7a3e8b85d33f54355b964a7fd1ffa361531c5626", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "61e7fb88134413ce324703852a01f81dda0cb96d4ae07a44cf845c9b1de8baa0", + "n": 20, + "accuracy": 0.7, + "true_positive_n": 13, + "true_positive_correct": 9 + }, + { + "group": "6201b69130be0b440362fc95b4cc2ef6c4c856543d9beb9584d43899e72d0283", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "629dffdc159a2dbb566dab56a6f17419a63aa2b0815020338d686277ff90f623", + "n": 20, + "accuracy": 0.4, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "63b98074727eecae10e68ab106f9c5f2ec98bd2347e0c3eaa72d5557c1ae1b64", + "n": 7, + "accuracy": 0.5714285714285714, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "63deca8572354a01b85154c34b64122772c89023b59c7931064bb347d8389e7c", + "n": 14, + "accuracy": 0.5714285714285714, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "659e4d7e826303db51376a34ed1ee1187305d8b4fcd2ba86de0f1df79cbe9d09", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 20, + "true_positive_correct": 20 + }, + { + "group": "681a13deadc6006e0d4d996f2b006e35b1d71fdeaaebfe98db36b84ce9e1bb82", + "n": 9, + "accuracy": 0.8888888888888888, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6972efb6c1c8c5719fc446f6b4e9bbf812c8274c04ddce38ae95cf5b06e65e10", + "n": 12, + "accuracy": 0.9166666666666666, + "true_positive_n": 5, + "true_positive_correct": 5 + }, + { + "group": "6d2a8e5968c904fbeceb057dc6d374d5cd1ba38019b8d31da27cc658c12aa476", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6d5ea32830050e4ac2840eed05d6b46ec9d7a78f65fbf85d0742d83cc628fa9c", + "n": 20, + "accuracy": 0.4, + "true_positive_n": 7, + "true_positive_correct": 0 + }, + { + "group": "6d74b56adb5791668e05d491f03f083060be7b184f47660a3b9cc14d7a32b511", + "n": 13, + "accuracy": 1.0, + "true_positive_n": 13, + "true_positive_correct": 13 + }, + { + "group": "6e73fd09bd5175004650b33517e33671c77ebb649384e14ffd95309ac3e567a7", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6f07feecb02e317c2db269ed9ec22266aa9195e11612cf7362bcdd1f3c9a7d3c", + "n": 20, + "accuracy": 0.95, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "6fc0443881957355817ec4b23a2d3027d6ce1ba949d0c313ac5134c5a33d7752", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "725e8bad3bf121140d3f1e8bb8625fac12766a1a7ebea88a432b39f8af246111", + "n": 20, + "accuracy": 0.95, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "745320acbe1a562a24246846ea4e7c92fe29683dd9e75636a6183142147be543", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "77af40ba5530d0abf3b141de72a4c48f4d594f8bc50515d71d71bb0ef57ba861", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "798e40bc3bfcfea7bb0935e395494ac4836e1d49f294b2ca803f7e234a478493", + "n": 4, + "accuracy": 0.5, + "true_positive_n": 4, + "true_positive_correct": 2 + }, + { + "group": "79e0a2063b1c1395f9032d7bcca4a7987b25c134851f2b7087535b66973f8767", + "n": 3, + "accuracy": 0.3333333333333333, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "7c2c1e933c21cfd9f0534226991df5542463f0d2c40138378024cbbabc1f205f", + "n": 20, + "accuracy": 0.55, + "true_positive_n": 7, + "true_positive_correct": 1 + }, + { + "group": "7d092ad55211811544dc5993b69d906906fce9350e187508760b25a549bbc9ad", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 5, + "true_positive_correct": 0 + }, + { + "group": "8269675fe1c1fcb6500b8262056237a7eb277d8d7822e63cc1b335c9b3587400", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 20, + "true_positive_correct": 0 + }, + { + "group": "8433ad4a51679bb9fc21396e090e351720fb331d25292de538c1e1a8ca524bab", + "n": 8, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8d353f585c30ae854c944e56bcab8cfff771273fefcde8d929e6faf7cb75488f", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8da5a2b78094becb52357364f9f1ea558c4052e869e9636c912fa88cce650666", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8e11912bbdeb4ce0b56ca82a5e0ad37a9159b78dd620607541dc93cb9ac8e02e", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "8f019450f830cbc42ded96cce0b7b4c1ee38439a9e198406ffb38ba5e0524fd7", + "n": 4, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "90994dcfeb71c7a87a4784556f5bf0bdb0b542c15a5f6b92da96e8020c4f1bf1", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "94d54be3b552337f2fc772ce8ff09d2b668ec497130ded8b947506a120acfb4e", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 14, + "true_positive_correct": 14 + }, + { + "group": "97d7c3f3b10a157cdb2791ff320a99a7d1856f8bbdd7ee020ad9ddbe7ec14b8e", + "n": 13, + "accuracy": 0.15384615384615385, + "true_positive_n": 13, + "true_positive_correct": 2 + }, + { + "group": "97f8ab62aee5d5d53245a565b836e892a9a7cda46748be23097421abb76ee39a", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 6, + "true_positive_correct": 0 + }, + { + "group": "99a049ff4aba4bc20df5f424b24efb9bab9118c192fd17f29b31ec910e497def", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "9b54c928a43ef6ff8b5b53cfeaad5e554106ed4d6718d08bc4f7440e968414af", + "n": 20, + "accuracy": 0.5, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "9bcc91cc05b92cb30cead1b81acb706497144e4815da75642e99d3fd7ebfd154", + "n": 11, + "accuracy": 0.9090909090909091, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "9d2f045465f59eb2bcb8a37818341ce3635a125bced7bfbcb7bf9acdd80c1a71", + "n": 20, + "accuracy": 0.0, + "true_positive_n": 20, + "true_positive_correct": 0 + }, + { + "group": "9e08a63fef975cba0f7fe19b9931144a9480e43b5b78497f97cfe4079c916045", + "n": 10, + "accuracy": 0.5, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "9e38ecba2d710c7a9e3c2bef6abd472773b0edae845c5af26f317a4f5a2a3bc2", + "n": 19, + "accuracy": 0.42105263157894735, + "true_positive_n": 7, + "true_positive_correct": 6 + }, + { + "group": "a04b59b050c050568baa9abccd205e73547f1e5111afaabdbcbe5949c3511ec6", + "n": 20, + "accuracy": 0.05, + "true_positive_n": 3, + "true_positive_correct": 0 + }, + { + "group": "a1e75cda474e69786231632ed64953c19d60658191ea63553964e1d27015a7ce", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a52745698cb48cb1a28ac9bf212d724174c320e4648c9bececd83422b3364c1d", + "n": 19, + "accuracy": 0.10526315789473684, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a5a3b2d9f1ca0af015ed0584c90b28830aa11e86e8311c7f8f32f26b6d4f2768", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "a68f2269c6412f5885bdd26d3023dae3411476f5dd5b2e103355d4e1b0a41bb5", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a7b13e8d802fa5d584ad83d610714c858aab70e0c1f005324a2a91e7a369aed1", + "n": 16, + "accuracy": 0.0625, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "a94db659229b95e557d461f6de9071da0762515528aa3a77d5cb870fcf0f914b", + "n": 20, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "aab21c537c79f9eee1a053d0d9f7fbea9eb1451e166156d59dd6c464104a7992", + "n": 10, + "accuracy": 0.5, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "abd52b07ebdf5d9235ca1239d9c3acbe23eb7cabdbc661e0dc93653479a095f0", + "n": 7, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "aefb87718e7fc3725adba69a4f789e9de7d638ec0a0d1f12fcff8a8735fb622c", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b095632bbe63fbdc009ed98e30a50899f6a8edc165e3be8a924a7f0f600be4c0", + "n": 20, + "accuracy": 0.45, + "true_positive_n": 18, + "true_positive_correct": 7 + }, + { + "group": "b1b47d1978d292c9b80319bd2b531d2f2af206e3bcabf9aa0c46472112638ba9", + "n": 6, + "accuracy": 0.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "b7fa0bdd60d2f2321a27ad4ef1c84b2258bb60cb55372ea8aa56643d6d7a6d2b", + "n": 20, + "accuracy": 0.05, + "true_positive_n": 15, + "true_positive_correct": 0 + }, + { + "group": "b8b4a7b5769be3d4db661d88d40336ade544fcac1e5bd2ec6606c09f83e5263c", + "n": 7, + "accuracy": 0.7142857142857143, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "b9063c091cb8deaab4ea7fc341aebc169033dfb16792a7f0f253ce89ce556510", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "bc9c57f088052889f45c0b709137c494d747464872fca72f16cfdde538281d64", + "n": 15, + "accuracy": 1.0, + "true_positive_n": 6, + "true_positive_correct": 6 + }, + { + "group": "bd4aa774324b8affeb489e7b796b6f32a9a2f46442394a8f5042c1967eac05dc", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "c0371b16a587352143c383ca9799634e07feb94a2df75ef8830adb86efe153c1", + "n": 7, + "accuracy": 0.5714285714285714, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "c045d0efa4ee5ab2be1d352625f16f6fb9e5cfbe90e02574363e83cd27bc5109", + "n": 1, + "accuracy": 0.0, + "true_positive_n": 1, + "true_positive_correct": 0 + }, + { + "group": "c98451abd4304660daabdb977564d36fdad533b4458c467bd2b97d80a651955f", + "n": 3, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "cb85febf11725dacf437b764d64bf097c9c2bab08e74ac2eeadb496ccd51b5e8", + "n": 16, + "accuracy": 0.875, + "true_positive_n": 3, + "true_positive_correct": 1 + }, + { + "group": "cfe165abb396d68223741b950f77ec55bd59d4c200bd310d7e9a956548c1590e", + "n": 8, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d530b979376d67460a1a3dcafcc95c91e1a51ce2d97124dc2050c2030ae223b7", + "n": 20, + "accuracy": 0.1, + "true_positive_n": 12, + "true_positive_correct": 0 + }, + { + "group": "d5496d95b9bcd56d2d382354f44af6098e30a6b132c6bd2ccbac429ee48d7d3b", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d6e9a943c91ab8d80d913c89d053cf89f2fe4bcd7df1963f3fd5a70bf56f3deb", + "n": 12, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d794d72182a0c6f8a3009e01f6d607875a60752bf7f05920823cc55729bcbf3c", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "d82b9f42323202feb6db7ed45f9990070e72e41705aeb5aa64d60a3fd7549f17", + "n": 2, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "dfc85bf0674a0000b0dfbcf621419502a19c4fffabf786b4a5e74a69776a1f9d", + "n": 5, + "accuracy": 0.4, + "true_positive_n": 3, + "true_positive_correct": 1 + }, + { + "group": "e35b0e3bc4a28a3bcf49fd9fe5c6e3c6ece59e921ecabf1904c9459cb7b84ad1", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "e6e40718557ace0b1764c2b7ce7021ff2b42e61db43cd59c7a54132f69807b2d", + "n": 7, + "accuracy": 0.5714285714285714, + "true_positive_n": 1, + "true_positive_correct": 1 + }, + { + "group": "f5b769ca29423ada31602cdd07bba0ab057d937256622788fa0872f6fe095865", + "n": 4, + "accuracy": 0.75, + "true_positive_n": 2, + "true_positive_correct": 2 + }, + { + "group": "f72e0a3b72341d46d7eb5ab932fcce07cf34702afa40d217c5e997bef74764be", + "n": 15, + "accuracy": 0.6666666666666666, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "f9476bc6e284b58a4d168d14b188b2ca1636ebe4a0adfcb0c40060bed41781c8", + "n": 3, + "accuracy": 0.3333333333333333, + "true_positive_n": 2, + "true_positive_correct": 0 + }, + { + "group": "f97f37793bfaa1b487125bd7b1f34bff0fed50146a5606962c763220f17bb6de", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fb39d2600c554fe3ac1eddb65704bc34c6102ca0fbb4a77744ac525869d35be4", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fcd64091f029e53dd4a770300ecf446dbf5bca8cc267148f28b198584caeae46", + "n": 1, + "accuracy": 1.0, + "true_positive_n": 0, + "true_positive_correct": 0 + }, + { + "group": "fe703c5eb241d3542fcc57e11688b3641c4f1f0b9506a720b42a9b9031382154", + "n": 19, + "accuracy": 0.7368421052631579, + "true_positive_n": 2, + "true_positive_correct": 0 + } + ], + "correct": 638 + } + }, + "swarm_static": { + "raw": { + "n": 354, + "accuracy": 0.9745762711864406, + "nll": 0.06671770193368179, + "brier": 0.03589778460324393, + "correct": 345 + }, + "global_calibrated": { + "n": 354, + "accuracy": 0.9745762711864406, + "nll": 0.08603305952328294, + "brier": 0.03970483443698841, + "correct": 345 + }, + "families": { + "binding": { + "raw": { + "n": 164, + "accuracy": 0.9512195121951219, + "nll": 0.12986993405061373, + "brier": 0.06690797882642434, + "correct": 156 + }, + "global_calibrated": { + "n": 164, + "accuracy": 0.9512195121951219, + "nll": 0.17000841292483287, + "brier": 0.07668135319778648, + "correct": 156 + } + }, + "direct": { + "raw": { + "n": 134, + "accuracy": 1.0, + "nll": 0.0002463532494033971, + "brier": 5.956582865744346e-06, + "correct": 134 + }, + "global_calibrated": { + "n": 134, + "accuracy": 1.0, + "nll": 0.003880609414537556, + "brier": 0.00015479616373253736, + "correct": 134 + } + }, + "flow": { + "raw": { + "n": 56, + "accuracy": 0.9821428571428571, + "nll": 0.040828320800047294, + "brier": 0.030966232855549104, + "correct": 55 + }, + "global_calibrated": { + "n": 56, + "accuracy": 0.9821428571428571, + "nll": 0.03668431589324172, + "brier": 0.02605404964851352, + "correct": 55 + } } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 26, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" } }, - "calibrated": { - "parent": { - "acc": 0.3111111111111111, - "nll": 1.1519141587962758, - "brier": 0.6972118196346561, - "ece": 0.12505951037027957 - }, - "candidate": { - "acc": 0.3888888888888889, - "nll": 1.0987834540411583, - "brier": 0.6666360545746582, - "ece": 0.03892731596092908 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.07777777777777778, - "ci95": [ - -0.12229166666666667, - 0.26746313102665914 - ], - "parent_ci95": [ - 0.20512820512820512, - 0.425 - ], - "candidate_ci95": [ - 0.28888888888888886, - 0.48984374999999974 - ] - }, - "nll": { - "candidate_minus_parent": -0.0531307047551175, - "ci95": [ - -0.13131384096372054, - 0.02030876595375617 - ], - "parent_ci95": [ - 1.0836231878965614, - 1.2232119820149001 - ], - "candidate_ci95": [ - 1.087965251457569, - 1.10803281863799 - ] - }, - "brier": { - "candidate_minus_parent": -0.030575765059997817, - "ci95": [ - -0.08053924500142515, - 0.016278720227622917 - ], - "parent_ci95": [ - 0.6536499526341443, - 0.7418442945165746 - ], - "candidate_ci95": [ - 0.6592384181572574, - 0.6728962291231819 - ] - }, - "ece": { - "candidate_minus_parent": -0.0861321944093505, - "ci95": [ - -0.1587048303302882, - 0.024643558296109955 - ], - "parent_ci95": [ - 0.02664428234558302, - 0.22892663757370982 - ], - "candidate_ci95": [ - 0.002359617988633989, - 0.14107288255595657 - ] + "manual": { + "raw": { + "n": 83, + "accuracy": 0.7831325301204819, + "nll": 1.1015953331868638, + "brier": 0.38672041228910636, + "correct": 65 + }, + "global_calibrated": { + "n": 83, + "accuracy": 0.7831325301204819, + "nll": 0.8197108651023933, + "brier": 0.3693842986364894, + "correct": 65 + }, + "families": { + "binding": { + "raw": { + "n": 40, + "accuracy": 0.8, + "nll": 0.906880605987712, + "brier": 0.3722274315061049, + "correct": 32 + }, + "global_calibrated": { + "n": 40, + "accuracy": 0.8, + "nll": 0.7146033829092314, + "brier": 0.3514071574500127, + "correct": 32 + } + }, + "flow": { + "raw": { + "n": 40, + "accuracy": 0.75, + "nll": 1.3789279125484732, + "brier": 0.43021742389448914, + "correct": 30 + }, + "global_calibrated": { + "n": 40, + "accuracy": 0.75, + "nll": 0.9861678633098097, + "brier": 0.4150648418384071, + "correct": 30 + } + }, + "prerequisite": { + "raw": { + "n": 3, + "accuracy": 1.0, + "nll": 2.3971020760583907e-05, + "brier": 1.3240221828953951e-09, + "correct": 3 + }, + "global_calibrated": { + "n": 3, + "accuracy": 1.0, + "nll": 0.0017173182456664555, + "brier": 5.60509727845392e-06, + "correct": 3 + } } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 26, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "sysmon": { + "raw": { + "n": 64, + "accuracy": 0.96875, + "nll": 0.08322377416424186, + "brier": 0.045695299016738855, + "correct": 62 + }, + "global_calibrated": { + "n": 64, + "accuracy": 0.96875, + "nll": 0.11024750472054996, + "brier": 0.04954806836760281, + "correct": 62 + } + }, + "general": { + "raw": { + "n": 105, + "accuracy": 0.819047619047619, + "nll": 0.5574328669578856, + "brier": 0.29243113448917374, + "correct": 86 + }, + "global_calibrated": { + "n": 105, + "accuracy": 0.819047619047619, + "nll": 0.42292499037031256, + "brier": 0.24649933996243034, + "correct": 86 } } + } + } + }, + "admissions": { + "parent": { + "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-swarmtraces-4b-v2/00-checkpoint", + "files": { + "result.json": "46b3ee2a68f250ba1d433c25612083bb8235a03e617d90707fa796f86ae2cbf4", + "calibration/rows.json": "9a33b44bee9cb63a635ac1516bcab9f67fbe56b15a72995a8c2b54a321ad4dd6", + "development/rows.json": "d882c91290398ba96cf39de9128ea5a973cf8d79d20ca84f5249f9f1c1cd5300", + "provenance.json": "39898e5e6ef87112f653436a9b6b80555da10a01b0d203b3f5eb29000dddae3a", + "calibration/report.json": "15dc07e79e0121e7345ece52b028220fbbf9123609a2eee5b331c9d372370383", + "development/report.json": "02fc22461d5b44f88173b90e0fffe97905b4eeb132eec79088c9a8396ce73134", + "transfer/rows.json": "5b624c0b0f91940332045523703dd5d52761a5ff931f2c9f1751185883976368", + "transfer/report.json": "f44bc7be71a3b37ded9e35f6fa382c1907c70fc959c15bc712b76f024ac726cc" }, - "full/iid/transactions_3_plus": { - "windows": 133, - "episodes": 93, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 24, - "4": 26, - "5": 41, - "6": 38, - "7": 4 + "provenance": { + "config": {}, + "config_sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "suite_sha256": "61e3c1898b8a03eb178355d3c8c5050e85bd217285c700df9b4506ec9e3680f8", + "source_hashes": { + "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb", + "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582", + "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a", + "kev/benchmark.py": "8d486659a9036102386b077fcbd197fc0d85ae7752a4711666d6b3a95622b26f", + "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519", + "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59", + "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435", + "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", + "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", + "kev/data.py": "a5a5e200f56b60e4bf30a195a4cc87a74afddae15999ec48c7543a4f9b59f71c", + "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4", + "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c", + "kev/experiment.py": "1a5ca47d2b3d236e53c87e8c4197704f162c6692aa008521d31ea491945693f0", + "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243", + "kev/metrics.py": "db0930d470c0ce16f16dcaf726d49a7a103973a5296cefc5259d1e68707d882b", + "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909", + "kev/model.py": "eac0f56c807b85b5c4077483d922e6bb03f07890de2835efd9372cf20cd01b8e", + "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02", + "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3", + "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa", + "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735", + "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3", + "kev/suite.py": "c5e5e27f2fed871b547a021d180b0a7f4a0542a488cce4ee02ad936f433441db", + "kev/train.py": "a2d85632d73760fb3efe51627ff4fde6af3100ed1c34e75d290c1b00c9899aa9", + "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763", + "modal_app.py": "639577258f4d581daefa51dad639961280ce92de9da256316ab4124f005a53fb", + "pyproject.toml": "39ec40f14c88598dee7ac98fb859a5a44e22337235a3064ebd5e89036ce81e30", + "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466" }, - "raw": { - "parent": { - "acc": 0.3118279569892473, - "nll": 1.8007600408262734, - "brier": 0.9912122366031354, - "ece": 0.41777594278222113 - }, - "candidate": { - "acc": 0.34050179211469533, - "nll": 1.0897724637391926, - "brier": 0.6608081864876919, - "ece": 0.06945969138416776 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.02867383512544802, - "ci95": [ - -0.00347222222222221, - 0.06666666666666671 - ], - "parent_ci95": [ - 0.28888888888888886, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3263157894736842, - 0.35555555555555557 - ] - }, - "nll": { - "candidate_minus_parent": -0.7109875770870808, - "ci95": [ - -0.8249378106097072, - -0.5928019108611865 - ], - "parent_ci95": [ - 1.6800381011917274, - 1.9200895008447456 - ], - "candidate_ci95": [ - 1.0736534993392457, - 1.1068153124221158 - ] - }, - "brier": { - "candidate_minus_parent": -0.3304040501154435, - "ci95": [ - -0.3793116868541628, - -0.28256738524317954 - ], - "parent_ci95": [ - 0.9404112325282276, - 1.0398186656005743 - ], - "candidate_ci95": [ - 0.6494179878255043, - 0.6727965288233754 - ] - }, - "ece": { - "candidate_minus_parent": -0.3483162513980534, - "ci95": [ - -0.40399898613809715, - -0.29044166032303326 - ], - "parent_ci95": [ - 0.38626474035396524, - 0.473126989756557 - ], - "candidate_ci95": [ - 0.05996149153641762, - 0.11740475732652438 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } + "git_commit": "449101a53d4385bac3aef7df4fee5a0d93248953", + "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36", + "torch": "2.8.0+cu128", + "device": "cuda", + "gpu": "NVIDIA H100 80GB HBM3", + "legacy_checkpoint": true, + "measured_checkpoint": { + "requested": "/runs/sev-r2-curriculum-repair-4b-v1/02-trial-2/checkpoint", + "resolved": "/runs/sev-r2-curriculum-repair-4b-v1/02-trial-2/checkpoint", + "head_sha256": "5a91ed82f11c6d0c8ab0a910e0c85770119e0034039379aa601d17a3ec7664d6", + "adapter_sha256": "0452010abdcc58f7c39a47a1562a1c812bf85f025d4a6c105c93d486051765de", + "inference_temperature": 1.0 + } + }, + "experiment_gates": { + "passed": true, + "checks": { + "complete_coverage": true, + "isolation_and_packing": true, + "transfer_complete": true, + "heldout_pairs_at_least_70pct": true, + "transfer_confident_errors_below_10pct": true }, - "calibrated": { - "parent": { - "acc": 0.3118279569892473, - "nll": 1.171633543085405, - "brier": 0.7154649932664467, - "ece": 0.14465601993335808 - }, - "candidate": { - "acc": 0.34050179211469533, - "nll": 1.088970247788087, - "brier": 0.6599466665496634, - "ece": 0.055631274925033 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.02867383512544802, - "ci95": [ - -0.00347222222222221, - 0.06666666666666671 - ], - "parent_ci95": [ - 0.28888888888888886, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3263157894736842, - 0.35555555555555557 - ] - }, - "nll": { - "candidate_minus_parent": -0.08266329529731808, - "ci95": [ - -0.10527160530287753, - -0.060813425309605006 - ], - "parent_ci95": [ - 1.1497915115579906, - 1.1953183319652614 - ], - "candidate_ci95": [ - 1.0798257200325903, - 1.0975630581184335 - ] - }, - "brier": { - "candidate_minus_parent": -0.05551832671678336, - "ci95": [ - -0.07142755322765607, - -0.040168620031008025 - ], - "parent_ci95": [ - 0.7003199621121581, - 0.7316456618117463 - ], - "candidate_ci95": [ - 0.6534735207307115, - 0.6661281632812587 - ] - }, - "ece": { - "candidate_minus_parent": -0.08902474500832508, - "ci95": [ - -0.15777008194974856, - -0.032028872805473535 - ], - "parent_ci95": [ - 0.11917374104868377, - 0.19726621817887172 - ], - "candidate_ci95": [ - 0.02223594121005041, - 0.1014834181087044 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/all/all": { - "windows": 148, - "episodes": 93, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 28, - "2": 23, - "3": 20, - "4": 41, - "5": 14, - "6": 15, - "7": 7 - }, - "raw": { - "parent": { - "acc": 0.3064516129032258, - "nll": 1.703684494187649, - "brier": 0.9537202801199026, - "ece": 0.4129881914164849 - }, - "candidate": { - "acc": 0.3333333333333333, - "nll": 1.091192212148124, - "brier": 0.6614718541960379, - "ece": 0.06541034126839912 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.026881720430107503, - "ci95": [ - -0.012152777777777736, - 0.07222222222222224 - ], - "parent_ci95": [ - 0.2777777777777778, - 0.328125 - ], - "candidate_ci95": [ - 0.3120567375886525, - 0.3518518518518518 - ] - }, - "nll": { - "candidate_minus_parent": -0.6124922820395251, - "ci95": [ - -0.7118571153712705, - -0.5197225076836384 - ], - "parent_ci95": [ - 1.6099049018770706, - 1.8069063026502517 - ], - "candidate_ci95": [ - 1.0755336874829131, - 1.108735959121408 - ] - }, - "brier": { - "candidate_minus_parent": -0.2922484259238647, - "ci95": [ - -0.3463368357881872, - -0.24465821940647325 - ], - "parent_ci95": [ - 0.906523256055364, - 1.006988521580945 - ], - "candidate_ci95": [ - 0.6498969445817571, - 0.6745257883818703 - ] - }, - "ece": { - "candidate_minus_parent": -0.3475778501480858, - "ci95": [ - -0.39278455794407824, - -0.2746709901865958 - ], - "parent_ci95": [ - 0.38440294829434885, - 0.4484787433230376 - ], - "candidate_ci95": [ - 0.04616834389587247, - 0.12818960972025695 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3064516129032258, - "nll": 1.1529331119038668, - "brier": 0.7033109417652029, - "ece": 0.13902595443424992 - }, - "candidate": { - "acc": 0.3333333333333333, - "nll": 1.0916634954036812, - "brier": 0.661974002661268, - "ece": 0.05073703452140449 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.026881720430107503, - "ci95": [ - -0.012152777777777736, - 0.07222222222222224 - ], - "parent_ci95": [ - 0.2777777777777778, - 0.328125 - ], - "candidate_ci95": [ - 0.3120567375886525, - 0.3518518518518518 - ] - }, - "nll": { - "candidate_minus_parent": -0.061269616500185586, - "ci95": [ - -0.0815360160872045, - -0.042796021724386325 - ], - "parent_ci95": [ - 1.1342530883801942, - 1.1746089245476712 - ], - "candidate_ci95": [ - 1.0841653219251408, - 1.099847160901972 - ] - }, - "brier": { - "candidate_minus_parent": -0.04133693910393488, - "ci95": [ - -0.054945952000430426, - -0.0283071065059587 - ], - "parent_ci95": [ - 0.6901203817927404, - 0.7179063393968591 - ], - "candidate_ci95": [ - 0.6566300081821984, - 0.6678751694064711 - ] - }, - "ece": { - "candidate_minus_parent": -0.08828891991284543, - "ci95": [ - -0.13567816051255593, - -0.03214176910943423 - ], - "parent_ci95": [ - 0.11565183132825865, - 0.17029153615689344 - ], - "candidate_ci95": [ - 0.020278667356217008, - 0.09905536546083844 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/all/transactions_1_2": { - "windows": 51, - "episodes": 49, - "latent_groups": 27, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 28, - "2": 23 - }, - "raw": { - "parent": { - "acc": 0.3469387755102041, - "nll": 1.5751052969851513, - "brier": 0.8802942551419999, - "ece": 0.35030273260170447 - }, - "candidate": { - "acc": 0.2857142857142857, - "nll": 1.1068816661995116, - "brier": 0.6720929561492037, - "ece": 0.08071820859513017 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.06122448979591838, - "ci95": [ - -0.20010204081632654, - 0.08510638297872342 - ], - "parent_ci95": [ - 0.2640947888589398, - 0.4318181818181818 - ], - "candidate_ci95": [ - 0.19990196078431374, - 0.36958534621578093 - ] - }, - "nll": { - "candidate_minus_parent": -0.4682236307856398, - "ci95": [ - -0.6874408585543053, - -0.25059914312634723 - ], - "parent_ci95": [ - 1.3660307447207953, - 1.7863481180513798 - ], - "candidate_ci95": [ - 1.0954611962384864, - 1.1205296358359653 - ] - }, - "brier": { - "candidate_minus_parent": -0.20820129899279616, - "ci95": [ - -0.3214504033746869, - -0.09218633279444037 - ], - "parent_ci95": [ - 0.7681875137354262, - 0.9905470857848545 - ], - "candidate_ci95": [ - 0.6642986501083062, - 0.6812307682914585 - ] - }, - "ece": { - "candidate_minus_parent": -0.2695845240065743, - "ci95": [ - -0.3904225741241218, - -0.13182069065959087 - ], - "parent_ci95": [ - 0.2684022133739064, - 0.4331878336118929 - ], - "candidate_ci95": [ - 0.015381404617433004, - 0.16679240726105588 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 27, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3469387755102041, - "nll": 1.1306501839467744, - "brier": 0.6866799099052595, - "ece": 0.1003196557889501 - }, - "candidate": { - "acc": 0.2857142857142857, - "nll": 1.101825014834903, - "brier": 0.6687795902311308, - "ece": 0.0639734338035598 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.06122448979591838, - "ci95": [ - -0.20010204081632654, - 0.08510638297872342 - ], - "parent_ci95": [ - 0.2640947888589398, - 0.4318181818181818 - ], - "candidate_ci95": [ - 0.19990196078431374, - 0.36958534621578093 - ] - }, - "nll": { - "candidate_minus_parent": -0.028825169111871496, - "ci95": [ - -0.08287051694186583, - 0.02922920870399774 - ], - "parent_ci95": [ - 1.0776980040903783, - 1.1807414922599382 - ], - "candidate_ci95": [ - 1.0959848691935723, - 1.1086465278602198 - ] - }, - "brier": { - "candidate_minus_parent": -0.01790031967412875, - "ci95": [ - -0.05386955921741074, - 0.020204736241254864 - ], - "parent_ci95": [ - 0.6515174635881781, - 0.7204922749907883 - ], - "candidate_ci95": [ - 0.6648958894226078, - 0.6733526771528309 - ] - }, - "ece": { - "candidate_minus_parent": -0.036346221985390295, - "ci95": [ - -0.1650347028291009, - 0.09366396409555656 - ], - "parent_ci95": [ - 0.024516912704449893, - 0.1972711957182588 - ], - "candidate_ci95": [ - 0.0035596289835879147, - 0.1498403229449324 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 27, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/all/transactions_3_plus": { - "windows": 97, - "episodes": 75, - "latent_groups": 29, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 20, - "4": 41, - "5": 14, - "6": 15, - "7": 7 - }, - "raw": { - "parent": { - "acc": 0.28, - "nll": 1.7666116286336437, - "brier": 0.9763766869892648, - "ece": 0.4315626097972386 - }, - "candidate": { - "acc": 0.36666666666666664, - "nll": 1.0864975658938738, - "brier": 0.6587212588511653, - "ece": 0.044820232890561064 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.08666666666666661, - "ci95": [ - 0.024691358024691357, - 0.16911764705882354 - ], - "parent_ci95": [ - 0.23680340557275542, - 0.3125 - ], - "candidate_ci95": [ - 0.3291139240506329, - 0.41304347826086957 - ] - }, - "nll": { - "candidate_minus_parent": -0.6801140627397699, - "ci95": [ - -0.852694438368166, - -0.5364090061436311 - ], - "parent_ci95": [ - 1.6363811561091484, - 1.9277416812066965 - ], - "candidate_ci95": [ - 1.0565640880278935, - 1.1122708307393117 - ] - }, - "brier": { - "candidate_minus_parent": -0.31765542813809955, - "ci95": [ - -0.4055508030920121, - -0.24603262483477328 - ], - "parent_ci95": [ - 0.9125704809328496, - 1.0576784187561903 - ], - "candidate_ci95": [ - 0.6375226558255674, - 0.677367412345545 - ] - }, - "ece": { - "candidate_minus_parent": -0.3867423769066775, - "ci95": [ - -0.43813835904088466, - -0.299041074051707 - ], - "parent_ci95": [ - 0.38732821796041755, - 0.4880029459977231 - ], - "candidate_ci95": [ - 0.027281818992051166, - 0.12376763110665068 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.28, - "nll": 1.1695120669775345, - "brier": 0.7138176453650833, - "ece": 0.16388566024997767 - }, - "candidate": { - "acc": 0.36666666666666664, - "nll": 1.0880931619979777, - "brier": 0.6597052729288035, - "ece": 0.020713258604670517 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.08666666666666661, - "ci95": [ - 0.024691358024691357, - 0.16911764705882354 - ], - "parent_ci95": [ - 0.23680340557275542, - 0.3125 - ], - "candidate_ci95": [ - 0.3291139240506329, - 0.41304347826086957 - ] - }, - "nll": { - "candidate_minus_parent": -0.08141890497955684, - "ci95": [ - -0.12091422490095782, - -0.04948136248698705 - ], - "parent_ci95": [ - 1.143868559330941, - 1.2022605906182802 - ], - "candidate_ci95": [ - 1.0734995012679627, - 1.1004991802450863 - ] - }, - "brier": { - "candidate_minus_parent": -0.054112372436279754, - "ci95": [ - -0.08087458322037622, - -0.03322586306298517 - ], - "parent_ci95": [ - 0.6965512084324665, - 0.7354555338983582 - ], - "candidate_ci95": [ - 0.6493246096824813, - 0.6685547109622129 - ] - }, - "ece": { - "candidate_minus_parent": -0.14317240164530715, - "ci95": [ - -0.17492207253743788, - -0.08263253263041898 - ], - "parent_ci95": [ - 0.12860078933903504, - 0.20826683422962747 - ], - "candidate_ci95": [ - 0.009737212666042754, - 0.07540991325472185 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/iid/all": { - "windows": 148, - "episodes": 93, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 28, - "2": 23, - "3": 20, - "4": 41, - "5": 14, - "6": 15, - "7": 7 - }, - "raw": { - "parent": { - "acc": 0.3064516129032258, - "nll": 1.703684494187649, - "brier": 0.9537202801199026, - "ece": 0.4129881914164849 - }, - "candidate": { - "acc": 0.3333333333333333, - "nll": 1.091192212148124, - "brier": 0.6614718541960379, - "ece": 0.06541034126839912 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.026881720430107503, - "ci95": [ - -0.012152777777777736, - 0.07222222222222224 - ], - "parent_ci95": [ - 0.2777777777777778, - 0.328125 - ], - "candidate_ci95": [ - 0.3120567375886525, - 0.3518518518518518 - ] - }, - "nll": { - "candidate_minus_parent": -0.6124922820395251, - "ci95": [ - -0.7118571153712705, - -0.5197225076836384 - ], - "parent_ci95": [ - 1.6099049018770706, - 1.8069063026502517 - ], - "candidate_ci95": [ - 1.0755336874829131, - 1.108735959121408 - ] - }, - "brier": { - "candidate_minus_parent": -0.2922484259238647, - "ci95": [ - -0.3463368357881872, - -0.24465821940647325 - ], - "parent_ci95": [ - 0.906523256055364, - 1.006988521580945 - ], - "candidate_ci95": [ - 0.6498969445817571, - 0.6745257883818703 - ] - }, - "ece": { - "candidate_minus_parent": -0.3475778501480858, - "ci95": [ - -0.39278455794407824, - -0.2746709901865958 - ], - "parent_ci95": [ - 0.38440294829434885, - 0.4484787433230376 - ], - "candidate_ci95": [ - 0.04616834389587247, - 0.12818960972025695 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3064516129032258, - "nll": 1.1529331119038668, - "brier": 0.7033109417652029, - "ece": 0.13902595443424992 - }, - "candidate": { - "acc": 0.3333333333333333, - "nll": 1.0916634954036812, - "brier": 0.661974002661268, - "ece": 0.05073703452140449 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.026881720430107503, - "ci95": [ - -0.012152777777777736, - 0.07222222222222224 - ], - "parent_ci95": [ - 0.2777777777777778, - 0.328125 - ], - "candidate_ci95": [ - 0.3120567375886525, - 0.3518518518518518 - ] - }, - "nll": { - "candidate_minus_parent": -0.061269616500185586, - "ci95": [ - -0.0815360160872045, - -0.042796021724386325 - ], - "parent_ci95": [ - 1.1342530883801942, - 1.1746089245476712 - ], - "candidate_ci95": [ - 1.0841653219251408, - 1.099847160901972 - ] - }, - "brier": { - "candidate_minus_parent": -0.04133693910393488, - "ci95": [ - -0.054945952000430426, - -0.0283071065059587 - ], - "parent_ci95": [ - 0.6901203817927404, - 0.7179063393968591 - ], - "candidate_ci95": [ - 0.6566300081821984, - 0.6678751694064711 - ] - }, - "ece": { - "candidate_minus_parent": -0.08828891991284543, - "ci95": [ - -0.13567816051255593, - -0.03214176910943423 - ], - "parent_ci95": [ - 0.11565183132825865, - 0.17029153615689344 - ], - "candidate_ci95": [ - 0.020278667356217008, - 0.09905536546083844 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/iid/transactions_1_2": { - "windows": 51, - "episodes": 49, - "latent_groups": 27, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 28, - "2": 23 - }, - "raw": { - "parent": { - "acc": 0.3469387755102041, - "nll": 1.5751052969851513, - "brier": 0.8802942551419999, - "ece": 0.35030273260170447 - }, - "candidate": { - "acc": 0.2857142857142857, - "nll": 1.1068816661995116, - "brier": 0.6720929561492037, - "ece": 0.08071820859513017 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.06122448979591838, - "ci95": [ - -0.20010204081632654, - 0.08510638297872342 - ], - "parent_ci95": [ - 0.2640947888589398, - 0.4318181818181818 - ], - "candidate_ci95": [ - 0.19990196078431374, - 0.36958534621578093 - ] - }, - "nll": { - "candidate_minus_parent": -0.4682236307856398, - "ci95": [ - -0.6874408585543053, - -0.25059914312634723 - ], - "parent_ci95": [ - 1.3660307447207953, - 1.7863481180513798 - ], - "candidate_ci95": [ - 1.0954611962384864, - 1.1205296358359653 - ] - }, - "brier": { - "candidate_minus_parent": -0.20820129899279616, - "ci95": [ - -0.3214504033746869, - -0.09218633279444037 - ], - "parent_ci95": [ - 0.7681875137354262, - 0.9905470857848545 - ], - "candidate_ci95": [ - 0.6642986501083062, - 0.6812307682914585 - ] - }, - "ece": { - "candidate_minus_parent": -0.2695845240065743, - "ci95": [ - -0.3904225741241218, - -0.13182069065959087 - ], - "parent_ci95": [ - 0.2684022133739064, - 0.4331878336118929 - ], - "candidate_ci95": [ - 0.015381404617433004, - 0.16679240726105588 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 27, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3469387755102041, - "nll": 1.1306501839467744, - "brier": 0.6866799099052595, - "ece": 0.1003196557889501 - }, - "candidate": { - "acc": 0.2857142857142857, - "nll": 1.101825014834903, - "brier": 0.6687795902311308, - "ece": 0.0639734338035598 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.06122448979591838, - "ci95": [ - -0.20010204081632654, - 0.08510638297872342 - ], - "parent_ci95": [ - 0.2640947888589398, - 0.4318181818181818 - ], - "candidate_ci95": [ - 0.19990196078431374, - 0.36958534621578093 - ] - }, - "nll": { - "candidate_minus_parent": -0.028825169111871496, - "ci95": [ - -0.08287051694186583, - 0.02922920870399774 - ], - "parent_ci95": [ - 1.0776980040903783, - 1.1807414922599382 - ], - "candidate_ci95": [ - 1.0959848691935723, - 1.1086465278602198 - ] - }, - "brier": { - "candidate_minus_parent": -0.01790031967412875, - "ci95": [ - -0.05386955921741074, - 0.020204736241254864 - ], - "parent_ci95": [ - 0.6515174635881781, - 0.7204922749907883 - ], - "candidate_ci95": [ - 0.6648958894226078, - 0.6733526771528309 - ] - }, - "ece": { - "candidate_minus_parent": -0.036346221985390295, - "ci95": [ - -0.1650347028291009, - 0.09366396409555656 - ], - "parent_ci95": [ - 0.024516912704449893, - 0.1972711957182588 - ], - "candidate_ci95": [ - 0.0035596289835879147, - 0.1498403229449324 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 27, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/iid/transactions_3_plus": { - "windows": 97, - "episodes": 75, - "latent_groups": 29, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 20, - "4": 41, - "5": 14, - "6": 15, - "7": 7 - }, - "raw": { - "parent": { - "acc": 0.28, - "nll": 1.7666116286336437, - "brier": 0.9763766869892648, - "ece": 0.4315626097972386 - }, - "candidate": { - "acc": 0.36666666666666664, - "nll": 1.0864975658938738, - "brier": 0.6587212588511653, - "ece": 0.044820232890561064 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.08666666666666661, - "ci95": [ - 0.024691358024691357, - 0.16911764705882354 - ], - "parent_ci95": [ - 0.23680340557275542, - 0.3125 - ], - "candidate_ci95": [ - 0.3291139240506329, - 0.41304347826086957 - ] - }, - "nll": { - "candidate_minus_parent": -0.6801140627397699, - "ci95": [ - -0.852694438368166, - -0.5364090061436311 - ], - "parent_ci95": [ - 1.6363811561091484, - 1.9277416812066965 - ], - "candidate_ci95": [ - 1.0565640880278935, - 1.1122708307393117 - ] - }, - "brier": { - "candidate_minus_parent": -0.31765542813809955, - "ci95": [ - -0.4055508030920121, - -0.24603262483477328 - ], - "parent_ci95": [ - 0.9125704809328496, - 1.0576784187561903 - ], - "candidate_ci95": [ - 0.6375226558255674, - 0.677367412345545 - ] - }, - "ece": { - "candidate_minus_parent": -0.3867423769066775, - "ci95": [ - -0.43813835904088466, - -0.299041074051707 - ], - "parent_ci95": [ - 0.38732821796041755, - 0.4880029459977231 - ], - "candidate_ci95": [ - 0.027281818992051166, - 0.12376763110665068 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.28, - "nll": 1.1695120669775345, - "brier": 0.7138176453650833, - "ece": 0.16388566024997767 - }, - "candidate": { - "acc": 0.36666666666666664, - "nll": 1.0880931619979777, - "brier": 0.6597052729288035, - "ece": 0.020713258604670517 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.08666666666666661, - "ci95": [ - 0.024691358024691357, - 0.16911764705882354 - ], - "parent_ci95": [ - 0.23680340557275542, - 0.3125 - ], - "candidate_ci95": [ - 0.3291139240506329, - 0.41304347826086957 - ] - }, - "nll": { - "candidate_minus_parent": -0.08141890497955684, - "ci95": [ - -0.12091422490095782, - -0.04948136248698705 - ], - "parent_ci95": [ - 1.143868559330941, - 1.2022605906182802 - ], - "candidate_ci95": [ - 1.0734995012679627, - 1.1004991802450863 - ] - }, - "brier": { - "candidate_minus_parent": -0.054112372436279754, - "ci95": [ - -0.08087458322037622, - -0.03322586306298517 - ], - "parent_ci95": [ - 0.6965512084324665, - 0.7354555338983582 - ], - "candidate_ci95": [ - 0.6493246096824813, - 0.6685547109622129 - ] - }, - "ece": { - "candidate_minus_parent": -0.14317240164530715, - "ci95": [ - -0.17492207253743788, - -0.08263253263041898 - ], - "parent_ci95": [ - 0.12860078933903504, - 0.20826683422962747 - ], - "candidate_ci95": [ - 0.009737212666042754, - 0.07540991325472185 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/all/all": { - "windows": 152, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 27, - "2": 18, - "3": 24, - "4": 18, - "5": 42, - "6": 20, - "7": 3 - }, - "raw": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.6730827021190828, - "brier": 0.9271168690450743, - "ece": 0.38029185976877056 - }, - "candidate": { - "acc": 0.3489583333333333, - "nll": 1.0971521242555076, - "brier": 0.6658498861990656, - "ece": 0.046307694308096524 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.015625, - "ci95": [ - -0.00347222222222221, - 0.04171006944444442 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3298611111111111, - 0.37504340277777776 - ] - }, - "nll": { - "candidate_minus_parent": -0.5759305778635753, - "ci95": [ - -0.6646879978333292, - -0.4844753658625933 - ], - "parent_ci95": [ - 1.575759527226761, - 1.7691376887298536 - ], - "candidate_ci95": [ - 1.081862216974002, - 1.1148355704805166 - ] - }, - "brier": { - "candidate_minus_parent": -0.2612669828460087, - "ci95": [ - -0.2970389567438269, - -0.22634178418755768 - ], - "parent_ci95": [ - 0.8887463241160158, - 0.9676440292791192 - ], - "candidate_ci95": [ - 0.6550728672970835, - 0.6783244126276657 - ] - }, - "ece": { - "candidate_minus_parent": -0.33398416546067405, - "ci95": [ - -0.3670358118919271, - -0.2816153662787594 - ], - "parent_ci95": [ - 0.35101328311005003, - 0.42006858974136496 - ], - "candidate_ci95": [ - 0.024396575640987968, - 0.09070083025698743 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.1468427816323261, - "brier": 0.6981381709092208, - "ece": 0.10938489002382056 - }, - "candidate": { - "acc": 0.3489583333333333, - "nll": 1.0937687968736984, - "brier": 0.663354598517312, - "ece": 0.019037355871080977 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.015625, - "ci95": [ - -0.00347222222222221, - 0.04171006944444442 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3298611111111111, - 0.37504340277777776 - ] - }, - "nll": { - "candidate_minus_parent": -0.05307398475862768, - "ci95": [ - -0.06656368181703105, - -0.04064767349269174 - ], - "parent_ci95": [ - 1.131378684831578, - 1.165777015737088 - ], - "candidate_ci95": [ - 1.0857767794541715, - 1.102658732735975 - ] - }, - "brier": { - "candidate_minus_parent": -0.03478357239190877, - "ci95": [ - -0.04339052468546176, - -0.026690749871374275 - ], - "parent_ci95": [ - 0.6877235900544243, - 0.7106051033335131 - ], - "candidate_ci95": [ - 0.6576958496027633, - 0.66980743900186 - ] - }, - "ece": { - "candidate_minus_parent": -0.09034753415273958, - "ci95": [ - -0.13920480289089912, - -0.05692976019192213 - ], - "parent_ci95": [ - 0.10134604571995985, - 0.15887011140561966 - ], - "candidate_ci95": [ - 0.0056846758306226585, - 0.05005291241006455 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/all/transactions_1_2": { - "windows": 45, - "episodes": 42, - "latent_groups": 23, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 27, - "2": 18 - }, - "raw": { - "parent": { - "acc": 0.30952380952380953, - "nll": 1.706998903645994, - "brier": 0.9212914627123131, - "ece": 0.3949219477940342 - }, - "candidate": { - "acc": 0.40476190476190477, - "nll": 1.0960532065103286, - "brier": 0.6644578498966356, - "ece": 0.0515361557378001 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.09523809523809523, - "ci95": [ - -0.10256410256410253, - 0.27088068181818176 - ], - "parent_ci95": [ - 0.20512820512820512, - 0.4286525974025973 - ], - "candidate_ci95": [ - 0.309478021978022, - 0.5 - ] - }, - "nll": { - "candidate_minus_parent": -0.6109456971356655, - "ci95": [ - -0.8672339381424512, - -0.3446385051769663 - ], - "parent_ci95": [ - 1.4471843049380726, - 1.9533243982325792 - ], - "candidate_ci95": [ - 1.0794300015550526, - 1.1111172236579347 - ] - }, - "brier": { - "candidate_minus_parent": -0.2568336128156775, - "ci95": [ - -0.3817285469348027, - -0.10872128206234798 - ], - "parent_ci95": [ - 0.7821920851336587, - 1.044244265219641 - ], - "candidate_ci95": [ - 0.6532393456517698, - 0.6748604671268419 - ] - }, - "ece": { - "candidate_minus_parent": -0.3433857920562341, - "ci95": [ - -0.41543430112359964, - -0.24068696649838917 - ], - "parent_ci95": [ - 0.29114499760835, - 0.49447586904298535 - ], - "candidate_ci95": [ - 0.007441025650636024, - 0.1472801418978136 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 23, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.30952380952380953, - "nll": 1.158429211230326, - "brier": 0.7024608643728002, - "ece": 0.1313191003771914 - }, - "candidate": { - "acc": 0.40476190476190477, - "nll": 1.0963778788315839, - "brier": 0.665038046030096, - "ece": 0.05532362839396711 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.09523809523809523, - "ci95": [ - -0.10256410256410253, - 0.27088068181818176 - ], - "parent_ci95": [ - 0.20512820512820512, - 0.4286525974025973 - ], - "candidate_ci95": [ - 0.309478021978022, - 0.5 - ] - }, - "nll": { - "candidate_minus_parent": -0.062051332398742165, - "ci95": [ - -0.1297682515406164, - 0.008765126062274106 - ], - "parent_ci95": [ - 1.0925163708898833, - 1.2213072824062752 - ], - "candidate_ci95": [ - 1.0882109002505502, - 1.1040158253555261 - ] - }, - "brier": { - "candidate_minus_parent": -0.03742281834270422, - "ci95": [ - -0.08094726147904728, - 0.010873660088425609 - ], - "parent_ci95": [ - 0.6587283528455579, - 0.7432459424477127 - ], - "candidate_ci95": [ - 0.6594927791043819, - 0.6701795801066367 - ] - }, - "ece": { - "candidate_minus_parent": -0.0759954719832243, - "ci95": [ - -0.15320873787904174, - 0.022610033698028144 - ], - "parent_ci95": [ - 0.022382031820028063, - 0.23260449715420847 - ], - "candidate_ci95": [ - 0.0017389176176505234, - 0.1504230264430541 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 23, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/all/transactions_3_plus": { - "windows": 107, - "episodes": 82, - "latent_groups": 29, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 24, - "4": 18, - "5": 42, - "6": 20, - "7": 3 - }, - "raw": { - "parent": { - "acc": 0.3048780487804878, - "nll": 1.7535274874757134, - "brier": 0.9723947060770417, - "ece": 0.3981191573176271 - }, - "candidate": { - "acc": 0.3556910569105691, - "nll": 1.0900251597170125, - "brier": 0.6611333300051692, - "ece": 0.05145502825345785 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.05081300813008133, - "ci95": [ - 0.007843137254901988, - 0.10685654008438818 - ], - "parent_ci95": [ - 0.26582278481012656, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3373004201680672, - 0.3782051282051282 - ] - }, - "nll": { - "candidate_minus_parent": -0.6635023277587009, - "ci95": [ - -0.8021584250061905, - -0.5285921340395728 - ], - "parent_ci95": [ - 1.6133860025871591, - 1.8956209466593967 - ], - "candidate_ci95": [ - 1.0718745522776276, - 1.1109697671138516 - ] - }, - "brier": { - "candidate_minus_parent": -0.31126137607187254, - "ci95": [ - -0.372040771277964, - -0.25304541752048104 - ], - "parent_ci95": [ - 0.9123559470337855, - 1.0348753860773976 - ], - "candidate_ci95": [ - 0.6480259058780655, - 0.6761502857169847 - ] - }, - "ece": { - "candidate_minus_parent": -0.3466641290641692, - "ci95": [ - -0.4069328888385726, - -0.27068127803160935 - ], - "parent_ci95": [ - 0.3522997946155402, - 0.45991410556399387 - ], - "candidate_ci95": [ - 0.03254665904019653, - 0.13149811142527223 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3048780487804878, - "nll": 1.1655763567531447, - "brier": 0.7114093944169242, - "ece": 0.13822239288209293 - }, - "candidate": { - "acc": 0.3556910569105691, - "nll": 1.0893491093060668, - "brier": 0.6603500514339845, - "ece": 0.018875857923770827 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.05081300813008133, - "ci95": [ - 0.007843137254901988, - 0.10685654008438818 - ], - "parent_ci95": [ - 0.26582278481012656, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3373004201680672, - 0.3782051282051282 - ] - }, - "nll": { - "candidate_minus_parent": -0.07622724744707798, - "ci95": [ - -0.10280872643452063, - -0.05400454009262088 - ], - "parent_ci95": [ - 1.1411099048409659, - 1.1933966733283992 - ], - "candidate_ci95": [ - 1.079626128827174, - 1.0995762112259768 - ] - }, - "brier": { - "candidate_minus_parent": -0.05105934298293968, - "ci95": [ - -0.06877127001477751, - -0.03603397210843332 - ], - "parent_ci95": [ - 0.694767814212102, - 0.7297861619784731 - ], - "candidate_ci95": [ - 0.6533625375594448, - 0.6676370103584927 - ] - }, - "ece": { - "candidate_minus_parent": -0.11934653495832209, - "ci95": [ - -0.17057654135184128, - -0.06356932675465357 - ], - "parent_ci95": [ - 0.1076936579270957, - 0.18985276072341845 - ], - "candidate_ci95": [ - 0.005858512718669687, - 0.054791348636670234 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/iid/all": { - "windows": 152, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 27, - "2": 18, - "3": 24, - "4": 18, - "5": 42, - "6": 20, - "7": 3 - }, - "raw": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.6730827021190828, - "brier": 0.9271168690450743, - "ece": 0.38029185976877056 - }, - "candidate": { - "acc": 0.3489583333333333, - "nll": 1.0971521242555076, - "brier": 0.6658498861990656, - "ece": 0.046307694308096524 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.015625, - "ci95": [ - -0.00347222222222221, - 0.04171006944444442 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3298611111111111, - 0.37504340277777776 - ] - }, - "nll": { - "candidate_minus_parent": -0.5759305778635753, - "ci95": [ - -0.6646879978333292, - -0.4844753658625933 - ], - "parent_ci95": [ - 1.575759527226761, - 1.7691376887298536 - ], - "candidate_ci95": [ - 1.081862216974002, - 1.1148355704805166 - ] - }, - "brier": { - "candidate_minus_parent": -0.2612669828460087, - "ci95": [ - -0.2970389567438269, - -0.22634178418755768 - ], - "parent_ci95": [ - 0.8887463241160158, - 0.9676440292791192 - ], - "candidate_ci95": [ - 0.6550728672970835, - 0.6783244126276657 - ] - }, - "ece": { - "candidate_minus_parent": -0.33398416546067405, - "ci95": [ - -0.3670358118919271, - -0.2816153662787594 - ], - "parent_ci95": [ - 0.35101328311005003, - 0.42006858974136496 - ], - "candidate_ci95": [ - 0.024396575640987968, - 0.09070083025698743 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.1468427816323261, - "brier": 0.6981381709092208, - "ece": 0.10938489002382056 - }, - "candidate": { - "acc": 0.3489583333333333, - "nll": 1.0937687968736984, - "brier": 0.663354598517312, - "ece": 0.019037355871080977 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.015625, - "ci95": [ - -0.00347222222222221, - 0.04171006944444442 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3298611111111111, - 0.37504340277777776 - ] - }, - "nll": { - "candidate_minus_parent": -0.05307398475862768, - "ci95": [ - -0.06656368181703105, - -0.04064767349269174 - ], - "parent_ci95": [ - 1.131378684831578, - 1.165777015737088 - ], - "candidate_ci95": [ - 1.0857767794541715, - 1.102658732735975 - ] - }, - "brier": { - "candidate_minus_parent": -0.03478357239190877, - "ci95": [ - -0.04339052468546176, - -0.026690749871374275 - ], - "parent_ci95": [ - 0.6877235900544243, - 0.7106051033335131 - ], - "candidate_ci95": [ - 0.6576958496027633, - 0.66980743900186 - ] - }, - "ece": { - "candidate_minus_parent": -0.09034753415273958, - "ci95": [ - -0.13920480289089912, - -0.05692976019192213 - ], - "parent_ci95": [ - 0.10134604571995985, - 0.15887011140561966 - ], - "candidate_ci95": [ - 0.0056846758306226585, - 0.05005291241006455 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/iid/transactions_1_2": { - "windows": 45, - "episodes": 42, - "latent_groups": 23, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 27, - "2": 18 - }, - "raw": { - "parent": { - "acc": 0.30952380952380953, - "nll": 1.706998903645994, - "brier": 0.9212914627123131, - "ece": 0.3949219477940342 - }, - "candidate": { - "acc": 0.40476190476190477, - "nll": 1.0960532065103286, - "brier": 0.6644578498966356, - "ece": 0.0515361557378001 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.09523809523809523, - "ci95": [ - -0.10256410256410253, - 0.27088068181818176 - ], - "parent_ci95": [ - 0.20512820512820512, - 0.4286525974025973 - ], - "candidate_ci95": [ - 0.309478021978022, - 0.5 - ] - }, - "nll": { - "candidate_minus_parent": -0.6109456971356655, - "ci95": [ - -0.8672339381424512, - -0.3446385051769663 - ], - "parent_ci95": [ - 1.4471843049380726, - 1.9533243982325792 - ], - "candidate_ci95": [ - 1.0794300015550526, - 1.1111172236579347 - ] - }, - "brier": { - "candidate_minus_parent": -0.2568336128156775, - "ci95": [ - -0.3817285469348027, - -0.10872128206234798 - ], - "parent_ci95": [ - 0.7821920851336587, - 1.044244265219641 - ], - "candidate_ci95": [ - 0.6532393456517698, - 0.6748604671268419 - ] - }, - "ece": { - "candidate_minus_parent": -0.3433857920562341, - "ci95": [ - -0.41543430112359964, - -0.24068696649838917 - ], - "parent_ci95": [ - 0.29114499760835, - 0.49447586904298535 - ], - "candidate_ci95": [ - 0.007441025650636024, - 0.1472801418978136 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 23, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.30952380952380953, - "nll": 1.158429211230326, - "brier": 0.7024608643728002, - "ece": 0.1313191003771914 - }, - "candidate": { - "acc": 0.40476190476190477, - "nll": 1.0963778788315839, - "brier": 0.665038046030096, - "ece": 0.05532362839396711 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.09523809523809523, - "ci95": [ - -0.10256410256410253, - 0.27088068181818176 - ], - "parent_ci95": [ - 0.20512820512820512, - 0.4286525974025973 - ], - "candidate_ci95": [ - 0.309478021978022, - 0.5 - ] - }, - "nll": { - "candidate_minus_parent": -0.062051332398742165, - "ci95": [ - -0.1297682515406164, - 0.008765126062274106 - ], - "parent_ci95": [ - 1.0925163708898833, - 1.2213072824062752 - ], - "candidate_ci95": [ - 1.0882109002505502, - 1.1040158253555261 - ] - }, - "brier": { - "candidate_minus_parent": -0.03742281834270422, - "ci95": [ - -0.08094726147904728, - 0.010873660088425609 - ], - "parent_ci95": [ - 0.6587283528455579, - 0.7432459424477127 - ], - "candidate_ci95": [ - 0.6594927791043819, - 0.6701795801066367 - ] - }, - "ece": { - "candidate_minus_parent": -0.0759954719832243, - "ci95": [ - -0.15320873787904174, - 0.022610033698028144 - ], - "parent_ci95": [ - 0.022382031820028063, - 0.23260449715420847 - ], - "candidate_ci95": [ - 0.0017389176176505234, - 0.1504230264430541 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 23, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/iid/transactions_3_plus": { - "windows": 107, - "episodes": 82, - "latent_groups": 29, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 24, - "4": 18, - "5": 42, - "6": 20, - "7": 3 - }, - "raw": { - "parent": { - "acc": 0.3048780487804878, - "nll": 1.7535274874757134, - "brier": 0.9723947060770417, - "ece": 0.3981191573176271 - }, - "candidate": { - "acc": 0.3556910569105691, - "nll": 1.0900251597170125, - "brier": 0.6611333300051692, - "ece": 0.05145502825345785 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.05081300813008133, - "ci95": [ - 0.007843137254901988, - 0.10685654008438818 - ], - "parent_ci95": [ - 0.26582278481012656, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3373004201680672, - 0.3782051282051282 - ] - }, - "nll": { - "candidate_minus_parent": -0.6635023277587009, - "ci95": [ - -0.8021584250061905, - -0.5285921340395728 - ], - "parent_ci95": [ - 1.6133860025871591, - 1.8956209466593967 - ], - "candidate_ci95": [ - 1.0718745522776276, - 1.1109697671138516 - ] - }, - "brier": { - "candidate_minus_parent": -0.31126137607187254, - "ci95": [ - -0.372040771277964, - -0.25304541752048104 - ], - "parent_ci95": [ - 0.9123559470337855, - 1.0348753860773976 - ], - "candidate_ci95": [ - 0.6480259058780655, - 0.6761502857169847 - ] - }, - "ece": { - "candidate_minus_parent": -0.3466641290641692, - "ci95": [ - -0.4069328888385726, - -0.27068127803160935 - ], - "parent_ci95": [ - 0.3522997946155402, - 0.45991410556399387 - ], - "candidate_ci95": [ - 0.03254665904019653, - 0.13149811142527223 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3048780487804878, - "nll": 1.1655763567531447, - "brier": 0.7114093944169242, - "ece": 0.13822239288209293 - }, - "candidate": { - "acc": 0.3556910569105691, - "nll": 1.0893491093060668, - "brier": 0.6603500514339845, - "ece": 0.018875857923770827 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.05081300813008133, - "ci95": [ - 0.007843137254901988, - 0.10685654008438818 - ], - "parent_ci95": [ - 0.26582278481012656, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3373004201680672, - 0.3782051282051282 - ] - }, - "nll": { - "candidate_minus_parent": -0.07622724744707798, - "ci95": [ - -0.10280872643452063, - -0.05400454009262088 - ], - "parent_ci95": [ - 1.1411099048409659, - 1.1933966733283992 - ], - "candidate_ci95": [ - 1.079626128827174, - 1.0995762112259768 - ] - }, - "brier": { - "candidate_minus_parent": -0.05105934298293968, - "ci95": [ - -0.06877127001477751, - -0.03603397210843332 - ], - "parent_ci95": [ - 0.694767814212102, - 0.7297861619784731 - ], - "candidate_ci95": [ - 0.6533625375594448, - 0.6676370103584927 - ] - }, - "ece": { - "candidate_minus_parent": -0.11934653495832209, - "ci95": [ - -0.17057654135184128, - -0.06356932675465357 - ], - "parent_ci95": [ - 0.1076936579270957, - 0.18985276072341845 - ], - "candidate_ci95": [ - 0.005858512718669687, - 0.054791348636670234 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - } - }, - "development": { - "full/all/all": { - "windows": 475, - "episodes": 192, - "latent_groups": 64, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 78, - "2": 59, - "3": 68, - "4": 97, - "5": 107, - "6": 53, - "7": 13 - }, - "raw": { - "parent": { - "acc": 0.3346354166666667, - "nll": 1.7350104903591685, - "brier": 0.931591212015464, - "ece": 0.39189449934680637 - }, - "candidate": { - "acc": 0.33940972222222215, - "nll": 1.1081342789041253, - "brier": 0.6726546422375012, - "ece": 0.05665595823559088 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.004774305555555469, - "ci95": [ - -0.0039171006944444964, - 0.01388888888888895 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3372395833333333 - ], - "candidate_ci95": [ - 0.3311631944444445, - 0.3481011284722223 - ] - }, - "nll": { - "candidate_minus_parent": -0.6268762114550432, - "ci95": [ - -0.6790899859377322, - -0.5749093794570713 - ], - "parent_ci95": [ - 1.6811977782718885, - 1.7866107928562296 - ], - "candidate_ci95": [ - 1.096188224934354, - 1.1214289900464145 - ] - }, - "brier": { - "candidate_minus_parent": -0.25893656977796287, - "ci95": [ - -0.28024348498046525, - -0.23624394979708485 - ], - "parent_ci95": [ - 0.9092211056538766, - 0.9509438619550101 - ], - "candidate_ci95": [ - 0.6646108272384416, - 0.6815656878154813 - ] - }, - "ece": { - "candidate_minus_parent": -0.3352385411112155, - "ci95": [ - -0.35128286511395124, - -0.3108043566620138 - ], - "parent_ci95": [ - 0.3781534407523819, - 0.4056166546749857 - ], - "candidate_ci95": [ - 0.04569651446212632, - 0.07531915688931973 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 64, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3346354166666667, - "nll": 1.1449771461576341, - "brier": 0.6962720420695204, - "ece": 0.11982150416096793 - }, - "candidate": { - "acc": 0.33940972222222215, - "nll": 1.0986515810875779, - "brier": 0.6665575258233994, - "ece": 0.02460719607488689 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.004774305555555469, - "ci95": [ - -0.0039171006944444964, - 0.01388888888888895 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3372395833333333 - ], - "candidate_ci95": [ - 0.3311631944444445, - 0.3481011284722223 - ] - }, - "nll": { - "candidate_minus_parent": -0.04632556507005625, - "ci95": [ - -0.05491666451540952, - -0.03719116377048172 - ], - "parent_ci95": [ - 1.1365041969928562, - 1.1526221927426767 - ], - "candidate_ci95": [ - 1.0931171370730473, - 1.1043579084594954 - ] - }, - "brier": { - "candidate_minus_parent": -0.02971451624612098, - "ci95": [ - -0.0361325958427297, - -0.022840468111426974 - ], - "parent_ci95": [ - 0.6901428934681388, - 0.7015872230804511 - ], - "candidate_ci95": [ - 0.6627210641166765, - 0.670591035712894 - ] - }, - "ece": { - "candidate_minus_parent": -0.09521430808608104, - "ci95": [ - -0.10827761667473262, - -0.07017375862249017 - ], - "parent_ci95": [ - 0.11338467129999455, - 0.12711460228131327 - ], - "candidate_ci95": [ - 0.015289219037466687, - 0.047426377351587314 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 64, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "full/all/transactions_1_2": { - "windows": 137, - "episodes": 120, - "latent_groups": 55, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 78, - "2": 59 - }, - "raw": { - "parent": { - "acc": 0.3416666666666667, - "nll": 1.684599582747552, - "brier": 0.9005110955171775, - "ece": 0.36682455700518823 - }, - "candidate": { - "acc": 0.35, - "nll": 1.101116661104652, - "brier": 0.6683340927580801, - "ece": 0.013849288001388559 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.008333333333333304, - "ci95": [ - -0.0630993897122929, - 0.08622563718140923 - ], - "parent_ci95": [ - 0.30274651144861614, - 0.3793103448275862 - ], - "candidate_ci95": [ - 0.30251612272816103, - 0.39655172413793105 - ] - }, - "nll": { - "candidate_minus_parent": -0.5834829216429, - "ci95": [ - -0.7015545794752215, - -0.4644986033568784 - ], - "parent_ci95": [ - 1.5712100799782325, - 1.7988729386677607 - ], - "candidate_ci95": [ - 1.0929978575928703, - 1.1103106386867923 - ] - }, - "brier": { - "candidate_minus_parent": -0.23217700275909736, - "ci95": [ - -0.2852689584801007, - -0.18326137387219782 - ], - "parent_ci95": [ - 0.8520279193111497, - 0.9521012018246955 - ], - "candidate_ci95": [ - 0.6628975480894769, - 0.6745206693158341 - ] - }, - "ece": { - "candidate_minus_parent": -0.35297526900379966, - "ci95": [ - -0.3985098064069664, - -0.2794380675101525 - ], - "parent_ci95": [ - 0.32695543451399345, - 0.40934861556604873 - ], - "candidate_ci95": [ - 0.0010193241456591974, - 0.06189194519210392 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 55, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3416666666666667, - "nll": 1.1455507388151853, - "brier": 0.6944973866276738, - "ece": 0.10320926869972151 - }, - "candidate": { - "acc": 0.35, - "nll": 1.0991255871933145, - "brier": 0.6670069728868838, - "ece": 0.0015285428355831414 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.008333333333333304, - "ci95": [ - -0.0630993897122929, - 0.08622563718140923 - ], - "parent_ci95": [ - 0.30274651144861614, - 0.3793103448275862 - ], - "candidate_ci95": [ - 0.30251612272816103, - 0.39655172413793105 - ] - }, - "nll": { - "candidate_minus_parent": -0.046425151621870775, - "ci95": [ - -0.07472376843975126, - -0.01622713172066983 - ], - "parent_ci95": [ - 1.1183217393157816, - 1.1716799673702034 - ], - "candidate_ci95": [ - 1.0951166991154377, - 1.103752703626571 - ] - }, - "brier": { - "candidate_minus_parent": -0.027490413740789976, - "ci95": [ - -0.045069969488215435, - -0.009417163544500267 - ], - "parent_ci95": [ - 0.6786844067119482, - 0.7107307082378528 - ], - "candidate_ci95": [ - 0.6643381100021499, - 0.6700878067256317 - ] - }, - "ece": { - "candidate_minus_parent": -0.10168072586413837, - "ci95": [ - -0.1332521717224303, - -0.03987470346219365 - ], - "parent_ci95": [ - 0.06925090184928052, - 0.14806137845187706 - ], - "candidate_ci95": [ - 0.0008313869138119382, - 0.055754750153097166 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 55, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "full/all/transactions_3_plus": { - "windows": 338, - "episodes": 191, - "latent_groups": 64, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 68, - "4": 97, - "5": 107, - "6": 53, - "7": 13 - }, - "raw": { - "parent": { - "acc": 0.33682373472949395, - "nll": 1.761452391457942, - "brier": 0.9442724162451026, - "ece": 0.3950515362300004 - }, - "candidate": { - "acc": 0.33726003490401396, - "nll": 1.1147148219060148, - "brier": 0.6763215604784556, - "ece": 0.0745558174365904 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.00043630017452001013, - "ci95": [ - -0.017454303297510847, - 0.01571561182009885 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.34219858156028377 - ], - "candidate_ci95": [ - 0.3223684210526316, - 0.3511308014288831 - ] - }, - "nll": { - "candidate_minus_parent": -0.6467375695519273, - "ci95": [ - -0.7267278992967698, - -0.5706812911968864 - ], - "parent_ci95": [ - 1.6806548862561872, - 1.8396636722233164 - ], - "candidate_ci95": [ - 1.0973732716449693, - 1.133455218416589 - ] - }, - "brier": { - "candidate_minus_parent": -0.267950855766647, - "ci95": [ - -0.29774336359130227, - -0.23610420078841077 - ], - "parent_ci95": [ - 0.9135761873571212, - 0.9723497753763009 - ], - "candidate_ci95": [ - 0.6646927948643595, - 0.6886955188028272 - ] - }, - "ece": { - "candidate_minus_parent": -0.32049571879341, - "ci95": [ - -0.34709872603547504, - -0.28679970747810446 - ], - "parent_ci95": [ - 0.3758519650838023, - 0.41288541640531323 - ], - "candidate_ci95": [ - 0.05472470170964465, - 0.10244079848991547 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 64, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.33682373472949395, - "nll": 1.1463297791564957, - "brier": 0.6983178566024099, - "ece": 0.12125128514353993 - }, - "candidate": { - "acc": 0.33726003490401396, - "nll": 1.1003470550538683, - "brier": 0.6675203677297942, - "ece": 0.03228487048620194 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.00043630017452001013, - "ci95": [ - -0.017454303297510847, - 0.01571561182009885 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.34219858156028377 - ], - "candidate_ci95": [ - 0.3223684210526316, - 0.3511308014288831 - ] - }, - "nll": { - "candidate_minus_parent": -0.04598272410262738, - "ci95": [ - -0.05915958918677809, - -0.031511613609357336 - ], - "parent_ci95": [ - 1.1334658573821124, - 1.1579355387417785 - ], - "candidate_ci95": [ - 1.0925136173057282, - 1.10792523882204 - ] - }, - "brier": { - "candidate_minus_parent": -0.030797488872615686, - "ci95": [ - -0.04002894682173265, - -0.020329667638605768 - ], - "parent_ci95": [ - 0.6893249796002362, - 0.7065566761962964 - ], - "candidate_ci95": [ - 0.6621069035390456, - 0.6727332808445797 - ] - }, - "ece": { - "candidate_minus_parent": -0.08896641465733798, - "ci95": [ - -0.11056724092783564, - -0.05915526650772133 - ], - "parent_ci95": [ - 0.1130570173875826, - 0.13525987313590213 - ], - "candidate_ci95": [ - 0.01819560542779522, - 0.06057786312676982 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 64, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "full/iid/all": { - "windows": 196, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 34, - "2": 16, - "3": 29, - "4": 36, - "5": 41, - "6": 33, - "7": 7 - }, - "raw": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.7124523573731933, - "brier": 0.9303227524837441, - "ece": 0.40316209525167973 - }, - "candidate": { - "acc": 0.34027777777777773, - "nll": 1.1141859538535557, - "brier": 0.6767587666407532, - "ece": 0.06582449248115932 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.00694444444444442, - "ci95": [ - 0.0, - 0.017361111111111105 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3333333333333333, - 0.3506944444444444 - ] - }, - "nll": { - "candidate_minus_parent": -0.5982664035196377, - "ci95": [ - -0.678565019721147, - -0.5198138791551941 - ], - "parent_ci95": [ - 1.6411516919873779, - 1.7821554572493035 - ], - "candidate_ci95": [ - 1.0932970868965697, - 1.137785374629282 - ] - }, - "brier": { - "candidate_minus_parent": -0.2535639858429909, - "ci95": [ - -0.28821447671246375, - -0.21795240813330616 - ], - "parent_ci95": [ - 0.9005605959813189, - 0.9600325280394629 - ], - "candidate_ci95": [ - 0.6623924005681695, - 0.6929750831612848 - ] - }, - "ece": { - "candidate_minus_parent": -0.3373376027705204, - "ci95": [ - -0.3670063589933437, - -0.3062944837628755 - ], - "parent_ci95": [ - 0.38457183848965565, - 0.4223828872836136 - ], - "candidate_ci95": [ - 0.04670505012772756, - 0.08779944049177654 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.1395642135708888, - "brier": 0.69244983786886, - "ece": 0.12259199189028035 - }, - "candidate": { - "acc": 0.34027777777777773, - "nll": 1.0997952972117553, - "brier": 0.6673327111796757, - "ece": 0.02908590049678272 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.00694444444444442, - "ci95": [ - 0.0, - 0.017361111111111105 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3333333333333333, - 0.3506944444444444 - ] - }, - "nll": { - "candidate_minus_parent": -0.03976891635913349, - "ci95": [ - -0.0541526606128827, - -0.025168846040066223 - ], - "parent_ci95": [ - 1.128130974471412, - 1.1506218033263493 - ], - "candidate_ci95": [ - 1.0898406047820128, - 1.110098733221803 - ] - }, - "brier": { - "candidate_minus_parent": -0.02511712668918431, - "ci95": [ - -0.035736637934652135, - -0.01491019414543638 - ], - "parent_ci95": [ - 0.6844439570327298, - 0.700775171664165 - ], - "candidate_ci95": [ - 0.6604923908009386, - 0.674458957910712 - ] - }, - "ece": { - "candidate_minus_parent": -0.09350609139349762, - "ci95": [ - -0.1122592755940263, - -0.06097575023830561 - ], - "parent_ci95": [ - 0.1151864856000442, - 0.13576747842445488 - ], - "candidate_ci95": [ - 0.016975309751749497, - 0.060832712514700425 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "full/iid/transactions_1_2": { - "windows": 50, - "episodes": 47, - "latent_groups": 24, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 34, - "2": 16 - }, - "raw": { - "parent": { - "acc": 0.3617021276595745, - "nll": 1.606278791640521, - "brier": 0.8668143694211096, - "ece": 0.31867378407842417 - }, - "candidate": { - "acc": 0.3617021276595745, - "nll": 1.0951264944863086, - "brier": 0.6643659084321083, - "ece": 0.0006462525101058933 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0, - "ci95": [ - -0.14583333333333337, - 0.14893617021276595 - ], - "parent_ci95": [ - 0.2856366459627329, - 0.4473684210526316 - ], - "candidate_ci95": [ - 0.27498774509803925, - 0.4524350649350649 - ] - }, - "nll": { - "candidate_minus_parent": -0.5111522971542124, - "ci95": [ - -0.6998260346042728, - -0.2966956346082348 - ], - "parent_ci95": [ - 1.4021310428301128, - 1.7895754867565086 - ], - "candidate_ci95": [ - 1.0799163978490631, - 1.1093613884301694 - ] - }, - "brier": { - "candidate_minus_parent": -0.20244846098900127, - "ci95": [ - -0.29625351971246794, - -0.10311579551210913 - ], - "parent_ci95": [ - 0.7727146159309324, - 0.9559588376055745 - ], - "candidate_ci95": [ - 0.6540730551374172, - 0.6741568414911062 - ] - }, - "ece": { - "candidate_minus_parent": -0.31802753156831826, - "ci95": [ - -0.37493368571558633, - -0.18510516049850553 - ], - "parent_ci95": [ - 0.24598050008228753, - 0.40578506275793336 - ], - "candidate_ci95": [ - 0.0009102569552842368, - 0.10216654573754363 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 24, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3617021276595745, - "nll": 1.1428425886049745, - "brier": 0.6920183105504162, - "ece": 0.08398588611848011 - }, - "candidate": { - "acc": 0.3617021276595745, - "nll": 1.0962314698157283, - "brier": 0.6650822310706771, - "ece": 0.013976683295737926 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0, - "ci95": [ - -0.14583333333333337, - 0.14893617021276595 - ], - "parent_ci95": [ - 0.2856366459627329, - 0.4473684210526316 - ], - "candidate_ci95": [ - 0.27498774509803925, - 0.4524350649350649 - ] - }, - "nll": { - "candidate_minus_parent": -0.046611118789246175, - "ci95": [ - -0.09561600806355856, - 0.005996069868429332 - ], - "parent_ci95": [ - 1.0945409291305586, - 1.1870865967481357 - ], - "candidate_ci95": [ - 1.088695192927692, - 1.1032106815760592 - ] - }, - "brier": { - "candidate_minus_parent": -0.026936079479739083, - "ci95": [ - -0.05750168546113095, - 0.00643858200710272 - ], - "parent_ci95": [ - 0.6620531653451835, - 0.7198120893298922 - ], - "candidate_ci95": [ - 0.6600092587271856, - 0.6698176402356754 - ] - }, - "ece": { - "candidate_minus_parent": -0.07000920282274219, - "ci95": [ - -0.14608908633724968, - 0.031151963204560304 - ], - "parent_ci95": [ - 0.025589163228596955, - 0.17726881365237537 - ], - "candidate_ci95": [ - 0.0009573572755788792, - 0.10847699460005204 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 24, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "full/iid/transactions_3_plus": { - "windows": 146, - "episodes": 95, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 29, - "4": 36, - "5": 41, - "6": 33, - "7": 7 - }, - "raw": { - "parent": { - "acc": 0.3368421052631579, - "nll": 1.7461010824909342, - "brier": 0.9415185117661182, - "ece": 0.4125392328057667 - }, - "candidate": { - "acc": 0.3368421052631579, - "nll": 1.1253460058972915, - "brier": 0.6834591680376513, - "ece": 0.08535066950123138 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0, - "ci95": [ - -0.02688172043010756, - 0.02105263157894738 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.34408602150537637 - ], - "candidate_ci95": [ - 0.3172043010752688, - 0.35789473684210527 - ] - }, - "nll": { - "candidate_minus_parent": -0.6207550765936427, - "ci95": [ - -0.7290660550006571, - -0.5173842578992225 - ], - "parent_ci95": [ - 1.6480276717710194, - 1.8467160693786326 - ], - "candidate_ci95": [ - 1.096204580918968, - 1.1579777731219882 - ] - }, - "brier": { - "candidate_minus_parent": -0.25805934372846695, - "ci95": [ - -0.30214923830076107, - -0.2142367363497392 - ], - "parent_ci95": [ - 0.901522270321034, - 0.9804060439548726 - ], - "candidate_ci95": [ - 0.6638689493859028, - 0.7057101836439521 - ] - }, - "ece": { - "candidate_minus_parent": -0.32718856330453533, - "ci95": [ - -0.3690141814092924, - -0.27888935367683226 - ], - "parent_ci95": [ - 0.38536173699373244, - 0.43754207835021325 - ], - "candidate_ci95": [ - 0.05528992404234269, - 0.1255413762737284 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3368421052631579, - "nll": 1.1390392002640022, - "brier": 0.6927147679259726, - "ece": 0.12558728946394143 - }, - "candidate": { - "acc": 0.3368421052631579, - "nll": 1.1027991703996407, - "brier": 0.6691990167866013, - "ece": 0.04061751773318837 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0, - "ci95": [ - -0.02688172043010756, - 0.02105263157894738 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.34408602150537637 - ], - "candidate_ci95": [ - 0.3172043010752688, - 0.35789473684210527 - ] - }, - "nll": { - "candidate_minus_parent": -0.03624002986436148, - "ci95": [ - -0.05793745836843981, - -0.01500581910141239 - ], - "parent_ci95": [ - 1.1229187761318027, - 1.1566543123195199 - ], - "candidate_ci95": [ - 1.0900328870684717, - 1.1166802658184218 - ] - }, - "brier": { - "candidate_minus_parent": -0.02351575113937132, - "ci95": [ - -0.03837682981567418, - -0.008820176281655752 - ], - "parent_ci95": [ - 0.6814368470195351, - 0.7046973404906302 - ], - "candidate_ci95": [ - 0.660377113283333, - 0.6787569507705161 - ] - }, - "ece": { - "candidate_minus_parent": -0.08496977173075307, - "ci95": [ - -0.11741964777848636, - -0.0519524894099493 - ], - "parent_ci95": [ - 0.11532377688961362, - 0.14439980032708952 - ], - "candidate_ci95": [ - 0.020732295614394415, - 0.07355313445514028 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "full/heldout_task/all": { - "windows": 279, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff" - ], - "transaction_count_histogram": { - "1": 44, - "2": 43, - "3": 39, - "4": 61, - "5": 66, - "6": 20, - "7": 6 - }, - "raw": { - "parent": { - "acc": 0.3359375, - "nll": 1.7575686233451444, - "brier": 0.9328596715471843, - "ece": 0.38062690344193295 - }, - "candidate": { - "acc": 0.3385416666666667, - "nll": 1.1020826039546938, - "brier": 0.668550517834249, - "ece": 0.06767651412584706 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.002604166666666685, - "ci95": [ - -0.012152777777777735, - 0.018229166666666685 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3411458333333333 - ], - "candidate_ci95": [ - 0.32463107638888883, - 0.3532986111111111 - ] - }, - "nll": { - "candidate_minus_parent": -0.6554860193904506, - "ci95": [ - -0.7191252619922539, - -0.5925014370566785 - ], - "parent_ci95": [ - 1.6936048834769095, - 1.8243095747652294 - ], - "candidate_ci95": [ - 1.0909955635083948, - 1.1148722338770993 - ] - }, - "brier": { - "candidate_minus_parent": -0.2643091537129353, - "ci95": [ - -0.28718910807802955, - -0.23881191694116374 - ], - "parent_ci95": [ - 0.9086730578648818, - 0.9557397760683705 - ], - "candidate_ci95": [ - 0.6602464516032025, - 0.6782263566624996 - ] - }, - "ece": { - "candidate_minus_parent": -0.3129503893160859, - "ci95": [ - -0.34660047272419636, - -0.2808933130120532 - ], - "parent_ci95": [ - 0.36558694479372333, - 0.39906087104179055 - ], - "candidate_ci95": [ - 0.04217660861029081, - 0.09521924116463996 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3359375, - "nll": 1.150390078744378, - "brier": 0.7000942462701806, - "ece": 0.11705101643165557 - }, - "candidate": { - "acc": 0.3385416666666667, - "nll": 1.0975078649634016, - "brier": 0.6657823404671229, - "ece": 0.03921704440404435 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.002604166666666685, - "ci95": [ - -0.012152777777777735, - 0.018229166666666685 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3411458333333333 - ], - "candidate_ci95": [ - 0.32463107638888883, - 0.3532986111111111 - ] - }, - "nll": { - "candidate_minus_parent": -0.05288221378097635, - "ci95": [ - -0.062443593811575986, - -0.04190493793031174 - ], - "parent_ci95": [ - 1.1410787694045743, - 1.1593915727679052 - ], - "candidate_ci95": [ - 1.091928362600967, - 1.1035442157202624 - ] - }, - "brier": { - "candidate_minus_parent": -0.03431190580305765, - "ci95": [ - -0.04143880164165249, - -0.02639595094126181 - ], - "parent_ci95": [ - 0.6935373129262572, - 0.706784911160115 - ], - "candidate_ci95": [ - 0.6617785284376593, - 0.6702643963140505 - ] - }, - "ece": { - "candidate_minus_parent": -0.07783397202761122, - "ci95": [ - -0.11444951609054546, - -0.04209215226387993 - ], - "parent_ci95": [ - 0.1088484189991902, - 0.13090558733426144 - ], - "candidate_ci95": [ - 0.010719860472098343, - 0.07335949517961429 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "full/heldout_task/transactions_1_2": { - "windows": 87, - "episodes": 73, - "latent_groups": 31, - "sources": [ - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff" - ], - "transaction_count_histogram": { - "1": 44, - "2": 43 - }, - "raw": { - "parent": { - "acc": 0.3287671232876712, - "nll": 1.7350252975698874, - "brier": 0.9222062479351931, - "ece": 0.3978257395744748 - }, - "candidate": { - "acc": 0.3424657534246575, - "nll": 1.104973343721942, - "brier": 0.6708889511597332, - "ece": 0.022349872495776038 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0136986301369863, - "ci95": [ - -0.0684931506849315, - 0.09999999999999998 - ], - "parent_ci95": [ - 0.27937091503267975, - 0.37142857142857144 - ], - "candidate_ci95": [ - 0.2875740466493891, - 0.39478905597326647 - ] - }, - "nll": { - "candidate_minus_parent": -0.6300519538479454, - "ci95": [ - -0.7871149224166658, - -0.4734227265150399 - ], - "parent_ci95": [ - 1.5816266494205868, - 1.8894676459462878 - ], - "candidate_ci95": [ - 1.0959260895443086, - 1.115504433417772 - ] - }, - "brier": { - "candidate_minus_parent": -0.2513172967754599, - "ci95": [ - -0.31816111614285403, - -0.19052235180482108 - ], - "parent_ci95": [ - 0.8620497635983301, - 0.9910377048377865 - ], - "candidate_ci95": [ - 0.6649016420955892, - 0.6780008413641927 - ] - }, - "ece": { - "candidate_minus_parent": -0.3754758670786988, - "ci95": [ - -0.43912023298958897, - -0.29252862461462614 - ], - "parent_ci95": [ - 0.35259500914090686, - 0.4548247635753083 - ], - "candidate_ci95": [ - 0.001328926908109236, - 0.0803031654645048 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 31, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3287671232876712, - "nll": 1.1472943423751842, - "brier": 0.6960935041020725, - "ece": 0.12388419826016156 - }, - "candidate": { - "acc": 0.3424657534246575, - "nll": 1.1009889230391574, - "brier": 0.668246190220606, - "ece": 0.006486013351091863 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0136986301369863, - "ci95": [ - -0.0684931506849315, - 0.09999999999999998 - ], - "parent_ci95": [ - 0.27937091503267975, - 0.37142857142857144 - ], - "candidate_ci95": [ - 0.2875740466493891, - 0.39478905597326647 - ] - }, - "nll": { - "candidate_minus_parent": -0.04630541933602683, - "ci95": [ - -0.08443920761641914, - -0.006965185387646984 - ], - "parent_ci95": [ - 1.110315934404331, - 1.1821887526257167 - ], - "candidate_ci95": [ - 1.096498439293462, - 1.10627108114853 - ] - }, - "brier": { - "candidate_minus_parent": -0.02784731388146644, - "ci95": [ - -0.050883589448877234, - -0.004163943536963318 - ], - "parent_ci95": [ - 0.6743476108077922, - 0.7173112637797384 - ], - "candidate_ci95": [ - 0.6652351601435463, - 0.6717803374395981 - ] - }, - "ece": { - "candidate_minus_parent": -0.1173981849090697, - "ci95": [ - -0.15950237235508946, - -0.03587379472521817 - ], - "parent_ci95": [ - 0.08056367869157598, - 0.17544693067276984 - ], - "candidate_ci95": [ - 0.0008863391142034598, - 0.0688510654467535 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 31, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "full/heldout_task/transactions_3_plus": { - "windows": 192, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff" - ], - "transaction_count_histogram": { - "3": 39, - "4": 61, - "5": 66, - "6": 20, - "7": 6 - }, - "raw": { - "parent": { - "acc": 0.3368055555555555, - "nll": 1.7766437909565445, - "brier": 0.9469976342190978, - "ece": 0.37774600316023155 - }, - "candidate": { - "acc": 0.33767361111111116, - "nll": 1.104194379414647, - "brier": 0.6692583029980016, - "ece": 0.08433455087754062 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0008680555555556357, - "ci95": [ - -0.01825086805555552, - 0.02172309027777771 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.34375 - ], - "candidate_ci95": [ - 0.3185763888888889, - 0.3567925347222223 - ] - }, - "nll": { - "candidate_minus_parent": -0.6724494115418975, - "ci95": [ - -0.7846533438325851, - -0.5642174507384142 - ], - "parent_ci95": [ - 1.669932285314896, - 1.895277090283664 - ], - "candidate_ci95": [ - 1.0893008766296481, - 1.1201245297548965 - ] - }, - "brier": { - "candidate_minus_parent": -0.2777393312210962, - "ci95": [ - -0.3149989606409033, - -0.23795002418428113 - ], - "parent_ci95": [ - 0.9083316689667666, - 0.9850896377946536 - ], - "candidate_ci95": [ - 0.6582977845907898, - 0.6809407849295769 - ] - }, - "ece": { - "candidate_minus_parent": -0.29341145228269094, - "ci95": [ - -0.3390404291788445, - -0.2490592554209443 - ], - "parent_ci95": [ - 0.35449375236020964, - 0.405508705226858 - ], - "candidate_ci95": [ - 0.048565854925756505, - 0.12331123938076059 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3368055555555555, - "nll": 1.1535444145188587, - "brier": 0.7038625797718008, - "ece": 0.11696044753480923 - }, - "candidate": { - "acc": 0.33767361111111116, - "nll": 1.0979204825762812, - "brier": 0.6658592046006618, - "ece": 0.05210446832070306 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0008680555555556357, - "ci95": [ - -0.01825086805555552, - 0.02172309027777771 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.34375 - ], - "candidate_ci95": [ - 0.3185763888888889, - 0.3567925347222223 - ] - }, - "nll": { - "candidate_minus_parent": -0.05562393194257753, - "ci95": [ - -0.07104980441804248, - -0.03889109071886262 - ], - "parent_ci95": [ - 1.1376932604226322, - 1.169471515491799 - ], - "candidate_ci95": [ - 1.0903705192268702, - 1.1055069702159943 - ] - }, - "brier": { - "candidate_minus_parent": -0.03800337517113894, - "ci95": [ - -0.04911188392942332, - -0.025785478747248913 - ], - "parent_ci95": [ - 0.6925366834896836, - 0.7152639596153512 - ], - "candidate_ci95": [ - 0.6605140359970081, - 0.6714526182780927 - ] - }, - "ece": { - "candidate_minus_parent": -0.06485597921410617, - "ci95": [ - -0.13292318496375544, - -0.019393761824628347 - ], - "parent_ci95": [ - 0.10688745756195825, - 0.15464785735718722 - ], - "candidate_ci95": [ - 0.013049959604135488, - 0.09738629236158458 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/all/all": { - "windows": 390, - "episodes": 191, - "latent_groups": 64, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 72, - "2": 62, - "3": 47, - "4": 97, - "5": 59, - "6": 43, - "7": 10 - }, - "raw": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.6254954852975034, - "brier": 0.9013043985586006, - "ece": 0.3597505684514547 - }, - "candidate": { - "acc": 0.33682373472949384, - "nll": 1.110266485995317, - "brier": 0.674771349616321, - "ece": 0.0596522660066575 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.003490401396160525, - "ci95": [ - -0.012347320724980352, - 0.02083333333333337 - ], - "parent_ci95": [ - 0.328125, - 0.3386243386243386 - ], - "candidate_ci95": [ - 0.3245149911816579, - 0.3524305555555555 - ] - }, - "nll": { - "candidate_minus_parent": -0.5152289993021864, - "ci95": [ - -0.5676505768552339, - -0.455085286081279 - ], - "parent_ci95": [ - 1.5684667289278473, - 1.6767592517653676 - ], - "candidate_ci95": [ - 1.1009605550043402, - 1.1207292524341428 - ] - }, - "brier": { - "candidate_minus_parent": -0.22653304894227955, - "ci95": [ - -0.250654103376682, - -0.1982401278563772 - ], - "parent_ci95": [ - 0.8760171160204571, - 0.9243132548480782 - ], - "candidate_ci95": [ - 0.6681397780663684, - 0.6824428981963307 - ] - }, - "ece": { - "candidate_minus_parent": -0.3000983024447972, - "ci95": [ - -0.32770194535473435, - -0.2727976391786125 - ], - "parent_ci95": [ - 0.3443639080838946, - 0.3816678968317757 - ], - "candidate_ci95": [ - 0.04335549755037792, - 0.07888579260595374 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 64, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.1407396953848146, - "brier": 0.6932489922362768, - "ece": 0.10409138149272482 - }, - "candidate": { - "acc": 0.33682373472949384, - "nll": 1.100190627600681, - "brier": 0.6676671025761404, - "ece": 0.02738107127463787 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.003490401396160525, - "ci95": [ - -0.012347320724980352, - 0.02083333333333337 - ], - "parent_ci95": [ - 0.328125, - 0.3386243386243386 - ], - "candidate_ci95": [ - 0.3245149911816579, - 0.3524305555555555 - ] - }, - "nll": { - "candidate_minus_parent": -0.040549067784133586, - "ci95": [ - -0.05069348217992202, - -0.027742540759487525 - ], - "parent_ci95": [ - 1.130108915954547, - 1.1501606809449922 - ], - "candidate_ci95": [ - 1.0957942757745658, - 1.1047245026075414 - ] - }, - "brier": { - "candidate_minus_parent": -0.025581889660136414, - "ci95": [ - -0.03237704094818407, - -0.01685801379643218 - ], - "parent_ci95": [ - 0.6860842681004884, - 0.6994286590282218 - ], - "candidate_ci95": [ - 0.664585723129944, - 0.6710185880855628 - ] - }, - "ece": { - "candidate_minus_parent": -0.07671031021808694, - "ci95": [ - -0.10213027408881704, - -0.05682353057788049 - ], - "parent_ci95": [ - 0.09783329602202052, - 0.12698925540903008 - ], - "candidate_ci95": [ - 0.013073190167377237, - 0.04491293661386301 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 64, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/all/transactions_1_2": { - "windows": 134, - "episodes": 121, - "latent_groups": 57, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 72, - "2": 62 - }, - "raw": { - "parent": { - "acc": 0.3140495867768595, - "nll": 1.6966350165378898, - "brier": 0.9281100519662293, - "ece": 0.39070076147798016 - }, - "candidate": { - "acc": 0.3305785123966942, - "nll": 1.1037310641630889, - "brier": 0.6700335760698483, - "ece": 0.03344612557316424 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.01652892561983471, - "ci95": [ - -0.06065340909090909, - 0.09823826741440374 - ], - "parent_ci95": [ - 0.2710146315305653, - 0.3566029900332226 - ], - "candidate_ci95": [ - 0.28203125, - 0.38098141695702664 - ] - }, - "nll": { - "candidate_minus_parent": -0.592903952374801, - "ci95": [ - -0.7578241062153577, - -0.4453913213688957 - ], - "parent_ci95": [ - 1.5539056137761456, - 1.8553697934380764 - ], - "candidate_ci95": [ - 1.0939309592978625, - 1.112084254804725 - ] - }, - "brier": { - "candidate_minus_parent": -0.25807647589638105, - "ci95": [ - -0.32915327406715944, - -0.19480923599509453 - ], - "parent_ci95": [ - 0.8666195602754322, - 0.9969135674266023 - ], - "candidate_ci95": [ - 0.6633850945977552, - 0.6756742502168676 - ] - }, - "ece": { - "candidate_minus_parent": -0.35725463590481593, - "ci95": [ - -0.4240395940757665, - -0.2692138271611469 - ], - "parent_ci95": [ - 0.3425586091205118, - 0.4420175276007634 - ], - "candidate_ci95": [ - 0.006504235656975715, - 0.08739692224084111 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 57, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3140495867768595, - "nll": 1.1504443915732265, - "brier": 0.6995600023788842, - "ece": 0.13361834319582644 - }, - "candidate": { - "acc": 0.3305785123966942, - "nll": 1.1004000643424592, - "brier": 0.6678439429866679, - "ece": 0.01795538869843083 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.01652892561983471, - "ci95": [ - -0.06065340909090909, - 0.09823826741440374 - ], - "parent_ci95": [ - 0.2710146315305653, - 0.3566029900332226 - ], - "candidate_ci95": [ - 0.28203125, - 0.38098141695702664 - ] - }, - "nll": { - "candidate_minus_parent": -0.05004432723076735, - "ci95": [ - -0.08637979380099192, - -0.01635323304221903 - ], - "parent_ci95": [ - 1.1184560351049317, - 1.1843136014307336 - ], - "candidate_ci95": [ - 1.0955398254088504, - 1.104605325738805 - ] - }, - "brier": { - "candidate_minus_parent": -0.03171605939221622, - "ci95": [ - -0.05485986502916156, - -0.010145809030184057 - ], - "parent_ci95": [ - 0.6797308847750702, - 0.7216905416020233 - ], - "candidate_ci95": [ - 0.6645759025183346, - 0.670651762712452 - ] - }, - "ece": { - "candidate_minus_parent": -0.11566295449739561, - "ci95": [ - -0.17571785910270954, - -0.04267184188290237 - ], - "parent_ci95": [ - 0.09250511763520454, - 0.18969974473657997 - ], - "candidate_ci95": [ - 0.0008354544586435885, - 0.0662625819853173 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 57, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/all/transactions_3_plus": { - "windows": 256, - "episodes": 171, - "latent_groups": 60, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 47, - "4": 97, - "5": 59, - "6": 43, - "7": 10 - }, - "raw": { - "parent": { - "acc": 0.3362573099415205, - "nll": 1.614608076963336, - "brier": 0.8961514757519852, - "ece": 0.3541944709177928 - }, - "candidate": { - "acc": 0.3391812865497076, - "nll": 1.1112555150418164, - "brier": 0.6756569607510187, - "ece": 0.07742608781497969 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0029239766081871066, - "ci95": [ - -0.01845350620630618, - 0.025873411978221402 - ], - "parent_ci95": [ - 0.32514235596925467, - 0.3473053892215569 - ], - "candidate_ci95": [ - 0.3216374269005848, - 0.3588349771046143 - ] - }, - "nll": { - "candidate_minus_parent": -0.5033525619215196, - "ci95": [ - -0.5760397492442256, - -0.43086321079813233 - ], - "parent_ci95": [ - 1.5425398515836162, - 1.686522408649482 - ], - "candidate_ci95": [ - 1.0965239071870951, - 1.1262398187506255 - ] - }, - "brier": { - "candidate_minus_parent": -0.2204945150009665, - "ci95": [ - -0.2523669636611384, - -0.18626853044480648 - ], - "parent_ci95": [ - 0.8641108521283106, - 0.9272656271309757 - ], - "candidate_ci95": [ - 0.6651284837028695, - 0.6867827894711732 - ] - }, - "ece": { - "candidate_minus_parent": -0.27676838310281315, - "ci95": [ - -0.30600041253595806, - -0.2349006043734538 - ], - "parent_ci95": [ - 0.33462179449357915, - 0.3824874686221771 - ], - "candidate_ci95": [ - 0.059624438787811616, - 0.11708948440265422 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 60, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3362573099415205, - "nll": 1.1376072963375419, - "brier": 0.6911222461269452, - "ece": 0.10123581136238233 - }, - "candidate": { - "acc": 0.3391812865497076, - "nll": 1.0987252403049783, - "brier": 0.6666289153322652, - "ece": 0.03490897494051337 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0029239766081871066, - "ci95": [ - -0.01845350620630618, - 0.025873411978221402 - ], - "parent_ci95": [ - 0.32514235596925467, - 0.3473053892215569 - ], - "candidate_ci95": [ - 0.3216374269005848, - 0.3588349771046143 - ] - }, - "nll": { - "candidate_minus_parent": -0.03888205603256356, - "ci95": [ - -0.05334088872101057, - -0.023639423053128283 - ], - "parent_ci95": [ - 1.1234643869963845, - 1.151497368963336 - ], - "candidate_ci95": [ - 1.0912678426417426, - 1.1063863964992202 - ] - }, - "brier": { - "candidate_minus_parent": -0.02449333079467997, - "ci95": [ - -0.034341580370048086, - -0.014420557386953376 - ], - "parent_ci95": [ - 0.6821943361952916, - 0.7000595417618332 - ], - "candidate_ci95": [ - 0.6613182918503591, - 0.6721973513310285 - ] - }, - "ece": { - "candidate_minus_parent": -0.06632683642186896, - "ci95": [ - -0.09359934652536785, - -0.031535166093947214 - ], - "parent_ci95": [ - 0.09033077027996042, - 0.12980451404512994 - ], - "candidate_ci95": [ - 0.017559246375493297, - 0.06955580862493253 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 60, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/iid/all": { - "windows": 161, - "episodes": 95, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 40, - "2": 24, - "3": 16, - "4": 18, - "5": 34, - "6": 24, - "7": 5 - }, - "raw": { - "parent": { - "acc": 0.3368421052631579, - "nll": 1.551428140539595, - "brier": 0.8809803470371403, - "ece": 0.3406805981123081 - }, - "candidate": { - "acc": 0.3263157894736842, - "nll": 1.1150678033880261, - "brier": 0.6783550595886768, - "ece": 0.07539941511313847 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.010526315789473661, - "ci95": [ - -0.03225806451612906, - 0.0 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.34408602150537637 - ], - "candidate_ci95": [ - 0.3118279569892473, - 0.3333333333333333 - ] - }, - "nll": { - "candidate_minus_parent": -0.43636033715156897, - "ci95": [ - -0.52810736804393, - -0.35009373129710897 - ], - "parent_ci95": [ - 1.4734442923463753, - 1.6369683513055724 - ], - "candidate_ci95": [ - 1.1016114662784458, - 1.1311523228494405 - ] - }, - "brier": { - "candidate_minus_parent": -0.20262528744846353, - "ci95": [ - -0.24471129020825447, - -0.16181052531141543 - ], - "parent_ci95": [ - 0.8460266474128768, - 0.9182401289315173 - ], - "candidate_ci95": [ - 0.669141068426313, - 0.6899658362473933 - ] - }, - "ece": { - "candidate_minus_parent": -0.26528118299916964, - "ci95": [ - -0.3059540812627167, - -0.22379753414226744 - ], - "parent_ci95": [ - 0.3192079421820447, - 0.3744168051211305 - ], - "candidate_ci95": [ - 0.05657089719716896, - 0.10463612356640144 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3368421052631579, - "nll": 1.1327163015658464, - "brier": 0.6883296731396646, - "ece": 0.09406803499836544 - }, - "candidate": { - "acc": 0.3263157894736842, - "nll": 1.1016997587704727, - "brier": 0.668802101067866, - "ece": 0.04215808199067718 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.010526315789473661, - "ci95": [ - -0.03225806451612906, - 0.0 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.34408602150537637 - ], - "candidate_ci95": [ - 0.3118279569892473, - 0.3333333333333333 - ] - }, - "nll": { - "candidate_minus_parent": -0.03101654279537369, - "ci95": [ - -0.05106434760087051, - -0.01233452322119823 - ], - "parent_ci95": [ - 1.116523591816024, - 1.1513212455338053 - ], - "candidate_ci95": [ - 1.09579883169201, - 1.1083455886338627 - ] - }, - "brier": { - "candidate_minus_parent": -0.01952757207179867, - "ci95": [ - -0.03254090267138453, - -0.00646752127970882 - ], - "parent_ci95": [ - 0.6778213296967531, - 0.7005165818006102 - ], - "candidate_ci95": [ - 0.6646066548969416, - 0.6736772012134988 - ] - }, - "ece": { - "candidate_minus_parent": -0.05190995300768826, - "ci95": [ - -0.10673619115606205, - -0.030280802684953916 - ], - "parent_ci95": [ - 0.08622401738775379, - 0.14015676652081643 - ], - "candidate_ci95": [ - 0.028781555727489527, - 0.06163593652826543 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/iid/transactions_1_2": { - "windows": 64, - "episodes": 60, - "latent_groups": 28, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 40, - "2": 24 - }, - "raw": { - "parent": { - "acc": 0.35, - "nll": 1.4883528089163451, - "brier": 0.8532207141394844, - "ece": 0.31972939169340536 - }, - "candidate": { - "acc": 0.3, - "nll": 1.1069882187922748, - "brier": 0.6722975273946177, - "ece": 0.06515841770023822 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.04999999999999999, - "ci95": [ - -0.1552146720757268, - 0.050021186440677934 - ], - "parent_ci95": [ - 0.2857142857142857, - 0.41509433962264153 - ], - "candidate_ci95": [ - 0.23214285714285715, - 0.35714285714285715 - ] - }, - "nll": { - "candidate_minus_parent": -0.3813645901240703, - "ci95": [ - -0.5156136554817409, - -0.23211164333094048 - ], - "parent_ci95": [ - 1.348725806837395, - 1.6150471936420967 - ], - "candidate_ci95": [ - 1.0928709149905234, - 1.1223015993915004 - ] - }, - "brier": { - "candidate_minus_parent": -0.18092318674486674, - "ci95": [ - -0.24475220111464377, - -0.10601283595766012 - ], - "parent_ci95": [ - 0.7791314208070413, - 0.9170941114976302 - ], - "candidate_ci95": [ - 0.6626849801685101, - 0.6826421567984873 - ] - }, - "ece": { - "candidate_minus_parent": -0.25457097399316714, - "ci95": [ - -0.3438415036852427, - -0.13863113460598453 - ], - "parent_ci95": [ - 0.25302953070124085, - 0.3843034725335647 - ], - "candidate_ci95": [ - 0.018820144360438072, - 0.1404389973818657 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 28, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.35, - "nll": 1.119540482516699, - "brier": 0.6805384450179266, - "ece": 0.07997479652210188 - }, - "candidate": { - "acc": 0.3, - "nll": 1.1019631899703175, - "brier": 0.6689075031173503, - "ece": 0.04907290355077636 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.04999999999999999, - "ci95": [ - -0.1552146720757268, - 0.050021186440677934 - ], - "parent_ci95": [ - 0.2857142857142857, - 0.41509433962264153 - ], - "candidate_ci95": [ - 0.23214285714285715, - 0.35714285714285715 - ] - }, - "nll": { - "candidate_minus_parent": -0.017577292546381518, - "ci95": [ - -0.05136216005923647, - 0.020599176977015705 - ], - "parent_ci95": [ - 1.0859349092471269, - 1.1508901652452113 - ], - "candidate_ci95": [ - 1.0948957293740824, - 1.1095137465094964 - ] - }, - "brier": { - "candidate_minus_parent": -0.011630941900576341, - "ci95": [ - -0.033248613554045965, - 0.012886900196317849 - ], - "parent_ci95": [ - 0.6590430467723951, - 0.7001408588651075 - ], - "candidate_ci95": [ - 0.6641347720982478, - 0.6740106780892229 - ] - }, - "ece": { - "candidate_minus_parent": -0.030901892971325513, - "ci95": [ - -0.1261490236410924, - 0.05221893446720805 - ], - "parent_ci95": [ - 0.0346809913133188, - 0.15421564281325367 - ], - "candidate_ci95": [ - 0.003870947592434356, - 0.11648710012476678 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 28, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/iid/transactions_3_plus": { - "windows": 97, - "episodes": 75, - "latent_groups": 28, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 16, - "4": 18, - "5": 34, - "6": 24, - "7": 5 - }, - "raw": { - "parent": { - "acc": 0.3466666666666667, - "nll": 1.5680014012828554, - "brier": 0.8735192333226294, - "ece": 0.33890123655052196 - }, - "candidate": { - "acc": 0.3333333333333333, - "nll": 1.1236101735524817, - "brier": 0.684959148963268, - "ece": 0.11333522504688955 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.013333333333333364, - "ci95": [ - -0.06331142486205778, - 0.02857142857142858 - ], - "parent_ci95": [ - 0.32432432432432434, - 0.3699021526418786 - ], - "candidate_ci95": [ - 0.2972972972972973, - 0.3684210526315789 - ] - }, - "nll": { - "candidate_minus_parent": -0.44439122773037365, - "ci95": [ - -0.5961601664795805, - -0.3053275116294062 - ], - "parent_ci95": [ - 1.4424888114143912, - 1.7028062250692066 - ], - "candidate_ci95": [ - 1.1014624518540175, - 1.1451365874380015 - ] - }, - "brier": { - "candidate_minus_parent": -0.18856008435936134, - "ci95": [ - -0.2549635152100391, - -0.1225264364772087 - ], - "parent_ci95": [ - 0.8155310992154277, - 0.9311213349034488 - ], - "candidate_ci95": [ - 0.6690087344244944, - 0.7016534820846407 - ] - }, - "ece": { - "candidate_minus_parent": -0.22556601150363242, - "ci95": [ - -0.3077557235943786, - -0.12794339981102823 - ], - "parent_ci95": [ - 0.30080941430475944, - 0.3967571784864277 - ], - "candidate_ci95": [ - 0.06366678330424731, - 0.19623783341680856 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 28, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3466666666666667, - "nll": 1.131763207180215, - "brier": 0.6863143705776773, - "ece": 0.0877502271071839 - }, - "candidate": { - "acc": 0.3333333333333333, - "nll": 1.1035049943710618, - "brier": 0.6701746076014911, - "ece": 0.048907726748390154 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.013333333333333364, - "ci95": [ - -0.06331142486205778, - 0.02857142857142858 - ], - "parent_ci95": [ - 0.32432432432432434, - 0.3699021526418786 - ], - "candidate_ci95": [ - 0.2972972972972973, - 0.3684210526315789 - ] - }, - "nll": { - "candidate_minus_parent": -0.028258212809153305, - "ci95": [ - -0.05962702570649112, - 0.004027120527058854 - ], - "parent_ci95": [ - 1.1049092475977573, - 1.16036054023137 - ], - "candidate_ci95": [ - 1.0930108046380778, - 1.11365111458743 - ] - }, - "brier": { - "candidate_minus_parent": -0.016139762976186245, - "ci95": [ - -0.036028815996373954, - 0.006768874748515453 - ], - "parent_ci95": [ - 0.6687286849072276, - 0.7042716085169389 - ], - "candidate_ci95": [ - 0.6627336792543009, - 0.6777495706240455 - ] - }, - "ece": { - "candidate_minus_parent": -0.03884250035879375, - "ci95": [ - -0.1186266649620203, - 0.005548265333439938 - ], - "parent_ci95": [ - 0.0664247161683894, - 0.15976734020796518 - ], - "candidate_ci95": [ - 0.017378841072555818, - 0.08763460844669049 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 28, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/heldout_task/all": { - "windows": 229, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff" - ], - "transaction_count_histogram": { - "1": 32, - "2": 38, - "3": 31, - "4": 79, - "5": 25, - "6": 19, - "7": 5 - }, - "raw": { - "parent": { - "acc": 0.3298611111111111, - "nll": 1.698791295214183, - "brier": 0.921416741210046, - "ece": 0.3793138394559325 - }, - "candidate": { - "acc": 0.34722222222222215, - "nll": 1.105515182325449, - "brier": 0.6712249699561775, - "ece": 0.049518278502379665 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.01736111111111105, - "ci95": [ - 0.0, - 0.04513888888888884 - ], - "parent_ci95": [ - 0.3229166666666667, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3333333333333333, - 0.375 - ] - }, - "nll": { - "candidate_minus_parent": -0.593276112888734, - "ci95": [ - -0.6483202339957271, - -0.5433594179811996 - ], - "parent_ci95": [ - 1.6470309252925026, - 1.7596016187674182 - ], - "candidate_ci95": [ - 1.09282458371758, - 1.1184693544582691 - ] - }, - "brier": { - "candidate_minus_parent": -0.2501917712538685, - "ci95": [ - -0.27432881472656895, - -0.2296314348840799 - ], - "parent_ci95": [ - 0.9010265175449971, - 0.9494995621888368 - ], - "candidate_ci95": [ - 0.662110057644752, - 0.6806020442997136 - ] - }, - "ece": { - "candidate_minus_parent": -0.32979556095355284, - "ci95": [ - -0.357606466251182, - -0.2954258621683766 - ], - "parent_ci95": [ - 0.3635599452789377, - 0.40427440215523025 - ], - "candidate_ci95": [ - 0.025157637704134143, - 0.07769176694025634 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3298611111111111, - "nll": 1.1486795121848348, - "brier": 0.6981170684256327, - "ece": 0.11401031812776787 - }, - "candidate": { - "acc": 0.34722222222222215, - "nll": 1.098697216547241, - "brier": 0.6665439269853703, - "ece": 0.02349564669394129 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.01736111111111105, - "ci95": [ - 0.0, - 0.04513888888888884 - ], - "parent_ci95": [ - 0.3229166666666667, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3333333333333333, - 0.375 - ] - }, - "nll": { - "candidate_minus_parent": -0.04998229563759371, - "ci95": [ - -0.05935456056647385, - -0.0402773394943774 - ], - "parent_ci95": [ - 1.138189705911166, - 1.1595554900162879 - ], - "candidate_ci95": [ - 1.0919381937392876, - 1.105469476604823 - ] - }, - "brier": { - "candidate_minus_parent": -0.03157314144026235, - "ci95": [ - -0.038181326557168584, - -0.02542550191942243 - ], - "parent_ci95": [ - 0.691945596634429, - 0.7057214717840435 - ], - "candidate_ci95": [ - 0.6616884540923037, - 0.67147476440236 - ] - }, - "ece": { - "candidate_minus_parent": -0.09051467143382658, - "ci95": [ - -0.1264153633635337, - -0.06263934365622244 - ], - "parent_ci95": [ - 0.10808498983169598, - 0.14582065312155512 - ], - "candidate_ci95": [ - 0.00582896423162496, - 0.05429685894844026 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/heldout_task/transactions_1_2": { - "windows": 70, - "episodes": 61, - "latent_groups": 29, - "sources": [ - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff" - ], - "transaction_count_histogram": { - "1": 32, - "2": 38 - }, - "raw": { - "parent": { - "acc": 0.2786885245901639, - "nll": 1.9015027617394087, - "brier": 1.0017716957302407, - "ece": 0.4605086661841192 - }, - "candidate": { - "acc": 0.36065573770491804, - "nll": 1.1005273055114302, - "brier": 0.6678067387012225, - "ece": 0.002253707087517676 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.08196721311475413, - "ci95": [ - -0.045508658008657964, - 0.22033898305084745 - ], - "parent_ci95": [ - 0.21052631578947367, - 0.3442622950819672 - ], - "candidate_ci95": [ - 0.2878787878787879, - 0.4444444444444444 - ] - }, - "nll": { - "candidate_minus_parent": -0.8009754562279785, - "ci95": [ - -1.0944341558459387, - -0.5388799527475848 - ], - "parent_ci95": [ - 1.6494540797886756, - 2.1890376902151942 - ], - "candidate_ci95": [ - 1.0886223917630717, - 1.1124574888145347 - ] - }, - "brier": { - "candidate_minus_parent": -0.3339649570290182, - "ci95": [ - -0.4616763420076285, - -0.22106294852825745 - ], - "parent_ci95": [ - 0.8926639965028754, - 1.1242562183743325 - ], - "candidate_ci95": [ - 0.6597937031097704, - 0.6758501818214387 - ] - }, - "ece": { - "candidate_minus_parent": -0.4582549590966015, - "ci95": [ - -0.5064246196985348, - -0.31884644225687725 - ], - "parent_ci95": [ - 0.384158071834183, - 0.543965119130656 - ], - "candidate_ci95": [ - 0.001666947178718863, - 0.08851886178851265 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.2786885245901639, - "nll": 1.1808416791698115, - "brier": 0.7182697309306456, - "ece": 0.18638248746506375 - }, - "candidate": { - "acc": 0.36065573770491804, - "nll": 1.0988625637248937, - "brier": 0.6667978182679637, - "ece": 0.012652002959613952 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.08196721311475413, - "ci95": [ - -0.045508658008657964, - 0.22033898305084745 - ], - "parent_ci95": [ - 0.21052631578947367, - 0.3442622950819672 - ], - "candidate_ci95": [ - 0.2878787878787879, - 0.4444444444444444 - ] - }, - "nll": { - "candidate_minus_parent": -0.08197911544491787, - "ci95": [ - -0.147780102043434, - -0.021967052574866506 - ], - "parent_ci95": [ - 1.124289880805791, - 1.2425389171949466 - ], - "candidate_ci95": [ - 1.0929146615274494, - 1.1048799230089281 - ] - }, - "brier": { - "candidate_minus_parent": -0.05147191266268192, - "ci95": [ - -0.09260013917655888, - -0.013927776686896477 - ], - "parent_ci95": [ - 0.6834250582786119, - 0.757755005187276 - ], - "candidate_ci95": [ - 0.6628269859395758, - 0.6708305753576195 - ] - }, - "ece": { - "candidate_minus_parent": -0.1737304845054498, - "ci95": [ - -0.22450514563273516, - -0.07083593966067232 - ], - "parent_ci95": [ - 0.12131586807809672, - 0.2658403693964864 - ], - "candidate_ci95": [ - 0.0020626829750478443, - 0.09761867629624886 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "prefix_loss/heldout_task/transactions_3_plus": { - "windows": 159, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff" - ], - "transaction_count_histogram": { - "3": 31, - "4": 79, - "5": 25, - "6": 19, - "7": 5 - }, - "raw": { - "parent": { - "acc": 0.328125, - "nll": 1.6510195423387115, - "brier": 0.9138329151499195, - "ece": 0.3696085163632797 - }, - "candidate": { - "acc": 0.34375, - "nll": 1.1016034380803594, - "brier": 0.6683896262101987, - "ece": 0.07443605121945863 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.015625, - "ci95": [ - 0.0, - 0.041666666666666685 - ], - "parent_ci95": [ - 0.3177083333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3333333333333333, - 0.375 - ] - }, - "nll": { - "candidate_minus_parent": -0.5494161042583521, - "ci95": [ - -0.6179119075607382, - -0.48969539347820834 - ], - "parent_ci95": [ - 1.5799320401902752, - 1.735727089509863 - ], - "candidate_ci95": [ - 1.0803359724129493, - 1.1229626211532155 - ] - }, - "brier": { - "candidate_minus_parent": -0.24544328893972078, - "ci95": [ - -0.2750370542612782, - -0.22039742851750838 - ], - "parent_ci95": [ - 0.8864907256145825, - 0.9472722248543136 - ], - "candidate_ci95": [ - 0.6527527862224815, - 0.684133775429557 - ] - }, - "ece": { - "candidate_minus_parent": -0.29517246514382106, - "ci95": [ - -0.33283090348061833, - -0.2434390979172202 - ], - "parent_ci95": [ - 0.3498905546275224, - 0.39792126396644434 - ], - "candidate_ci95": [ - 0.04465048942192652, - 0.12030922907162292 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.328125, - "nll": 1.142172990991703, - "brier": 0.6948783988998107, - "ece": 0.11177142406175611 - }, - "candidate": { - "acc": 0.34375, - "nll": 1.0949910574408515, - "brier": 0.6638588432469327, - "ece": 0.04976727430001198 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.015625, - "ci95": [ - 0.0, - 0.041666666666666685 - ], - "parent_ci95": [ - 0.3177083333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3333333333333333, - 0.375 - ] - }, - "nll": { - "candidate_minus_parent": -0.04718193355085165, - "ci95": [ - -0.05846609201493588, - -0.03613562441133395 - ], - "parent_ci95": [ - 1.1293883233608148, - 1.1572591727556825 - ], - "candidate_ci95": [ - 1.0829251309123857, - 1.1064490416782484 - ] - }, - "brier": { - "candidate_minus_parent": -0.03101955565287806, - "ci95": [ - -0.03892913486669872, - -0.023066865238842306 - ], - "parent_ci95": [ - 0.6866324652855914, - 0.7047785964984155 - ], - "candidate_ci95": [ - 0.6551340116156167, - 0.6719950282068228 - ] - }, - "ece": { - "candidate_minus_parent": -0.06200414976174413, - "ci95": [ - -0.11525241336311848, - -0.012050859906388515 - ], - "parent_ci95": [ - 0.10303629178806634, - 0.14801943896591382 - ], - "candidate_ci95": [ - 0.013060497608093698, - 0.10147202300148765 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/all/all": { - "windows": 400, - "episodes": 192, - "latent_groups": 64, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 60, - "2": 60, - "3": 62, - "4": 74, - "5": 95, - "6": 41, - "7": 8 - }, - "raw": { - "parent": { - "acc": 0.3359375, - "nll": 1.7406074901136879, - "brier": 0.9401657076473695, - "ece": 0.38460155194384615 - }, - "candidate": { - "acc": 0.33550347222222227, - "nll": 1.1065801142748992, - "brier": 0.6719468244560088, - "ece": 0.05413551116313254 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.0004340277777777346, - "ci95": [ - -0.009548611111111107, - 0.008691406250000124 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3411458333333333 - ], - "candidate_ci95": [ - 0.3285590277777778, - 0.34375 - ] - }, - "nll": { - "candidate_minus_parent": -0.6340273758387887, - "ci95": [ - -0.6949692587650983, - -0.5763771561528886 - ], - "parent_ci95": [ - 1.683495559665557, - 1.8013055567222513 - ], - "candidate_ci95": [ - 1.096576736428186, - 1.1163791634819291 - ] - }, - "brier": { - "candidate_minus_parent": -0.26821888319136067, - "ci95": [ - -0.28957641859448985, - -0.24767413467914176 - ], - "parent_ci95": [ - 0.9192347540480064, - 0.9606597003615783 - ], - "candidate_ci95": [ - 0.6653425533094902, - 0.6783600881844497 - ] - }, - "ece": { - "candidate_minus_parent": -0.3304660407807136, - "ci95": [ - -0.3513774582804656, - -0.30953687802232904 - ], - "parent_ci95": [ - 0.37061568281624446, - 0.4036105845599263 - ], - "candidate_ci95": [ - 0.04543937738620642, - 0.07021350266475344 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 64, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3359375, - "nll": 1.1514101009185052, - "brier": 0.7007255316703395, - "ece": 0.1318957337257902 - }, - "candidate": { - "acc": 0.33550347222222227, - "nll": 1.0992675611044904, - "brier": 0.667040087036328, - "ece": 0.02521041186312405 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.0004340277777777346, - "ci95": [ - -0.009548611111111107, - 0.008691406250000124 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3411458333333333 - ], - "candidate_ci95": [ - 0.3285590277777778, - 0.34375 - ] - }, - "nll": { - "candidate_minus_parent": -0.05214253981401473, - "ci95": [ - -0.0611047205371555, - -0.042689065801267405 - ], - "parent_ci95": [ - 1.1420658463454039, - 1.1607064611272824 - ], - "candidate_ci95": [ - 1.094538571741435, - 1.1036078394289575 - ] - }, - "brier": { - "candidate_minus_parent": -0.03368544463401146, - "ci95": [ - -0.03992924731720826, - -0.027343189646057253 - ], - "parent_ci95": [ - 0.6942205689664265, - 0.7070023487666626 - ], - "candidate_ci95": [ - 0.6637669919722485, - 0.6700643112167214 - ] - }, - "ece": { - "candidate_minus_parent": -0.10668532186266615, - "ci95": [ - -0.13534219619205876, - -0.08066785804743311 - ], - "parent_ci95": [ - 0.11254079231923728, - 0.15998342150277217 - ], - "candidate_ci95": [ - 0.017516573977361352, - 0.04110521778685313 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 64, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/all/transactions_1_2": { - "windows": 120, - "episodes": 111, - "latent_groups": 53, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 60, - "2": 60 - }, - "raw": { - "parent": { - "acc": 0.35135135135135137, - "nll": 1.6960520792205063, - "brier": 0.9024482031715041, - "ece": 0.35711202622916166 - }, - "candidate": { - "acc": 0.34234234234234234, - "nll": 1.101448747899018, - "brier": 0.6684951405755102, - "ece": 0.0222143113151399 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.009009009009009028, - "ci95": [ - -0.09176435609921849, - 0.0762087912087912 - ], - "parent_ci95": [ - 0.30357142857142855, - 0.39814814814814814 - ], - "candidate_ci95": [ - 0.2935661183169883, - 0.3944954128440367 - ] - }, - "nll": { - "candidate_minus_parent": -0.5946033313214882, - "ci95": [ - -0.7581216779522638, - -0.44897828070830953 - ], - "parent_ci95": [ - 1.5569891830770557, - 1.8546291869076958 - ], - "candidate_ci95": [ - 1.0910651991677789, - 1.111816018257437 - ] - }, - "brier": { - "candidate_minus_parent": -0.2339530625959939, - "ci95": [ - -0.30292693258046355, - -0.16878874992495604 - ], - "parent_ci95": [ - 0.8386540155629556, - 0.9687019878832628 - ], - "candidate_ci95": [ - 0.6615977027563771, - 0.6754919205401331 - ] - }, - "ece": { - "candidate_minus_parent": -0.33489771491402176, - "ci95": [ - -0.39049483430670984, - -0.24899811558459425 - ], - "parent_ci95": [ - 0.3060085541080475, - 0.40839431898214706 - ], - "candidate_ci95": [ - 0.004200428773137663, - 0.07224785599092538 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 53, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.35135135135135137, - "nll": 1.1497208714203155, - "brier": 0.697066459482084, - "ece": 0.09289332764803161 - }, - "candidate": { - "acc": 0.34234234234234234, - "nll": 1.0992606854025573, - "brier": 0.6670804067459685, - "ece": 0.006460488670376252 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.009009009009009028, - "ci95": [ - -0.09176435609921849, - 0.0762087912087912 - ], - "parent_ci95": [ - 0.30357142857142855, - 0.39814814814814814 - ], - "candidate_ci95": [ - 0.2935661183169883, - 0.3944954128440367 - ] - }, - "nll": { - "candidate_minus_parent": -0.05046018601775826, - "ci95": [ - -0.08893543253969396, - -0.016119766171313037 - ], - "parent_ci95": [ - 1.1186675405813775, - 1.185234804225732 - ], - "candidate_ci95": [ - 1.0940675461919098, - 1.104439816406138 - ] - }, - "brier": { - "candidate_minus_parent": -0.029986052736115543, - "ci95": [ - -0.053591089547911044, - -0.008604571076369208 - ], - "parent_ci95": [ - 0.6774031678585275, - 0.7194912987064691 - ], - "candidate_ci95": [ - 0.6636148710664281, - 0.6705104599698513 - ] - }, - "ece": { - "candidate_minus_parent": -0.08643283897765536, - "ci95": [ - -0.1284744458005143, - -0.00567387954157532 - ], - "parent_ci95": [ - 0.04892543439467153, - 0.14708183067036984 - ], - "candidate_ci95": [ - 0.0007245024633393655, - 0.058453570890178434 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 53, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/all/transactions_3_plus": { - "windows": 280, - "episodes": 187, - "latent_groups": 63, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 62, - "4": 74, - "5": 95, - "6": 41, - "7": 8 - }, - "raw": { - "parent": { - "acc": 0.3342245989304813, - "nll": 1.7555289505985587, - "brier": 0.9477640503549342, - "ece": 0.38925499483042736 - }, - "candidate": { - "acc": 0.3351158645276292, - "nll": 1.108188740668899, - "brier": 0.6725390323728005, - "ece": 0.066716952019375 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0008912655971479166, - "ci95": [ - -0.02216312056737585, - 0.024064171122994638 - ], - "parent_ci95": [ - 0.32432432432432434, - 0.34324324324324323 - ], - "candidate_ci95": [ - 0.3163970109060242, - 0.35294117647058826 - ] - }, - "nll": { - "candidate_minus_parent": -0.6473402099296597, - "ci95": [ - -0.733757998665238, - -0.5683007253955933 - ], - "parent_ci95": [ - 1.6771038718491593, - 1.8425302947893092 - ], - "candidate_ci95": [ - 1.0938231412925323, - 1.1228092182280043 - ] - }, - "brier": { - "candidate_minus_parent": -0.2752250179821336, - "ci95": [ - -0.3048612158832415, - -0.24746805643997863 - ], - "parent_ci95": [ - 0.9208384001562874, - 0.9762679410110967 - ], - "candidate_ci95": [ - 0.6630851718543517, - 0.6817288697755652 - ] - }, - "ece": { - "candidate_minus_parent": -0.32253804281105236, - "ci95": [ - -0.35615534554677647, - -0.2874507941358827 - ], - "parent_ci95": [ - 0.36944755188573547, - 0.4157692041247423 - ], - "candidate_ci95": [ - 0.04695683962256579, - 0.09947176928230858 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 63, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3342245989304813, - "nll": 1.150591689158398, - "brier": 0.7008946904316202, - "ece": 0.1363262357343515 - }, - "candidate": { - "acc": 0.3351158645276292, - "nll": 1.0988193944708307, - "brier": 0.6665887963196561, - "ece": 0.030538238453360175 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.0008912655971479166, - "ci95": [ - -0.02216312056737585, - 0.024064171122994638 - ], - "parent_ci95": [ - 0.32432432432432434, - 0.34324324324324323 - ], - "candidate_ci95": [ - 0.3163970109060242, - 0.35294117647058826 - ] - }, - "nll": { - "candidate_minus_parent": -0.05177229468756739, - "ci95": [ - -0.06603192468881637, - -0.03842071619828359 - ], - "parent_ci95": [ - 1.1367162320580266, - 1.1636937233838849 - ], - "candidate_ci95": [ - 1.0919966713264804, - 1.1051700107081668 - ] - }, - "brier": { - "candidate_minus_parent": -0.034305894111964164, - "ci95": [ - -0.043947105187773686, - -0.024996850755553735 - ], - "parent_ci95": [ - 0.6912483686237404, - 0.710424380972464 - ], - "candidate_ci95": [ - 0.6619212202937902, - 0.6709416587645569 - ] - }, - "ece": { - "candidate_minus_parent": -0.10578799728099131, - "ci95": [ - -0.14528332484454656, - -0.06599811211293838 - ], - "parent_ci95": [ - 0.11310969153121071, - 0.17550582691120856 - ], - "candidate_ci95": [ - 0.013988469825566535, - 0.059886216530795705 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 63, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/iid/all": { - "windows": 169, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 28, - "2": 19, - "3": 32, - "4": 23, - "5": 37, - "6": 25, - "7": 5 - }, - "raw": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.725757697659655, - "brier": 0.9401047180979623, - "ece": 0.4020367409320484 - }, - "candidate": { - "acc": 0.3368055555555555, - "nll": 1.1040626060473822, - "brier": 0.6702717561426815, - "ece": 0.0683276279449332 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.00347222222222221, - "ci95": [ - 0.0, - 0.010416666666666685 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3333333333333333, - 0.34375 - ] - }, - "nll": { - "candidate_minus_parent": -0.6216950916122728, - "ci95": [ - -0.717380983626561, - -0.5371541588570768 - ], - "parent_ci95": [ - 1.646108455710435, - 1.8141616324863405 - ], - "candidate_ci95": [ - 1.0878025017305053, - 1.1213471984146075 - ] - }, - "brier": { - "candidate_minus_parent": -0.2698329619552807, - "ci95": [ - -0.3036907003221278, - -0.23773415677566542 - ], - "parent_ci95": [ - 0.9117529183450316, - 0.9728276694058462 - ], - "candidate_ci95": [ - 0.6590794699959776, - 0.681685864765193 - ] - }, - "ece": { - "candidate_minus_parent": -0.3337091129871152, - "ci95": [ - -0.36812145646765443, - -0.2983093108505633 - ], - "parent_ci95": [ - 0.37645003059960197, - 0.42770359084715837 - ], - "candidate_ci95": [ - 0.04981607684558695, - 0.08984381483279809 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.3333333333333333, - "nll": 1.1454640711429278, - "brier": 0.6964743796858682, - "ece": 0.13260620058724457 - }, - "candidate": { - "acc": 0.3368055555555555, - "nll": 1.0968550632456722, - "brier": 0.6653896553980921, - "ece": 0.02770294957160892 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.00347222222222221, - "ci95": [ - 0.0, - 0.010416666666666685 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3333333333333333 - ], - "candidate_ci95": [ - 0.3333333333333333, - 0.34375 - ] - }, - "nll": { - "candidate_minus_parent": -0.04860900789725564, - "ci95": [ - -0.06429208242319576, - -0.035905118558082774 - ], - "parent_ci95": [ - 1.133997196994798, - 1.1597081636865096 - ], - "candidate_ci95": [ - 1.0890255440357801, - 1.1043556773200416 - ] - }, - "brier": { - "candidate_minus_parent": -0.03108472428777609, - "ci95": [ - -0.042035188518183615, - -0.022463586773767412 - ], - "parent_ci95": [ - 0.6888515776201788, - 0.7064283693874138 - ], - "candidate_ci95": [ - 0.6599514223773825, - 0.6705037492073507 - ] - }, - "ece": { - "candidate_minus_parent": -0.10490325101563565, - "ci95": [ - -0.15451502648279997, - -0.07484636724625014 - ], - "parent_ci95": [ - 0.11485842839206964, - 0.18210567599060168 - ], - "candidate_ci95": [ - 0.018669307044195796, - 0.05262544209859075 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/iid/transactions_1_2": { - "windows": 47, - "episodes": 46, - "latent_groups": 24, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "1": 28, - "2": 19 - }, - "raw": { - "parent": { - "acc": 0.391304347826087, - "nll": 1.5129882776373453, - "brier": 0.8186307187425507, - "ece": 0.28917804828375704 - }, - "candidate": { - "acc": 0.32608695652173914, - "nll": 1.1015960837269156, - "brier": 0.668776812291835, - "ece": 0.03962849828397818 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.06521739130434784, - "ci95": [ - -0.20930232558139533, - 0.0667045454545454 - ], - "parent_ci95": [ - 0.3125, - 0.4865038115038115 - ], - "candidate_ci95": [ - 0.25, - 0.40009615384615377 - ] - }, - "nll": { - "candidate_minus_parent": -0.4113921939104297, - "ci95": [ - -0.6205679145739176, - -0.2101180318055222 - ], - "parent_ci95": [ - 1.3216366772080308, - 1.7139671734715645 - ], - "candidate_ci95": [ - 1.0852654892339724, - 1.1183779806860223 - ] - }, - "brier": { - "candidate_minus_parent": -0.1498539064507156, - "ci95": [ - -0.24867992033193947, - -0.043559138339383484 - ], - "parent_ci95": [ - 0.7190015074213191, - 0.9143874547417223 - ], - "candidate_ci95": [ - 0.6579463944990259, - 0.6799861375851854 - ] - }, - "ece": { - "candidate_minus_parent": -0.24954954999977885, - "ci95": [ - -0.3412014486136007, - -0.10647175778303841 - ], - "parent_ci95": [ - 0.20010121025847302, - 0.36941998397149045 - ], - "candidate_ci95": [ - 0.004974082473479452, - 0.11742284705331912 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 24, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.391304347826087, - "nll": 1.120753313789249, - "brier": 0.6776321501881324, - "ece": 0.040942654704029205 - }, - "candidate": { - "acc": 0.32608695652173914, - "nll": 1.09934156032772, - "brier": 0.6671793581123403, - "ece": 0.02327882216570413 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.06521739130434784, - "ci95": [ - -0.20930232558139533, - 0.0667045454545454 - ], - "parent_ci95": [ - 0.3125, - 0.4865038115038115 - ], - "candidate_ci95": [ - 0.25, - 0.40009615384615377 - ] - }, - "nll": { - "candidate_minus_parent": -0.021411753461529015, - "ci95": [ - -0.0742078488590243, - 0.031503542983940015 - ], - "parent_ci95": [ - 1.073362241891128, - 1.168284094132113 - ], - "candidate_ci95": [ - 1.0912596679638558, - 1.107740706599038 - ] - }, - "brier": { - "candidate_minus_parent": -0.010452792075792083, - "ci95": [ - -0.04319876483562449, - 0.024794671496791775 - ], - "parent_ci95": [ - 0.6464336025025834, - 0.7068355883848134 - ], - "candidate_ci95": [ - 0.6618441948255462, - 0.6727848577253428 - ] - }, - "ece": { - "candidate_minus_parent": -0.017663832538325076, - "ci95": [ - -0.10255297223563613, - 0.0762596890938096 - ], - "parent_ci95": [ - 0.0024123742064083987, - 0.12018630791428743 - ], - "candidate_ci95": [ - 0.001062319425773476, - 0.10029696521217593 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 24, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } - }, - "sampled_loss/iid/transactions_3_plus": { - "windows": 122, - "episodes": 91, - "latent_groups": 31, - "sources": [ - "sev_behavioral_artifact_handoff", - "sev_behavioral_authorized_export", - "sev_behavioral_lookup_catalog", - "sev_behavioral_update_export" - ], - "transaction_count_histogram": { - "3": 32, - "4": 23, - "5": 37, - "6": 25, - "7": 5 - }, - "raw": { - "parent": { - "acc": 0.32967032967032966, - "nll": 1.737592255031958, - "brier": 0.9498280924219143, - "ece": 0.41336371029633723 - }, - "candidate": { - "acc": 0.33516483516483514, - "nll": 1.1091061258354027, - "brier": 0.6728955720361526, - "ece": 0.086883523890482 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.005494505494505475, - "ci95": [ - -0.02777777777777779, - 0.03888888888888886 - ], - "parent_ci95": [ - 0.3111111111111111, - 0.34444444444444444 - ], - "candidate_ci95": [ - 0.3146067415730337, - 0.3532608695652174 - ] - }, - "nll": { - "candidate_minus_parent": -0.6284861291965553, - "ci95": [ - -0.7365761589792299, - -0.5299836225042109 - ], - "parent_ci95": [ - 1.6481255398405608, - 1.835985485646765 - ], - "candidate_ci95": [ - 1.0860489866208896, - 1.1354374741301707 - ] - }, - "brier": { - "candidate_minus_parent": -0.27693252038576177, - "ci95": [ - -0.32272262614292246, - -0.2379724134329891 - ], - "parent_ci95": [ - 0.9130870924640325, - 0.9881958378896699 - ], - "candidate_ci95": [ - 0.6577998309311673, - 0.6887578066446614 - ] - }, - "ece": { - "candidate_minus_parent": -0.32648018640585524, - "ci95": [ - -0.3761642813685217, - -0.25983029483154757 - ], - "parent_ci95": [ - 0.38081116400769355, - 0.4491761367813116 - ], - "candidate_ci95": [ - 0.05816060461883278, - 0.13675444810712686 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 31, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - }, - "calibrated": { - "parent": { - "acc": 0.32967032967032966, - "nll": 1.1438484259633617, - "brier": 0.695746062044986, - "ece": 0.14019819112070317 - }, - "candidate": { - "acc": 0.33516483516483514, - "nll": 1.0973027167047875, - "brier": 0.6654772404419866, - "ece": 0.03627728484177039 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.005494505494505475, - "ci95": [ - -0.02777777777777779, - 0.03888888888888886 - ], - "parent_ci95": [ - 0.3111111111111111, - 0.34444444444444444 - ], - "candidate_ci95": [ - 0.3146067415730337, - 0.3532608695652174 - ] - }, - "nll": { - "candidate_minus_parent": -0.04654570925857415, - "ci95": [ - -0.06762029227679757, - -0.026984482065314105 - ], - "parent_ci95": [ - 1.125887311305212, - 1.1627207604459122 - ], - "candidate_ci95": [ - 1.0864801553091392, - 1.1078573621419587 - ] - }, - "brier": { - "candidate_minus_parent": -0.030268821602999374, - "ci95": [ - -0.04420502400536354, - -0.01727495129952463 - ], - "parent_ci95": [ - 0.6846871811700046, - 0.7085095782172175 - ], - "candidate_ci95": [ - 0.6581439135738535, - 0.6725572237106584 - ] - }, - "ece": { - "candidate_minus_parent": -0.10392090627893277, - "ci95": [ - -0.17733269976239296, - -0.06015447573499777 - ], - "parent_ci95": [ - 0.11589770798809797, - 0.2102353079649942 - ], - "candidate_ci95": [ - 0.018755743706560857, - 0.06865304317626338 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 31, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } - } + "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." }, - "sampled_loss/heldout_task/all": { - "windows": 231, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff" - ], - "transaction_count_histogram": { - "1": 32, - "2": 41, - "3": 30, - "4": 51, - "5": 58, - "6": 16, - "7": 3 + "weight_verification": "Recorded checkpoint digests; model weights were not opened by this report" + }, + "candidate": { + "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-swarmtraces-4b-retention-v1/00-trial-0", + "files": { + "result.json": "9e6143f4cf6055c8644458f6920b7138a1a348cc254422e9d3173c268b97c4c6", + "calibration/rows.json": "5dc5633f59c9da863561e1ae34fbc1c87199053cf1397d4592b921ec5ff69790", + "development/rows.json": "9c969c91604dbe733626246efcdecee0c3706d84d5e7504d07f586ee2b518542", + "provenance.json": "5712631ff496a274abe8abecc84f8f259c8e58d6bf1f4a73638d969f2cce8227", + "calibration/report.json": "4f0cae5eb45b35029fe8b65241b5a8976c6a11a7bb5453b8111a6da995c4c40f", + "development/report.json": "f12c5bd74787f4c0daac0f93807f6f2e7c537509a6cc05fb98828e31a58aab20", + "checkpoint/training_config.json": "daf81a93394c6cd9e38cc4b0e049f54910048ffebf4de34f6ded2339d85478cb", + "transfer/rows.json": "2987ed490118f383c96f590852e4a4a4ef72721d50887f8fe4ff7e406670ba37", + "transfer/report.json": "31345b7b6f60a2036039d233f0c484dc7abac1285e1b5db0fe7444d82c9014e4" + }, + "provenance": { + "config": { + "epochs": 1, + "seed": 4, + "lr": 2.5e-06, + "lora": 16, + "accum": 2, + "batch": 4, + "perm_kl": 0, + "perm_frac": 0.3, + "ord_w": 0.0, + "p_none": 0, + "p_none_distract": 0, + "p_distract": 0, + "p_none_pair": 0, + "synthetic_repeat": 1, + "public_frac": 1, + "head_lr": 0.0, + "weight_decay": 0.01, + "anchor_w": 0.0, + "label_smoothing": 0.0, + "brier_w": 0.0, + "focal_gamma": 0.0, + "base": "Qwen/Qwen3.5-4B-Base", + "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b", + "init_from": "/runs/sev-r2-curriculum-repair-4b-v1/02-trial-2/checkpoint", + "lora_targets": "all", + "head_dim": 256, + "weights_dtype": "fp32", + "dtype": "bf16", + "checkpointing": 1, + "option_isolation": 0, + "special_embeddings": 0 }, - "raw": { - "parent": { - "acc": 0.3385416666666667, - "nll": 1.7554572825677204, - "brier": 0.9402266971967772, - "ece": 0.367367651579607 - }, - "candidate": { - "acc": 0.33420138888888884, - "nll": 1.1090976225024158, - "brier": 0.6736218927693362, - "ece": 0.055902489562076274 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.004340277777777846, - "ci95": [ - -0.02256944444444442, - 0.01215277777777768 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3489583333333333 - ], - "candidate_ci95": [ - 0.3211805555555555, - 0.34722222222222227 - ] - }, - "nll": { - "candidate_minus_parent": -0.6463596600653045, - "ci95": [ - -0.7234564302872076, - -0.5713167884632395 - ], - "parent_ci95": [ - 1.678905602327295, - 1.8366894657861392 - ], - "candidate_ci95": [ - 1.099442596350794, - 1.1198317908611226 - ] - }, - "brier": { - "candidate_minus_parent": -0.26660480442744106, - "ci95": [ - -0.29319111488031036, - -0.240211311858877 - ], - "parent_ci95": [ - 0.9137964635276594, - 0.96882149817189 - ], - "candidate_ci95": [ - 0.6668101247461546, - 0.6816392474818874 - ] - }, - "ece": { - "candidate_minus_parent": -0.3114651620175307, - "ci95": [ - -0.3436920042267205, - -0.28518855784843594 - ], - "parent_ci95": [ - 0.35036920812646866, - 0.39004405490227895 - ], - "candidate_ci95": [ - 0.037515519119150936, - 0.07724749162261044 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } + "config_sha256": "958d0b20b3ffd1c572e0abf221383a450e0f8e8d3fff3cb745235a371e6a025d", + "suite_sha256": "61e3c1898b8a03eb178355d3c8c5050e85bd217285c700df9b4506ec9e3680f8", + "source_hashes": { + "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb", + "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582", + "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a", + "kev/benchmark.py": "8d486659a9036102386b077fcbd197fc0d85ae7752a4711666d6b3a95622b26f", + "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519", + "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59", + "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435", + "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", + "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", + "kev/data.py": "a5a5e200f56b60e4bf30a195a4cc87a74afddae15999ec48c7543a4f9b59f71c", + "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4", + "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c", + "kev/experiment.py": "1a5ca47d2b3d236e53c87e8c4197704f162c6692aa008521d31ea491945693f0", + "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243", + "kev/metrics.py": "db0930d470c0ce16f16dcaf726d49a7a103973a5296cefc5259d1e68707d882b", + "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909", + "kev/model.py": "eac0f56c807b85b5c4077483d922e6bb03f07890de2835efd9372cf20cd01b8e", + "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02", + "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3", + "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa", + "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735", + "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3", + "kev/suite.py": "c5e5e27f2fed871b547a021d180b0a7f4a0542a488cce4ee02ad936f433441db", + "kev/train.py": "a2d85632d73760fb3efe51627ff4fde6af3100ed1c34e75d290c1b00c9899aa9", + "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763", + "modal_app.py": "639577258f4d581daefa51dad639961280ce92de9da256316ab4124f005a53fb", + "pyproject.toml": "39ec40f14c88598dee7ac98fb859a5a44e22337235a3064ebd5e89036ce81e30", + "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466" }, - "calibrated": { - "parent": { - "acc": 0.3385416666666667, - "nll": 1.1573561306940823, - "brier": 0.7049766836548104, - "ece": 0.13118526686433576 - }, - "candidate": { - "acc": 0.33420138888888884, - "nll": 1.1016800589633087, - "brier": 0.6686905186745636, - "ece": 0.029678489842914945 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.004340277777777846, - "ci95": [ - -0.02256944444444442, - 0.01215277777777768 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3489583333333333 - ], - "candidate_ci95": [ - 0.3211805555555555, - 0.34722222222222227 - ] - }, - "nll": { - "candidate_minus_parent": -0.0556760717307736, - "ci95": [ - -0.06793569023212345, - -0.04323814096919579 - ], - "parent_ci95": [ - 1.1444930201587054, - 1.1710769227292688 - ], - "candidate_ci95": [ - 1.0968679616282462, - 1.1066588132568023 - ] - }, - "brier": { - "candidate_minus_parent": -0.03628616498024673, - "ci95": [ - -0.044616583193091544, - -0.028245525790847326 - ], - "parent_ci95": [ - 0.696442208712845, - 0.7142081513405784 - ], - "candidate_ci95": [ - 0.6652185510566648, - 0.6722788224530039 - ] - }, - "ece": { - "candidate_minus_parent": -0.10150677702142082, - "ci95": [ - -0.14658130666733923, - -0.06451661944813686 - ], - "parent_ci95": [ - 0.10649446224605773, - 0.17110719018645643 - ], - "candidate_ci95": [ - 0.012983333179381055, - 0.0535721921541909 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } + "git_commit": "76e180880160721082a065bf1882495e756a643a", + "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36", + "torch": "2.8.0+cu128", + "device": "cuda", + "gpu": "NVIDIA H100 80GB HBM3", + "legacy_checkpoint": false, + "measured_checkpoint": { + "requested": "/runs/sev-r2-swarmtraces-4b-retention-v1/00-trial-0/checkpoint", + "resolved": "/runs/sev-r2-swarmtraces-4b-retention-v1/00-trial-0/checkpoint", + "head_sha256": "b29abc817f2340aa510fcd4b2c813455ef38102508120ffc35f7917ce359c27e", + "adapter_sha256": "b5afd73584d1098cccb0a1bd42afc40e2de3c3bc3074a5173a642b6d4b880420", + "inference_temperature": 1.0 } }, - "sampled_loss/heldout_task/transactions_1_2": { - "windows": 73, - "episodes": 65, - "latent_groups": 29, - "sources": [ - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff" + "experiment_gates": { + "passed": true, + "checks": { + "complete_coverage": true, + "isolation_and_packing": true, + "transfer_complete": true, + "heldout_pairs_at_least_70pct": true, + "transfer_confident_errors_below_10pct": true + }, + "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." + }, + "weight_verification": "Recorded checkpoint digests; model weights were not opened by this report" + } + }, + "training_resources": { + "wall_seconds": 1043.7473661899567, + "records_seen": 6438, + "requested_records": 6438, + "truncated_records": 0, + "rejected_records": 0, + "optimizer_steps": 805, + "forward_tokens": 1447855, + "peak_device_bytes": 22227358208, + "device": "cuda", + "dtype": "bf16", + "batch": 4, + "peak_rss_bytes": 31557873664 + }, + "paired_raw": { + "excytin": { + "groups": 1, + "ci95": null, + "point_deltas": { + "acc": 0.0, + "nll": -0.0010619269814233934, + "brier": -8.441910898222422e-05 + }, + "reason": "One incident group; no generalization interval estimated" + }, + "guide": { + "acc": { + "micro_acc_delta": 0.012987012987012991, + "ci95": [ + -0.005102694924123447, + 0.028847543352601156 + ], + "samples": 2000, + "groups": 23, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "nll": { + "micro_nll_delta": 0.0806574651079075, + "ci95": [ + -0.016061369620481, + 0.1993407432639239 + ], + "samples": 2000, + "groups": 23, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "brier": { + "micro_brier_delta": 0.04629405344979143, + "ci95": [ + -0.005474368002895286, + 0.10610974848164327 + ], + "samples": 2000, + "groups": 23, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + } + }, + "native": { + "acc": { + "micro_acc_delta": 0.0, + "ci95": [ + 0.0, + 0.0 + ], + "samples": 2000, + "groups": 4, + "aggregation": "micro", + "unit": "Whole capture group across source families", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "nll": { + "micro_nll_delta": -0.004541798865699442, + "ci95": [ + -0.007275272740407906, + -0.003258894034076542 + ], + "samples": 2000, + "groups": 4, + "aggregation": "micro", + "unit": "Whole capture group across source families", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "brier": { + "micro_brier_delta": -0.0006570378773440486, + "ci95": [ + -0.0018239016582102339, + -0.00012501403854909632 + ], + "samples": 2000, + "groups": 4, + "aggregation": "micro", + "unit": "Whole capture group across source families", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + } + }, + "guide_detector": { + "acc": { + "micro_acc_delta": 0.014097744360902276, + "ci95": [ + -0.010733268142241958, + 0.04307115301628897 + ], + "samples": 2000, + "groups": 114, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "nll": { + "micro_nll_delta": 0.010318565978466143, + "ci95": [ + -0.02562492366637801, + 0.044691969172975764 + ], + "samples": 2000, + "groups": 114, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "brier": { + "micro_brier_delta": 0.005962989482191983, + "ci95": [ + -0.01664672866743052, + 0.028271086382067975 + ], + "samples": 2000, + "groups": 114, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + } + }, + "swarm_static": { + "acc": { + "micro_acc_delta": 0.05932203389830504, + "ci95": [ + 0.034480288509040564, + 0.08547197640117996 + ], + "samples": 2000, + "groups": 176, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "nll": { + "micro_nll_delta": -0.17308792668740217, + "ci95": [ + -0.23906732648623316, + -0.11508206848561314 + ], + "samples": 2000, + "groups": 176, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "brier": { + "micro_brier_delta": -0.08362752812460522, + "ci95": [ + -0.11395797861253039, + -0.05612652240109466 + ], + "samples": 2000, + "groups": 176, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + } + }, + "manual": { + "acc": { + "micro_acc_delta": 0.0843373493975903, + "ci95": [ + 0.034628911857593994, + 0.14634146341463417 + ], + "samples": 2000, + "groups": 40, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "nll": { + "micro_nll_delta": -0.15959679081720557, + "ci95": [ + -0.29781254216292613, + -0.037512233422527574 + ], + "samples": 2000, + "groups": 40, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "brier": { + "micro_brier_delta": -0.12862065982564036, + "ci95": [ + -0.20555506872375667, + -0.0596051281217685 + ], + "samples": 2000, + "groups": 40, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + } + }, + "sysmon": { + "acc": { + "micro_acc_delta": 0.0, + "ci95": [ + 0.0, + 0.0 + ], + "samples": 2000, + "groups": 8, + "aggregation": "micro", + "unit": "Whole capture group across source families", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "nll": { + "micro_nll_delta": -0.0014288934300975553, + "ci95": [ + -0.011110020952993583, + 0.013753809583785515 + ], + "samples": 2000, + "groups": 8, + "aggregation": "micro", + "unit": "Whole capture group across source families", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "brier": { + "micro_brier_delta": 0.0007729154617278078, + "ci95": [ + -0.002342110229997109, + 0.005453425939115126 + ], + "samples": 2000, + "groups": 8, + "aggregation": "micro", + "unit": "Whole capture group across source families", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + } + }, + "general": { + "acc": { + "micro_acc_delta": 0.0, + "ci95": [ + 0.0, + 0.0 + ], + "samples": 2000, + "groups": 64, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "nll": { + "micro_nll_delta": 0.05578705356138791, + "ci95": [ + 0.02406632980677795, + 0.08982514373984912 + ], + "samples": 2000, + "groups": 64, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + }, + "brier": { + "micro_brier_delta": 0.015313433968751122, + "ci95": [ + 0.004495110348257341, + 0.02583795985424143 ], - "transaction_count_histogram": { - "1": 32, - "2": 41 + "samples": 2000, + "groups": 64, + "aggregation": "micro", + "unit": "Whole source/group; all original variants included", + "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample" + } + } + }, + "paired_guide_decisions": { + "guide": { + "samples": 2000, + "seed": 20260927, + "groups": 23, + "unit": "Whole organization, sampled with replacement; all incidents remain together.", + "method": "Paired percentile bootstrap; fixed three-class macro-F1, absent-class F1 zero. TP recall excludes draws with no true positives.", + "statistics": { + "macro_f1": { + "delta": 0.013536133799551708, + "ci95": [ + -0.016836824631839598, + 0.05791991746908926 + ], + "valid_samples": 2000 }, - "raw": { - "parent": { - "acc": 0.3230769230769231, - "nll": 1.82560492341782, - "brier": 0.9617651921519934, - "ece": 0.40518837985206335 - }, - "candidate": { - "acc": 0.35384615384615387, - "nll": 1.101344479466967, - "brier": 0.6682958036685728, - "ece": 0.009890425152577426 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.03076923076923077, - "ci95": [ - -0.07249040920716111, - 0.14516129032258063 - ], - "parent_ci95": [ - 0.26666666666666666, - 0.373134328358209 - ], - "candidate_ci95": [ - 0.2835820895522388, - 0.42309322033898306 - ] - }, - "nll": { - "candidate_minus_parent": -0.7242604439508531, - "ci95": [ - -0.957232645851412, - -0.525437308197973 - ], - "parent_ci95": [ - 1.6314021132204477, - 2.0514856371260306 - ], - "candidate_ci95": [ - 1.0882392331642454, - 1.1148529173040966 - ] - }, - "brier": { - "candidate_minus_parent": -0.29346938848342063, - "ci95": [ - -0.3908429661994472, - -0.20344320094316598 - ], - "parent_ci95": [ - 0.8735299166326217, - 1.0532749805380457 - ], - "candidate_ci95": [ - 0.6595671989614225, - 0.6775008638361005 - ] - }, - "ece": { - "candidate_minus_parent": -0.3952979546994859, - "ci95": [ - -0.441695614734844, - -0.2833377661075722 - ], - "parent_ci95": [ - 0.3438845701232617, - 0.4694929446306306 - ], - "candidate_ci95": [ - 0.00208642196278144, - 0.08624279806717236 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } + "true_positive_recall": { + "delta": -0.02702702702702703, + "ci95": [ + -0.047619047619047616, + 0.0 + ], + "valid_samples": 2000 }, - "calibrated": { - "parent": { - "acc": 0.3230769230769231, - "nll": 1.170220989128455, - "brier": 0.7108199706747266, - "ece": 0.129658419269941 - }, - "candidate": { - "acc": 0.35384615384615387, - "nll": 1.0992034508401352, - "brier": 0.6670103796251514, - "ece": 0.005441716572471168 - }, - "paired": { - "acc": { - "candidate_minus_parent": 0.03076923076923077, - "ci95": [ - -0.07249040920716111, - 0.14516129032258063 - ], - "parent_ci95": [ - 0.26666666666666666, - 0.373134328358209 - ], - "candidate_ci95": [ - 0.2835820895522388, - 0.42309322033898306 - ] - }, - "nll": { - "candidate_minus_parent": -0.0710175382883198, - "ci95": [ - -0.12245128408798325, - -0.02299298460609988 - ], - "parent_ci95": [ - 1.1241940339495027, - 1.2172337233951056 - ], - "candidate_ci95": [ - 1.0926773710164572, - 1.105859396115898 - ] - }, - "brier": { - "candidate_minus_parent": -0.04380959104957516, - "ci95": [ - -0.07619559868356404, - -0.013991724849913031 - ], - "parent_ci95": [ - 0.682306683307591, - 0.7410407625903449 - ], - "candidate_ci95": [ - 0.6626434656543708, - 0.6714627601489861 - ] - }, - "ece": { - "candidate_minus_parent": -0.12421670269746984, - "ci95": [ - -0.16877106313977253, - -0.03649558296884139 - ], - "parent_ci95": [ - 0.08237175233001533, - 0.19524531874276807 - ], - "candidate_ci95": [ - 0.0009436284449954635, - 0.07885721568333505 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 29, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } + "equal_organization_accuracy": { + "delta": 0.006521739130434745, + "ci95": [ + -0.0021739130434783593, + 0.015217391304347794 + ], + "valid_samples": 2000 } - }, - "sampled_loss/heldout_task/transactions_3_plus": { - "windows": 158, - "episodes": 96, - "latent_groups": 32, - "sources": [ - "sev_behavioral_retry_export", - "sev_behavioral_stale_handoff" - ], - "transaction_count_histogram": { - "3": 30, - "4": 51, - "5": 58, - "6": 16, - "7": 3 + } + }, + "guide_detector": { + "samples": 2000, + "seed": 20260927, + "groups": 114, + "unit": "Whole organization, sampled with replacement; all incidents remain together.", + "method": "Paired percentile bootstrap; fixed three-class macro-F1, absent-class F1 zero. TP recall excludes draws with no true positives.", + "statistics": { + "macro_f1": { + "delta": 0.0266170267250998, + "ci95": [ + -0.005222855290995668, + 0.06622974286948383 + ], + "valid_samples": 2000 }, - "raw": { - "parent": { - "acc": 0.3385416666666667, - "nll": 1.7725314432710662, - "brier": 0.9458075104789425, - "ece": 0.3679623325707097 - }, - "candidate": { - "acc": 0.3350694444444445, - "nll": 1.107319135979817, - "brier": 0.672201062483582, - "ece": 0.0675443921109179 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.00347222222222221, - "ci95": [ - -0.039930555555555525, - 0.03298611111111105 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3489583333333333 - ], - "candidate_ci95": [ - 0.3038194444444445, - 0.3697916666666667 - ] - }, - "nll": { - "candidate_minus_parent": -0.6652123072912492, - "ci95": [ - -0.8025975641357458, - -0.5516005543207456 - ], - "parent_ci95": [ - 1.660404920994282, - 1.9139018837933768 - ], - "candidate_ci95": [ - 1.0923167803855924, - 1.123221355468193 - ] - }, - "brier": { - "candidate_minus_parent": -0.2736064479953605, - "ci95": [ - -0.3138128898431603, - -0.23674963543085267 - ], - "parent_ci95": [ - 0.9083144203125041, - 0.9882132562854329 - ], - "candidate_ci95": [ - 0.6617359212586741, - 0.6836821973843982 - ] - }, - "ece": { - "candidate_minus_parent": -0.3004179404597918, - "ci95": [ - -0.3536256938359151, - -0.2540426675462728 - ], - "parent_ci95": [ - 0.3427788587563052, - 0.4034734267267112 - ], - "candidate_ci95": [ - 0.03351414265736119, - 0.10671035016969924 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } + "true_positive_recall": { + "delta": 0.10035842293906805, + "ci95": [ + 0.033210332103321055, + 0.194811225925777 + ], + "valid_samples": 2000 }, - "calibrated": { - "parent": { - "acc": 0.3385416666666667, - "nll": 1.1569837407286923, - "brier": 0.7057751610897839, - "ece": 0.13265594469103895 - }, - "candidate": { - "acc": 0.3350694444444445, - "nll": 1.100257078603226, - "brier": 0.6676424586620296, - "ece": 0.037382529727641785 - }, - "paired": { - "acc": { - "candidate_minus_parent": -0.00347222222222221, - "ci95": [ - -0.039930555555555525, - 0.03298611111111105 - ], - "parent_ci95": [ - 0.3333333333333333, - 0.3489583333333333 - ], - "candidate_ci95": [ - 0.3038194444444445, - 0.3697916666666667 - ] - }, - "nll": { - "candidate_minus_parent": -0.05672666212546629, - "ci95": [ - -0.0763045424395524, - -0.04000481087581666 - ], - "parent_ci95": [ - 1.1384890247157313, - 1.1790609577960465 - ], - "candidate_ci95": [ - 1.0924179530820977, - 1.1083763076809625 - ] - }, - "brier": { - "candidate_minus_parent": -0.03813270242775424, - "ci95": [ - -0.051958591175575164, - -0.026782914028508145 - ], - "parent_ci95": [ - 0.693406489808031, - 0.7212784754522964 - ], - "candidate_ci95": [ - 0.6621406053699213, - 0.6732328817854741 - ] - }, - "ece": { - "candidate_minus_parent": -0.09527341496339717, - "ci95": [ - -0.15369852233059553, - -0.03826622807493024 - ], - "parent_ci95": [ - 0.10441488602576846, - 0.17836289667024915 - ], - "candidate_ci95": [ - 0.008393098027866705, - 0.08302721483572066 - ] - } - }, - "bootstrap": { - "samples": 1000, - "seed": 0, - "latent_groups": 32, - "unit": "group_id alone; every source, episode and view in a group moves together", - "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", - "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" - } + "equal_organization_accuracy": { + "delta": 0.024447391002711205, + "ci95": [ + -0.0024186565385442365, + 0.05694784972373788 + ], + "valid_samples": 2000 } } } }, - "interpretation": [ - "The preregistered primary is equal-episode full-view IID development accuracy/NLL. Combined development, heldout-task/prior shift, loss views and transaction-count panels are secondary.", - "Every episode has equal weight within each panel; its eligible windows share that weight. Probabilities are not pooled into an episode ensemble.", - "Length is the number of observed transactions in _meta.request_ids, not a window's ordinal index. An episode can occur in both length panels.", - "Raw logits are restored using recorded inference_temperature; each model's one diagnostic temperature is fit only on its full-view calibration origin rows.", - "Calibration panels are in-sample for temperature fitting. Development labels never select temperatures; bootstrap intervals condition on the fitted temperatures.", - "Group resampling ignores source boundaries. Matched origins and every episode/window/view in a latent group remain dependent.", - "heldout_task combines unfamiliar tasks with a v2 prior distribution shift in the same controller implementation, not an implementation holdout.", - "Targets describe authored simulated policies. This is a development comparison, not a real-traffic accuracy claim or publication gate." - ] + "gates": { + "passed": false, + "checks": { + "manual_correct_gain": { + "actual": 7, + "bound": 3, + "operator": ">=", + "passed": true + }, + "native_correct": { + "actual": 409, + "bound": 398, + "operator": ">=", + "passed": true + }, + "excytin_correct": { + "actual": 398, + "bound": 391, + "operator": ">=", + "passed": true + }, + "sysmon_correct": { + "actual": 62, + "bound": 60, + "operator": ">=", + "passed": true + }, + "general_correct": { + "actual": 86, + "bound": 85, + "operator": ">=", + "passed": true + }, + "manual_raw_nll": { + "actual": 1.1015953331868638, + "bound": 1.3111921240040694, + "operator": "<=", + "passed": true + }, + "manual_raw_brier": { + "actual": 0.38672041228910636, + "bound": 0.5353410721147468, + "operator": "<=", + "passed": true + }, + "manual_global_calibrated_nll": { + "actual": 0.8197108651023933, + "bound": 1.0802111739617255, + "operator": "<=", + "passed": true + }, + "manual_global_calibrated_brier": { + "actual": 0.3693842986364894, + "bound": 0.5033988288027265, + "operator": "<=", + "passed": true + }, + "native_raw_nll": { + "actual": 0.008457485545022507, + "bound": 0.06299928441072196, + "operator": "<=", + "passed": true + }, + "native_raw_brier": { + "actual": 0.0035344210430228854, + "bound": 0.024191458920366934, + "operator": "<=", + "passed": true + }, + "native_global_calibrated_nll": { + "actual": 0.028769043116345983, + "bound": 0.08020084130636222, + "operator": "<=", + "passed": true + }, + "native_global_calibrated_brier": { + "actual": 0.006284804103393485, + "bound": 0.02669768749600822, + "operator": "<=", + "passed": true + }, + "excytin_raw_nll": { + "actual": 0.0009678808724190956, + "bound": 0.052029807853842494, + "operator": "<=", + "passed": true + }, + "excytin_raw_brier": { + "actual": 4.8256567178328636e-05, + "bound": 0.020132675676160554, + "operator": "<=", + "passed": true + }, + "excytin_global_calibrated_nll": { + "actual": 0.009470865385844369, + "bound": 0.05820502390654013, + "operator": "<=", + "passed": true + }, + "excytin_global_calibrated_brier": { + "actual": 0.0006565384935901168, + "bound": 0.02062640684122579, + "operator": "<=", + "passed": true + }, + "sysmon_raw_nll": { + "actual": 0.08322377416424186, + "bound": 0.13465266759433944, + "operator": "<=", + "passed": true + }, + "sysmon_raw_brier": { + "actual": 0.045695299016738855, + "bound": 0.06492238355501105, + "operator": "<=", + "passed": true + }, + "sysmon_global_calibrated_nll": { + "actual": 0.11024750472054996, + "bound": 0.15719175042876138, + "operator": "<=", + "passed": true + }, + "sysmon_global_calibrated_brier": { + "actual": 0.04954806836760281, + "bound": 0.06844225884031123, + "operator": "<=", + "passed": true + }, + "general_raw_nll": { + "actual": 0.5574328669578856, + "bound": 0.5516458133964977, + "operator": "<=", + "passed": false + }, + "general_raw_brier": { + "actual": 0.29243113448917374, + "bound": 0.29711770052042263, + "operator": "<=", + "passed": true + }, + "general_global_calibrated_nll": { + "actual": 0.42292499037031256, + "bound": 0.4765669988063534, + "operator": "<=", + "passed": true + }, + "general_global_calibrated_brier": { + "actual": 0.24649933996243034, + "bound": 0.26785251439683744, + "operator": "<=", + "passed": true + }, + "guide_raw_nll": { + "actual": 0.8563265185203435, + "bound": 0.8256690534124361, + "operator": "<=", + "passed": false + }, + "guide_raw_brier": { + "actual": 0.5255142828500304, + "bound": 0.499220229400239, + "operator": "<=", + "passed": false + }, + "guide_global_calibrated_nll": { + "actual": 0.7793526965750157, + "bound": 0.8008521265465799, + "operator": "<=", + "passed": true + }, + "guide_global_calibrated_brier": { + "actual": 0.4718856823661165, + "bound": 0.4766593103734014, + "operator": "<=", + "passed": true + }, + "guide_detector_raw_nll": { + "actual": 0.8276776090882748, + "bound": 0.8673590431098086, + "operator": "<=", + "passed": true + }, + "guide_detector_raw_brier": { + "actual": 0.48019952548397726, + "bound": 0.49423653600178535, + "operator": "<=", + "passed": true + }, + "guide_detector_global_calibrated_nll": { + "actual": 0.785071927880167, + "bound": 0.8476013493654482, + "operator": "<=", + "passed": true + }, + "guide_detector_global_calibrated_brier": { + "actual": 0.46075730837789786, + "bound": 0.4889190783897773, + "operator": "<=", + "passed": true + }, + "guide_macro_f1": { + "actual": 0.4613423970886119, + "bound": 0.4278062632890602, + "operator": ">=", + "passed": true + }, + "guide_true_positive_correct": { + "actual": 7, + "bound": 6, + "operator": ">=", + "passed": true + }, + "guide_detector_macro_f1": { + "actual": 0.5681608349992006, + "bound": 0.5215438082741007, + "operator": ">=", + "passed": true + }, + "guide_detector_true_positive_correct": { + "actual": 101, + "bound": 71, + "operator": ">=", + "passed": true + } + }, + "interpretation": "Fixed descriptive engineering gates for a research continuation; not statistical noninferiority or a field detection claim." + }, + "protocol_sha256": "acc38c3ec612344b8bf732f8bacf60f3ebd58815d735982ad25c748be3480f6a", + "scorer_sha256": "967b5bf646812d95167d3d88e1b4df7405da6d74983f8aceefb283512bc29285", + "protocol": { + "version": 3, + "registered_date": "2026-09-27", + "study": "sev-r2-swarmtraces-4b-retention-v1", + "suite": "evals/sev/swarmtraces-curriculum-v1", + "plan": "experiments/sev-swarmtraces-4b-retention-v1.json", + "transfer": "evals/sev/swarmtraces-transfer-v1", + "original_transfer": "evals/sev/security-transfer-v1", + "checkpoints": { + "parent": "/runs/sev-r2-curriculum-repair-4b-v1/02-trial-2/checkpoint", + "candidate": "/runs/sev-r2-swarmtraces-4b-retention-v1/00-trial-0/checkpoint" + }, + "parent_checkpoint_hashes": { + "head_sha256": "5a91ed82f11c6d0c8ab0a910e0c85770119e0034039379aa601d17a3ec7664d6", + "adapter_sha256": "0452010abdcc58f7c39a47a1562a1c812bf85f025d4a6c105c93d486051765de" + }, + "panels": { + "excytin": { + "suite": "evals/sev/excytin-native-v1" + }, + "guide": { + "suite": "evals/sev/guide-v1" + }, + "native": { + "suite": "evals/sev/sysmon-native-v2" + }, + "guide_detector": { + "suite": "evals/sev/guide-detector-v1" + }, + "swarm_static": { + "suite": "evals/sev/swarmtraces-static-v1" + }, + "manual": { + "suite": "evals/sev/swarmtraces-manual-v1" + } + }, + "gates": { + "manual_min_correct_gain": 3, + "minimum_correct": { + "native": 398, + "excytin": 391, + "sysmon": 60, + "general": 85 + }, + "probability_retention_domains": [ + "manual", + "native", + "excytin", + "sysmon", + "general", + "guide", + "guide_detector" + ], + "nll_margin": 0.05, + "brier_margin": 0.02, + "guide_macro_f1_margin": 0.02, + "guide_tp_count_margin": 2 + }, + "calibration": "One micro-NLL temperature for each model on identical union calibration; unchanged on all evaluation panels. Raw and calibrated metrics reported separately.", + "primary": "Independent manually annotated complete-program panel. Development screen, not final field validation.", + "secondary": "Parser-derived held-out programs, grouped by exact text, source reconstruction and whole-program AST templates.", + "interpretation": "Continuation adoption criteria only. Prior GUIDE shortcomings remain separate; this experiment does not establish a general release. No final test is used.", + "bootstrap": { + "samples": 2000, + "seed": 20260927 + }, + "timeout_seconds": 3600, + "maximum_reservation_usd": "5.68", + "input_sha256": { + "docs/reviews/sev-swarmtraces-curriculum-review.md": "5ba6f9682dbfe1782309c934e05c487825731c544ce23f293d131927b059c309", + "docs/reviews/sev-swarmtraces-scorer-v1.md": "d63c433da881fd9a03a2b3d065533dd19f68f0f905bbe7c3aa042d0ccfb63447", + "docs/sev-swarmtraces-round-v1.md": "bdddd3fe1c8f4776710bff45e1f257c66e48f2bb198ca641aaa51f0db053bf35", + "evals/sev/curriculum-repair-v1/manifest.json": "5ae5dbfd48e9a2717397c0197ed8db016db87d69abc30d51813878ef734beba3", + "evals/sev/excytin-native-v1/manifest.json": "1a537e295dce0df278b7648160674bdabcec5d1e6d8adc80c203c1dae9dcaf5c", + "evals/sev/guide-detector-v1/manifest.json": "507cefca384f2a88aa7172f54757dc06bfd6f815e9501c522fcbf071f05be714", + "evals/sev/guide-v1/manifest.json": "50245db125492792319f4fc477ed5b356819f0bf32567bfc059f5285ae58b069", + "evals/sev/security-transfer-v1/manifest.json": "6b2d546ac337946bfce4c85c7634b6f749f98bff0dc84e6c7f2317a4f6eddaea", + "evals/sev/swarmtraces-curriculum-v1/calibration.jsonl": "8504531b423590392325f9e1d5fd291280122e6ca038483268e05bfe94970f77", + "evals/sev/swarmtraces-curriculum-v1/development.jsonl": "69d4aec321f985901ab0ca3ecccf072c66d60479924111d663d5f71d7e877c84", + "evals/sev/swarmtraces-curriculum-v1/manifest.json": "61e3c1898b8a03eb178355d3c8c5050e85bd217285c700df9b4506ec9e3680f8", + "evals/sev/swarmtraces-curriculum-v1/train.jsonl": "d6faa54db510a9285d4c2c2bc1f0c9095108a42b71460ad7ce83bccfcf199b92", + "evals/sev/swarmtraces-manual-v1/annotations.json": "e5f3412722153a5de7e4ee6ee3aed9844f0c062a8afc028b396e65c794314206", + "evals/sev/swarmtraces-manual-v1/calibration.jsonl": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "evals/sev/swarmtraces-manual-v1/development.jsonl": "d0e5832305f1f04e82a16fe3f643a3518061817858736297b6cc3f7423ad4055", + "evals/sev/swarmtraces-manual-v1/manifest.json": "6c1f892e39bd722dea54042f2139514720021faf33698aba9eb7cd399b67324e", + "evals/sev/swarmtraces-manual-v1/protection.json": "abd1e0ebf71e0edd2d94ded564d52f1555667a3fbf6fb828f3e44e1d7feb173d", + "evals/sev/swarmtraces-manual-v1/train.jsonl": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "evals/sev/swarmtraces-manual-v1/validation.json": "b4302a4fa05444ac1714c91ad0daf4e63dc66e8e4065fbc52e78bb6d79de0fea", + "evals/sev/swarmtraces-static-v1/calibration.jsonl": "7754bac053cd7a2bfb2303d9e1a591381a2a25925f6f728d7995a6fbb343dc5d", + "evals/sev/swarmtraces-static-v1/development.jsonl": "1847a598a428615a416c2c11d5bce0f6a129bd926684cbbe3bd5ccd7b4ec1321", + "evals/sev/swarmtraces-static-v1/manifest.json": "1f304228dbbcad5b4aab5d9ce582738b8b7109214c1fdfd56c617fa5f1db583b", + "evals/sev/swarmtraces-static-v1/train.jsonl": "c09de8a213c5cd6b3506c9b3ccea60074115052617ce71307a24d99d067ef8bd", + "evals/sev/swarmtraces-transfer-v1/calibration.jsonl": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "evals/sev/swarmtraces-transfer-v1/development.jsonl": "e312086e75f6c3b8367be5b949e163f9d48a5634a1f304f555d9f063685c9190", + "evals/sev/swarmtraces-transfer-v1/manifest.json": "1caf2255850930113bc56b708d0881ed22d3df9ba1868e966f9454595c2cefa7", + "evals/sev/swarmtraces-transfer-v1/train.jsonl": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "evals/sev/sysmon-native-v2/manifest.json": "e8c3b5dd306525e88e17b70f410892744bf99f2ed18fb14f42b910db52e558b1", + "experiments/sev-swarmtraces-4b-v1.json": "87e70bce1e76ccddd4a60866f3eb93e946a6f950de062cb2290db58e4ca1aee1", + "experiments/sev-swarmtraces-assembly-v1.json": "1915555716b0c993e1dfebc8928c1a8188265f1869db1884873f28c85d363851", + "runs/sev-curriculum-repair-v1/comparison.json": "623c2c4b2e348eab8a5975e48a94a4e9b0d2c98e7f7a43d6024a87779485b042", + "runs/sev-swarmtraces-review-20260927/draft4-admission.json": "ff1bcfbfb7448d4c60b4d86038ec5c7b6057c3e9e833a3b51d01ee66337cb318", + "runs/sev-swarmtraces-review-20260927/final-admission-receipt.json": "5f037732f74e09e0e46bbebbe95ad45146f89060d96ad9cbccfa59568ac5fad2", + "runs/sev-swarmtraces-review-20260927/panel-protection.json": "abd1e0ebf71e0edd2d94ded564d52f1555667a3fbf6fb828f3e44e1d7feb173d", + "runs/sev-swarmtraces-review-20260927/shared-fingerprints.jsonl": "452985f0efabc3b97a1f40a1e6a3eec57dd881ebeb448049f2de9120b491055b", + "scripts/assemble_sev_curriculum.py": "147b9f2a5133761d3a30acb08c90ba65cf9f4a278fed0e6310d39b5922eb551a", + "scripts/build_sev_swarmtraces.py": "08d60fcdfd9557633b5293e2b5e3b69758756f50ecedfa3e997ef310d396267c", + "scripts/integrate_sev_swarmtraces.py": "08ec5f630da2e8987eae7d6f93cd0cf7c674a0f528e3c2597f6886d01075a8ed", + "scripts/score_sev_curriculum_repair.py": "f9456677fbfc5f97106ce16c8bd31ab8818513778ab527e366a808ba655506b5", + "scripts/score_sev_guide.py": "1fc0d5e94af1fb456c571e48f312b8ff833dc2db53a5d53db1ba1f5b00cb8527", + "scripts/score_sev_mixed_public.py": "c2bbbef53c567065ee7c694f355b26af7b43696cf3244d89bac835dba4081192", + "scripts/score_sev_security.py": "83351d2299aa308611e14679e4bb77845e40519a8a01248bc651547f844f172b", + "scripts/score_sev_swarmtraces.py": "967b5bf646812d95167d3d88e1b4df7405da6d74983f8aceefb283512bc29285", + "scripts/swarmtraces_ast.mjs": "1c1035149778b411eda4ca95a5cf822afd6a0117064c6f6687130fc113ea81e3", + "modal_app.py": "639577258f4d581daefa51dad639961280ce92de9da256316ab4124f005a53fb", + "tests/test_modal_input_mount.py": "6fcc9ccc9da13f72646538e1342728b7e935bf7ebb156af704f0ad0f4843d3e3", + "runs/sev-swarmtraces-intake-20260927/failed-launch-v1.json": "6702c11219de850dba1723cbad354276c0db691bc5ba9125ce49ca1f67e5edfe", + "experiments/sev-swarmtraces-4b-retention-v1.json": "8dea92d4ab89f7aa09bcdae6c1299ea06a8875cfc09d657601b64331ac4377a4", + "docs/sev-swarmtraces-retention-v1.md": "39b57384b91a850420cc624d0081b2d1de7551539145e19af764854ed31b059c", + "experiments/sev-swarmtraces-protocol-v2.json": "62680e8f7965b981efe8d9281fc0b0dbd5b8f0759a871bad68e249365377538a", + "runs/sev-swarmtraces-v2/comparison.json": "df7a4433b1320e98f5a2cec59f792c2ac641aa74fa69a6e5306b7341f3219b57", + "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/provenance.json": "39898e5e6ef87112f653436a9b6b80555da10a01b0d203b3f5eb29000dddae3a", + "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/result.json": "46b3ee2a68f250ba1d433c25612083bb8235a03e617d90707fa796f86ae2cbf4", + "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/calibration/rows.json": "9a33b44bee9cb63a635ac1516bcab9f67fbe56b15a72995a8c2b54a321ad4dd6", + "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/calibration/report.json": "15dc07e79e0121e7345ece52b028220fbbf9123609a2eee5b331c9d372370383", + "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/development/rows.json": "d882c91290398ba96cf39de9128ea5a973cf8d79d20ca84f5249f9f1c1cd5300", + "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/development/report.json": "02fc22461d5b44f88173b90e0fffe97905b4eeb132eec79088c9a8396ce73134", + "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/transfer/rows.json": "5b624c0b0f91940332045523703dd5d52761a5ff931f2c9f1751185883976368", + "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/transfer/report.json": "f44bc7be71a3b37ded9e35f6fa382c1907c70fc959c15bc712b76f024ac726cc", + "docs/reviews/sev-swarmtraces-results-v2.md": "bbfc48b93b09dafbdac5d44576d7c852fa07fcb021e587560109c5c1516313f8", + "runs/sev-swarmtraces-review-20260927/results-v2-final-receipt.json": "6d474088bbd9bbe725ea7b0fb17f4e82d31ec2f95bfa07b3a6994b7a93a5ce52", + "runs/sev-swarmtraces-review-20260927/results-v2-review.json": "dc5f2a90997f3d4c13fc65a63e1eb8939b8d9cae8dc278ff5c1707f8e588f397", + "runs/sev-swarmtraces-review-20260927/audit-results-v2.py": "5e9966bd8a51312e85a162e17d3ce71da87b2b9a7000641a407fa56ccb93be87" + }, + "retained_suite": "evals/sev/curriculum-repair-v1", + "admission_limitations": [ + "No earlier-response credential examples survive automatic admission.", + "Five execution-flow and eleven unavailable-context examples are training-only; automatic development does not test these classes.", + "The manual panel is a fixed development screen, not a locked final test or field detection evaluation." + ], + "maximum_total_reservations_including_failed_launch_usd": "28.40", + "preceding_development_screen": { + "protocol": "experiments/sev-swarmtraces-protocol-v2.json", + "protocol_sha256": "62680e8f7965b981efe8d9281fc0b0dbd5b8f0759a871bad68e249365377538a", + "result": "runs/sev-swarmtraces-v2/comparison.json", + "result_sha256": "df7a4433b1320e98f5a2cec59f792c2ac641aa74fa69a6e5306b7341f3219b57", + "decision": "Keep the same parent, data and gates. Test one continuation at half the learning rate after the older GUIDE panel failed raw NLL, raw Brier and true-positive retention checks. The manual panel is reused development data, not fresh confirmation." + }, + "reused_parent_evaluation": "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint", + "expected_training_resources": { + "records_seen": 6438, + "optimizer_steps": 805, + "forward_tokens": 1447855, + "truncated_records": 0, + "rejected_records": 0 + } + } }