| { |
| "selected": "head_lora_rlcd", |
| "seed_run": "artifacts/runs/e4b-v142", |
| "method": "Proper-scoring REINFORCE with Gaussian logit exploration", |
| "gates": { |
| "seed": { |
| "eligible": true, |
| "score": 0.0 |
| }, |
| "head_rlcd": { |
| "eligible": true, |
| "preservation_gate": true, |
| "probability_gate": true, |
| "score": 0.013723702557236433, |
| "mean_nll": 0.2912874264480955, |
| "mean_brier": 0.17143807150110177, |
| "mean_ece": 0.026519458902999775 |
| }, |
| "head_lora_rlcd": { |
| "eligible": true, |
| "preservation_gate": true, |
| "probability_gate": true, |
| "score": 0.02410640078446494, |
| "mean_nll": 0.2851054899756657, |
| "mean_brier": 0.1656363697259328, |
| "mean_ece": 0.02280542785381634 |
| } |
| }, |
| "results": { |
| "seed": { |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.097443699836731, |
| "groups": { |
| "choice:2": { |
| "temperature": 0.7726892229265028, |
| "raw_temperature": 0.4564978778362274, |
| "n": 80, |
| "shrinkage_weight": 0.4 |
| }, |
| "choice:3-5": { |
| "temperature": 1.0735645605398847, |
| "raw_temperature": 1.0707285404205322, |
| "n": 998, |
| "shrinkage_weight": 0.8926654740608229 |
| }, |
| "choice:6-10": { |
| "temperature": 1.0213380776161125, |
| "raw_temperature": 0.9631407260894775, |
| "n": 147, |
| "shrinkage_weight": 0.550561797752809 |
| }, |
| "noul:*": { |
| "temperature": 1.18076753369684, |
| "raw_temperature": 1.1976659297943115, |
| "n": 618, |
| "shrinkage_weight": 0.8373983739837398 |
| }, |
| "score:*": { |
| "temperature": 1.1118863993359898, |
| "raw_temperature": 1.1230534315109253, |
| "n": 157, |
| "shrinkage_weight": 0.5667870036101083 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "sets": { |
| "validation": { |
| "n": 2000, |
| "accuracy": 0.8635, |
| "nll": 0.33747788503553516, |
| "brier": 0.1907344097228753, |
| "ece": 0.016734494030475625, |
| "mean_confidence": 0.8650541846752167, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.097443699836731, |
| "groups": { |
| "choice:2": { |
| "temperature": 0.7726892229265028, |
| "raw_temperature": 0.4564978778362274, |
| "n": 80, |
| "shrinkage_weight": 0.4 |
| }, |
| "choice:3-5": { |
| "temperature": 1.0735645605398847, |
| "raw_temperature": 1.0707285404205322, |
| "n": 998, |
| "shrinkage_weight": 0.8926654740608229 |
| }, |
| "choice:6-10": { |
| "temperature": 1.0213380776161125, |
| "raw_temperature": 0.9631407260894775, |
| "n": 147, |
| "shrinkage_weight": 0.550561797752809 |
| }, |
| "noul:*": { |
| "temperature": 1.18076753369684, |
| "raw_temperature": 1.1976659297943115, |
| "n": 618, |
| "shrinkage_weight": 0.8373983739837398 |
| }, |
| "score:*": { |
| "temperature": 1.1118863993359898, |
| "raw_temperature": 1.1230534315109253, |
| "n": 157, |
| "shrinkage_weight": 0.5667870036101083 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "PolyAI/banking77": { |
| "n": 136, |
| "accuracy": 0.9338235294117647, |
| "nll": 0.18829089664733842, |
| "brier": 0.10488018020936762, |
| "ece": 0.05482878689380254, |
| "mean_confidence": 0.9316614732584533 |
| }, |
| "Rowan/hellaswag": { |
| "n": 153, |
| "accuracy": 0.934640522875817, |
| "nll": 0.20100579900196738, |
| "brier": 0.11092448101729271, |
| "ece": 0.050551106921987575, |
| "mean_confidence": 0.9167824553118812 |
| }, |
| "alisawuffles/WANLI": { |
| "n": 158, |
| "accuracy": 0.7911392405063291, |
| "nll": 0.4983649758763517, |
| "brier": 0.2937829899183501, |
| "ece": 0.06604516147812708, |
| "mean_confidence": 0.7893789567147629 |
| }, |
| "cais/mmlu": { |
| "n": 154, |
| "accuracy": 0.8766233766233766, |
| "nll": 0.4075193296471239, |
| "brier": 0.19814945253592872, |
| "ece": 0.07114563940407395, |
| "mean_confidence": 0.8730270781300284 |
| }, |
| "google/boolq": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.3304804692638724, |
| "brier": 0.19049125629416552, |
| "ece": 0.04868968210968311, |
| "mean_confidence": 0.8886790147014693 |
| }, |
| "google/boolq-negation": { |
| "n": 153, |
| "accuracy": 0.8627450980392157, |
| "nll": 0.29803780987980516, |
| "brier": 0.1722142930618895, |
| "ece": 0.06692373012405595, |
| "mean_confidence": 0.8571316631011714 |
| }, |
| "nvidia/HelpSteer2": { |
| "n": 157, |
| "accuracy": 0.535031847133758, |
| "nll": 0.9802866589491535, |
| "brier": 0.5608620777678718, |
| "ece": 0.1083361166677657, |
| "mean_confidence": 0.5841244707821281 |
| }, |
| "nvidia/HelpSteer2-threshold": { |
| "n": 158, |
| "accuracy": 0.879746835443038, |
| "nll": 0.3300814368538087, |
| "brier": 0.19343432728677312, |
| "ece": 0.06685320712343043, |
| "mean_confidence": 0.832719200019595 |
| }, |
| "nyu-mll/multi_nli": { |
| "n": 157, |
| "accuracy": 0.8662420382165605, |
| "nll": 0.3357255889909567, |
| "brier": 0.19144842998859854, |
| "ece": 0.07937940025025873, |
| "mean_confidence": 0.84814400467903 |
| }, |
| "product-science/xlam-function-calling-60k-raw": { |
| "n": 154, |
| "accuracy": 1.0, |
| "nll": 0.002737168537177378, |
| "brier": 0.0005628128288157573, |
| "ece": 0.0025677526151979472, |
| "mean_confidence": 0.9974322473848021 |
| }, |
| "product-science/xlam-hard": { |
| "n": 154, |
| "accuracy": 0.961038961038961, |
| "nll": 0.105508735974156, |
| "brier": 0.06104789821045359, |
| "ece": 0.03465073526679697, |
| "mean_confidence": 0.9694648733386746 |
| }, |
| "stanfordnlp/snli": { |
| "n": 159, |
| "accuracy": 0.8553459119496856, |
| "nll": 0.3588936702368793, |
| "brier": 0.2055985772370396, |
| "ece": 0.053331098466549334, |
| "mean_confidence": 0.8946507621861104 |
| }, |
| "synthetic/nli-haystack": { |
| "n": 154, |
| "accuracy": 0.8766233766233766, |
| "nll": 0.31448460003064777, |
| "brier": 0.1750220485890215, |
| "ece": 0.058161640709096796, |
| "mean_confidence": 0.8783779376512998 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 1225, |
| "accuracy": 0.8906122448979592, |
| "nll": 0.2910091881993051, |
| "brier": 0.16159634624408029, |
| "ece": 0.018170039288851744, |
| "mean_confidence": 0.8901425058257824 |
| }, |
| "noul": { |
| "n": 618, |
| "accuracy": 0.8932038834951457, |
| "nll": 0.2662856133849587, |
| "brier": 0.15446270078834373, |
| "ece": 0.022636081794319064, |
| "mean_confidence": 0.8866929737881164 |
| }, |
| "score": { |
| "n": 157, |
| "accuracy": 0.535031847133758, |
| "nll": 0.9802866589491535, |
| "brier": 0.5608620777678718, |
| "ece": 0.1083361166677657, |
| "mean_confidence": 0.5841244707821281 |
| } |
| }, |
| "by_transform": { |
| "agent.invocation.correct.v1": { |
| "n": 59, |
| "accuracy": 0.9322033898305084, |
| "nll": 0.19396388833805667, |
| "brier": 0.10943394663022984, |
| "ece": 0.06162244788670942, |
| "mean_confidence": 0.9474405292737282 |
| }, |
| "agent.invocation.corrupt_argument.v1": { |
| "n": 47, |
| "accuracy": 1.0, |
| "nll": 0.007110317709528782, |
| "brier": 0.00041410416924144496, |
| "ece": 0.007002635205045669, |
| "mean_confidence": 0.9929973647949544 |
| }, |
| "agent.invocation.wrong_tool.v1": { |
| "n": 48, |
| "accuracy": 0.9583333333333334, |
| "nll": 0.09313106241097557, |
| "brier": 0.06094397035983216, |
| "ece": 0.048730698724587775, |
| "mean_confidence": 0.9734942317008972 |
| }, |
| "commonsense.choice.v2": { |
| "n": 153, |
| "accuracy": 0.934640522875817, |
| "nll": 0.20100579900196738, |
| "brier": 0.11092448101729271, |
| "ece": 0.050551106921987575, |
| "mean_confidence": 0.9167824553118812 |
| }, |
| "intent.choice.v3": { |
| "n": 136, |
| "accuracy": 0.9338235294117647, |
| "nll": 0.18829089664733842, |
| "brier": 0.10488018020936762, |
| "ece": 0.05482878689380254, |
| "mean_confidence": 0.9316614732584533 |
| }, |
| "knowledge.choice.v1": { |
| "n": 154, |
| "accuracy": 0.8766233766233766, |
| "nll": 0.4075193296471239, |
| "brier": 0.19814945253592872, |
| "ece": 0.07114563940407395, |
| "mean_confidence": 0.8730270781300284 |
| }, |
| "nli.choice.v4": { |
| "n": 157, |
| "accuracy": 0.8662420382165605, |
| "nll": 0.3357255889909567, |
| "brier": 0.19144842998859854, |
| "ece": 0.07937940025025873, |
| "mean_confidence": 0.84814400467903 |
| }, |
| "nli.hard.v4": { |
| "n": 158, |
| "accuracy": 0.7911392405063291, |
| "nll": 0.4983649758763517, |
| "brier": 0.2937829899183501, |
| "ece": 0.06604516147812708, |
| "mean_confidence": 0.7893789567147629 |
| }, |
| "nli.haystack.absent.v1": { |
| "n": 9, |
| "accuracy": 1.0, |
| "nll": 0.05771497719817691, |
| "brier": 0.012067315275392806, |
| "ece": 0.05367900265587699, |
| "mean_confidence": 0.946320997344123 |
| }, |
| "nli.haystack.present.v1": { |
| "n": 145, |
| "accuracy": 0.8689655172413793, |
| "nll": 0.33042202489611144, |
| "brier": 0.18513648031193639, |
| "ece": 0.060484279846322944, |
| "mean_confidence": 0.8741607822220901 |
| }, |
| "nli.snli.v1": { |
| "n": 159, |
| "accuracy": 0.8553459119496856, |
| "nll": 0.3588936702368793, |
| "brier": 0.2055985772370396, |
| "ece": 0.053331098466549334, |
| "mean_confidence": 0.8946507621861104 |
| }, |
| "noul.negated_answer.v1": { |
| "n": 153, |
| "accuracy": 0.8627450980392157, |
| "nll": 0.29803780987980516, |
| "brier": 0.1722142930618895, |
| "ece": 0.06692373012405595, |
| "mean_confidence": 0.8571316631011714 |
| }, |
| "noul.v3": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.3304804692638724, |
| "brier": 0.19049125629416552, |
| "ece": 0.04868968210968311, |
| "mean_confidence": 0.8886790147014693 |
| }, |
| "score.coherence.threshold.v1": { |
| "n": 35, |
| "accuracy": 0.9428571428571428, |
| "nll": 0.30217631511124116, |
| "brier": 0.16118863898768074, |
| "ece": 0.13178183862141202, |
| "mean_confidence": 0.8401974422591073 |
| }, |
| "score.coherence.v3": { |
| "n": 32, |
| "accuracy": 0.5625, |
| "nll": 0.9295596550218761, |
| "brier": 0.5722244664793834, |
| "ece": 0.20393101125955582, |
| "mean_confidence": 0.694971589371562 |
| }, |
| "score.complexity.threshold.v1": { |
| "n": 30, |
| "accuracy": 0.9333333333333333, |
| "nll": 0.27167421360500155, |
| "brier": 0.14001424941464696, |
| "ece": 0.082729438940684, |
| "mean_confidence": 0.9072748144467672 |
| }, |
| "score.complexity.v3": { |
| "n": 27, |
| "accuracy": 0.5925925925925926, |
| "nll": 0.7247205337992421, |
| "brier": 0.4520837900539239, |
| "ece": 0.1485392583741082, |
| "mean_confidence": 0.6281530835010387 |
| }, |
| "score.correctness.threshold.v1": { |
| "n": 30, |
| "accuracy": 0.7333333333333333, |
| "nll": 0.44957111608237027, |
| "brier": 0.30260899337008595, |
| "ece": 0.13217389583587646, |
| "mean_confidence": 0.7230223735173543 |
| }, |
| "score.correctness.v3": { |
| "n": 30, |
| "accuracy": 0.4, |
| "nll": 1.2207635447382927, |
| "brier": 0.6627602388461431, |
| "ece": 0.19082429458697636, |
| "mean_confidence": 0.4697169209520022 |
| }, |
| "score.helpfulness.threshold.v1": { |
| "n": 30, |
| "accuracy": 0.8, |
| "nll": 0.41629887875169513, |
| "brier": 0.27174266742852826, |
| "ece": 0.1563175916671753, |
| "mean_confidence": 0.7734385410944621 |
| }, |
| "score.helpfulness.v3": { |
| "n": 36, |
| "accuracy": 0.5277777777777778, |
| "nll": 1.0197049867775705, |
| "brier": 0.5488159826232327, |
| "ece": 0.2696710663537184, |
| "mean_confidence": 0.44966551951236194 |
| }, |
| "score.verbosity.threshold.v1": { |
| "n": 33, |
| "accuracy": 0.9696969696969697, |
| "nll": 0.2257687800253431, |
| "brier": 0.10575891031343913, |
| "ece": 0.07801289088798288, |
| "mean_confidence": 0.9106257955233256 |
| }, |
| "score.verbosity.v3": { |
| "n": 32, |
| "accuracy": 0.59375, |
| "nll": 0.9768548817373812, |
| "brier": 0.5593037003418431, |
| "ece": 0.16308739501982927, |
| "mean_confidence": 0.6946516083553433 |
| }, |
| "tool.choice.single.v3": { |
| "n": 154, |
| "accuracy": 1.0, |
| "nll": 0.002737168537177378, |
| "brier": 0.0005628128288157573, |
| "ece": 0.0025677526151979472, |
| "mean_confidence": 0.9974322473848021 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 698, |
| "accuracy": 0.9054441260744985, |
| "nll": 0.23576630212333755, |
| "brier": 0.13675923951146768, |
| "ece": 0.020041158684345223, |
| "mean_confidence": 0.8996789226559309 |
| }, |
| "3-5": { |
| "n": 1155, |
| "accuracy": 0.8285714285714286, |
| "nll": 0.41972496986994157, |
| "brier": 0.23527893114692047, |
| "ece": 0.017945458233614507, |
| "mean_confidence": 0.8350021731027794 |
| }, |
| "6-10": { |
| "n": 147, |
| "accuracy": 0.9387755102040817, |
| "nll": 0.17420782985849112, |
| "brier": 0.09703200538811604, |
| "ece": 0.05073268012124669, |
| "mean_confidence": 0.9367685129447859 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 1999, |
| "accuracy": 0.863431715857929, |
| "nll": 0.3375232138199196, |
| "brier": 0.19078194724154862, |
| "ece": 0.016852297354722528, |
| "mean_confidence": 0.865096109905739 |
| }, |
| "truncated": { |
| "n": 1, |
| "accuracy": 1.0, |
| "nll": 0.2468656450510025, |
| "brier": 0.09570690989494324, |
| "ece": 0.2187543511390686, |
| "mean_confidence": 0.7812456488609314 |
| } |
| }, |
| "by_dataset_truncation": { |
| "PolyAI/banking77|not_truncated": { |
| "n": 136, |
| "accuracy": 0.9338235294117647, |
| "nll": 0.18829089664733842, |
| "brier": 0.10488018020936762, |
| "ece": 0.05482878689380254, |
| "mean_confidence": 0.9316614732584533 |
| }, |
| "Rowan/hellaswag|not_truncated": { |
| "n": 153, |
| "accuracy": 0.934640522875817, |
| "nll": 0.20100579900196738, |
| "brier": 0.11092448101729271, |
| "ece": 0.050551106921987575, |
| "mean_confidence": 0.9167824553118812 |
| }, |
| "alisawuffles/WANLI|not_truncated": { |
| "n": 158, |
| "accuracy": 0.7911392405063291, |
| "nll": 0.4983649758763517, |
| "brier": 0.2937829899183501, |
| "ece": 0.06604516147812708, |
| "mean_confidence": 0.7893789567147629 |
| }, |
| "cais/mmlu|not_truncated": { |
| "n": 154, |
| "accuracy": 0.8766233766233766, |
| "nll": 0.4075193296471239, |
| "brier": 0.19814945253592872, |
| "ece": 0.07114563940407395, |
| "mean_confidence": 0.8730270781300284 |
| }, |
| "google/boolq-negation|not_truncated": { |
| "n": 153, |
| "accuracy": 0.8627450980392157, |
| "nll": 0.29803780987980516, |
| "brier": 0.1722142930618895, |
| "ece": 0.06692373012405595, |
| "mean_confidence": 0.8571316631011714 |
| }, |
| "google/boolq|not_truncated": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.3304804692638724, |
| "brier": 0.19049125629416552, |
| "ece": 0.04868968210968311, |
| "mean_confidence": 0.8886790147014693 |
| }, |
| "nvidia/HelpSteer2-threshold|not_truncated": { |
| "n": 157, |
| "accuracy": 0.8789808917197452, |
| "nll": 0.3306114737442724, |
| "brier": 0.19405679491347266, |
| "ece": 0.06588568391313974, |
| "mean_confidence": 0.8330470570333445 |
| }, |
| "nvidia/HelpSteer2-threshold|truncated": { |
| "n": 1, |
| "accuracy": 1.0, |
| "nll": 0.2468656450510025, |
| "brier": 0.09570690989494324, |
| "ece": 0.2187543511390686, |
| "mean_confidence": 0.7812456488609314 |
| }, |
| "nvidia/HelpSteer2|not_truncated": { |
| "n": 157, |
| "accuracy": 0.535031847133758, |
| "nll": 0.9802866589491535, |
| "brier": 0.5608620777678718, |
| "ece": 0.1083361166677657, |
| "mean_confidence": 0.5841244707821281 |
| }, |
| "nyu-mll/multi_nli|not_truncated": { |
| "n": 157, |
| "accuracy": 0.8662420382165605, |
| "nll": 0.3357255889909567, |
| "brier": 0.19144842998859854, |
| "ece": 0.07937940025025873, |
| "mean_confidence": 0.84814400467903 |
| }, |
| "product-science/xlam-function-calling-60k-raw|not_truncated": { |
| "n": 154, |
| "accuracy": 1.0, |
| "nll": 0.002737168537177378, |
| "brier": 0.0005628128288157573, |
| "ece": 0.0025677526151979472, |
| "mean_confidence": 0.9974322473848021 |
| }, |
| "product-science/xlam-hard|not_truncated": { |
| "n": 154, |
| "accuracy": 0.961038961038961, |
| "nll": 0.105508735974156, |
| "brier": 0.06104789821045359, |
| "ece": 0.03465073526679697, |
| "mean_confidence": 0.9694648733386746 |
| }, |
| "stanfordnlp/snli|not_truncated": { |
| "n": 159, |
| "accuracy": 0.8553459119496856, |
| "nll": 0.3588936702368793, |
| "brier": 0.2055985772370396, |
| "ece": 0.053331098466549334, |
| "mean_confidence": 0.8946507621861104 |
| }, |
| "synthetic/nli-haystack|not_truncated": { |
| "n": 154, |
| "accuracy": 0.8766233766233766, |
| "nll": 0.31448460003064777, |
| "brier": 0.1750220485890215, |
| "ece": 0.058161640709096796, |
| "mean_confidence": 0.8783779376512998 |
| } |
| }, |
| "by_kind_truncation": { |
| "choice|not_truncated": { |
| "n": 1225, |
| "accuracy": 0.8906122448979592, |
| "nll": 0.2910091881993051, |
| "brier": 0.16159634624408029, |
| "ece": 0.018170039288851744, |
| "mean_confidence": 0.8901425058257824 |
| }, |
| "noul|not_truncated": { |
| "n": 617, |
| "accuracy": 0.893030794165316, |
| "nll": 0.26631708821208017, |
| "brier": 0.1545579289745567, |
| "ece": 0.02302731426260657, |
| "mean_confidence": 0.886863877070008 |
| }, |
| "noul|truncated": { |
| "n": 1, |
| "accuracy": 1.0, |
| "nll": 0.2468656450510025, |
| "brier": 0.09570690989494324, |
| "ece": 0.2187543511390686, |
| "mean_confidence": 0.7812456488609314 |
| }, |
| "score|not_truncated": { |
| "n": 157, |
| "accuracy": 0.535031847133758, |
| "nll": 0.9802866589491535, |
| "brier": 0.5608620777678718, |
| "ece": 0.1083361166677657, |
| "mean_confidence": 0.5841244707821281 |
| } |
| }, |
| "nli_confusion": { |
| "n": 628, |
| "labels": [ |
| "entailment", |
| "neutral", |
| "contradiction" |
| ], |
| "matrix": { |
| "entailment": { |
| "entailment": 173, |
| "neutral": 23, |
| "contradiction": 3 |
| }, |
| "neutral": { |
| "entailment": 21, |
| "neutral": 203, |
| "contradiction": 15 |
| }, |
| "contradiction": { |
| "entailment": 2, |
| "neutral": 32, |
| "contradiction": 156 |
| } |
| } |
| } |
| }, |
| "schema": { |
| "n": 2000, |
| "accuracy": 0.758, |
| "nll": 0.5204962961420965, |
| "brier": 0.32018979337543463, |
| "ece": 0.07037245657294991, |
| "mean_confidence": 0.8196629058793187, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.097443699836731, |
| "groups": { |
| "choice:2": { |
| "temperature": 0.7726892229265028, |
| "raw_temperature": 0.4564978778362274, |
| "n": 80, |
| "shrinkage_weight": 0.4 |
| }, |
| "choice:3-5": { |
| "temperature": 1.0735645605398847, |
| "raw_temperature": 1.0707285404205322, |
| "n": 998, |
| "shrinkage_weight": 0.8926654740608229 |
| }, |
| "choice:6-10": { |
| "temperature": 1.0213380776161125, |
| "raw_temperature": 0.9631407260894775, |
| "n": 147, |
| "shrinkage_weight": 0.550561797752809 |
| }, |
| "noul:*": { |
| "temperature": 1.18076753369684, |
| "raw_temperature": 1.1976659297943115, |
| "n": 618, |
| "shrinkage_weight": 0.8373983739837398 |
| }, |
| "score:*": { |
| "temperature": 1.1118863993359898, |
| "raw_temperature": 1.1230534315109253, |
| "n": 157, |
| "shrinkage_weight": 0.5667870036101083 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "schema-firewall/PolyAI/banking77": { |
| "n": 46, |
| "accuracy": 0.8043478260869565, |
| "nll": 0.35612511238073435, |
| "brier": 0.22549919875508087, |
| "ece": 0.10526247128196384, |
| "mean_confidence": 0.8808949822964875 |
| }, |
| "schema-firewall/Rowan/hellaswag": { |
| "n": 130, |
| "accuracy": 0.9, |
| "nll": 0.20414897868758125, |
| "brier": 0.11925340885983747, |
| "ece": 0.05373616493665252, |
| "mean_confidence": 0.9159407689021184 |
| }, |
| "schema-firewall/alisawuffles/WANLI": { |
| "n": 209, |
| "accuracy": 0.7655502392344498, |
| "nll": 0.5446630402214148, |
| "brier": 0.32441898222513327, |
| "ece": 0.059890149027536935, |
| "mean_confidence": 0.7769627089135385 |
| }, |
| "schema-firewall/cais/mmlu": { |
| "n": 134, |
| "accuracy": 0.917910447761194, |
| "nll": 0.22831852445149217, |
| "brier": 0.11953794011554492, |
| "ece": 0.049322252620511986, |
| "mean_confidence": 0.909431281596867 |
| }, |
| "schema-firewall/google/boolq": { |
| "n": 36, |
| "accuracy": 0.8888888888888888, |
| "nll": 0.44322766041700057, |
| "brier": 0.22839449674166368, |
| "ece": 0.1561499618821674, |
| "mean_confidence": 0.8758690738015704 |
| }, |
| "schema-firewall/google/boolq-negation": { |
| "n": 60, |
| "accuracy": 0.5166666666666667, |
| "nll": 1.2234002001583577, |
| "brier": 0.7434715281813321, |
| "ece": 0.34772479931513467, |
| "mean_confidence": 0.8540057341257731 |
| }, |
| "schema-firewall/nvidia/HelpSteer2": { |
| "n": 270, |
| "accuracy": 0.6296296296296297, |
| "nll": 0.8983366615242429, |
| "brier": 0.496752599419819, |
| "ece": 0.07374763350795817, |
| "mean_confidence": 0.5930154732531971 |
| }, |
| "schema-firewall/nvidia/HelpSteer2-threshold": { |
| "n": 260, |
| "accuracy": 0.5230769230769231, |
| "nll": 0.8322646450712524, |
| "brier": 0.6063095804116249, |
| "ece": 0.2711401492357254, |
| "mean_confidence": 0.7942170723126485 |
| }, |
| "schema-firewall/nyu-mll/multi_nli": { |
| "n": 200, |
| "accuracy": 0.865, |
| "nll": 0.41195048167137427, |
| "brier": 0.21928038816042317, |
| "ece": 0.061470215469598743, |
| "mean_confidence": 0.8334396223723889 |
| }, |
| "schema-firewall/product-science/xlam-function-calling-60k-raw": { |
| "n": 81, |
| "accuracy": 1.0, |
| "nll": 0.0050883564066314295, |
| "brier": 0.001193973693776232, |
| "ece": 0.004606815032017043, |
| "mean_confidence": 0.9953931849679829 |
| }, |
| "schema-firewall/product-science/xlam-hard": { |
| "n": 211, |
| "accuracy": 0.6303317535545023, |
| "nll": 0.5777726572084735, |
| "brier": 0.400591680919437, |
| "ece": 0.21581395203468354, |
| "mean_confidence": 0.846145705589186 |
| }, |
| "schema-firewall/stanfordnlp/snli": { |
| "n": 208, |
| "accuracy": 0.8846153846153846, |
| "nll": 0.26919846923891994, |
| "brier": 0.15986069018861843, |
| "ece": 0.04227677503457438, |
| "mean_confidence": 0.8858483952398484 |
| }, |
| "schema-firewall/synthetic/nli-haystack": { |
| "n": 155, |
| "accuracy": 0.896774193548387, |
| "nll": 0.28797892007766474, |
| "brier": 0.15715603314421323, |
| "ece": 0.06688037726186936, |
| "mean_confidence": 0.8773749347656004 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 1163, |
| "accuracy": 0.8718830610490111, |
| "nll": 0.3188156900246091, |
| "brier": 0.1816513806989808, |
| "ece": 0.026625588907830587, |
| "mean_confidence": 0.8696533281128671 |
| }, |
| "noul": { |
| "n": 567, |
| "accuracy": 0.5855379188712522, |
| "nll": 0.754248758684351, |
| "brier": 0.5202748309605, |
| "ece": 0.23951466205469096, |
| "mean_confidence": 0.8250525809259431 |
| }, |
| "score": { |
| "n": 270, |
| "accuracy": 0.6296296296296297, |
| "nll": 0.8983366615242429, |
| "brier": 0.496752599419819, |
| "ece": 0.07374763350795817, |
| "mean_confidence": 0.5930154732531971 |
| } |
| }, |
| "by_transform": { |
| "schema.firewall.distractor_drop.v1": { |
| "n": 179, |
| "accuracy": 0.9162011173184358, |
| "nll": 0.17996553745532837, |
| "brier": 0.10203871893360637, |
| "ece": 0.041837152822057584, |
| "mean_confidence": 0.9357914165411582 |
| }, |
| "schema.firewall.instruction.v1": { |
| "n": 1395, |
| "accuracy": 0.8523297491039427, |
| "nll": 0.3699874161727043, |
| "brier": 0.21063317768835207, |
| "ece": 0.02584742985959546, |
| "mean_confidence": 0.8404050774151279 |
| }, |
| "schema.firewall.noul_inversion.v1": { |
| "n": 292, |
| "accuracy": 0.2808219178082192, |
| "nll": 1.247713702195005, |
| "brier": 0.8843806009206997, |
| "ece": 0.47954579167170064, |
| "mean_confidence": 0.7603677094799198 |
| }, |
| "schema.firewall.score_scale.v1": { |
| "n": 134, |
| "accuracy": 0.6044776119402985, |
| "nll": 0.9575665259539191, |
| "brier": 0.5227032665496887, |
| "ece": 0.07600586627846333, |
| "mean_confidence": 0.5778111495188812 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 594, |
| "accuracy": 0.6043771043771043, |
| "nll": 0.7199648861602912, |
| "brier": 0.4966259750298848, |
| "ece": 0.22862747000524092, |
| "mean_confidence": 0.8330045743823453 |
| }, |
| "3-5": { |
| "n": 1375, |
| "accuracy": 0.8218181818181818, |
| "nll": 0.44010872633975223, |
| "brier": 0.24742196818368095, |
| "ece": 0.032660349141467676, |
| "mean_confidence": 0.8119605057998137 |
| }, |
| "6-10": { |
| "n": 31, |
| "accuracy": 0.8709677419354839, |
| "nll": 0.263998425413569, |
| "brier": 0.16705004292117379, |
| "ece": 0.09422032871553974, |
| "mean_confidence": 0.905658035509048 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 1998, |
| "accuracy": 0.7587587587587588, |
| "nll": 0.5200072703881736, |
| "brier": 0.3198700578515258, |
| "ece": 0.0704587096193651, |
| "mean_confidence": 0.8200187474653289 |
| }, |
| "truncated": { |
| "n": 2, |
| "accuracy": 0.0, |
| "nll": 1.0090330243110657, |
| "brier": 0.6396055817604065, |
| "ece": 0.4641771614551544, |
| "mean_confidence": 0.4641771614551544 |
| } |
| }, |
| "by_dataset_truncation": { |
| "schema-firewall/PolyAI/banking77|not_truncated": { |
| "n": 46, |
| "accuracy": 0.8043478260869565, |
| "nll": 0.35612511238073435, |
| "brier": 0.22549919875508087, |
| "ece": 0.10526247128196384, |
| "mean_confidence": 0.8808949822964875 |
| }, |
| "schema-firewall/Rowan/hellaswag|not_truncated": { |
| "n": 130, |
| "accuracy": 0.9, |
| "nll": 0.20414897868758125, |
| "brier": 0.11925340885983747, |
| "ece": 0.05373616493665252, |
| "mean_confidence": 0.9159407689021184 |
| }, |
| "schema-firewall/alisawuffles/WANLI|not_truncated": { |
| "n": 209, |
| "accuracy": 0.7655502392344498, |
| "nll": 0.5446630402214148, |
| "brier": 0.32441898222513327, |
| "ece": 0.059890149027536935, |
| "mean_confidence": 0.7769627089135385 |
| }, |
| "schema-firewall/cais/mmlu|not_truncated": { |
| "n": 134, |
| "accuracy": 0.917910447761194, |
| "nll": 0.22831852445149217, |
| "brier": 0.11953794011554492, |
| "ece": 0.049322252620511986, |
| "mean_confidence": 0.909431281596867 |
| }, |
| "schema-firewall/google/boolq-negation|not_truncated": { |
| "n": 60, |
| "accuracy": 0.5166666666666667, |
| "nll": 1.2234002001583577, |
| "brier": 0.7434715281813321, |
| "ece": 0.34772479931513467, |
| "mean_confidence": 0.8540057341257731 |
| }, |
| "schema-firewall/google/boolq|not_truncated": { |
| "n": 36, |
| "accuracy": 0.8888888888888888, |
| "nll": 0.44322766041700057, |
| "brier": 0.22839449674166368, |
| "ece": 0.1561499618821674, |
| "mean_confidence": 0.8758690738015704 |
| }, |
| "schema-firewall/nvidia/HelpSteer2-threshold|not_truncated": { |
| "n": 260, |
| "accuracy": 0.5230769230769231, |
| "nll": 0.8322646450712524, |
| "brier": 0.6063095804116249, |
| "ece": 0.2711401492357254, |
| "mean_confidence": 0.7942170723126485 |
| }, |
| "schema-firewall/nvidia/HelpSteer2|not_truncated": { |
| "n": 268, |
| "accuracy": 0.6343283582089553, |
| "nll": 0.8975105692646397, |
| "brier": 0.49568653238742655, |
| "ece": 0.07441585716694149, |
| "mean_confidence": 0.5939769531919885 |
| }, |
| "schema-firewall/nvidia/HelpSteer2|truncated": { |
| "n": 2, |
| "accuracy": 0.0, |
| "nll": 1.0090330243110657, |
| "brier": 0.6396055817604065, |
| "ece": 0.4641771614551544, |
| "mean_confidence": 0.4641771614551544 |
| }, |
| "schema-firewall/nyu-mll/multi_nli|not_truncated": { |
| "n": 200, |
| "accuracy": 0.865, |
| "nll": 0.41195048167137427, |
| "brier": 0.21928038816042317, |
| "ece": 0.061470215469598743, |
| "mean_confidence": 0.8334396223723889 |
| }, |
| "schema-firewall/product-science/xlam-function-calling-60k-raw|not_truncated": { |
| "n": 81, |
| "accuracy": 1.0, |
| "nll": 0.0050883564066314295, |
| "brier": 0.001193973693776232, |
| "ece": 0.004606815032017043, |
| "mean_confidence": 0.9953931849679829 |
| }, |
| "schema-firewall/product-science/xlam-hard|not_truncated": { |
| "n": 211, |
| "accuracy": 0.6303317535545023, |
| "nll": 0.5777726572084735, |
| "brier": 0.400591680919437, |
| "ece": 0.21581395203468354, |
| "mean_confidence": 0.846145705589186 |
| }, |
| "schema-firewall/stanfordnlp/snli|not_truncated": { |
| "n": 208, |
| "accuracy": 0.8846153846153846, |
| "nll": 0.26919846923891994, |
| "brier": 0.15986069018861843, |
| "ece": 0.04227677503457438, |
| "mean_confidence": 0.8858483952398484 |
| }, |
| "schema-firewall/synthetic/nli-haystack|not_truncated": { |
| "n": 155, |
| "accuracy": 0.896774193548387, |
| "nll": 0.28797892007766474, |
| "brier": 0.15715603314421323, |
| "ece": 0.06688037726186936, |
| "mean_confidence": 0.8773749347656004 |
| } |
| }, |
| "by_kind_truncation": { |
| "choice|not_truncated": { |
| "n": 1163, |
| "accuracy": 0.8718830610490111, |
| "nll": 0.3188156900246091, |
| "brier": 0.1816513806989808, |
| "ece": 0.026625588907830587, |
| "mean_confidence": 0.8696533281128671 |
| }, |
| "noul|not_truncated": { |
| "n": 567, |
| "accuracy": 0.5855379188712522, |
| "nll": 0.754248758684351, |
| "brier": 0.5202748309605, |
| "ece": 0.23951466205469096, |
| "mean_confidence": 0.8250525809259431 |
| }, |
| "score|not_truncated": { |
| "n": 268, |
| "accuracy": 0.6343283582089553, |
| "nll": 0.8975105692646397, |
| "brier": 0.49568653238742655, |
| "ece": 0.07441585716694149, |
| "mean_confidence": 0.5939769531919885 |
| }, |
| "score|truncated": { |
| "n": 2, |
| "accuracy": 0.0, |
| "nll": 1.0090330243110657, |
| "brier": 0.6396055817604065, |
| "ece": 0.4641771614551544, |
| "mean_confidence": 0.4641771614551544 |
| } |
| }, |
| "nli_confusion": { |
| "n": 772, |
| "labels": [ |
| "entailment", |
| "neutral", |
| "contradiction" |
| ], |
| "matrix": { |
| "entailment": { |
| "entailment": 208, |
| "neutral": 26, |
| "contradiction": 3 |
| }, |
| "neutral": { |
| "entailment": 40, |
| "neutral": 259, |
| "contradiction": 19 |
| }, |
| "contradiction": { |
| "entailment": 4, |
| "neutral": 24, |
| "contradiction": 189 |
| } |
| } |
| } |
| }, |
| "firewall": { |
| "n": 2000, |
| "accuracy": 0.8725, |
| "nll": 0.34271046637929975, |
| "brier": 0.2024325927257114, |
| "ece": 0.021594500750303322, |
| "mean_confidence": 0.8701762627661228, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.097443699836731, |
| "groups": { |
| "choice:2": { |
| "temperature": 0.7726892229265028, |
| "raw_temperature": 0.4564978778362274, |
| "n": 80, |
| "shrinkage_weight": 0.4 |
| }, |
| "choice:3-5": { |
| "temperature": 1.0735645605398847, |
| "raw_temperature": 1.0707285404205322, |
| "n": 998, |
| "shrinkage_weight": 0.8926654740608229 |
| }, |
| "choice:6-10": { |
| "temperature": 1.0213380776161125, |
| "raw_temperature": 0.9631407260894775, |
| "n": 147, |
| "shrinkage_weight": 0.550561797752809 |
| }, |
| "noul:*": { |
| "temperature": 1.18076753369684, |
| "raw_temperature": 1.1976659297943115, |
| "n": 618, |
| "shrinkage_weight": 0.8373983739837398 |
| }, |
| "score:*": { |
| "temperature": 1.1118863993359898, |
| "raw_temperature": 1.1230534315109253, |
| "n": 157, |
| "shrinkage_weight": 0.5667870036101083 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "firewall/tommccoy1-hans": { |
| "n": 2000, |
| "accuracy": 0.8725, |
| "nll": 0.34271046637929975, |
| "brier": 0.2024325927257114, |
| "ece": 0.021594500750303322, |
| "mean_confidence": 0.8701762627661228 |
| } |
| }, |
| "by_kind": { |
| "noul": { |
| "n": 2000, |
| "accuracy": 0.8725, |
| "nll": 0.34271046637929975, |
| "brier": 0.2024325927257114, |
| "ece": 0.021594500750303322, |
| "mean_confidence": 0.8701762627661228 |
| } |
| }, |
| "by_transform": { |
| "nli.firewall.hans.v1": { |
| "n": 2000, |
| "accuracy": 0.8725, |
| "nll": 0.34271046637929975, |
| "brier": 0.2024325927257114, |
| "ece": 0.021594500750303322, |
| "mean_confidence": 0.8701762627661228 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 2000, |
| "accuracy": 0.8725, |
| "nll": 0.34271046637929975, |
| "brier": 0.2024325927257114, |
| "ece": 0.021594500750303322, |
| "mean_confidence": 0.8701762627661228 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 2000, |
| "accuracy": 0.8725, |
| "nll": 0.34271046637929975, |
| "brier": 0.2024325927257114, |
| "ece": 0.021594500750303322, |
| "mean_confidence": 0.8701762627661228 |
| } |
| }, |
| "by_dataset_truncation": { |
| "firewall/tommccoy1-hans|not_truncated": { |
| "n": 2000, |
| "accuracy": 0.8725, |
| "nll": 0.34271046637929975, |
| "brier": 0.2024325927257114, |
| "ece": 0.021594500750303322, |
| "mean_confidence": 0.8701762627661228 |
| } |
| }, |
| "by_kind_truncation": { |
| "noul|not_truncated": { |
| "n": 2000, |
| "accuracy": 0.8725, |
| "nll": 0.34271046637929975, |
| "brier": 0.2024325927257114, |
| "ece": 0.021594500750303322, |
| "mean_confidence": 0.8701762627661228 |
| } |
| } |
| }, |
| "confusability": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.00341335700372578, |
| "brier": 7.767048636123954e-05, |
| "ece": 0.003391873708793125, |
| "mean_confidence": 0.9966081262912069, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.097443699836731, |
| "groups": { |
| "choice:2": { |
| "temperature": 0.7726892229265028, |
| "raw_temperature": 0.4564978778362274, |
| "n": 80, |
| "shrinkage_weight": 0.4 |
| }, |
| "choice:3-5": { |
| "temperature": 1.0735645605398847, |
| "raw_temperature": 1.0707285404205322, |
| "n": 998, |
| "shrinkage_weight": 0.8926654740608229 |
| }, |
| "choice:6-10": { |
| "temperature": 1.0213380776161125, |
| "raw_temperature": 0.9631407260894775, |
| "n": 147, |
| "shrinkage_weight": 0.550561797752809 |
| }, |
| "noul:*": { |
| "temperature": 1.18076753369684, |
| "raw_temperature": 1.1976659297943115, |
| "n": 618, |
| "shrinkage_weight": 0.8373983739837398 |
| }, |
| "score:*": { |
| "temperature": 1.1118863993359898, |
| "raw_temperature": 1.1230534315109253, |
| "n": 157, |
| "shrinkage_weight": 0.5667870036101083 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "v142-confusability": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.00341335700372578, |
| "brier": 7.767048636123954e-05, |
| "ece": 0.003391873708793125, |
| "mean_confidence": 0.9966081262912069 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 1000, |
| "accuracy": 1.0, |
| "nll": 0.0030758743519090784, |
| "brier": 8.714231419299279e-05, |
| "ece": 0.0030530961751937813, |
| "mean_confidence": 0.9969469038248062 |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0011829274808405899, |
| "brier": 3.475759723983174e-06, |
| "ece": 0.0011820575594901905, |
| "mean_confidence": 0.9988179424405098 |
| }, |
| "score": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.00733119978569448, |
| "brier": 0.00010450607383972965, |
| "ece": 0.007295577526092556, |
| "mean_confidence": 0.9927044224739074 |
| } |
| }, |
| "by_transform": { |
| "coreference": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.005743390580028063, |
| "brier": 0.00021617284226891798, |
| "ece": 0.0056866601109504256, |
| "mean_confidence": 0.9943133398890496 |
| }, |
| "entity_binding": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.00016332407969457563, |
| "brier": 4.9414514606826996e-08, |
| "ece": 0.00016330808401110453, |
| "mean_confidence": 0.9998366919159889 |
| }, |
| "negation": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0011829274808405899, |
| "brier": 3.475759723983174e-06, |
| "ece": 0.0011820575594901905, |
| "mean_confidence": 0.9988179424405098 |
| }, |
| "numeric_boundary": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.00014066748185541654, |
| "brier": 6.167312968208272e-08, |
| "ece": 0.00014065206050872803, |
| "mean_confidence": 0.9998593479394913 |
| }, |
| "ordinal_boundary": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.00733119978569448, |
| "brier": 0.00010450607383972965, |
| "ece": 0.007295577526092556, |
| "mean_confidence": 0.9927044224739074 |
| }, |
| "temporal_state": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.007766418813844211, |
| "brier": 0.00015806605823172503, |
| "ece": 0.007725143730640371, |
| "mean_confidence": 0.9922748562693596 |
| }, |
| "tool_argument_validity": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0015655708041231264, |
| "brier": 6.1361582820032e-05, |
| "ece": 0.001549716889858277, |
| "mean_confidence": 0.9984502831101417 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 400, |
| "accuracy": 1.0, |
| "nll": 0.0006617974813480032, |
| "brier": 1.7687164268326283e-06, |
| "ece": 0.0006613548099995148, |
| "mean_confidence": 0.9993386451900005 |
| }, |
| "3-5": { |
| "n": 1000, |
| "accuracy": 1.0, |
| "nll": 0.004513980812676891, |
| "brier": 0.0001080311943350023, |
| "ece": 0.004484081268310569, |
| "mean_confidence": 0.9955159187316894 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.00341335700372578, |
| "brier": 7.767048636123954e-05, |
| "ece": 0.003391873708793125, |
| "mean_confidence": 0.9966081262912069 |
| } |
| }, |
| "by_dataset_truncation": { |
| "v142-confusability|not_truncated": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.00341335700372578, |
| "brier": 7.767048636123954e-05, |
| "ece": 0.003391873708793125, |
| "mean_confidence": 0.9966081262912069 |
| } |
| }, |
| "by_kind_truncation": { |
| "choice|not_truncated": { |
| "n": 1000, |
| "accuracy": 1.0, |
| "nll": 0.0030758743519090784, |
| "brier": 8.714231419299279e-05, |
| "ece": 0.0030530961751937813, |
| "mean_confidence": 0.9969469038248062 |
| }, |
| "noul|not_truncated": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0011829274808405899, |
| "brier": 3.475759723983174e-06, |
| "ece": 0.0011820575594901905, |
| "mean_confidence": 0.9988179424405098 |
| }, |
| "score|not_truncated": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.00733119978569448, |
| "brier": 0.00010450607383972965, |
| "ece": 0.007295577526092556, |
| "mean_confidence": 0.9927044224739074 |
| } |
| } |
| } |
| }, |
| "latency": { |
| "validation": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 3647.106461352203, |
| "effective_ms_per_example": 455.8883076690254, |
| "skipped_unusable": 0 |
| }, |
| "schema": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 4148.647082126699, |
| "effective_ms_per_example": 518.5808852658374, |
| "skipped_unusable": 0 |
| }, |
| "firewall": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 335.46002532273997, |
| "effective_ms_per_example": 41.932503165342496, |
| "skipped_unusable": 0 |
| }, |
| "confusability": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 658.9870701618826, |
| "effective_ms_per_example": 82.37338377023532, |
| "skipped_unusable": 0 |
| } |
| } |
| }, |
| "head_rlcd": { |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.0621812343597412, |
| "groups": { |
| "choice:2": { |
| "temperature": 0.7285364649335917, |
| "raw_temperature": 0.3852079510688782, |
| "n": 71, |
| "shrinkage_weight": 0.3717277486910995 |
| }, |
| "choice:3-5": { |
| "temperature": 1.0312828950285642, |
| "raw_temperature": 1.0276613235473633, |
| "n": 1007, |
| "shrinkage_weight": 0.8935226264418811 |
| }, |
| "choice:6-10": { |
| "temperature": 1.018117407640952, |
| "raw_temperature": 0.9839622974395752, |
| "n": 149, |
| "shrinkage_weight": 0.5539033457249071 |
| }, |
| "noul:*": { |
| "temperature": 1.12278263121421, |
| "raw_temperature": 1.1349645853042603, |
| "n": 617, |
| "shrinkage_weight": 0.8371777476255088 |
| }, |
| "score:*": { |
| "temperature": 1.1063119547040072, |
| "raw_temperature": 1.1415023803710938, |
| "n": 156, |
| "shrinkage_weight": 0.5652173913043478 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "sets": { |
| "validation": { |
| "n": 2000, |
| "accuracy": 0.8705, |
| "nll": 0.3296415164664375, |
| "brier": 0.1840096441006748, |
| "ece": 0.015826731696724952, |
| "mean_confidence": 0.8667946958690882, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.0621812343597412, |
| "groups": { |
| "choice:2": { |
| "temperature": 0.7285364649335917, |
| "raw_temperature": 0.3852079510688782, |
| "n": 71, |
| "shrinkage_weight": 0.3717277486910995 |
| }, |
| "choice:3-5": { |
| "temperature": 1.0312828950285642, |
| "raw_temperature": 1.0276613235473633, |
| "n": 1007, |
| "shrinkage_weight": 0.8935226264418811 |
| }, |
| "choice:6-10": { |
| "temperature": 1.018117407640952, |
| "raw_temperature": 0.9839622974395752, |
| "n": 149, |
| "shrinkage_weight": 0.5539033457249071 |
| }, |
| "noul:*": { |
| "temperature": 1.12278263121421, |
| "raw_temperature": 1.1349645853042603, |
| "n": 617, |
| "shrinkage_weight": 0.8371777476255088 |
| }, |
| "score:*": { |
| "temperature": 1.1063119547040072, |
| "raw_temperature": 1.1415023803710938, |
| "n": 156, |
| "shrinkage_weight": 0.5652173913043478 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "PolyAI/banking77": { |
| "n": 136, |
| "accuracy": 0.9338235294117647, |
| "nll": 0.18870303818857792, |
| "brier": 0.10519016720641593, |
| "ece": 0.05590041091336926, |
| "mean_confidence": 0.9336026931071982 |
| }, |
| "Rowan/hellaswag": { |
| "n": 153, |
| "accuracy": 0.9215686274509803, |
| "nll": 0.2399784766869879, |
| "brier": 0.11934398814653138, |
| "ece": 0.040469242272034245, |
| "mean_confidence": 0.9290768474924798 |
| }, |
| "alisawuffles/WANLI": { |
| "n": 160, |
| "accuracy": 0.80625, |
| "nll": 0.5381927386275492, |
| "brier": 0.30977863866601185, |
| "ece": 0.06804261095821856, |
| "mean_confidence": 0.7853333316743374 |
| }, |
| "cais/mmlu": { |
| "n": 154, |
| "accuracy": 0.8831168831168831, |
| "nll": 0.3485408564913748, |
| "brier": 0.17917580716248588, |
| "ece": 0.06689213796869503, |
| "mean_confidence": 0.8730696820980542 |
| }, |
| "google/boolq": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.33914438616278964, |
| "brier": 0.19378073975049515, |
| "ece": 0.04670275738036708, |
| "mean_confidence": 0.8901991384481293 |
| }, |
| "google/boolq-negation": { |
| "n": 153, |
| "accuracy": 0.8627450980392157, |
| "nll": 0.29727962399145064, |
| "brier": 0.17111000223924158, |
| "ece": 0.06959474671120737, |
| "mean_confidence": 0.8627709676237667 |
| }, |
| "nvidia/HelpSteer2": { |
| "n": 156, |
| "accuracy": 0.5448717948717948, |
| "nll": 1.00863150984813, |
| "brier": 0.549988131922407, |
| "ece": 0.07374643553525977, |
| "mean_confidence": 0.5746784233129941 |
| }, |
| "nvidia/HelpSteer2-threshold": { |
| "n": 157, |
| "accuracy": 0.8853503184713376, |
| "nll": 0.31764734924213284, |
| "brier": 0.1801272323022331, |
| "ece": 0.05705484463151096, |
| "mean_confidence": 0.8431880648728389 |
| }, |
| "nyu-mll/multi_nli": { |
| "n": 157, |
| "accuracy": 0.8853503184713376, |
| "nll": 0.2977778198298328, |
| "brier": 0.17204729962497461, |
| "ece": 0.0343798731163049, |
| "mean_confidence": 0.8596658488367773 |
| }, |
| "product-science/xlam-function-calling-60k-raw": { |
| "n": 153, |
| "accuracy": 1.0, |
| "nll": 0.0033377126531411883, |
| "brier": 0.001834733447133328, |
| "ece": 0.002718002188439416, |
| "mean_confidence": 0.9972819978115606 |
| }, |
| "product-science/xlam-hard": { |
| "n": 154, |
| "accuracy": 0.9935064935064936, |
| "nll": 0.0357667315122369, |
| "brier": 0.018634809771057987, |
| "ece": 0.020999968051910404, |
| "mean_confidence": 0.9805593196447794 |
| }, |
| "stanfordnlp/snli": { |
| "n": 159, |
| "accuracy": 0.8805031446540881, |
| "nll": 0.3138472866193749, |
| "brier": 0.1782226289040226, |
| "ece": 0.04512691535289933, |
| "mean_confidence": 0.8852375029767834 |
| }, |
| "synthetic/nli-haystack": { |
| "n": 155, |
| "accuracy": 0.864516129032258, |
| "nll": 0.32164501180992494, |
| "brier": 0.19284149839020814, |
| "ece": 0.02968011844542717, |
| "mean_confidence": 0.8698015168789894 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 1227, |
| "accuracy": 0.895680521597392, |
| "nll": 0.28458429776615213, |
| "brier": 0.15912234712374151, |
| "ece": 0.016193954776532293, |
| "mean_confidence": 0.8902595518027852 |
| }, |
| "noul": { |
| "n": 617, |
| "accuracy": 0.9027552674230146, |
| "nll": 0.24757144900729028, |
| "brier": 0.14096923776438128, |
| "ece": 0.031755420241981885, |
| "mean_confidence": 0.8939887968222557 |
| }, |
| "score": { |
| "n": 156, |
| "accuracy": 0.5448717948717948, |
| "nll": 1.00863150984813, |
| "brier": 0.549988131922407, |
| "ece": 0.07374643553525977, |
| "mean_confidence": 0.5746784233129941 |
| } |
| }, |
| "by_transform": { |
| "agent.invocation.correct.v1": { |
| "n": 50, |
| "accuracy": 1.0, |
| "nll": 0.04205997154116631, |
| "brier": 0.01677228283144359, |
| "ece": 0.03617538213729857, |
| "mean_confidence": 0.9638246178627015 |
| }, |
| "agent.invocation.corrupt_argument.v1": { |
| "n": 61, |
| "accuracy": 0.9836065573770492, |
| "nll": 0.041105691575422335, |
| "brier": 0.025920638532433198, |
| "ece": 0.02006097699775066, |
| "mean_confidence": 0.9925722235538921 |
| }, |
| "agent.invocation.wrong_tool.v1": { |
| "n": 43, |
| "accuracy": 1.0, |
| "nll": 0.020875136970358272, |
| "brier": 0.010464828202147104, |
| "ece": 0.017023286154103805, |
| "mean_confidence": 0.9829767138458961 |
| }, |
| "commonsense.choice.v2": { |
| "n": 153, |
| "accuracy": 0.9215686274509803, |
| "nll": 0.2399784766869879, |
| "brier": 0.11934398814653138, |
| "ece": 0.040469242272034245, |
| "mean_confidence": 0.9290768474924798 |
| }, |
| "intent.choice.v3": { |
| "n": 136, |
| "accuracy": 0.9338235294117647, |
| "nll": 0.18870303818857792, |
| "brier": 0.10519016720641593, |
| "ece": 0.05590041091336926, |
| "mean_confidence": 0.9336026931071982 |
| }, |
| "knowledge.choice.v1": { |
| "n": 154, |
| "accuracy": 0.8831168831168831, |
| "nll": 0.3485408564913748, |
| "brier": 0.17917580716248588, |
| "ece": 0.06689213796869503, |
| "mean_confidence": 0.8730696820980542 |
| }, |
| "nli.choice.v4": { |
| "n": 157, |
| "accuracy": 0.8853503184713376, |
| "nll": 0.2977778198298328, |
| "brier": 0.17204729962497461, |
| "ece": 0.0343798731163049, |
| "mean_confidence": 0.8596658488367773 |
| }, |
| "nli.hard.v4": { |
| "n": 160, |
| "accuracy": 0.80625, |
| "nll": 0.5381927386275492, |
| "brier": 0.30977863866601185, |
| "ece": 0.06804261095821856, |
| "mean_confidence": 0.7853333316743374 |
| }, |
| "nli.haystack.absent.v1": { |
| "n": 13, |
| "accuracy": 0.9230769230769231, |
| "nll": 0.293056732795846, |
| "brier": 0.1554580526787434, |
| "ece": 0.09880676177831796, |
| "mean_confidence": 0.9119220834511977 |
| }, |
| "nli.haystack.present.v1": { |
| "n": 142, |
| "accuracy": 0.8591549295774648, |
| "nll": 0.324262248621073, |
| "brier": 0.19626392651872251, |
| "ece": 0.028329969921582165, |
| "mean_confidence": 0.8659454086716746 |
| }, |
| "nli.snli.v1": { |
| "n": 159, |
| "accuracy": 0.8805031446540881, |
| "nll": 0.3138472866193749, |
| "brier": 0.1782226289040226, |
| "ece": 0.04512691535289933, |
| "mean_confidence": 0.8852375029767834 |
| }, |
| "noul.negated_answer.v1": { |
| "n": 153, |
| "accuracy": 0.8627450980392157, |
| "nll": 0.29727962399145064, |
| "brier": 0.17111000223924158, |
| "ece": 0.06959474671120737, |
| "mean_confidence": 0.8627709676237667 |
| }, |
| "noul.v3": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.33914438616278964, |
| "brier": 0.19378073975049515, |
| "ece": 0.04670275738036708, |
| "mean_confidence": 0.8901991384481293 |
| }, |
| "score.coherence.threshold.v1": { |
| "n": 29, |
| "accuracy": 0.9655172413793104, |
| "nll": 0.19833977429742186, |
| "brier": 0.10644655702463783, |
| "ece": 0.1178100828466744, |
| "mean_confidence": 0.8477071585326359 |
| }, |
| "score.coherence.v3": { |
| "n": 27, |
| "accuracy": 0.8148148148148148, |
| "nll": 0.4967265730654752, |
| "brier": 0.2842161387757019, |
| "ece": 0.099791513548957, |
| "mean_confidence": 0.7278836170832316 |
| }, |
| "score.complexity.threshold.v1": { |
| "n": 36, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.32385558721661156, |
| "brier": 0.17315642751181157, |
| "ece": 0.1148082067569097, |
| "mean_confidence": 0.8971044040388532 |
| }, |
| "score.complexity.v3": { |
| "n": 40, |
| "accuracy": 0.6, |
| "nll": 0.9197511970996857, |
| "brier": 0.5391126235947012, |
| "ece": 0.10441315844655036, |
| "mean_confidence": 0.6230462364852428 |
| }, |
| "score.correctness.threshold.v1": { |
| "n": 29, |
| "accuracy": 0.7931034482758621, |
| "nll": 0.342922273381003, |
| "brier": 0.21508417756232465, |
| "ece": 0.14295518809351426, |
| "mean_confidence": 0.7817615517254534 |
| }, |
| "score.correctness.v3": { |
| "n": 37, |
| "accuracy": 0.35135135135135137, |
| "nll": 1.3171428655450408, |
| "brier": 0.7058205044752842, |
| "ece": 0.20661347017094894, |
| "mean_confidence": 0.45046308274204666 |
| }, |
| "score.helpfulness.threshold.v1": { |
| "n": 25, |
| "accuracy": 0.8, |
| "nll": 0.44108421333134173, |
| "brier": 0.28551679542288183, |
| "ece": 0.13267860651016236, |
| "mean_confidence": 0.7273315024375916 |
| }, |
| "score.helpfulness.v3": { |
| "n": 28, |
| "accuracy": 0.42857142857142855, |
| "nll": 1.2584992029837199, |
| "brier": 0.6579760227884565, |
| "ece": 0.2361379388187613, |
| "mean_confidence": 0.43264825482453617 |
| }, |
| "score.verbosity.threshold.v1": { |
| "n": 38, |
| "accuracy": 0.9210526315789473, |
| "nll": 0.30231915724365727, |
| "brier": 0.14694823401135326, |
| "ece": 0.07115527046354195, |
| "mean_confidence": 0.9117601968740162 |
| }, |
| "score.verbosity.v3": { |
| "n": 24, |
| "accuracy": 0.5833333333333334, |
| "nll": 0.9655244362850984, |
| "brier": 0.5008800243958831, |
| "ece": 0.10565355420112608, |
| "mean_confidence": 0.6789100716511408 |
| }, |
| "tool.choice.single.v3": { |
| "n": 153, |
| "accuracy": 1.0, |
| "nll": 0.0033377126531411883, |
| "brier": 0.001834733447133328, |
| "ece": 0.002718002188439416, |
| "mean_confidence": 0.9972819978115606 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 688, |
| "accuracy": 0.9127906976744186, |
| "nll": 0.22202356312992652, |
| "brier": 0.1264215409403835, |
| "ece": 0.02847742297968196, |
| "mean_confidence": 0.9049279947093752 |
| }, |
| "3-5": { |
| "n": 1163, |
| "accuracy": 0.8366294067067928, |
| "nll": 0.4134522701136551, |
| "brier": 0.22935059510468528, |
| "ece": 0.013319709643350023, |
| "mean_confidence": 0.8349535806058095 |
| }, |
| "6-10": { |
| "n": 149, |
| "accuracy": 0.9395973154362416, |
| "nll": 0.17238812991479716, |
| "brier": 0.09601695253434102, |
| "ece": 0.05117117458541918, |
| "mean_confidence": 0.9392477659971122 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 1999, |
| "accuracy": 0.8704352176088044, |
| "nll": 0.3295081802774739, |
| "brier": 0.18389991385929264, |
| "ece": 0.015609992585103528, |
| "mean_confidence": 0.8669527163351697 |
| }, |
| "truncated": { |
| "n": 1, |
| "accuracy": 1.0, |
| "nll": 0.5961805582046509, |
| "brier": 0.40336039662361145, |
| "ece": 0.4490882158279419, |
| "mean_confidence": 0.5509117841720581 |
| } |
| }, |
| "by_dataset_truncation": { |
| "PolyAI/banking77|not_truncated": { |
| "n": 136, |
| "accuracy": 0.9338235294117647, |
| "nll": 0.18870303818857792, |
| "brier": 0.10519016720641593, |
| "ece": 0.05590041091336926, |
| "mean_confidence": 0.9336026931071982 |
| }, |
| "Rowan/hellaswag|not_truncated": { |
| "n": 153, |
| "accuracy": 0.9215686274509803, |
| "nll": 0.2399784766869879, |
| "brier": 0.11934398814653138, |
| "ece": 0.040469242272034245, |
| "mean_confidence": 0.9290768474924798 |
| }, |
| "alisawuffles/WANLI|not_truncated": { |
| "n": 160, |
| "accuracy": 0.80625, |
| "nll": 0.5381927386275492, |
| "brier": 0.30977863866601185, |
| "ece": 0.06804261095821856, |
| "mean_confidence": 0.7853333316743374 |
| }, |
| "cais/mmlu|not_truncated": { |
| "n": 154, |
| "accuracy": 0.8831168831168831, |
| "nll": 0.3485408564913748, |
| "brier": 0.17917580716248588, |
| "ece": 0.06689213796869503, |
| "mean_confidence": 0.8730696820980542 |
| }, |
| "google/boolq-negation|not_truncated": { |
| "n": 153, |
| "accuracy": 0.8627450980392157, |
| "nll": 0.29727962399145064, |
| "brier": 0.17111000223924158, |
| "ece": 0.06959474671120737, |
| "mean_confidence": 0.8627709676237667 |
| }, |
| "google/boolq|not_truncated": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.33914438616278964, |
| "brier": 0.19378073975049515, |
| "ece": 0.04670275738036708, |
| "mean_confidence": 0.8901991384481293 |
| }, |
| "nvidia/HelpSteer2-threshold|not_truncated": { |
| "n": 156, |
| "accuracy": 0.8846153846153846, |
| "nll": 0.3158618799539116, |
| "brier": 0.17869625047966017, |
| "ece": 0.0545418102007646, |
| "mean_confidence": 0.8450616307747669 |
| }, |
| "nvidia/HelpSteer2-threshold|truncated": { |
| "n": 1, |
| "accuracy": 1.0, |
| "nll": 0.5961805582046509, |
| "brier": 0.40336039662361145, |
| "ece": 0.4490882158279419, |
| "mean_confidence": 0.5509117841720581 |
| }, |
| "nvidia/HelpSteer2|not_truncated": { |
| "n": 156, |
| "accuracy": 0.5448717948717948, |
| "nll": 1.00863150984813, |
| "brier": 0.549988131922407, |
| "ece": 0.07374643553525977, |
| "mean_confidence": 0.5746784233129941 |
| }, |
| "nyu-mll/multi_nli|not_truncated": { |
| "n": 157, |
| "accuracy": 0.8853503184713376, |
| "nll": 0.2977778198298328, |
| "brier": 0.17204729962497461, |
| "ece": 0.0343798731163049, |
| "mean_confidence": 0.8596658488367773 |
| }, |
| "product-science/xlam-function-calling-60k-raw|not_truncated": { |
| "n": 153, |
| "accuracy": 1.0, |
| "nll": 0.0033377126531411883, |
| "brier": 0.001834733447133328, |
| "ece": 0.002718002188439416, |
| "mean_confidence": 0.9972819978115606 |
| }, |
| "product-science/xlam-hard|not_truncated": { |
| "n": 154, |
| "accuracy": 0.9935064935064936, |
| "nll": 0.0357667315122369, |
| "brier": 0.018634809771057987, |
| "ece": 0.020999968051910404, |
| "mean_confidence": 0.9805593196447794 |
| }, |
| "stanfordnlp/snli|not_truncated": { |
| "n": 159, |
| "accuracy": 0.8805031446540881, |
| "nll": 0.3138472866193749, |
| "brier": 0.1782226289040226, |
| "ece": 0.04512691535289933, |
| "mean_confidence": 0.8852375029767834 |
| }, |
| "synthetic/nli-haystack|not_truncated": { |
| "n": 155, |
| "accuracy": 0.864516129032258, |
| "nll": 0.32164501180992494, |
| "brier": 0.19284149839020814, |
| "ece": 0.02968011844542717, |
| "mean_confidence": 0.8698015168789894 |
| } |
| }, |
| "by_kind_truncation": { |
| "choice|not_truncated": { |
| "n": 1227, |
| "accuracy": 0.895680521597392, |
| "nll": 0.28458429776615213, |
| "brier": 0.15912234712374151, |
| "ece": 0.016193954776532293, |
| "mean_confidence": 0.8902595518027852 |
| }, |
| "noul|not_truncated": { |
| "n": 616, |
| "accuracy": 0.9025974025974026, |
| "nll": 0.24700552512872315, |
| "brier": 0.1405432780909085, |
| "ece": 0.031077931937459235, |
| "mean_confidence": 0.89454574002461 |
| }, |
| "noul|truncated": { |
| "n": 1, |
| "accuracy": 1.0, |
| "nll": 0.5961805582046509, |
| "brier": 0.40336039662361145, |
| "ece": 0.4490882158279419, |
| "mean_confidence": 0.5509117841720581 |
| }, |
| "score|not_truncated": { |
| "n": 156, |
| "accuracy": 0.5448717948717948, |
| "nll": 1.00863150984813, |
| "brier": 0.549988131922407, |
| "ece": 0.07374643553525977, |
| "mean_confidence": 0.5746784233129941 |
| } |
| }, |
| "nli_confusion": { |
| "n": 631, |
| "labels": [ |
| "entailment", |
| "neutral", |
| "contradiction" |
| ], |
| "matrix": { |
| "entailment": { |
| "entailment": 187, |
| "neutral": 24, |
| "contradiction": 2 |
| }, |
| "neutral": { |
| "entailment": 21, |
| "neutral": 214, |
| "contradiction": 20 |
| }, |
| "contradiction": { |
| "entailment": 1, |
| "neutral": 21, |
| "contradiction": 141 |
| } |
| } |
| } |
| }, |
| "schema": { |
| "n": 2000, |
| "accuracy": 0.7645, |
| "nll": 0.5092308543568372, |
| "brier": 0.31048622908528783, |
| "ece": 0.06814040819555524, |
| "mean_confidence": 0.8320965484306216, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.0621812343597412, |
| "groups": { |
| "choice:2": { |
| "temperature": 0.7285364649335917, |
| "raw_temperature": 0.3852079510688782, |
| "n": 71, |
| "shrinkage_weight": 0.3717277486910995 |
| }, |
| "choice:3-5": { |
| "temperature": 1.0312828950285642, |
| "raw_temperature": 1.0276613235473633, |
| "n": 1007, |
| "shrinkage_weight": 0.8935226264418811 |
| }, |
| "choice:6-10": { |
| "temperature": 1.018117407640952, |
| "raw_temperature": 0.9839622974395752, |
| "n": 149, |
| "shrinkage_weight": 0.5539033457249071 |
| }, |
| "noul:*": { |
| "temperature": 1.12278263121421, |
| "raw_temperature": 1.1349645853042603, |
| "n": 617, |
| "shrinkage_weight": 0.8371777476255088 |
| }, |
| "score:*": { |
| "temperature": 1.1063119547040072, |
| "raw_temperature": 1.1415023803710938, |
| "n": 156, |
| "shrinkage_weight": 0.5652173913043478 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "schema-firewall/PolyAI/banking77": { |
| "n": 46, |
| "accuracy": 0.8043478260869565, |
| "nll": 0.35900251572955155, |
| "brier": 0.2278477099184452, |
| "ece": 0.11447685244290724, |
| "mean_confidence": 0.8849650835213454 |
| }, |
| "schema-firewall/Rowan/hellaswag": { |
| "n": 130, |
| "accuracy": 0.9076923076923077, |
| "nll": 0.20391861355223787, |
| "brier": 0.12032575284317323, |
| "ece": 0.05504847719119148, |
| "mean_confidence": 0.9189063984614152 |
| }, |
| "schema-firewall/alisawuffles/WANLI": { |
| "n": 214, |
| "accuracy": 0.7570093457943925, |
| "nll": 0.5224599649424656, |
| "brier": 0.3108746913783881, |
| "ece": 0.08684171750166704, |
| "mean_confidence": 0.7917580997275415 |
| }, |
| "schema-firewall/cais/mmlu": { |
| "n": 134, |
| "accuracy": 0.917910447761194, |
| "nll": 0.2270736379013853, |
| "brier": 0.11838710439602669, |
| "ece": 0.05173284882929787, |
| "mean_confidence": 0.9138704791887483 |
| }, |
| "schema-firewall/google/boolq": { |
| "n": 36, |
| "accuracy": 0.8888888888888888, |
| "nll": 0.45656878421868896, |
| "brier": 0.2262111558433004, |
| "ece": 0.13308524257606935, |
| "mean_confidence": 0.8821887258026335 |
| }, |
| "schema-firewall/google/boolq-negation": { |
| "n": 60, |
| "accuracy": 0.5166666666666667, |
| "nll": 1.2670575649982008, |
| "brier": 0.7512862990629401, |
| "ece": 0.35960156718889874, |
| "mean_confidence": 0.860178780555725 |
| }, |
| "schema-firewall/nvidia/HelpSteer2": { |
| "n": 257, |
| "accuracy": 0.6108949416342413, |
| "nll": 0.8855918734569957, |
| "brier": 0.49536697754948056, |
| "ece": 0.09132106381863471, |
| "mean_confidence": 0.5927287479898808 |
| }, |
| "schema-firewall/nvidia/HelpSteer2-threshold": { |
| "n": 254, |
| "accuracy": 0.5433070866141733, |
| "nll": 0.8280847921860258, |
| "brier": 0.595998754226859, |
| "ece": 0.26783039860838037, |
| "mean_confidence": 0.8042075009796563 |
| }, |
| "schema-firewall/nyu-mll/multi_nli": { |
| "n": 216, |
| "accuracy": 0.8703703703703703, |
| "nll": 0.35441261314122974, |
| "brier": 0.1908318395181865, |
| "ece": 0.05234303628956832, |
| "mean_confidence": 0.8628430830107795 |
| }, |
| "schema-firewall/product-science/xlam-function-calling-60k-raw": { |
| "n": 81, |
| "accuracy": 1.0, |
| "nll": 0.004432377281443713, |
| "brier": 0.0009832953881987287, |
| "ece": 0.004043651215824124, |
| "mean_confidence": 0.9959563487841759 |
| }, |
| "schema-firewall/product-science/xlam-hard": { |
| "n": 207, |
| "accuracy": 0.6521739130434783, |
| "nll": 0.5901507024798354, |
| "brier": 0.3917848867662268, |
| "ece": 0.21952186143340696, |
| "mean_confidence": 0.8716957744768852 |
| }, |
| "schema-firewall/stanfordnlp/snli": { |
| "n": 212, |
| "accuracy": 0.8915094339622641, |
| "nll": 0.28470200158541425, |
| "brier": 0.16288797511131478, |
| "ece": 0.05838072018803289, |
| "mean_confidence": 0.8921123490018664 |
| }, |
| "schema-firewall/synthetic/nli-haystack": { |
| "n": 153, |
| "accuracy": 0.9019607843137255, |
| "nll": 0.28354985410883243, |
| "brier": 0.15433488102038004, |
| "ece": 0.06596269872453478, |
| "mean_confidence": 0.8859285342147927 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 1186, |
| "accuracy": 0.8735244519392917, |
| "nll": 0.3085243966904598, |
| "brier": 0.17534503393928574, |
| "ece": 0.025424960818499803, |
| "mean_confidence": 0.8800864731029153 |
| }, |
| "noul": { |
| "n": 557, |
| "accuracy": 0.6032315978456014, |
| "nll": 0.762934762585891, |
| "brier": 0.5129334554548768, |
| "ece": 0.23712612482546905, |
| "mean_confidence": 0.8403577226710705 |
| }, |
| "score": { |
| "n": 257, |
| "accuracy": 0.6108949416342413, |
| "nll": 0.8855918734569957, |
| "brier": 0.49536697754948056, |
| "ece": 0.09132106381863471, |
| "mean_confidence": 0.5927287479898808 |
| } |
| }, |
| "by_transform": { |
| "schema.firewall.distractor_drop.v1": { |
| "n": 179, |
| "accuracy": 0.9162011173184358, |
| "nll": 0.18095475863384314, |
| "brier": 0.10300702612023682, |
| "ece": 0.04983094084862223, |
| "mean_confidence": 0.9392977393539258 |
| }, |
| "schema.firewall.instruction.v1": { |
| "n": 1419, |
| "accuracy": 0.8611698379140239, |
| "nll": 0.34634081345778583, |
| "brier": 0.19550318506250666, |
| "ece": 0.02325636315421375, |
| "mean_confidence": 0.853838113273761 |
| }, |
| "schema.firewall.noul_inversion.v1": { |
| "n": 276, |
| "accuracy": 0.2572463768115942, |
| "nll": 1.357916534608365, |
| "brier": 0.9378885064367711, |
| "ece": 0.5137078725341437, |
| "mean_confidence": 0.770954249345738 |
| }, |
| "schema.firewall.score_scale.v1": { |
| "n": 126, |
| "accuracy": 0.5714285714285714, |
| "nll": 0.9510176910294427, |
| "brier": 0.5258567707524413, |
| "ece": 0.10069926556140658, |
| "mean_confidence": 0.568882110809523 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 584, |
| "accuracy": 0.6215753424657534, |
| "nll": 0.7276621770525656, |
| "brier": 0.48921906625380607, |
| "ece": 0.2261630186479385, |
| "mean_confidence": 0.8477383611136919 |
| }, |
| "3-5": { |
| "n": 1385, |
| "accuracy": 0.8223826714801444, |
| "nll": 0.42260318807209163, |
| "brier": 0.23831324855760574, |
| "ece": 0.029846963028184778, |
| "mean_confidence": 0.8238253159320742 |
| }, |
| "6-10": { |
| "n": 31, |
| "accuracy": 0.8709677419354839, |
| "nll": 0.2645671559719103, |
| "brier": 0.16789271696997024, |
| "ece": 0.09428581979966932, |
| "mean_confidence": 0.9069623033846578 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 2000, |
| "accuracy": 0.7645, |
| "nll": 0.5092308543568372, |
| "brier": 0.31048622908528783, |
| "ece": 0.06814040819555524, |
| "mean_confidence": 0.8320965484306216 |
| } |
| }, |
| "by_dataset_truncation": { |
| "schema-firewall/PolyAI/banking77|not_truncated": { |
| "n": 46, |
| "accuracy": 0.8043478260869565, |
| "nll": 0.35900251572955155, |
| "brier": 0.2278477099184452, |
| "ece": 0.11447685244290724, |
| "mean_confidence": 0.8849650835213454 |
| }, |
| "schema-firewall/Rowan/hellaswag|not_truncated": { |
| "n": 130, |
| "accuracy": 0.9076923076923077, |
| "nll": 0.20391861355223787, |
| "brier": 0.12032575284317323, |
| "ece": 0.05504847719119148, |
| "mean_confidence": 0.9189063984614152 |
| }, |
| "schema-firewall/alisawuffles/WANLI|not_truncated": { |
| "n": 214, |
| "accuracy": 0.7570093457943925, |
| "nll": 0.5224599649424656, |
| "brier": 0.3108746913783881, |
| "ece": 0.08684171750166704, |
| "mean_confidence": 0.7917580997275415 |
| }, |
| "schema-firewall/cais/mmlu|not_truncated": { |
| "n": 134, |
| "accuracy": 0.917910447761194, |
| "nll": 0.2270736379013853, |
| "brier": 0.11838710439602669, |
| "ece": 0.05173284882929787, |
| "mean_confidence": 0.9138704791887483 |
| }, |
| "schema-firewall/google/boolq-negation|not_truncated": { |
| "n": 60, |
| "accuracy": 0.5166666666666667, |
| "nll": 1.2670575649982008, |
| "brier": 0.7512862990629401, |
| "ece": 0.35960156718889874, |
| "mean_confidence": 0.860178780555725 |
| }, |
| "schema-firewall/google/boolq|not_truncated": { |
| "n": 36, |
| "accuracy": 0.8888888888888888, |
| "nll": 0.45656878421868896, |
| "brier": 0.2262111558433004, |
| "ece": 0.13308524257606935, |
| "mean_confidence": 0.8821887258026335 |
| }, |
| "schema-firewall/nvidia/HelpSteer2-threshold|not_truncated": { |
| "n": 254, |
| "accuracy": 0.5433070866141733, |
| "nll": 0.8280847921860258, |
| "brier": 0.595998754226859, |
| "ece": 0.26783039860838037, |
| "mean_confidence": 0.8042075009796563 |
| }, |
| "schema-firewall/nvidia/HelpSteer2|not_truncated": { |
| "n": 257, |
| "accuracy": 0.6108949416342413, |
| "nll": 0.8855918734569957, |
| "brier": 0.49536697754948056, |
| "ece": 0.09132106381863471, |
| "mean_confidence": 0.5927287479898808 |
| }, |
| "schema-firewall/nyu-mll/multi_nli|not_truncated": { |
| "n": 216, |
| "accuracy": 0.8703703703703703, |
| "nll": 0.35441261314122974, |
| "brier": 0.1908318395181865, |
| "ece": 0.05234303628956832, |
| "mean_confidence": 0.8628430830107795 |
| }, |
| "schema-firewall/product-science/xlam-function-calling-60k-raw|not_truncated": { |
| "n": 81, |
| "accuracy": 1.0, |
| "nll": 0.004432377281443713, |
| "brier": 0.0009832953881987287, |
| "ece": 0.004043651215824124, |
| "mean_confidence": 0.9959563487841759 |
| }, |
| "schema-firewall/product-science/xlam-hard|not_truncated": { |
| "n": 207, |
| "accuracy": 0.6521739130434783, |
| "nll": 0.5901507024798354, |
| "brier": 0.3917848867662268, |
| "ece": 0.21952186143340696, |
| "mean_confidence": 0.8716957744768852 |
| }, |
| "schema-firewall/stanfordnlp/snli|not_truncated": { |
| "n": 212, |
| "accuracy": 0.8915094339622641, |
| "nll": 0.28470200158541425, |
| "brier": 0.16288797511131478, |
| "ece": 0.05838072018803289, |
| "mean_confidence": 0.8921123490018664 |
| }, |
| "schema-firewall/synthetic/nli-haystack|not_truncated": { |
| "n": 153, |
| "accuracy": 0.9019607843137255, |
| "nll": 0.28354985410883243, |
| "brier": 0.15433488102038004, |
| "ece": 0.06596269872453478, |
| "mean_confidence": 0.8859285342147927 |
| } |
| }, |
| "by_kind_truncation": { |
| "choice|not_truncated": { |
| "n": 1186, |
| "accuracy": 0.8735244519392917, |
| "nll": 0.3085243966904598, |
| "brier": 0.17534503393928574, |
| "ece": 0.025424960818499803, |
| "mean_confidence": 0.8800864731029153 |
| }, |
| "noul|not_truncated": { |
| "n": 557, |
| "accuracy": 0.6032315978456014, |
| "nll": 0.762934762585891, |
| "brier": 0.5129334554548768, |
| "ece": 0.23712612482546905, |
| "mean_confidence": 0.8403577226710705 |
| }, |
| "score|not_truncated": { |
| "n": 257, |
| "accuracy": 0.6108949416342413, |
| "nll": 0.8855918734569957, |
| "brier": 0.49536697754948056, |
| "ece": 0.09132106381863471, |
| "mean_confidence": 0.5927287479898808 |
| } |
| }, |
| "nli_confusion": { |
| "n": 795, |
| "labels": [ |
| "entailment", |
| "neutral", |
| "contradiction" |
| ], |
| "matrix": { |
| "entailment": { |
| "entailment": 222, |
| "neutral": 30, |
| "contradiction": 4 |
| }, |
| "neutral": { |
| "entailment": 40, |
| "neutral": 255, |
| "contradiction": 18 |
| }, |
| "contradiction": { |
| "entailment": 1, |
| "neutral": 25, |
| "contradiction": 200 |
| } |
| } |
| } |
| }, |
| "firewall": { |
| "n": 2000, |
| "accuracy": 0.8785, |
| "nll": 0.32404716382944027, |
| "brier": 0.19121665542999927, |
| "ece": 0.019892057061195338, |
| "mean_confidence": 0.8801644538640976, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.0621812343597412, |
| "groups": { |
| "choice:2": { |
| "temperature": 0.7285364649335917, |
| "raw_temperature": 0.3852079510688782, |
| "n": 71, |
| "shrinkage_weight": 0.3717277486910995 |
| }, |
| "choice:3-5": { |
| "temperature": 1.0312828950285642, |
| "raw_temperature": 1.0276613235473633, |
| "n": 1007, |
| "shrinkage_weight": 0.8935226264418811 |
| }, |
| "choice:6-10": { |
| "temperature": 1.018117407640952, |
| "raw_temperature": 0.9839622974395752, |
| "n": 149, |
| "shrinkage_weight": 0.5539033457249071 |
| }, |
| "noul:*": { |
| "temperature": 1.12278263121421, |
| "raw_temperature": 1.1349645853042603, |
| "n": 617, |
| "shrinkage_weight": 0.8371777476255088 |
| }, |
| "score:*": { |
| "temperature": 1.1063119547040072, |
| "raw_temperature": 1.1415023803710938, |
| "n": 156, |
| "shrinkage_weight": 0.5652173913043478 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "firewall/tommccoy1-hans": { |
| "n": 2000, |
| "accuracy": 0.8785, |
| "nll": 0.32404716382944027, |
| "brier": 0.19121665542999927, |
| "ece": 0.019892057061195338, |
| "mean_confidence": 0.8801644538640976 |
| } |
| }, |
| "by_kind": { |
| "noul": { |
| "n": 2000, |
| "accuracy": 0.8785, |
| "nll": 0.32404716382944027, |
| "brier": 0.19121665542999927, |
| "ece": 0.019892057061195338, |
| "mean_confidence": 0.8801644538640976 |
| } |
| }, |
| "by_transform": { |
| "nli.firewall.hans.v1": { |
| "n": 2000, |
| "accuracy": 0.8785, |
| "nll": 0.32404716382944027, |
| "brier": 0.19121665542999927, |
| "ece": 0.019892057061195338, |
| "mean_confidence": 0.8801644538640976 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 2000, |
| "accuracy": 0.8785, |
| "nll": 0.32404716382944027, |
| "brier": 0.19121665542999927, |
| "ece": 0.019892057061195338, |
| "mean_confidence": 0.8801644538640976 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 2000, |
| "accuracy": 0.8785, |
| "nll": 0.32404716382944027, |
| "brier": 0.19121665542999927, |
| "ece": 0.019892057061195338, |
| "mean_confidence": 0.8801644538640976 |
| } |
| }, |
| "by_dataset_truncation": { |
| "firewall/tommccoy1-hans|not_truncated": { |
| "n": 2000, |
| "accuracy": 0.8785, |
| "nll": 0.32404716382944027, |
| "brier": 0.19121665542999927, |
| "ece": 0.019892057061195338, |
| "mean_confidence": 0.8801644538640976 |
| } |
| }, |
| "by_kind_truncation": { |
| "noul|not_truncated": { |
| "n": 2000, |
| "accuracy": 0.8785, |
| "nll": 0.32404716382944027, |
| "brier": 0.19121665542999927, |
| "ece": 0.019892057061195338, |
| "mean_confidence": 0.8801644538640976 |
| } |
| } |
| }, |
| "confusability": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.0022301711396669685, |
| "brier": 3.975738844519126e-05, |
| "ece": 0.002218638658523564, |
| "mean_confidence": 0.9977813613414764, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.0621812343597412, |
| "groups": { |
| "choice:2": { |
| "temperature": 0.7285364649335917, |
| "raw_temperature": 0.3852079510688782, |
| "n": 71, |
| "shrinkage_weight": 0.3717277486910995 |
| }, |
| "choice:3-5": { |
| "temperature": 1.0312828950285642, |
| "raw_temperature": 1.0276613235473633, |
| "n": 1007, |
| "shrinkage_weight": 0.8935226264418811 |
| }, |
| "choice:6-10": { |
| "temperature": 1.018117407640952, |
| "raw_temperature": 0.9839622974395752, |
| "n": 149, |
| "shrinkage_weight": 0.5539033457249071 |
| }, |
| "noul:*": { |
| "temperature": 1.12278263121421, |
| "raw_temperature": 1.1349645853042603, |
| "n": 617, |
| "shrinkage_weight": 0.8371777476255088 |
| }, |
| "score:*": { |
| "temperature": 1.1063119547040072, |
| "raw_temperature": 1.1415023803710938, |
| "n": 156, |
| "shrinkage_weight": 0.5652173913043478 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "v142-confusability": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.0022301711396669685, |
| "brier": 3.975738844519126e-05, |
| "ece": 0.002218638658523564, |
| "mean_confidence": 0.9977813613414764 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 1000, |
| "accuracy": 1.0, |
| "nll": 0.001581937825678324, |
| "brier": 3.273811125628399e-05, |
| "ece": 0.0015734966397285, |
| "mean_confidence": 0.9984265033602715 |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0006024025060469285, |
| "brier": 8.392574431326238e-07, |
| "ece": 0.000602192580699934, |
| "mean_confidence": 0.9993978074193001 |
| }, |
| "score": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.007099106343230232, |
| "brier": 0.00011377190539178627, |
| "ece": 0.007060794830322292, |
| "mean_confidence": 0.9929392051696777 |
| } |
| }, |
| "by_transform": { |
| "coreference": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0031666626091100625, |
| "brier": 8.986054856441328e-05, |
| "ece": 0.00314338117837909, |
| "mean_confidence": 0.9968566188216209 |
| }, |
| "entity_binding": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 5.6076028463394324e-05, |
| "brier": 6.457422713664063e-09, |
| "ece": 5.607396364215589e-05, |
| "mean_confidence": 0.9999439260363578 |
| }, |
| "negation": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0006024025060469285, |
| "brier": 8.392574431326238e-07, |
| "ece": 0.000602192580699934, |
| "mean_confidence": 0.9993978074193001 |
| }, |
| "numeric_boundary": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 8.565005251966796e-05, |
| "brier": 2.4895733549690215e-08, |
| "ece": 8.564382791520497e-05, |
| "mean_confidence": 0.9999143561720848 |
| }, |
| "ordinal_boundary": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.007099106343230232, |
| "brier": 0.00011377190539178627, |
| "ece": 0.007060794830322292, |
| "mean_confidence": 0.9929392051696777 |
| }, |
| "temporal_state": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0034405127598438412, |
| "brier": 3.449978205878779e-05, |
| "ece": 0.0034316426515579312, |
| "mean_confidence": 0.9965683573484421 |
| }, |
| "tool_argument_validity": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0011607876784546533, |
| "brier": 3.929887250195552e-05, |
| "ece": 0.0011507415771484508, |
| "mean_confidence": 0.9988492584228515 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 400, |
| "accuracy": 1.0, |
| "nll": 0.0003440262792832982, |
| "brier": 4.32076588341157e-07, |
| "ece": 0.0003439182043075695, |
| "mean_confidence": 0.9996560817956924 |
| }, |
| "3-5": { |
| "n": 1000, |
| "accuracy": 1.0, |
| "nll": 0.0029846290838204366, |
| "brier": 5.5487513187931304e-05, |
| "ece": 0.002968526840209984, |
| "mean_confidence": 0.99703147315979 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.0022301711396669685, |
| "brier": 3.975738844519126e-05, |
| "ece": 0.002218638658523564, |
| "mean_confidence": 0.9977813613414764 |
| } |
| }, |
| "by_dataset_truncation": { |
| "v142-confusability|not_truncated": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.0022301711396669685, |
| "brier": 3.975738844519126e-05, |
| "ece": 0.002218638658523564, |
| "mean_confidence": 0.9977813613414764 |
| } |
| }, |
| "by_kind_truncation": { |
| "choice|not_truncated": { |
| "n": 1000, |
| "accuracy": 1.0, |
| "nll": 0.001581937825678324, |
| "brier": 3.273811125628399e-05, |
| "ece": 0.0015734966397285, |
| "mean_confidence": 0.9984265033602715 |
| }, |
| "noul|not_truncated": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0006024025060469285, |
| "brier": 8.392574431326238e-07, |
| "ece": 0.000602192580699934, |
| "mean_confidence": 0.9993978074193001 |
| }, |
| "score|not_truncated": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.007099106343230232, |
| "brier": 0.00011377190539178627, |
| "ece": 0.007060794830322292, |
| "mean_confidence": 0.9929392051696777 |
| } |
| } |
| } |
| }, |
| "latency": { |
| "validation": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 3746.8793283732375, |
| "effective_ms_per_example": 468.3599160466547, |
| "skipped_unusable": 0 |
| }, |
| "schema": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 3996.121177363326, |
| "effective_ms_per_example": 499.51514717041573, |
| "skipped_unusable": 0 |
| }, |
| "firewall": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 335.970186118735, |
| "effective_ms_per_example": 41.996273264841875, |
| "skipped_unusable": 0 |
| }, |
| "confusability": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 658.859060741961, |
| "effective_ms_per_example": 82.35738259274513, |
| "skipped_unusable": 0 |
| } |
| } |
| }, |
| "head_lora_rlcd": { |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.2130241394042969, |
| "groups": { |
| "choice:2": { |
| "temperature": 1.131178691354451, |
| "raw_temperature": 0.9999842643737793, |
| "n": 68, |
| "shrinkage_weight": 0.3617021276595745 |
| }, |
| "choice:3-5": { |
| "temperature": 1.229406620643328, |
| "raw_temperature": 1.2313735485076904, |
| "n": 1007, |
| "shrinkage_weight": 0.8935226264418811 |
| }, |
| "choice:6-10": { |
| "temperature": 1.1287873284307708, |
| "raw_temperature": 1.0643781423568726, |
| "n": 147, |
| "shrinkage_weight": 0.550561797752809 |
| }, |
| "noul:*": { |
| "temperature": 1.2522414435081368, |
| "raw_temperature": 1.259989619255066, |
| "n": 619, |
| "shrinkage_weight": 0.8376184032476319 |
| }, |
| "score:*": { |
| "temperature": 1.1708337953943981, |
| "raw_temperature": 1.1399664878845215, |
| "n": 159, |
| "shrinkage_weight": 0.5698924731182796 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "sets": { |
| "validation": { |
| "n": 2000, |
| "accuracy": 0.874, |
| "nll": 0.3413633666240774, |
| "brier": 0.18889581276680206, |
| "ece": 0.022634778514504437, |
| "mean_confidence": 0.8636561000198126, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.2130241394042969, |
| "groups": { |
| "choice:2": { |
| "temperature": 1.131178691354451, |
| "raw_temperature": 0.9999842643737793, |
| "n": 68, |
| "shrinkage_weight": 0.3617021276595745 |
| }, |
| "choice:3-5": { |
| "temperature": 1.229406620643328, |
| "raw_temperature": 1.2313735485076904, |
| "n": 1007, |
| "shrinkage_weight": 0.8935226264418811 |
| }, |
| "choice:6-10": { |
| "temperature": 1.1287873284307708, |
| "raw_temperature": 1.0643781423568726, |
| "n": 147, |
| "shrinkage_weight": 0.550561797752809 |
| }, |
| "noul:*": { |
| "temperature": 1.2522414435081368, |
| "raw_temperature": 1.259989619255066, |
| "n": 619, |
| "shrinkage_weight": 0.8376184032476319 |
| }, |
| "score:*": { |
| "temperature": 1.1708337953943981, |
| "raw_temperature": 1.1399664878845215, |
| "n": 159, |
| "shrinkage_weight": 0.5698924731182796 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "PolyAI/banking77": { |
| "n": 136, |
| "accuracy": 0.9338235294117647, |
| "nll": 0.18907353777312358, |
| "brier": 0.10504818881286704, |
| "ece": 0.06510368646944269, |
| "mean_confidence": 0.9324855239075773 |
| }, |
| "Rowan/hellaswag": { |
| "n": 154, |
| "accuracy": 0.922077922077922, |
| "nll": 0.21983189224844757, |
| "brier": 0.11673031982369635, |
| "ece": 0.040538395186523364, |
| "mean_confidence": 0.9012584014759435 |
| }, |
| "alisawuffles/WANLI": { |
| "n": 156, |
| "accuracy": 0.7564102564102564, |
| "nll": 0.5891167478373226, |
| "brier": 0.35234179292866685, |
| "ece": 0.06832877985942057, |
| "mean_confidence": 0.7601621380219092 |
| }, |
| "cais/mmlu": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.40682872855269997, |
| "brier": 0.20491553394237635, |
| "ece": 0.04900335096845434, |
| "mean_confidence": 0.8429947429233127 |
| }, |
| "google/boolq": { |
| "n": 153, |
| "accuracy": 0.8758169934640523, |
| "nll": 0.3271605430153393, |
| "brier": 0.18819155338973212, |
| "ece": 0.05587063856374201, |
| "mean_confidence": 0.8991910511372136 |
| }, |
| "google/boolq-negation": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.3173225029292547, |
| "brier": 0.18408045429418515, |
| "ece": 0.0648853922981063, |
| "mean_confidence": 0.8740317903312982 |
| }, |
| "nvidia/HelpSteer2": { |
| "n": 159, |
| "accuracy": 0.660377358490566, |
| "nll": 0.8923262642614497, |
| "brier": 0.4767587937402931, |
| "ece": 0.0820022630241682, |
| "mean_confidence": 0.6223008767613825 |
| }, |
| "nvidia/HelpSteer2-threshold": { |
| "n": 158, |
| "accuracy": 0.8860759493670886, |
| "nll": 0.24275886275381134, |
| "brier": 0.14648212109415587, |
| "ece": 0.04942092601257034, |
| "mean_confidence": 0.8698332471938073 |
| }, |
| "nyu-mll/multi_nli": { |
| "n": 157, |
| "accuracy": 0.910828025477707, |
| "nll": 0.3416944928808363, |
| "brier": 0.17827376386410335, |
| "ece": 0.08725254399001975, |
| "mean_confidence": 0.844191101705952 |
| }, |
| "product-science/xlam-function-calling-60k-raw": { |
| "n": 154, |
| "accuracy": 1.0, |
| "nll": 0.0060333324190392805, |
| "brier": 0.002306750856381662, |
| "ece": 0.005282950865757921, |
| "mean_confidence": 0.9947170491342421 |
| }, |
| "product-science/xlam-hard": { |
| "n": 155, |
| "accuracy": 0.967741935483871, |
| "nll": 0.06760209277854301, |
| "brier": 0.040693268881268306, |
| "ece": 0.02164438040025776, |
| "mean_confidence": 0.9642918621340105 |
| }, |
| "stanfordnlp/snli": { |
| "n": 158, |
| "accuracy": 0.8607594936708861, |
| "nll": 0.405171424059103, |
| "brier": 0.22694251017077752, |
| "ece": 0.06340998675249801, |
| "mean_confidence": 0.8710687118995039 |
| }, |
| "synthetic/nli-haystack": { |
| "n": 154, |
| "accuracy": 0.8636363636363636, |
| "nll": 0.39674904061384597, |
| "brier": 0.21296215491428802, |
| "ece": 0.03454946155671952, |
| "mean_confidence": 0.8677647721457791 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 1222, |
| "accuracy": 0.8887070376432079, |
| "nll": 0.32193676473427585, |
| "brier": 0.17641385298182077, |
| "ece": 0.02652165657077016, |
| "mean_confidence": 0.8757482603948362 |
| }, |
| "noul": { |
| "n": 619, |
| "accuracy": 0.8998384491114702, |
| "nll": 0.23819084123634773, |
| "brier": 0.13959490950745163, |
| "ece": 0.017604535538853695, |
| "mean_confidence": 0.9017802688724967 |
| }, |
| "score": { |
| "n": 159, |
| "accuracy": 0.660377358490566, |
| "nll": 0.8923262642614497, |
| "brier": 0.4767587937402931, |
| "ece": 0.0820022630241682, |
| "mean_confidence": 0.6223008767613825 |
| } |
| }, |
| "by_transform": { |
| "agent.invocation.correct.v1": { |
| "n": 52, |
| "accuracy": 0.9230769230769231, |
| "nll": 0.1550768981419074, |
| "brier": 0.09798712961044377, |
| "ece": 0.06195802413500273, |
| "mean_confidence": 0.9290768962640029 |
| }, |
| "agent.invocation.corrupt_argument.v1": { |
| "n": 58, |
| "accuracy": 1.0, |
| "nll": 0.015422354992740016, |
| "brier": 0.007456194708894294, |
| "ece": 0.01274871106805474, |
| "mean_confidence": 0.9872512889319452 |
| }, |
| "agent.invocation.wrong_tool.v1": { |
| "n": 45, |
| "accuracy": 0.9777777777777777, |
| "nll": 0.03377397972702359, |
| "brier": 0.017325925416392034, |
| "ece": 0.02705154154035782, |
| "mean_confidence": 0.9753925614886814 |
| }, |
| "commonsense.choice.v2": { |
| "n": 154, |
| "accuracy": 0.922077922077922, |
| "nll": 0.21983189224844757, |
| "brier": 0.11673031982369635, |
| "ece": 0.040538395186523364, |
| "mean_confidence": 0.9012584014759435 |
| }, |
| "intent.choice.v3": { |
| "n": 136, |
| "accuracy": 0.9338235294117647, |
| "nll": 0.18907353777312358, |
| "brier": 0.10504818881286704, |
| "ece": 0.06510368646944269, |
| "mean_confidence": 0.9324855239075773 |
| }, |
| "knowledge.choice.v1": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.40682872855269997, |
| "brier": 0.20491553394237635, |
| "ece": 0.04900335096845434, |
| "mean_confidence": 0.8429947429233127 |
| }, |
| "nli.choice.v4": { |
| "n": 157, |
| "accuracy": 0.910828025477707, |
| "nll": 0.3416944928808363, |
| "brier": 0.17827376386410335, |
| "ece": 0.08725254399001975, |
| "mean_confidence": 0.844191101705952 |
| }, |
| "nli.hard.v4": { |
| "n": 156, |
| "accuracy": 0.7564102564102564, |
| "nll": 0.5891167478373226, |
| "brier": 0.35234179292866685, |
| "ece": 0.06832877985942057, |
| "mean_confidence": 0.7601621380219092 |
| }, |
| "nli.haystack.absent.v1": { |
| "n": 6, |
| "accuracy": 1.0, |
| "nll": 0.12007932023455699, |
| "brier": 0.042778262247641884, |
| "ece": 0.10573691129684447, |
| "mean_confidence": 0.8942630887031555 |
| }, |
| "nli.haystack.present.v1": { |
| "n": 148, |
| "accuracy": 0.8581081081081081, |
| "nll": 0.40796538062922255, |
| "brier": 0.2198615019142872, |
| "ece": 0.038296295178903136, |
| "mean_confidence": 0.8666905160691287 |
| }, |
| "nli.snli.v1": { |
| "n": 158, |
| "accuracy": 0.8607594936708861, |
| "nll": 0.405171424059103, |
| "brier": 0.22694251017077752, |
| "ece": 0.06340998675249801, |
| "mean_confidence": 0.8710687118995039 |
| }, |
| "noul.negated_answer.v1": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.3173225029292547, |
| "brier": 0.18408045429418515, |
| "ece": 0.0648853922981063, |
| "mean_confidence": 0.8740317903312982 |
| }, |
| "noul.v3": { |
| "n": 153, |
| "accuracy": 0.8758169934640523, |
| "nll": 0.3271605430153393, |
| "brier": 0.18819155338973212, |
| "ece": 0.05587063856374201, |
| "mean_confidence": 0.8991910511372136 |
| }, |
| "score.coherence.threshold.v1": { |
| "n": 36, |
| "accuracy": 0.9444444444444444, |
| "nll": 0.15781310862965053, |
| "brier": 0.08520178628396025, |
| "ece": 0.08752633300092487, |
| "mean_confidence": 0.8862901313437356 |
| }, |
| "score.coherence.v3": { |
| "n": 36, |
| "accuracy": 0.8611111111111112, |
| "nll": 0.49905938903490704, |
| "brier": 0.27527502324018216, |
| "ece": 0.14887905451986522, |
| "mean_confidence": 0.7211348878012763 |
| }, |
| "score.complexity.threshold.v1": { |
| "n": 28, |
| "accuracy": 0.9285714285714286, |
| "nll": 0.19121888622508518, |
| "brier": 0.10774182653299899, |
| "ece": 0.1085933659757887, |
| "mean_confidence": 0.9150230714253017 |
| }, |
| "score.complexity.v3": { |
| "n": 30, |
| "accuracy": 0.8, |
| "nll": 0.6180950621763865, |
| "brier": 0.36170755798618, |
| "ece": 0.1601276993751526, |
| "mean_confidence": 0.6420709570248921 |
| }, |
| "score.correctness.threshold.v1": { |
| "n": 30, |
| "accuracy": 0.8, |
| "nll": 0.3484654218579332, |
| "brier": 0.23496840521499204, |
| "ece": 0.13440750638643903, |
| "mean_confidence": 0.8265082140763601 |
| }, |
| "score.correctness.v3": { |
| "n": 22, |
| "accuracy": 0.5, |
| "nll": 1.0662444193254819, |
| "brier": 0.5692707571116361, |
| "ece": 0.18789786439050324, |
| "mean_confidence": 0.49144711751829495 |
| }, |
| "score.helpfulness.threshold.v1": { |
| "n": 30, |
| "accuracy": 0.8, |
| "nll": 0.3833232717588544, |
| "brier": 0.2426174452722383, |
| "ece": 0.1406406263510386, |
| "mean_confidence": 0.7806143343448639 |
| }, |
| "score.helpfulness.v3": { |
| "n": 32, |
| "accuracy": 0.40625, |
| "nll": 1.2641449673101306, |
| "brier": 0.6717381589114666, |
| "ece": 0.10654595633968711, |
| "mean_confidence": 0.4571774401701987 |
| }, |
| "score.verbosity.threshold.v1": { |
| "n": 34, |
| "accuracy": 0.9411764705882353, |
| "nll": 0.15784819945966935, |
| "brier": 0.08036953438568162, |
| "ece": 0.047484531122095476, |
| "mean_confidence": 0.9321437022265266 |
| }, |
| "score.verbosity.v3": { |
| "n": 39, |
| "accuracy": 0.6666666666666666, |
| "nll": 1.0630992299471147, |
| "brier": 0.5390754586372238, |
| "ece": 0.19610086618325653, |
| "mean_confidence": 0.7251625657081604 |
| }, |
| "tool.choice.single.v3": { |
| "n": 154, |
| "accuracy": 1.0, |
| "nll": 0.0060333324190392805, |
| "brier": 0.002306750856381662, |
| "ece": 0.005282950865757921, |
| "mean_confidence": 0.9947170491342421 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 687, |
| "accuracy": 0.9097525473071325, |
| "nll": 0.21461692523512385, |
| "brier": 0.1257776561957685, |
| "ece": 0.015864489380449164, |
| "mean_confidence": 0.9114996939246831 |
| }, |
| "3-5": { |
| "n": 1166, |
| "accuracy": 0.8447684391080618, |
| "nll": 0.4370019674408599, |
| "brier": 0.2376457747592827, |
| "ece": 0.03868178026447169, |
| "mean_confidence": 0.8261751901888724 |
| }, |
| "6-10": { |
| "n": 147, |
| "accuracy": 0.9387755102040817, |
| "nll": 0.1751062011944359, |
| "brier": 0.09719321331828265, |
| "ece": 0.0604115242049808, |
| "mean_confidence": 0.9373580853955276 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 1999, |
| "accuracy": 0.8744372186093047, |
| "nll": 0.34017474692719035, |
| "brier": 0.18811760026757368, |
| "ece": 0.022178890631758277, |
| "mean_confidence": 0.8636209331583297 |
| }, |
| "truncated": { |
| "n": 1, |
| "accuracy": 0.0, |
| "nll": 2.717414140701294, |
| "brier": 1.7445425987243652, |
| "ece": 0.933954656124115, |
| "mean_confidence": 0.933954656124115 |
| } |
| }, |
| "by_dataset_truncation": { |
| "PolyAI/banking77|not_truncated": { |
| "n": 136, |
| "accuracy": 0.9338235294117647, |
| "nll": 0.18907353777312358, |
| "brier": 0.10504818881286704, |
| "ece": 0.06510368646944269, |
| "mean_confidence": 0.9324855239075773 |
| }, |
| "Rowan/hellaswag|not_truncated": { |
| "n": 154, |
| "accuracy": 0.922077922077922, |
| "nll": 0.21983189224844757, |
| "brier": 0.11673031982369635, |
| "ece": 0.040538395186523364, |
| "mean_confidence": 0.9012584014759435 |
| }, |
| "alisawuffles/WANLI|not_truncated": { |
| "n": 156, |
| "accuracy": 0.7564102564102564, |
| "nll": 0.5891167478373226, |
| "brier": 0.35234179292866685, |
| "ece": 0.06832877985942057, |
| "mean_confidence": 0.7601621380219092 |
| }, |
| "cais/mmlu|not_truncated": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.40682872855269997, |
| "brier": 0.20491553394237635, |
| "ece": 0.04900335096845434, |
| "mean_confidence": 0.8429947429233127 |
| }, |
| "google/boolq-negation|not_truncated": { |
| "n": 153, |
| "accuracy": 0.869281045751634, |
| "nll": 0.3173225029292547, |
| "brier": 0.18408045429418515, |
| "ece": 0.0648853922981063, |
| "mean_confidence": 0.8740317903312982 |
| }, |
| "google/boolq|not_truncated": { |
| "n": 153, |
| "accuracy": 0.8758169934640523, |
| "nll": 0.3271605430153393, |
| "brier": 0.18819155338973212, |
| "ece": 0.05587063856374201, |
| "mean_confidence": 0.8991910511372136 |
| }, |
| "nvidia/HelpSteer2-threshold|not_truncated": { |
| "n": 157, |
| "accuracy": 0.89171974522293, |
| "nll": 0.22699672722548342, |
| "brier": 0.13630339193727556, |
| "ece": 0.055684464752294466, |
| "mean_confidence": 0.8694248305764168 |
| }, |
| "nvidia/HelpSteer2-threshold|truncated": { |
| "n": 1, |
| "accuracy": 0.0, |
| "nll": 2.717414140701294, |
| "brier": 1.7445425987243652, |
| "ece": 0.933954656124115, |
| "mean_confidence": 0.933954656124115 |
| }, |
| "nvidia/HelpSteer2|not_truncated": { |
| "n": 159, |
| "accuracy": 0.660377358490566, |
| "nll": 0.8923262642614497, |
| "brier": 0.4767587937402931, |
| "ece": 0.0820022630241682, |
| "mean_confidence": 0.6223008767613825 |
| }, |
| "nyu-mll/multi_nli|not_truncated": { |
| "n": 157, |
| "accuracy": 0.910828025477707, |
| "nll": 0.3416944928808363, |
| "brier": 0.17827376386410335, |
| "ece": 0.08725254399001975, |
| "mean_confidence": 0.844191101705952 |
| }, |
| "product-science/xlam-function-calling-60k-raw|not_truncated": { |
| "n": 154, |
| "accuracy": 1.0, |
| "nll": 0.0060333324190392805, |
| "brier": 0.002306750856381662, |
| "ece": 0.005282950865757921, |
| "mean_confidence": 0.9947170491342421 |
| }, |
| "product-science/xlam-hard|not_truncated": { |
| "n": 155, |
| "accuracy": 0.967741935483871, |
| "nll": 0.06760209277854301, |
| "brier": 0.040693268881268306, |
| "ece": 0.02164438040025776, |
| "mean_confidence": 0.9642918621340105 |
| }, |
| "stanfordnlp/snli|not_truncated": { |
| "n": 158, |
| "accuracy": 0.8607594936708861, |
| "nll": 0.405171424059103, |
| "brier": 0.22694251017077752, |
| "ece": 0.06340998675249801, |
| "mean_confidence": 0.8710687118995039 |
| }, |
| "synthetic/nli-haystack|not_truncated": { |
| "n": 154, |
| "accuracy": 0.8636363636363636, |
| "nll": 0.39674904061384597, |
| "brier": 0.21296215491428802, |
| "ece": 0.03454946155671952, |
| "mean_confidence": 0.8677647721457791 |
| } |
| }, |
| "by_kind_truncation": { |
| "choice|not_truncated": { |
| "n": 1222, |
| "accuracy": 0.8887070376432079, |
| "nll": 0.32193676473427585, |
| "brier": 0.17641385298182077, |
| "ece": 0.02652165657077016, |
| "mean_confidence": 0.8757482603948362 |
| }, |
| "noul|not_truncated": { |
| "n": 618, |
| "accuracy": 0.901294498381877, |
| "nll": 0.23417915304951123, |
| "brier": 0.13699790677409093, |
| "ece": 0.019144275331188614, |
| "mean_confidence": 0.9017282067572029 |
| }, |
| "noul|truncated": { |
| "n": 1, |
| "accuracy": 0.0, |
| "nll": 2.717414140701294, |
| "brier": 1.7445425987243652, |
| "ece": 0.933954656124115, |
| "mean_confidence": 0.933954656124115 |
| }, |
| "score|not_truncated": { |
| "n": 159, |
| "accuracy": 0.660377358490566, |
| "nll": 0.8923262642614497, |
| "brier": 0.4767587937402931, |
| "ece": 0.0820022630241682, |
| "mean_confidence": 0.6223008767613825 |
| } |
| }, |
| "nli_confusion": { |
| "n": 625, |
| "labels": [ |
| "entailment", |
| "neutral", |
| "contradiction" |
| ], |
| "matrix": { |
| "entailment": { |
| "entailment": 179, |
| "neutral": 28, |
| "contradiction": 6 |
| }, |
| "neutral": { |
| "entailment": 17, |
| "neutral": 178, |
| "contradiction": 17 |
| }, |
| "contradiction": { |
| "entailment": 0, |
| "neutral": 27, |
| "contradiction": 173 |
| } |
| } |
| } |
| }, |
| "schema": { |
| "n": 2000, |
| "accuracy": 0.777, |
| "nll": 0.48716114530798577, |
| "brier": 0.2934196735615337, |
| "ece": 0.046331627137959026, |
| "mean_confidence": 0.8208303308263422, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.2130241394042969, |
| "groups": { |
| "choice:2": { |
| "temperature": 1.131178691354451, |
| "raw_temperature": 0.9999842643737793, |
| "n": 68, |
| "shrinkage_weight": 0.3617021276595745 |
| }, |
| "choice:3-5": { |
| "temperature": 1.229406620643328, |
| "raw_temperature": 1.2313735485076904, |
| "n": 1007, |
| "shrinkage_weight": 0.8935226264418811 |
| }, |
| "choice:6-10": { |
| "temperature": 1.1287873284307708, |
| "raw_temperature": 1.0643781423568726, |
| "n": 147, |
| "shrinkage_weight": 0.550561797752809 |
| }, |
| "noul:*": { |
| "temperature": 1.2522414435081368, |
| "raw_temperature": 1.259989619255066, |
| "n": 619, |
| "shrinkage_weight": 0.8376184032476319 |
| }, |
| "score:*": { |
| "temperature": 1.1708337953943981, |
| "raw_temperature": 1.1399664878845215, |
| "n": 159, |
| "shrinkage_weight": 0.5698924731182796 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "schema-firewall/PolyAI/banking77": { |
| "n": 46, |
| "accuracy": 0.8043478260869565, |
| "nll": 0.35682467644105403, |
| "brier": 0.2239034981356199, |
| "ece": 0.12990026823852374, |
| "mean_confidence": 0.8811038130003473 |
| }, |
| "schema-firewall/Rowan/hellaswag": { |
| "n": 130, |
| "accuracy": 0.9076923076923077, |
| "nll": 0.20287465754243106, |
| "brier": 0.11677193438268157, |
| "ece": 0.047463341171924855, |
| "mean_confidence": 0.9111302653184303 |
| }, |
| "schema-firewall/alisawuffles/WANLI": { |
| "n": 205, |
| "accuracy": 0.7219512195121951, |
| "nll": 0.6024273006109203, |
| "brier": 0.36376239454186315, |
| "ece": 0.06989647847850149, |
| "mean_confidence": 0.7581731124622065 |
| }, |
| "schema-firewall/cais/mmlu": { |
| "n": 134, |
| "accuracy": 0.917910447761194, |
| "nll": 0.2275545184470434, |
| "brier": 0.11969006923772431, |
| "ece": 0.05594150722026825, |
| "mean_confidence": 0.9056024651474027 |
| }, |
| "schema-firewall/google/boolq": { |
| "n": 36, |
| "accuracy": 0.8888888888888888, |
| "nll": 0.42648325838106554, |
| "brier": 0.2071607477258826, |
| "ece": 0.13994142909844717, |
| "mean_confidence": 0.8997285084591972 |
| }, |
| "schema-firewall/google/boolq-negation": { |
| "n": 60, |
| "accuracy": 0.5166666666666667, |
| "nll": 1.2815287677299543, |
| "brier": 0.7706017639355196, |
| "ece": 0.36829513410727177, |
| "mean_confidence": 0.8849618007739385 |
| }, |
| "schema-firewall/nvidia/HelpSteer2": { |
| "n": 273, |
| "accuracy": 0.608058608058608, |
| "nll": 0.9225735204778748, |
| "brier": 0.502879177783053, |
| "ece": 0.05189086431330377, |
| "mean_confidence": 0.6023068452601904 |
| }, |
| "schema-firewall/nvidia/HelpSteer2-threshold": { |
| "n": 263, |
| "accuracy": 0.6121673003802282, |
| "nll": 0.7050396920068494, |
| "brier": 0.5057733078411408, |
| "ece": 0.20569920426539143, |
| "mean_confidence": 0.8013089970037511 |
| }, |
| "schema-firewall/nyu-mll/multi_nli": { |
| "n": 214, |
| "accuracy": 0.8691588785046729, |
| "nll": 0.3600308844127701, |
| "brier": 0.19274686407703126, |
| "ece": 0.06172461933064685, |
| "mean_confidence": 0.8438963853867254 |
| }, |
| "schema-firewall/product-science/xlam-function-calling-60k-raw": { |
| "n": 81, |
| "accuracy": 1.0, |
| "nll": 0.0060919241069499275, |
| "brier": 0.00158877624618902, |
| "ece": 0.005427672539228313, |
| "mean_confidence": 0.9945723274607717 |
| }, |
| "schema-firewall/product-science/xlam-hard": { |
| "n": 202, |
| "accuracy": 0.7673267326732673, |
| "nll": 0.3468692268006905, |
| "brier": 0.24214135508483428, |
| "ece": 0.09207318767462629, |
| "mean_confidence": 0.8529215121623313 |
| }, |
| "schema-firewall/stanfordnlp/snli": { |
| "n": 202, |
| "accuracy": 0.8811881188118812, |
| "nll": 0.2818646928258609, |
| "brier": 0.16245338230385975, |
| "ece": 0.040539493360141704, |
| "mean_confidence": 0.8788761552607659 |
| }, |
| "schema-firewall/synthetic/nli-haystack": { |
| "n": 154, |
| "accuracy": 0.8961038961038961, |
| "nll": 0.28225524406940616, |
| "brier": 0.15355019212951115, |
| "ece": 0.0682181323116476, |
| "mean_confidence": 0.8718665600210042 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 1166, |
| "accuracy": 0.8653516295025729, |
| "nll": 0.3213735620484178, |
| "brier": 0.1834721313419164, |
| "ece": 0.029624137982700523, |
| "mean_confidence": 0.8651016697057524 |
| }, |
| "noul": { |
| "n": 561, |
| "accuracy": 0.6755793226381461, |
| "nll": 0.6198540930072308, |
| "brier": 0.4200086032863093, |
| "ece": 0.15957637436674665, |
| "mean_confidence": 0.8351556970048928 |
| }, |
| "score": { |
| "n": 273, |
| "accuracy": 0.608058608058608, |
| "nll": 0.9225735204778748, |
| "brier": 0.502879177783053, |
| "ece": 0.05189086431330377, |
| "mean_confidence": 0.6023068452601904 |
| } |
| }, |
| "by_transform": { |
| "schema.firewall.distractor_drop.v1": { |
| "n": 179, |
| "accuracy": 0.9162011173184358, |
| "nll": 0.18068733821303656, |
| "brier": 0.1011414483847378, |
| "ece": 0.032388108236163664, |
| "mean_confidence": 0.9310932723836526 |
| }, |
| "schema.firewall.instruction.v1": { |
| "n": 1416, |
| "accuracy": 0.8509887005649718, |
| "nll": 0.36921639851671667, |
| "brier": 0.2073376011020019, |
| "ece": 0.02585129172535939, |
| "mean_confidence": 0.8416002465801388 |
| }, |
| "schema.firewall.noul_inversion.v1": { |
| "n": 273, |
| "accuracy": 0.3882783882783883, |
| "nll": 1.0767006237783057, |
| "brier": 0.7550764729788241, |
| "ece": 0.35771637259822187, |
| "mean_confidence": 0.7459947608766102 |
| }, |
| "schema.firewall.score_scale.v1": { |
| "n": 132, |
| "accuracy": 0.5984848484848485, |
| "nll": 0.9487088370052251, |
| "brier": 0.5227962695344379, |
| "ece": 0.10626354102383959, |
| "mean_confidence": 0.6032761137819651 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 588, |
| "accuracy": 0.6904761904761905, |
| "nll": 0.5913940336594421, |
| "brier": 0.4007224980486256, |
| "ece": 0.15224626021725784, |
| "mean_confidence": 0.8427224506934484 |
| }, |
| "3-5": { |
| "n": 1381, |
| "accuracy": 0.8117306299782766, |
| "nll": 0.44767054287509334, |
| "brier": 0.25053120708128784, |
| "ece": 0.027608664106148723, |
| "mean_confidence": 0.8095618236397246 |
| }, |
| "6-10": { |
| "n": 31, |
| "accuracy": 0.8709677419354839, |
| "nll": 0.26934126173276335, |
| "brier": 0.16873939649087583, |
| "ece": 0.11212440363822443, |
| "mean_confidence": 0.9075800709186062 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 1999, |
| "accuracy": 0.7773886943471736, |
| "nll": 0.4869243776422231, |
| "brier": 0.2932593345994733, |
| "ece": 0.04612398467372813, |
| "mean_confidence": 0.8210101314354921 |
| }, |
| "truncated": { |
| "n": 1, |
| "accuracy": 0.0, |
| "nll": 0.9604597091674805, |
| "brier": 0.613937258720398, |
| "ece": 0.46140891313552856, |
| "mean_confidence": 0.46140891313552856 |
| } |
| }, |
| "by_dataset_truncation": { |
| "schema-firewall/PolyAI/banking77|not_truncated": { |
| "n": 46, |
| "accuracy": 0.8043478260869565, |
| "nll": 0.35682467644105403, |
| "brier": 0.2239034981356199, |
| "ece": 0.12990026823852374, |
| "mean_confidence": 0.8811038130003473 |
| }, |
| "schema-firewall/Rowan/hellaswag|not_truncated": { |
| "n": 130, |
| "accuracy": 0.9076923076923077, |
| "nll": 0.20287465754243106, |
| "brier": 0.11677193438268157, |
| "ece": 0.047463341171924855, |
| "mean_confidence": 0.9111302653184303 |
| }, |
| "schema-firewall/alisawuffles/WANLI|not_truncated": { |
| "n": 205, |
| "accuracy": 0.7219512195121951, |
| "nll": 0.6024273006109203, |
| "brier": 0.36376239454186315, |
| "ece": 0.06989647847850149, |
| "mean_confidence": 0.7581731124622065 |
| }, |
| "schema-firewall/cais/mmlu|not_truncated": { |
| "n": 134, |
| "accuracy": 0.917910447761194, |
| "nll": 0.2275545184470434, |
| "brier": 0.11969006923772431, |
| "ece": 0.05594150722026825, |
| "mean_confidence": 0.9056024651474027 |
| }, |
| "schema-firewall/google/boolq-negation|not_truncated": { |
| "n": 60, |
| "accuracy": 0.5166666666666667, |
| "nll": 1.2815287677299543, |
| "brier": 0.7706017639355196, |
| "ece": 0.36829513410727177, |
| "mean_confidence": 0.8849618007739385 |
| }, |
| "schema-firewall/google/boolq|not_truncated": { |
| "n": 36, |
| "accuracy": 0.8888888888888888, |
| "nll": 0.42648325838106554, |
| "brier": 0.2071607477258826, |
| "ece": 0.13994142909844717, |
| "mean_confidence": 0.8997285084591972 |
| }, |
| "schema-firewall/nvidia/HelpSteer2-threshold|not_truncated": { |
| "n": 263, |
| "accuracy": 0.6121673003802282, |
| "nll": 0.7050396920068494, |
| "brier": 0.5057733078411408, |
| "ece": 0.20569920426539143, |
| "mean_confidence": 0.8013089970037511 |
| }, |
| "schema-firewall/nvidia/HelpSteer2|not_truncated": { |
| "n": 272, |
| "accuracy": 0.6102941176470589, |
| "nll": 0.9224342330194572, |
| "brier": 0.502470876014901, |
| "ece": 0.050385283251457344, |
| "mean_confidence": 0.60282485236359 |
| }, |
| "schema-firewall/nvidia/HelpSteer2|truncated": { |
| "n": 1, |
| "accuracy": 0.0, |
| "nll": 0.9604597091674805, |
| "brier": 0.613937258720398, |
| "ece": 0.46140891313552856, |
| "mean_confidence": 0.46140891313552856 |
| }, |
| "schema-firewall/nyu-mll/multi_nli|not_truncated": { |
| "n": 214, |
| "accuracy": 0.8691588785046729, |
| "nll": 0.3600308844127701, |
| "brier": 0.19274686407703126, |
| "ece": 0.06172461933064685, |
| "mean_confidence": 0.8438963853867254 |
| }, |
| "schema-firewall/product-science/xlam-function-calling-60k-raw|not_truncated": { |
| "n": 81, |
| "accuracy": 1.0, |
| "nll": 0.0060919241069499275, |
| "brier": 0.00158877624618902, |
| "ece": 0.005427672539228313, |
| "mean_confidence": 0.9945723274607717 |
| }, |
| "schema-firewall/product-science/xlam-hard|not_truncated": { |
| "n": 202, |
| "accuracy": 0.7673267326732673, |
| "nll": 0.3468692268006905, |
| "brier": 0.24214135508483428, |
| "ece": 0.09207318767462629, |
| "mean_confidence": 0.8529215121623313 |
| }, |
| "schema-firewall/stanfordnlp/snli|not_truncated": { |
| "n": 202, |
| "accuracy": 0.8811881188118812, |
| "nll": 0.2818646928258609, |
| "brier": 0.16245338230385975, |
| "ece": 0.040539493360141704, |
| "mean_confidence": 0.8788761552607659 |
| }, |
| "schema-firewall/synthetic/nli-haystack|not_truncated": { |
| "n": 154, |
| "accuracy": 0.8961038961038961, |
| "nll": 0.28225524406940616, |
| "brier": 0.15355019212951115, |
| "ece": 0.0682181323116476, |
| "mean_confidence": 0.8718665600210042 |
| } |
| }, |
| "by_kind_truncation": { |
| "choice|not_truncated": { |
| "n": 1166, |
| "accuracy": 0.8653516295025729, |
| "nll": 0.3213735620484178, |
| "brier": 0.1834721313419164, |
| "ece": 0.029624137982700523, |
| "mean_confidence": 0.8651016697057524 |
| }, |
| "noul|not_truncated": { |
| "n": 561, |
| "accuracy": 0.6755793226381461, |
| "nll": 0.6198540930072308, |
| "brier": 0.4200086032863093, |
| "ece": 0.15957637436674665, |
| "mean_confidence": 0.8351556970048928 |
| }, |
| "score|not_truncated": { |
| "n": 272, |
| "accuracy": 0.6102941176470589, |
| "nll": 0.9224342330194572, |
| "brier": 0.502470876014901, |
| "ece": 0.050385283251457344, |
| "mean_confidence": 0.60282485236359 |
| }, |
| "score|truncated": { |
| "n": 1, |
| "accuracy": 0.0, |
| "nll": 0.9604597091674805, |
| "brier": 0.613937258720398, |
| "ece": 0.46140891313552856, |
| "mean_confidence": 0.46140891313552856 |
| } |
| }, |
| "nli_confusion": { |
| "n": 775, |
| "labels": [ |
| "entailment", |
| "neutral", |
| "contradiction" |
| ], |
| "matrix": { |
| "entailment": { |
| "entailment": 201, |
| "neutral": 38, |
| "contradiction": 1 |
| }, |
| "neutral": { |
| "entailment": 31, |
| "neutral": 263, |
| "contradiction": 22 |
| }, |
| "contradiction": { |
| "entailment": 4, |
| "neutral": 29, |
| "contradiction": 186 |
| } |
| } |
| } |
| }, |
| "firewall": { |
| "n": 2000, |
| "accuracy": 0.889, |
| "nll": 0.3097366738639539, |
| "brier": 0.1801932942780604, |
| "ece": 0.02010454627871515, |
| "mean_confidence": 0.8843429988920689, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.2130241394042969, |
| "groups": { |
| "choice:2": { |
| "temperature": 1.131178691354451, |
| "raw_temperature": 0.9999842643737793, |
| "n": 68, |
| "shrinkage_weight": 0.3617021276595745 |
| }, |
| "choice:3-5": { |
| "temperature": 1.229406620643328, |
| "raw_temperature": 1.2313735485076904, |
| "n": 1007, |
| "shrinkage_weight": 0.8935226264418811 |
| }, |
| "choice:6-10": { |
| "temperature": 1.1287873284307708, |
| "raw_temperature": 1.0643781423568726, |
| "n": 147, |
| "shrinkage_weight": 0.550561797752809 |
| }, |
| "noul:*": { |
| "temperature": 1.2522414435081368, |
| "raw_temperature": 1.259989619255066, |
| "n": 619, |
| "shrinkage_weight": 0.8376184032476319 |
| }, |
| "score:*": { |
| "temperature": 1.1708337953943981, |
| "raw_temperature": 1.1399664878845215, |
| "n": 159, |
| "shrinkage_weight": 0.5698924731182796 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "firewall/tommccoy1-hans": { |
| "n": 2000, |
| "accuracy": 0.889, |
| "nll": 0.3097366738639539, |
| "brier": 0.1801932942780604, |
| "ece": 0.02010454627871515, |
| "mean_confidence": 0.8843429988920689 |
| } |
| }, |
| "by_kind": { |
| "noul": { |
| "n": 2000, |
| "accuracy": 0.889, |
| "nll": 0.3097366738639539, |
| "brier": 0.1801932942780604, |
| "ece": 0.02010454627871515, |
| "mean_confidence": 0.8843429988920689 |
| } |
| }, |
| "by_transform": { |
| "nli.firewall.hans.v1": { |
| "n": 2000, |
| "accuracy": 0.889, |
| "nll": 0.3097366738639539, |
| "brier": 0.1801932942780604, |
| "ece": 0.02010454627871515, |
| "mean_confidence": 0.8843429988920689 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 2000, |
| "accuracy": 0.889, |
| "nll": 0.3097366738639539, |
| "brier": 0.1801932942780604, |
| "ece": 0.02010454627871515, |
| "mean_confidence": 0.8843429988920689 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 2000, |
| "accuracy": 0.889, |
| "nll": 0.3097366738639539, |
| "brier": 0.1801932942780604, |
| "ece": 0.02010454627871515, |
| "mean_confidence": 0.8843429988920689 |
| } |
| }, |
| "by_dataset_truncation": { |
| "firewall/tommccoy1-hans|not_truncated": { |
| "n": 2000, |
| "accuracy": 0.889, |
| "nll": 0.3097366738639539, |
| "brier": 0.1801932942780604, |
| "ece": 0.02010454627871515, |
| "mean_confidence": 0.8843429988920689 |
| } |
| }, |
| "by_kind_truncation": { |
| "noul|not_truncated": { |
| "n": 2000, |
| "accuracy": 0.889, |
| "nll": 0.3097366738639539, |
| "brier": 0.1801932942780604, |
| "ece": 0.02010454627871515, |
| "mean_confidence": 0.8843429988920689 |
| } |
| } |
| }, |
| "confusability": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.0021607741066457364, |
| "brier": 3.6698297335017013e-05, |
| "ece": 0.0021507594840867515, |
| "mean_confidence": 0.9978492405159133, |
| "calibration": { |
| "version": 2, |
| "type": "primitive_option_temperature", |
| "global_temperature": 1.2130241394042969, |
| "groups": { |
| "choice:2": { |
| "temperature": 1.131178691354451, |
| "raw_temperature": 0.9999842643737793, |
| "n": 68, |
| "shrinkage_weight": 0.3617021276595745 |
| }, |
| "choice:3-5": { |
| "temperature": 1.229406620643328, |
| "raw_temperature": 1.2313735485076904, |
| "n": 1007, |
| "shrinkage_weight": 0.8935226264418811 |
| }, |
| "choice:6-10": { |
| "temperature": 1.1287873284307708, |
| "raw_temperature": 1.0643781423568726, |
| "n": 147, |
| "shrinkage_weight": 0.550561797752809 |
| }, |
| "noul:*": { |
| "temperature": 1.2522414435081368, |
| "raw_temperature": 1.259989619255066, |
| "n": 619, |
| "shrinkage_weight": 0.8376184032476319 |
| }, |
| "score:*": { |
| "temperature": 1.1708337953943981, |
| "raw_temperature": 1.1399664878845215, |
| "n": 159, |
| "shrinkage_weight": 0.5698924731182796 |
| } |
| }, |
| "min_group": 50, |
| "prior_strength": 120.0, |
| "n": 2000 |
| }, |
| "by_dataset": { |
| "v142-confusability": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.0021607741066457364, |
| "brier": 3.6698297335017013e-05, |
| "ece": 0.0021507594840867515, |
| "mean_confidence": 0.9978492405159133 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 1000, |
| "accuracy": 1.0, |
| "nll": 0.001977618149085174, |
| "brier": 4.1289017551536136e-05, |
| "ece": 0.001966866195201821, |
| "mean_confidence": 0.9980331338047981 |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0006506792725122068, |
| "brier": 9.928545260606826e-07, |
| "ece": 0.0006504309177398637, |
| "mean_confidence": 0.9993495690822601 |
| }, |
| "score": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.004586648728582077, |
| "brier": 4.945013906137774e-05, |
| "ece": 0.00457055449485777, |
| "mean_confidence": 0.9954294455051422 |
| } |
| }, |
| "by_transform": { |
| "coreference": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.00249042555813503, |
| "brier": 4.369134036505073e-05, |
| "ece": 0.002478986680507611, |
| "mean_confidence": 0.9975210133194924 |
| }, |
| "entity_binding": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0001403224484056409, |
| "brier": 3.782463963375449e-08, |
| "ece": 0.0001403102278709767, |
| "mean_confidence": 0.999859689772129 |
| }, |
| "negation": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0006506792725122068, |
| "brier": 9.928545260606826e-07, |
| "ece": 0.0006504309177398637, |
| "mean_confidence": 0.9993495690822601 |
| }, |
| "numeric_boundary": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0012902074388694018, |
| "brier": 4.61289219387595e-06, |
| "ece": 0.0012890523672104415, |
| "mean_confidence": 0.9987109476327896 |
| }, |
| "ordinal_boundary": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.004586648728582077, |
| "brier": 4.945013906137774e-05, |
| "ece": 0.00457055449485777, |
| "mean_confidence": 0.9954294455051422 |
| }, |
| "temporal_state": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.003792114706011489, |
| "brier": 3.8238258427440995e-05, |
| "ece": 0.0037821048498153376, |
| "mean_confidence": 0.9962178951501847 |
| }, |
| "tool_argument_validity": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.002175020594004309, |
| "brier": 0.00011986477213167924, |
| "ece": 0.0021438768506050277, |
| "mean_confidence": 0.997856123149395 |
| } |
| }, |
| "by_option_bucket": { |
| "2": { |
| "n": 400, |
| "accuracy": 1.0, |
| "nll": 0.0009704433556908043, |
| "brier": 2.8028733599683165e-06, |
| "ece": 0.0009697416424750971, |
| "mean_confidence": 0.9990302583575249 |
| }, |
| "3-5": { |
| "n": 1000, |
| "accuracy": 1.0, |
| "nll": 0.002636906407027709, |
| "brier": 5.025646692503649e-05, |
| "ece": 0.0026231666207314017, |
| "mean_confidence": 0.9973768333792686 |
| } |
| }, |
| "by_truncation": { |
| "not_truncated": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.0021607741066457364, |
| "brier": 3.6698297335017013e-05, |
| "ece": 0.0021507594840867515, |
| "mean_confidence": 0.9978492405159133 |
| } |
| }, |
| "by_dataset_truncation": { |
| "v142-confusability|not_truncated": { |
| "n": 1400, |
| "accuracy": 1.0, |
| "nll": 0.0021607741066457364, |
| "brier": 3.6698297335017013e-05, |
| "ece": 0.0021507594840867515, |
| "mean_confidence": 0.9978492405159133 |
| } |
| }, |
| "by_kind_truncation": { |
| "choice|not_truncated": { |
| "n": 1000, |
| "accuracy": 1.0, |
| "nll": 0.001977618149085174, |
| "brier": 4.1289017551536136e-05, |
| "ece": 0.001966866195201821, |
| "mean_confidence": 0.9980331338047981 |
| }, |
| "noul|not_truncated": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.0006506792725122068, |
| "brier": 9.928545260606826e-07, |
| "ece": 0.0006504309177398637, |
| "mean_confidence": 0.9993495690822601 |
| }, |
| "score|not_truncated": { |
| "n": 200, |
| "accuracy": 1.0, |
| "nll": 0.004586648728582077, |
| "brier": 4.945013906137774e-05, |
| "ece": 0.00457055449485777, |
| "mean_confidence": 0.9954294455051422 |
| } |
| } |
| } |
| }, |
| "latency": { |
| "validation": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 3659.3674976518378, |
| "effective_ms_per_example": 457.4209372064797, |
| "skipped_unusable": 0 |
| }, |
| "schema": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 4174.043139163288, |
| "effective_ms_per_example": 521.755392395411, |
| "skipped_unusable": 0 |
| }, |
| "firewall": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 336.11437101254705, |
| "effective_ms_per_example": 42.01429637656838, |
| "skipped_unusable": 0 |
| }, |
| "confusability": { |
| "eval_batch_size": 8, |
| "batch_latency_ms_mean": 658.7701784420226, |
| "effective_ms_per_example": 82.34627230525282, |
| "skipped_unusable": 0 |
| } |
| } |
| } |
| }, |
| "lora_mode": "auto" |
| } |
|
|
