| { |
| "schema": "kapi-hf-eval-summary/1", |
| "kind": "homologation_v1", |
| "label": "Internal evaluation, synthetic out-of-distribution homologation; LLM- and grammar-written, not written by people; not fully independent of the v3 training data. Aggregates only; no item text.", |
| "model": "esilva/KapiCoder-Mongo-Small", |
| "format": "homolog-v1-eval-1", |
| "set_sha256": "aca0aae51533a95d830c1f296d621f9203a7d76475658fb1e2dcc8f389495332", |
| "n": 6030, |
| "models": { |
| "base_1.5b": { |
| "overall": { |
| "n": 6030, |
| "exact_match": 0.1856, |
| "normalized_match": 0.1874, |
| "edit_similarity": 0.3732, |
| "accepted_prefix_tokens": 2.2488, |
| "syntax_valid": 0.6774, |
| "mongo_valid": 0.6048, |
| "exec_valid": 0.4915, |
| "ceiling_exec_valid": 0.9307, |
| "exec_over_ceiling": 0.5281, |
| "exec_given_ref_ok": 0.5276, |
| "n_ref_ok": 5612, |
| "nonexistent_fields_rate": 0.0818, |
| "empty_rate": 0.0141, |
| "normalized_match_ci95": [ |
| 0.1778, |
| 0.1974 |
| ], |
| "exec_over_ceiling_ci95": [ |
| 0.5151, |
| 0.5409 |
| ] |
| }, |
| "by_context_kind": { |
| "full": { |
| "n": 3720, |
| "exact_match": 0.1392, |
| "normalized_match": 0.1398, |
| "edit_similarity": 0.3221, |
| "accepted_prefix_tokens": 2.1505, |
| "syntax_valid": 0.6231, |
| "mongo_valid": 0.5567, |
| "exec_valid": 0.45, |
| "ceiling_exec_valid": 0.9237, |
| "exec_over_ceiling": 0.4872, |
| "exec_given_ref_ok": 0.4863, |
| "n_ref_ok": 3436, |
| "nonexistent_fields_rate": 0.0715, |
| "empty_rate": 0.0108 |
| }, |
| "irrelevant": { |
| "n": 1155, |
| "exact_match": 0.2442, |
| "normalized_match": 0.2485, |
| "edit_similarity": 0.43, |
| "accepted_prefix_tokens": 2.3446, |
| "syntax_valid": 0.7238, |
| "mongo_valid": 0.6442, |
| "exec_valid": 0.5342, |
| "ceiling_exec_valid": 0.9455, |
| "exec_over_ceiling": 0.565, |
| "exec_given_ref_ok": 0.565, |
| "n_ref_ok": 1092, |
| "nonexistent_fields_rate": 0.0961, |
| "empty_rate": 0.0208 |
| }, |
| "none": { |
| "n": 1155, |
| "exact_match": 0.2762, |
| "normalized_match": 0.2797, |
| "edit_similarity": 0.4809, |
| "accepted_prefix_tokens": 2.4693, |
| "syntax_valid": 0.8061, |
| "mongo_valid": 0.7203, |
| "exec_valid": 0.5827, |
| "ceiling_exec_valid": 0.9385, |
| "exec_over_ceiling": 0.6209, |
| "exec_given_ref_ok": 0.6208, |
| "n_ref_ok": 1084, |
| "nonexistent_fields_rate": 0.1004, |
| "empty_rate": 0.0182 |
| } |
| }, |
| "by_cut_kind": { |
| "key": { |
| "n": 959, |
| "exact_match": 0.1846, |
| "normalized_match": 0.1856, |
| "edit_similarity": 0.353, |
| "accepted_prefix_tokens": 1.4943, |
| "syntax_valid": 0.6486, |
| "mongo_valid": 0.6173, |
| "exec_valid": 0.5036, |
| "ceiling_exec_valid": 0.9166, |
| "exec_over_ceiling": 0.5494, |
| "exec_given_ref_ok": 0.5495, |
| "n_ref_ok": 879, |
| "nonexistent_fields_rate": 0.0563, |
| "empty_rate": 0.0 |
| }, |
| "member": { |
| "n": 600, |
| "exact_match": 0.4583, |
| "normalized_match": 0.4583, |
| "edit_similarity": 0.5483, |
| "accepted_prefix_tokens": 1.2783, |
| "syntax_valid": 0.85, |
| "mongo_valid": 0.6467, |
| "exec_valid": 0.575, |
| "ceiling_exec_valid": 0.89, |
| "exec_over_ceiling": 0.6461, |
| "exec_given_ref_ok": 0.6461, |
| "n_ref_ok": 534, |
| "nonexistent_fields_rate": 0.0767, |
| "empty_rate": 0.0 |
| }, |
| "multiline": { |
| "n": 480, |
| "exact_match": 0.0187, |
| "normalized_match": 0.025, |
| "edit_similarity": 0.3463, |
| "accepted_prefix_tokens": 3.7896, |
| "syntax_valid": 0.6167, |
| "mongo_valid": 0.5542, |
| "exec_valid": 0.4188, |
| "ceiling_exec_valid": 0.9583, |
| "exec_over_ceiling": 0.437, |
| "exec_given_ref_ok": 0.437, |
| "n_ref_ok": 460, |
| "nonexistent_fields_rate": 0.1437, |
| "empty_rate": 0.0 |
| }, |
| "operator": { |
| "n": 841, |
| "exact_match": 0.2331, |
| "normalized_match": 0.2342, |
| "edit_similarity": 0.3963, |
| "accepted_prefix_tokens": 2.4388, |
| "syntax_valid": 0.6338, |
| "mongo_valid": 0.5945, |
| "exec_valid": 0.5125, |
| "ceiling_exec_valid": 0.9489, |
| "exec_over_ceiling": 0.5401, |
| "exec_given_ref_ok": 0.5401, |
| "n_ref_ok": 798, |
| "nonexistent_fields_rate": 0.0559, |
| "empty_rate": 0.0 |
| }, |
| "partial": { |
| "n": 600, |
| "exact_match": 0.3333, |
| "normalized_match": 0.3333, |
| "edit_similarity": 0.4636, |
| "accepted_prefix_tokens": 2.0317, |
| "syntax_valid": 0.72, |
| "mongo_valid": 0.6367, |
| "exec_valid": 0.4717, |
| "ceiling_exec_valid": 0.91, |
| "exec_over_ceiling": 0.5184, |
| "exec_given_ref_ok": 0.5183, |
| "n_ref_ok": 546, |
| "nonexistent_fields_rate": 0.1217, |
| "empty_rate": 0.01 |
| }, |
| "rest": { |
| "n": 720, |
| "exact_match": 0.0778, |
| "normalized_match": 0.0778, |
| "edit_similarity": 0.2764, |
| "accepted_prefix_tokens": 2.2569, |
| "syntax_valid": 0.7056, |
| "mongo_valid": 0.6028, |
| "exec_valid": 0.475, |
| "ceiling_exec_valid": 0.9319, |
| "exec_over_ceiling": 0.5097, |
| "exec_given_ref_ok": 0.5097, |
| "n_ref_ok": 671, |
| "nonexistent_fields_rate": 0.0528, |
| "empty_rate": 0.1069 |
| }, |
| "stage": { |
| "n": 720, |
| "exact_match": 0.0222, |
| "normalized_match": 0.0306, |
| "edit_similarity": 0.3191, |
| "accepted_prefix_tokens": 3.5264, |
| "syntax_valid": 0.6056, |
| "mongo_valid": 0.5903, |
| "exec_valid": 0.4139, |
| "ceiling_exec_valid": 0.9486, |
| "exec_over_ceiling": 0.4363, |
| "exec_given_ref_ok": 0.4363, |
| "n_ref_ok": 683, |
| "nonexistent_fields_rate": 0.1361, |
| "empty_rate": 0.0028 |
| }, |
| "stmt_start": { |
| "n": 269, |
| "exact_match": 0.1227, |
| "normalized_match": 0.1227, |
| "edit_similarity": 0.4425, |
| "accepted_prefix_tokens": 2.6729, |
| "syntax_valid": 0.8439, |
| "mongo_valid": 0.6208, |
| "exec_valid": 0.6283, |
| "ceiling_exec_valid": 0.9777, |
| "exec_over_ceiling": 0.6426, |
| "exec_given_ref_ok": 0.6312, |
| "n_ref_ok": 263, |
| "nonexistent_fields_rate": 0.1004, |
| "empty_rate": 0.0 |
| }, |
| "value": { |
| "n": 841, |
| "exact_match": 0.1867, |
| "normalized_match": 0.1867, |
| "edit_similarity": 0.306, |
| "accepted_prefix_tokens": 1.6504, |
| "syntax_valid": 0.6195, |
| "mongo_valid": 0.5862, |
| "exec_valid": 0.4899, |
| "ceiling_exec_valid": 0.9251, |
| "exec_over_ceiling": 0.5296, |
| "exec_given_ref_ok": 0.5296, |
| "n_ref_ok": 778, |
| "nonexistent_fields_rate": 0.0488, |
| "empty_rate": 0.0 |
| } |
| }, |
| "by_domain": { |
| "adtech": { |
| "n": 3057, |
| "exact_match": 0.1835, |
| "normalized_match": 0.1861, |
| "edit_similarity": 0.3734, |
| "accepted_prefix_tokens": 2.2512, |
| "syntax_valid": 0.6817, |
| "mongo_valid": 0.6042, |
| "exec_valid": 0.4953, |
| "ceiling_exec_valid": 0.9303, |
| "exec_over_ceiling": 0.5324, |
| "exec_given_ref_ok": 0.5313, |
| "n_ref_ok": 2844, |
| "nonexistent_fields_rate": 0.0818, |
| "empty_rate": 0.0144 |
| }, |
| "telecom": { |
| "n": 2973, |
| "exact_match": 0.1877, |
| "normalized_match": 0.1887, |
| "edit_similarity": 0.373, |
| "accepted_prefix_tokens": 2.2462, |
| "syntax_valid": 0.6731, |
| "mongo_valid": 0.6054, |
| "exec_valid": 0.4877, |
| "ceiling_exec_valid": 0.931, |
| "exec_over_ceiling": 0.5238, |
| "exec_given_ref_ok": 0.5238, |
| "n_ref_ok": 2768, |
| "nonexistent_fields_rate": 0.0817, |
| "empty_rate": 0.0138 |
| } |
| }, |
| "by_author": { |
| "grammar": { |
| "n": 3220, |
| "exact_match": 0.1261, |
| "normalized_match": 0.1261, |
| "edit_similarity": 0.2908, |
| "accepted_prefix_tokens": 1.8161, |
| "syntax_valid": 0.5894, |
| "mongo_valid": 0.5509, |
| "exec_valid": 0.3829, |
| "ceiling_exec_valid": 0.8702, |
| "exec_over_ceiling": 0.44, |
| "exec_given_ref_ok": 0.439, |
| "n_ref_ok": 2802, |
| "nonexistent_fields_rate": 0.0761, |
| "empty_rate": 0.0081 |
| }, |
| "llm": { |
| "n": 2810, |
| "exact_match": 0.2537, |
| "normalized_match": 0.2577, |
| "edit_similarity": 0.4676, |
| "accepted_prefix_tokens": 2.7445, |
| "syntax_valid": 0.7783, |
| "mongo_valid": 0.6665, |
| "exec_valid": 0.616, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.616, |
| "exec_given_ref_ok": 0.616, |
| "n_ref_ok": 2810, |
| "nonexistent_fields_rate": 0.0883, |
| "empty_rate": 0.021 |
| } |
| }, |
| "by_author_context": { |
| "grammar/full": { |
| "n": 2121, |
| "exact_match": 0.099, |
| "normalized_match": 0.099, |
| "edit_similarity": 0.2651, |
| "accepted_prefix_tokens": 1.9043, |
| "syntax_valid": 0.5413, |
| "mongo_valid": 0.5101, |
| "exec_valid": 0.3446, |
| "ceiling_exec_valid": 0.8661, |
| "exec_over_ceiling": 0.3979, |
| "exec_given_ref_ok": 0.3963, |
| "n_ref_ok": 1837, |
| "nonexistent_fields_rate": 0.0707, |
| "empty_rate": 0.0057 |
| }, |
| "grammar/irrelevant": { |
| "n": 553, |
| "exact_match": 0.1374, |
| "normalized_match": 0.1374, |
| "edit_similarity": 0.2987, |
| "accepted_prefix_tokens": 1.5443, |
| "syntax_valid": 0.6239, |
| "mongo_valid": 0.5714, |
| "exec_valid": 0.4014, |
| "ceiling_exec_valid": 0.8861, |
| "exec_over_ceiling": 0.453, |
| "exec_given_ref_ok": 0.4531, |
| "n_ref_ok": 490, |
| "nonexistent_fields_rate": 0.0922, |
| "empty_rate": 0.0163 |
| }, |
| "grammar/none": { |
| "n": 546, |
| "exact_match": 0.2198, |
| "normalized_match": 0.2198, |
| "edit_similarity": 0.3827, |
| "accepted_prefix_tokens": 1.7491, |
| "syntax_valid": 0.7418, |
| "mongo_valid": 0.6886, |
| "exec_valid": 0.5128, |
| "ceiling_exec_valid": 0.87, |
| "exec_over_ceiling": 0.5894, |
| "exec_given_ref_ok": 0.5895, |
| "n_ref_ok": 475, |
| "nonexistent_fields_rate": 0.0806, |
| "empty_rate": 0.0092 |
| }, |
| "llm/full": { |
| "n": 1599, |
| "exact_match": 0.1926, |
| "normalized_match": 0.1939, |
| "edit_similarity": 0.3977, |
| "accepted_prefix_tokens": 2.4772, |
| "syntax_valid": 0.7317, |
| "mongo_valid": 0.6185, |
| "exec_valid": 0.5897, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.5897, |
| "exec_given_ref_ok": 0.5897, |
| "n_ref_ok": 1599, |
| "nonexistent_fields_rate": 0.0725, |
| "empty_rate": 0.0175 |
| }, |
| "llm/irrelevant": { |
| "n": 602, |
| "exact_match": 0.3422, |
| "normalized_match": 0.3505, |
| "edit_similarity": 0.5507, |
| "accepted_prefix_tokens": 3.0797, |
| "syntax_valid": 0.8156, |
| "mongo_valid": 0.711, |
| "exec_valid": 0.6561, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.6561, |
| "exec_given_ref_ok": 0.6561, |
| "n_ref_ok": 602, |
| "nonexistent_fields_rate": 0.0997, |
| "empty_rate": 0.0249 |
| }, |
| "llm/none": { |
| "n": 609, |
| "exact_match": 0.3268, |
| "normalized_match": 0.3333, |
| "edit_similarity": 0.5689, |
| "accepted_prefix_tokens": 3.1149, |
| "syntax_valid": 0.8637, |
| "mongo_valid": 0.7488, |
| "exec_valid": 0.6453, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.6453, |
| "exec_given_ref_ok": 0.6453, |
| "n_ref_ok": 609, |
| "nonexistent_fields_rate": 0.1182, |
| "empty_rate": 0.0263 |
| } |
| } |
| }, |
| "t1_distill": { |
| "overall": { |
| "n": 6030, |
| "exact_match": 0.3687, |
| "normalized_match": 0.3697, |
| "edit_similarity": 0.6188, |
| "accepted_prefix_tokens": 2.7891, |
| "syntax_valid": 0.9902, |
| "mongo_valid": 0.962, |
| "exec_valid": 0.7859, |
| "ceiling_exec_valid": 0.9307, |
| "exec_over_ceiling": 0.8444, |
| "exec_given_ref_ok": 0.8437, |
| "n_ref_ok": 5612, |
| "nonexistent_fields_rate": 0.1007, |
| "empty_rate": 0.0017, |
| "normalized_match_ci95": [ |
| 0.3576, |
| 0.382 |
| ], |
| "exec_over_ceiling_ci95": [ |
| 0.8342, |
| 0.854 |
| ] |
| }, |
| "by_context_kind": { |
| "full": { |
| "n": 3720, |
| "exact_match": 0.3151, |
| "normalized_match": 0.3153, |
| "edit_similarity": 0.5746, |
| "accepted_prefix_tokens": 2.7946, |
| "syntax_valid": 0.9892, |
| "mongo_valid": 0.9613, |
| "exec_valid": 0.7879, |
| "ceiling_exec_valid": 0.9237, |
| "exec_over_ceiling": 0.853, |
| "exec_given_ref_ok": 0.8519, |
| "n_ref_ok": 3436, |
| "nonexistent_fields_rate": 0.0919, |
| "empty_rate": 0.0022 |
| }, |
| "irrelevant": { |
| "n": 1155, |
| "exact_match": 0.4511, |
| "normalized_match": 0.4545, |
| "edit_similarity": 0.689, |
| "accepted_prefix_tokens": 2.7307, |
| "syntax_valid": 0.9913, |
| "mongo_valid": 0.9645, |
| "exec_valid": 0.7931, |
| "ceiling_exec_valid": 0.9455, |
| "exec_over_ceiling": 0.8388, |
| "exec_given_ref_ok": 0.8388, |
| "n_ref_ok": 1092, |
| "nonexistent_fields_rate": 0.1134, |
| "empty_rate": 0.0 |
| }, |
| "none": { |
| "n": 1155, |
| "exact_match": 0.4589, |
| "normalized_match": 0.4597, |
| "edit_similarity": 0.6908, |
| "accepted_prefix_tokens": 2.8294, |
| "syntax_valid": 0.9922, |
| "mongo_valid": 0.9619, |
| "exec_valid": 0.7723, |
| "ceiling_exec_valid": 0.9385, |
| "exec_over_ceiling": 0.8229, |
| "exec_given_ref_ok": 0.8229, |
| "n_ref_ok": 1084, |
| "nonexistent_fields_rate": 0.116, |
| "empty_rate": 0.0017 |
| } |
| }, |
| "by_cut_kind": { |
| "key": { |
| "n": 959, |
| "exact_match": 0.3566, |
| "normalized_match": 0.3566, |
| "edit_similarity": 0.5737, |
| "accepted_prefix_tokens": 1.8749, |
| "syntax_valid": 0.9958, |
| "mongo_valid": 0.9698, |
| "exec_valid": 0.7737, |
| "ceiling_exec_valid": 0.9166, |
| "exec_over_ceiling": 0.8441, |
| "exec_given_ref_ok": 0.8441, |
| "n_ref_ok": 879, |
| "nonexistent_fields_rate": 0.1157, |
| "empty_rate": 0.0 |
| }, |
| "member": { |
| "n": 600, |
| "exact_match": 0.785, |
| "normalized_match": 0.785, |
| "edit_similarity": 0.8609, |
| "accepted_prefix_tokens": 1.95, |
| "syntax_valid": 0.9917, |
| "mongo_valid": 0.9567, |
| "exec_valid": 0.8233, |
| "ceiling_exec_valid": 0.89, |
| "exec_over_ceiling": 0.9251, |
| "exec_given_ref_ok": 0.9251, |
| "n_ref_ok": 534, |
| "nonexistent_fields_rate": 0.0417, |
| "empty_rate": 0.0 |
| }, |
| "multiline": { |
| "n": 480, |
| "exact_match": 0.0229, |
| "normalized_match": 0.0271, |
| "edit_similarity": 0.4367, |
| "accepted_prefix_tokens": 4.6229, |
| "syntax_valid": 0.9917, |
| "mongo_valid": 0.9521, |
| "exec_valid": 0.65, |
| "ceiling_exec_valid": 0.9583, |
| "exec_over_ceiling": 0.6783, |
| "exec_given_ref_ok": 0.6783, |
| "n_ref_ok": 460, |
| "nonexistent_fields_rate": 0.25, |
| "empty_rate": 0.0 |
| }, |
| "operator": { |
| "n": 841, |
| "exact_match": 0.4554, |
| "normalized_match": 0.4554, |
| "edit_similarity": 0.6871, |
| "accepted_prefix_tokens": 2.9382, |
| "syntax_valid": 0.9881, |
| "mongo_valid": 0.9584, |
| "exec_valid": 0.8502, |
| "ceiling_exec_valid": 0.9489, |
| "exec_over_ceiling": 0.896, |
| "exec_given_ref_ok": 0.896, |
| "n_ref_ok": 798, |
| "nonexistent_fields_rate": 0.0606, |
| "empty_rate": 0.0 |
| }, |
| "partial": { |
| "n": 600, |
| "exact_match": 0.6083, |
| "normalized_match": 0.6083, |
| "edit_similarity": 0.7702, |
| "accepted_prefix_tokens": 2.38, |
| "syntax_valid": 0.9933, |
| "mongo_valid": 0.9633, |
| "exec_valid": 0.735, |
| "ceiling_exec_valid": 0.91, |
| "exec_over_ceiling": 0.8077, |
| "exec_given_ref_ok": 0.8077, |
| "n_ref_ok": 546, |
| "nonexistent_fields_rate": 0.1333, |
| "empty_rate": 0.0017 |
| }, |
| "rest": { |
| "n": 720, |
| "exact_match": 0.2875, |
| "normalized_match": 0.2875, |
| "edit_similarity": 0.5632, |
| "accepted_prefix_tokens": 2.6264, |
| "syntax_valid": 0.9792, |
| "mongo_valid": 0.9458, |
| "exec_valid": 0.7556, |
| "ceiling_exec_valid": 0.9319, |
| "exec_over_ceiling": 0.8108, |
| "exec_given_ref_ok": 0.8107, |
| "n_ref_ok": 671, |
| "nonexistent_fields_rate": 0.0764, |
| "empty_rate": 0.0097 |
| }, |
| "stage": { |
| "n": 720, |
| "exact_match": 0.0347, |
| "normalized_match": 0.0403, |
| "edit_similarity": 0.4278, |
| "accepted_prefix_tokens": 4.6514, |
| "syntax_valid": 0.9875, |
| "mongo_valid": 0.9778, |
| "exec_valid": 0.7264, |
| "ceiling_exec_valid": 0.9486, |
| "exec_over_ceiling": 0.7658, |
| "exec_given_ref_ok": 0.7657, |
| "n_ref_ok": 683, |
| "nonexistent_fields_rate": 0.1583, |
| "empty_rate": 0.0028 |
| }, |
| "stmt_start": { |
| "n": 269, |
| "exact_match": 0.2379, |
| "normalized_match": 0.2379, |
| "edit_similarity": 0.5815, |
| "accepted_prefix_tokens": 3.7212, |
| "syntax_valid": 0.9777, |
| "mongo_valid": 0.9368, |
| "exec_valid": 0.7732, |
| "ceiling_exec_valid": 0.9777, |
| "exec_over_ceiling": 0.7908, |
| "exec_given_ref_ok": 0.7757, |
| "n_ref_ok": 263, |
| "nonexistent_fields_rate": 0.1561, |
| "empty_rate": 0.0 |
| }, |
| "value": { |
| "n": 841, |
| "exact_match": 0.4221, |
| "normalized_match": 0.4221, |
| "edit_similarity": 0.6483, |
| "accepted_prefix_tokens": 1.7729, |
| "syntax_valid": 0.9976, |
| "mongo_valid": 0.9738, |
| "exec_valid": 0.9037, |
| "ceiling_exec_valid": 0.9251, |
| "exec_over_ceiling": 0.9769, |
| "exec_given_ref_ok": 0.9769, |
| "n_ref_ok": 778, |
| "nonexistent_fields_rate": 0.0107, |
| "empty_rate": 0.0 |
| } |
| }, |
| "by_domain": { |
| "adtech": { |
| "n": 3057, |
| "exact_match": 0.3749, |
| "normalized_match": 0.3759, |
| "edit_similarity": 0.6244, |
| "accepted_prefix_tokens": 2.8911, |
| "syntax_valid": 0.9908, |
| "mongo_valid": 0.9627, |
| "exec_valid": 0.7975, |
| "ceiling_exec_valid": 0.9303, |
| "exec_over_ceiling": 0.8573, |
| "exec_given_ref_ok": 0.8558, |
| "n_ref_ok": 2844, |
| "nonexistent_fields_rate": 0.0932, |
| "empty_rate": 0.002 |
| }, |
| "telecom": { |
| "n": 2973, |
| "exact_match": 0.3623, |
| "normalized_match": 0.3633, |
| "edit_similarity": 0.613, |
| "accepted_prefix_tokens": 2.6842, |
| "syntax_valid": 0.9896, |
| "mongo_valid": 0.9613, |
| "exec_valid": 0.774, |
| "ceiling_exec_valid": 0.931, |
| "exec_over_ceiling": 0.8314, |
| "exec_given_ref_ok": 0.8313, |
| "n_ref_ok": 2768, |
| "nonexistent_fields_rate": 0.1083, |
| "empty_rate": 0.0013 |
| } |
| }, |
| "by_author": { |
| "grammar": { |
| "n": 3220, |
| "exact_match": 0.3758, |
| "normalized_match": 0.3761, |
| "edit_similarity": 0.633, |
| "accepted_prefix_tokens": 3.027, |
| "syntax_valid": 0.9975, |
| "mongo_valid": 0.9969, |
| "exec_valid": 0.7898, |
| "ceiling_exec_valid": 0.8702, |
| "exec_over_ceiling": 0.9076, |
| "exec_given_ref_ok": 0.9061, |
| "n_ref_ok": 2802, |
| "nonexistent_fields_rate": 0.0724, |
| "empty_rate": 0.0 |
| }, |
| "llm": { |
| "n": 2810, |
| "exact_match": 0.3605, |
| "normalized_match": 0.3623, |
| "edit_similarity": 0.6025, |
| "accepted_prefix_tokens": 2.5164, |
| "syntax_valid": 0.9819, |
| "mongo_valid": 0.9221, |
| "exec_valid": 0.7815, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.7815, |
| "exec_given_ref_ok": 0.7815, |
| "n_ref_ok": 2810, |
| "nonexistent_fields_rate": 0.1331, |
| "empty_rate": 0.0036 |
| } |
| }, |
| "by_author_context": { |
| "grammar/full": { |
| "n": 2121, |
| "exact_match": 0.3215, |
| "normalized_match": 0.322, |
| "edit_similarity": 0.5933, |
| "accepted_prefix_tokens": 3.1061, |
| "syntax_valid": 0.9972, |
| "mongo_valid": 0.9972, |
| "exec_valid": 0.7845, |
| "ceiling_exec_valid": 0.8661, |
| "exec_over_ceiling": 0.9058, |
| "exec_given_ref_ok": 0.9036, |
| "n_ref_ok": 1837, |
| "nonexistent_fields_rate": 0.0726, |
| "empty_rate": 0.0 |
| }, |
| "grammar/irrelevant": { |
| "n": 553, |
| "exact_match": 0.4629, |
| "normalized_match": 0.4629, |
| "edit_similarity": 0.6965, |
| "accepted_prefix_tokens": 2.8101, |
| "syntax_valid": 0.9982, |
| "mongo_valid": 0.9964, |
| "exec_valid": 0.8065, |
| "ceiling_exec_valid": 0.8861, |
| "exec_over_ceiling": 0.9102, |
| "exec_given_ref_ok": 0.9102, |
| "n_ref_ok": 490, |
| "nonexistent_fields_rate": 0.0687, |
| "empty_rate": 0.0 |
| }, |
| "grammar/none": { |
| "n": 546, |
| "exact_match": 0.4982, |
| "normalized_match": 0.4982, |
| "edit_similarity": 0.7231, |
| "accepted_prefix_tokens": 2.9396, |
| "syntax_valid": 0.9982, |
| "mongo_valid": 0.9963, |
| "exec_valid": 0.793, |
| "ceiling_exec_valid": 0.87, |
| "exec_over_ceiling": 0.9115, |
| "exec_given_ref_ok": 0.9116, |
| "n_ref_ok": 475, |
| "nonexistent_fields_rate": 0.0751, |
| "empty_rate": 0.0 |
| }, |
| "llm/full": { |
| "n": 1599, |
| "exact_match": 0.3064, |
| "normalized_match": 0.3064, |
| "edit_similarity": 0.5499, |
| "accepted_prefix_tokens": 2.3815, |
| "syntax_valid": 0.9787, |
| "mongo_valid": 0.9137, |
| "exec_valid": 0.7924, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.7924, |
| "exec_given_ref_ok": 0.7924, |
| "n_ref_ok": 1599, |
| "nonexistent_fields_rate": 0.1176, |
| "empty_rate": 0.005 |
| }, |
| "llm/irrelevant": { |
| "n": 602, |
| "exact_match": 0.4402, |
| "normalized_match": 0.4468, |
| "edit_similarity": 0.6822, |
| "accepted_prefix_tokens": 2.6578, |
| "syntax_valid": 0.985, |
| "mongo_valid": 0.9352, |
| "exec_valid": 0.7807, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.7807, |
| "exec_given_ref_ok": 0.7807, |
| "n_ref_ok": 602, |
| "nonexistent_fields_rate": 0.1545, |
| "empty_rate": 0.0 |
| }, |
| "llm/none": { |
| "n": 609, |
| "exact_match": 0.4236, |
| "normalized_match": 0.4253, |
| "edit_similarity": 0.6618, |
| "accepted_prefix_tokens": 2.7307, |
| "syntax_valid": 0.9869, |
| "mongo_valid": 0.931, |
| "exec_valid": 0.7537, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.7537, |
| "exec_given_ref_ok": 0.7537, |
| "n_ref_ok": 609, |
| "nonexistent_fields_rate": 0.1527, |
| "empty_rate": 0.0033 |
| } |
| } |
| }, |
| "t1_v3": { |
| "overall": { |
| "n": 6030, |
| "exact_match": 0.4516, |
| "normalized_match": 0.4537, |
| "edit_similarity": 0.7011, |
| "accepted_prefix_tokens": 3.8992, |
| "syntax_valid": 0.9972, |
| "mongo_valid": 0.9718, |
| "exec_valid": 0.8307, |
| "ceiling_exec_valid": 0.9307, |
| "exec_over_ceiling": 0.8926, |
| "exec_given_ref_ok": 0.892, |
| "n_ref_ok": 5612, |
| "nonexistent_fields_rate": 0.0788, |
| "empty_rate": 0.0, |
| "normalized_match_ci95": [ |
| 0.4412, |
| 0.4663 |
| ], |
| "exec_over_ceiling_ci95": [ |
| 0.8839, |
| 0.9007 |
| ] |
| }, |
| "by_context_kind": { |
| "full": { |
| "n": 3720, |
| "exact_match": 0.3812, |
| "normalized_match": 0.3828, |
| "edit_similarity": 0.6471, |
| "accepted_prefix_tokens": 3.7911, |
| "syntax_valid": 0.9965, |
| "mongo_valid": 0.9704, |
| "exec_valid": 0.8204, |
| "ceiling_exec_valid": 0.9237, |
| "exec_over_ceiling": 0.8882, |
| "exec_given_ref_ok": 0.8874, |
| "n_ref_ok": 3436, |
| "nonexistent_fields_rate": 0.075, |
| "empty_rate": 0.0 |
| }, |
| "irrelevant": { |
| "n": 1155, |
| "exact_match": 0.5524, |
| "normalized_match": 0.555, |
| "edit_similarity": 0.782, |
| "accepted_prefix_tokens": 3.8952, |
| "syntax_valid": 0.9965, |
| "mongo_valid": 0.974, |
| "exec_valid": 0.8511, |
| "ceiling_exec_valid": 0.9455, |
| "exec_over_ceiling": 0.9002, |
| "exec_given_ref_ok": 0.9002, |
| "n_ref_ok": 1092, |
| "nonexistent_fields_rate": 0.0823, |
| "empty_rate": 0.0 |
| }, |
| "none": { |
| "n": 1155, |
| "exact_match": 0.5775, |
| "normalized_match": 0.581, |
| "edit_similarity": 0.7941, |
| "accepted_prefix_tokens": 4.2511, |
| "syntax_valid": 1.0, |
| "mongo_valid": 0.974, |
| "exec_valid": 0.8433, |
| "ceiling_exec_valid": 0.9385, |
| "exec_over_ceiling": 0.8986, |
| "exec_given_ref_ok": 0.8985, |
| "n_ref_ok": 1084, |
| "nonexistent_fields_rate": 0.0874, |
| "empty_rate": 0.0 |
| } |
| }, |
| "by_cut_kind": { |
| "key": { |
| "n": 959, |
| "exact_match": 0.439, |
| "normalized_match": 0.44, |
| "edit_similarity": 0.6624, |
| "accepted_prefix_tokens": 2.4703, |
| "syntax_valid": 0.999, |
| "mongo_valid": 0.975, |
| "exec_valid": 0.8373, |
| "ceiling_exec_valid": 0.9166, |
| "exec_over_ceiling": 0.9135, |
| "exec_given_ref_ok": 0.9135, |
| "n_ref_ok": 879, |
| "nonexistent_fields_rate": 0.0688, |
| "empty_rate": 0.0 |
| }, |
| "member": { |
| "n": 600, |
| "exact_match": 0.835, |
| "normalized_match": 0.835, |
| "edit_similarity": 0.895, |
| "accepted_prefix_tokens": 2.17, |
| "syntax_valid": 0.9983, |
| "mongo_valid": 0.9617, |
| "exec_valid": 0.8267, |
| "ceiling_exec_valid": 0.89, |
| "exec_over_ceiling": 0.9289, |
| "exec_given_ref_ok": 0.9288, |
| "n_ref_ok": 534, |
| "nonexistent_fields_rate": 0.04, |
| "empty_rate": 0.0 |
| }, |
| "multiline": { |
| "n": 480, |
| "exact_match": 0.1062, |
| "normalized_match": 0.1125, |
| "edit_similarity": 0.5706, |
| "accepted_prefix_tokens": 8.2354, |
| "syntax_valid": 0.9812, |
| "mongo_valid": 0.9646, |
| "exec_valid": 0.7021, |
| "ceiling_exec_valid": 0.9583, |
| "exec_over_ceiling": 0.7327, |
| "exec_given_ref_ok": 0.7326, |
| "n_ref_ok": 460, |
| "nonexistent_fields_rate": 0.1896, |
| "empty_rate": 0.0 |
| }, |
| "operator": { |
| "n": 841, |
| "exact_match": 0.5743, |
| "normalized_match": 0.5755, |
| "edit_similarity": 0.7736, |
| "accepted_prefix_tokens": 3.8359, |
| "syntax_valid": 1.0, |
| "mongo_valid": 0.9762, |
| "exec_valid": 0.8859, |
| "ceiling_exec_valid": 0.9489, |
| "exec_over_ceiling": 0.9336, |
| "exec_given_ref_ok": 0.9336, |
| "n_ref_ok": 798, |
| "nonexistent_fields_rate": 0.0523, |
| "empty_rate": 0.0 |
| }, |
| "partial": { |
| "n": 600, |
| "exact_match": 0.6567, |
| "normalized_match": 0.6567, |
| "edit_similarity": 0.8213, |
| "accepted_prefix_tokens": 2.8333, |
| "syntax_valid": 0.9983, |
| "mongo_valid": 0.9717, |
| "exec_valid": 0.7817, |
| "ceiling_exec_valid": 0.91, |
| "exec_over_ceiling": 0.859, |
| "exec_given_ref_ok": 0.859, |
| "n_ref_ok": 546, |
| "nonexistent_fields_rate": 0.1133, |
| "empty_rate": 0.0 |
| }, |
| "rest": { |
| "n": 720, |
| "exact_match": 0.4, |
| "normalized_match": 0.4, |
| "edit_similarity": 0.6704, |
| "accepted_prefix_tokens": 4.0847, |
| "syntax_valid": 1.0, |
| "mongo_valid": 0.9667, |
| "exec_valid": 0.8375, |
| "ceiling_exec_valid": 0.9319, |
| "exec_over_ceiling": 0.8987, |
| "exec_given_ref_ok": 0.8987, |
| "n_ref_ok": 671, |
| "nonexistent_fields_rate": 0.0681, |
| "empty_rate": 0.0 |
| }, |
| "stage": { |
| "n": 720, |
| "exact_match": 0.0958, |
| "normalized_match": 0.1056, |
| "edit_similarity": 0.5092, |
| "accepted_prefix_tokens": 6.6472, |
| "syntax_valid": 0.9958, |
| "mongo_valid": 0.9847, |
| "exec_valid": 0.7778, |
| "ceiling_exec_valid": 0.9486, |
| "exec_over_ceiling": 0.8199, |
| "exec_given_ref_ok": 0.8199, |
| "n_ref_ok": 683, |
| "nonexistent_fields_rate": 0.1389, |
| "empty_rate": 0.0 |
| }, |
| "stmt_start": { |
| "n": 269, |
| "exact_match": 0.368, |
| "normalized_match": 0.3717, |
| "edit_similarity": 0.7071, |
| "accepted_prefix_tokens": 4.8662, |
| "syntax_valid": 0.9963, |
| "mongo_valid": 0.9517, |
| "exec_valid": 0.8699, |
| "ceiling_exec_valid": 0.9777, |
| "exec_over_ceiling": 0.8897, |
| "exec_given_ref_ok": 0.8783, |
| "n_ref_ok": 263, |
| "nonexistent_fields_rate": 0.0892, |
| "empty_rate": 0.0 |
| }, |
| "value": { |
| "n": 841, |
| "exact_match": 0.4958, |
| "normalized_match": 0.4958, |
| "edit_similarity": 0.7118, |
| "accepted_prefix_tokens": 2.2901, |
| "syntax_valid": 0.9988, |
| "mongo_valid": 0.975, |
| "exec_valid": 0.9061, |
| "ceiling_exec_valid": 0.9251, |
| "exec_over_ceiling": 0.9795, |
| "exec_given_ref_ok": 0.9794, |
| "n_ref_ok": 778, |
| "nonexistent_fields_rate": 0.0107, |
| "empty_rate": 0.0 |
| } |
| }, |
| "by_domain": { |
| "adtech": { |
| "n": 3057, |
| "exact_match": 0.4544, |
| "normalized_match": 0.4576, |
| "edit_similarity": 0.7054, |
| "accepted_prefix_tokens": 4.0203, |
| "syntax_valid": 0.9974, |
| "mongo_valid": 0.9702, |
| "exec_valid": 0.8328, |
| "ceiling_exec_valid": 0.9303, |
| "exec_over_ceiling": 0.8952, |
| "exec_given_ref_ok": 0.8942, |
| "n_ref_ok": 2844, |
| "nonexistent_fields_rate": 0.0772, |
| "empty_rate": 0.0 |
| }, |
| "telecom": { |
| "n": 2973, |
| "exact_match": 0.4487, |
| "normalized_match": 0.4497, |
| "edit_similarity": 0.6967, |
| "accepted_prefix_tokens": 3.7746, |
| "syntax_valid": 0.997, |
| "mongo_valid": 0.9734, |
| "exec_valid": 0.8285, |
| "ceiling_exec_valid": 0.931, |
| "exec_over_ceiling": 0.8899, |
| "exec_given_ref_ok": 0.8898, |
| "n_ref_ok": 2768, |
| "nonexistent_fields_rate": 0.0804, |
| "empty_rate": 0.0 |
| } |
| }, |
| "by_author": { |
| "grammar": { |
| "n": 3220, |
| "exact_match": 0.3835, |
| "normalized_match": 0.3835, |
| "edit_similarity": 0.641, |
| "accepted_prefix_tokens": 3.0596, |
| "syntax_valid": 0.9988, |
| "mongo_valid": 0.9981, |
| "exec_valid": 0.8, |
| "ceiling_exec_valid": 0.8702, |
| "exec_over_ceiling": 0.9193, |
| "exec_given_ref_ok": 0.9183, |
| "n_ref_ok": 2802, |
| "nonexistent_fields_rate": 0.0528, |
| "empty_rate": 0.0 |
| }, |
| "llm": { |
| "n": 2810, |
| "exact_match": 0.5295, |
| "normalized_match": 0.5342, |
| "edit_similarity": 0.77, |
| "accepted_prefix_tokens": 4.8612, |
| "syntax_valid": 0.9954, |
| "mongo_valid": 0.9416, |
| "exec_valid": 0.8658, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.8658, |
| "exec_given_ref_ok": 0.8658, |
| "n_ref_ok": 2810, |
| "nonexistent_fields_rate": 0.1085, |
| "empty_rate": 0.0 |
| } |
| }, |
| "by_author_context": { |
| "grammar/full": { |
| "n": 2121, |
| "exact_match": 0.3234, |
| "normalized_match": 0.3234, |
| "edit_similarity": 0.5932, |
| "accepted_prefix_tokens": 3.075, |
| "syntax_valid": 0.9986, |
| "mongo_valid": 0.9976, |
| "exec_valid": 0.7888, |
| "ceiling_exec_valid": 0.8661, |
| "exec_over_ceiling": 0.9107, |
| "exec_given_ref_ok": 0.9091, |
| "n_ref_ok": 1837, |
| "nonexistent_fields_rate": 0.0528, |
| "empty_rate": 0.0 |
| }, |
| "grammar/irrelevant": { |
| "n": 553, |
| "exact_match": 0.4792, |
| "normalized_match": 0.4792, |
| "edit_similarity": 0.7223, |
| "accepted_prefix_tokens": 3.0362, |
| "syntax_valid": 0.9982, |
| "mongo_valid": 0.9982, |
| "exec_valid": 0.83, |
| "ceiling_exec_valid": 0.8861, |
| "exec_over_ceiling": 0.9367, |
| "exec_given_ref_ok": 0.9367, |
| "n_ref_ok": 490, |
| "nonexistent_fields_rate": 0.0488, |
| "empty_rate": 0.0 |
| }, |
| "grammar/none": { |
| "n": 546, |
| "exact_match": 0.5201, |
| "normalized_match": 0.5201, |
| "edit_similarity": 0.7445, |
| "accepted_prefix_tokens": 3.0238, |
| "syntax_valid": 1.0, |
| "mongo_valid": 1.0, |
| "exec_valid": 0.8132, |
| "ceiling_exec_valid": 0.87, |
| "exec_over_ceiling": 0.9347, |
| "exec_given_ref_ok": 0.9347, |
| "n_ref_ok": 475, |
| "nonexistent_fields_rate": 0.0568, |
| "empty_rate": 0.0 |
| }, |
| "llm/full": { |
| "n": 1599, |
| "exact_match": 0.4578, |
| "normalized_match": 0.4615, |
| "edit_similarity": 0.7186, |
| "accepted_prefix_tokens": 4.7411, |
| "syntax_valid": 0.9937, |
| "mongo_valid": 0.9343, |
| "exec_valid": 0.8624, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.8624, |
| "exec_given_ref_ok": 0.8624, |
| "n_ref_ok": 1599, |
| "nonexistent_fields_rate": 0.1044, |
| "empty_rate": 0.0 |
| }, |
| "llm/irrelevant": { |
| "n": 602, |
| "exact_match": 0.6196, |
| "normalized_match": 0.6246, |
| "edit_similarity": 0.8369, |
| "accepted_prefix_tokens": 4.6844, |
| "syntax_valid": 0.995, |
| "mongo_valid": 0.9518, |
| "exec_valid": 0.8704, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.8704, |
| "exec_given_ref_ok": 0.8704, |
| "n_ref_ok": 602, |
| "nonexistent_fields_rate": 0.113, |
| "empty_rate": 0.0 |
| }, |
| "llm/none": { |
| "n": 609, |
| "exact_match": 0.6289, |
| "normalized_match": 0.6355, |
| "edit_similarity": 0.8386, |
| "accepted_prefix_tokens": 5.3514, |
| "syntax_valid": 1.0, |
| "mongo_valid": 0.9507, |
| "exec_valid": 0.8703, |
| "ceiling_exec_valid": 1.0, |
| "exec_over_ceiling": 0.8703, |
| "exec_given_ref_ok": 0.8703, |
| "n_ref_ok": 609, |
| "nonexistent_fields_rate": 0.1149, |
| "empty_rate": 0.0 |
| } |
| } |
| } |
| }, |
| "val_v2c": { |
| "base_1.5b": { |
| "n": 2000, |
| "normalized_match": 0.142, |
| "exact_match": 0.142, |
| "edit_similarity": 0.3057, |
| "syntax_valid": 0.667, |
| "mongo_valid": 0.595, |
| "exec_valid": 0.316, |
| "ceiling_exec_valid": 0.6255, |
| "exec_over_ceiling": 0.5052, |
| "nonexistent_fields_rate": 0.051, |
| "nonexistent_fields_full": 0.0376, |
| "by_context_kind": { |
| "full": { |
| "normalized_match": 0.1475, |
| "exec_valid": 0.3364 |
| }, |
| "irrelevant": { |
| "normalized_match": 0.1061, |
| "exec_valid": 0.2172 |
| }, |
| "none": { |
| "normalized_match": 0.129, |
| "exec_valid": 0.2258 |
| } |
| } |
| }, |
| "t1_distill": { |
| "n": 2000, |
| "normalized_match": 0.4105, |
| "exact_match": 0.4105, |
| "edit_similarity": 0.6568, |
| "syntax_valid": 0.9955, |
| "mongo_valid": 0.9945, |
| "exec_valid": 0.595, |
| "ceiling_exec_valid": 0.6255, |
| "exec_over_ceiling": 0.9512, |
| "nonexistent_fields_rate": 0.029, |
| "nonexistent_fields_full": 0.0158, |
| "by_context_kind": { |
| "full": { |
| "normalized_match": 0.4159, |
| "exec_valid": 0.6199 |
| }, |
| "irrelevant": { |
| "normalized_match": 0.4091, |
| "exec_valid": 0.5051 |
| }, |
| "none": { |
| "normalized_match": 0.3548, |
| "exec_valid": 0.4452 |
| } |
| } |
| }, |
| "t1_v3": { |
| "n": 2000, |
| "normalized_match": 0.41, |
| "exact_match": 0.41, |
| "edit_similarity": 0.6536, |
| "syntax_valid": 0.998, |
| "mongo_valid": 0.9975, |
| "exec_valid": 0.6015, |
| "ceiling_exec_valid": 0.6255, |
| "exec_over_ceiling": 0.9616, |
| "nonexistent_fields_rate": 0.021, |
| "nonexistent_fields_full": 0.0121, |
| "by_context_kind": { |
| "full": { |
| "normalized_match": 0.4183, |
| "exec_valid": 0.6211 |
| }, |
| "irrelevant": { |
| "normalized_match": 0.3838, |
| "exec_valid": 0.5303 |
| }, |
| "none": { |
| "normalized_match": 0.3548, |
| "exec_valid": 0.4839 |
| } |
| } |
| } |
| }, |
| "pairs": { |
| "t1_distill_vs_base_1.5b": { |
| "context_kind": { |
| "n_aligned": 6030, |
| "overall": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 2.2487562189054726, |
| "mean_b": 2.789054726368159, |
| "delta": 0.5402985074626866, |
| "ci_low": 0.4485820895522389, |
| "ci_high": 0.6321766169154228, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 0.37318669983416247, |
| "mean_b": 0.6188011608623549, |
| "delta": 0.24561446102819237, |
| "ci_low": 0.23526142454394694, |
| "ci_high": 0.2561746745439469, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.1855721393034826, |
| "rate_b": 0.3686567164179104, |
| "delta": 0.18308457711442785, |
| "better": 1307, |
| "worse": 203, |
| "equal": 4520, |
| "p": 1.4405600645418062e-197, |
| "ci_low": 0.17147180762852407, |
| "ci_high": 0.19502487562189055, |
| "superior": true, |
| "p_holm": 2.5930081161752513e-196 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.49154228855721394, |
| "rate_b": 0.7859038142620233, |
| "delta": 0.2943615257048093, |
| "better": 2104, |
| "worse": 329, |
| "equal": 3597, |
| "p": 9.56375726e-316, |
| "ci_low": 0.2800995024875622, |
| "ci_high": 0.3086235489220564, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.9127514525e-314 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.6048092868988392, |
| "rate_b": 0.9620232172470978, |
| "delta": 0.3572139303482587, |
| "better": 2194, |
| "worse": 40, |
| "equal": 3796, |
| "p": 0.0, |
| "ci_low": 0.3449419568822554, |
| "ci_high": 0.369983416252073, |
| "superior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.9182421227197347, |
| "rate_b": 0.8993366500829187, |
| "delta": -0.01890547263681592, |
| "better": 305, |
| "worse": 419, |
| "equal": 5306, |
| "p": 2.5795404069987098e-05, |
| "ci_low": -0.02769485903814262, |
| "ci_high": -0.010281923714759536, |
| "superior": false, |
| "p_holm": 0.00010318161627994839 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.18739635157545606, |
| "rate_b": 0.36965174129353234, |
| "delta": 0.18225538971807628, |
| "better": 1308, |
| "worse": 209, |
| "equal": 4513, |
| "p": 2.0345053924323034e-194, |
| "ci_low": 0.17064676616915422, |
| "ci_high": 0.19419568822553898, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 3.4586591671349157e-193 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.6774461028192371, |
| "rate_b": 0.9902155887230514, |
| "delta": 0.31276948590381426, |
| "better": 1919, |
| "worse": 33, |
| "equal": 4078, |
| "p": 0.0, |
| "ci_low": 0.30082918739635156, |
| "ci_high": 0.3250414593698176, |
| "superior": true, |
| "p_holm": 0.0 |
| } |
| ], |
| "groups": { |
| "full": { |
| "n": 3720, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 3720, |
| "lower_is_better": false, |
| "mean_a": 2.150537634408602, |
| "mean_b": 2.7946236559139783, |
| "delta": 0.6440860215053763, |
| "ci_low": 0.5311760752688172, |
| "ci_high": 0.7588709677419355, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 3720, |
| "lower_is_better": false, |
| "mean_a": 0.3220927956989247, |
| "mean_b": 0.5746390860215054, |
| "delta": 0.25254629032258064, |
| "ci_low": 0.23996857459677418, |
| "ci_high": 0.26556272849462365, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.139247311827957, |
| "rate_b": 0.3150537634408602, |
| "delta": 0.17580645161290323, |
| "better": 735, |
| "worse": 81, |
| "equal": 2904, |
| "p": 1.0249568878429672e-132, |
| "ci_low": 0.1620967741935484, |
| "ci_high": 0.18978494623655914, |
| "superior": true, |
| "p_holm": 1.6399310205487475e-131 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.45, |
| "rate_b": 0.7879032258064517, |
| "delta": 0.3379032258064516, |
| "better": 1451, |
| "worse": 194, |
| "equal": 2075, |
| "p": 6.806754608883945e-238, |
| "ci_low": 0.3193548387096774, |
| "ci_high": 0.35618279569892475, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.2932833756879496e-236 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.5567204301075269, |
| "rate_b": 0.9612903225806452, |
| "delta": 0.40456989247311825, |
| "better": 1528, |
| "worse": 23, |
| "equal": 2169, |
| "p": 0.0, |
| "ci_low": 0.3881720430107527, |
| "ci_high": 0.4206989247311828, |
| "superior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.928494623655914, |
| "rate_b": 0.9080645161290323, |
| "delta": -0.02043010752688172, |
| "better": 174, |
| "worse": 250, |
| "equal": 3296, |
| "p": 0.00026101422945458636, |
| "ci_low": -0.03118279569892473, |
| "ci_high": -0.00967741935483871, |
| "superior": false, |
| "p_holm": 0.0007830426883637591 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.13978494623655913, |
| "rate_b": 0.3153225806451613, |
| "delta": 0.17553763440860215, |
| "better": 736, |
| "worse": 83, |
| "equal": 2901, |
| "p": 1.4039628267736683e-131, |
| "ci_low": 0.16182795698924732, |
| "ci_high": 0.18951612903225806, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 2.1059442401605025e-130 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.6231182795698925, |
| "rate_b": 0.989247311827957, |
| "delta": 0.36612903225806454, |
| "better": 1385, |
| "worse": 23, |
| "equal": 2312, |
| "p": 0.0, |
| "ci_low": 0.3497311827956989, |
| "ci_high": 0.381989247311828, |
| "superior": true, |
| "p_holm": 0.0 |
| } |
| ] |
| }, |
| "irrelevant": { |
| "n": 1155, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 2.3445887445887448, |
| "mean_b": 2.7307359307359307, |
| "delta": 0.38614718614718613, |
| "ci_low": 0.1774891774891775, |
| "ci_high": 0.6043290043290044, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 0.4300482251082251, |
| "mean_b": 0.6890493506493506, |
| "delta": 0.25900112554112553, |
| "ci_low": 0.2331739329004329, |
| "ci_high": 0.2849483398268398, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.24415584415584415, |
| "rate_b": 0.4510822510822511, |
| "delta": 0.20692640692640693, |
| "better": 290, |
| "worse": 51, |
| "equal": 814, |
| "p": 1.0054457699912198e-41, |
| "ci_low": 0.17835497835497835, |
| "ci_high": 0.2354978354978355, |
| "superior": true, |
| "p_holm": 9.049011929920978e-41 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.5341991341991342, |
| "rate_b": 0.793073593073593, |
| "delta": 0.25887445887445887, |
| "better": 357, |
| "worse": 58, |
| "equal": 740, |
| "p": 1.2878161280010718e-53, |
| "ci_low": 0.22857142857142856, |
| "ci_high": 0.29004329004329005, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.2878161280010719e-52 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.6441558441558441, |
| "rate_b": 0.9645021645021645, |
| "delta": 0.3203463203463203, |
| "better": 378, |
| "worse": 8, |
| "equal": 769, |
| "p": 1.4728552360527823e-100, |
| "ci_low": 0.29264069264069265, |
| "ci_high": 0.34805194805194806, |
| "superior": true, |
| "p_holm": 2.0619973304738953e-99 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.9038961038961039, |
| "rate_b": 0.8865800865800866, |
| "delta": -0.017316017316017316, |
| "better": 64, |
| "worse": 84, |
| "equal": 1007, |
| "p": 0.11803924678172636, |
| "ci_low": -0.0380952380952381, |
| "ci_high": 0.0025974025974025974, |
| "superior": false, |
| "p_holm": 0.23607849356345273 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.24848484848484848, |
| "rate_b": 0.45454545454545453, |
| "delta": 0.20606060606060606, |
| "better": 290, |
| "worse": 52, |
| "equal": 813, |
| "p": 3.3200986933768434e-41, |
| "ci_low": 0.1774891774891775, |
| "ci_high": 0.23463203463203464, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 2.6560789547014747e-40 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.7238095238095238, |
| "rate_b": 0.9913419913419913, |
| "delta": 0.2675324675324675, |
| "better": 313, |
| "worse": 4, |
| "equal": 838, |
| "p": 3.1321784783752422e-87, |
| "ci_low": 0.24155844155844156, |
| "ci_high": 0.2935064935064935, |
| "superior": true, |
| "p_holm": 4.071832021887815e-86 |
| } |
| ] |
| }, |
| "none": { |
| "n": 1155, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 2.469264069264069, |
| "mean_b": 2.8294372294372296, |
| "delta": 0.3601731601731602, |
| "ci_low": 0.16277056277056276, |
| "ci_high": 0.5627705627705628, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 0.48088735930735926, |
| "mean_b": 0.690789264069264, |
| "delta": 0.20990190476190476, |
| "ci_low": 0.18358663203463205, |
| "ci_high": 0.2359251753246753, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.2761904761904762, |
| "rate_b": 0.4588744588744589, |
| "delta": 0.18268398268398267, |
| "better": 282, |
| "worse": 71, |
| "equal": 802, |
| "p": 6.925330255223634e-31, |
| "ci_low": 0.1523809523809524, |
| "ci_high": 0.21298701298701297, |
| "superior": true, |
| "p_holm": 4.15519815313418e-30 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.5826839826839827, |
| "rate_b": 0.7722943722943723, |
| "delta": 0.18961038961038962, |
| "better": 296, |
| "worse": 77, |
| "equal": 782, |
| "p": 2.1825485902965193e-31, |
| "ci_low": 0.15844155844155844, |
| "ci_high": 0.22077922077922077, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.5277840132075635e-30 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.7203463203463204, |
| "rate_b": 0.9619047619047619, |
| "delta": 0.24155844155844156, |
| "better": 288, |
| "worse": 9, |
| "equal": 858, |
| "p": 3.5538904469071796e-73, |
| "ci_low": 0.2164285714285716, |
| "ci_high": 0.2675324675324675, |
| "superior": true, |
| "p_holm": 4.2646685362886157e-72 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.8995670995670996, |
| "rate_b": 0.883982683982684, |
| "delta": -0.015584415584415584, |
| "better": 67, |
| "worse": 85, |
| "equal": 1003, |
| "p": 0.16772199317919748, |
| "ci_low": -0.03636363636363636, |
| "ci_high": 0.004329004329004329, |
| "superior": false, |
| "p_holm": 0.23607849356345273 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.27965367965367965, |
| "rate_b": 0.4597402597402597, |
| "delta": 0.1800865800865801, |
| "better": 282, |
| "worse": 74, |
| "equal": 799, |
| "p": 1.0098733511899805e-29, |
| "ci_low": 0.1497835497835498, |
| "ci_high": 0.21038961038961038, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 5.049366755949903e-29 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.806060606060606, |
| "rate_b": 0.9922077922077922, |
| "delta": 0.18614718614718614, |
| "better": 221, |
| "worse": 6, |
| "equal": 928, |
| "p": 1.6941577103211408e-57, |
| "ci_low": 0.16363636363636364, |
| "ci_high": 0.20952380952380953, |
| "superior": true, |
| "p_holm": 1.863573481353255e-56 |
| } |
| ] |
| } |
| }, |
| "groups_skipped": {} |
| }, |
| "author": { |
| "n_aligned": 6030, |
| "overall": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 2.2487562189054726, |
| "mean_b": 2.789054726368159, |
| "delta": 0.5402985074626866, |
| "ci_low": 0.4485820895522389, |
| "ci_high": 0.6321766169154228, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 0.37318669983416247, |
| "mean_b": 0.6188011608623549, |
| "delta": 0.24561446102819237, |
| "ci_low": 0.23526142454394694, |
| "ci_high": 0.2561746745439469, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.1855721393034826, |
| "rate_b": 0.3686567164179104, |
| "delta": 0.18308457711442785, |
| "better": 1307, |
| "worse": 203, |
| "equal": 4520, |
| "p": 1.4405600645418062e-197, |
| "ci_low": 0.17147180762852407, |
| "ci_high": 0.19502487562189055, |
| "superior": true, |
| "p_holm": 1.4405600645418063e-196 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.49154228855721394, |
| "rate_b": 0.7859038142620233, |
| "delta": 0.2943615257048093, |
| "better": 2104, |
| "worse": 329, |
| "equal": 3597, |
| "p": 9.56375726e-316, |
| "ci_low": 0.2800995024875622, |
| "ci_high": 0.3086235489220564, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.3389260167e-314 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.6048092868988392, |
| "rate_b": 0.9620232172470978, |
| "delta": 0.3572139303482587, |
| "better": 2194, |
| "worse": 40, |
| "equal": 3796, |
| "p": 0.0, |
| "ci_low": 0.3449419568822554, |
| "ci_high": 0.369983416252073, |
| "superior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.9182421227197347, |
| "rate_b": 0.8993366500829187, |
| "delta": -0.01890547263681592, |
| "better": 305, |
| "worse": 419, |
| "equal": 5306, |
| "p": 2.5795404069987098e-05, |
| "ci_low": -0.02769485903814262, |
| "ci_high": -0.010281923714759536, |
| "superior": false, |
| "p_holm": 5.1590808139974196e-05 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.18739635157545606, |
| "rate_b": 0.36965174129353234, |
| "delta": 0.18225538971807628, |
| "better": 1308, |
| "worse": 209, |
| "equal": 4513, |
| "p": 2.0345053924323034e-194, |
| "ci_low": 0.17064676616915422, |
| "ci_high": 0.19419568822553898, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.831054853189073e-193 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.6774461028192371, |
| "rate_b": 0.9902155887230514, |
| "delta": 0.31276948590381426, |
| "better": 1919, |
| "worse": 33, |
| "equal": 4078, |
| "p": 0.0, |
| "ci_low": 0.30082918739635156, |
| "ci_high": 0.3250414593698176, |
| "superior": true, |
| "p_holm": 0.0 |
| } |
| ], |
| "groups": { |
| "grammar": { |
| "n": 3220, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 3220, |
| "lower_is_better": false, |
| "mean_a": 1.8161490683229813, |
| "mean_b": 3.0270186335403726, |
| "delta": 1.2108695652173913, |
| "ci_low": 1.0956521739130434, |
| "ci_high": 1.3301242236024844, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 3220, |
| "lower_is_better": false, |
| "mean_a": 0.2908079192546584, |
| "mean_b": 0.6330357142857143, |
| "delta": 0.34222779503105594, |
| "ci_low": 0.3284316257763975, |
| "ci_high": 0.3557838066770186, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.12608695652173912, |
| "rate_b": 0.37577639751552794, |
| "delta": 0.2496894409937888, |
| "better": 839, |
| "worse": 35, |
| "equal": 2346, |
| "p": 7.216279331318949e-201, |
| "ci_low": 0.23385093167701862, |
| "ci_high": 0.265527950310559, |
| "superior": true, |
| "p_holm": 7.937907264450844e-200 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.3829192546583851, |
| "rate_b": 0.7897515527950311, |
| "delta": 0.40683229813664595, |
| "better": 1397, |
| "worse": 87, |
| "equal": 1736, |
| "p": 1.1874189546903685e-304, |
| "ci_low": 0.38819875776397517, |
| "ci_high": 0.4254658385093168, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.543644641097479e-303 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.5509316770186335, |
| "rate_b": 0.9968944099378882, |
| "delta": 0.44596273291925465, |
| "better": 1438, |
| "worse": 2, |
| "equal": 1780, |
| "p": 0.0, |
| "ci_low": 0.4282608695652174, |
| "ci_high": 0.46366459627329193, |
| "superior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.9239130434782609, |
| "rate_b": 0.927639751552795, |
| "delta": 0.0037267080745341614, |
| "better": 183, |
| "worse": 171, |
| "equal": 2866, |
| "p": 0.5588488229617014, |
| "ci_low": -0.007763975155279503, |
| "ci_high": 0.015217391304347827, |
| "superior": false, |
| "p_holm": 0.5588488229617014 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.12608695652173912, |
| "rate_b": 0.3760869565217391, |
| "delta": 0.25, |
| "better": 840, |
| "worse": 35, |
| "equal": 2345, |
| "p": 3.758285052644206e-201, |
| "ci_low": 0.2341614906832298, |
| "ci_high": 0.26583850931677017, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 4.509942063173047e-200 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.5894409937888199, |
| "rate_b": 0.9975155279503105, |
| "delta": 0.4080745341614907, |
| "better": 1315, |
| "worse": 1, |
| "equal": 1904, |
| "p": 0.0, |
| "ci_low": 0.3906832298136646, |
| "ci_high": 0.4254658385093168, |
| "superior": true, |
| "p_holm": 0.0 |
| } |
| ] |
| }, |
| "llm": { |
| "n": 2810, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 2810, |
| "lower_is_better": false, |
| "mean_a": 2.7444839857651244, |
| "mean_b": 2.516370106761566, |
| "delta": -0.22811387900355873, |
| "ci_low": -0.3608540925266904, |
| "ci_high": -0.09217081850533808, |
| "superior": false |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 2810, |
| "lower_is_better": false, |
| "mean_a": 0.46758516014234874, |
| "mean_b": 0.6024896797153025, |
| "delta": 0.13490451957295374, |
| "ci_low": 0.11944684519572954, |
| "ci_high": 0.15020412010676157, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.2537366548042705, |
| "rate_b": 0.3604982206405694, |
| "delta": 0.10676156583629894, |
| "better": 468, |
| "worse": 168, |
| "equal": 2174, |
| "p": 1.1582932129215375e-33, |
| "ci_low": 0.08967971530249111, |
| "ci_high": 0.12384341637010676, |
| "superior": true, |
| "p_holm": 5.7914660646076875e-33 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.6160142348754448, |
| "rate_b": 0.7814946619217081, |
| "delta": 0.16548042704626334, |
| "better": 707, |
| "worse": 242, |
| "equal": 1861, |
| "p": 1.9060073309706465e-53, |
| "ci_low": 0.14483985765124555, |
| "ci_high": 0.18612099644128113, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.1436043985823879e-52 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.6665480427046263, |
| "rate_b": 0.9220640569395018, |
| "delta": 0.25551601423487547, |
| "better": 756, |
| "worse": 38, |
| "equal": 2016, |
| "p": 2.4517989932927237e-174, |
| "ci_low": 0.23843416370106763, |
| "ci_high": 0.27259786476868325, |
| "superior": true, |
| "p_holm": 1.961439194634179e-173 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.9117437722419929, |
| "rate_b": 0.8669039145907473, |
| "delta": -0.044839857651245554, |
| "better": 122, |
| "worse": 248, |
| "equal": 2440, |
| "p": 5.329604207306828e-11, |
| "ci_low": -0.058362989323843414, |
| "ci_high": -0.03131672597864769, |
| "superior": false, |
| "p_holm": 1.5988812621920485e-10 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.2576512455516014, |
| "rate_b": 0.3622775800711744, |
| "delta": 0.10462633451957296, |
| "better": 468, |
| "worse": 174, |
| "equal": 2168, |
| "p": 4.964410510210872e-32, |
| "ci_low": 0.08754448398576513, |
| "ci_high": 0.12206405693950179, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.985764204084349e-31 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.7782918149466193, |
| "rate_b": 0.9818505338078292, |
| "delta": 0.20355871886120996, |
| "better": 604, |
| "worse": 32, |
| "equal": 2174, |
| "p": 6.539367210840716e-138, |
| "ci_low": 0.18753558718861219, |
| "ci_high": 0.2195729537366548, |
| "superior": true, |
| "p_holm": 4.577557047588501e-137 |
| } |
| ] |
| } |
| }, |
| "groups_skipped": {} |
| } |
| }, |
| "t1_v3_vs_t1_distill": { |
| "context_kind": { |
| "n_aligned": 6030, |
| "overall": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 2.789054726368159, |
| "mean_b": 3.8991708126036486, |
| "delta": 1.1101160862354893, |
| "ci_low": 0.9971807628524046, |
| "ci_high": 1.2273673300165837, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 0.6188011608623549, |
| "mean_b": 0.7011165505804311, |
| "delta": 0.08231538971807628, |
| "ci_low": 0.07611636194029851, |
| "ci_high": 0.08865810199004974, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.3686567164179104, |
| "rate_b": 0.451575456053068, |
| "delta": 0.08291873963515754, |
| "better": 600, |
| "worse": 100, |
| "equal": 5330, |
| "p": 9.348895347113809e-88, |
| "ci_low": 0.07462686567164178, |
| "ci_high": 0.09137645107794362, |
| "superior": true, |
| "p_holm": 2.150245929836176e-86 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.7859038142620233, |
| "rate_b": 0.8306799336650083, |
| "delta": 0.04477611940298507, |
| "better": 442, |
| "worse": 172, |
| "equal": 5416, |
| "p": 2.414329124019748e-28, |
| "ci_low": 0.03681592039800995, |
| "ci_high": 0.052570480928689885, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 4.587225335637521e-27 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.9620232172470978, |
| "rate_b": 0.9718076285240465, |
| "delta": 0.00978441127694859, |
| "better": 82, |
| "worse": 23, |
| "equal": 5925, |
| "p": 5.956502026439826e-09, |
| "ci_low": 0.006467661691542288, |
| "ci_high": 0.013101160862354893, |
| "superior": true, |
| "p_holm": 6.552152229083809e-08 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.8993366500829187, |
| "rate_b": 0.9212271973466003, |
| "delta": 0.021890547263681594, |
| "better": 296, |
| "worse": 164, |
| "equal": 5570, |
| "p": 7.782239199341524e-10, |
| "ci_low": 0.015091210613598673, |
| "ci_high": 0.028855721393034824, |
| "superior": true, |
| "p_holm": 9.338687039209828e-09 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.36965174129353234, |
| "rate_b": 0.4537313432835821, |
| "delta": 0.08407960199004975, |
| "better": 608, |
| "worse": 101, |
| "equal": 5321, |
| "p": 4.35933007640598e-89, |
| "ci_low": 0.075787728026534, |
| "ci_high": 0.09253731343283582, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.046239218337435e-87 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.9902155887230514, |
| "rate_b": 0.9971807628524046, |
| "delta": 0.006965174129353234, |
| "better": 59, |
| "worse": 17, |
| "equal": 5954, |
| "p": 1.3968718113140024e-06, |
| "ci_low": 0.0041459369817578775, |
| "ci_high": 0.00978441127694859, |
| "superior": true, |
| "p_holm": 1.3968718113140024e-05 |
| } |
| ], |
| "groups": { |
| "full": { |
| "n": 3720, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 3720, |
| "lower_is_better": false, |
| "mean_a": 2.7946236559139783, |
| "mean_b": 3.7911290322580644, |
| "delta": 0.996505376344086, |
| "ci_low": 0.850268817204301, |
| "ci_high": 1.1508131720430108, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 3720, |
| "lower_is_better": false, |
| "mean_a": 0.5746390860215054, |
| "mean_b": 0.6471357795698924, |
| "delta": 0.0724966935483871, |
| "ci_low": 0.06445746639784947, |
| "ci_high": 0.08039365658602149, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.3150537634408602, |
| "rate_b": 0.3811827956989247, |
| "delta": 0.06612903225806452, |
| "better": 303, |
| "worse": 57, |
| "equal": 3360, |
| "p": 1.2206754770942516e-41, |
| "ci_low": 0.05618279569892473, |
| "ci_high": 0.0760752688172043, |
| "superior": true, |
| "p_holm": 2.5634185018979285e-40 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.7879032258064517, |
| "rate_b": 0.8204301075268817, |
| "delta": 0.032526881720430105, |
| "better": 252, |
| "worse": 131, |
| "equal": 3337, |
| "p": 6.314946543298559e-10, |
| "ci_low": 0.022311827956989248, |
| "ci_high": 0.04247311827956989, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 8.209430506288127e-09 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.9612903225806452, |
| "rate_b": 0.9704301075268817, |
| "delta": 0.00913978494623656, |
| "better": 52, |
| "worse": 18, |
| "equal": 3650, |
| "p": 5.849547318695842e-05, |
| "ci_low": 0.004838709677419355, |
| "ci_high": 0.013709677419354839, |
| "superior": true, |
| "p_holm": 0.0005264592586826258 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.9080645161290323, |
| "rate_b": 0.925, |
| "delta": 0.016935483870967744, |
| "better": 168, |
| "worse": 105, |
| "equal": 3447, |
| "p": 0.00016477217809691969, |
| "ci_low": 0.008064516129032258, |
| "ci_high": 0.025537634408602152, |
| "superior": true, |
| "p_holm": 0.0011534052466784378 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.3153225806451613, |
| "rate_b": 0.3827956989247312, |
| "delta": 0.0674731182795699, |
| "better": 309, |
| "worse": 58, |
| "equal": 3353, |
| "p": 1.678335502796201e-42, |
| "ci_low": 0.057526881720430106, |
| "ci_high": 0.07741935483870968, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 3.692338106151642e-41 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.989247311827957, |
| "rate_b": 0.996505376344086, |
| "delta": 0.007258064516129033, |
| "better": 40, |
| "worse": 13, |
| "equal": 3667, |
| "p": 0.0002685401188191605, |
| "ci_low": 0.0034946236559139786, |
| "ci_high": 0.01129032258064516, |
| "superior": true, |
| "p_holm": 0.0013427005940958026 |
| } |
| ] |
| }, |
| "irrelevant": { |
| "n": 1155, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 2.7307359307359307, |
| "mean_b": 3.895238095238095, |
| "delta": 1.1645021645021645, |
| "ci_low": 0.9246753246753247, |
| "ci_high": 1.4130086580086578, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 0.6890493506493506, |
| "mean_b": 0.7820035497835498, |
| "delta": 0.09295419913419914, |
| "ci_low": 0.07836375541125543, |
| "ci_high": 0.10772298051948052, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.4510822510822511, |
| "rate_b": 0.5523809523809524, |
| "delta": 0.1012987012987013, |
| "better": 140, |
| "worse": 23, |
| "equal": 992, |
| "p": 1.1750354186024887e-21, |
| "ci_low": 0.08051948051948052, |
| "ci_high": 0.12294372294372294, |
| "superior": true, |
| "p_holm": 1.997560211624231e-20 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.793073593073593, |
| "rate_b": 0.8510822510822511, |
| "delta": 0.05800865800865801, |
| "better": 88, |
| "worse": 21, |
| "equal": 1046, |
| "p": 6.112691638091427e-11, |
| "ci_low": 0.04069264069264069, |
| "ci_high": 0.07532467532467532, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 8.557768293327998e-10 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.9645021645021645, |
| "rate_b": 0.974025974025974, |
| "delta": 0.009523809523809525, |
| "better": 16, |
| "worse": 5, |
| "equal": 1134, |
| "p": 0.02660369873046875, |
| "ci_low": 0.0017316017316017316, |
| "ci_high": 0.017316017316017316, |
| "superior": true, |
| "p_holm": 0.0532073974609375 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.8865800865800866, |
| "rate_b": 0.9177489177489178, |
| "delta": 0.03116883116883117, |
| "better": 64, |
| "worse": 28, |
| "equal": 1063, |
| "p": 0.00022250112957580387, |
| "ci_low": 0.015584415584415584, |
| "ci_high": 0.047619047619047616, |
| "superior": true, |
| "p_holm": 0.0013350067774548233 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.45454545454545453, |
| "rate_b": 0.554978354978355, |
| "delta": 0.10043290043290043, |
| "better": 140, |
| "worse": 24, |
| "equal": 991, |
| "p": 4.04851304031129e-21, |
| "ci_low": 0.07965367965367966, |
| "ci_high": 0.12207792207792208, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 6.477620864498064e-20 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.9913419913419913, |
| "rate_b": 0.9965367965367965, |
| "delta": 0.005194805194805195, |
| "better": 10, |
| "worse": 4, |
| "equal": 1141, |
| "p": 0.1795654296875, |
| "ci_low": -0.0008658008658008658, |
| "ci_high": 0.011255411255411256, |
| "superior": false, |
| "p_holm": 0.1795654296875 |
| } |
| ] |
| }, |
| "none": { |
| "n": 1155, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 2.8294372294372296, |
| "mean_b": 4.251082251082251, |
| "delta": 1.4216450216450216, |
| "ci_low": 1.1523809523809523, |
| "ci_high": 1.7073593073593074, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 0.690789264069264, |
| "mean_b": 0.7940896969696969, |
| "delta": 0.10330043290043289, |
| "ci_low": 0.08851722943722945, |
| "ci_high": 0.11877723160173158, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.4588744588744589, |
| "rate_b": 0.5774891774891775, |
| "delta": 0.11861471861471862, |
| "better": 157, |
| "worse": 20, |
| "equal": 978, |
| "p": 1.4656075428029792e-27, |
| "ci_low": 0.09783549783549783, |
| "ci_high": 0.14025974025974025, |
| "superior": true, |
| "p_holm": 2.6380935770453625e-26 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.7722943722943723, |
| "rate_b": 0.8432900432900433, |
| "delta": 0.070995670995671, |
| "better": 102, |
| "worse": 20, |
| "equal": 1033, |
| "p": 1.9641987858977157e-14, |
| "ci_low": 0.05281385281385281, |
| "ci_high": 0.08917748917748917, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 2.946298178846574e-13 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.9619047619047619, |
| "rate_b": 0.974025974025974, |
| "delta": 0.012121212121212121, |
| "better": 14, |
| "worse": 0, |
| "equal": 1141, |
| "p": 0.0001220703125, |
| "ci_low": 0.006060606060606061, |
| "ci_high": 0.01904761904761905, |
| "superior": true, |
| "p_holm": 0.0009765625 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.883982683982684, |
| "rate_b": 0.9125541125541126, |
| "delta": 0.02857142857142857, |
| "better": 64, |
| "worse": 31, |
| "equal": 1060, |
| "p": 0.0009246444117719318, |
| "ci_low": 0.012121212121212121, |
| "ci_high": 0.045021645021645025, |
| "superior": true, |
| "p_holm": 0.0036985776470877274 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.4597402597402597, |
| "rate_b": 0.580952380952381, |
| "delta": 0.12121212121212122, |
| "better": 159, |
| "worse": 19, |
| "equal": 977, |
| "p": 1.0293288050049541e-28, |
| "ci_low": 0.10043290043290043, |
| "ci_high": 0.14285714285714285, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 2.0586576100099084e-27 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.9922077922077922, |
| "rate_b": 1.0, |
| "delta": 0.007792207792207792, |
| "better": 9, |
| "worse": 0, |
| "equal": 1146, |
| "p": 0.00390625, |
| "ci_low": 0.003463203463203463, |
| "ci_high": 0.012987012987012988, |
| "superior": true, |
| "p_holm": 0.01171875 |
| } |
| ] |
| } |
| }, |
| "groups_skipped": {} |
| }, |
| "author": { |
| "n_aligned": 6030, |
| "overall": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 2.789054726368159, |
| "mean_b": 3.8991708126036486, |
| "delta": 1.1101160862354893, |
| "ci_low": 0.9971807628524046, |
| "ci_high": 1.2273673300165837, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 0.6188011608623549, |
| "mean_b": 0.7011165505804311, |
| "delta": 0.08231538971807628, |
| "ci_low": 0.07611636194029851, |
| "ci_high": 0.08865810199004974, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.3686567164179104, |
| "rate_b": 0.451575456053068, |
| "delta": 0.08291873963515754, |
| "better": 600, |
| "worse": 100, |
| "equal": 5330, |
| "p": 9.348895347113809e-88, |
| "ci_low": 0.07462686567164178, |
| "ci_high": 0.09137645107794362, |
| "superior": true, |
| "p_holm": 1.4023343020670715e-86 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.7859038142620233, |
| "rate_b": 0.8306799336650083, |
| "delta": 0.04477611940298507, |
| "better": 442, |
| "worse": 172, |
| "equal": 5416, |
| "p": 2.414329124019748e-28, |
| "ci_low": 0.03681592039800995, |
| "ci_high": 0.052570480928689885, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 3.1386278612256722e-27 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.9620232172470978, |
| "rate_b": 0.9718076285240465, |
| "delta": 0.00978441127694859, |
| "better": 82, |
| "worse": 23, |
| "equal": 5925, |
| "p": 5.956502026439826e-09, |
| "ci_low": 0.006467661691542288, |
| "ci_high": 0.013101160862354893, |
| "superior": true, |
| "p_holm": 5.956502026439826e-08 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.8993366500829187, |
| "rate_b": 0.9212271973466003, |
| "delta": 0.021890547263681594, |
| "better": 296, |
| "worse": 164, |
| "equal": 5570, |
| "p": 7.782239199341524e-10, |
| "ci_low": 0.015091210613598673, |
| "ci_high": 0.028855721393034824, |
| "superior": true, |
| "p_holm": 9.338687039209828e-09 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.36965174129353234, |
| "rate_b": 0.4537313432835821, |
| "delta": 0.08407960199004975, |
| "better": 608, |
| "worse": 101, |
| "equal": 5321, |
| "p": 4.35933007640598e-89, |
| "ci_low": 0.075787728026534, |
| "ci_high": 0.09253731343283582, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 6.974928122249568e-88 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.9902155887230514, |
| "rate_b": 0.9971807628524046, |
| "delta": 0.006965174129353234, |
| "better": 59, |
| "worse": 17, |
| "equal": 5954, |
| "p": 1.3968718113140024e-06, |
| "ci_low": 0.0041459369817578775, |
| "ci_high": 0.00978441127694859, |
| "superior": true, |
| "p_holm": 1.1174974490512019e-05 |
| } |
| ], |
| "groups": { |
| "grammar": { |
| "n": 3220, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 3220, |
| "lower_is_better": false, |
| "mean_a": 3.0270186335403726, |
| "mean_b": 3.0596273291925464, |
| "delta": 0.03260869565217391, |
| "ci_low": -0.04503105590062112, |
| "ci_high": 0.11055900621118013, |
| "superior": false |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 3220, |
| "lower_is_better": false, |
| "mean_a": 0.6330357142857143, |
| "mean_b": 0.6410397204968944, |
| "delta": 0.008004006211180125, |
| "ci_low": 0.0015777158385093183, |
| "ci_high": 0.014488523291925461, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.37577639751552794, |
| "rate_b": 0.38354037267080743, |
| "delta": 0.007763975155279503, |
| "better": 95, |
| "worse": 70, |
| "equal": 3055, |
| "p": 0.06137832710948886, |
| "ci_low": 0.0, |
| "ci_high": 0.015838509316770187, |
| "superior": false, |
| "p_holm": 0.24551330843795544 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.7897515527950311, |
| "rate_b": 0.8, |
| "delta": 0.010248447204968944, |
| "better": 112, |
| "worse": 79, |
| "equal": 3029, |
| "p": 0.020347077476115385, |
| "ci_low": 0.0021661490683230504, |
| "ci_high": 0.018944099378881987, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 0.10173538738057693 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.9968944099378882, |
| "rate_b": 0.9981366459627329, |
| "delta": 0.0012422360248447205, |
| "better": 10, |
| "worse": 6, |
| "equal": 3204, |
| "p": 0.454498291015625, |
| "ci_low": -0.0012422360248447205, |
| "ci_high": 0.0037267080745341614, |
| "superior": false, |
| "p_holm": 0.775390625 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.927639751552795, |
| "rate_b": 0.9472049689440993, |
| "delta": 0.01956521739130435, |
| "better": 115, |
| "worse": 52, |
| "equal": 3053, |
| "p": 1.2145886270456976e-06, |
| "ci_low": 0.011490683229813664, |
| "ci_high": 0.02732919254658385, |
| "superior": true, |
| "p_holm": 1.0931297643411278e-05 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.3760869565217391, |
| "rate_b": 0.38354037267080743, |
| "delta": 0.007453416149068323, |
| "better": 95, |
| "worse": 71, |
| "equal": 3054, |
| "p": 0.0739190926121619, |
| "ci_low": -0.00031055900621118014, |
| "ci_high": 0.015527950310559006, |
| "superior": false, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 0.24551330843795544 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.9975155279503105, |
| "rate_b": 0.9987577639751553, |
| "delta": 0.0012422360248447205, |
| "better": 8, |
| "worse": 4, |
| "equal": 3208, |
| "p": 0.3876953125, |
| "ci_low": -0.0009316770186335404, |
| "ci_high": 0.0034161490683229812, |
| "superior": false, |
| "p_holm": 0.775390625 |
| } |
| ] |
| }, |
| "llm": { |
| "n": 2810, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 2810, |
| "lower_is_better": false, |
| "mean_a": 2.516370106761566, |
| "mean_b": 4.8612099644128115, |
| "delta": 2.3448398576512455, |
| "ci_low": 2.126325622775801, |
| "ci_high": 2.5722419928825624, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 2810, |
| "lower_is_better": false, |
| "mean_a": 0.6024896797153025, |
| "mean_b": 0.7699590391459075, |
| "delta": 0.16746935943060498, |
| "ci_low": 0.15663636298932385, |
| "ci_high": 0.17816107740213522, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.3604982206405694, |
| "rate_b": 0.5295373665480427, |
| "delta": 0.16903914590747332, |
| "better": 505, |
| "worse": 30, |
| "equal": 2275, |
| "p": 2.2061014161757547e-112, |
| "ci_low": 0.15409252669039145, |
| "ci_high": 0.18398576512455517, |
| "superior": true, |
| "p_holm": 3.750372407498783e-111 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.7814946619217081, |
| "rate_b": 0.8658362989323843, |
| "delta": 0.08434163701067615, |
| "better": 330, |
| "worse": 93, |
| "equal": 2387, |
| "p": 3.4779869095107494e-32, |
| "ci_low": 0.0701067615658363, |
| "ci_high": 0.09822064056939502, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 4.869181673315049e-31 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.9220640569395018, |
| "rate_b": 0.9416370106761566, |
| "delta": 0.019572953736654804, |
| "better": 72, |
| "worse": 17, |
| "equal": 2721, |
| "p": 3.1727390190435945e-09, |
| "ci_low": 0.013167259786476869, |
| "ci_high": 0.02597864768683274, |
| "superior": true, |
| "p_holm": 3.490012920947954e-08 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.8669039145907473, |
| "rate_b": 0.891459074733096, |
| "delta": 0.024555160142348754, |
| "better": 181, |
| "worse": 112, |
| "equal": 2517, |
| "p": 6.617443552871609e-05, |
| "ci_low": 0.012455516014234875, |
| "ci_high": 0.036298932384341634, |
| "superior": true, |
| "p_holm": 0.00039704661317229654 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.3622775800711744, |
| "rate_b": 0.5341637010676157, |
| "delta": 0.1718861209964413, |
| "better": 513, |
| "worse": 30, |
| "equal": 2267, |
| "p": 1.3606411844347843e-114, |
| "ci_low": 0.15693950177935942, |
| "ci_high": 0.18683274021352314, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 2.4491541319826116e-113 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.9818505338078292, |
| "rate_b": 0.995373665480427, |
| "delta": 0.013523131672597865, |
| "better": 51, |
| "worse": 13, |
| "equal": 2746, |
| "p": 1.8809621564294015e-06, |
| "ci_low": 0.007829181494661922, |
| "ci_high": 0.019217081850533807, |
| "superior": true, |
| "p_holm": 1.316673509500581e-05 |
| } |
| ] |
| } |
| }, |
| "groups_skipped": {} |
| } |
| }, |
| "t1_v3_vs_base_1.5b": { |
| "context_kind": { |
| "n_aligned": 6030, |
| "overall": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 2.2487562189054726, |
| "mean_b": 3.8991708126036486, |
| "delta": 1.6504145936981758, |
| "ci_low": 1.5303482587064676, |
| "ci_high": 1.7753026533996683, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 0.37318669983416247, |
| "mean_b": 0.7011165505804311, |
| "delta": 0.3279298507462686, |
| "ci_low": 0.3182463428689884, |
| "ci_high": 0.3378565907960199, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.1855721393034826, |
| "rate_b": 0.451575456053068, |
| "delta": 0.2660033167495854, |
| "better": 1686, |
| "worse": 82, |
| "equal": 4262, |
| "p": 0.0, |
| "ci_low": 0.25422885572139303, |
| "ci_high": 0.2779436152570481, |
| "superior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.49154228855721394, |
| "rate_b": 0.8306799336650083, |
| "delta": 0.33913764510779437, |
| "better": 2251, |
| "worse": 206, |
| "equal": 3573, |
| "p": 0.0, |
| "ci_low": 0.32553897180762853, |
| "ci_high": 0.3529021558872305, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.6048092868988392, |
| "rate_b": 0.9718076285240465, |
| "delta": 0.3669983416252073, |
| "better": 2225, |
| "worse": 12, |
| "equal": 3793, |
| "p": 0.0, |
| "ci_low": 0.35505804311774464, |
| "ci_high": 0.37960199004975126, |
| "superior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.9182421227197347, |
| "rate_b": 0.9212271973466003, |
| "delta": 0.0029850746268656717, |
| "better": 331, |
| "worse": 313, |
| "equal": 5386, |
| "p": 0.5029560522139299, |
| "ci_low": -0.005306799336650083, |
| "ci_high": 0.011111111111111112, |
| "superior": false, |
| "p_holm": 1.0 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.18739635157545606, |
| "rate_b": 0.4537313432835821, |
| "delta": 0.266334991708126, |
| "better": 1687, |
| "worse": 81, |
| "equal": 4262, |
| "p": 0.0, |
| "ci_low": 0.2543946932006633, |
| "ci_high": 0.2782794361525704, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.6774461028192371, |
| "rate_b": 0.9971807628524046, |
| "delta": 0.3197346600331675, |
| "better": 1939, |
| "worse": 11, |
| "equal": 4080, |
| "p": 0.0, |
| "ci_low": 0.3081260364842454, |
| "ci_high": 0.3320066334991708, |
| "superior": true, |
| "p_holm": 0.0 |
| } |
| ], |
| "groups": { |
| "full": { |
| "n": 3720, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 3720, |
| "lower_is_better": false, |
| "mean_a": 2.150537634408602, |
| "mean_b": 3.7911290322580644, |
| "delta": 1.6405913978494624, |
| "ci_low": 1.4833333333333334, |
| "ci_high": 1.8016129032258064, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 3720, |
| "lower_is_better": false, |
| "mean_a": 0.3220927956989247, |
| "mean_b": 0.6471357795698924, |
| "delta": 0.32504298387096775, |
| "ci_low": 0.31256386626344085, |
| "ci_high": 0.33756841397849463, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.139247311827957, |
| "rate_b": 0.3811827956989247, |
| "delta": 0.24193548387096775, |
| "better": 938, |
| "worse": 38, |
| "equal": 2744, |
| "p": 1.1949364281529616e-225, |
| "ci_low": 0.22768817204301076, |
| "ci_high": 0.2564516129032258, |
| "superior": true, |
| "p_holm": 1.7924046422294423e-224 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.45, |
| "rate_b": 0.8204301075268817, |
| "delta": 0.3704301075268817, |
| "better": 1533, |
| "worse": 155, |
| "equal": 2032, |
| "p": 4.001190348593105e-285, |
| "ci_low": 0.3521505376344086, |
| "ci_high": 0.38844086021505375, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 6.802023592608279e-284 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.5567204301075269, |
| "rate_b": 0.9704301075268817, |
| "delta": 0.4137096774193548, |
| "better": 1545, |
| "worse": 6, |
| "equal": 2169, |
| "p": 0.0, |
| "ci_low": 0.39731182795698927, |
| "ci_high": 0.42983870967741933, |
| "superior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.928494623655914, |
| "rate_b": 0.925, |
| "delta": -0.0034946236559139786, |
| "better": 194, |
| "worse": 207, |
| "equal": 3319, |
| "p": 0.5490587682630458, |
| "ci_low": -0.013978494623655914, |
| "ci_high": 0.006989247311827957, |
| "superior": false, |
| "p_holm": 1.0 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.13978494623655913, |
| "rate_b": 0.3827956989247312, |
| "delta": 0.24301075268817204, |
| "better": 942, |
| "worse": 38, |
| "equal": 2740, |
| "p": 8.748941920297441e-227, |
| "ci_low": 0.22849462365591397, |
| "ci_high": 0.2575268817204301, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.3998307072475906e-225 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 3720, |
| "lower_is_better": false, |
| "rate_a": 0.6231182795698925, |
| "rate_b": 0.996505376344086, |
| "delta": 0.37338709677419357, |
| "better": 1396, |
| "worse": 7, |
| "equal": 2317, |
| "p": 0.0, |
| "ci_low": 0.357258064516129, |
| "ci_high": 0.38924731182795697, |
| "superior": true, |
| "p_holm": 0.0 |
| } |
| ] |
| }, |
| "irrelevant": { |
| "n": 1155, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 2.3445887445887448, |
| "mean_b": 3.895238095238095, |
| "delta": 1.5506493506493506, |
| "ci_low": 1.3142857142857143, |
| "ci_high": 1.7948051948051948, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 0.4300482251082251, |
| "mean_b": 0.7820035497835498, |
| "delta": 0.35195532467532464, |
| "ci_low": 0.32793832034632037, |
| "ci_high": 0.3758575714285714, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.24415584415584415, |
| "rate_b": 0.5523809523809524, |
| "delta": 0.30822510822510824, |
| "better": 375, |
| "worse": 19, |
| "equal": 761, |
| "p": 5.694924653737641e-87, |
| "ci_low": 0.2804978354978357, |
| "ci_high": 0.3367965367965368, |
| "superior": true, |
| "p_holm": 6.83390958448517e-86 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.5341991341991342, |
| "rate_b": 0.8510822510822511, |
| "delta": 0.3168831168831169, |
| "better": 392, |
| "worse": 26, |
| "equal": 737, |
| "p": 5.0158096301156385e-85, |
| "ci_low": 0.2883116883116883, |
| "ci_high": 0.3463203463203463, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 4.514228667104075e-84 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.6441558441558441, |
| "rate_b": 0.974025974025974, |
| "delta": 0.32987012987012987, |
| "better": 386, |
| "worse": 5, |
| "equal": 764, |
| "p": 2.981856573542977e-107, |
| "ci_low": 0.30303030303030304, |
| "ci_high": 0.3575757575757576, |
| "superior": true, |
| "p_holm": 4.174599202960168e-106 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.9038961038961039, |
| "rate_b": 0.9177489177489178, |
| "delta": 0.013852813852813853, |
| "better": 74, |
| "worse": 58, |
| "equal": 1023, |
| "p": 0.19149877187347097, |
| "ci_low": -0.005194805194805195, |
| "ci_high": 0.033766233766233764, |
| "superior": false, |
| "p_holm": 0.7346286876070992 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.24848484848484848, |
| "rate_b": 0.554978354978355, |
| "delta": 0.3064935064935065, |
| "better": 373, |
| "worse": 19, |
| "equal": 763, |
| "p": 2.0638822530610425e-86, |
| "ci_low": 0.2779220779220779, |
| "ci_high": 0.33506493506493507, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 2.0638822530610427e-85 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.7238095238095238, |
| "rate_b": 0.9965367965367965, |
| "delta": 0.2727272727272727, |
| "better": 319, |
| "worse": 4, |
| "equal": 832, |
| "p": 5.2758083314590226e-89, |
| "ci_low": 0.24675324675324675, |
| "ci_high": 0.2987012987012987, |
| "superior": true, |
| "p_holm": 6.858550830896729e-88 |
| } |
| ] |
| }, |
| "none": { |
| "n": 1155, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 2.469264069264069, |
| "mean_b": 4.251082251082251, |
| "delta": 1.7818181818181817, |
| "ci_low": 1.4987012987012986, |
| "ci_high": 2.084848484848485, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 1155, |
| "lower_is_better": false, |
| "mean_a": 0.48088735930735926, |
| "mean_b": 0.7940896969696969, |
| "delta": 0.3132023376623376, |
| "ci_low": 0.290192645021645, |
| "ci_high": 0.33632416017316014, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.2761904761904762, |
| "rate_b": 0.5774891774891775, |
| "delta": 0.3012987012987013, |
| "better": 373, |
| "worse": 25, |
| "equal": 757, |
| "p": 9.844313337086202e-81, |
| "ci_low": 0.2727272727272727, |
| "ci_high": 0.32987012987012987, |
| "superior": true, |
| "p_holm": 6.891019335960342e-80 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.5826839826839827, |
| "rate_b": 0.8432900432900433, |
| "delta": 0.2606060606060606, |
| "better": 326, |
| "worse": 25, |
| "equal": 804, |
| "p": 5.442582279000005e-68, |
| "ci_low": 0.2329004329004329, |
| "ci_high": 0.2891774891774892, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 3.265549367400003e-67 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.7203463203463204, |
| "rate_b": 0.974025974025974, |
| "delta": 0.25367965367965367, |
| "better": 294, |
| "worse": 1, |
| "equal": 860, |
| "p": 9.299786660660013e-87, |
| "ci_low": 0.22857142857142856, |
| "ci_high": 0.27965367965367965, |
| "superior": true, |
| "p_holm": 1.0229765326726015e-85 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.8995670995670996, |
| "rate_b": 0.9125541125541126, |
| "delta": 0.012987012987012988, |
| "better": 63, |
| "worse": 48, |
| "equal": 1044, |
| "p": 0.1836571719017748, |
| "ci_low": -0.004329004329004329, |
| "ci_high": 0.03116883116883117, |
| "superior": false, |
| "p_holm": 0.7346286876070992 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.27965367965367965, |
| "rate_b": 0.580952380952381, |
| "delta": 0.3012987012987013, |
| "better": 372, |
| "worse": 24, |
| "equal": 759, |
| "p": 2.3177249009102542e-81, |
| "ci_low": 0.2727272727272727, |
| "ci_high": 0.32987012987012987, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 1.8541799207282034e-80 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 1155, |
| "lower_is_better": false, |
| "rate_a": 0.806060606060606, |
| "rate_b": 1.0, |
| "delta": 0.19393939393939394, |
| "better": 224, |
| "worse": 0, |
| "equal": 931, |
| "p": 7.418412301374843e-68, |
| "ci_low": 0.17227272727272747, |
| "ci_high": 0.21731601731601732, |
| "superior": true, |
| "p_holm": 3.7092061506874214e-67 |
| } |
| ] |
| } |
| }, |
| "groups_skipped": {} |
| }, |
| "author": { |
| "n_aligned": 6030, |
| "overall": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 2.2487562189054726, |
| "mean_b": 3.8991708126036486, |
| "delta": 1.6504145936981758, |
| "ci_low": 1.5303482587064676, |
| "ci_high": 1.7753026533996683, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 6030, |
| "lower_is_better": false, |
| "mean_a": 0.37318669983416247, |
| "mean_b": 0.7011165505804311, |
| "delta": 0.3279298507462686, |
| "ci_low": 0.3182463428689884, |
| "ci_high": 0.3378565907960199, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.1855721393034826, |
| "rate_b": 0.451575456053068, |
| "delta": 0.2660033167495854, |
| "better": 1686, |
| "worse": 82, |
| "equal": 4262, |
| "p": 0.0, |
| "ci_low": 0.25422885572139303, |
| "ci_high": 0.2779436152570481, |
| "superior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.49154228855721394, |
| "rate_b": 0.8306799336650083, |
| "delta": 0.33913764510779437, |
| "better": 2251, |
| "worse": 206, |
| "equal": 3573, |
| "p": 0.0, |
| "ci_low": 0.32553897180762853, |
| "ci_high": 0.3529021558872305, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.6048092868988392, |
| "rate_b": 0.9718076285240465, |
| "delta": 0.3669983416252073, |
| "better": 2225, |
| "worse": 12, |
| "equal": 3793, |
| "p": 0.0, |
| "ci_low": 0.35505804311774464, |
| "ci_high": 0.37960199004975126, |
| "superior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.9182421227197347, |
| "rate_b": 0.9212271973466003, |
| "delta": 0.0029850746268656717, |
| "better": 331, |
| "worse": 313, |
| "equal": 5386, |
| "p": 0.5029560522139299, |
| "ci_low": -0.005306799336650083, |
| "ci_high": 0.011111111111111112, |
| "superior": false, |
| "p_holm": 0.5029560522139299 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.18739635157545606, |
| "rate_b": 0.4537313432835821, |
| "delta": 0.266334991708126, |
| "better": 1687, |
| "worse": 81, |
| "equal": 4262, |
| "p": 0.0, |
| "ci_low": 0.2543946932006633, |
| "ci_high": 0.2782794361525704, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 6030, |
| "lower_is_better": false, |
| "rate_a": 0.6774461028192371, |
| "rate_b": 0.9971807628524046, |
| "delta": 0.3197346600331675, |
| "better": 1939, |
| "worse": 11, |
| "equal": 4080, |
| "p": 0.0, |
| "ci_low": 0.3081260364842454, |
| "ci_high": 0.3320066334991708, |
| "superior": true, |
| "p_holm": 0.0 |
| } |
| ], |
| "groups": { |
| "grammar": { |
| "n": 3220, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 3220, |
| "lower_is_better": false, |
| "mean_a": 1.8161490683229813, |
| "mean_b": 3.0596273291925464, |
| "delta": 1.2434782608695651, |
| "ci_low": 1.1235947204968944, |
| "ci_high": 1.3670885093167702, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 3220, |
| "lower_is_better": false, |
| "mean_a": 0.2908079192546584, |
| "mean_b": 0.6410397204968944, |
| "delta": 0.350231801242236, |
| "ci_low": 0.3362704782608696, |
| "ci_high": 0.36421224689440995, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.12608695652173912, |
| "rate_b": 0.38354037267080743, |
| "delta": 0.2574534161490683, |
| "better": 864, |
| "worse": 35, |
| "equal": 2321, |
| "p": 5.876926180008631e-208, |
| "ci_low": 0.24161490683229814, |
| "ci_high": 0.2732919254658385, |
| "superior": true, |
| "p_holm": 5.2892335620077676e-207 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.3829192546583851, |
| "rate_b": 0.8, |
| "delta": 0.41708074534161493, |
| "better": 1421, |
| "worse": 78, |
| "equal": 1721, |
| "p": 7.1556e-320, |
| "ci_low": 0.3984472049689441, |
| "ci_high": 0.4360248447204969, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 7.8711e-319 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.5509316770186335, |
| "rate_b": 0.9981366459627329, |
| "delta": 0.4472049689440994, |
| "better": 1442, |
| "worse": 2, |
| "equal": 1776, |
| "p": 0.0, |
| "ci_low": 0.4295031055900621, |
| "ci_high": 0.4649068322981366, |
| "superior": true, |
| "p_holm": 0.0 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.9239130434782609, |
| "rate_b": 0.9472049689440993, |
| "delta": 0.023291925465838508, |
| "better": 196, |
| "worse": 121, |
| "equal": 2903, |
| "p": 2.9877855768950636e-05, |
| "ci_low": 0.012732919254658385, |
| "ci_high": 0.033850931677018636, |
| "superior": true, |
| "p_holm": 8.96335673068519e-05 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.12608695652173912, |
| "rate_b": 0.38354037267080743, |
| "delta": 0.2574534161490683, |
| "better": 864, |
| "worse": 35, |
| "equal": 2321, |
| "p": 5.876926180008631e-208, |
| "ci_low": 0.24161490683229814, |
| "ci_high": 0.2732919254658385, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 5.2892335620077676e-207 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 3220, |
| "lower_is_better": false, |
| "rate_a": 0.5894409937888199, |
| "rate_b": 0.9987577639751553, |
| "delta": 0.4093167701863354, |
| "better": 1320, |
| "worse": 2, |
| "equal": 1898, |
| "p": 0.0, |
| "ci_low": 0.3919254658385093, |
| "ci_high": 0.4267080745341615, |
| "superior": true, |
| "p_holm": 0.0 |
| } |
| ] |
| }, |
| "llm": { |
| "n": 2810, |
| "metrics": [ |
| { |
| "metric": "accepted_prefix_tokens", |
| "kind": "continuous", |
| "n": 2810, |
| "lower_is_better": false, |
| "mean_a": 2.7444839857651244, |
| "mean_b": 4.8612099644128115, |
| "delta": 2.1167259786476866, |
| "ci_low": 1.903202846975089, |
| "ci_high": 2.3387900355871887, |
| "superior": true |
| }, |
| { |
| "metric": "edit_similarity", |
| "kind": "continuous", |
| "n": 2810, |
| "lower_is_better": false, |
| "mean_a": 0.46758516014234874, |
| "mean_b": 0.7699590391459075, |
| "delta": 0.3023738790035587, |
| "ci_low": 0.28847639679715303, |
| "ci_high": 0.31637307918149465, |
| "superior": true |
| }, |
| { |
| "metric": "exact_match", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.2537366548042705, |
| "rate_b": 0.5295373665480427, |
| "delta": 0.27580071174377224, |
| "better": 822, |
| "worse": 47, |
| "equal": 1941, |
| "p": 7.990635342893872e-184, |
| "ci_low": 0.25836298932384344, |
| "ci_high": 0.29323843416370104, |
| "superior": true, |
| "p_holm": 4.794381205736324e-183 |
| }, |
| { |
| "metric": "exec_valid", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.6160142348754448, |
| "rate_b": 0.8658362989323843, |
| "delta": 0.2498220640569395, |
| "better": 830, |
| "worse": 128, |
| "equal": 1852, |
| "p": 1.4303288564350014e-126, |
| "ci_low": 0.2302491103202847, |
| "ci_high": 0.2701067615658363, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 5.7213154257400056e-126 |
| }, |
| { |
| "metric": "mongo_valid", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.6665480427046263, |
| "rate_b": 0.9416370106761566, |
| "delta": 0.27508896797153026, |
| "better": 783, |
| "worse": 10, |
| "equal": 2017, |
| "p": 9.954887411232013e-217, |
| "ci_low": 0.2580071174377224, |
| "ci_high": 0.29145907473309607, |
| "superior": true, |
| "p_holm": 9.954887411232013e-216 |
| }, |
| { |
| "metric": "no_nonexistent_field", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.9117437722419929, |
| "rate_b": 0.891459074733096, |
| "delta": -0.020284697508896797, |
| "better": 135, |
| "worse": 192, |
| "equal": 2483, |
| "p": 0.0019114402812567247, |
| "ci_low": -0.03309608540925267, |
| "ci_high": -0.007829181494661922, |
| "superior": false, |
| "p_holm": 0.0038228805625134493 |
| }, |
| { |
| "metric": "normalized_match", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.2576512455516014, |
| "rate_b": 0.5341637010676157, |
| "delta": 0.2765124555160142, |
| "better": 823, |
| "worse": 46, |
| "equal": 1941, |
| "p": 4.5571131842281796e-185, |
| "ci_low": 0.2587188612099644, |
| "ci_high": 0.29359430604982206, |
| "superior": true, |
| "margin": 0.02, |
| "noninferior": true, |
| "p_holm": 3.1899792289597256e-184 |
| }, |
| { |
| "metric": "syntax_valid", |
| "kind": "binary", |
| "n": 2810, |
| "lower_is_better": false, |
| "rate_a": 0.7782918149466193, |
| "rate_b": 0.995373665480427, |
| "delta": 0.21708185053380782, |
| "better": 619, |
| "worse": 9, |
| "equal": 2182, |
| "p": 7.201076790162522e-170, |
| "ci_low": 0.201423487544484, |
| "ci_high": 0.23274021352313168, |
| "superior": true, |
| "p_holm": 3.600538395081261e-169 |
| } |
| ] |
| } |
| }, |
| "groups_skipped": {} |
| } |
| } |
| }, |
| "status_distribution": { |
| "base_1.5b": { |
| "ok": 3404, |
| "syntax": 1442, |
| "error": 1027, |
| "no_exec": 26, |
| "empty": 85, |
| "static": 45, |
| "timeout": 1 |
| }, |
| "t1_distill": { |
| "ok": 5328, |
| "error": 520, |
| "static": 98, |
| "syntax": 49, |
| "no_exec": 25, |
| "empty": 10 |
| }, |
| "t1_v3": { |
| "ok": 5478, |
| "error": 453, |
| "static": 82, |
| "syntax": 16, |
| "timeout": 1 |
| }, |
| "_reference": { |
| "ok": 5612, |
| "error": 418 |
| } |
| }, |
| "readonly_proof": { |
| "base_1.5b": { |
| "dbHash_before_equals_after": true, |
| "dbs_checked": 28, |
| "proxy_write_commands_forwarded": {}, |
| "maxTimeMS_injected": 3000 |
| }, |
| "t1_distill": { |
| "dbHash_before_equals_after": true, |
| "dbs_checked": 28, |
| "proxy_write_commands_forwarded": {}, |
| "maxTimeMS_injected": 3000 |
| }, |
| "t1_v3": { |
| "dbHash_before_equals_after": true, |
| "dbs_checked": 28, |
| "proxy_write_commands_forwarded": {}, |
| "maxTimeMS_injected": 3000 |
| }, |
| "_reference": { |
| "dbHash_before_equals_after": true, |
| "dbs_checked": 28, |
| "proxy_write_commands_forwarded": {}, |
| "maxTimeMS_injected": 3000 |
| } |
| }, |
| "definitions": { |
| "normalized_match": "pred == middle com espacos normalizados", |
| "exec_valid": "prefix+pred+suffix executa (mongosh via proxy RO, status ok, comando de leitura enviado) sem campo/colecao inexistente no plano capturado e pred nao vazia", |
| "ceiling_exec_valid": "mesma validacao sobre prefix+middle+suffix", |
| "exec_over_ceiling": "exec_valid / ceiling_exec_valid (razao de medias)", |
| "nonexistent_fields_rate": "itens cujo plano capturado cita campo/colecao fora do schema, sobre todos os itens", |
| "json_dialect": "dialect json executa como db.getCollection(c).aggregate(EJSON.parse(texto))" |
| } |
| } |
|
|
