{ "schema": "kapi-hf-eval-summary/1", "kind": "homologation_v1", "label": "Internal evaluation, synthetic out-of-distribution homologation; LLM- and grammar-written, not written by people; not fully independent of the v3 training data. Aggregates only; no item text.", "model": "esilva/KapiCoder-Mongo-Small", "format": "homolog-v1-eval-1", "set_sha256": "aca0aae51533a95d830c1f296d621f9203a7d76475658fb1e2dcc8f389495332", "n": 6030, "models": { "base_1.5b": { "overall": { "n": 6030, "exact_match": 0.1856, "normalized_match": 0.1874, "edit_similarity": 0.3732, "accepted_prefix_tokens": 2.2488, "syntax_valid": 0.6774, "mongo_valid": 0.6048, "exec_valid": 0.4915, "ceiling_exec_valid": 0.9307, "exec_over_ceiling": 0.5281, "exec_given_ref_ok": 0.5276, "n_ref_ok": 5612, "nonexistent_fields_rate": 0.0818, "empty_rate": 0.0141, "normalized_match_ci95": [ 0.1778, 0.1974 ], "exec_over_ceiling_ci95": [ 0.5151, 0.5409 ] }, "by_context_kind": { "full": { "n": 3720, "exact_match": 0.1392, "normalized_match": 0.1398, "edit_similarity": 0.3221, "accepted_prefix_tokens": 2.1505, "syntax_valid": 0.6231, "mongo_valid": 0.5567, "exec_valid": 0.45, "ceiling_exec_valid": 0.9237, "exec_over_ceiling": 0.4872, "exec_given_ref_ok": 0.4863, "n_ref_ok": 3436, "nonexistent_fields_rate": 0.0715, "empty_rate": 0.0108 }, "irrelevant": { "n": 1155, "exact_match": 0.2442, "normalized_match": 0.2485, "edit_similarity": 0.43, "accepted_prefix_tokens": 2.3446, "syntax_valid": 0.7238, "mongo_valid": 0.6442, "exec_valid": 0.5342, "ceiling_exec_valid": 0.9455, "exec_over_ceiling": 0.565, "exec_given_ref_ok": 0.565, "n_ref_ok": 1092, "nonexistent_fields_rate": 0.0961, "empty_rate": 0.0208 }, "none": { "n": 1155, "exact_match": 0.2762, "normalized_match": 0.2797, "edit_similarity": 0.4809, "accepted_prefix_tokens": 2.4693, "syntax_valid": 0.8061, "mongo_valid": 0.7203, "exec_valid": 0.5827, "ceiling_exec_valid": 0.9385, "exec_over_ceiling": 0.6209, "exec_given_ref_ok": 0.6208, "n_ref_ok": 1084, "nonexistent_fields_rate": 0.1004, "empty_rate": 0.0182 } }, "by_cut_kind": { "key": { "n": 959, "exact_match": 0.1846, "normalized_match": 0.1856, "edit_similarity": 0.353, "accepted_prefix_tokens": 1.4943, "syntax_valid": 0.6486, "mongo_valid": 0.6173, "exec_valid": 0.5036, "ceiling_exec_valid": 0.9166, "exec_over_ceiling": 0.5494, "exec_given_ref_ok": 0.5495, "n_ref_ok": 879, "nonexistent_fields_rate": 0.0563, "empty_rate": 0.0 }, "member": { "n": 600, "exact_match": 0.4583, "normalized_match": 0.4583, "edit_similarity": 0.5483, "accepted_prefix_tokens": 1.2783, "syntax_valid": 0.85, "mongo_valid": 0.6467, "exec_valid": 0.575, "ceiling_exec_valid": 0.89, "exec_over_ceiling": 0.6461, "exec_given_ref_ok": 0.6461, "n_ref_ok": 534, "nonexistent_fields_rate": 0.0767, "empty_rate": 0.0 }, "multiline": { "n": 480, "exact_match": 0.0187, "normalized_match": 0.025, "edit_similarity": 0.3463, "accepted_prefix_tokens": 3.7896, "syntax_valid": 0.6167, "mongo_valid": 0.5542, "exec_valid": 0.4188, "ceiling_exec_valid": 0.9583, "exec_over_ceiling": 0.437, "exec_given_ref_ok": 0.437, "n_ref_ok": 460, "nonexistent_fields_rate": 0.1437, "empty_rate": 0.0 }, "operator": { "n": 841, "exact_match": 0.2331, "normalized_match": 0.2342, "edit_similarity": 0.3963, "accepted_prefix_tokens": 2.4388, "syntax_valid": 0.6338, "mongo_valid": 0.5945, "exec_valid": 0.5125, "ceiling_exec_valid": 0.9489, "exec_over_ceiling": 0.5401, "exec_given_ref_ok": 0.5401, "n_ref_ok": 798, "nonexistent_fields_rate": 0.0559, "empty_rate": 0.0 }, "partial": { "n": 600, "exact_match": 0.3333, "normalized_match": 0.3333, "edit_similarity": 0.4636, "accepted_prefix_tokens": 2.0317, "syntax_valid": 0.72, "mongo_valid": 0.6367, "exec_valid": 0.4717, "ceiling_exec_valid": 0.91, "exec_over_ceiling": 0.5184, "exec_given_ref_ok": 0.5183, "n_ref_ok": 546, "nonexistent_fields_rate": 0.1217, "empty_rate": 0.01 }, "rest": { "n": 720, "exact_match": 0.0778, "normalized_match": 0.0778, "edit_similarity": 0.2764, "accepted_prefix_tokens": 2.2569, "syntax_valid": 0.7056, "mongo_valid": 0.6028, "exec_valid": 0.475, "ceiling_exec_valid": 0.9319, "exec_over_ceiling": 0.5097, "exec_given_ref_ok": 0.5097, "n_ref_ok": 671, "nonexistent_fields_rate": 0.0528, "empty_rate": 0.1069 }, "stage": { "n": 720, "exact_match": 0.0222, "normalized_match": 0.0306, "edit_similarity": 0.3191, "accepted_prefix_tokens": 3.5264, "syntax_valid": 0.6056, "mongo_valid": 0.5903, "exec_valid": 0.4139, "ceiling_exec_valid": 0.9486, "exec_over_ceiling": 0.4363, "exec_given_ref_ok": 0.4363, "n_ref_ok": 683, "nonexistent_fields_rate": 0.1361, "empty_rate": 0.0028 }, "stmt_start": { "n": 269, "exact_match": 0.1227, "normalized_match": 0.1227, "edit_similarity": 0.4425, "accepted_prefix_tokens": 2.6729, "syntax_valid": 0.8439, "mongo_valid": 0.6208, "exec_valid": 0.6283, "ceiling_exec_valid": 0.9777, "exec_over_ceiling": 0.6426, "exec_given_ref_ok": 0.6312, "n_ref_ok": 263, "nonexistent_fields_rate": 0.1004, "empty_rate": 0.0 }, "value": { "n": 841, "exact_match": 0.1867, "normalized_match": 0.1867, "edit_similarity": 0.306, "accepted_prefix_tokens": 1.6504, "syntax_valid": 0.6195, "mongo_valid": 0.5862, "exec_valid": 0.4899, "ceiling_exec_valid": 0.9251, "exec_over_ceiling": 0.5296, "exec_given_ref_ok": 0.5296, "n_ref_ok": 778, "nonexistent_fields_rate": 0.0488, "empty_rate": 0.0 } }, "by_domain": { "adtech": { "n": 3057, "exact_match": 0.1835, "normalized_match": 0.1861, "edit_similarity": 0.3734, "accepted_prefix_tokens": 2.2512, "syntax_valid": 0.6817, "mongo_valid": 0.6042, "exec_valid": 0.4953, "ceiling_exec_valid": 0.9303, "exec_over_ceiling": 0.5324, "exec_given_ref_ok": 0.5313, "n_ref_ok": 2844, "nonexistent_fields_rate": 0.0818, "empty_rate": 0.0144 }, "telecom": { "n": 2973, "exact_match": 0.1877, "normalized_match": 0.1887, "edit_similarity": 0.373, "accepted_prefix_tokens": 2.2462, "syntax_valid": 0.6731, "mongo_valid": 0.6054, "exec_valid": 0.4877, "ceiling_exec_valid": 0.931, "exec_over_ceiling": 0.5238, "exec_given_ref_ok": 0.5238, "n_ref_ok": 2768, "nonexistent_fields_rate": 0.0817, "empty_rate": 0.0138 } }, "by_author": { "grammar": { "n": 3220, "exact_match": 0.1261, "normalized_match": 0.1261, "edit_similarity": 0.2908, "accepted_prefix_tokens": 1.8161, "syntax_valid": 0.5894, "mongo_valid": 0.5509, "exec_valid": 0.3829, "ceiling_exec_valid": 0.8702, "exec_over_ceiling": 0.44, "exec_given_ref_ok": 0.439, "n_ref_ok": 2802, "nonexistent_fields_rate": 0.0761, "empty_rate": 0.0081 }, "llm": { "n": 2810, "exact_match": 0.2537, "normalized_match": 0.2577, "edit_similarity": 0.4676, "accepted_prefix_tokens": 2.7445, "syntax_valid": 0.7783, "mongo_valid": 0.6665, "exec_valid": 0.616, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.616, "exec_given_ref_ok": 0.616, "n_ref_ok": 2810, "nonexistent_fields_rate": 0.0883, "empty_rate": 0.021 } }, "by_author_context": { "grammar/full": { "n": 2121, "exact_match": 0.099, "normalized_match": 0.099, "edit_similarity": 0.2651, "accepted_prefix_tokens": 1.9043, "syntax_valid": 0.5413, "mongo_valid": 0.5101, "exec_valid": 0.3446, "ceiling_exec_valid": 0.8661, "exec_over_ceiling": 0.3979, "exec_given_ref_ok": 0.3963, "n_ref_ok": 1837, "nonexistent_fields_rate": 0.0707, "empty_rate": 0.0057 }, "grammar/irrelevant": { "n": 553, "exact_match": 0.1374, "normalized_match": 0.1374, "edit_similarity": 0.2987, "accepted_prefix_tokens": 1.5443, "syntax_valid": 0.6239, "mongo_valid": 0.5714, "exec_valid": 0.4014, "ceiling_exec_valid": 0.8861, "exec_over_ceiling": 0.453, "exec_given_ref_ok": 0.4531, "n_ref_ok": 490, "nonexistent_fields_rate": 0.0922, "empty_rate": 0.0163 }, "grammar/none": { "n": 546, "exact_match": 0.2198, "normalized_match": 0.2198, "edit_similarity": 0.3827, "accepted_prefix_tokens": 1.7491, "syntax_valid": 0.7418, "mongo_valid": 0.6886, "exec_valid": 0.5128, "ceiling_exec_valid": 0.87, "exec_over_ceiling": 0.5894, "exec_given_ref_ok": 0.5895, "n_ref_ok": 475, "nonexistent_fields_rate": 0.0806, "empty_rate": 0.0092 }, "llm/full": { "n": 1599, "exact_match": 0.1926, "normalized_match": 0.1939, "edit_similarity": 0.3977, "accepted_prefix_tokens": 2.4772, "syntax_valid": 0.7317, "mongo_valid": 0.6185, "exec_valid": 0.5897, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.5897, "exec_given_ref_ok": 0.5897, "n_ref_ok": 1599, "nonexistent_fields_rate": 0.0725, "empty_rate": 0.0175 }, "llm/irrelevant": { "n": 602, "exact_match": 0.3422, "normalized_match": 0.3505, "edit_similarity": 0.5507, "accepted_prefix_tokens": 3.0797, "syntax_valid": 0.8156, "mongo_valid": 0.711, "exec_valid": 0.6561, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.6561, "exec_given_ref_ok": 0.6561, "n_ref_ok": 602, "nonexistent_fields_rate": 0.0997, "empty_rate": 0.0249 }, "llm/none": { "n": 609, "exact_match": 0.3268, "normalized_match": 0.3333, "edit_similarity": 0.5689, "accepted_prefix_tokens": 3.1149, "syntax_valid": 0.8637, "mongo_valid": 0.7488, "exec_valid": 0.6453, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.6453, "exec_given_ref_ok": 0.6453, "n_ref_ok": 609, "nonexistent_fields_rate": 0.1182, "empty_rate": 0.0263 } } }, "t1_distill": { "overall": { "n": 6030, "exact_match": 0.3687, "normalized_match": 0.3697, "edit_similarity": 0.6188, "accepted_prefix_tokens": 2.7891, "syntax_valid": 0.9902, "mongo_valid": 0.962, "exec_valid": 0.7859, "ceiling_exec_valid": 0.9307, "exec_over_ceiling": 0.8444, "exec_given_ref_ok": 0.8437, "n_ref_ok": 5612, "nonexistent_fields_rate": 0.1007, "empty_rate": 0.0017, "normalized_match_ci95": [ 0.3576, 0.382 ], "exec_over_ceiling_ci95": [ 0.8342, 0.854 ] }, "by_context_kind": { "full": { "n": 3720, "exact_match": 0.3151, "normalized_match": 0.3153, "edit_similarity": 0.5746, "accepted_prefix_tokens": 2.7946, "syntax_valid": 0.9892, "mongo_valid": 0.9613, "exec_valid": 0.7879, "ceiling_exec_valid": 0.9237, "exec_over_ceiling": 0.853, "exec_given_ref_ok": 0.8519, "n_ref_ok": 3436, "nonexistent_fields_rate": 0.0919, "empty_rate": 0.0022 }, "irrelevant": { "n": 1155, "exact_match": 0.4511, "normalized_match": 0.4545, "edit_similarity": 0.689, "accepted_prefix_tokens": 2.7307, "syntax_valid": 0.9913, "mongo_valid": 0.9645, "exec_valid": 0.7931, "ceiling_exec_valid": 0.9455, "exec_over_ceiling": 0.8388, "exec_given_ref_ok": 0.8388, "n_ref_ok": 1092, "nonexistent_fields_rate": 0.1134, "empty_rate": 0.0 }, "none": { "n": 1155, "exact_match": 0.4589, "normalized_match": 0.4597, "edit_similarity": 0.6908, "accepted_prefix_tokens": 2.8294, "syntax_valid": 0.9922, "mongo_valid": 0.9619, "exec_valid": 0.7723, "ceiling_exec_valid": 0.9385, "exec_over_ceiling": 0.8229, "exec_given_ref_ok": 0.8229, "n_ref_ok": 1084, "nonexistent_fields_rate": 0.116, "empty_rate": 0.0017 } }, "by_cut_kind": { "key": { "n": 959, "exact_match": 0.3566, "normalized_match": 0.3566, "edit_similarity": 0.5737, "accepted_prefix_tokens": 1.8749, "syntax_valid": 0.9958, "mongo_valid": 0.9698, "exec_valid": 0.7737, "ceiling_exec_valid": 0.9166, "exec_over_ceiling": 0.8441, "exec_given_ref_ok": 0.8441, "n_ref_ok": 879, "nonexistent_fields_rate": 0.1157, "empty_rate": 0.0 }, "member": { "n": 600, "exact_match": 0.785, "normalized_match": 0.785, "edit_similarity": 0.8609, "accepted_prefix_tokens": 1.95, "syntax_valid": 0.9917, "mongo_valid": 0.9567, "exec_valid": 0.8233, "ceiling_exec_valid": 0.89, "exec_over_ceiling": 0.9251, "exec_given_ref_ok": 0.9251, "n_ref_ok": 534, "nonexistent_fields_rate": 0.0417, "empty_rate": 0.0 }, "multiline": { "n": 480, "exact_match": 0.0229, "normalized_match": 0.0271, "edit_similarity": 0.4367, "accepted_prefix_tokens": 4.6229, "syntax_valid": 0.9917, "mongo_valid": 0.9521, "exec_valid": 0.65, "ceiling_exec_valid": 0.9583, "exec_over_ceiling": 0.6783, "exec_given_ref_ok": 0.6783, "n_ref_ok": 460, "nonexistent_fields_rate": 0.25, "empty_rate": 0.0 }, "operator": { "n": 841, "exact_match": 0.4554, "normalized_match": 0.4554, "edit_similarity": 0.6871, "accepted_prefix_tokens": 2.9382, "syntax_valid": 0.9881, "mongo_valid": 0.9584, "exec_valid": 0.8502, "ceiling_exec_valid": 0.9489, "exec_over_ceiling": 0.896, "exec_given_ref_ok": 0.896, "n_ref_ok": 798, "nonexistent_fields_rate": 0.0606, "empty_rate": 0.0 }, "partial": { "n": 600, "exact_match": 0.6083, "normalized_match": 0.6083, "edit_similarity": 0.7702, "accepted_prefix_tokens": 2.38, "syntax_valid": 0.9933, "mongo_valid": 0.9633, "exec_valid": 0.735, "ceiling_exec_valid": 0.91, "exec_over_ceiling": 0.8077, "exec_given_ref_ok": 0.8077, "n_ref_ok": 546, "nonexistent_fields_rate": 0.1333, "empty_rate": 0.0017 }, "rest": { "n": 720, "exact_match": 0.2875, "normalized_match": 0.2875, "edit_similarity": 0.5632, "accepted_prefix_tokens": 2.6264, "syntax_valid": 0.9792, "mongo_valid": 0.9458, "exec_valid": 0.7556, "ceiling_exec_valid": 0.9319, "exec_over_ceiling": 0.8108, "exec_given_ref_ok": 0.8107, "n_ref_ok": 671, "nonexistent_fields_rate": 0.0764, "empty_rate": 0.0097 }, "stage": { "n": 720, "exact_match": 0.0347, "normalized_match": 0.0403, "edit_similarity": 0.4278, "accepted_prefix_tokens": 4.6514, "syntax_valid": 0.9875, "mongo_valid": 0.9778, "exec_valid": 0.7264, "ceiling_exec_valid": 0.9486, "exec_over_ceiling": 0.7658, "exec_given_ref_ok": 0.7657, "n_ref_ok": 683, "nonexistent_fields_rate": 0.1583, "empty_rate": 0.0028 }, "stmt_start": { "n": 269, "exact_match": 0.2379, "normalized_match": 0.2379, "edit_similarity": 0.5815, "accepted_prefix_tokens": 3.7212, "syntax_valid": 0.9777, "mongo_valid": 0.9368, "exec_valid": 0.7732, "ceiling_exec_valid": 0.9777, "exec_over_ceiling": 0.7908, "exec_given_ref_ok": 0.7757, "n_ref_ok": 263, "nonexistent_fields_rate": 0.1561, "empty_rate": 0.0 }, "value": { "n": 841, "exact_match": 0.4221, "normalized_match": 0.4221, "edit_similarity": 0.6483, "accepted_prefix_tokens": 1.7729, "syntax_valid": 0.9976, "mongo_valid": 0.9738, "exec_valid": 0.9037, "ceiling_exec_valid": 0.9251, "exec_over_ceiling": 0.9769, "exec_given_ref_ok": 0.9769, "n_ref_ok": 778, "nonexistent_fields_rate": 0.0107, "empty_rate": 0.0 } }, "by_domain": { "adtech": { "n": 3057, "exact_match": 0.3749, "normalized_match": 0.3759, "edit_similarity": 0.6244, "accepted_prefix_tokens": 2.8911, "syntax_valid": 0.9908, "mongo_valid": 0.9627, "exec_valid": 0.7975, "ceiling_exec_valid": 0.9303, "exec_over_ceiling": 0.8573, "exec_given_ref_ok": 0.8558, "n_ref_ok": 2844, "nonexistent_fields_rate": 0.0932, "empty_rate": 0.002 }, "telecom": { "n": 2973, "exact_match": 0.3623, "normalized_match": 0.3633, "edit_similarity": 0.613, "accepted_prefix_tokens": 2.6842, "syntax_valid": 0.9896, "mongo_valid": 0.9613, "exec_valid": 0.774, "ceiling_exec_valid": 0.931, "exec_over_ceiling": 0.8314, "exec_given_ref_ok": 0.8313, "n_ref_ok": 2768, "nonexistent_fields_rate": 0.1083, "empty_rate": 0.0013 } }, "by_author": { "grammar": { "n": 3220, "exact_match": 0.3758, "normalized_match": 0.3761, "edit_similarity": 0.633, "accepted_prefix_tokens": 3.027, "syntax_valid": 0.9975, "mongo_valid": 0.9969, "exec_valid": 0.7898, "ceiling_exec_valid": 0.8702, "exec_over_ceiling": 0.9076, "exec_given_ref_ok": 0.9061, "n_ref_ok": 2802, "nonexistent_fields_rate": 0.0724, "empty_rate": 0.0 }, "llm": { "n": 2810, "exact_match": 0.3605, "normalized_match": 0.3623, "edit_similarity": 0.6025, "accepted_prefix_tokens": 2.5164, "syntax_valid": 0.9819, "mongo_valid": 0.9221, "exec_valid": 0.7815, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.7815, "exec_given_ref_ok": 0.7815, "n_ref_ok": 2810, "nonexistent_fields_rate": 0.1331, "empty_rate": 0.0036 } }, "by_author_context": { "grammar/full": { "n": 2121, "exact_match": 0.3215, "normalized_match": 0.322, "edit_similarity": 0.5933, "accepted_prefix_tokens": 3.1061, "syntax_valid": 0.9972, "mongo_valid": 0.9972, "exec_valid": 0.7845, "ceiling_exec_valid": 0.8661, "exec_over_ceiling": 0.9058, "exec_given_ref_ok": 0.9036, "n_ref_ok": 1837, "nonexistent_fields_rate": 0.0726, "empty_rate": 0.0 }, "grammar/irrelevant": { "n": 553, "exact_match": 0.4629, "normalized_match": 0.4629, "edit_similarity": 0.6965, "accepted_prefix_tokens": 2.8101, "syntax_valid": 0.9982, "mongo_valid": 0.9964, "exec_valid": 0.8065, "ceiling_exec_valid": 0.8861, "exec_over_ceiling": 0.9102, "exec_given_ref_ok": 0.9102, "n_ref_ok": 490, "nonexistent_fields_rate": 0.0687, "empty_rate": 0.0 }, "grammar/none": { "n": 546, "exact_match": 0.4982, "normalized_match": 0.4982, "edit_similarity": 0.7231, "accepted_prefix_tokens": 2.9396, "syntax_valid": 0.9982, "mongo_valid": 0.9963, "exec_valid": 0.793, "ceiling_exec_valid": 0.87, "exec_over_ceiling": 0.9115, "exec_given_ref_ok": 0.9116, "n_ref_ok": 475, "nonexistent_fields_rate": 0.0751, "empty_rate": 0.0 }, "llm/full": { "n": 1599, "exact_match": 0.3064, "normalized_match": 0.3064, "edit_similarity": 0.5499, "accepted_prefix_tokens": 2.3815, "syntax_valid": 0.9787, "mongo_valid": 0.9137, "exec_valid": 0.7924, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.7924, "exec_given_ref_ok": 0.7924, "n_ref_ok": 1599, "nonexistent_fields_rate": 0.1176, "empty_rate": 0.005 }, "llm/irrelevant": { "n": 602, "exact_match": 0.4402, "normalized_match": 0.4468, "edit_similarity": 0.6822, "accepted_prefix_tokens": 2.6578, "syntax_valid": 0.985, "mongo_valid": 0.9352, "exec_valid": 0.7807, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.7807, "exec_given_ref_ok": 0.7807, "n_ref_ok": 602, "nonexistent_fields_rate": 0.1545, "empty_rate": 0.0 }, "llm/none": { "n": 609, "exact_match": 0.4236, "normalized_match": 0.4253, "edit_similarity": 0.6618, "accepted_prefix_tokens": 2.7307, "syntax_valid": 0.9869, "mongo_valid": 0.931, "exec_valid": 0.7537, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.7537, "exec_given_ref_ok": 0.7537, "n_ref_ok": 609, "nonexistent_fields_rate": 0.1527, "empty_rate": 0.0033 } } }, "t1_v3": { "overall": { "n": 6030, "exact_match": 0.4516, "normalized_match": 0.4537, "edit_similarity": 0.7011, "accepted_prefix_tokens": 3.8992, "syntax_valid": 0.9972, "mongo_valid": 0.9718, "exec_valid": 0.8307, "ceiling_exec_valid": 0.9307, "exec_over_ceiling": 0.8926, "exec_given_ref_ok": 0.892, "n_ref_ok": 5612, "nonexistent_fields_rate": 0.0788, "empty_rate": 0.0, "normalized_match_ci95": [ 0.4412, 0.4663 ], "exec_over_ceiling_ci95": [ 0.8839, 0.9007 ] }, "by_context_kind": { "full": { "n": 3720, "exact_match": 0.3812, "normalized_match": 0.3828, "edit_similarity": 0.6471, "accepted_prefix_tokens": 3.7911, "syntax_valid": 0.9965, "mongo_valid": 0.9704, "exec_valid": 0.8204, "ceiling_exec_valid": 0.9237, "exec_over_ceiling": 0.8882, "exec_given_ref_ok": 0.8874, "n_ref_ok": 3436, "nonexistent_fields_rate": 0.075, "empty_rate": 0.0 }, "irrelevant": { "n": 1155, "exact_match": 0.5524, "normalized_match": 0.555, "edit_similarity": 0.782, "accepted_prefix_tokens": 3.8952, "syntax_valid": 0.9965, "mongo_valid": 0.974, "exec_valid": 0.8511, "ceiling_exec_valid": 0.9455, "exec_over_ceiling": 0.9002, "exec_given_ref_ok": 0.9002, "n_ref_ok": 1092, "nonexistent_fields_rate": 0.0823, "empty_rate": 0.0 }, "none": { "n": 1155, "exact_match": 0.5775, "normalized_match": 0.581, "edit_similarity": 0.7941, "accepted_prefix_tokens": 4.2511, "syntax_valid": 1.0, "mongo_valid": 0.974, "exec_valid": 0.8433, "ceiling_exec_valid": 0.9385, "exec_over_ceiling": 0.8986, "exec_given_ref_ok": 0.8985, "n_ref_ok": 1084, "nonexistent_fields_rate": 0.0874, "empty_rate": 0.0 } }, "by_cut_kind": { "key": { "n": 959, "exact_match": 0.439, "normalized_match": 0.44, "edit_similarity": 0.6624, "accepted_prefix_tokens": 2.4703, "syntax_valid": 0.999, "mongo_valid": 0.975, "exec_valid": 0.8373, "ceiling_exec_valid": 0.9166, "exec_over_ceiling": 0.9135, "exec_given_ref_ok": 0.9135, "n_ref_ok": 879, "nonexistent_fields_rate": 0.0688, "empty_rate": 0.0 }, "member": { "n": 600, "exact_match": 0.835, "normalized_match": 0.835, "edit_similarity": 0.895, "accepted_prefix_tokens": 2.17, "syntax_valid": 0.9983, "mongo_valid": 0.9617, "exec_valid": 0.8267, "ceiling_exec_valid": 0.89, "exec_over_ceiling": 0.9289, "exec_given_ref_ok": 0.9288, "n_ref_ok": 534, "nonexistent_fields_rate": 0.04, "empty_rate": 0.0 }, "multiline": { "n": 480, "exact_match": 0.1062, "normalized_match": 0.1125, "edit_similarity": 0.5706, "accepted_prefix_tokens": 8.2354, "syntax_valid": 0.9812, "mongo_valid": 0.9646, "exec_valid": 0.7021, "ceiling_exec_valid": 0.9583, "exec_over_ceiling": 0.7327, "exec_given_ref_ok": 0.7326, "n_ref_ok": 460, "nonexistent_fields_rate": 0.1896, "empty_rate": 0.0 }, "operator": { "n": 841, "exact_match": 0.5743, "normalized_match": 0.5755, "edit_similarity": 0.7736, "accepted_prefix_tokens": 3.8359, "syntax_valid": 1.0, "mongo_valid": 0.9762, "exec_valid": 0.8859, "ceiling_exec_valid": 0.9489, "exec_over_ceiling": 0.9336, "exec_given_ref_ok": 0.9336, "n_ref_ok": 798, "nonexistent_fields_rate": 0.0523, "empty_rate": 0.0 }, "partial": { "n": 600, "exact_match": 0.6567, "normalized_match": 0.6567, "edit_similarity": 0.8213, "accepted_prefix_tokens": 2.8333, "syntax_valid": 0.9983, "mongo_valid": 0.9717, "exec_valid": 0.7817, "ceiling_exec_valid": 0.91, "exec_over_ceiling": 0.859, "exec_given_ref_ok": 0.859, "n_ref_ok": 546, "nonexistent_fields_rate": 0.1133, "empty_rate": 0.0 }, "rest": { "n": 720, "exact_match": 0.4, "normalized_match": 0.4, "edit_similarity": 0.6704, "accepted_prefix_tokens": 4.0847, "syntax_valid": 1.0, "mongo_valid": 0.9667, "exec_valid": 0.8375, "ceiling_exec_valid": 0.9319, "exec_over_ceiling": 0.8987, "exec_given_ref_ok": 0.8987, "n_ref_ok": 671, "nonexistent_fields_rate": 0.0681, "empty_rate": 0.0 }, "stage": { "n": 720, "exact_match": 0.0958, "normalized_match": 0.1056, "edit_similarity": 0.5092, "accepted_prefix_tokens": 6.6472, "syntax_valid": 0.9958, "mongo_valid": 0.9847, "exec_valid": 0.7778, "ceiling_exec_valid": 0.9486, "exec_over_ceiling": 0.8199, "exec_given_ref_ok": 0.8199, "n_ref_ok": 683, "nonexistent_fields_rate": 0.1389, "empty_rate": 0.0 }, "stmt_start": { "n": 269, "exact_match": 0.368, "normalized_match": 0.3717, "edit_similarity": 0.7071, "accepted_prefix_tokens": 4.8662, "syntax_valid": 0.9963, "mongo_valid": 0.9517, "exec_valid": 0.8699, "ceiling_exec_valid": 0.9777, "exec_over_ceiling": 0.8897, "exec_given_ref_ok": 0.8783, "n_ref_ok": 263, "nonexistent_fields_rate": 0.0892, "empty_rate": 0.0 }, "value": { "n": 841, "exact_match": 0.4958, "normalized_match": 0.4958, "edit_similarity": 0.7118, "accepted_prefix_tokens": 2.2901, "syntax_valid": 0.9988, "mongo_valid": 0.975, "exec_valid": 0.9061, "ceiling_exec_valid": 0.9251, "exec_over_ceiling": 0.9795, "exec_given_ref_ok": 0.9794, "n_ref_ok": 778, "nonexistent_fields_rate": 0.0107, "empty_rate": 0.0 } }, "by_domain": { "adtech": { "n": 3057, "exact_match": 0.4544, "normalized_match": 0.4576, "edit_similarity": 0.7054, "accepted_prefix_tokens": 4.0203, "syntax_valid": 0.9974, "mongo_valid": 0.9702, "exec_valid": 0.8328, "ceiling_exec_valid": 0.9303, "exec_over_ceiling": 0.8952, "exec_given_ref_ok": 0.8942, "n_ref_ok": 2844, "nonexistent_fields_rate": 0.0772, "empty_rate": 0.0 }, "telecom": { "n": 2973, "exact_match": 0.4487, "normalized_match": 0.4497, "edit_similarity": 0.6967, "accepted_prefix_tokens": 3.7746, "syntax_valid": 0.997, "mongo_valid": 0.9734, "exec_valid": 0.8285, "ceiling_exec_valid": 0.931, "exec_over_ceiling": 0.8899, "exec_given_ref_ok": 0.8898, "n_ref_ok": 2768, "nonexistent_fields_rate": 0.0804, "empty_rate": 0.0 } }, "by_author": { "grammar": { "n": 3220, "exact_match": 0.3835, "normalized_match": 0.3835, "edit_similarity": 0.641, "accepted_prefix_tokens": 3.0596, "syntax_valid": 0.9988, "mongo_valid": 0.9981, "exec_valid": 0.8, "ceiling_exec_valid": 0.8702, "exec_over_ceiling": 0.9193, "exec_given_ref_ok": 0.9183, "n_ref_ok": 2802, "nonexistent_fields_rate": 0.0528, "empty_rate": 0.0 }, "llm": { "n": 2810, "exact_match": 0.5295, "normalized_match": 0.5342, "edit_similarity": 0.77, "accepted_prefix_tokens": 4.8612, "syntax_valid": 0.9954, "mongo_valid": 0.9416, "exec_valid": 0.8658, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.8658, "exec_given_ref_ok": 0.8658, "n_ref_ok": 2810, "nonexistent_fields_rate": 0.1085, "empty_rate": 0.0 } }, "by_author_context": { "grammar/full": { "n": 2121, "exact_match": 0.3234, "normalized_match": 0.3234, "edit_similarity": 0.5932, "accepted_prefix_tokens": 3.075, "syntax_valid": 0.9986, "mongo_valid": 0.9976, "exec_valid": 0.7888, "ceiling_exec_valid": 0.8661, "exec_over_ceiling": 0.9107, "exec_given_ref_ok": 0.9091, "n_ref_ok": 1837, "nonexistent_fields_rate": 0.0528, "empty_rate": 0.0 }, "grammar/irrelevant": { "n": 553, "exact_match": 0.4792, "normalized_match": 0.4792, "edit_similarity": 0.7223, "accepted_prefix_tokens": 3.0362, "syntax_valid": 0.9982, "mongo_valid": 0.9982, "exec_valid": 0.83, "ceiling_exec_valid": 0.8861, "exec_over_ceiling": 0.9367, "exec_given_ref_ok": 0.9367, "n_ref_ok": 490, "nonexistent_fields_rate": 0.0488, "empty_rate": 0.0 }, "grammar/none": { "n": 546, "exact_match": 0.5201, "normalized_match": 0.5201, "edit_similarity": 0.7445, "accepted_prefix_tokens": 3.0238, "syntax_valid": 1.0, "mongo_valid": 1.0, "exec_valid": 0.8132, "ceiling_exec_valid": 0.87, "exec_over_ceiling": 0.9347, "exec_given_ref_ok": 0.9347, "n_ref_ok": 475, "nonexistent_fields_rate": 0.0568, "empty_rate": 0.0 }, "llm/full": { "n": 1599, "exact_match": 0.4578, "normalized_match": 0.4615, "edit_similarity": 0.7186, "accepted_prefix_tokens": 4.7411, "syntax_valid": 0.9937, "mongo_valid": 0.9343, "exec_valid": 0.8624, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.8624, "exec_given_ref_ok": 0.8624, "n_ref_ok": 1599, "nonexistent_fields_rate": 0.1044, "empty_rate": 0.0 }, "llm/irrelevant": { "n": 602, "exact_match": 0.6196, "normalized_match": 0.6246, "edit_similarity": 0.8369, "accepted_prefix_tokens": 4.6844, "syntax_valid": 0.995, "mongo_valid": 0.9518, "exec_valid": 0.8704, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.8704, "exec_given_ref_ok": 0.8704, "n_ref_ok": 602, "nonexistent_fields_rate": 0.113, "empty_rate": 0.0 }, "llm/none": { "n": 609, "exact_match": 0.6289, "normalized_match": 0.6355, "edit_similarity": 0.8386, "accepted_prefix_tokens": 5.3514, "syntax_valid": 1.0, "mongo_valid": 0.9507, "exec_valid": 0.8703, "ceiling_exec_valid": 1.0, "exec_over_ceiling": 0.8703, "exec_given_ref_ok": 0.8703, "n_ref_ok": 609, "nonexistent_fields_rate": 0.1149, "empty_rate": 0.0 } } } }, "val_v2c": { "base_1.5b": { "n": 2000, "normalized_match": 0.142, "exact_match": 0.142, "edit_similarity": 0.3057, "syntax_valid": 0.667, "mongo_valid": 0.595, "exec_valid": 0.316, "ceiling_exec_valid": 0.6255, "exec_over_ceiling": 0.5052, "nonexistent_fields_rate": 0.051, "nonexistent_fields_full": 0.0376, "by_context_kind": { "full": { "normalized_match": 0.1475, "exec_valid": 0.3364 }, "irrelevant": { "normalized_match": 0.1061, "exec_valid": 0.2172 }, "none": { "normalized_match": 0.129, "exec_valid": 0.2258 } } }, "t1_distill": { "n": 2000, "normalized_match": 0.4105, "exact_match": 0.4105, "edit_similarity": 0.6568, "syntax_valid": 0.9955, "mongo_valid": 0.9945, "exec_valid": 0.595, "ceiling_exec_valid": 0.6255, "exec_over_ceiling": 0.9512, "nonexistent_fields_rate": 0.029, "nonexistent_fields_full": 0.0158, "by_context_kind": { "full": { "normalized_match": 0.4159, "exec_valid": 0.6199 }, "irrelevant": { "normalized_match": 0.4091, "exec_valid": 0.5051 }, "none": { "normalized_match": 0.3548, "exec_valid": 0.4452 } } }, "t1_v3": { "n": 2000, "normalized_match": 0.41, "exact_match": 0.41, "edit_similarity": 0.6536, "syntax_valid": 0.998, "mongo_valid": 0.9975, "exec_valid": 0.6015, "ceiling_exec_valid": 0.6255, "exec_over_ceiling": 0.9616, "nonexistent_fields_rate": 0.021, "nonexistent_fields_full": 0.0121, "by_context_kind": { "full": { "normalized_match": 0.4183, "exec_valid": 0.6211 }, "irrelevant": { "normalized_match": 0.3838, "exec_valid": 0.5303 }, "none": { "normalized_match": 0.3548, "exec_valid": 0.4839 } } } }, "pairs": { "t1_distill_vs_base_1.5b": { "context_kind": { "n_aligned": 6030, "overall": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 2.2487562189054726, "mean_b": 2.789054726368159, "delta": 0.5402985074626866, "ci_low": 0.4485820895522389, "ci_high": 0.6321766169154228, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 0.37318669983416247, "mean_b": 0.6188011608623549, "delta": 0.24561446102819237, "ci_low": 0.23526142454394694, "ci_high": 0.2561746745439469, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.1855721393034826, "rate_b": 0.3686567164179104, "delta": 0.18308457711442785, "better": 1307, "worse": 203, "equal": 4520, "p": 1.4405600645418062e-197, "ci_low": 0.17147180762852407, "ci_high": 0.19502487562189055, "superior": true, "p_holm": 2.5930081161752513e-196 }, { "metric": "exec_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.49154228855721394, "rate_b": 0.7859038142620233, "delta": 0.2943615257048093, "better": 2104, "worse": 329, "equal": 3597, "p": 9.56375726e-316, "ci_low": 0.2800995024875622, "ci_high": 0.3086235489220564, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.9127514525e-314 }, { "metric": "mongo_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.6048092868988392, "rate_b": 0.9620232172470978, "delta": 0.3572139303482587, "better": 2194, "worse": 40, "equal": 3796, "p": 0.0, "ci_low": 0.3449419568822554, "ci_high": 0.369983416252073, "superior": true, "p_holm": 0.0 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.9182421227197347, "rate_b": 0.8993366500829187, "delta": -0.01890547263681592, "better": 305, "worse": 419, "equal": 5306, "p": 2.5795404069987098e-05, "ci_low": -0.02769485903814262, "ci_high": -0.010281923714759536, "superior": false, "p_holm": 0.00010318161627994839 }, { "metric": "normalized_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.18739635157545606, "rate_b": 0.36965174129353234, "delta": 0.18225538971807628, "better": 1308, "worse": 209, "equal": 4513, "p": 2.0345053924323034e-194, "ci_low": 0.17064676616915422, "ci_high": 0.19419568822553898, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 3.4586591671349157e-193 }, { "metric": "syntax_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.6774461028192371, "rate_b": 0.9902155887230514, "delta": 0.31276948590381426, "better": 1919, "worse": 33, "equal": 4078, "p": 0.0, "ci_low": 0.30082918739635156, "ci_high": 0.3250414593698176, "superior": true, "p_holm": 0.0 } ], "groups": { "full": { "n": 3720, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 3720, "lower_is_better": false, "mean_a": 2.150537634408602, "mean_b": 2.7946236559139783, "delta": 0.6440860215053763, "ci_low": 0.5311760752688172, "ci_high": 0.7588709677419355, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 3720, "lower_is_better": false, "mean_a": 0.3220927956989247, "mean_b": 0.5746390860215054, "delta": 0.25254629032258064, "ci_low": 0.23996857459677418, "ci_high": 0.26556272849462365, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.139247311827957, "rate_b": 0.3150537634408602, "delta": 0.17580645161290323, "better": 735, "worse": 81, "equal": 2904, "p": 1.0249568878429672e-132, "ci_low": 0.1620967741935484, "ci_high": 0.18978494623655914, "superior": true, "p_holm": 1.6399310205487475e-131 }, { "metric": "exec_valid", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.45, "rate_b": 0.7879032258064517, "delta": 0.3379032258064516, "better": 1451, "worse": 194, "equal": 2075, "p": 6.806754608883945e-238, "ci_low": 0.3193548387096774, "ci_high": 0.35618279569892475, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.2932833756879496e-236 }, { "metric": "mongo_valid", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.5567204301075269, "rate_b": 0.9612903225806452, "delta": 0.40456989247311825, "better": 1528, "worse": 23, "equal": 2169, "p": 0.0, "ci_low": 0.3881720430107527, "ci_high": 0.4206989247311828, "superior": true, "p_holm": 0.0 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.928494623655914, "rate_b": 0.9080645161290323, "delta": -0.02043010752688172, "better": 174, "worse": 250, "equal": 3296, "p": 0.00026101422945458636, "ci_low": -0.03118279569892473, "ci_high": -0.00967741935483871, "superior": false, "p_holm": 0.0007830426883637591 }, { "metric": "normalized_match", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.13978494623655913, "rate_b": 0.3153225806451613, "delta": 0.17553763440860215, "better": 736, "worse": 83, "equal": 2901, "p": 1.4039628267736683e-131, "ci_low": 0.16182795698924732, "ci_high": 0.18951612903225806, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 2.1059442401605025e-130 }, { "metric": "syntax_valid", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.6231182795698925, "rate_b": 0.989247311827957, "delta": 0.36612903225806454, "better": 1385, "worse": 23, "equal": 2312, "p": 0.0, "ci_low": 0.3497311827956989, "ci_high": 0.381989247311828, "superior": true, "p_holm": 0.0 } ] }, "irrelevant": { "n": 1155, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 2.3445887445887448, "mean_b": 2.7307359307359307, "delta": 0.38614718614718613, "ci_low": 0.1774891774891775, "ci_high": 0.6043290043290044, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 0.4300482251082251, "mean_b": 0.6890493506493506, "delta": 0.25900112554112553, "ci_low": 0.2331739329004329, "ci_high": 0.2849483398268398, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.24415584415584415, "rate_b": 0.4510822510822511, "delta": 0.20692640692640693, "better": 290, "worse": 51, "equal": 814, "p": 1.0054457699912198e-41, "ci_low": 0.17835497835497835, "ci_high": 0.2354978354978355, "superior": true, "p_holm": 9.049011929920978e-41 }, { "metric": "exec_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.5341991341991342, "rate_b": 0.793073593073593, "delta": 0.25887445887445887, "better": 357, "worse": 58, "equal": 740, "p": 1.2878161280010718e-53, "ci_low": 0.22857142857142856, "ci_high": 0.29004329004329005, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.2878161280010719e-52 }, { "metric": "mongo_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.6441558441558441, "rate_b": 0.9645021645021645, "delta": 0.3203463203463203, "better": 378, "worse": 8, "equal": 769, "p": 1.4728552360527823e-100, "ci_low": 0.29264069264069265, "ci_high": 0.34805194805194806, "superior": true, "p_holm": 2.0619973304738953e-99 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.9038961038961039, "rate_b": 0.8865800865800866, "delta": -0.017316017316017316, "better": 64, "worse": 84, "equal": 1007, "p": 0.11803924678172636, "ci_low": -0.0380952380952381, "ci_high": 0.0025974025974025974, "superior": false, "p_holm": 0.23607849356345273 }, { "metric": "normalized_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.24848484848484848, "rate_b": 0.45454545454545453, "delta": 0.20606060606060606, "better": 290, "worse": 52, "equal": 813, "p": 3.3200986933768434e-41, "ci_low": 0.1774891774891775, "ci_high": 0.23463203463203464, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 2.6560789547014747e-40 }, { "metric": "syntax_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.7238095238095238, "rate_b": 0.9913419913419913, "delta": 0.2675324675324675, "better": 313, "worse": 4, "equal": 838, "p": 3.1321784783752422e-87, "ci_low": 0.24155844155844156, "ci_high": 0.2935064935064935, "superior": true, "p_holm": 4.071832021887815e-86 } ] }, "none": { "n": 1155, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 2.469264069264069, "mean_b": 2.8294372294372296, "delta": 0.3601731601731602, "ci_low": 0.16277056277056276, "ci_high": 0.5627705627705628, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 0.48088735930735926, "mean_b": 0.690789264069264, "delta": 0.20990190476190476, "ci_low": 0.18358663203463205, "ci_high": 0.2359251753246753, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.2761904761904762, "rate_b": 0.4588744588744589, "delta": 0.18268398268398267, "better": 282, "worse": 71, "equal": 802, "p": 6.925330255223634e-31, "ci_low": 0.1523809523809524, "ci_high": 0.21298701298701297, "superior": true, "p_holm": 4.15519815313418e-30 }, { "metric": "exec_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.5826839826839827, "rate_b": 0.7722943722943723, "delta": 0.18961038961038962, "better": 296, "worse": 77, "equal": 782, "p": 2.1825485902965193e-31, "ci_low": 0.15844155844155844, "ci_high": 0.22077922077922077, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.5277840132075635e-30 }, { "metric": "mongo_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.7203463203463204, "rate_b": 0.9619047619047619, "delta": 0.24155844155844156, "better": 288, "worse": 9, "equal": 858, "p": 3.5538904469071796e-73, "ci_low": 0.2164285714285716, "ci_high": 0.2675324675324675, "superior": true, "p_holm": 4.2646685362886157e-72 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.8995670995670996, "rate_b": 0.883982683982684, "delta": -0.015584415584415584, "better": 67, "worse": 85, "equal": 1003, "p": 0.16772199317919748, "ci_low": -0.03636363636363636, "ci_high": 0.004329004329004329, "superior": false, "p_holm": 0.23607849356345273 }, { "metric": "normalized_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.27965367965367965, "rate_b": 0.4597402597402597, "delta": 0.1800865800865801, "better": 282, "worse": 74, "equal": 799, "p": 1.0098733511899805e-29, "ci_low": 0.1497835497835498, "ci_high": 0.21038961038961038, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 5.049366755949903e-29 }, { "metric": "syntax_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.806060606060606, "rate_b": 0.9922077922077922, "delta": 0.18614718614718614, "better": 221, "worse": 6, "equal": 928, "p": 1.6941577103211408e-57, "ci_low": 0.16363636363636364, "ci_high": 0.20952380952380953, "superior": true, "p_holm": 1.863573481353255e-56 } ] } }, "groups_skipped": {} }, "author": { "n_aligned": 6030, "overall": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 2.2487562189054726, "mean_b": 2.789054726368159, "delta": 0.5402985074626866, "ci_low": 0.4485820895522389, "ci_high": 0.6321766169154228, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 0.37318669983416247, "mean_b": 0.6188011608623549, "delta": 0.24561446102819237, "ci_low": 0.23526142454394694, "ci_high": 0.2561746745439469, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.1855721393034826, "rate_b": 0.3686567164179104, "delta": 0.18308457711442785, "better": 1307, "worse": 203, "equal": 4520, "p": 1.4405600645418062e-197, "ci_low": 0.17147180762852407, "ci_high": 0.19502487562189055, "superior": true, "p_holm": 1.4405600645418063e-196 }, { "metric": "exec_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.49154228855721394, "rate_b": 0.7859038142620233, "delta": 0.2943615257048093, "better": 2104, "worse": 329, "equal": 3597, "p": 9.56375726e-316, "ci_low": 0.2800995024875622, "ci_high": 0.3086235489220564, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.3389260167e-314 }, { "metric": "mongo_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.6048092868988392, "rate_b": 0.9620232172470978, "delta": 0.3572139303482587, "better": 2194, "worse": 40, "equal": 3796, "p": 0.0, "ci_low": 0.3449419568822554, "ci_high": 0.369983416252073, "superior": true, "p_holm": 0.0 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.9182421227197347, "rate_b": 0.8993366500829187, "delta": -0.01890547263681592, "better": 305, "worse": 419, "equal": 5306, "p": 2.5795404069987098e-05, "ci_low": -0.02769485903814262, "ci_high": -0.010281923714759536, "superior": false, "p_holm": 5.1590808139974196e-05 }, { "metric": "normalized_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.18739635157545606, "rate_b": 0.36965174129353234, "delta": 0.18225538971807628, "better": 1308, "worse": 209, "equal": 4513, "p": 2.0345053924323034e-194, "ci_low": 0.17064676616915422, "ci_high": 0.19419568822553898, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.831054853189073e-193 }, { "metric": "syntax_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.6774461028192371, "rate_b": 0.9902155887230514, "delta": 0.31276948590381426, "better": 1919, "worse": 33, "equal": 4078, "p": 0.0, "ci_low": 0.30082918739635156, "ci_high": 0.3250414593698176, "superior": true, "p_holm": 0.0 } ], "groups": { "grammar": { "n": 3220, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 3220, "lower_is_better": false, "mean_a": 1.8161490683229813, "mean_b": 3.0270186335403726, "delta": 1.2108695652173913, "ci_low": 1.0956521739130434, "ci_high": 1.3301242236024844, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 3220, "lower_is_better": false, "mean_a": 0.2908079192546584, "mean_b": 0.6330357142857143, "delta": 0.34222779503105594, "ci_low": 0.3284316257763975, "ci_high": 0.3557838066770186, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.12608695652173912, "rate_b": 0.37577639751552794, "delta": 0.2496894409937888, "better": 839, "worse": 35, "equal": 2346, "p": 7.216279331318949e-201, "ci_low": 0.23385093167701862, "ci_high": 0.265527950310559, "superior": true, "p_holm": 7.937907264450844e-200 }, { "metric": "exec_valid", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.3829192546583851, "rate_b": 0.7897515527950311, "delta": 0.40683229813664595, "better": 1397, "worse": 87, "equal": 1736, "p": 1.1874189546903685e-304, "ci_low": 0.38819875776397517, "ci_high": 0.4254658385093168, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.543644641097479e-303 }, { "metric": "mongo_valid", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.5509316770186335, "rate_b": 0.9968944099378882, "delta": 0.44596273291925465, "better": 1438, "worse": 2, "equal": 1780, "p": 0.0, "ci_low": 0.4282608695652174, "ci_high": 0.46366459627329193, "superior": true, "p_holm": 0.0 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.9239130434782609, "rate_b": 0.927639751552795, "delta": 0.0037267080745341614, "better": 183, "worse": 171, "equal": 2866, "p": 0.5588488229617014, "ci_low": -0.007763975155279503, "ci_high": 0.015217391304347827, "superior": false, "p_holm": 0.5588488229617014 }, { "metric": "normalized_match", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.12608695652173912, "rate_b": 0.3760869565217391, "delta": 0.25, "better": 840, "worse": 35, "equal": 2345, "p": 3.758285052644206e-201, "ci_low": 0.2341614906832298, "ci_high": 0.26583850931677017, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 4.509942063173047e-200 }, { "metric": "syntax_valid", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.5894409937888199, "rate_b": 0.9975155279503105, "delta": 0.4080745341614907, "better": 1315, "worse": 1, "equal": 1904, "p": 0.0, "ci_low": 0.3906832298136646, "ci_high": 0.4254658385093168, "superior": true, "p_holm": 0.0 } ] }, "llm": { "n": 2810, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 2810, "lower_is_better": false, "mean_a": 2.7444839857651244, "mean_b": 2.516370106761566, "delta": -0.22811387900355873, "ci_low": -0.3608540925266904, "ci_high": -0.09217081850533808, "superior": false }, { "metric": "edit_similarity", "kind": "continuous", "n": 2810, "lower_is_better": false, "mean_a": 0.46758516014234874, "mean_b": 0.6024896797153025, "delta": 0.13490451957295374, "ci_low": 0.11944684519572954, "ci_high": 0.15020412010676157, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.2537366548042705, "rate_b": 0.3604982206405694, "delta": 0.10676156583629894, "better": 468, "worse": 168, "equal": 2174, "p": 1.1582932129215375e-33, "ci_low": 0.08967971530249111, "ci_high": 0.12384341637010676, "superior": true, "p_holm": 5.7914660646076875e-33 }, { "metric": "exec_valid", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.6160142348754448, "rate_b": 0.7814946619217081, "delta": 0.16548042704626334, "better": 707, "worse": 242, "equal": 1861, "p": 1.9060073309706465e-53, "ci_low": 0.14483985765124555, "ci_high": 0.18612099644128113, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.1436043985823879e-52 }, { "metric": "mongo_valid", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.6665480427046263, "rate_b": 0.9220640569395018, "delta": 0.25551601423487547, "better": 756, "worse": 38, "equal": 2016, "p": 2.4517989932927237e-174, "ci_low": 0.23843416370106763, "ci_high": 0.27259786476868325, "superior": true, "p_holm": 1.961439194634179e-173 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.9117437722419929, "rate_b": 0.8669039145907473, "delta": -0.044839857651245554, "better": 122, "worse": 248, "equal": 2440, "p": 5.329604207306828e-11, "ci_low": -0.058362989323843414, "ci_high": -0.03131672597864769, "superior": false, "p_holm": 1.5988812621920485e-10 }, { "metric": "normalized_match", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.2576512455516014, "rate_b": 0.3622775800711744, "delta": 0.10462633451957296, "better": 468, "worse": 174, "equal": 2168, "p": 4.964410510210872e-32, "ci_low": 0.08754448398576513, "ci_high": 0.12206405693950179, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.985764204084349e-31 }, { "metric": "syntax_valid", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.7782918149466193, "rate_b": 0.9818505338078292, "delta": 0.20355871886120996, "better": 604, "worse": 32, "equal": 2174, "p": 6.539367210840716e-138, "ci_low": 0.18753558718861219, "ci_high": 0.2195729537366548, "superior": true, "p_holm": 4.577557047588501e-137 } ] } }, "groups_skipped": {} } }, "t1_v3_vs_t1_distill": { "context_kind": { "n_aligned": 6030, "overall": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 2.789054726368159, "mean_b": 3.8991708126036486, "delta": 1.1101160862354893, "ci_low": 0.9971807628524046, "ci_high": 1.2273673300165837, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 0.6188011608623549, "mean_b": 0.7011165505804311, "delta": 0.08231538971807628, "ci_low": 0.07611636194029851, "ci_high": 0.08865810199004974, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.3686567164179104, "rate_b": 0.451575456053068, "delta": 0.08291873963515754, "better": 600, "worse": 100, "equal": 5330, "p": 9.348895347113809e-88, "ci_low": 0.07462686567164178, "ci_high": 0.09137645107794362, "superior": true, "p_holm": 2.150245929836176e-86 }, { "metric": "exec_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.7859038142620233, "rate_b": 0.8306799336650083, "delta": 0.04477611940298507, "better": 442, "worse": 172, "equal": 5416, "p": 2.414329124019748e-28, "ci_low": 0.03681592039800995, "ci_high": 0.052570480928689885, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 4.587225335637521e-27 }, { "metric": "mongo_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.9620232172470978, "rate_b": 0.9718076285240465, "delta": 0.00978441127694859, "better": 82, "worse": 23, "equal": 5925, "p": 5.956502026439826e-09, "ci_low": 0.006467661691542288, "ci_high": 0.013101160862354893, "superior": true, "p_holm": 6.552152229083809e-08 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.8993366500829187, "rate_b": 0.9212271973466003, "delta": 0.021890547263681594, "better": 296, "worse": 164, "equal": 5570, "p": 7.782239199341524e-10, "ci_low": 0.015091210613598673, "ci_high": 0.028855721393034824, "superior": true, "p_holm": 9.338687039209828e-09 }, { "metric": "normalized_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.36965174129353234, "rate_b": 0.4537313432835821, "delta": 0.08407960199004975, "better": 608, "worse": 101, "equal": 5321, "p": 4.35933007640598e-89, "ci_low": 0.075787728026534, "ci_high": 0.09253731343283582, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.046239218337435e-87 }, { "metric": "syntax_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.9902155887230514, "rate_b": 0.9971807628524046, "delta": 0.006965174129353234, "better": 59, "worse": 17, "equal": 5954, "p": 1.3968718113140024e-06, "ci_low": 0.0041459369817578775, "ci_high": 0.00978441127694859, "superior": true, "p_holm": 1.3968718113140024e-05 } ], "groups": { "full": { "n": 3720, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 3720, "lower_is_better": false, "mean_a": 2.7946236559139783, "mean_b": 3.7911290322580644, "delta": 0.996505376344086, "ci_low": 0.850268817204301, "ci_high": 1.1508131720430108, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 3720, "lower_is_better": false, "mean_a": 0.5746390860215054, "mean_b": 0.6471357795698924, "delta": 0.0724966935483871, "ci_low": 0.06445746639784947, "ci_high": 0.08039365658602149, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.3150537634408602, "rate_b": 0.3811827956989247, "delta": 0.06612903225806452, "better": 303, "worse": 57, "equal": 3360, "p": 1.2206754770942516e-41, "ci_low": 0.05618279569892473, "ci_high": 0.0760752688172043, "superior": true, "p_holm": 2.5634185018979285e-40 }, { "metric": "exec_valid", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.7879032258064517, "rate_b": 0.8204301075268817, "delta": 0.032526881720430105, "better": 252, "worse": 131, "equal": 3337, "p": 6.314946543298559e-10, "ci_low": 0.022311827956989248, "ci_high": 0.04247311827956989, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 8.209430506288127e-09 }, { "metric": "mongo_valid", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.9612903225806452, "rate_b": 0.9704301075268817, "delta": 0.00913978494623656, "better": 52, "worse": 18, "equal": 3650, "p": 5.849547318695842e-05, "ci_low": 0.004838709677419355, "ci_high": 0.013709677419354839, "superior": true, "p_holm": 0.0005264592586826258 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.9080645161290323, "rate_b": 0.925, "delta": 0.016935483870967744, "better": 168, "worse": 105, "equal": 3447, "p": 0.00016477217809691969, "ci_low": 0.008064516129032258, "ci_high": 0.025537634408602152, "superior": true, "p_holm": 0.0011534052466784378 }, { "metric": "normalized_match", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.3153225806451613, "rate_b": 0.3827956989247312, "delta": 0.0674731182795699, "better": 309, "worse": 58, "equal": 3353, "p": 1.678335502796201e-42, "ci_low": 0.057526881720430106, "ci_high": 0.07741935483870968, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 3.692338106151642e-41 }, { "metric": "syntax_valid", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.989247311827957, "rate_b": 0.996505376344086, "delta": 0.007258064516129033, "better": 40, "worse": 13, "equal": 3667, "p": 0.0002685401188191605, "ci_low": 0.0034946236559139786, "ci_high": 0.01129032258064516, "superior": true, "p_holm": 0.0013427005940958026 } ] }, "irrelevant": { "n": 1155, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 2.7307359307359307, "mean_b": 3.895238095238095, "delta": 1.1645021645021645, "ci_low": 0.9246753246753247, "ci_high": 1.4130086580086578, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 0.6890493506493506, "mean_b": 0.7820035497835498, "delta": 0.09295419913419914, "ci_low": 0.07836375541125543, "ci_high": 0.10772298051948052, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.4510822510822511, "rate_b": 0.5523809523809524, "delta": 0.1012987012987013, "better": 140, "worse": 23, "equal": 992, "p": 1.1750354186024887e-21, "ci_low": 0.08051948051948052, "ci_high": 0.12294372294372294, "superior": true, "p_holm": 1.997560211624231e-20 }, { "metric": "exec_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.793073593073593, "rate_b": 0.8510822510822511, "delta": 0.05800865800865801, "better": 88, "worse": 21, "equal": 1046, "p": 6.112691638091427e-11, "ci_low": 0.04069264069264069, "ci_high": 0.07532467532467532, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 8.557768293327998e-10 }, { "metric": "mongo_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.9645021645021645, "rate_b": 0.974025974025974, "delta": 0.009523809523809525, "better": 16, "worse": 5, "equal": 1134, "p": 0.02660369873046875, "ci_low": 0.0017316017316017316, "ci_high": 0.017316017316017316, "superior": true, "p_holm": 0.0532073974609375 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.8865800865800866, "rate_b": 0.9177489177489178, "delta": 0.03116883116883117, "better": 64, "worse": 28, "equal": 1063, "p": 0.00022250112957580387, "ci_low": 0.015584415584415584, "ci_high": 0.047619047619047616, "superior": true, "p_holm": 0.0013350067774548233 }, { "metric": "normalized_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.45454545454545453, "rate_b": 0.554978354978355, "delta": 0.10043290043290043, "better": 140, "worse": 24, "equal": 991, "p": 4.04851304031129e-21, "ci_low": 0.07965367965367966, "ci_high": 0.12207792207792208, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 6.477620864498064e-20 }, { "metric": "syntax_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.9913419913419913, "rate_b": 0.9965367965367965, "delta": 0.005194805194805195, "better": 10, "worse": 4, "equal": 1141, "p": 0.1795654296875, "ci_low": -0.0008658008658008658, "ci_high": 0.011255411255411256, "superior": false, "p_holm": 0.1795654296875 } ] }, "none": { "n": 1155, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 2.8294372294372296, "mean_b": 4.251082251082251, "delta": 1.4216450216450216, "ci_low": 1.1523809523809523, "ci_high": 1.7073593073593074, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 0.690789264069264, "mean_b": 0.7940896969696969, "delta": 0.10330043290043289, "ci_low": 0.08851722943722945, "ci_high": 0.11877723160173158, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.4588744588744589, "rate_b": 0.5774891774891775, "delta": 0.11861471861471862, "better": 157, "worse": 20, "equal": 978, "p": 1.4656075428029792e-27, "ci_low": 0.09783549783549783, "ci_high": 0.14025974025974025, "superior": true, "p_holm": 2.6380935770453625e-26 }, { "metric": "exec_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.7722943722943723, "rate_b": 0.8432900432900433, "delta": 0.070995670995671, "better": 102, "worse": 20, "equal": 1033, "p": 1.9641987858977157e-14, "ci_low": 0.05281385281385281, "ci_high": 0.08917748917748917, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 2.946298178846574e-13 }, { "metric": "mongo_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.9619047619047619, "rate_b": 0.974025974025974, "delta": 0.012121212121212121, "better": 14, "worse": 0, "equal": 1141, "p": 0.0001220703125, "ci_low": 0.006060606060606061, "ci_high": 0.01904761904761905, "superior": true, "p_holm": 0.0009765625 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.883982683982684, "rate_b": 0.9125541125541126, "delta": 0.02857142857142857, "better": 64, "worse": 31, "equal": 1060, "p": 0.0009246444117719318, "ci_low": 0.012121212121212121, "ci_high": 0.045021645021645025, "superior": true, "p_holm": 0.0036985776470877274 }, { "metric": "normalized_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.4597402597402597, "rate_b": 0.580952380952381, "delta": 0.12121212121212122, "better": 159, "worse": 19, "equal": 977, "p": 1.0293288050049541e-28, "ci_low": 0.10043290043290043, "ci_high": 0.14285714285714285, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 2.0586576100099084e-27 }, { "metric": "syntax_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.9922077922077922, "rate_b": 1.0, "delta": 0.007792207792207792, "better": 9, "worse": 0, "equal": 1146, "p": 0.00390625, "ci_low": 0.003463203463203463, "ci_high": 0.012987012987012988, "superior": true, "p_holm": 0.01171875 } ] } }, "groups_skipped": {} }, "author": { "n_aligned": 6030, "overall": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 2.789054726368159, "mean_b": 3.8991708126036486, "delta": 1.1101160862354893, "ci_low": 0.9971807628524046, "ci_high": 1.2273673300165837, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 0.6188011608623549, "mean_b": 0.7011165505804311, "delta": 0.08231538971807628, "ci_low": 0.07611636194029851, "ci_high": 0.08865810199004974, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.3686567164179104, "rate_b": 0.451575456053068, "delta": 0.08291873963515754, "better": 600, "worse": 100, "equal": 5330, "p": 9.348895347113809e-88, "ci_low": 0.07462686567164178, "ci_high": 0.09137645107794362, "superior": true, "p_holm": 1.4023343020670715e-86 }, { "metric": "exec_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.7859038142620233, "rate_b": 0.8306799336650083, "delta": 0.04477611940298507, "better": 442, "worse": 172, "equal": 5416, "p": 2.414329124019748e-28, "ci_low": 0.03681592039800995, "ci_high": 0.052570480928689885, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 3.1386278612256722e-27 }, { "metric": "mongo_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.9620232172470978, "rate_b": 0.9718076285240465, "delta": 0.00978441127694859, "better": 82, "worse": 23, "equal": 5925, "p": 5.956502026439826e-09, "ci_low": 0.006467661691542288, "ci_high": 0.013101160862354893, "superior": true, "p_holm": 5.956502026439826e-08 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.8993366500829187, "rate_b": 0.9212271973466003, "delta": 0.021890547263681594, "better": 296, "worse": 164, "equal": 5570, "p": 7.782239199341524e-10, "ci_low": 0.015091210613598673, "ci_high": 0.028855721393034824, "superior": true, "p_holm": 9.338687039209828e-09 }, { "metric": "normalized_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.36965174129353234, "rate_b": 0.4537313432835821, "delta": 0.08407960199004975, "better": 608, "worse": 101, "equal": 5321, "p": 4.35933007640598e-89, "ci_low": 0.075787728026534, "ci_high": 0.09253731343283582, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 6.974928122249568e-88 }, { "metric": "syntax_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.9902155887230514, "rate_b": 0.9971807628524046, "delta": 0.006965174129353234, "better": 59, "worse": 17, "equal": 5954, "p": 1.3968718113140024e-06, "ci_low": 0.0041459369817578775, "ci_high": 0.00978441127694859, "superior": true, "p_holm": 1.1174974490512019e-05 } ], "groups": { "grammar": { "n": 3220, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 3220, "lower_is_better": false, "mean_a": 3.0270186335403726, "mean_b": 3.0596273291925464, "delta": 0.03260869565217391, "ci_low": -0.04503105590062112, "ci_high": 0.11055900621118013, "superior": false }, { "metric": "edit_similarity", "kind": "continuous", "n": 3220, "lower_is_better": false, "mean_a": 0.6330357142857143, "mean_b": 0.6410397204968944, "delta": 0.008004006211180125, "ci_low": 0.0015777158385093183, "ci_high": 0.014488523291925461, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.37577639751552794, "rate_b": 0.38354037267080743, "delta": 0.007763975155279503, "better": 95, "worse": 70, "equal": 3055, "p": 0.06137832710948886, "ci_low": 0.0, "ci_high": 0.015838509316770187, "superior": false, "p_holm": 0.24551330843795544 }, { "metric": "exec_valid", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.7897515527950311, "rate_b": 0.8, "delta": 0.010248447204968944, "better": 112, "worse": 79, "equal": 3029, "p": 0.020347077476115385, "ci_low": 0.0021661490683230504, "ci_high": 0.018944099378881987, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 0.10173538738057693 }, { "metric": "mongo_valid", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.9968944099378882, "rate_b": 0.9981366459627329, "delta": 0.0012422360248447205, "better": 10, "worse": 6, "equal": 3204, "p": 0.454498291015625, "ci_low": -0.0012422360248447205, "ci_high": 0.0037267080745341614, "superior": false, "p_holm": 0.775390625 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.927639751552795, "rate_b": 0.9472049689440993, "delta": 0.01956521739130435, "better": 115, "worse": 52, "equal": 3053, "p": 1.2145886270456976e-06, "ci_low": 0.011490683229813664, "ci_high": 0.02732919254658385, "superior": true, "p_holm": 1.0931297643411278e-05 }, { "metric": "normalized_match", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.3760869565217391, "rate_b": 0.38354037267080743, "delta": 0.007453416149068323, "better": 95, "worse": 71, "equal": 3054, "p": 0.0739190926121619, "ci_low": -0.00031055900621118014, "ci_high": 0.015527950310559006, "superior": false, "margin": 0.02, "noninferior": true, "p_holm": 0.24551330843795544 }, { "metric": "syntax_valid", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.9975155279503105, "rate_b": 0.9987577639751553, "delta": 0.0012422360248447205, "better": 8, "worse": 4, "equal": 3208, "p": 0.3876953125, "ci_low": -0.0009316770186335404, "ci_high": 0.0034161490683229812, "superior": false, "p_holm": 0.775390625 } ] }, "llm": { "n": 2810, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 2810, "lower_is_better": false, "mean_a": 2.516370106761566, "mean_b": 4.8612099644128115, "delta": 2.3448398576512455, "ci_low": 2.126325622775801, "ci_high": 2.5722419928825624, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 2810, "lower_is_better": false, "mean_a": 0.6024896797153025, "mean_b": 0.7699590391459075, "delta": 0.16746935943060498, "ci_low": 0.15663636298932385, "ci_high": 0.17816107740213522, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.3604982206405694, "rate_b": 0.5295373665480427, "delta": 0.16903914590747332, "better": 505, "worse": 30, "equal": 2275, "p": 2.2061014161757547e-112, "ci_low": 0.15409252669039145, "ci_high": 0.18398576512455517, "superior": true, "p_holm": 3.750372407498783e-111 }, { "metric": "exec_valid", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.7814946619217081, "rate_b": 0.8658362989323843, "delta": 0.08434163701067615, "better": 330, "worse": 93, "equal": 2387, "p": 3.4779869095107494e-32, "ci_low": 0.0701067615658363, "ci_high": 0.09822064056939502, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 4.869181673315049e-31 }, { "metric": "mongo_valid", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.9220640569395018, "rate_b": 0.9416370106761566, "delta": 0.019572953736654804, "better": 72, "worse": 17, "equal": 2721, "p": 3.1727390190435945e-09, "ci_low": 0.013167259786476869, "ci_high": 0.02597864768683274, "superior": true, "p_holm": 3.490012920947954e-08 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.8669039145907473, "rate_b": 0.891459074733096, "delta": 0.024555160142348754, "better": 181, "worse": 112, "equal": 2517, "p": 6.617443552871609e-05, "ci_low": 0.012455516014234875, "ci_high": 0.036298932384341634, "superior": true, "p_holm": 0.00039704661317229654 }, { "metric": "normalized_match", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.3622775800711744, "rate_b": 0.5341637010676157, "delta": 0.1718861209964413, "better": 513, "worse": 30, "equal": 2267, "p": 1.3606411844347843e-114, "ci_low": 0.15693950177935942, "ci_high": 0.18683274021352314, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 2.4491541319826116e-113 }, { "metric": "syntax_valid", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.9818505338078292, "rate_b": 0.995373665480427, "delta": 0.013523131672597865, "better": 51, "worse": 13, "equal": 2746, "p": 1.8809621564294015e-06, "ci_low": 0.007829181494661922, "ci_high": 0.019217081850533807, "superior": true, "p_holm": 1.316673509500581e-05 } ] } }, "groups_skipped": {} } }, "t1_v3_vs_base_1.5b": { "context_kind": { "n_aligned": 6030, "overall": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 2.2487562189054726, "mean_b": 3.8991708126036486, "delta": 1.6504145936981758, "ci_low": 1.5303482587064676, "ci_high": 1.7753026533996683, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 0.37318669983416247, "mean_b": 0.7011165505804311, "delta": 0.3279298507462686, "ci_low": 0.3182463428689884, "ci_high": 0.3378565907960199, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.1855721393034826, "rate_b": 0.451575456053068, "delta": 0.2660033167495854, "better": 1686, "worse": 82, "equal": 4262, "p": 0.0, "ci_low": 0.25422885572139303, "ci_high": 0.2779436152570481, "superior": true, "p_holm": 0.0 }, { "metric": "exec_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.49154228855721394, "rate_b": 0.8306799336650083, "delta": 0.33913764510779437, "better": 2251, "worse": 206, "equal": 3573, "p": 0.0, "ci_low": 0.32553897180762853, "ci_high": 0.3529021558872305, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 0.0 }, { "metric": "mongo_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.6048092868988392, "rate_b": 0.9718076285240465, "delta": 0.3669983416252073, "better": 2225, "worse": 12, "equal": 3793, "p": 0.0, "ci_low": 0.35505804311774464, "ci_high": 0.37960199004975126, "superior": true, "p_holm": 0.0 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.9182421227197347, "rate_b": 0.9212271973466003, "delta": 0.0029850746268656717, "better": 331, "worse": 313, "equal": 5386, "p": 0.5029560522139299, "ci_low": -0.005306799336650083, "ci_high": 0.011111111111111112, "superior": false, "p_holm": 1.0 }, { "metric": "normalized_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.18739635157545606, "rate_b": 0.4537313432835821, "delta": 0.266334991708126, "better": 1687, "worse": 81, "equal": 4262, "p": 0.0, "ci_low": 0.2543946932006633, "ci_high": 0.2782794361525704, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 0.0 }, { "metric": "syntax_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.6774461028192371, "rate_b": 0.9971807628524046, "delta": 0.3197346600331675, "better": 1939, "worse": 11, "equal": 4080, "p": 0.0, "ci_low": 0.3081260364842454, "ci_high": 0.3320066334991708, "superior": true, "p_holm": 0.0 } ], "groups": { "full": { "n": 3720, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 3720, "lower_is_better": false, "mean_a": 2.150537634408602, "mean_b": 3.7911290322580644, "delta": 1.6405913978494624, "ci_low": 1.4833333333333334, "ci_high": 1.8016129032258064, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 3720, "lower_is_better": false, "mean_a": 0.3220927956989247, "mean_b": 0.6471357795698924, "delta": 0.32504298387096775, "ci_low": 0.31256386626344085, "ci_high": 0.33756841397849463, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.139247311827957, "rate_b": 0.3811827956989247, "delta": 0.24193548387096775, "better": 938, "worse": 38, "equal": 2744, "p": 1.1949364281529616e-225, "ci_low": 0.22768817204301076, "ci_high": 0.2564516129032258, "superior": true, "p_holm": 1.7924046422294423e-224 }, { "metric": "exec_valid", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.45, "rate_b": 0.8204301075268817, "delta": 0.3704301075268817, "better": 1533, "worse": 155, "equal": 2032, "p": 4.001190348593105e-285, "ci_low": 0.3521505376344086, "ci_high": 0.38844086021505375, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 6.802023592608279e-284 }, { "metric": "mongo_valid", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.5567204301075269, "rate_b": 0.9704301075268817, "delta": 0.4137096774193548, "better": 1545, "worse": 6, "equal": 2169, "p": 0.0, "ci_low": 0.39731182795698927, "ci_high": 0.42983870967741933, "superior": true, "p_holm": 0.0 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.928494623655914, "rate_b": 0.925, "delta": -0.0034946236559139786, "better": 194, "worse": 207, "equal": 3319, "p": 0.5490587682630458, "ci_low": -0.013978494623655914, "ci_high": 0.006989247311827957, "superior": false, "p_holm": 1.0 }, { "metric": "normalized_match", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.13978494623655913, "rate_b": 0.3827956989247312, "delta": 0.24301075268817204, "better": 942, "worse": 38, "equal": 2740, "p": 8.748941920297441e-227, "ci_low": 0.22849462365591397, "ci_high": 0.2575268817204301, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.3998307072475906e-225 }, { "metric": "syntax_valid", "kind": "binary", "n": 3720, "lower_is_better": false, "rate_a": 0.6231182795698925, "rate_b": 0.996505376344086, "delta": 0.37338709677419357, "better": 1396, "worse": 7, "equal": 2317, "p": 0.0, "ci_low": 0.357258064516129, "ci_high": 0.38924731182795697, "superior": true, "p_holm": 0.0 } ] }, "irrelevant": { "n": 1155, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 2.3445887445887448, "mean_b": 3.895238095238095, "delta": 1.5506493506493506, "ci_low": 1.3142857142857143, "ci_high": 1.7948051948051948, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 0.4300482251082251, "mean_b": 0.7820035497835498, "delta": 0.35195532467532464, "ci_low": 0.32793832034632037, "ci_high": 0.3758575714285714, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.24415584415584415, "rate_b": 0.5523809523809524, "delta": 0.30822510822510824, "better": 375, "worse": 19, "equal": 761, "p": 5.694924653737641e-87, "ci_low": 0.2804978354978357, "ci_high": 0.3367965367965368, "superior": true, "p_holm": 6.83390958448517e-86 }, { "metric": "exec_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.5341991341991342, "rate_b": 0.8510822510822511, "delta": 0.3168831168831169, "better": 392, "worse": 26, "equal": 737, "p": 5.0158096301156385e-85, "ci_low": 0.2883116883116883, "ci_high": 0.3463203463203463, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 4.514228667104075e-84 }, { "metric": "mongo_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.6441558441558441, "rate_b": 0.974025974025974, "delta": 0.32987012987012987, "better": 386, "worse": 5, "equal": 764, "p": 2.981856573542977e-107, "ci_low": 0.30303030303030304, "ci_high": 0.3575757575757576, "superior": true, "p_holm": 4.174599202960168e-106 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.9038961038961039, "rate_b": 0.9177489177489178, "delta": 0.013852813852813853, "better": 74, "worse": 58, "equal": 1023, "p": 0.19149877187347097, "ci_low": -0.005194805194805195, "ci_high": 0.033766233766233764, "superior": false, "p_holm": 0.7346286876070992 }, { "metric": "normalized_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.24848484848484848, "rate_b": 0.554978354978355, "delta": 0.3064935064935065, "better": 373, "worse": 19, "equal": 763, "p": 2.0638822530610425e-86, "ci_low": 0.2779220779220779, "ci_high": 0.33506493506493507, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 2.0638822530610427e-85 }, { "metric": "syntax_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.7238095238095238, "rate_b": 0.9965367965367965, "delta": 0.2727272727272727, "better": 319, "worse": 4, "equal": 832, "p": 5.2758083314590226e-89, "ci_low": 0.24675324675324675, "ci_high": 0.2987012987012987, "superior": true, "p_holm": 6.858550830896729e-88 } ] }, "none": { "n": 1155, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 2.469264069264069, "mean_b": 4.251082251082251, "delta": 1.7818181818181817, "ci_low": 1.4987012987012986, "ci_high": 2.084848484848485, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 1155, "lower_is_better": false, "mean_a": 0.48088735930735926, "mean_b": 0.7940896969696969, "delta": 0.3132023376623376, "ci_low": 0.290192645021645, "ci_high": 0.33632416017316014, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.2761904761904762, "rate_b": 0.5774891774891775, "delta": 0.3012987012987013, "better": 373, "worse": 25, "equal": 757, "p": 9.844313337086202e-81, "ci_low": 0.2727272727272727, "ci_high": 0.32987012987012987, "superior": true, "p_holm": 6.891019335960342e-80 }, { "metric": "exec_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.5826839826839827, "rate_b": 0.8432900432900433, "delta": 0.2606060606060606, "better": 326, "worse": 25, "equal": 804, "p": 5.442582279000005e-68, "ci_low": 0.2329004329004329, "ci_high": 0.2891774891774892, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 3.265549367400003e-67 }, { "metric": "mongo_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.7203463203463204, "rate_b": 0.974025974025974, "delta": 0.25367965367965367, "better": 294, "worse": 1, "equal": 860, "p": 9.299786660660013e-87, "ci_low": 0.22857142857142856, "ci_high": 0.27965367965367965, "superior": true, "p_holm": 1.0229765326726015e-85 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.8995670995670996, "rate_b": 0.9125541125541126, "delta": 0.012987012987012988, "better": 63, "worse": 48, "equal": 1044, "p": 0.1836571719017748, "ci_low": -0.004329004329004329, "ci_high": 0.03116883116883117, "superior": false, "p_holm": 0.7346286876070992 }, { "metric": "normalized_match", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.27965367965367965, "rate_b": 0.580952380952381, "delta": 0.3012987012987013, "better": 372, "worse": 24, "equal": 759, "p": 2.3177249009102542e-81, "ci_low": 0.2727272727272727, "ci_high": 0.32987012987012987, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 1.8541799207282034e-80 }, { "metric": "syntax_valid", "kind": "binary", "n": 1155, "lower_is_better": false, "rate_a": 0.806060606060606, "rate_b": 1.0, "delta": 0.19393939393939394, "better": 224, "worse": 0, "equal": 931, "p": 7.418412301374843e-68, "ci_low": 0.17227272727272747, "ci_high": 0.21731601731601732, "superior": true, "p_holm": 3.7092061506874214e-67 } ] } }, "groups_skipped": {} }, "author": { "n_aligned": 6030, "overall": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 2.2487562189054726, "mean_b": 3.8991708126036486, "delta": 1.6504145936981758, "ci_low": 1.5303482587064676, "ci_high": 1.7753026533996683, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 6030, "lower_is_better": false, "mean_a": 0.37318669983416247, "mean_b": 0.7011165505804311, "delta": 0.3279298507462686, "ci_low": 0.3182463428689884, "ci_high": 0.3378565907960199, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.1855721393034826, "rate_b": 0.451575456053068, "delta": 0.2660033167495854, "better": 1686, "worse": 82, "equal": 4262, "p": 0.0, "ci_low": 0.25422885572139303, "ci_high": 0.2779436152570481, "superior": true, "p_holm": 0.0 }, { "metric": "exec_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.49154228855721394, "rate_b": 0.8306799336650083, "delta": 0.33913764510779437, "better": 2251, "worse": 206, "equal": 3573, "p": 0.0, "ci_low": 0.32553897180762853, "ci_high": 0.3529021558872305, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 0.0 }, { "metric": "mongo_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.6048092868988392, "rate_b": 0.9718076285240465, "delta": 0.3669983416252073, "better": 2225, "worse": 12, "equal": 3793, "p": 0.0, "ci_low": 0.35505804311774464, "ci_high": 0.37960199004975126, "superior": true, "p_holm": 0.0 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.9182421227197347, "rate_b": 0.9212271973466003, "delta": 0.0029850746268656717, "better": 331, "worse": 313, "equal": 5386, "p": 0.5029560522139299, "ci_low": -0.005306799336650083, "ci_high": 0.011111111111111112, "superior": false, "p_holm": 0.5029560522139299 }, { "metric": "normalized_match", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.18739635157545606, "rate_b": 0.4537313432835821, "delta": 0.266334991708126, "better": 1687, "worse": 81, "equal": 4262, "p": 0.0, "ci_low": 0.2543946932006633, "ci_high": 0.2782794361525704, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 0.0 }, { "metric": "syntax_valid", "kind": "binary", "n": 6030, "lower_is_better": false, "rate_a": 0.6774461028192371, "rate_b": 0.9971807628524046, "delta": 0.3197346600331675, "better": 1939, "worse": 11, "equal": 4080, "p": 0.0, "ci_low": 0.3081260364842454, "ci_high": 0.3320066334991708, "superior": true, "p_holm": 0.0 } ], "groups": { "grammar": { "n": 3220, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 3220, "lower_is_better": false, "mean_a": 1.8161490683229813, "mean_b": 3.0596273291925464, "delta": 1.2434782608695651, "ci_low": 1.1235947204968944, "ci_high": 1.3670885093167702, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 3220, "lower_is_better": false, "mean_a": 0.2908079192546584, "mean_b": 0.6410397204968944, "delta": 0.350231801242236, "ci_low": 0.3362704782608696, "ci_high": 0.36421224689440995, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.12608695652173912, "rate_b": 0.38354037267080743, "delta": 0.2574534161490683, "better": 864, "worse": 35, "equal": 2321, "p": 5.876926180008631e-208, "ci_low": 0.24161490683229814, "ci_high": 0.2732919254658385, "superior": true, "p_holm": 5.2892335620077676e-207 }, { "metric": "exec_valid", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.3829192546583851, "rate_b": 0.8, "delta": 0.41708074534161493, "better": 1421, "worse": 78, "equal": 1721, "p": 7.1556e-320, "ci_low": 0.3984472049689441, "ci_high": 0.4360248447204969, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 7.8711e-319 }, { "metric": "mongo_valid", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.5509316770186335, "rate_b": 0.9981366459627329, "delta": 0.4472049689440994, "better": 1442, "worse": 2, "equal": 1776, "p": 0.0, "ci_low": 0.4295031055900621, "ci_high": 0.4649068322981366, "superior": true, "p_holm": 0.0 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.9239130434782609, "rate_b": 0.9472049689440993, "delta": 0.023291925465838508, "better": 196, "worse": 121, "equal": 2903, "p": 2.9877855768950636e-05, "ci_low": 0.012732919254658385, "ci_high": 0.033850931677018636, "superior": true, "p_holm": 8.96335673068519e-05 }, { "metric": "normalized_match", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.12608695652173912, "rate_b": 0.38354037267080743, "delta": 0.2574534161490683, "better": 864, "worse": 35, "equal": 2321, "p": 5.876926180008631e-208, "ci_low": 0.24161490683229814, "ci_high": 0.2732919254658385, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 5.2892335620077676e-207 }, { "metric": "syntax_valid", "kind": "binary", "n": 3220, "lower_is_better": false, "rate_a": 0.5894409937888199, "rate_b": 0.9987577639751553, "delta": 0.4093167701863354, "better": 1320, "worse": 2, "equal": 1898, "p": 0.0, "ci_low": 0.3919254658385093, "ci_high": 0.4267080745341615, "superior": true, "p_holm": 0.0 } ] }, "llm": { "n": 2810, "metrics": [ { "metric": "accepted_prefix_tokens", "kind": "continuous", "n": 2810, "lower_is_better": false, "mean_a": 2.7444839857651244, "mean_b": 4.8612099644128115, "delta": 2.1167259786476866, "ci_low": 1.903202846975089, "ci_high": 2.3387900355871887, "superior": true }, { "metric": "edit_similarity", "kind": "continuous", "n": 2810, "lower_is_better": false, "mean_a": 0.46758516014234874, "mean_b": 0.7699590391459075, "delta": 0.3023738790035587, "ci_low": 0.28847639679715303, "ci_high": 0.31637307918149465, "superior": true }, { "metric": "exact_match", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.2537366548042705, "rate_b": 0.5295373665480427, "delta": 0.27580071174377224, "better": 822, "worse": 47, "equal": 1941, "p": 7.990635342893872e-184, "ci_low": 0.25836298932384344, "ci_high": 0.29323843416370104, "superior": true, "p_holm": 4.794381205736324e-183 }, { "metric": "exec_valid", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.6160142348754448, "rate_b": 0.8658362989323843, "delta": 0.2498220640569395, "better": 830, "worse": 128, "equal": 1852, "p": 1.4303288564350014e-126, "ci_low": 0.2302491103202847, "ci_high": 0.2701067615658363, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 5.7213154257400056e-126 }, { "metric": "mongo_valid", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.6665480427046263, "rate_b": 0.9416370106761566, "delta": 0.27508896797153026, "better": 783, "worse": 10, "equal": 2017, "p": 9.954887411232013e-217, "ci_low": 0.2580071174377224, "ci_high": 0.29145907473309607, "superior": true, "p_holm": 9.954887411232013e-216 }, { "metric": "no_nonexistent_field", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.9117437722419929, "rate_b": 0.891459074733096, "delta": -0.020284697508896797, "better": 135, "worse": 192, "equal": 2483, "p": 0.0019114402812567247, "ci_low": -0.03309608540925267, "ci_high": -0.007829181494661922, "superior": false, "p_holm": 0.0038228805625134493 }, { "metric": "normalized_match", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.2576512455516014, "rate_b": 0.5341637010676157, "delta": 0.2765124555160142, "better": 823, "worse": 46, "equal": 1941, "p": 4.5571131842281796e-185, "ci_low": 0.2587188612099644, "ci_high": 0.29359430604982206, "superior": true, "margin": 0.02, "noninferior": true, "p_holm": 3.1899792289597256e-184 }, { "metric": "syntax_valid", "kind": "binary", "n": 2810, "lower_is_better": false, "rate_a": 0.7782918149466193, "rate_b": 0.995373665480427, "delta": 0.21708185053380782, "better": 619, "worse": 9, "equal": 2182, "p": 7.201076790162522e-170, "ci_low": 0.201423487544484, "ci_high": 0.23274021352313168, "superior": true, "p_holm": 3.600538395081261e-169 } ] } }, "groups_skipped": {} } } }, "status_distribution": { "base_1.5b": { "ok": 3404, "syntax": 1442, "error": 1027, "no_exec": 26, "empty": 85, "static": 45, "timeout": 1 }, "t1_distill": { "ok": 5328, "error": 520, "static": 98, "syntax": 49, "no_exec": 25, "empty": 10 }, "t1_v3": { "ok": 5478, "error": 453, "static": 82, "syntax": 16, "timeout": 1 }, "_reference": { "ok": 5612, "error": 418 } }, "readonly_proof": { "base_1.5b": { "dbHash_before_equals_after": true, "dbs_checked": 28, "proxy_write_commands_forwarded": {}, "maxTimeMS_injected": 3000 }, "t1_distill": { "dbHash_before_equals_after": true, "dbs_checked": 28, "proxy_write_commands_forwarded": {}, "maxTimeMS_injected": 3000 }, "t1_v3": { "dbHash_before_equals_after": true, "dbs_checked": 28, "proxy_write_commands_forwarded": {}, "maxTimeMS_injected": 3000 }, "_reference": { "dbHash_before_equals_after": true, "dbs_checked": 28, "proxy_write_commands_forwarded": {}, "maxTimeMS_injected": 3000 } }, "definitions": { "normalized_match": "pred == middle com espacos normalizados", "exec_valid": "prefix+pred+suffix executa (mongosh via proxy RO, status ok, comando de leitura enviado) sem campo/colecao inexistente no plano capturado e pred nao vazia", "ceiling_exec_valid": "mesma validacao sobre prefix+middle+suffix", "exec_over_ceiling": "exec_valid / ceiling_exec_valid (razao de medias)", "nonexistent_fields_rate": "itens cujo plano capturado cita campo/colecao fora do schema, sobre todos os itens", "json_dialect": "dialect json executa como db.getCollection(c).aggregate(EJSON.parse(texto))" } }