{ "model": "OpenSML-JSONLite-175M-Instruct", "checkpoint": "main", "selected_stage": "joint JSON extraction SFT", "step": 300, "model_sha256": "781597baeb9b51d535fc27cd6f941dbc202e01d62c8d963587a3f3d565128560", "output_mode": "native-json", "development": { "metrics": { "all": { "absent_accuracy": 0.9211356466876972, "count": 240, "counts": { "absent_correct": 292, "absent_total": 317, "exact": 178, "false_abstention": 25, "field_correct": 632, "fields": 714, "fn": 65, "fp": 47, "json_parse_ok": 240, "missed_abstention": 25, "natural_stop": 240, "nonrepetitive_nonempty": 240, "predicted_values": 504, "present_correct": 340, "present_total": 397, "schema_ok": 239, "token_limited": 0, "tp": 457, "unsupported_values": 6 }, "exact": 0.7416666666666667, "f1": 0.8908382066276802, "false_abstention_rate": 0.06297229219143577, "field_accuracy": 0.8851540616246498, "json_parse_rate": 1.0, "missed_abstention_rate": 0.07886435331230283, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.04828121321412861, "precision": 0.9067460317460317, "present_accuracy": 0.8564231738035264, "recall": 0.8754789272030651, "schema_valid_rate": 0.9958333333333333, "token_limited_rate": 0.0, "unsupported_value_rate": 0.011904761904761904 }, "domain/AddToPlaylist": { "absent_accuracy": 0.8974358974358975, "count": 28, "counts": { "absent_correct": 35, "absent_total": 39, "exact": 19, "false_abstention": 1, "field_correct": 81, "fields": 92, "fn": 7, "fp": 10, "json_parse_ok": 28, "missed_abstention": 4, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 56, "present_correct": 46, "present_total": 53, "schema_ok": 28, "token_limited": 0, "tp": 46, "unsupported_values": 2 }, "exact": 0.6785714285714286, "f1": 0.8440366972477065, "false_abstention_rate": 0.018867924528301886, "field_accuracy": 0.8804347826086957, "json_parse_rate": 1.0, "missed_abstention_rate": 0.10256410256410256, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.0638373876364702, "precision": 0.8214285714285714, "present_accuracy": 0.8679245283018868, "recall": 0.8679245283018868, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.03571428571428571 }, "domain/BookRestaurant": { "absent_accuracy": 0.9454545454545454, "count": 28, "counts": { "absent_correct": 52, "absent_total": 55, "exact": 22, "false_abstention": 1, "field_correct": 99, "fields": 105, "fn": 3, "fp": 5, "json_parse_ok": 28, "missed_abstention": 3, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 60, "present_correct": 47, "present_total": 50, "schema_ok": 28, "token_limited": 0, "tp": 55, "unsupported_values": 1 }, "exact": 0.7857142857142857, "f1": 0.9322033898305084, "false_abstention_rate": 0.02, "field_accuracy": 0.9428571428571428, "json_parse_rate": 1.0, "missed_abstention_rate": 0.05454545454545454, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.01643673108524776, "precision": 0.9166666666666666, "present_accuracy": 0.94, "recall": 0.9482758620689655, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.016666666666666666 }, "domain/GetWeather": { "absent_accuracy": 0.9090909090909091, "count": 28, "counts": { "absent_correct": 30, "absent_total": 33, "exact": 20, "false_abstention": 5, "field_correct": 64, "fields": 73, "fn": 6, "fp": 3, "json_parse_ok": 28, "missed_abstention": 3, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 59, "present_correct": 34, "present_total": 40, "schema_ok": 28, "token_limited": 0, "tp": 56, "unsupported_values": 0 }, "exact": 0.7142857142857143, "f1": 0.9256198347107438, "false_abstention_rate": 0.125, "field_accuracy": 0.8767123287671232, "json_parse_rate": 1.0, "missed_abstention_rate": 0.09090909090909091, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.037961417684398056, "precision": 0.9491525423728814, "present_accuracy": 0.85, "recall": 0.9032258064516129, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "domain/PlayMusic": { "absent_accuracy": 0.8, "count": 28, "counts": { "absent_correct": 28, "absent_total": 35, "exact": 19, "false_abstention": 5, "field_correct": 62, "fields": 74, "fn": 5, "fp": 7, "json_parse_ok": 28, "missed_abstention": 7, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 56, "present_correct": 34, "present_total": 39, "schema_ok": 28, "token_limited": 0, "tp": 49, "unsupported_values": 0 }, "exact": 0.6785714285714286, "f1": 0.8909090909090909, "false_abstention_rate": 0.1282051282051282, "field_accuracy": 0.8378378378378378, "json_parse_rate": 1.0, "missed_abstention_rate": 0.2, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.028358824200820112, "precision": 0.875, "present_accuracy": 0.8717948717948718, "recall": 0.9074074074074074, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "domain/RateBook": { "absent_accuracy": 0.9761904761904762, "count": 28, "counts": { "absent_correct": 41, "absent_total": 42, "exact": 23, "false_abstention": 2, "field_correct": 104, "fields": 110, "fn": 5, "fp": 4, "json_parse_ok": 28, "missed_abstention": 1, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 67, "present_correct": 63, "present_total": 68, "schema_ok": 28, "token_limited": 0, "tp": 63, "unsupported_values": 1 }, "exact": 0.8214285714285714, "f1": 0.9333333333333335, "false_abstention_rate": 0.029411764705882353, "field_accuracy": 0.9454545454545454, "json_parse_rate": 1.0, "missed_abstention_rate": 0.023809523809523808, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.010672787178918952, "precision": 0.9402985074626866, "present_accuracy": 0.9264705882352942, "recall": 0.9264705882352942, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.014925373134328358 }, "domain/SearchCreativeWork": { "absent_accuracy": 1.0, "count": 28, "counts": { "absent_correct": 9, "absent_total": 9, "exact": 26, "false_abstention": 0, "field_correct": 43, "fields": 45, "fn": 2, "fp": 2, "json_parse_ok": 28, "missed_abstention": 0, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 36, "present_correct": 34, "present_total": 36, "schema_ok": 28, "token_limited": 0, "tp": 34, "unsupported_values": 0 }, "exact": 0.9285714285714286, "f1": 0.9444444444444444, "false_abstention_rate": 0.0, "field_accuracy": 0.9555555555555556, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.011042242548553207, "precision": 0.9444444444444444, "present_accuracy": 0.9444444444444444, "recall": 0.9444444444444444, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "domain/SearchScreeningEvent": { "absent_accuracy": 0.9454545454545454, "count": 28, "counts": { "absent_correct": 52, "absent_total": 55, "exact": 23, "false_abstention": 1, "field_correct": 100, "fields": 108, "fn": 8, "fp": 6, "json_parse_ok": 28, "missed_abstention": 3, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 64, "present_correct": 48, "present_total": 53, "schema_ok": 28, "token_limited": 0, "tp": 58, "unsupported_values": 0 }, "exact": 0.8214285714285714, "f1": 0.8923076923076922, "false_abstention_rate": 0.018867924528301886, "field_accuracy": 0.9259259259259259, "json_parse_rate": 1.0, "missed_abstention_rate": 0.05454545454545454, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.018851709860817727, "precision": 0.90625, "present_accuracy": 0.9056603773584906, "recall": 0.8787878787878788, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "domain/cardinality": { "absent_accuracy": 1.0, "count": 8, "counts": { "absent_correct": 8, "absent_total": 8, "exact": 8, "false_abstention": 0, "field_correct": 16, "fields": 16, "fn": 0, "fp": 0, "json_parse_ok": 8, "missed_abstention": 0, "natural_stop": 8, "nonrepetitive_nonempty": 8, "predicted_values": 12, "present_correct": 8, "present_total": 8, "schema_ok": 8, "token_limited": 0, "tp": 12, "unsupported_values": 0 }, "exact": 1.0, "f1": 1.0, "false_abstention_rate": 0.0, "field_accuracy": 1.0, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.012686168280197307, "precision": 1.0, "present_accuracy": 1.0, "recall": 1.0, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "domain/mixsnips": { "absent_accuracy": 1.0, "count": 8, "counts": { "absent_correct": 8, "absent_total": 8, "exact": 1, "false_abstention": 4, "field_correct": 12, "fields": 24, "fn": 14, "fp": 4, "json_parse_ok": 8, "missed_abstention": 0, "natural_stop": 8, "nonrepetitive_nonempty": 8, "predicted_values": 12, "present_correct": 4, "present_total": 16, "schema_ok": 7, "token_limited": 0, "tp": 8, "unsupported_values": 0 }, "exact": 0.125, "f1": 0.4705882352941177, "false_abstention_rate": 0.25, "field_accuracy": 0.5, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.378448877716437, "precision": 0.6666666666666666, "present_accuracy": 0.25, "recall": 0.36363636363636365, "schema_valid_rate": 0.875, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "domain/observed_lists": { "absent_accuracy": 1.0, "count": 12, "counts": { "absent_correct": 12, "absent_total": 12, "exact": 9, "false_abstention": 0, "field_correct": 21, "fields": 24, "fn": 4, "fp": 0, "json_parse_ok": 12, "missed_abstention": 0, "natural_stop": 12, "nonrepetitive_nonempty": 12, "predicted_values": 49, "present_correct": 9, "present_total": 12, "schema_ok": 12, "token_limited": 0, "tp": 49, "unsupported_values": 0 }, "exact": 0.75, "f1": 0.9607843137254902, "false_abstention_rate": 0.0, "field_accuracy": 0.875, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.04184045293368399, "precision": 1.0, "present_accuracy": 0.75, "recall": 0.9245283018867925, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "domain/playground": { "absent_accuracy": 1.0, "count": 4, "counts": { "absent_correct": 2, "absent_total": 2, "exact": 3, "false_abstention": 0, "field_correct": 8, "fields": 9, "fn": 3, "fp": 1, "json_parse_ok": 4, "missed_abstention": 0, "natural_stop": 4, "nonrepetitive_nonempty": 4, "predicted_values": 9, "present_correct": 6, "present_total": 7, "schema_ok": 4, "token_limited": 0, "tp": 8, "unsupported_values": 1 }, "exact": 0.75, "f1": 0.7999999999999999, "false_abstention_rate": 0.0, "field_accuracy": 0.8888888888888888, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.03125538246240467, "precision": 0.8888888888888888, "present_accuracy": 0.8571428571428571, "recall": 0.7272727272727273, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.1111111111111111 }, "domain/roles": { "absent_accuracy": 0.8181818181818182, "count": 8, "counts": { "absent_correct": 9, "absent_total": 11, "exact": 5, "false_abstention": 2, "field_correct": 16, "fields": 22, "fn": 4, "fp": 3, "json_parse_ok": 8, "missed_abstention": 2, "natural_stop": 8, "nonrepetitive_nonempty": 8, "predicted_values": 22, "present_correct": 7, "present_total": 11, "schema_ok": 8, "token_limited": 0, "tp": 19, "unsupported_values": 1 }, "exact": 0.625, "f1": 0.8444444444444444, "false_abstention_rate": 0.18181818181818182, "field_accuracy": 0.7272727272727273, "json_parse_rate": 1.0, "missed_abstention_rate": 0.18181818181818182, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.21559197976603173, "precision": 0.8636363636363636, "present_accuracy": 0.6363636363636364, "recall": 0.8260869565217391, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.045454545454545456 }, "domain/single_lists": { "absent_accuracy": 0.75, "count": 4, "counts": { "absent_correct": 6, "absent_total": 8, "exact": 0, "false_abstention": 4, "field_correct": 6, "fields": 12, "fn": 4, "fp": 2, "json_parse_ok": 4, "missed_abstention": 2, "natural_stop": 4, "nonrepetitive_nonempty": 4, "predicted_values": 2, "present_correct": 0, "present_total": 4, "schema_ok": 4, "token_limited": 0, "tp": 0, "unsupported_values": 0 }, "exact": 0.0, "f1": 0.0, "false_abstention_rate": 1.0, "field_accuracy": 0.5, "json_parse_rate": 1.0, "missed_abstention_rate": 0.25, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.21651429869234562, "precision": 0.0, "present_accuracy": 0.0, "recall": 0.0, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "macro_exact": 0.6675824175824177, "paired_requests": { "accuracy": 0.7045454545454546, "both_correct": 31, "count": 44 }, "panel/legacy": { "absent_accuracy": 0.9076923076923077, "count": 128, "counts": { "absent_correct": 118, "absent_total": 130, "exact": 96, "false_abstention": 11, "field_correct": 267, "fields": 310, "fn": 36, "fp": 24, "json_parse_ok": 128, "missed_abstention": 12, "natural_stop": 128, "nonrepetitive_nonempty": 128, "predicted_values": 235, "present_correct": 149, "present_total": 180, "schema_ok": 127, "token_limited": 0, "tp": 211, "unsupported_values": 4 }, "exact": 0.75, "f1": 0.8755186721991701, "false_abstention_rate": 0.06111111111111111, "field_accuracy": 0.8612903225806452, "json_parse_rate": 1.0, "missed_abstention_rate": 0.09230769230769231, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.06681022582381502, "precision": 0.8978723404255319, "present_accuracy": 0.8277777777777777, "recall": 0.854251012145749, "schema_valid_rate": 0.9921875, "token_limited_rate": 0.0, "unsupported_value_rate": 0.01702127659574468 }, "panel/new_prose": { "absent_accuracy": 0.93048128342246, "count": 112, "counts": { "absent_correct": 174, "absent_total": 187, "exact": 82, "false_abstention": 14, "field_correct": 365, "fields": 404, "fn": 29, "fp": 23, "json_parse_ok": 112, "missed_abstention": 13, "natural_stop": 112, "nonrepetitive_nonempty": 112, "predicted_values": 269, "present_correct": 191, "present_total": 217, "schema_ok": 112, "token_limited": 0, "tp": 246, "unsupported_values": 2 }, "exact": 0.7321428571428571, "f1": 0.9044117647058822, "false_abstention_rate": 0.06451612903225806, "field_accuracy": 0.9034653465346535, "json_parse_rate": 1.0, "missed_abstention_rate": 0.06951871657754011, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.027105198803058426, "precision": 0.9144981412639405, "present_accuracy": 0.880184331797235, "recall": 0.8945454545454545, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.007434944237918215 }, "variant/alias": { "absent_accuracy": 0.9078947368421053, "count": 46, "counts": { "absent_correct": 69, "absent_total": 76, "exact": 29, "false_abstention": 11, "field_correct": 131, "fields": 157, "fn": 20, "fp": 14, "json_parse_ok": 46, "missed_abstention": 7, "natural_stop": 46, "nonrepetitive_nonempty": 46, "predicted_values": 101, "present_correct": 62, "present_total": 81, "schema_ok": 46, "token_limited": 0, "tp": 87, "unsupported_values": 1 }, "exact": 0.6304347826086957, "f1": 0.8365384615384616, "false_abstention_rate": 0.13580246913580246, "field_accuracy": 0.8343949044585988, "json_parse_rate": 1.0, "missed_abstention_rate": 0.09210526315789473, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.06243703520158306, "precision": 0.8613861386138614, "present_accuracy": 0.7654320987654321, "recall": 0.8130841121495327, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.009900990099009901 }, "variant/canonical": { "absent_accuracy": 0.9235294117647059, "count": 149, "counts": { "absent_correct": 157, "absent_total": 170, "exact": 118, "false_abstention": 8, "field_correct": 369, "fields": 405, "fn": 27, "fp": 24, "json_parse_ok": 149, "missed_abstention": 13, "natural_stop": 149, "nonrepetitive_nonempty": 149, "predicted_values": 304, "present_correct": 212, "present_total": 235, "schema_ok": 149, "token_limited": 0, "tp": 280, "unsupported_values": 4 }, "exact": 0.7919463087248322, "f1": 0.9165302782324058, "false_abstention_rate": 0.03404255319148936, "field_accuracy": 0.9111111111111111, "json_parse_rate": 1.0, "missed_abstention_rate": 0.07647058823529412, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.043735888344098986, "precision": 0.9210526315789473, "present_accuracy": 0.902127659574468, "recall": 0.9120521172638436, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.013157894736842105 }, "variant/nullable": { "absent_accuracy": 0.9295774647887324, "count": 45, "counts": { "absent_correct": 66, "absent_total": 71, "exact": 31, "false_abstention": 6, "field_correct": 132, "fields": 152, "fn": 18, "fp": 9, "json_parse_ok": 45, "missed_abstention": 5, "natural_stop": 45, "nonrepetitive_nonempty": 45, "predicted_values": 99, "present_correct": 66, "present_total": 81, "schema_ok": 44, "token_limited": 0, "tp": 90, "unsupported_values": 1 }, "exact": 0.6888888888888889, "f1": 0.8695652173913043, "false_abstention_rate": 0.07407407407407407, "field_accuracy": 0.868421052631579, "json_parse_rate": 1.0, "missed_abstention_rate": 0.07042253521126761, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.04886089308549547, "precision": 0.9090909090909091, "present_accuracy": 0.8148148148148148, "recall": 0.8333333333333334, "schema_valid_rate": 0.9777777777777777, "token_limited_rate": 0.0, "unsupported_value_rate": 0.010101010101010102 } }, "retention": { "gained": [ "snips:AddToPlaylist:train:954:view0:native:canonical", "snips:AddToPlaylist:train:1715:view0:native:canonical", "snips:AddToPlaylist:train:1564:view0:native:canonical", "focus110:development:roles:61", "focus110:development:roles:36", "focus110:development:roles:33", "focus109:development:lists:43", "focus109:development:lists:38", "focus109:development:lists:40", "cardinality105:development:16:alias", "cardinality105:development:37:alias", "cardinality105:development:10:alias", "mixsnips:dev:738:focused102:canonical", "joint128:development:PlayMusic:1:alias", "joint128:development:PlayMusic:5:nullable", "joint128:development:PlayMusic:7:alias", "joint128:development:PlayMusic:8:nullable", "joint128:development:PlayMusic:11:nullable", "joint128:development:GetWeather:1:alias", "joint128:development:GetWeather:3:canonical", "joint128:development:GetWeather:7:alias", "joint128:development:GetWeather:8:nullable", "joint128:development:GetWeather:9:canonical", "joint128:development:GetWeather:13:alias", "joint128:development:BookRestaurant:0:canonical", "joint128:development:BookRestaurant:1:alias", "joint128:development:BookRestaurant:3:canonical", "joint128:development:BookRestaurant:5:nullable", "joint128:development:BookRestaurant:9:canonical", "joint128:development:AddToPlaylist:1:alias", "joint128:development:AddToPlaylist:2:nullable", "joint128:development:AddToPlaylist:4:alias", "joint128:development:AddToPlaylist:9:canonical", "joint128:development:AddToPlaylist:11:nullable", "joint128:development:AddToPlaylist:13:alias", "joint128:development:AddToPlaylist:14:nullable", "joint128:development:RateBook:1:alias", "joint128:development:RateBook:6:canonical", "joint128:development:RateBook:10:alias", "joint128:development:RateBook:11:nullable", "joint128:development:RateBook:12:canonical", "joint128:development:RateBook:13:alias", "joint128:development:SearchCreativeWork:1:alias", "joint128:development:SearchCreativeWork:15:canonical", "joint128:development:SearchScreeningEvent:0:canonical", "joint128:development:SearchScreeningEvent:1:alias", "joint128:development:SearchScreeningEvent:3:canonical", "joint128:development:SearchScreeningEvent:5:nullable", "joint128:development:SearchScreeningEvent:7:alias", "joint128:development:SearchScreeningEvent:9:canonical", "joint128:development:SearchScreeningEvent:11:nullable", "joint128:development:SearchScreeningEvent:13:alias" ], "lost_previously_correct": [ "snips:BookRestaurant:train:246:view1:native:canonical", "snips:RateBook:train:1057:view0:native:canonical", "snips:RateBook:train:1057:view1:native:canonical", "focus109:development:lists:80", "focus109:development:lists:77", "focus109:development:lists:83" ], "new_prose_correct_delta": 37, "paired_correct_delta": -1, "panels": { "all": { "error_deltas": { "false_abstention": -3, "missed_abstention": -2, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.01538461538461533, "exact": 0.0546875, "f1": 0.04434984103033901, "field_accuracy": 0.029032258064516148, "json_parse_rate": 0.0234375, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.0048490846115784025, "present_accuracy": 0.03888888888888886, "recall": 0.07692307692307687, "schema_valid_rate": 0.0625 }, "passed": true }, "domain/AddToPlaylist": { "error_deltas": { "false_abstention": -2, "missed_abstention": 0, "token_limited": 0, "unsupported_values": -1 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": 0.24999999999999994, "f1": 0.11988304093567259, "field_accuracy": 0.09375, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.07777777777777783, "present_accuracy": 0.16666666666666663, "recall": 0.16666666666666663, "schema_valid_rate": 0.0 }, "passed": true }, "domain/BookRestaurant": { "error_deltas": { "false_abstention": 0, "missed_abstention": 1, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": -0.07692307692307687, "exact": -0.08333333333333337, "f1": -0.026890756302521024, "field_accuracy": -0.03448275862068961, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": -0.04678362573099415, "present_accuracy": 0.0, "recall": 0.0, "schema_valid_rate": 0.0 }, "passed": false }, "domain/GetWeather": { "error_deltas": { "false_abstention": 0, "missed_abstention": 0, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": 0.0, "f1": 0.0, "field_accuracy": 0.0, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.0, "present_accuracy": 0.0, "recall": 0.0, "schema_valid_rate": 0.0 }, "passed": true }, "domain/PlayMusic": { "error_deltas": { "false_abstention": 0, "missed_abstention": 0, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": 0.0, "f1": 0.0, "field_accuracy": 0.0, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.0, "present_accuracy": 0.0, "recall": 0.0, "schema_valid_rate": 0.0 }, "passed": true }, "domain/RateBook": { "error_deltas": { "false_abstention": 0, "missed_abstention": 0, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": -0.16666666666666663, "f1": -0.09523809523809523, "field_accuracy": -0.05882352941176472, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": -0.09523809523809523, "present_accuracy": -0.09523809523809523, "recall": -0.09523809523809523, "schema_valid_rate": 0.0 }, "passed": false }, "domain/SearchCreativeWork": { "error_deltas": { "false_abstention": 0, "missed_abstention": 0, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": 0.0, "f1": 0.0, "field_accuracy": 0.0, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.0, "present_accuracy": 0.0, "recall": 0.0, "schema_valid_rate": 0.0 }, "passed": true }, "domain/SearchScreeningEvent": { "error_deltas": { "false_abstention": 0, "missed_abstention": 0, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": 0.0, "f1": 0.0, "field_accuracy": 0.0, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.0, "present_accuracy": 0.0, "recall": 0.0, "schema_valid_rate": 0.0 }, "passed": true }, "domain/cardinality": { "error_deltas": { "false_abstention": 0, "missed_abstention": -2, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.25, "exact": 0.375, "f1": 0.33333333333333337, "field_accuracy": 0.3125, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.33333333333333337, "present_accuracy": 0.375, "recall": 0.33333333333333337, "schema_valid_rate": 0.375 }, "passed": true }, "domain/mixsnips": { "error_deltas": { "false_abstention": 0, "missed_abstention": 0, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": 0.125, "f1": -0.05882352941176466, "field_accuracy": 0.0, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": -0.08333333333333337, "present_accuracy": 0.0, "recall": -0.04545454545454547, "schema_valid_rate": 0.125 }, "passed": false }, "domain/observed_lists": { "error_deltas": { "false_abstention": 0, "missed_abstention": -2, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.16666666666666663, "exact": 0.0, "f1": 0.07657378740970067, "field_accuracy": 0.08333333333333337, "json_parse_rate": 0.16666666666666663, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.0, "present_accuracy": 0.0, "recall": 0.13207547169811318, "schema_valid_rate": 0.16666666666666663 }, "passed": true }, "domain/playground": { "error_deltas": { "false_abstention": 0, "missed_abstention": 0, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": 0.0, "f1": 0.0, "field_accuracy": 0.0, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.0, "present_accuracy": 0.0, "recall": 0.0, "schema_valid_rate": 0.0 }, "passed": true }, "domain/roles": { "error_deltas": { "false_abstention": -1, "missed_abstention": 0, "token_limited": 0, "unsupported_values": 1 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": 0.375, "f1": 0.23333333333333328, "field_accuracy": 0.13636363636363635, "json_parse_rate": 0.125, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.0174825174825175, "present_accuracy": 0.2727272727272727, "recall": 0.34782608695652173, "schema_valid_rate": 0.25 }, "passed": false }, "domain/single_lists": { "error_deltas": { "false_abstention": 0, "missed_abstention": 1, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": -0.125, "exact": 0.0, "f1": 0.0, "field_accuracy": -0.08333333333333337, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.0, "present_accuracy": 0.0, "recall": 0.0, "schema_valid_rate": 0.0 }, "passed": false }, "variant/alias": { "error_deltas": { "false_abstention": 0, "missed_abstention": -2, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.1333333333333333, "exact": 0.4545454545454545, "f1": 0.380392156862745, "field_accuracy": 0.20689655172413796, "json_parse_rate": 0.09090909090909094, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": 0.3857142857142857, "present_accuracy": 0.2857142857142857, "recall": 0.35000000000000003, "schema_valid_rate": 0.4545454545454546 }, "passed": true }, "variant/canonical": { "error_deltas": { "false_abstention": -4, "missed_abstention": 0, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": 0.028037383177570097, "f1": 0.029791422893334252, "field_accuracy": 0.01945525291828798, "json_parse_rate": 0.01869158878504673, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": -0.012469023085952768, "present_accuracy": 0.0326797385620915, "recall": 0.06796116504854377, "schema_valid_rate": 0.01869158878504673 }, "passed": false }, "variant/nullable": { "error_deltas": { "false_abstention": 1, "missed_abstention": 0, "token_limited": 0, "unsupported_values": 0 }, "metric_deltas": { "absent_accuracy": 0.0, "exact": -0.09999999999999998, "f1": -0.0637560637560638, "field_accuracy": -0.08333333333333337, "json_parse_rate": 0.0, "natural_stop_rate": 0.0, "nonrepetitive_nonempty_rate": 0.0, "precision": -0.01388888888888884, "present_accuracy": -0.15384615384615385, "recall": -0.09523809523809523, "schema_valid_rate": 0.09999999999999998 }, "passed": false } }, "passed": false, "scope": "Finite observed request/metric retention gates; cannot guarantee universal no-regression." } }, "heldout_1444": { "metrics": { "all": { "absent_accuracy": 0.9521625163826999, "count": 1444, "counts": { "absent_correct": 1453, "absent_total": 1526, "exact": 1271, "false_abstention": 53, "field_correct": 3293, "fields": 3497, "fn": 131, "fp": 151, "json_parse_ok": 1444, "missed_abstention": 73, "natural_stop": 1444, "nonrepetitive_nonempty": 1444, "predicted_values": 1991, "present_correct": 1840, "present_total": 1971, "schema_ok": 1444, "token_limited": 0, "tp": 1840, "unsupported_values": 24 }, "exact": 0.8801939058171745, "f1": 0.928823826350328, "false_abstention_rate": 0.026889903602232368, "field_accuracy": 0.9416642836717186, "json_parse_rate": 1.0, "missed_abstention_rate": 0.04783748361730013, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.024015552303623188, "precision": 0.9241587142139628, "present_accuracy": 0.9335362760020294, "recall": 0.9335362760020294, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.012054244098442994 }, "domain/AddToPlaylist": { "absent_accuracy": 0.9364406779661016, "count": 206, "counts": { "absent_correct": 221, "absent_total": 236, "exact": 172, "false_abstention": 8, "field_correct": 481, "fields": 526, "fn": 30, "fp": 37, "json_parse_ok": 206, "missed_abstention": 15, "natural_stop": 206, "nonrepetitive_nonempty": 206, "predicted_values": 297, "present_correct": 260, "present_total": 290, "schema_ok": 206, "token_limited": 0, "tp": 260, "unsupported_values": 14 }, "exact": 0.8349514563106796, "f1": 0.8858603066439523, "false_abstention_rate": 0.027586206896551724, "field_accuracy": 0.9144486692015209, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0635593220338983, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.041144459200014236, "precision": 0.8754208754208754, "present_accuracy": 0.896551724137931, "recall": 0.896551724137931, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.04713804713804714 }, "domain/BookRestaurant": { "absent_accuracy": 0.932, "count": 208, "counts": { "absent_correct": 233, "absent_total": 250, "exact": 172, "false_abstention": 8, "field_correct": 528, "fields": 569, "fn": 24, "fp": 33, "json_parse_ok": 208, "missed_abstention": 17, "natural_stop": 208, "nonrepetitive_nonempty": 208, "predicted_values": 328, "present_correct": 295, "present_total": 319, "schema_ok": 208, "token_limited": 0, "tp": 295, "unsupported_values": 1 }, "exact": 0.8269230769230769, "f1": 0.9119010819165378, "false_abstention_rate": 0.025078369905956112, "field_accuracy": 0.9279437609841827, "json_parse_rate": 1.0, "missed_abstention_rate": 0.068, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.027245943393189427, "precision": 0.899390243902439, "present_accuracy": 0.9247648902821317, "recall": 0.9247648902821317, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.003048780487804878 }, "domain/GetWeather": { "absent_accuracy": 0.943089430894309, "count": 206, "counts": { "absent_correct": 232, "absent_total": 246, "exact": 178, "false_abstention": 5, "field_correct": 493, "fields": 525, "fn": 18, "fp": 27, "json_parse_ok": 206, "missed_abstention": 14, "natural_stop": 206, "nonrepetitive_nonempty": 206, "predicted_values": 288, "present_correct": 261, "present_total": 279, "schema_ok": 206, "token_limited": 0, "tp": 261, "unsupported_values": 5 }, "exact": 0.8640776699029126, "f1": 0.9206349206349206, "false_abstention_rate": 0.017921146953405017, "field_accuracy": 0.939047619047619, "json_parse_rate": 1.0, "missed_abstention_rate": 0.056910569105691054, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.022860561515487453, "precision": 0.90625, "present_accuracy": 0.9354838709677419, "recall": 0.9354838709677419, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.017361111111111112 }, "domain/PlayMusic": { "absent_accuracy": 0.9296875, "count": 208, "counts": { "absent_correct": 238, "absent_total": 256, "exact": 164, "false_abstention": 24, "field_correct": 468, "fields": 523, "fn": 37, "fp": 31, "json_parse_ok": 208, "missed_abstention": 18, "natural_stop": 208, "nonrepetitive_nonempty": 208, "predicted_values": 261, "present_correct": 230, "present_total": 267, "schema_ok": 208, "token_limited": 0, "tp": 230, "unsupported_values": 0 }, "exact": 0.7884615384615384, "f1": 0.8712121212121211, "false_abstention_rate": 0.0898876404494382, "field_accuracy": 0.8948374760994264, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0703125, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.03961348310237484, "precision": 0.8812260536398467, "present_accuracy": 0.8614232209737828, "recall": 0.8614232209737828, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "domain/RateBook": { "absent_accuracy": 0.983739837398374, "count": 204, "counts": { "absent_correct": 242, "absent_total": 246, "exact": 195, "false_abstention": 1, "field_correct": 558, "fields": 567, "fn": 5, "fp": 8, "json_parse_ok": 204, "missed_abstention": 4, "natural_stop": 204, "nonrepetitive_nonempty": 204, "predicted_values": 324, "present_correct": 316, "present_total": 321, "schema_ok": 204, "token_limited": 0, "tp": 316, "unsupported_values": 1 }, "exact": 0.9558823529411765, "f1": 0.9798449612403102, "false_abstention_rate": 0.003115264797507788, "field_accuracy": 0.9841269841269841, "json_parse_rate": 1.0, "missed_abstention_rate": 0.016260162601626018, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.008335007565976554, "precision": 0.9753086419753086, "present_accuracy": 0.9844236760124611, "recall": 0.9844236760124611, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0030864197530864196 }, "domain/SearchCreativeWork": { "absent_accuracy": 1.0, "count": 208, "counts": { "absent_correct": 39, "absent_total": 39, "exact": 199, "false_abstention": 2, "field_correct": 277, "fields": 286, "fn": 9, "fp": 7, "json_parse_ok": 208, "missed_abstention": 0, "natural_stop": 208, "nonrepetitive_nonempty": 208, "predicted_values": 245, "present_correct": 238, "present_total": 247, "schema_ok": 208, "token_limited": 0, "tp": 238, "unsupported_values": 2 }, "exact": 0.9567307692307693, "f1": 0.9674796747967479, "false_abstention_rate": 0.008097165991902834, "field_accuracy": 0.9685314685314685, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.01792181849521932, "precision": 0.9714285714285714, "present_accuracy": 0.9635627530364372, "recall": 0.9635627530364372, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.00816326530612245 }, "domain/SearchScreeningEvent": { "absent_accuracy": 0.9802371541501976, "count": 204, "counts": { "absent_correct": 248, "absent_total": 253, "exact": 191, "false_abstention": 5, "field_correct": 488, "fields": 501, "fn": 8, "fp": 8, "json_parse_ok": 204, "missed_abstention": 5, "natural_stop": 204, "nonrepetitive_nonempty": 204, "predicted_values": 248, "present_correct": 240, "present_total": 248, "schema_ok": 204, "token_limited": 0, "tp": 240, "unsupported_values": 1 }, "exact": 0.9362745098039216, "f1": 0.967741935483871, "false_abstention_rate": 0.020161290322580645, "field_accuracy": 0.9740518962075848, "json_parse_rate": 1.0, "missed_abstention_rate": 0.019762845849802372, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.01058128802694281, "precision": 0.967741935483871, "present_accuracy": 0.967741935483871, "recall": 0.967741935483871, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.004032258064516129 }, "macro_exact": 0.8804716247962964, "paired_requests": { "accuracy": 0.80835734870317, "both_correct": 561, "count": 694 }, "panel/original1444": { "absent_accuracy": 0.9521625163826999, "count": 1444, "counts": { "absent_correct": 1453, "absent_total": 1526, "exact": 1271, "false_abstention": 53, "field_correct": 3293, "fields": 3497, "fn": 131, "fp": 151, "json_parse_ok": 1444, "missed_abstention": 73, "natural_stop": 1444, "nonrepetitive_nonempty": 1444, "predicted_values": 1991, "present_correct": 1840, "present_total": 1971, "schema_ok": 1444, "token_limited": 0, "tp": 1840, "unsupported_values": 24 }, "exact": 0.8801939058171745, "f1": 0.928823826350328, "false_abstention_rate": 0.026889903602232368, "field_accuracy": 0.9416642836717186, "json_parse_rate": 1.0, "missed_abstention_rate": 0.04783748361730013, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.024015552303623188, "precision": 0.9241587142139628, "present_accuracy": 0.9335362760020294, "recall": 0.9335362760020294, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.012054244098442994 }, "variant/alias": { "absent_accuracy": 0.9310344827586207, "count": 28, "counts": { "absent_correct": 27, "absent_total": 29, "exact": 23, "false_abstention": 1, "field_correct": 70, "fields": 75, "fn": 3, "fp": 4, "json_parse_ok": 28, "missed_abstention": 2, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 47, "present_correct": 43, "present_total": 46, "schema_ok": 28, "token_limited": 0, "tp": 43, "unsupported_values": 2 }, "exact": 0.8214285714285714, "f1": 0.924731182795699, "false_abstention_rate": 0.021739130434782608, "field_accuracy": 0.9333333333333333, "json_parse_rate": 1.0, "missed_abstention_rate": 0.06896551724137931, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.051775694317517, "precision": 0.9148936170212766, "present_accuracy": 0.9347826086956522, "recall": 0.9347826086956522, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0425531914893617 }, "variant/canonical": { "absent_accuracy": 0.952316076294278, "count": 1388, "counts": { "absent_correct": 1398, "absent_total": 1468, "exact": 1224, "false_abstention": 50, "field_correct": 3154, "fields": 3347, "fn": 123, "fp": 143, "json_parse_ok": 1388, "missed_abstention": 70, "natural_stop": 1388, "nonrepetitive_nonempty": 1388, "predicted_values": 1899, "present_correct": 1756, "present_total": 1879, "schema_ok": 1388, "token_limited": 0, "tp": 1756, "unsupported_values": 21 }, "exact": 0.8818443804034583, "f1": 0.9295923769190046, "false_abstention_rate": 0.026609898882384245, "field_accuracy": 0.9423364206752316, "json_parse_rate": 1.0, "missed_abstention_rate": 0.047683923705722074, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.022789464743013806, "precision": 0.9246972090573986, "present_accuracy": 0.9345396487493347, "recall": 0.9345396487493347, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.011058451816745656 }, "variant/nullable": { "absent_accuracy": 0.9655172413793104, "count": 28, "counts": { "absent_correct": 28, "absent_total": 29, "exact": 24, "false_abstention": 2, "field_correct": 69, "fields": 75, "fn": 5, "fp": 4, "json_parse_ok": 28, "missed_abstention": 1, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 45, "present_correct": 41, "present_total": 46, "schema_ok": 28, "token_limited": 0, "tp": 41, "unsupported_values": 1 }, "exact": 0.8571428571428571, "f1": 0.9010989010989011, "false_abstention_rate": 0.043478260869565216, "field_accuracy": 0.92, "json_parse_rate": 1.0, "missed_abstention_rate": 0.034482758620689655, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.05703432222279454, "precision": 0.9111111111111111, "present_accuracy": 0.8913043478260869, "recall": 0.8913043478260869, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.022222222222222223 } }, "previous_metrics": { "all": { "absent_accuracy": 0.927916120576671, "count": 1444, "counts": { "absent_correct": 1416, "absent_total": 1526, "exact": 1199, "false_abstention": 107, "field_correct": 3195, "fields": 3497, "fn": 192, "fp": 186, "json_parse_ok": 1442, "missed_abstention": 110, "natural_stop": 1444, "nonrepetitive_nonempty": 1444, "predicted_values": 1965, "present_correct": 1779, "present_total": 1971, "schema_ok": 1441, "token_limited": 0, "tp": 1779, "unsupported_values": 27 }, "exact": 0.8303324099722992, "f1": 0.9039634146341463, "false_abstention_rate": 0.05428716387620497, "field_accuracy": 0.9136402630826422, "json_parse_rate": 0.9986149584487535, "missed_abstention_rate": 0.07208387942332896, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.030197499356996216, "precision": 0.9053435114503817, "present_accuracy": 0.9025875190258752, "recall": 0.9025875190258752, "schema_valid_rate": 0.9979224376731302, "token_limited_rate": 0.0, "unsupported_value_rate": 0.013740458015267175 }, "domain/AddToPlaylist": { "absent_accuracy": 0.9152542372881356, "count": 206, "counts": { "absent_correct": 216, "absent_total": 236, "exact": 165, "false_abstention": 12, "field_correct": 473, "fields": 526, "fn": 33, "fp": 41, "json_parse_ok": 206, "missed_abstention": 20, "natural_stop": 206, "nonrepetitive_nonempty": 206, "predicted_values": 298, "present_correct": 257, "present_total": 290, "schema_ok": 206, "token_limited": 0, "tp": 257, "unsupported_values": 15 }, "exact": 0.8009708737864077, "f1": 0.8741496598639455, "false_abstention_rate": 0.041379310344827586, "field_accuracy": 0.8992395437262357, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0847457627118644, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.04620553536913629, "precision": 0.8624161073825504, "present_accuracy": 0.8862068965517241, "recall": 0.8862068965517241, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.050335570469798654 }, "domain/BookRestaurant": { "absent_accuracy": 0.908, "count": 208, "counts": { "absent_correct": 227, "absent_total": 250, "exact": 162, "false_abstention": 16, "field_correct": 510, "fields": 569, "fn": 36, "fp": 43, "json_parse_ok": 208, "missed_abstention": 23, "natural_stop": 208, "nonrepetitive_nonempty": 208, "predicted_values": 326, "present_correct": 283, "present_total": 319, "schema_ok": 208, "token_limited": 0, "tp": 283, "unsupported_values": 3 }, "exact": 0.7788461538461539, "f1": 0.8775193798449612, "false_abstention_rate": 0.050156739811912224, "field_accuracy": 0.8963093145869947, "json_parse_rate": 1.0, "missed_abstention_rate": 0.092, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.03511890941109204, "precision": 0.8680981595092024, "present_accuracy": 0.8871473354231975, "recall": 0.8871473354231975, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.009202453987730062 }, "domain/GetWeather": { "absent_accuracy": 0.9186991869918699, "count": 206, "counts": { "absent_correct": 226, "absent_total": 246, "exact": 164, "false_abstention": 15, "field_correct": 473, "fields": 525, "fn": 32, "fp": 32, "json_parse_ok": 205, "missed_abstention": 20, "natural_stop": 206, "nonrepetitive_nonempty": 206, "predicted_values": 279, "present_correct": 247, "present_total": 279, "schema_ok": 204, "token_limited": 0, "tp": 247, "unsupported_values": 3 }, "exact": 0.7961165048543689, "f1": 0.8853046594982079, "false_abstention_rate": 0.053763440860215055, "field_accuracy": 0.900952380952381, "json_parse_rate": 0.9951456310679612, "missed_abstention_rate": 0.08130081300813008, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.030885711256629398, "precision": 0.8853046594982079, "present_accuracy": 0.8853046594982079, "recall": 0.8853046594982079, "schema_valid_rate": 0.9902912621359223, "token_limited_rate": 0.0, "unsupported_value_rate": 0.010752688172043012 }, "domain/PlayMusic": { "absent_accuracy": 0.94140625, "count": 208, "counts": { "absent_correct": 241, "absent_total": 256, "exact": 153, "false_abstention": 45, "field_correct": 449, "fields": 523, "fn": 59, "fp": 25, "json_parse_ok": 207, "missed_abstention": 15, "natural_stop": 208, "nonrepetitive_nonempty": 208, "predicted_values": 233, "present_correct": 208, "present_total": 267, "schema_ok": 207, "token_limited": 0, "tp": 208, "unsupported_values": 1 }, "exact": 0.7355769230769231, "f1": 0.8320000000000001, "false_abstention_rate": 0.16853932584269662, "field_accuracy": 0.858508604206501, "json_parse_rate": 0.9951923076923077, "missed_abstention_rate": 0.05859375, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.0454767054272032, "precision": 0.8927038626609443, "present_accuracy": 0.7790262172284644, "recall": 0.7790262172284644, "schema_valid_rate": 0.9951923076923077, "token_limited_rate": 0.0, "unsupported_value_rate": 0.004291845493562232 }, "domain/RateBook": { "absent_accuracy": 0.9796747967479674, "count": 204, "counts": { "absent_correct": 241, "absent_total": 246, "exact": 193, "false_abstention": 3, "field_correct": 556, "fields": 567, "fn": 6, "fp": 8, "json_parse_ok": 204, "missed_abstention": 5, "natural_stop": 204, "nonrepetitive_nonempty": 204, "predicted_values": 323, "present_correct": 315, "present_total": 321, "schema_ok": 204, "token_limited": 0, "tp": 315, "unsupported_values": 1 }, "exact": 0.946078431372549, "f1": 0.9782608695652175, "false_abstention_rate": 0.009345794392523364, "field_accuracy": 0.9805996472663139, "json_parse_rate": 1.0, "missed_abstention_rate": 0.02032520325203252, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.010863955307286233, "precision": 0.9752321981424149, "present_accuracy": 0.9813084112149533, "recall": 0.9813084112149533, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0030959752321981426 }, "domain/SearchCreativeWork": { "absent_accuracy": 0.9487179487179487, "count": 208, "counts": { "absent_correct": 37, "absent_total": 39, "exact": 197, "false_abstention": 2, "field_correct": 275, "fields": 286, "fn": 9, "fp": 9, "json_parse_ok": 208, "missed_abstention": 2, "natural_stop": 208, "nonrepetitive_nonempty": 208, "predicted_values": 247, "present_correct": 238, "present_total": 247, "schema_ok": 208, "token_limited": 0, "tp": 238, "unsupported_values": 0 }, "exact": 0.9471153846153846, "f1": 0.9635627530364372, "false_abstention_rate": 0.008097165991902834, "field_accuracy": 0.9615384615384616, "json_parse_rate": 1.0, "missed_abstention_rate": 0.05128205128205128, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.019084845725116723, "precision": 0.9635627530364372, "present_accuracy": 0.9635627530364372, "recall": 0.9635627530364372, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "domain/SearchScreeningEvent": { "absent_accuracy": 0.9011857707509882, "count": 204, "counts": { "absent_correct": 228, "absent_total": 253, "exact": 165, "false_abstention": 14, "field_correct": 459, "fields": 501, "fn": 17, "fp": 28, "json_parse_ok": 204, "missed_abstention": 25, "natural_stop": 204, "nonrepetitive_nonempty": 204, "predicted_values": 259, "present_correct": 231, "present_total": 248, "schema_ok": 204, "token_limited": 0, "tp": 231, "unsupported_values": 4 }, "exact": 0.8088235294117647, "f1": 0.9112426035502958, "false_abstention_rate": 0.056451612903225805, "field_accuracy": 0.9161676646706587, "json_parse_rate": 1.0, "missed_abstention_rate": 0.09881422924901186, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.023404948954503554, "precision": 0.8918918918918919, "present_accuracy": 0.9314516129032258, "recall": 0.9314516129032258, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.015444015444015444 }, "macro_exact": 0.8305039715662217, "paired_requests": { "accuracy": 0.7377521613832853, "both_correct": 512, "count": 694 }, "panel/original1444": { "absent_accuracy": 0.927916120576671, "count": 1444, "counts": { "absent_correct": 1416, "absent_total": 1526, "exact": 1199, "false_abstention": 107, "field_correct": 3195, "fields": 3497, "fn": 192, "fp": 186, "json_parse_ok": 1442, "missed_abstention": 110, "natural_stop": 1444, "nonrepetitive_nonempty": 1444, "predicted_values": 1965, "present_correct": 1779, "present_total": 1971, "schema_ok": 1441, "token_limited": 0, "tp": 1779, "unsupported_values": 27 }, "exact": 0.8303324099722992, "f1": 0.9039634146341463, "false_abstention_rate": 0.05428716387620497, "field_accuracy": 0.9136402630826422, "json_parse_rate": 0.9986149584487535, "missed_abstention_rate": 0.07208387942332896, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.030197499356996216, "precision": 0.9053435114503817, "present_accuracy": 0.9025875190258752, "recall": 0.9025875190258752, "schema_valid_rate": 0.9979224376731302, "token_limited_rate": 0.0, "unsupported_value_rate": 0.013740458015267175 }, "variant/alias": { "absent_accuracy": 0.7931034482758621, "count": 28, "counts": { "absent_correct": 23, "absent_total": 29, "exact": 21, "false_abstention": 1, "field_correct": 61, "fields": 75, "fn": 8, "fp": 5, "json_parse_ok": 26, "missed_abstention": 6, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 43, "present_correct": 38, "present_total": 46, "schema_ok": 26, "token_limited": 0, "tp": 38, "unsupported_values": 2 }, "exact": 0.75, "f1": 0.853932584269663, "false_abstention_rate": 0.021739130434782608, "field_accuracy": 0.8133333333333334, "json_parse_rate": 0.9285714285714286, "missed_abstention_rate": 0.20689655172413793, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.05955322041908013, "precision": 0.8837209302325582, "present_accuracy": 0.8260869565217391, "recall": 0.8260869565217391, "schema_valid_rate": 0.9285714285714286, "token_limited_rate": 0.0, "unsupported_value_rate": 0.046511627906976744 }, "variant/canonical": { "absent_accuracy": 0.9339237057220708, "count": 1388, "counts": { "absent_correct": 1371, "absent_total": 1468, "exact": 1163, "false_abstention": 100, "field_correct": 3075, "fields": 3347, "fn": 175, "fp": 172, "json_parse_ok": 1388, "missed_abstention": 97, "natural_stop": 1388, "nonrepetitive_nonempty": 1388, "predicted_values": 1876, "present_correct": 1704, "present_total": 1879, "schema_ok": 1388, "token_limited": 0, "tp": 1704, "unsupported_values": 23 }, "exact": 0.8378962536023055, "f1": 0.9075898801597869, "false_abstention_rate": 0.05321979776476849, "field_accuracy": 0.9187331939049895, "json_parse_rate": 1.0, "missed_abstention_rate": 0.06607629427792916, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.028413429219706483, "precision": 0.908315565031983, "present_accuracy": 0.9068653539116551, "recall": 0.9068653539116551, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.012260127931769723 }, "variant/nullable": { "absent_accuracy": 0.7586206896551724, "count": 28, "counts": { "absent_correct": 22, "absent_total": 29, "exact": 15, "false_abstention": 6, "field_correct": 59, "fields": 75, "fn": 9, "fp": 9, "json_parse_ok": 28, "missed_abstention": 7, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 46, "present_correct": 37, "present_total": 46, "schema_ok": 27, "token_limited": 0, "tp": 37, "unsupported_values": 2 }, "exact": 0.5357142857142857, "f1": 0.8043478260869565, "false_abstention_rate": 0.13043478260869565, "field_accuracy": 0.7866666666666666, "json_parse_rate": 1.0, "missed_abstention_rate": 0.2413793103448276, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.08928068367198908, "precision": 0.8043478260869565, "present_accuracy": 0.8043478260869565, "recall": 0.8043478260869565, "schema_valid_rate": 0.9642857142857143, "token_limited_rate": 0.0, "unsupported_value_rate": 0.043478260869565216 } }, "improved_requests": 96, "regressed_requests": 24 }, "playground_60": { "metrics": { "all": { "absent_accuracy": 0.9322033898305084, "count": 60, "counts": { "absent_correct": 55, "absent_total": 59, "exact": 49, "false_abstention": 2, "field_correct": 136, "fields": 147, "fn": 9, "fp": 10, "json_parse_ok": 60, "missed_abstention": 4, "natural_stop": 60, "nonrepetitive_nonempty": 60, "predicted_values": 109, "present_correct": 81, "present_total": 88, "schema_ok": 60, "token_limited": 0, "tp": 99, "unsupported_values": 6 }, "exact": 0.8166666666666667, "f1": 0.912442396313364, "false_abstention_rate": 0.022727272727272728, "field_accuracy": 0.9251700680272109, "json_parse_rate": 1.0, "missed_abstention_rate": 0.06779661016949153, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.046341597845215195, "precision": 0.908256880733945, "present_accuracy": 0.9204545454545454, "recall": 0.9166666666666666, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.05504587155963303 }, "domain/playground": { "absent_accuracy": 0.9322033898305084, "count": 60, "counts": { "absent_correct": 55, "absent_total": 59, "exact": 49, "false_abstention": 2, "field_correct": 136, "fields": 147, "fn": 9, "fp": 10, "json_parse_ok": 60, "missed_abstention": 4, "natural_stop": 60, "nonrepetitive_nonempty": 60, "predicted_values": 109, "present_correct": 81, "present_total": 88, "schema_ok": 60, "token_limited": 0, "tp": 99, "unsupported_values": 6 }, "exact": 0.8166666666666667, "f1": 0.912442396313364, "false_abstention_rate": 0.022727272727272728, "field_accuracy": 0.9251700680272109, "json_parse_rate": 1.0, "missed_abstention_rate": 0.06779661016949153, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.046341597845215195, "precision": 0.908256880733945, "present_accuracy": 0.9204545454545454, "recall": 0.9166666666666666, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.05504587155963303 }, "macro_exact": 0.8166666666666667, "paired_requests": { "accuracy": 0.0, "both_correct": 0, "count": 0 }, "panel/playground60": { "absent_accuracy": 0.9322033898305084, "count": 60, "counts": { "absent_correct": 55, "absent_total": 59, "exact": 49, "false_abstention": 2, "field_correct": 136, "fields": 147, "fn": 9, "fp": 10, "json_parse_ok": 60, "missed_abstention": 4, "natural_stop": 60, "nonrepetitive_nonempty": 60, "predicted_values": 109, "present_correct": 81, "present_total": 88, "schema_ok": 60, "token_limited": 0, "tp": 99, "unsupported_values": 6 }, "exact": 0.8166666666666667, "f1": 0.912442396313364, "false_abstention_rate": 0.022727272727272728, "field_accuracy": 0.9251700680272109, "json_parse_rate": 1.0, "missed_abstention_rate": 0.06779661016949153, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.046341597845215195, "precision": 0.908256880733945, "present_accuracy": 0.9204545454545454, "recall": 0.9166666666666666, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.05504587155963303 }, "variant/canonical": { "absent_accuracy": 0.9322033898305084, "count": 60, "counts": { "absent_correct": 55, "absent_total": 59, "exact": 49, "false_abstention": 2, "field_correct": 136, "fields": 147, "fn": 9, "fp": 10, "json_parse_ok": 60, "missed_abstention": 4, "natural_stop": 60, "nonrepetitive_nonempty": 60, "predicted_values": 109, "present_correct": 81, "present_total": 88, "schema_ok": 60, "token_limited": 0, "tp": 99, "unsupported_values": 6 }, "exact": 0.8166666666666667, "f1": 0.912442396313364, "false_abstention_rate": 0.022727272727272728, "field_accuracy": 0.9251700680272109, "json_parse_rate": 1.0, "missed_abstention_rate": 0.06779661016949153, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.046341597845215195, "precision": 0.908256880733945, "present_accuracy": 0.9204545454545454, "recall": 0.9166666666666666, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.05504587155963303 } }, "previous_metrics": { "all": { "absent_accuracy": 0.8983050847457628, "count": 60, "counts": { "absent_correct": 53, "absent_total": 59, "exact": 45, "false_abstention": 3, "field_correct": 129, "fields": 147, "fn": 22, "fp": 13, "json_parse_ok": 58, "missed_abstention": 6, "natural_stop": 60, "nonrepetitive_nonempty": 60, "predicted_values": 99, "present_correct": 76, "present_total": 88, "schema_ok": 58, "token_limited": 0, "tp": 86, "unsupported_values": 7 }, "exact": 0.75, "f1": 0.8309178743961353, "false_abstention_rate": 0.03409090909090909, "field_accuracy": 0.8775510204081632, "json_parse_rate": 0.9666666666666667, "missed_abstention_rate": 0.1016949152542373, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.07145761149877217, "precision": 0.8686868686868687, "present_accuracy": 0.8636363636363636, "recall": 0.7962962962962963, "schema_valid_rate": 0.9666666666666667, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0707070707070707 }, "domain/playground": { "absent_accuracy": 0.8983050847457628, "count": 60, "counts": { "absent_correct": 53, "absent_total": 59, "exact": 45, "false_abstention": 3, "field_correct": 129, "fields": 147, "fn": 22, "fp": 13, "json_parse_ok": 58, "missed_abstention": 6, "natural_stop": 60, "nonrepetitive_nonempty": 60, "predicted_values": 99, "present_correct": 76, "present_total": 88, "schema_ok": 58, "token_limited": 0, "tp": 86, "unsupported_values": 7 }, "exact": 0.75, "f1": 0.8309178743961353, "false_abstention_rate": 0.03409090909090909, "field_accuracy": 0.8775510204081632, "json_parse_rate": 0.9666666666666667, "missed_abstention_rate": 0.1016949152542373, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.07145761149877217, "precision": 0.8686868686868687, "present_accuracy": 0.8636363636363636, "recall": 0.7962962962962963, "schema_valid_rate": 0.9666666666666667, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0707070707070707 }, "macro_exact": 0.75, "paired_requests": { "accuracy": 0.0, "both_correct": 0, "count": 0 }, "panel/playground60": { "absent_accuracy": 0.8983050847457628, "count": 60, "counts": { "absent_correct": 53, "absent_total": 59, "exact": 45, "false_abstention": 3, "field_correct": 129, "fields": 147, "fn": 22, "fp": 13, "json_parse_ok": 58, "missed_abstention": 6, "natural_stop": 60, "nonrepetitive_nonempty": 60, "predicted_values": 99, "present_correct": 76, "present_total": 88, "schema_ok": 58, "token_limited": 0, "tp": 86, "unsupported_values": 7 }, "exact": 0.75, "f1": 0.8309178743961353, "false_abstention_rate": 0.03409090909090909, "field_accuracy": 0.8775510204081632, "json_parse_rate": 0.9666666666666667, "missed_abstention_rate": 0.1016949152542373, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.07145761149877217, "precision": 0.8686868686868687, "present_accuracy": 0.8636363636363636, "recall": 0.7962962962962963, "schema_valid_rate": 0.9666666666666667, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0707070707070707 }, "variant/canonical": { "absent_accuracy": 0.8983050847457628, "count": 60, "counts": { "absent_correct": 53, "absent_total": 59, "exact": 45, "false_abstention": 3, "field_correct": 129, "fields": 147, "fn": 22, "fp": 13, "json_parse_ok": 58, "missed_abstention": 6, "natural_stop": 60, "nonrepetitive_nonempty": 60, "predicted_values": 99, "present_correct": 76, "present_total": 88, "schema_ok": 58, "token_limited": 0, "tp": 86, "unsupported_values": 7 }, "exact": 0.75, "f1": 0.8309178743961353, "false_abstention_rate": 0.03409090909090909, "field_accuracy": 0.8775510204081632, "json_parse_rate": 0.9666666666666667, "missed_abstention_rate": 0.1016949152542373, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.07145761149877217, "precision": 0.8686868686868687, "present_accuracy": 0.8636363636363636, "recall": 0.7962962962962963, "schema_valid_rate": 0.9666666666666667, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0707070707070707 } }, "subgroups": { "original1444": { "parent105_64": {}, "replay128_200": {}, "replay128_300": {} }, "playground60": { "parent105_64": { "added6": { "absent_accuracy": 0.6666666666666666, "count": 6, "counts": { "absent_correct": 4, "absent_total": 6, "exact": 1, "false_abstention": 0, "field_correct": 5, "fields": 12, "fn": 15, "fp": 4, "json_parse_ok": 4, "missed_abstention": 2, "natural_stop": 6, "nonrepetitive_nonempty": 6, "predicted_values": 13, "present_correct": 1, "present_total": 6, "schema_ok": 4, "token_limited": 0, "tp": 9, "unsupported_values": 4 }, "exact": 0.16666666666666666, "f1": 0.48648648648648646, "false_abstention_rate": 0.0, "field_accuracy": 0.4166666666666667, "json_parse_rate": 0.6666666666666666, "missed_abstention_rate": 0.3333333333333333, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.16294185755153498, "precision": 0.6923076923076923, "present_accuracy": 0.16666666666666666, "recall": 0.375, "schema_valid_rate": 0.6666666666666666, "token_limited_rate": 0.0, "unsupported_value_rate": 0.3076923076923077 }, "array3": { "absent_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 2, "absent_total": 2, "exact": 1, "false_abstention": 0, "field_correct": 3, "fields": 4, "fn": 1, "fp": 1, "json_parse_ok": 2, "missed_abstention": 0, "natural_stop": 2, "nonrepetitive_nonempty": 2, "predicted_values": 6, "present_correct": 1, "present_total": 2, "schema_ok": 2, "token_limited": 0, "tp": 5, "unsupported_values": 1 }, "exact": 0.5, "f1": 0.8333333333333334, "false_abstention_rate": 0.0, "field_accuracy": 0.75, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.12317727319896221, "precision": 0.8333333333333334, "present_accuracy": 0.5, "recall": 0.8333333333333334, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.16666666666666666 }, "array4": { "absent_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 1, "absent_total": 2, "exact": 0, "false_abstention": 0, "field_correct": 1, "fields": 4, "fn": 6, "fp": 2, "json_parse_ok": 1, "missed_abstention": 1, "natural_stop": 2, "nonrepetitive_nonempty": 2, "predicted_values": 4, "present_correct": 0, "present_total": 2, "schema_ok": 1, "token_limited": 0, "tp": 2, "unsupported_values": 2 }, "exact": 0.0, "f1": 0.3333333333333333, "false_abstention_rate": 0.0, "field_accuracy": 0.25, "json_parse_rate": 0.5, "missed_abstention_rate": 0.5, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.1759970560669899, "precision": 0.5, "present_accuracy": 0.0, "recall": 0.25, "schema_valid_rate": 0.5, "token_limited_rate": 0.0, "unsupported_value_rate": 0.5 }, "array5": { "absent_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 1, "absent_total": 2, "exact": 0, "false_abstention": 0, "field_correct": 1, "fields": 4, "fn": 8, "fp": 1, "json_parse_ok": 1, "missed_abstention": 1, "natural_stop": 2, "nonrepetitive_nonempty": 2, "predicted_values": 3, "present_correct": 0, "present_total": 2, "schema_ok": 1, "token_limited": 0, "tp": 2, "unsupported_values": 1 }, "exact": 0.0, "f1": 0.30769230769230765, "false_abstention_rate": 0.0, "field_accuracy": 0.25, "json_parse_rate": 0.5, "missed_abstention_rate": 0.5, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.1896512433886528, "precision": 0.6666666666666666, "present_accuracy": 0.0, "recall": 0.2, "schema_valid_rate": 0.5, "token_limited_rate": 0.0, "unsupported_value_rate": 0.3333333333333333 }, "original54": { "absent_accuracy": 0.9245283018867925, "count": 54, "counts": { "absent_correct": 49, "absent_total": 53, "exact": 44, "false_abstention": 3, "field_correct": 124, "fields": 135, "fn": 7, "fp": 9, "json_parse_ok": 54, "missed_abstention": 4, "natural_stop": 54, "nonrepetitive_nonempty": 54, "predicted_values": 86, "present_correct": 75, "present_total": 82, "schema_ok": 54, "token_limited": 0, "tp": 77, "unsupported_values": 3 }, "exact": 0.8148148148148148, "f1": 0.9058823529411765, "false_abstention_rate": 0.036585365853658534, "field_accuracy": 0.9185185185185185, "json_parse_rate": 1.0, "missed_abstention_rate": 0.07547169811320754, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.06129269527068741, "precision": 0.8953488372093024, "present_accuracy": 0.9146341463414634, "recall": 0.9166666666666666, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.03488372093023256 } }, "replay128_200": { "added6": { "absent_accuracy": 1.0, "count": 6, "counts": { "absent_correct": 6, "absent_total": 6, "exact": 3, "false_abstention": 0, "field_correct": 9, "fields": 12, "fn": 5, "fp": 3, "json_parse_ok": 6, "missed_abstention": 0, "natural_stop": 6, "nonrepetitive_nonempty": 6, "predicted_values": 22, "present_correct": 3, "present_total": 6, "schema_ok": 6, "token_limited": 0, "tp": 19, "unsupported_values": 3 }, "exact": 0.5, "f1": 0.8260869565217391, "false_abstention_rate": 0.0, "field_accuracy": 0.75, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.10507260775193572, "precision": 0.8636363636363636, "present_accuracy": 0.5, "recall": 0.7916666666666666, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.13636363636363635 }, "array3": { "absent_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 2, "absent_total": 2, "exact": 1, "false_abstention": 0, "field_correct": 3, "fields": 4, "fn": 1, "fp": 1, "json_parse_ok": 2, "missed_abstention": 0, "natural_stop": 2, "nonrepetitive_nonempty": 2, "predicted_values": 6, "present_correct": 1, "present_total": 2, "schema_ok": 2, "token_limited": 0, "tp": 5, "unsupported_values": 1 }, "exact": 0.5, "f1": 0.8333333333333334, "false_abstention_rate": 0.0, "field_accuracy": 0.75, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.0854416680522263, "precision": 0.8333333333333334, "present_accuracy": 0.5, "recall": 0.8333333333333334, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.16666666666666666 }, "array4": { "absent_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 2, "absent_total": 2, "exact": 1, "false_abstention": 0, "field_correct": 3, "fields": 4, "fn": 1, "fp": 1, "json_parse_ok": 2, "missed_abstention": 0, "natural_stop": 2, "nonrepetitive_nonempty": 2, "predicted_values": 8, "present_correct": 1, "present_total": 2, "schema_ok": 2, "token_limited": 0, "tp": 7, "unsupported_values": 1 }, "exact": 0.5, "f1": 0.875, "false_abstention_rate": 0.0, "field_accuracy": 0.75, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.1231730468571186, "precision": 0.875, "present_accuracy": 0.5, "recall": 0.875, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.125 }, "array5": { "absent_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 2, "absent_total": 2, "exact": 1, "false_abstention": 0, "field_correct": 3, "fields": 4, "fn": 3, "fp": 1, "json_parse_ok": 2, "missed_abstention": 0, "natural_stop": 2, "nonrepetitive_nonempty": 2, "predicted_values": 8, "present_correct": 1, "present_total": 2, "schema_ok": 2, "token_limited": 0, "tp": 7, "unsupported_values": 1 }, "exact": 0.5, "f1": 0.7777777777777777, "false_abstention_rate": 0.0, "field_accuracy": 0.75, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.10660310834646225, "precision": 0.875, "present_accuracy": 0.5, "recall": 0.7, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.125 }, "original54": { "absent_accuracy": 0.9056603773584906, "count": 54, "counts": { "absent_correct": 48, "absent_total": 53, "exact": 45, "false_abstention": 1, "field_correct": 126, "fields": 135, "fn": 4, "fp": 9, "json_parse_ok": 54, "missed_abstention": 5, "natural_stop": 54, "nonrepetitive_nonempty": 54, "predicted_values": 89, "present_correct": 78, "present_total": 82, "schema_ok": 54, "token_limited": 0, "tp": 80, "unsupported_values": 3 }, "exact": 0.8333333333333334, "f1": 0.9248554913294796, "false_abstention_rate": 0.012195121951219513, "field_accuracy": 0.9333333333333333, "json_parse_rate": 1.0, "missed_abstention_rate": 0.09433962264150944, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.038012701103407716, "precision": 0.898876404494382, "present_accuracy": 0.9512195121951219, "recall": 0.9523809523809523, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.033707865168539325 } }, "replay128_300": { "added6": { "absent_accuracy": 1.0, "count": 6, "counts": { "absent_correct": 6, "absent_total": 6, "exact": 3, "false_abstention": 0, "field_correct": 9, "fields": 12, "fn": 5, "fp": 3, "json_parse_ok": 6, "missed_abstention": 0, "natural_stop": 6, "nonrepetitive_nonempty": 6, "predicted_values": 22, "present_correct": 3, "present_total": 6, "schema_ok": 6, "token_limited": 0, "tp": 19, "unsupported_values": 3 }, "exact": 0.5, "f1": 0.8260869565217391, "false_abstention_rate": 0.0, "field_accuracy": 0.75, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.10285404759148757, "precision": 0.8636363636363636, "present_accuracy": 0.5, "recall": 0.7916666666666666, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.13636363636363635 }, "array3": { "absent_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 2, "absent_total": 2, "exact": 1, "false_abstention": 0, "field_correct": 3, "fields": 4, "fn": 1, "fp": 1, "json_parse_ok": 2, "missed_abstention": 0, "natural_stop": 2, "nonrepetitive_nonempty": 2, "predicted_values": 6, "present_correct": 1, "present_total": 2, "schema_ok": 2, "token_limited": 0, "tp": 5, "unsupported_values": 1 }, "exact": 0.5, "f1": 0.8333333333333334, "false_abstention_rate": 0.0, "field_accuracy": 0.75, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.08301506005227566, "precision": 0.8333333333333334, "present_accuracy": 0.5, "recall": 0.8333333333333334, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.16666666666666666 }, "array4": { "absent_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 2, "absent_total": 2, "exact": 1, "false_abstention": 0, "field_correct": 3, "fields": 4, "fn": 1, "fp": 1, "json_parse_ok": 2, "missed_abstention": 0, "natural_stop": 2, "nonrepetitive_nonempty": 2, "predicted_values": 8, "present_correct": 1, "present_total": 2, "schema_ok": 2, "token_limited": 0, "tp": 7, "unsupported_values": 1 }, "exact": 0.5, "f1": 0.875, "false_abstention_rate": 0.0, "field_accuracy": 0.75, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.1220800168812275, "precision": 0.875, "present_accuracy": 0.5, "recall": 0.875, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.125 }, "array5": { "absent_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 2, "absent_total": 2, "exact": 1, "false_abstention": 0, "field_correct": 3, "fields": 4, "fn": 3, "fp": 1, "json_parse_ok": 2, "missed_abstention": 0, "natural_stop": 2, "nonrepetitive_nonempty": 2, "predicted_values": 8, "present_correct": 1, "present_total": 2, "schema_ok": 2, "token_limited": 0, "tp": 7, "unsupported_values": 1 }, "exact": 0.5, "f1": 0.7777777777777777, "false_abstention_rate": 0.0, "field_accuracy": 0.75, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.10346706584095955, "precision": 0.875, "present_accuracy": 0.5, "recall": 0.7, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.125 }, "original54": { "absent_accuracy": 0.9245283018867925, "count": 54, "counts": { "absent_correct": 49, "absent_total": 53, "exact": 46, "false_abstention": 2, "field_correct": 127, "fields": 135, "fn": 4, "fp": 7, "json_parse_ok": 54, "missed_abstention": 4, "natural_stop": 54, "nonrepetitive_nonempty": 54, "predicted_values": 87, "present_correct": 78, "present_total": 82, "schema_ok": 54, "token_limited": 0, "tp": 80, "unsupported_values": 3 }, "exact": 0.8518518518518519, "f1": 0.935672514619883, "false_abstention_rate": 0.024390243902439025, "field_accuracy": 0.9407407407407408, "json_parse_rate": 1.0, "missed_abstention_rate": 0.07547169811320754, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.040062436762296046, "precision": 0.9195402298850575, "present_accuracy": 0.9512195121951219, "recall": 0.9523809523809523, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.034482758620689655 } } } } }, "protocol": { "constrained_decoding": false, "json_assembly": false, "max_tokens": 256, "one_generation_per_request": true, "output_repair": false, "prompt_truncation": false, "repetition_penalty": 1, "temperature": 0 }, "scope": "Previously observed development, SNIPS-derived held-out and playground research panels, not fresh blind external benchmarks. Source text, schema and fixed instruction only; gold values excluded. One raw generation; no JSON assembly, repair or constrained decoding.", "selection": "Owner-selected aggregate improvement; strict zero-regression gate did not pass.", "training_prefix": { "records": 19200, "serialized_positions": 6153194, "supervised_assistant_positions": 633561, "components": { "original_json": 13640, "new_prose": 4871, "cardinality_replay": 689 } } }