diff --git "a/JSONLITE_RESULTS.json" "b/JSONLITE_RESULTS.json" --- "a/JSONLITE_RESULTS.json" +++ "b/JSONLITE_RESULTS.json" @@ -1,3955 +1,3177 @@ { "model": "OpenSML-JSONLite-175M-Instruct", "checkpoint": "main", - "selected_stage": "additional extraction SFT", - "step": 64, - "model_sha256": "655fb5cfba009163cd8a7f7fe7b170b998bb452c217b1b473cfe10b33de0ccd1", + "selected_stage": "joint JSON extraction SFT", + "step": 300, + "model_sha256": "781597baeb9b51d535fc27cd6f941dbc202e01d62c8d963587a3f3d565128560", "output_mode": "native-json", "development": { "metrics": { "all": { - "absent_accuracy": 0.9051094890510949, - "count": 404, + "absent_accuracy": 0.9211356466876972, + "count": 240, "counts": { - "absent_correct": 372, - "absent_total": 411, - "exact": 254, - "false_abstention": 70, - "field_correct": 769, - "fields": 980, - "fn": 217, - "fp": 96, - "json_parse_ok": 401, - "missed_abstention": 39, - "natural_stop": 403, - "nonrepetitive_nonempty": 403, - "predicted_values": 591, - "present_correct": 397, - "present_total": 569, - "schema_ok": 371, - "token_limited": 1, - "tp": 495, - "unsupported_values": 16 - }, - "exact": 0.6287128712871287, - "f1": 0.7597851112816577, - "false_abstention_rate": 0.12302284710017575, - "field_accuracy": 0.7846938775510204, - "json_parse_rate": 0.9925742574257426, - "missed_abstention_rate": 0.0948905109489051, - "natural_stop_rate": 0.9975247524752475, - "nonrepetitive_nonempty_rate": 0.9975247524752475, - "output_nll": 0.1001320060895523, - "precision": 0.8375634517766497, - "present_accuracy": 0.6977152899824253, - "recall": 0.6952247191011236, - "schema_valid_rate": 0.9183168316831684, - "token_limited_rate": 0.0024752475247524753, - "unsupported_value_rate": 0.02707275803722504 - }, - "cardinality/0": { - "absent_accuracy": 0.8333333333333334, - "count": 12, - "counts": { - "absent_correct": 10, - "absent_total": 12, - "exact": 10, - "false_abstention": 0, - "field_correct": 21, - "fields": 24, - "fn": 1, - "fp": 1, - "json_parse_ok": 12, - "missed_abstention": 2, - "natural_stop": 12, - "nonrepetitive_nonempty": 12, - "predicted_values": 12, - "present_correct": 11, - "present_total": 12, - "schema_ok": 10, - "token_limited": 0, - "tp": 11, - "unsupported_values": 0 - }, - "exact": 0.8333333333333334, - "f1": 0.9166666666666666, - "false_abstention_rate": 0.0, - "field_accuracy": 0.875, + "absent_correct": 292, + "absent_total": 317, + "exact": 178, + "false_abstention": 25, + "field_correct": 632, + "fields": 714, + "fn": 65, + "fp": 47, + "json_parse_ok": 240, + "missed_abstention": 25, + "natural_stop": 240, + "nonrepetitive_nonempty": 240, + "predicted_values": 504, + "present_correct": 340, + "present_total": 397, + "schema_ok": 239, + "token_limited": 0, + "tp": 457, + "unsupported_values": 6 + }, + "exact": 0.7416666666666667, + "f1": 0.8908382066276802, + "false_abstention_rate": 0.06297229219143577, + "field_accuracy": 0.8851540616246498, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.16666666666666666, + "missed_abstention_rate": 0.07886435331230283, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.03547299994776646, - "precision": 0.9166666666666666, - "present_accuracy": 0.9166666666666666, - "recall": 0.9166666666666666, - "schema_valid_rate": 0.8333333333333334, + "output_nll": 0.04828121321412861, + "precision": 0.9067460317460317, + "present_accuracy": 0.8564231738035264, + "recall": 0.8754789272030651, + "schema_valid_rate": 0.9958333333333333, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.0 + "unsupported_value_rate": 0.011904761904761904 }, - "cardinality/1": { - "absent_accuracy": 0.9166666666666666, - "count": 12, + "domain/AddToPlaylist": { + "absent_accuracy": 0.8974358974358975, + "count": 28, "counts": { - "absent_correct": 11, - "absent_total": 12, - "exact": 8, - "false_abstention": 0, - "field_correct": 20, - "fields": 24, - "fn": 3, - "fp": 4, - "json_parse_ok": 12, - "missed_abstention": 1, - "natural_stop": 12, - "nonrepetitive_nonempty": 12, - "predicted_values": 13, - "present_correct": 9, - "present_total": 12, - "schema_ok": 9, + "absent_correct": 35, + "absent_total": 39, + "exact": 19, + "false_abstention": 1, + "field_correct": 81, + "fields": 92, + "fn": 7, + "fp": 10, + "json_parse_ok": 28, + "missed_abstention": 4, + "natural_stop": 28, + "nonrepetitive_nonempty": 28, + "predicted_values": 56, + "present_correct": 46, + "present_total": 53, + "schema_ok": 28, "token_limited": 0, - "tp": 9, - "unsupported_values": 0 + "tp": 46, + "unsupported_values": 2 }, - "exact": 0.6666666666666666, - "f1": 0.7199999999999999, - "false_abstention_rate": 0.0, - "field_accuracy": 0.8333333333333334, + "exact": 0.6785714285714286, + "f1": 0.8440366972477065, + "false_abstention_rate": 0.018867924528301886, + "field_accuracy": 0.8804347826086957, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.08333333333333333, + "missed_abstention_rate": 0.10256410256410256, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.056408251325289406, - "precision": 0.6923076923076923, - "present_accuracy": 0.75, - "recall": 0.75, - "schema_valid_rate": 0.75, + "output_nll": 0.0638373876364702, + "precision": 0.8214285714285714, + "present_accuracy": 0.8679245283018868, + "recall": 0.8679245283018868, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.0 + "unsupported_value_rate": 0.03571428571428571 }, - "cardinality/2": { - "absent_accuracy": 0.8333333333333334, - "count": 12, + "domain/BookRestaurant": { + "absent_accuracy": 0.9454545454545454, + "count": 28, "counts": { - "absent_correct": 10, - "absent_total": 12, - "exact": 9, - "false_abstention": 0, - "field_correct": 19, - "fields": 24, - "fn": 6, - "fp": 6, - "json_parse_ok": 12, - "missed_abstention": 2, - "natural_stop": 12, - "nonrepetitive_nonempty": 12, - "predicted_values": 24, - "present_correct": 9, - "present_total": 12, - "schema_ok": 9, + "absent_correct": 52, + "absent_total": 55, + "exact": 22, + "false_abstention": 1, + "field_correct": 99, + "fields": 105, + "fn": 3, + "fp": 5, + "json_parse_ok": 28, + "missed_abstention": 3, + "natural_stop": 28, + "nonrepetitive_nonempty": 28, + "predicted_values": 60, + "present_correct": 47, + "present_total": 50, + "schema_ok": 28, "token_limited": 0, - "tp": 18, - "unsupported_values": 0 + "tp": 55, + "unsupported_values": 1 }, - "exact": 0.75, - "f1": 0.75, - "false_abstention_rate": 0.0, - "field_accuracy": 0.7916666666666666, + "exact": 0.7857142857142857, + "f1": 0.9322033898305084, + "false_abstention_rate": 0.02, + "field_accuracy": 0.9428571428571428, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.16666666666666666, + "missed_abstention_rate": 0.05454545454545454, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.06341762685527404, - "precision": 0.75, - "present_accuracy": 0.75, - "recall": 0.75, - "schema_valid_rate": 0.75, + "output_nll": 0.01643673108524776, + "precision": 0.9166666666666666, + "present_accuracy": 0.94, + "recall": 0.9482758620689655, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.0 + "unsupported_value_rate": 0.016666666666666666 }, - "cardinality/3": { - "absent_accuracy": 1.0, - "count": 12, + "domain/GetWeather": { + "absent_accuracy": 0.9090909090909091, + "count": 28, "counts": { - "absent_correct": 12, - "absent_total": 12, - "exact": 5, - "false_abstention": 0, - "field_correct": 17, - "fields": 24, - "fn": 14, - "fp": 14, - "json_parse_ok": 12, - "missed_abstention": 0, - "natural_stop": 12, - "nonrepetitive_nonempty": 12, - "predicted_values": 36, - "present_correct": 5, - "present_total": 12, - "schema_ok": 8, + "absent_correct": 30, + "absent_total": 33, + "exact": 20, + "false_abstention": 5, + "field_correct": 64, + "fields": 73, + "fn": 6, + "fp": 3, + "json_parse_ok": 28, + "missed_abstention": 3, + "natural_stop": 28, + "nonrepetitive_nonempty": 28, + "predicted_values": 59, + "present_correct": 34, + "present_total": 40, + "schema_ok": 28, "token_limited": 0, - "tp": 22, - "unsupported_values": 5 + "tp": 56, + "unsupported_values": 0 }, - "exact": 0.4166666666666667, - "f1": 0.6111111111111112, - "false_abstention_rate": 0.0, - "field_accuracy": 0.7083333333333334, + "exact": 0.7142857142857143, + "f1": 0.9256198347107438, + "false_abstention_rate": 0.125, + "field_accuracy": 0.8767123287671232, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.0, + "missed_abstention_rate": 0.09090909090909091, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.08486803613292675, - "precision": 0.6111111111111112, - "present_accuracy": 0.4166666666666667, - "recall": 0.6111111111111112, - "schema_valid_rate": 0.6666666666666666, + "output_nll": 0.037961417684398056, + "precision": 0.9491525423728814, + "present_accuracy": 0.85, + "recall": 0.9032258064516129, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.1388888888888889 - }, - "cardinality_contrasts": { - "all_four_correct": 2, - "both_multiple_correct": 4, - "groups": 12 + "unsupported_value_rate": 0.0 }, - "category/multi_artist": { - "absent_accuracy": 0.875, - "count": 32, + "domain/PlayMusic": { + "absent_accuracy": 0.8, + "count": 28, "counts": { "absent_correct": 28, - "absent_total": 32, - "exact": 13, - "false_abstention": 7, - "field_correct": 46, - "fields": 76, - "fn": 35, - "fp": 12, - "json_parse_ok": 31, - "missed_abstention": 4, - "natural_stop": 31, - "nonrepetitive_nonempty": 31, - "predicted_values": 63, - "present_correct": 18, - "present_total": 44, - "schema_ok": 27, - "token_limited": 1, - "tp": 51, - "unsupported_values": 7 - }, - "exact": 0.40625, - "f1": 0.6845637583892619, - "false_abstention_rate": 0.1590909090909091, - "field_accuracy": 0.6052631578947368, - "json_parse_rate": 0.96875, - "missed_abstention_rate": 0.125, - "natural_stop_rate": 0.96875, - "nonrepetitive_nonempty_rate": 0.96875, - "output_nll": 0.15398504153563408, - "precision": 0.8095238095238095, - "present_accuracy": 0.4090909090909091, - "recall": 0.5930232558139535, - "schema_valid_rate": 0.84375, - "token_limited_rate": 0.03125, - "unsupported_value_rate": 0.1111111111111111 - }, - "category/multi_other": { - "absent_accuracy": 0.8095238095238095, - "count": 84, - "counts": { - "absent_correct": 68, - "absent_total": 84, - "exact": 17, - "false_abstention": 26, - "field_correct": 112, - "fields": 216, - "fn": 124, - "fp": 47, - "json_parse_ok": 82, - "missed_abstention": 16, - "natural_stop": 84, - "nonrepetitive_nonempty": 84, - "predicted_values": 155, - "present_correct": 44, - "present_total": 132, - "schema_ok": 61, - "token_limited": 0, - "tp": 108, - "unsupported_values": 4 - }, - "exact": 0.20238095238095238, - "f1": 0.5581395348837209, - "false_abstention_rate": 0.19696969696969696, - "field_accuracy": 0.5185185185185185, - "json_parse_rate": 0.9761904761904762, - "missed_abstention_rate": 0.19047619047619047, - "natural_stop_rate": 1.0, - "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.25150625878346283, - "precision": 0.6967741935483871, - "present_accuracy": 0.3333333333333333, - "recall": 0.46551724137931033, - "schema_valid_rate": 0.7261904761904762, - "token_limited_rate": 0.0, - "unsupported_value_rate": 0.025806451612903226 - }, - "category/retention": { - "absent_accuracy": 0.9490196078431372, - "count": 252, - "counts": { - "absent_correct": 242, - "absent_total": 255, - "exact": 206, - "false_abstention": 24, - "field_correct": 553, - "fields": 604, - "fn": 38, - "fp": 26, - "json_parse_ok": 252, - "missed_abstention": 13, - "natural_stop": 252, - "nonrepetitive_nonempty": 252, - "predicted_values": 338, - "present_correct": 311, - "present_total": 349, - "schema_ok": 252, + "absent_total": 35, + "exact": 19, + "false_abstention": 5, + "field_correct": 62, + "fields": 74, + "fn": 5, + "fp": 7, + "json_parse_ok": 28, + "missed_abstention": 7, + "natural_stop": 28, + "nonrepetitive_nonempty": 28, + "predicted_values": 56, + "present_correct": 34, + "present_total": 39, + "schema_ok": 28, "token_limited": 0, - "tp": 312, - "unsupported_values": 4 + "tp": 49, + "unsupported_values": 0 }, - "exact": 0.8174603174603174, - "f1": 0.9069767441860466, - "false_abstention_rate": 0.06876790830945559, - "field_accuracy": 0.9155629139072847, + "exact": 0.6785714285714286, + "f1": 0.8909090909090909, + "false_abstention_rate": 0.1282051282051282, + "field_accuracy": 0.8378378378378378, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.050980392156862744, + "missed_abstention_rate": 0.2, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.0388590097979417, - "precision": 0.9230769230769231, - "present_accuracy": 0.8911174785100286, - "recall": 0.8914285714285715, + "output_nll": 0.028358824200820112, + "precision": 0.875, + "present_accuracy": 0.8717948717948718, + "recall": 0.9074074074074074, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.011834319526627219 + "unsupported_value_rate": 0.0 }, - "category/single_value": { - "absent_accuracy": 0.85, - "count": 36, + "domain/RateBook": { + "absent_accuracy": 0.9761904761904762, + "count": 28, "counts": { - "absent_correct": 34, - "absent_total": 40, - "exact": 18, - "false_abstention": 13, - "field_correct": 58, - "fields": 84, - "fn": 20, - "fp": 11, - "json_parse_ok": 36, - "missed_abstention": 6, - "natural_stop": 36, - "nonrepetitive_nonempty": 36, - "predicted_values": 35, - "present_correct": 24, - "present_total": 44, - "schema_ok": 31, + "absent_correct": 41, + "absent_total": 42, + "exact": 23, + "false_abstention": 2, + "field_correct": 104, + "fields": 110, + "fn": 5, + "fp": 4, + "json_parse_ok": 28, + "missed_abstention": 1, + "natural_stop": 28, + "nonrepetitive_nonempty": 28, + "predicted_values": 67, + "present_correct": 63, + "present_total": 68, + "schema_ok": 28, "token_limited": 0, - "tp": 24, + "tp": 63, "unsupported_values": 1 }, - "exact": 0.5, - "f1": 0.6075949367088607, - "false_abstention_rate": 0.29545454545454547, - "field_accuracy": 0.6904761904761905, + "exact": 0.8214285714285714, + "f1": 0.9333333333333335, + "false_abstention_rate": 0.029411764705882353, + "field_accuracy": 0.9454545454545454, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.15, + "missed_abstention_rate": 0.023809523809523808, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.12796702567074034, - "precision": 0.6857142857142857, - "present_accuracy": 0.5454545454545454, - "recall": 0.5454545454545454, - "schema_valid_rate": 0.8611111111111112, + "output_nll": 0.010672787178918952, + "precision": 0.9402985074626866, + "present_accuracy": 0.9264705882352942, + "recall": 0.9264705882352942, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.02857142857142857 + "unsupported_value_rate": 0.014925373134328358 }, - "domain/cardinality": { - "absent_accuracy": 0.8958333333333334, - "count": 48, + "domain/SearchCreativeWork": { + "absent_accuracy": 1.0, + "count": 28, "counts": { - "absent_correct": 43, - "absent_total": 48, - "exact": 32, + "absent_correct": 9, + "absent_total": 9, + "exact": 26, "false_abstention": 0, - "field_correct": 77, - "fields": 96, - "fn": 24, - "fp": 25, - "json_parse_ok": 48, - "missed_abstention": 5, - "natural_stop": 48, - "nonrepetitive_nonempty": 48, - "predicted_values": 85, + "field_correct": 43, + "fields": 45, + "fn": 2, + "fp": 2, + "json_parse_ok": 28, + "missed_abstention": 0, + "natural_stop": 28, + "nonrepetitive_nonempty": 28, + "predicted_values": 36, "present_correct": 34, - "present_total": 48, - "schema_ok": 36, + "present_total": 36, + "schema_ok": 28, "token_limited": 0, - "tp": 60, - "unsupported_values": 5 + "tp": 34, + "unsupported_values": 0 }, - "exact": 0.6666666666666666, - "f1": 0.7100591715976332, + "exact": 0.9285714285714286, + "f1": 0.9444444444444444, "false_abstention_rate": 0.0, - "field_accuracy": 0.8020833333333334, + "field_accuracy": 0.9555555555555556, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.10416666666666667, + "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.060041728565314166, - "precision": 0.7058823529411765, - "present_accuracy": 0.7083333333333334, - "recall": 0.7142857142857143, - "schema_valid_rate": 0.75, + "output_nll": 0.011042242548553207, + "precision": 0.9444444444444444, + "present_accuracy": 0.9444444444444444, + "recall": 0.9444444444444444, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.058823529411764705 + "unsupported_value_rate": 0.0 }, - "domain/mixsnips": { - "absent_accuracy": 0.71875, - "count": 64, + "domain/SearchScreeningEvent": { + "absent_accuracy": 0.9454545454545454, + "count": 28, "counts": { - "absent_correct": 46, - "absent_total": 64, - "exact": 0, - "false_abstention": 31, - "field_correct": 82, - "fields": 192, - "fn": 112, - "fp": 34, - "json_parse_ok": 62, - "missed_abstention": 18, - "natural_stop": 64, - "nonrepetitive_nonempty": 64, - "predicted_values": 108, - "present_correct": 36, - "present_total": 128, - "schema_ok": 46, + "absent_correct": 52, + "absent_total": 55, + "exact": 23, + "false_abstention": 1, + "field_correct": 100, + "fields": 108, + "fn": 8, + "fp": 6, + "json_parse_ok": 28, + "missed_abstention": 3, + "natural_stop": 28, + "nonrepetitive_nonempty": 28, + "predicted_values": 64, + "present_correct": 48, + "present_total": 53, + "schema_ok": 28, "token_limited": 0, - "tp": 74, - "unsupported_values": 2 + "tp": 58, + "unsupported_values": 0 }, - "exact": 0.0, - "f1": 0.5034013605442177, - "false_abstention_rate": 0.2421875, - "field_accuracy": 0.4270833333333333, - "json_parse_rate": 0.96875, - "missed_abstention_rate": 0.28125, + "exact": 0.8214285714285714, + "f1": 0.8923076923076922, + "false_abstention_rate": 0.018867924528301886, + "field_accuracy": 0.9259259259259259, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.05454545454545454, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.3572382861166261, - "precision": 0.6851851851851852, - "present_accuracy": 0.28125, - "recall": 0.3978494623655914, - "schema_valid_rate": 0.71875, + "output_nll": 0.018851709860817727, + "precision": 0.90625, + "present_accuracy": 0.9056603773584906, + "recall": 0.8787878787878788, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.018518518518518517 + "unsupported_value_rate": 0.0 }, - "domain/probe/single_request": { + "domain/cardinality": { "absent_accuracy": 1.0, "count": 8, "counts": { "absent_correct": 8, "absent_total": 8, - "exact": 7, + "exact": 8, "false_abstention": 0, - "field_correct": 15, + "field_correct": 16, "fields": 16, "fn": 0, - "fp": 1, + "fp": 0, "json_parse_ok": 8, "missed_abstention": 0, "natural_stop": 8, "nonrepetitive_nonempty": 8, - "predicted_values": 17, - "present_correct": 7, + "predicted_values": 12, + "present_correct": 8, "present_total": 8, "schema_ok": 8, "token_limited": 0, - "tp": 16, - "unsupported_values": 1 + "tp": 12, + "unsupported_values": 0 }, - "exact": 0.875, - "f1": 0.9696969696969697, + "exact": 1.0, + "f1": 1.0, "false_abstention_rate": 0.0, - "field_accuracy": 0.9375, + "field_accuracy": 1.0, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.04306051167077385, - "precision": 0.9411764705882353, - "present_accuracy": 0.875, + "output_nll": 0.012686168280197307, + "precision": 1.0, + "present_accuracy": 1.0, "recall": 1.0, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.058823529411764705 + "unsupported_value_rate": 0.0 }, - "domain/retention/AddToPlaylist": { - "absent_accuracy": 0.926829268292683, - "count": 36, + "domain/mixsnips": { + "absent_accuracy": 1.0, + "count": 8, "counts": { - "absent_correct": 38, - "absent_total": 41, - "exact": 21, - "false_abstention": 8, - "field_correct": 74, - "fields": 91, + "absent_correct": 8, + "absent_total": 8, + "exact": 1, + "false_abstention": 4, + "field_correct": 12, + "fields": 24, "fn": 14, - "fp": 9, - "json_parse_ok": 36, - "missed_abstention": 3, - "natural_stop": 36, - "nonrepetitive_nonempty": 36, - "predicted_values": 45, - "present_correct": 36, - "present_total": 50, - "schema_ok": 36, + "fp": 4, + "json_parse_ok": 8, + "missed_abstention": 0, + "natural_stop": 8, + "nonrepetitive_nonempty": 8, + "predicted_values": 12, + "present_correct": 4, + "present_total": 16, + "schema_ok": 7, "token_limited": 0, - "tp": 36, - "unsupported_values": 3 + "tp": 8, + "unsupported_values": 0 }, - "exact": 0.5833333333333334, - "f1": 0.7578947368421052, - "false_abstention_rate": 0.16, - "field_accuracy": 0.8131868131868132, + "exact": 0.125, + "f1": 0.4705882352941177, + "false_abstention_rate": 0.25, + "field_accuracy": 0.5, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.07317073170731707, + "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.0822503470375927, - "precision": 0.8, - "present_accuracy": 0.72, - "recall": 0.72, - "schema_valid_rate": 1.0, + "output_nll": 0.378448877716437, + "precision": 0.6666666666666666, + "present_accuracy": 0.25, + "recall": 0.36363636363636365, + "schema_valid_rate": 0.875, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.06666666666666667 + "unsupported_value_rate": 0.0 }, - "domain/retention/BookRestaurant": { - "absent_accuracy": 0.8604651162790697, - "count": 36, + "domain/observed_lists": { + "absent_accuracy": 1.0, + "count": 12, "counts": { - "absent_correct": 37, - "absent_total": 43, - "exact": 24, - "false_abstention": 4, - "field_correct": 82, - "fields": 95, - "fn": 7, - "fp": 9, - "json_parse_ok": 36, - "missed_abstention": 6, - "natural_stop": 36, - "nonrepetitive_nonempty": 36, - "predicted_values": 54, - "present_correct": 45, - "present_total": 52, - "schema_ok": 36, - "token_limited": 0, - "tp": 45, - "unsupported_values": 1 - }, - "exact": 0.6666666666666666, - "f1": 0.8490566037735849, - "false_abstention_rate": 0.07692307692307693, - "field_accuracy": 0.8631578947368421, - "json_parse_rate": 1.0, - "missed_abstention_rate": 0.13953488372093023, - "natural_stop_rate": 1.0, - "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.05120245015778993, - "precision": 0.8333333333333334, - "present_accuracy": 0.8653846153846154, - "recall": 0.8653846153846154, - "schema_valid_rate": 1.0, - "token_limited_rate": 0.0, - "unsupported_value_rate": 0.018518518518518517 - }, - "domain/retention/GetWeather": { - "absent_accuracy": 0.925, - "count": 36, - "counts": { - "absent_correct": 37, - "absent_total": 40, - "exact": 28, - "false_abstention": 5, - "field_correct": 85, - "fields": 94, - "fn": 6, - "fp": 3, - "json_parse_ok": 36, - "missed_abstention": 3, - "natural_stop": 36, - "nonrepetitive_nonempty": 36, - "predicted_values": 52, - "present_correct": 48, - "present_total": 54, - "schema_ok": 36, + "absent_correct": 12, + "absent_total": 12, + "exact": 9, + "false_abstention": 0, + "field_correct": 21, + "fields": 24, + "fn": 4, + "fp": 0, + "json_parse_ok": 12, + "missed_abstention": 0, + "natural_stop": 12, + "nonrepetitive_nonempty": 12, + "predicted_values": 49, + "present_correct": 9, + "present_total": 12, + "schema_ok": 12, "token_limited": 0, "tp": 49, "unsupported_values": 0 }, - "exact": 0.7777777777777778, - "f1": 0.9158878504672897, - "false_abstention_rate": 0.09259259259259259, - "field_accuracy": 0.9042553191489362, - "json_parse_rate": 1.0, - "missed_abstention_rate": 0.075, - "natural_stop_rate": 1.0, - "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.04159018842619844, - "precision": 0.9423076923076923, - "present_accuracy": 0.8888888888888888, - "recall": 0.8909090909090909, - "schema_valid_rate": 1.0, - "token_limited_rate": 0.0, - "unsupported_value_rate": 0.0 - }, - "domain/retention/PlayMusic": { - "absent_accuracy": 0.9767441860465116, - "count": 36, - "counts": { - "absent_correct": 42, - "absent_total": 43, - "exact": 30, - "false_abstention": 6, - "field_correct": 81, - "fields": 88, - "fn": 6, - "fp": 1, - "json_parse_ok": 36, - "missed_abstention": 1, - "natural_stop": 36, - "nonrepetitive_nonempty": 36, - "predicted_values": 40, - "present_correct": 39, - "present_total": 45, - "schema_ok": 36, - "token_limited": 0, - "tp": 39, - "unsupported_values": 0 - }, - "exact": 0.8333333333333334, - "f1": 0.9176470588235294, - "false_abstention_rate": 0.13333333333333333, - "field_accuracy": 0.9204545454545454, + "exact": 0.75, + "f1": 0.9607843137254902, + "false_abstention_rate": 0.0, + "field_accuracy": 0.875, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.023255813953488372, + "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.042106466590970136, - "precision": 0.975, - "present_accuracy": 0.8666666666666667, - "recall": 0.8666666666666667, + "output_nll": 0.04184045293368399, + "precision": 1.0, + "present_accuracy": 0.75, + "recall": 0.9245283018867925, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, - "domain/retention/RateBook": { + "domain/playground": { "absent_accuracy": 1.0, - "count": 36, + "count": 4, "counts": { - "absent_correct": 43, - "absent_total": 43, - "exact": 35, + "absent_correct": 2, + "absent_total": 2, + "exact": 3, "false_abstention": 0, - "field_correct": 95, - "fields": 96, - "fn": 1, + "field_correct": 8, + "fields": 9, + "fn": 3, "fp": 1, - "json_parse_ok": 36, + "json_parse_ok": 4, "missed_abstention": 0, - "natural_stop": 36, - "nonrepetitive_nonempty": 36, - "predicted_values": 53, - "present_correct": 52, - "present_total": 53, - "schema_ok": 36, - "token_limited": 0, - "tp": 52, - "unsupported_values": 0 + "natural_stop": 4, + "nonrepetitive_nonempty": 4, + "predicted_values": 9, + "present_correct": 6, + "present_total": 7, + "schema_ok": 4, + "token_limited": 0, + "tp": 8, + "unsupported_values": 1 }, - "exact": 0.9722222222222222, - "f1": 0.9811320754716981, + "exact": 0.75, + "f1": 0.7999999999999999, "false_abstention_rate": 0.0, - "field_accuracy": 0.9895833333333334, + "field_accuracy": 0.8888888888888888, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.014572561123511858, - "precision": 0.9811320754716981, - "present_accuracy": 0.9811320754716981, - "recall": 0.9811320754716981, + "output_nll": 0.03125538246240467, + "precision": 0.8888888888888888, + "present_accuracy": 0.8571428571428571, + "recall": 0.7272727272727273, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.0 + "unsupported_value_rate": 0.1111111111111111 }, - "domain/retention/SearchCreativeWork": { - "absent_accuracy": 1.0, - "count": 36, + "domain/roles": { + "absent_accuracy": 0.8181818181818182, + "count": 8, "counts": { - "absent_correct": 4, - "absent_total": 4, - "exact": 33, - "false_abstention": 1, - "field_correct": 44, - "fields": 47, - "fn": 3, - "fp": 2, - "json_parse_ok": 36, - "missed_abstention": 0, - "natural_stop": 36, - "nonrepetitive_nonempty": 36, - "predicted_values": 42, - "present_correct": 40, - "present_total": 43, - "schema_ok": 36, + "absent_correct": 9, + "absent_total": 11, + "exact": 5, + "false_abstention": 2, + "field_correct": 16, + "fields": 22, + "fn": 4, + "fp": 3, + "json_parse_ok": 8, + "missed_abstention": 2, + "natural_stop": 8, + "nonrepetitive_nonempty": 8, + "predicted_values": 22, + "present_correct": 7, + "present_total": 11, + "schema_ok": 8, "token_limited": 0, - "tp": 40, - "unsupported_values": 0 + "tp": 19, + "unsupported_values": 1 }, - "exact": 0.9166666666666666, - "f1": 0.9411764705882352, - "false_abstention_rate": 0.023255813953488372, - "field_accuracy": 0.9361702127659575, + "exact": 0.625, + "f1": 0.8444444444444444, + "false_abstention_rate": 0.18181818181818182, + "field_accuracy": 0.7272727272727273, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.0, + "missed_abstention_rate": 0.18181818181818182, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.017326170247138686, - "precision": 0.9523809523809523, - "present_accuracy": 0.9302325581395349, - "recall": 0.9302325581395349, + "output_nll": 0.21559197976603173, + "precision": 0.8636363636363636, + "present_accuracy": 0.6363636363636364, + "recall": 0.8260869565217391, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.0 + "unsupported_value_rate": 0.045454545454545456 }, - "domain/retention/SearchScreeningEvent": { - "absent_accuracy": 1.0, - "count": 36, + "domain/single_lists": { + "absent_accuracy": 0.75, + "count": 4, "counts": { - "absent_correct": 41, - "absent_total": 41, - "exact": 35, - "false_abstention": 0, - "field_correct": 92, - "fields": 93, - "fn": 1, - "fp": 1, - "json_parse_ok": 36, - "missed_abstention": 0, - "natural_stop": 36, - "nonrepetitive_nonempty": 36, - "predicted_values": 52, - "present_correct": 51, - "present_total": 52, - "schema_ok": 36, + "absent_correct": 6, + "absent_total": 8, + "exact": 0, + "false_abstention": 4, + "field_correct": 6, + "fields": 12, + "fn": 4, + "fp": 2, + "json_parse_ok": 4, + "missed_abstention": 2, + "natural_stop": 4, + "nonrepetitive_nonempty": 4, + "predicted_values": 2, + "present_correct": 0, + "present_total": 4, + "schema_ok": 4, "token_limited": 0, - "tp": 51, + "tp": 0, "unsupported_values": 0 }, - "exact": 0.9722222222222222, - "f1": 0.9807692307692307, - "false_abstention_rate": 0.0, - "field_accuracy": 0.989247311827957, + "exact": 0.0, + "f1": 0.0, + "false_abstention_rate": 1.0, + "field_accuracy": 0.5, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.0, + "missed_abstention_rate": 0.25, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.022964885002390172, - "precision": 0.9807692307692307, - "present_accuracy": 0.9807692307692307, - "recall": 0.9807692307692307, + "output_nll": 0.21651429869234562, + "precision": 0.0, + "present_accuracy": 0.0, + "recall": 0.0, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, - "domain/single_lists": { - "absent_accuracy": 0.9166666666666666, - "count": 32, - "counts": { - "absent_correct": 33, - "absent_total": 36, - "exact": 9, - "false_abstention": 15, - "field_correct": 42, - "fields": 72, - "fn": 43, - "fp": 10, - "json_parse_ok": 31, - "missed_abstention": 3, - "natural_stop": 31, - "nonrepetitive_nonempty": 31, - "predicted_values": 43, - "present_correct": 9, - "present_total": 36, - "schema_ok": 29, - "token_limited": 1, - "tp": 33, - "unsupported_values": 4 - }, - "exact": 0.28125, - "f1": 0.5546218487394958, - "false_abstention_rate": 0.4166666666666667, - "field_accuracy": 0.5833333333333334, - "json_parse_rate": 0.96875, - "missed_abstention_rate": 0.08333333333333333, - "natural_stop_rate": 0.96875, - "nonrepetitive_nonempty_rate": 0.96875, - "output_nll": 0.14284758172289003, - "precision": 0.7674418604651163, - "present_accuracy": 0.25, - "recall": 0.4342105263157895, - "schema_valid_rate": 0.90625, - "token_limited_rate": 0.03125, - "unsupported_value_rate": 0.09302325581395349 - }, - "macro_exact": 0.6859217171717172, - "multi_value_fields": { - "accuracy": 0.2459016393442623, - "artist_accuracy": 0.3611111111111111, - "artist_correct": 13, - "artist_fields": 36, - "correct": 30, - "fields": 122 - }, + "macro_exact": 0.6675824175824177, "paired_requests": { - "accuracy": 0.73, - "both_correct": 73, - "count": 100 + "accuracy": 0.7045454545454546, + "both_correct": 31, + "count": 44 }, - "playground": { - "absent_accuracy": 0.9245283018867925, - "count": 54, + "panel/legacy": { + "absent_accuracy": 0.9076923076923077, + "count": 128, "counts": { - "absent_correct": 49, - "absent_total": 53, - "exact": 44, - "false_abstention": 3, - "field_correct": 124, - "fields": 135, - "fn": 7, - "fp": 9, - "json_parse_ok": 54, - "missed_abstention": 4, - "natural_stop": 54, - "nonrepetitive_nonempty": 54, - "predicted_values": 86, - "present_correct": 75, - "present_total": 82, - "schema_ok": 54, - "token_limited": 0, - "tp": 77, - "unsupported_values": 3 + "absent_correct": 118, + "absent_total": 130, + "exact": 96, + "false_abstention": 11, + "field_correct": 267, + "fields": 310, + "fn": 36, + "fp": 24, + "json_parse_ok": 128, + "missed_abstention": 12, + "natural_stop": 128, + "nonrepetitive_nonempty": 128, + "predicted_values": 235, + "present_correct": 149, + "present_total": 180, + "schema_ok": 127, + "token_limited": 0, + "tp": 211, + "unsupported_values": 4 }, - "exact": 0.8148148148148148, - "f1": 0.9058823529411765, - "false_abstention_rate": 0.036585365853658534, - "field_accuracy": 0.9185185185185185, + "exact": 0.75, + "f1": 0.8755186721991701, + "false_abstention_rate": 0.06111111111111111, + "field_accuracy": 0.8612903225806452, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.07547169811320754, + "missed_abstention_rate": 0.09230769230769231, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.05903947115142795, - "precision": 0.8953488372093024, - "present_accuracy": 0.9146341463414634, - "recall": 0.9166666666666666, - "schema_valid_rate": 1.0, + "output_nll": 0.06681022582381502, + "precision": 0.8978723404255319, + "present_accuracy": 0.8277777777777777, + "recall": 0.854251012145749, + "schema_valid_rate": 0.9921875, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.03488372093023256 + "unsupported_value_rate": 0.01702127659574468 }, - "retention/variant/alias": { - "absent_accuracy": 0.96875, - "count": 28, + "panel/new_prose": { + "absent_accuracy": 0.93048128342246, + "count": 112, "counts": { - "absent_correct": 31, - "absent_total": 32, - "exact": 24, - "false_abstention": 3, - "field_correct": 64, - "fields": 68, - "fn": 3, - "fp": 1, - "json_parse_ok": 28, - "missed_abstention": 1, - "natural_stop": 28, - "nonrepetitive_nonempty": 28, - "predicted_values": 34, - "present_correct": 33, - "present_total": 36, - "schema_ok": 28, - "token_limited": 0, - "tp": 33, - "unsupported_values": 0 + "absent_correct": 174, + "absent_total": 187, + "exact": 82, + "false_abstention": 14, + "field_correct": 365, + "fields": 404, + "fn": 29, + "fp": 23, + "json_parse_ok": 112, + "missed_abstention": 13, + "natural_stop": 112, + "nonrepetitive_nonempty": 112, + "predicted_values": 269, + "present_correct": 191, + "present_total": 217, + "schema_ok": 112, + "token_limited": 0, + "tp": 246, + "unsupported_values": 2 }, - "exact": 0.8571428571428571, - "f1": 0.9428571428571428, - "false_abstention_rate": 0.08333333333333333, - "field_accuracy": 0.9411764705882353, + "exact": 0.7321428571428571, + "f1": 0.9044117647058822, + "false_abstention_rate": 0.06451612903225806, + "field_accuracy": 0.9034653465346535, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.03125, + "missed_abstention_rate": 0.06951871657754011, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.05814421801811217, - "precision": 0.9705882352941176, - "present_accuracy": 0.9166666666666666, - "recall": 0.9166666666666666, + "output_nll": 0.027105198803058426, + "precision": 0.9144981412639405, + "present_accuracy": 0.880184331797235, + "recall": 0.8945454545454545, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.0 + "unsupported_value_rate": 0.007434944237918215 }, - "retention/variant/canonical": { - "absent_accuracy": 0.9476439790575916, - "count": 196, + "variant/alias": { + "absent_accuracy": 0.9078947368421053, + "count": 46, "counts": { - "absent_correct": 181, - "absent_total": 191, - "exact": 162, - "false_abstention": 15, - "field_correct": 430, - "fields": 468, - "fn": 28, - "fp": 22, - "json_parse_ok": 196, - "missed_abstention": 10, - "natural_stop": 196, - "nonrepetitive_nonempty": 196, - "predicted_values": 272, - "present_correct": 249, - "present_total": 277, - "schema_ok": 196, + "absent_correct": 69, + "absent_total": 76, + "exact": 29, + "false_abstention": 11, + "field_correct": 131, + "fields": 157, + "fn": 20, + "fp": 14, + "json_parse_ok": 46, + "missed_abstention": 7, + "natural_stop": 46, + "nonrepetitive_nonempty": 46, + "predicted_values": 101, + "present_correct": 62, + "present_total": 81, + "schema_ok": 46, "token_limited": 0, - "tp": 250, - "unsupported_values": 4 + "tp": 87, + "unsupported_values": 1 }, - "exact": 0.826530612244898, - "f1": 0.909090909090909, - "false_abstention_rate": 0.05415162454873646, - "field_accuracy": 0.9188034188034188, + "exact": 0.6304347826086957, + "f1": 0.8365384615384616, + "false_abstention_rate": 0.13580246913580246, + "field_accuracy": 0.8343949044585988, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.05235602094240838, + "missed_abstention_rate": 0.09210526315789473, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.030436397921258337, - "precision": 0.9191176470588235, - "present_accuracy": 0.8989169675090253, - "recall": 0.8992805755395683, + "output_nll": 0.06243703520158306, + "precision": 0.8613861386138614, + "present_accuracy": 0.7654320987654321, + "recall": 0.8130841121495327, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.014705882352941176 + "unsupported_value_rate": 0.009900990099009901 }, - "retention/variant/nullable": { - "absent_accuracy": 0.9375, - "count": 28, + "variant/canonical": { + "absent_accuracy": 0.9235294117647059, + "count": 149, "counts": { - "absent_correct": 30, - "absent_total": 32, - "exact": 20, - "false_abstention": 6, - "field_correct": 59, - "fields": 68, - "fn": 7, - "fp": 3, - "json_parse_ok": 28, - "missed_abstention": 2, - "natural_stop": 28, - "nonrepetitive_nonempty": 28, - "predicted_values": 32, - "present_correct": 29, - "present_total": 36, - "schema_ok": 28, - "token_limited": 0, - "tp": 29, - "unsupported_values": 0 + "absent_correct": 157, + "absent_total": 170, + "exact": 118, + "false_abstention": 8, + "field_correct": 369, + "fields": 405, + "fn": 27, + "fp": 24, + "json_parse_ok": 149, + "missed_abstention": 13, + "natural_stop": 149, + "nonrepetitive_nonempty": 149, + "predicted_values": 304, + "present_correct": 212, + "present_total": 235, + "schema_ok": 149, + "token_limited": 0, + "tp": 280, + "unsupported_values": 4 }, - "exact": 0.7142857142857143, - "f1": 0.8529411764705882, - "false_abstention_rate": 0.16666666666666666, - "field_accuracy": 0.8676470588235294, + "exact": 0.7919463087248322, + "f1": 0.9165302782324058, + "false_abstention_rate": 0.03404255319148936, + "field_accuracy": 0.9111111111111111, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.0625, + "missed_abstention_rate": 0.07647058823529412, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.07853208471455478, - "precision": 0.90625, - "present_accuracy": 0.8055555555555556, - "recall": 0.8055555555555556, + "output_nll": 0.043735888344098986, + "precision": 0.9210526315789473, + "present_accuracy": 0.902127659574468, + "recall": 0.9120521172638436, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.0 + "unsupported_value_rate": 0.013157894736842105 }, - "variant/alias": { - "absent_accuracy": 0.8554216867469879, - "count": 77, + "variant/nullable": { + "absent_accuracy": 0.9295774647887324, + "count": 45, "counts": { - "absent_correct": 71, - "absent_total": 83, - "exact": 30, - "false_abstention": 28, - "field_correct": 121, - "fields": 191, - "fn": 85, - "fp": 33, - "json_parse_ok": 76, - "missed_abstention": 12, - "natural_stop": 77, - "nonrepetitive_nonempty": 77, + "absent_correct": 66, + "absent_total": 71, + "exact": 31, + "false_abstention": 6, + "field_correct": 132, + "fields": 152, + "fn": 18, + "fp": 9, + "json_parse_ok": 45, + "missed_abstention": 5, + "natural_stop": 45, + "nonrepetitive_nonempty": 45, "predicted_values": 99, - "present_correct": 50, - "present_total": 108, - "schema_ok": 65, - "token_limited": 0, - "tp": 66, - "unsupported_values": 3 - }, - "exact": 0.38961038961038963, - "f1": 0.528, - "false_abstention_rate": 0.25925925925925924, - "field_accuracy": 0.6335078534031413, - "json_parse_rate": 0.987012987012987, - "missed_abstention_rate": 0.14457831325301204, - "natural_stop_rate": 1.0, - "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.16598448270853222, - "precision": 0.6666666666666666, - "present_accuracy": 0.46296296296296297, - "recall": 0.4370860927152318, - "schema_valid_rate": 0.8441558441558441, - "token_limited_rate": 0.0, - "unsupported_value_rate": 0.030303030303030304 - }, - "variant/canonical": { - "absent_accuracy": 0.9397590361445783, - "count": 253, - "counts": { - "absent_correct": 234, - "absent_total": 249, - "exact": 187, - "false_abstention": 29, - "field_correct": 526, - "fields": 607, - "fn": 76, - "fp": 40, - "json_parse_ok": 253, - "missed_abstention": 15, - "natural_stop": 253, - "nonrepetitive_nonempty": 253, - "predicted_values": 376, - "present_correct": 292, - "present_total": 358, - "schema_ok": 253, + "present_correct": 66, + "present_total": 81, + "schema_ok": 44, "token_limited": 0, - "tp": 336, - "unsupported_values": 10 + "tp": 90, + "unsupported_values": 1 }, - "exact": 0.7391304347826086, - "f1": 0.852791878172589, - "false_abstention_rate": 0.08100558659217877, - "field_accuracy": 0.8665568369028006, + "exact": 0.6888888888888889, + "f1": 0.8695652173913043, + "false_abstention_rate": 0.07407407407407407, + "field_accuracy": 0.868421052631579, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.060240963855421686, + "missed_abstention_rate": 0.07042253521126761, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.06281126955392487, - "precision": 0.8936170212765957, - "present_accuracy": 0.8156424581005587, - "recall": 0.8155339805825242, - "schema_valid_rate": 1.0, + "output_nll": 0.04886089308549547, + "precision": 0.9090909090909091, + "present_accuracy": 0.8148148148148148, + "recall": 0.8333333333333334, + "schema_valid_rate": 0.9777777777777777, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.026595744680851064 - }, - "variant/nullable": { - "absent_accuracy": 0.8481012658227848, - "count": 74, - "counts": { - "absent_correct": 67, - "absent_total": 79, - "exact": 37, - "false_abstention": 13, - "field_correct": 122, - "fields": 182, - "fn": 56, - "fp": 23, - "json_parse_ok": 72, - "missed_abstention": 12, - "natural_stop": 73, - "nonrepetitive_nonempty": 73, - "predicted_values": 116, - "present_correct": 55, - "present_total": 103, - "schema_ok": 53, - "token_limited": 1, - "tp": 93, - "unsupported_values": 3 - }, - "exact": 0.5, - "f1": 0.7018867924528301, - "false_abstention_rate": 0.1262135922330097, - "field_accuracy": 0.6703296703296703, - "json_parse_rate": 0.972972972972973, - "missed_abstention_rate": 0.1518987341772152, - "natural_stop_rate": 0.9864864864864865, - "nonrepetitive_nonempty_rate": 0.9864864864864865, - "output_nll": 0.1592064066821508, - "precision": 0.8017241379310345, - "present_accuracy": 0.5339805825242718, - "recall": 0.6241610738255033, - "schema_valid_rate": 0.7162162162162162, - "token_limited_rate": 0.013513513513513514, - "unsupported_value_rate": 0.02586206896551724 + "unsupported_value_rate": 0.010101010101010102 } }, - "protocol": { - "constrained_decoding": false, - "json_assembly": false, - "max_tokens": 256, - "model_calls_per_request": 1, - "output_repair": false, - "prompt_truncation": false, - "repetition_penalty": 1, - "scorer": "strict-raw-json100-v1", - "serialization": "native-sft-chat-v1", - "temperature": 0 - }, - "scope": "Raw whole-object generation. Authored cardinality contrasts plus observed original/MixSNIPS/list controls. New authored values exclude existing evaluation source texts; split-disjoint values and phrasing for added controlled panels. BIO targets preserve annotated source order on space-reconstructed text; no independent gold perfection or external blind benchmark claim." + "retention": { + "gained": [ + "snips:AddToPlaylist:train:954:view0:native:canonical", + "snips:AddToPlaylist:train:1715:view0:native:canonical", + "snips:AddToPlaylist:train:1564:view0:native:canonical", + "focus110:development:roles:61", + "focus110:development:roles:36", + "focus110:development:roles:33", + "focus109:development:lists:43", + "focus109:development:lists:38", + "focus109:development:lists:40", + "cardinality105:development:16:alias", + "cardinality105:development:37:alias", + "cardinality105:development:10:alias", + "mixsnips:dev:738:focused102:canonical", + "joint128:development:PlayMusic:1:alias", + "joint128:development:PlayMusic:5:nullable", + "joint128:development:PlayMusic:7:alias", + "joint128:development:PlayMusic:8:nullable", + "joint128:development:PlayMusic:11:nullable", + "joint128:development:GetWeather:1:alias", + "joint128:development:GetWeather:3:canonical", + "joint128:development:GetWeather:7:alias", + "joint128:development:GetWeather:8:nullable", + "joint128:development:GetWeather:9:canonical", + "joint128:development:GetWeather:13:alias", + "joint128:development:BookRestaurant:0:canonical", + "joint128:development:BookRestaurant:1:alias", + "joint128:development:BookRestaurant:3:canonical", + "joint128:development:BookRestaurant:5:nullable", + "joint128:development:BookRestaurant:9:canonical", + "joint128:development:AddToPlaylist:1:alias", + "joint128:development:AddToPlaylist:2:nullable", + "joint128:development:AddToPlaylist:4:alias", + "joint128:development:AddToPlaylist:9:canonical", + "joint128:development:AddToPlaylist:11:nullable", + "joint128:development:AddToPlaylist:13:alias", + "joint128:development:AddToPlaylist:14:nullable", + "joint128:development:RateBook:1:alias", + "joint128:development:RateBook:6:canonical", + "joint128:development:RateBook:10:alias", + "joint128:development:RateBook:11:nullable", + "joint128:development:RateBook:12:canonical", + "joint128:development:RateBook:13:alias", + "joint128:development:SearchCreativeWork:1:alias", + "joint128:development:SearchCreativeWork:15:canonical", + "joint128:development:SearchScreeningEvent:0:canonical", + "joint128:development:SearchScreeningEvent:1:alias", + "joint128:development:SearchScreeningEvent:3:canonical", + "joint128:development:SearchScreeningEvent:5:nullable", + "joint128:development:SearchScreeningEvent:7:alias", + "joint128:development:SearchScreeningEvent:9:canonical", + "joint128:development:SearchScreeningEvent:11:nullable", + "joint128:development:SearchScreeningEvent:13:alias" + ], + "lost_previously_correct": [ + "snips:BookRestaurant:train:246:view1:native:canonical", + "snips:RateBook:train:1057:view0:native:canonical", + "snips:RateBook:train:1057:view1:native:canonical", + "focus109:development:lists:80", + "focus109:development:lists:77", + "focus109:development:lists:83" + ], + "new_prose_correct_delta": 37, + "paired_correct_delta": -1, + "panels": { + "all": { + "error_deltas": { + "false_abstention": -3, + "missed_abstention": -2, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.01538461538461533, + "exact": 0.0546875, + "f1": 0.04434984103033901, + "field_accuracy": 0.029032258064516148, + "json_parse_rate": 0.0234375, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.0048490846115784025, + "present_accuracy": 0.03888888888888886, + "recall": 0.07692307692307687, + "schema_valid_rate": 0.0625 + }, + "passed": true + }, + "domain/AddToPlaylist": { + "error_deltas": { + "false_abstention": -2, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": -1 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": 0.24999999999999994, + "f1": 0.11988304093567259, + "field_accuracy": 0.09375, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.07777777777777783, + "present_accuracy": 0.16666666666666663, + "recall": 0.16666666666666663, + "schema_valid_rate": 0.0 + }, + "passed": true + }, + "domain/BookRestaurant": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": 1, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": -0.07692307692307687, + "exact": -0.08333333333333337, + "f1": -0.026890756302521024, + "field_accuracy": -0.03448275862068961, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": -0.04678362573099415, + "present_accuracy": 0.0, + "recall": 0.0, + "schema_valid_rate": 0.0 + }, + "passed": false + }, + "domain/GetWeather": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": 0.0, + "f1": 0.0, + "field_accuracy": 0.0, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.0, + "present_accuracy": 0.0, + "recall": 0.0, + "schema_valid_rate": 0.0 + }, + "passed": true + }, + "domain/PlayMusic": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": 0.0, + "f1": 0.0, + "field_accuracy": 0.0, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.0, + "present_accuracy": 0.0, + "recall": 0.0, + "schema_valid_rate": 0.0 + }, + "passed": true + }, + "domain/RateBook": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": -0.16666666666666663, + "f1": -0.09523809523809523, + "field_accuracy": -0.05882352941176472, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": -0.09523809523809523, + "present_accuracy": -0.09523809523809523, + "recall": -0.09523809523809523, + "schema_valid_rate": 0.0 + }, + "passed": false + }, + "domain/SearchCreativeWork": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": 0.0, + "f1": 0.0, + "field_accuracy": 0.0, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.0, + "present_accuracy": 0.0, + "recall": 0.0, + "schema_valid_rate": 0.0 + }, + "passed": true + }, + "domain/SearchScreeningEvent": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": 0.0, + "f1": 0.0, + "field_accuracy": 0.0, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.0, + "present_accuracy": 0.0, + "recall": 0.0, + "schema_valid_rate": 0.0 + }, + "passed": true + }, + "domain/cardinality": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": -2, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.25, + "exact": 0.375, + "f1": 0.33333333333333337, + "field_accuracy": 0.3125, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.33333333333333337, + "present_accuracy": 0.375, + "recall": 0.33333333333333337, + "schema_valid_rate": 0.375 + }, + "passed": true + }, + "domain/mixsnips": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": 0.125, + "f1": -0.05882352941176466, + "field_accuracy": 0.0, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": -0.08333333333333337, + "present_accuracy": 0.0, + "recall": -0.04545454545454547, + "schema_valid_rate": 0.125 + }, + "passed": false + }, + "domain/observed_lists": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": -2, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.16666666666666663, + "exact": 0.0, + "f1": 0.07657378740970067, + "field_accuracy": 0.08333333333333337, + "json_parse_rate": 0.16666666666666663, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.0, + "present_accuracy": 0.0, + "recall": 0.13207547169811318, + "schema_valid_rate": 0.16666666666666663 + }, + "passed": true + }, + "domain/playground": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": 0.0, + "f1": 0.0, + "field_accuracy": 0.0, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.0, + "present_accuracy": 0.0, + "recall": 0.0, + "schema_valid_rate": 0.0 + }, + "passed": true + }, + "domain/roles": { + "error_deltas": { + "false_abstention": -1, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": 1 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": 0.375, + "f1": 0.23333333333333328, + "field_accuracy": 0.13636363636363635, + "json_parse_rate": 0.125, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.0174825174825175, + "present_accuracy": 0.2727272727272727, + "recall": 0.34782608695652173, + "schema_valid_rate": 0.25 + }, + "passed": false + }, + "domain/single_lists": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": 1, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": -0.125, + "exact": 0.0, + "f1": 0.0, + "field_accuracy": -0.08333333333333337, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.0, + "present_accuracy": 0.0, + "recall": 0.0, + "schema_valid_rate": 0.0 + }, + "passed": false + }, + "variant/alias": { + "error_deltas": { + "false_abstention": 0, + "missed_abstention": -2, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.1333333333333333, + "exact": 0.4545454545454545, + "f1": 0.380392156862745, + "field_accuracy": 0.20689655172413796, + "json_parse_rate": 0.09090909090909094, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": 0.3857142857142857, + "present_accuracy": 0.2857142857142857, + "recall": 0.35000000000000003, + "schema_valid_rate": 0.4545454545454546 + }, + "passed": true + }, + "variant/canonical": { + "error_deltas": { + "false_abstention": -4, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": 0.028037383177570097, + "f1": 0.029791422893334252, + "field_accuracy": 0.01945525291828798, + "json_parse_rate": 0.01869158878504673, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": -0.012469023085952768, + "present_accuracy": 0.0326797385620915, + "recall": 0.06796116504854377, + "schema_valid_rate": 0.01869158878504673 + }, + "passed": false + }, + "variant/nullable": { + "error_deltas": { + "false_abstention": 1, + "missed_abstention": 0, + "token_limited": 0, + "unsupported_values": 0 + }, + "metric_deltas": { + "absent_accuracy": 0.0, + "exact": -0.09999999999999998, + "f1": -0.0637560637560638, + "field_accuracy": -0.08333333333333337, + "json_parse_rate": 0.0, + "natural_stop_rate": 0.0, + "nonrepetitive_nonempty_rate": 0.0, + "precision": -0.01388888888888884, + "present_accuracy": -0.15384615384615385, + "recall": -0.09523809523809523, + "schema_valid_rate": 0.09999999999999998 + }, + "passed": false + } + }, + "passed": false, + "scope": "Finite observed request/metric retention gates; cannot guarantee universal no-regression." + } }, - "test": { + "heldout_1444": { "metrics": { "all": { - "absent_accuracy": 0.927916120576671, + "absent_accuracy": 0.9521625163826999, "count": 1444, "counts": { - "absent_correct": 1416, + "absent_correct": 1453, "absent_total": 1526, - "exact": 1199, - "false_abstention": 107, - "field_correct": 3195, + "exact": 1271, + "false_abstention": 53, + "field_correct": 3293, "fields": 3497, - "fn": 192, - "fp": 186, - "json_parse_ok": 1442, - "missed_abstention": 110, + "fn": 131, + "fp": 151, + "json_parse_ok": 1444, + "missed_abstention": 73, "natural_stop": 1444, "nonrepetitive_nonempty": 1444, - "predicted_values": 1965, - "present_correct": 1779, + "predicted_values": 1991, + "present_correct": 1840, "present_total": 1971, - "schema_ok": 1441, + "schema_ok": 1444, "token_limited": 0, - "tp": 1779, - "unsupported_values": 27 + "tp": 1840, + "unsupported_values": 24 }, - "exact": 0.8303324099722992, - "f1": 0.9039634146341463, - "false_abstention_rate": 0.05428716387620497, - "field_accuracy": 0.9136402630826422, - "json_parse_rate": 0.9986149584487535, - "missed_abstention_rate": 0.07208387942332896, + "exact": 0.8801939058171745, + "f1": 0.928823826350328, + "false_abstention_rate": 0.026889903602232368, + "field_accuracy": 0.9416642836717186, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.04783748361730013, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.02910132725713655, - "precision": 0.9053435114503817, - "present_accuracy": 0.9025875190258752, - "recall": 0.9025875190258752, - "schema_valid_rate": 0.9979224376731302, + "output_nll": 0.024015552303623188, + "precision": 0.9241587142139628, + "present_accuracy": 0.9335362760020294, + "recall": 0.9335362760020294, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.013740458015267175 + "unsupported_value_rate": 0.012054244098442994 }, "domain/AddToPlaylist": { - "absent_accuracy": 0.9152542372881356, + "absent_accuracy": 0.9364406779661016, "count": 206, "counts": { - "absent_correct": 216, + "absent_correct": 221, "absent_total": 236, - "exact": 165, - "false_abstention": 12, - "field_correct": 473, + "exact": 172, + "false_abstention": 8, + "field_correct": 481, "fields": 526, - "fn": 33, - "fp": 41, + "fn": 30, + "fp": 37, "json_parse_ok": 206, - "missed_abstention": 20, + "missed_abstention": 15, "natural_stop": 206, "nonrepetitive_nonempty": 206, - "predicted_values": 298, - "present_correct": 257, + "predicted_values": 297, + "present_correct": 260, "present_total": 290, "schema_ok": 206, "token_limited": 0, - "tp": 257, - "unsupported_values": 15 + "tp": 260, + "unsupported_values": 14 }, - "exact": 0.8009708737864077, - "f1": 0.8741496598639455, - "false_abstention_rate": 0.041379310344827586, - "field_accuracy": 0.8992395437262357, + "exact": 0.8349514563106796, + "f1": 0.8858603066439523, + "false_abstention_rate": 0.027586206896551724, + "field_accuracy": 0.9144486692015209, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.0847457627118644, + "missed_abstention_rate": 0.0635593220338983, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.044631248904427163, - "precision": 0.8624161073825504, - "present_accuracy": 0.8862068965517241, - "recall": 0.8862068965517241, + "output_nll": 0.041144459200014236, + "precision": 0.8754208754208754, + "present_accuracy": 0.896551724137931, + "recall": 0.896551724137931, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.050335570469798654 + "unsupported_value_rate": 0.04713804713804714 }, "domain/BookRestaurant": { - "absent_accuracy": 0.908, + "absent_accuracy": 0.932, "count": 208, "counts": { - "absent_correct": 227, + "absent_correct": 233, "absent_total": 250, - "exact": 162, - "false_abstention": 16, - "field_correct": 510, + "exact": 172, + "false_abstention": 8, + "field_correct": 528, "fields": 569, - "fn": 36, - "fp": 43, + "fn": 24, + "fp": 33, "json_parse_ok": 208, - "missed_abstention": 23, + "missed_abstention": 17, "natural_stop": 208, "nonrepetitive_nonempty": 208, - "predicted_values": 326, - "present_correct": 283, + "predicted_values": 328, + "present_correct": 295, "present_total": 319, "schema_ok": 208, "token_limited": 0, - "tp": 283, - "unsupported_values": 3 + "tp": 295, + "unsupported_values": 1 }, - "exact": 0.7788461538461539, - "f1": 0.8775193798449612, - "false_abstention_rate": 0.050156739811912224, - "field_accuracy": 0.8963093145869947, + "exact": 0.8269230769230769, + "f1": 0.9119010819165378, + "false_abstention_rate": 0.025078369905956112, + "field_accuracy": 0.9279437609841827, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.092, + "missed_abstention_rate": 0.068, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.03397235209894671, - "precision": 0.8680981595092024, - "present_accuracy": 0.8871473354231975, - "recall": 0.8871473354231975, + "output_nll": 0.027245943393189427, + "precision": 0.899390243902439, + "present_accuracy": 0.9247648902821317, + "recall": 0.9247648902821317, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.009202453987730062 + "unsupported_value_rate": 0.003048780487804878 }, "domain/GetWeather": { - "absent_accuracy": 0.9186991869918699, + "absent_accuracy": 0.943089430894309, "count": 206, "counts": { - "absent_correct": 226, + "absent_correct": 232, "absent_total": 246, - "exact": 164, - "false_abstention": 15, - "field_correct": 473, + "exact": 178, + "false_abstention": 5, + "field_correct": 493, "fields": 525, - "fn": 32, - "fp": 32, - "json_parse_ok": 205, - "missed_abstention": 20, + "fn": 18, + "fp": 27, + "json_parse_ok": 206, + "missed_abstention": 14, "natural_stop": 206, "nonrepetitive_nonempty": 206, - "predicted_values": 279, - "present_correct": 247, + "predicted_values": 288, + "present_correct": 261, "present_total": 279, - "schema_ok": 204, + "schema_ok": 206, "token_limited": 0, - "tp": 247, - "unsupported_values": 3 + "tp": 261, + "unsupported_values": 5 }, - "exact": 0.7961165048543689, - "f1": 0.8853046594982079, - "false_abstention_rate": 0.053763440860215055, - "field_accuracy": 0.900952380952381, - "json_parse_rate": 0.9951456310679612, - "missed_abstention_rate": 0.08130081300813008, + "exact": 0.8640776699029126, + "f1": 0.9206349206349206, + "false_abstention_rate": 0.017921146953405017, + "field_accuracy": 0.939047619047619, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.056910569105691054, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.02975186593454779, - "precision": 0.8853046594982079, - "present_accuracy": 0.8853046594982079, - "recall": 0.8853046594982079, - "schema_valid_rate": 0.9902912621359223, + "output_nll": 0.022860561515487453, + "precision": 0.90625, + "present_accuracy": 0.9354838709677419, + "recall": 0.9354838709677419, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.010752688172043012 + "unsupported_value_rate": 0.017361111111111112 }, "domain/PlayMusic": { - "absent_accuracy": 0.94140625, + "absent_accuracy": 0.9296875, "count": 208, "counts": { - "absent_correct": 241, + "absent_correct": 238, "absent_total": 256, - "exact": 153, - "false_abstention": 45, - "field_correct": 449, + "exact": 164, + "false_abstention": 24, + "field_correct": 468, "fields": 523, - "fn": 59, - "fp": 25, - "json_parse_ok": 207, - "missed_abstention": 15, + "fn": 37, + "fp": 31, + "json_parse_ok": 208, + "missed_abstention": 18, "natural_stop": 208, "nonrepetitive_nonempty": 208, - "predicted_values": 233, - "present_correct": 208, + "predicted_values": 261, + "present_correct": 230, "present_total": 267, - "schema_ok": 207, + "schema_ok": 208, "token_limited": 0, - "tp": 208, - "unsupported_values": 1 + "tp": 230, + "unsupported_values": 0 }, - "exact": 0.7355769230769231, - "f1": 0.8320000000000001, - "false_abstention_rate": 0.16853932584269662, - "field_accuracy": 0.858508604206501, - "json_parse_rate": 0.9951923076923077, - "missed_abstention_rate": 0.05859375, + "exact": 0.7884615384615384, + "f1": 0.8712121212121211, + "false_abstention_rate": 0.0898876404494382, + "field_accuracy": 0.8948374760994264, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0703125, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.04383864237080651, - "precision": 0.8927038626609443, - "present_accuracy": 0.7790262172284644, - "recall": 0.7790262172284644, - "schema_valid_rate": 0.9951923076923077, + "output_nll": 0.03961348310237484, + "precision": 0.8812260536398467, + "present_accuracy": 0.8614232209737828, + "recall": 0.8614232209737828, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.004291845493562232 + "unsupported_value_rate": 0.0 }, "domain/RateBook": { - "absent_accuracy": 0.9796747967479674, + "absent_accuracy": 0.983739837398374, "count": 204, "counts": { - "absent_correct": 241, + "absent_correct": 242, "absent_total": 246, - "exact": 193, - "false_abstention": 3, - "field_correct": 556, + "exact": 195, + "false_abstention": 1, + "field_correct": 558, "fields": 567, - "fn": 6, + "fn": 5, "fp": 8, "json_parse_ok": 204, - "missed_abstention": 5, + "missed_abstention": 4, "natural_stop": 204, "nonrepetitive_nonempty": 204, - "predicted_values": 323, - "present_correct": 315, + "predicted_values": 324, + "present_correct": 316, "present_total": 321, "schema_ok": 204, "token_limited": 0, - "tp": 315, + "tp": 316, "unsupported_values": 1 }, - "exact": 0.946078431372549, - "f1": 0.9782608695652175, - "false_abstention_rate": 0.009345794392523364, - "field_accuracy": 0.9805996472663139, + "exact": 0.9558823529411765, + "f1": 0.9798449612403102, + "false_abstention_rate": 0.003115264797507788, + "field_accuracy": 0.9841269841269841, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.02032520325203252, + "missed_abstention_rate": 0.016260162601626018, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.010552975621619992, - "precision": 0.9752321981424149, - "present_accuracy": 0.9813084112149533, - "recall": 0.9813084112149533, + "output_nll": 0.008335007565976554, + "precision": 0.9753086419753086, + "present_accuracy": 0.9844236760124611, + "recall": 0.9844236760124611, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.0030959752321981426 + "unsupported_value_rate": 0.0030864197530864196 }, "domain/SearchCreativeWork": { - "absent_accuracy": 0.9487179487179487, + "absent_accuracy": 1.0, "count": 208, "counts": { - "absent_correct": 37, + "absent_correct": 39, "absent_total": 39, - "exact": 197, + "exact": 199, "false_abstention": 2, - "field_correct": 275, + "field_correct": 277, "fields": 286, "fn": 9, - "fp": 9, + "fp": 7, "json_parse_ok": 208, - "missed_abstention": 2, + "missed_abstention": 0, "natural_stop": 208, "nonrepetitive_nonempty": 208, - "predicted_values": 247, + "predicted_values": 245, "present_correct": 238, "present_total": 247, "schema_ok": 208, "token_limited": 0, "tp": 238, - "unsupported_values": 0 + "unsupported_values": 2 }, - "exact": 0.9471153846153846, - "f1": 0.9635627530364372, + "exact": 0.9567307692307693, + "f1": 0.9674796747967479, "false_abstention_rate": 0.008097165991902834, - "field_accuracy": 0.9615384615384616, + "field_accuracy": 0.9685314685314685, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.05128205128205128, + "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.0180713094832754, - "precision": 0.9635627530364372, + "output_nll": 0.01792181849521932, + "precision": 0.9714285714285714, "present_accuracy": 0.9635627530364372, "recall": 0.9635627530364372, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.0 + "unsupported_value_rate": 0.00816326530612245 }, "domain/SearchScreeningEvent": { - "absent_accuracy": 0.9011857707509882, + "absent_accuracy": 0.9802371541501976, "count": 204, "counts": { - "absent_correct": 228, + "absent_correct": 248, "absent_total": 253, - "exact": 165, - "false_abstention": 14, - "field_correct": 459, + "exact": 191, + "false_abstention": 5, + "field_correct": 488, "fields": 501, - "fn": 17, - "fp": 28, + "fn": 8, + "fp": 8, "json_parse_ok": 204, - "missed_abstention": 25, + "missed_abstention": 5, "natural_stop": 204, "nonrepetitive_nonempty": 204, - "predicted_values": 259, - "present_correct": 231, + "predicted_values": 248, + "present_correct": 240, "present_total": 248, "schema_ok": 204, "token_limited": 0, - "tp": 231, - "unsupported_values": 4 + "tp": 240, + "unsupported_values": 1 }, - "exact": 0.8088235294117647, - "f1": 0.9112426035502958, - "false_abstention_rate": 0.056451612903225805, - "field_accuracy": 0.9161676646706587, + "exact": 0.9362745098039216, + "f1": 0.967741935483871, + "false_abstention_rate": 0.020161290322580645, + "field_accuracy": 0.9740518962075848, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.09881422924901186, + "missed_abstention_rate": 0.019762845849802372, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.02256406202664663, - "precision": 0.8918918918918919, - "present_accuracy": 0.9314516129032258, - "recall": 0.9314516129032258, + "output_nll": 0.01058128802694281, + "precision": 0.967741935483871, + "present_accuracy": 0.967741935483871, + "recall": 0.967741935483871, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.015444015444015444 + "unsupported_value_rate": 0.004032258064516129 }, - "macro_exact": 0.8305039715662217, + "macro_exact": 0.8804716247962964, "paired_requests": { - "accuracy": 0.7377521613832853, - "both_correct": 512, + "accuracy": 0.80835734870317, + "both_correct": 561, "count": 694 }, + "panel/original1444": { + "absent_accuracy": 0.9521625163826999, + "count": 1444, + "counts": { + "absent_correct": 1453, + "absent_total": 1526, + "exact": 1271, + "false_abstention": 53, + "field_correct": 3293, + "fields": 3497, + "fn": 131, + "fp": 151, + "json_parse_ok": 1444, + "missed_abstention": 73, + "natural_stop": 1444, + "nonrepetitive_nonempty": 1444, + "predicted_values": 1991, + "present_correct": 1840, + "present_total": 1971, + "schema_ok": 1444, + "token_limited": 0, + "tp": 1840, + "unsupported_values": 24 + }, + "exact": 0.8801939058171745, + "f1": 0.928823826350328, + "false_abstention_rate": 0.026889903602232368, + "field_accuracy": 0.9416642836717186, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.04783748361730013, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.024015552303623188, + "precision": 0.9241587142139628, + "present_accuracy": 0.9335362760020294, + "recall": 0.9335362760020294, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.012054244098442994 + }, "variant/alias": { - "absent_accuracy": 0.7931034482758621, + "absent_accuracy": 0.9310344827586207, "count": 28, "counts": { - "absent_correct": 23, + "absent_correct": 27, "absent_total": 29, - "exact": 21, + "exact": 23, "false_abstention": 1, - "field_correct": 61, + "field_correct": 70, "fields": 75, - "fn": 8, - "fp": 5, - "json_parse_ok": 26, - "missed_abstention": 6, + "fn": 3, + "fp": 4, + "json_parse_ok": 28, + "missed_abstention": 2, "natural_stop": 28, "nonrepetitive_nonempty": 28, - "predicted_values": 43, - "present_correct": 38, + "predicted_values": 47, + "present_correct": 43, "present_total": 46, - "schema_ok": 26, + "schema_ok": 28, "token_limited": 0, - "tp": 38, + "tp": 43, "unsupported_values": 2 }, - "exact": 0.75, - "f1": 0.853932584269663, + "exact": 0.8214285714285714, + "f1": 0.924731182795699, "false_abstention_rate": 0.021739130434782608, - "field_accuracy": 0.8133333333333334, - "json_parse_rate": 0.9285714285714286, - "missed_abstention_rate": 0.20689655172413793, + "field_accuracy": 0.9333333333333333, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.06896551724137931, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.05795656384933474, - "precision": 0.8837209302325582, - "present_accuracy": 0.8260869565217391, - "recall": 0.8260869565217391, - "schema_valid_rate": 0.9285714285714286, + "output_nll": 0.051775694317517, + "precision": 0.9148936170212766, + "present_accuracy": 0.9347826086956522, + "recall": 0.9347826086956522, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.046511627906976744 + "unsupported_value_rate": 0.0425531914893617 }, "variant/canonical": { - "absent_accuracy": 0.9339237057220708, + "absent_accuracy": 0.952316076294278, "count": 1388, "counts": { - "absent_correct": 1371, + "absent_correct": 1398, "absent_total": 1468, - "exact": 1163, - "false_abstention": 100, - "field_correct": 3075, + "exact": 1224, + "false_abstention": 50, + "field_correct": 3154, "fields": 3347, - "fn": 175, - "fp": 172, + "fn": 123, + "fp": 143, "json_parse_ok": 1388, - "missed_abstention": 97, + "missed_abstention": 70, "natural_stop": 1388, "nonrepetitive_nonempty": 1388, - "predicted_values": 1876, - "present_correct": 1704, + "predicted_values": 1899, + "present_correct": 1756, "present_total": 1879, "schema_ok": 1388, "token_limited": 0, - "tp": 1704, - "unsupported_values": 23 + "tp": 1756, + "unsupported_values": 21 }, - "exact": 0.8378962536023055, - "f1": 0.9075898801597869, - "false_abstention_rate": 0.05321979776476849, - "field_accuracy": 0.9187331939049895, + "exact": 0.8818443804034583, + "f1": 0.9295923769190046, + "false_abstention_rate": 0.026609898882384245, + "field_accuracy": 0.9423364206752316, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.06607629427792916, + "missed_abstention_rate": 0.047683923705722074, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.02737440389113602, - "precision": 0.908315565031983, - "present_accuracy": 0.9068653539116551, - "recall": 0.9068653539116551, + "output_nll": 0.022789464743013806, + "precision": 0.9246972090573986, + "present_accuracy": 0.9345396487493347, + "recall": 0.9345396487493347, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.012260127931769723 + "unsupported_value_rate": 0.011058451816745656 }, "variant/nullable": { - "absent_accuracy": 0.7586206896551724, + "absent_accuracy": 0.9655172413793104, "count": 28, "counts": { - "absent_correct": 22, + "absent_correct": 28, "absent_total": 29, - "exact": 15, - "false_abstention": 6, - "field_correct": 59, + "exact": 24, + "false_abstention": 2, + "field_correct": 69, "fields": 75, - "fn": 9, - "fp": 9, + "fn": 5, + "fp": 4, "json_parse_ok": 28, - "missed_abstention": 7, + "missed_abstention": 1, "natural_stop": 28, "nonrepetitive_nonempty": 28, - "predicted_values": 46, - "present_correct": 37, + "predicted_values": 45, + "present_correct": 41, "present_total": 46, - "schema_ok": 27, + "schema_ok": 28, "token_limited": 0, - "tp": 37, - "unsupported_values": 2 + "tp": 41, + "unsupported_values": 1 }, - "exact": 0.5357142857142857, - "f1": 0.8043478260869565, - "false_abstention_rate": 0.13043478260869565, - "field_accuracy": 0.7866666666666666, + "exact": 0.8571428571428571, + "f1": 0.9010989010989011, + "false_abstention_rate": 0.043478260869565216, + "field_accuracy": 0.92, "json_parse_rate": 1.0, - "missed_abstention_rate": 0.2413793103448276, + "missed_abstention_rate": 0.034482758620689655, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.08585214895096474, - "precision": 0.8043478260869565, - "present_accuracy": 0.8043478260869565, - "recall": 0.8043478260869565, - "schema_valid_rate": 0.9642857142857143, + "output_nll": 0.05703432222279454, + "precision": 0.9111111111111111, + "present_accuracy": 0.8913043478260869, + "recall": 0.8913043478260869, + "schema_valid_rate": 1.0, "token_limited_rate": 0.0, - "unsupported_value_rate": 0.043478260869565216 + "unsupported_value_rate": 0.022222222222222223 } }, - "protocol": { - "constrained_decoding": false, - "json_assembly": false, - "max_tokens": 256, - "model_calls_per_request": 1, - "output_repair": false, - "prompt_truncation": false, - "repetition_penalty": 1, - "scorer": "strict-raw-json100-v1", - "serialization": "native-sft-chat-v1", - "temperature": 0 - }, - "scope": "Raw model-generated JSON, source-span annotation agreement. Duplicate keys, nonfinite JSON, fences, trailing prose and schema violations fail exact correctness. Flat string arrays/nullable strings in seven SNIPS domains; observed research panels, not a new blind or external benchmark." - }, - "playground": { - "metrics": { - "absent_accuracy": 0.9245283018867925, - "count": 54, - "counts": { - "absent_correct": 49, - "absent_total": 53, - "exact": 44, - "false_abstention": 3, - "field_correct": 124, - "fields": 135, - "fn": 7, - "fp": 9, - "json_parse_ok": 54, - "missed_abstention": 4, - "natural_stop": 54, - "nonrepetitive_nonempty": 54, - "predicted_values": 86, - "present_correct": 75, - "present_total": 82, - "schema_ok": 54, - "token_limited": 0, - "tp": 77, - "unsupported_values": 3 - }, - "exact": 0.8148148148148148, - "f1": 0.9058823529411765, - "false_abstention_rate": 0.036585365853658534, - "field_accuracy": 0.9185185185185185, - "json_parse_rate": 1.0, - "missed_abstention_rate": 0.07547169811320754, - "natural_stop_rate": 1.0, - "nonrepetitive_nonempty_rate": 1.0, - "output_nll": 0.05903947115142795, - "precision": 0.8953488372093024, - "present_accuracy": 0.9146341463414634, - "recall": 0.9166666666666666, - "schema_valid_rate": 1.0, - "token_limited_rate": 0.0, - "unsupported_value_rate": 0.03488372093023256 - }, - "requests": 54, - "outputs": [ - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 0, - "false_abstention": 1, - "field_correct": 1, - "fields": 3, - "fn": 2, - "fp": 1, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 1, - "present_total": 3, - "schema_ok": 1, + "previous_metrics": { + "all": { + "absent_accuracy": 0.927916120576671, + "count": 1444, + "counts": { + "absent_correct": 1416, + "absent_total": 1526, + "exact": 1199, + "false_abstention": 107, + "field_correct": 3195, + "fields": 3497, + "fn": 192, + "fp": 186, + "json_parse_ok": 1442, + "missed_abstention": 110, + "natural_stop": 1444, + "nonrepetitive_nonempty": 1444, + "predicted_values": 1965, + "present_correct": 1779, + "present_total": 1971, + "schema_ok": 1441, "token_limited": 0, - "tp": 1, - "unsupported_values": 0 + "tp": 1779, + "unsupported_values": 27 }, - "id": "FIELD-01", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"playlist_owner\":[],\"playlist\":[\"Rock Gaming\"],\"entity_name\":[\"Mariem Hassan\"]}", - "tokens": 37, - "variant": "canonical" + "exact": 0.8303324099722992, + "f1": 0.9039634146341463, + "false_abstention_rate": 0.05428716387620497, + "field_accuracy": 0.9136402630826422, + "json_parse_rate": 0.9986149584487535, + "missed_abstention_rate": 0.07208387942332896, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.030197499356996216, + "precision": 0.9053435114503817, + "present_accuracy": 0.9025875190258752, + "recall": 0.9025875190258752, + "schema_valid_rate": 0.9979224376731302, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.013740458015267175 }, - { - "checks": { - "absent_correct": 1, - "absent_total": 2, - "exact": 0, - "false_abstention": 0, - "field_correct": 1, - "fields": 2, - "fn": 0, - "fp": 1, - "json_parse_ok": 1, - "missed_abstention": 1, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 0, - "present_total": 0, - "schema_ok": 1, + "domain/AddToPlaylist": { + "absent_accuracy": 0.9152542372881356, + "count": 206, + "counts": { + "absent_correct": 216, + "absent_total": 236, + "exact": 165, + "false_abstention": 12, + "field_correct": 473, + "fields": 526, + "fn": 33, + "fp": 41, + "json_parse_ok": 206, + "missed_abstention": 20, + "natural_stop": 206, + "nonrepetitive_nonempty": 206, + "predicted_values": 298, + "present_correct": 257, + "present_total": 290, + "schema_ok": 206, "token_limited": 0, - "tp": 0, - "unsupported_values": 0 + "tp": 257, + "unsupported_values": 15 }, - "id": "FIELD-02", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"artist\":[\"Mariem Hassan\"],\"music_item\":[]}", - "tokens": 23, - "variant": "canonical" + "exact": 0.8009708737864077, + "f1": 0.8741496598639455, + "false_abstention_rate": 0.041379310344827586, + "field_accuracy": 0.8992395437262357, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0847457627118644, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.04620553536913629, + "precision": 0.8624161073825504, + "present_accuracy": 0.8862068965517241, + "recall": 0.8862068965517241, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.050335570469798654 }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 1, - "fields": 1, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, + "domain/BookRestaurant": { + "absent_accuracy": 0.908, + "count": 208, + "counts": { + "absent_correct": 227, + "absent_total": 250, + "exact": 162, + "false_abstention": 16, + "field_correct": 510, + "fields": 569, + "fn": 36, + "fp": 43, + "json_parse_ok": 208, + "missed_abstention": 23, + "natural_stop": 208, + "nonrepetitive_nonempty": 208, + "predicted_values": 326, + "present_correct": 283, + "present_total": 319, + "schema_ok": 208, "token_limited": 0, - "tp": 1, - "unsupported_values": 0 + "tp": 283, + "unsupported_values": 3 }, - "id": "FIELD-03", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"playlist_owner\":[\"my\"]}", - "tokens": 13, - "variant": "canonical" + "exact": 0.7788461538461539, + "f1": 0.8775193798449612, + "false_abstention_rate": 0.050156739811912224, + "field_accuracy": 0.8963093145869947, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.092, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.03511890941109204, + "precision": 0.8680981595092024, + "present_accuracy": 0.8871473354231975, + "recall": 0.8871473354231975, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.009202453987730062 }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, + "domain/GetWeather": { + "absent_accuracy": 0.9186991869918699, + "count": 206, + "counts": { + "absent_correct": 226, + "absent_total": 246, + "exact": 164, + "false_abstention": 15, + "field_correct": 473, + "fields": 525, + "fn": 32, + "fp": 32, + "json_parse_ok": 205, + "missed_abstention": 20, + "natural_stop": 206, + "nonrepetitive_nonempty": 206, + "predicted_values": 279, + "present_correct": 247, + "present_total": 279, + "schema_ok": 204, "token_limited": 0, - "tp": 1, - "unsupported_values": 0 + "tp": 247, + "unsupported_values": 3 }, - "id": "FIELD-04", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"artist\":[],\"entity_name\":[],\"playlist\":[\"Digster Future Hits\"]}", - "tokens": 29, - "variant": "canonical" + "exact": 0.7961165048543689, + "f1": 0.8853046594982079, + "false_abstention_rate": 0.053763440860215055, + "field_accuracy": 0.900952380952381, + "json_parse_rate": 0.9951456310679612, + "missed_abstention_rate": 0.08130081300813008, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.030885711256629398, + "precision": 0.8853046594982079, + "present_accuracy": 0.8853046594982079, + "recall": 0.8853046594982079, + "schema_valid_rate": 0.9902912621359223, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.010752688172043012 }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 3, - "present_correct": 3, - "present_total": 3, - "schema_ok": 1, + "domain/PlayMusic": { + "absent_accuracy": 0.94140625, + "count": 208, + "counts": { + "absent_correct": 241, + "absent_total": 256, + "exact": 153, + "false_abstention": 45, + "field_correct": 449, + "fields": 523, + "fn": 59, + "fp": 25, + "json_parse_ok": 207, + "missed_abstention": 15, + "natural_stop": 208, + "nonrepetitive_nonempty": 208, + "predicted_values": 233, + "present_correct": 208, + "present_total": 267, + "schema_ok": 207, "token_limited": 0, - "tp": 3, - "unsupported_values": 0 + "tp": 208, + "unsupported_values": 1 }, - "id": "FIELD-05", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"state\":[\"AR\"],\"party_size_number\":[\"nine\"],\"restaurant_type\":[\"brasserie\"]}", - "tokens": 36, - "variant": "canonical" + "exact": 0.7355769230769231, + "f1": 0.8320000000000001, + "false_abstention_rate": 0.16853932584269662, + "field_accuracy": 0.858508604206501, + "json_parse_rate": 0.9951923076923077, + "missed_abstention_rate": 0.05859375, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.0454767054272032, + "precision": 0.8927038626609443, + "present_accuracy": 0.7790262172284644, + "recall": 0.7790262172284644, + "schema_valid_rate": 0.9951923076923077, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.004291845493562232 }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, + "domain/RateBook": { + "absent_accuracy": 0.9796747967479674, + "count": 204, + "counts": { + "absent_correct": 241, + "absent_total": 246, + "exact": 193, + "false_abstention": 3, + "field_correct": 556, + "fields": 567, + "fn": 6, + "fp": 8, + "json_parse_ok": 204, + "missed_abstention": 5, + "natural_stop": 204, + "nonrepetitive_nonempty": 204, + "predicted_values": 323, + "present_correct": 315, + "present_total": 321, + "schema_ok": 204, "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-06", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"party_size_number\":[\"nine\"],\"timeRange\":[],\"sort\":[]}", - "tokens": 27, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "FIELD-07", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"city\":[\"Wrigley\"],\"sort\":[\"best\"]}", - "tokens": 21, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 0, - "present_correct": 0, - "present_total": 0, - "schema_ok": 1, - "token_limited": 0, - "tp": 0, - "unsupported_values": 0 - }, - "id": "FIELD-08", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"poi\":[],\"timeRange\":[]}", - "tokens": 15, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "FIELD-09", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"state\":[\"Louisiana\"],\"condition_temperature\":[\"colder\"]}", - "tokens": 23, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-10", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"current_location\":[],\"geographic_poi\":[],\"condition_temperature\":[\"colder\"]}", - "tokens": 31, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 3, - "present_correct": 3, - "present_total": 3, - "schema_ok": 1, - "token_limited": 0, - "tp": 3, - "unsupported_values": 0 - }, - "id": "FIELD-11", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"timeRange\":[\"last August\"],\"geographic_poi\":[\"Stunt Ranch Santa Monica Mountains Reserve\"],\"condition_temperature\":[\"colder\"]}", - "tokens": 44, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-12", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"state\":[],\"geographic_poi\":[\"Stunt Ranch Santa Monica Mountains Reserve\"],\"current_location\":[]}", - "tokens": 34, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 3, - "present_correct": 3, - "present_total": 3, - "schema_ok": 1, - "token_limited": 0, - "tp": 3, - "unsupported_values": 0 - }, - "id": "FIELD-13", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"artist\":[\"Behzad Mirkhani\"],\"music_item\":[\"theme\"],\"sort\":[\"last\"]}", - "tokens": 38, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-14", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"playlist\":[],\"music_item\":[\"theme\"],\"service\":[]}", - "tokens": 25, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 3, - "present_correct": 3, - "present_total": 3, - "schema_ok": 1, - "token_limited": 0, - "tp": 3, - "unsupported_values": 0 - }, - "id": "FIELD-15", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"artist\":[\"Yui\"],\"music_item\":[\"track\"],\"service\":[\"Vimeo\"]}", - "tokens": 33, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 0, - "present_correct": 0, - "present_total": 0, - "schema_ok": 1, - "token_limited": 0, - "tp": 0, - "unsupported_values": 0 - }, - "id": "FIELD-16", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"playlist\":[],\"sort\":[]}", - "tokens": 14, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 3, - "present_correct": 3, - "present_total": 3, - "schema_ok": 1, - "token_limited": 0, - "tp": 3, - "unsupported_values": 0 - }, - "id": "FIELD-17", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"rating_value\":[\"3\"],\"rating_unit\":[\"stars\"],\"object_name\":[\"Things Fall Apart\"]}", - "tokens": 35, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-18", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"object_name\":[\"Things Fall Apart\"],\"object_select\":[],\"object_type\":[]}", - "tokens": 29, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 1, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-19", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"rating_value\":[\"1\"],\"object_name\":[]}", - "tokens": 19, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 0, - "present_correct": 0, - "present_total": 0, - "schema_ok": 1, - "token_limited": 0, - "tp": 0, - "unsupported_values": 0 - }, - "id": "FIELD-20", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"object_part_of_series_type\":[],\"object_name\":[]}", - "tokens": 23, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 1, - "fields": 1, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-21", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"object_name\":[\"Happiness Was Free\"]}", - "tokens": 16, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 1, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-22", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"object_type\":[],\"object_name\":[\"Happiness Was Free\"]}", - "tokens": 23, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "FIELD-23", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"object_name\":[\"Martin Morning\"],\"object_type\":[\"saga\"]}", - "tokens": 24, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 1, - "fields": 1, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-24", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"object_name\":[\"Martin Morning\"]}", - "tokens": 13, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "FIELD-25", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"movie_type\":[\"animated movies\"],\"spatial_relation\":[\"in the neighborhood\"]}", - "tokens": 29, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-26", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"movie_name\":[],\"timeRange\":[],\"movie_type\":[\"animated movies\"]}", - "tokens": 32, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "FIELD-27", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"timeRange\":[\"twenty two o'clock\"],\"movie_name\":[\"Mabel’s Dramatic Career\"]}", - "tokens": 36, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-28", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"timeRange\":[\"twenty two o'clock\"],\"spatial_relation\":[],\"object_location_type\":[]}", - "tokens": 35, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 1, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "FIELD-29", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"track\":[\"Blue Lantern\"],\"artist\":[\"Mira Sol\"],\"album\":[]}", - "tokens": 29, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 4, - "fields": 4, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 4, - "present_correct": 4, - "present_total": 4, - "schema_ok": 1, - "token_limited": 0, - "tp": 4, - "unsupported_values": 0 - }, - "id": "FIELD-30", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"restaurant_name\":[\"River Moon\"],\"city\":[\"Denver\"],\"party_size_number\":[\"four\"],\"timeRange\":[\"tomorrow\"]}", - "tokens": 49, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 1, - "exact": 0, - "false_abstention": 0, - "field_correct": 2, - "fields": 3, - "fn": 1, - "fp": 1, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 1, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 1 - }, - "id": "FIELD-31", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"city\":[\"Ose\"],\"timeRange\":[\"Friday\"],\"country\":[]}", - "tokens": 26, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 0, - "false_abstention": 1, - "field_correct": 3, - "fields": 4, - "fn": 1, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 3, - "present_correct": 3, - "present_total": 4, - "schema_ok": 1, - "token_limited": 0, - "tp": 3, - "unsupported_values": 0 - }, - "id": "FIELD-32", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"object_name\":[\"Glass Harbor\"],\"rating_value\":[\"four\"],\"best_rating\":[],\"rating_unit\":[\"stars\"]}", - "tokens": 42, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 1, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "FIELD-33", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"timeRange\":[],\"location_name\":[\"Maple Cinema\"],\"movie_name\":[\"Silver Orchard\"]}", - "tokens": 37, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "FIELD-34", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"entity_name\":[\"Night Steps\"],\"playlist\":[\"Evening Walks\"]}", - "tokens": 27, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 1, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "FIELD-35", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"artist\":[\"Mira Sol\",\"Theo Lane\"],\"album\":[]}", - "tokens": 24, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 1, - "exact": 0, - "false_abstention": 0, - "field_correct": 1, - "fields": 2, - "fn": 1, - "fp": 2, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 3, - "present_correct": 0, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 2 - }, - "id": "FIELD-36", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"city\":[\"Bergen\",\"Orange\",\"Lake\"],\"country\":[]}", - "tokens": 24, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 1, - "exact": 0, - "false_abstention": 1, - "field_correct": 2, - "fields": 3, - "fn": 1, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-37", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"cuisine\":[],\"city\":[\"Boston\"],\"restaurant_name\":[]}", - "tokens": 26, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 1, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "FIELD-38", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"track\":[],\"artist\":[\"Mira Sol\"],\"album\":[\"Quiet Signals\"]}", - "tokens": 30, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 1, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "FIELD-39", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"object_name\":[\"Paper Bridge\"],\"rating_value\":[]}", - "tokens": 20, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 3, - "absent_total": 3, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 0, - "present_correct": 0, - "present_total": 0, - "schema_ok": 1, - "token_limited": 0, - "tp": 0, - "unsupported_values": 0 - }, - "id": "FIELD-40", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"artist\":[],\"timeRange\":[],\"country\":[]}", - "tokens": 20, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "native:snips:AddToPlaylist:train:80:view1:native:alias", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"selected_playlist\":[\"night vibes\"],\"selected_artist\":[],\"selected_playlist_owner\":[]}", - "tokens": 36, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 2, - "exact": 0, - "false_abstention": 0, - "field_correct": 2, - "fields": 3, - "fn": 0, - "fp": 1, - "json_parse_ok": 1, - "missed_abstention": 1, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "native:snips:AddToPlaylist:train:80:view1:native:nullable", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"playlist_owner\":null,\"artist\":\"gn\",\"playlist\":\"night vibes\"}", - "tokens": 26, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 2, - "exact": 0, - "false_abstention": 0, - "field_correct": 2, - "fields": 3, - "fn": 0, - "fp": 1, - "json_parse_ok": 1, - "missed_abstention": 1, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "native:snips:BookRestaurant:train:684:view1:native:alias", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"selected_restaurant_name\":[\"mughlai restaurant\"],\"selected_timeRange\":[\"Nov. the 1st\"],\"selected_sort\":[]}", - "tokens": 47, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 1, - "absent_total": 2, - "exact": 0, - "false_abstention": 0, - "field_correct": 2, - "fields": 3, - "fn": 0, - "fp": 1, - "json_parse_ok": 1, - "missed_abstention": 1, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 - }, - "id": "native:snips:BookRestaurant:train:684:view1:native:nullable", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"restaurant_name\":\"mughlai restaurant\",\"timeRange\":\"Nov. the 1st\",\"sort\":null}", - "tokens": 34, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "native:snips:GetWeather:train:109:view0:native:alias", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"selected_condition_temperature\":[\"colder\"],\"selected_state\":[\"Louisiana\"]}", - "tokens": 29, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "native:snips:GetWeather:train:109:view0:native:nullable", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"state\":\"Louisiana\",\"condition_temperature\":\"colder\"}", - "tokens": 19, - "variant": "canonical" - }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, - "token_limited": 0, - "tp": 1, - "unsupported_values": 0 + "tp": 315, + "unsupported_values": 1 }, - "id": "native:snips:PlayMusic:train:801:view1:native:alias", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"selected_playlist\":[],\"selected_music_item\":[\"theme\"],\"selected_service\":[]}", - "tokens": 34, - "variant": "canonical" + "exact": 0.946078431372549, + "f1": 0.9782608695652175, + "false_abstention_rate": 0.009345794392523364, + "field_accuracy": 0.9805996472663139, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.02032520325203252, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.010863955307286233, + "precision": 0.9752321981424149, + "present_accuracy": 0.9813084112149533, + "recall": 0.9813084112149533, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.0030959752321981426 }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, + "domain/SearchCreativeWork": { + "absent_accuracy": 0.9487179487179487, + "count": 208, + "counts": { + "absent_correct": 37, + "absent_total": 39, + "exact": 197, + "false_abstention": 2, + "field_correct": 275, + "fields": 286, + "fn": 9, + "fp": 9, + "json_parse_ok": 208, + "missed_abstention": 2, + "natural_stop": 208, + "nonrepetitive_nonempty": 208, + "predicted_values": 247, + "present_correct": 238, + "present_total": 247, + "schema_ok": 208, "token_limited": 0, - "tp": 1, + "tp": 238, "unsupported_values": 0 }, - "id": "native:snips:PlayMusic:train:801:view1:native:nullable", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"playlist\":null,\"music_item\":\"theme\",\"service\":null}", - "tokens": 23, - "variant": "canonical" + "exact": 0.9471153846153846, + "f1": 0.9635627530364372, + "false_abstention_rate": 0.008097165991902834, + "field_accuracy": 0.9615384615384616, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.05128205128205128, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.019084845725116723, + "precision": 0.9635627530364372, + "present_accuracy": 0.9635627530364372, + "recall": 0.9635627530364372, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.0 }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, + "domain/SearchScreeningEvent": { + "absent_accuracy": 0.9011857707509882, + "count": 204, + "counts": { + "absent_correct": 228, + "absent_total": 253, + "exact": 165, + "false_abstention": 14, + "field_correct": 459, + "fields": 501, + "fn": 17, + "fp": 28, + "json_parse_ok": 204, + "missed_abstention": 25, + "natural_stop": 204, + "nonrepetitive_nonempty": 204, + "predicted_values": 259, + "present_correct": 231, + "present_total": 248, + "schema_ok": 204, "token_limited": 0, - "tp": 2, - "unsupported_values": 0 + "tp": 231, + "unsupported_values": 4 }, - "id": "native:snips:RateBook:train:293:view0:native:alias", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"selected_rating_value\":[\"2\"],\"selected_object_select\":[\"current\"]}", - "tokens": 29, - "variant": "canonical" + "exact": 0.8088235294117647, + "f1": 0.9112426035502958, + "false_abstention_rate": 0.056451612903225805, + "field_accuracy": 0.9161676646706587, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.09881422924901186, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.023404948954503554, + "precision": 0.8918918918918919, + "present_accuracy": 0.9314516129032258, + "recall": 0.9314516129032258, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.015444015444015444 }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 2, - "fields": 2, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 2, - "present_correct": 2, - "present_total": 2, - "schema_ok": 1, - "token_limited": 0, - "tp": 2, - "unsupported_values": 0 - }, - "id": "native:snips:RateBook:train:293:view0:native:nullable", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"object_select\":\"current\",\"rating_value\":\"2\"}", - "tokens": 19, - "variant": "canonical" + "macro_exact": 0.8305039715662217, + "paired_requests": { + "accuracy": 0.7377521613832853, + "both_correct": 512, + "count": 694 }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 1, - "fields": 1, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, + "panel/original1444": { + "absent_accuracy": 0.927916120576671, + "count": 1444, + "counts": { + "absent_correct": 1416, + "absent_total": 1526, + "exact": 1199, + "false_abstention": 107, + "field_correct": 3195, + "fields": 3497, + "fn": 192, + "fp": 186, + "json_parse_ok": 1442, + "missed_abstention": 110, + "natural_stop": 1444, + "nonrepetitive_nonempty": 1444, + "predicted_values": 1965, + "present_correct": 1779, + "present_total": 1971, + "schema_ok": 1441, "token_limited": 0, - "tp": 1, - "unsupported_values": 0 + "tp": 1779, + "unsupported_values": 27 }, - "id": "native:snips:SearchCreativeWork:train:602:view1:native:alias", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"selected_object_name\":[\"Playlist: The Very Best of Dolly Parton\"]}", - "tokens": 25, - "variant": "canonical" + "exact": 0.8303324099722992, + "f1": 0.9039634146341463, + "false_abstention_rate": 0.05428716387620497, + "field_accuracy": 0.9136402630826422, + "json_parse_rate": 0.9986149584487535, + "missed_abstention_rate": 0.07208387942332896, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.030197499356996216, + "precision": 0.9053435114503817, + "present_accuracy": 0.9025875190258752, + "recall": 0.9025875190258752, + "schema_valid_rate": 0.9979224376731302, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.013740458015267175 }, - { - "checks": { - "absent_correct": 0, - "absent_total": 0, - "exact": 1, - "false_abstention": 0, - "field_correct": 1, - "fields": 1, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, + "variant/alias": { + "absent_accuracy": 0.7931034482758621, + "count": 28, + "counts": { + "absent_correct": 23, + "absent_total": 29, + "exact": 21, + "false_abstention": 1, + "field_correct": 61, + "fields": 75, + "fn": 8, + "fp": 5, + "json_parse_ok": 26, + "missed_abstention": 6, + "natural_stop": 28, + "nonrepetitive_nonempty": 28, + "predicted_values": 43, + "present_correct": 38, + "present_total": 46, + "schema_ok": 26, "token_limited": 0, - "tp": 1, - "unsupported_values": 0 + "tp": 38, + "unsupported_values": 2 }, - "id": "native:snips:SearchCreativeWork:train:602:view1:native:nullable", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"object_name\":\"Playlist: The Very Best of Dolly Parton\"}", - "tokens": 20, - "variant": "canonical" + "exact": 0.75, + "f1": 0.853932584269663, + "false_abstention_rate": 0.021739130434782608, + "field_accuracy": 0.8133333333333334, + "json_parse_rate": 0.9285714285714286, + "missed_abstention_rate": 0.20689655172413793, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.05955322041908013, + "precision": 0.8837209302325582, + "present_accuracy": 0.8260869565217391, + "recall": 0.8260869565217391, + "schema_valid_rate": 0.9285714285714286, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.046511627906976744 }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 1, - "false_abstention": 0, - "field_correct": 3, - "fields": 3, - "fn": 0, - "fp": 0, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 1, - "present_total": 1, - "schema_ok": 1, + "variant/canonical": { + "absent_accuracy": 0.9339237057220708, + "count": 1388, + "counts": { + "absent_correct": 1371, + "absent_total": 1468, + "exact": 1163, + "false_abstention": 100, + "field_correct": 3075, + "fields": 3347, + "fn": 175, + "fp": 172, + "json_parse_ok": 1388, + "missed_abstention": 97, + "natural_stop": 1388, + "nonrepetitive_nonempty": 1388, + "predicted_values": 1876, + "present_correct": 1704, + "present_total": 1879, + "schema_ok": 1388, "token_limited": 0, - "tp": 1, - "unsupported_values": 0 + "tp": 1704, + "unsupported_values": 23 }, - "id": "native:snips:SearchScreeningEvent:train:897:view1:native:alias", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"selected_movie_type\":[\"animated movies\"],\"selected_movie_name\":[],\"selected_location_name\":[]}", - "tokens": 42, - "variant": "canonical" + "exact": 0.8378962536023055, + "f1": 0.9075898801597869, + "false_abstention_rate": 0.05321979776476849, + "field_accuracy": 0.9187331939049895, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.06607629427792916, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.028413429219706483, + "precision": 0.908315565031983, + "present_accuracy": 0.9068653539116551, + "recall": 0.9068653539116551, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.012260127931769723 }, - { - "checks": { - "absent_correct": 2, - "absent_total": 2, - "exact": 0, - "false_abstention": 0, - "field_correct": 2, - "fields": 3, - "fn": 1, - "fp": 1, - "json_parse_ok": 1, - "missed_abstention": 0, - "natural_stop": 1, - "nonrepetitive_nonempty": 1, - "predicted_values": 1, - "present_correct": 0, - "present_total": 1, - "schema_ok": 1, + "variant/nullable": { + "absent_accuracy": 0.7586206896551724, + "count": 28, + "counts": { + "absent_correct": 22, + "absent_total": 29, + "exact": 15, + "false_abstention": 6, + "field_correct": 59, + "fields": 75, + "fn": 9, + "fp": 9, + "json_parse_ok": 28, + "missed_abstention": 7, + "natural_stop": 28, + "nonrepetitive_nonempty": 28, + "predicted_values": 46, + "present_correct": 37, + "present_total": 46, + "schema_ok": 27, "token_limited": 0, - "tp": 0, - "unsupported_values": 0 + "tp": 37, + "unsupported_values": 2 }, - "id": "native:snips:SearchScreeningEvent:train:897:view1:native:nullable", - "source": "playground", - "stop_reason": "turn_end", - "text": "{\"movie_name\":null,\"movie_type\":\"movies\",\"location_name\":null}", - "tokens": 31, - "variant": "canonical" + "exact": 0.5357142857142857, + "f1": 0.8043478260869565, + "false_abstention_rate": 0.13043478260869565, + "field_accuracy": 0.7866666666666666, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.2413793103448276, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.08928068367198908, + "precision": 0.8043478260869565, + "present_accuracy": 0.8043478260869565, + "recall": 0.8043478260869565, + "schema_valid_rate": 0.9642857142857143, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.043478260869565216 } - ], - "scope": "Observed fixed local54 diagnostics; standalone parity verification recorded separately." - }, - "previous_release_comparison": { - "all": { - "candidate_correct": 1199, - "error_counts": { - "false_abstention": { - "candidate": 107, - "parent": 111 - }, - "json_parse_ok": { - "candidate": 1442, - "parent": 1442 - }, - "missed_abstention": { - "candidate": 110, - "parent": 115 - }, - "schema_ok": { - "candidate": 1441, - "parent": 1441 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 27, - "parent": 30 - } - }, - "metrics": { - "absent_accuracy": { - "candidate": 0.927916120576671, - "delta_percentage_points": 0.327653997378774, - "parent": 0.9246395806028833 - }, - "exact": { - "candidate": 0.8303324099722992, - "delta_percentage_points": 0.5540166204986208, - "parent": 0.824792243767313 - }, - "f1": { - "candidate": 0.9039634146341463, - "delta_percentage_points": -0.08347574907584532, - "parent": 0.9047981721249048 - }, - "field_accuracy": { - "candidate": 0.9136402630826422, - "delta_percentage_points": 0.05719187875321552, - "parent": 0.9130683442951101 - }, - "precision": { - "candidate": 0.9053435114503817, - "delta_percentage_points": -0.014429342766708597, - "parent": 0.9054878048780488 - }, - "recall": { - "candidate": 0.9025875190258752, - "delta_percentage_points": -0.15220700152206446, - "parent": 0.9041095890410958 - }, - "schema_valid_rate": { - "candidate": 0.9979224376731302, - "delta_percentage_points": 0.0, - "parent": 0.9979224376731302 - } - }, - "parent_correct": 1191, - "requests": 1444 - }, - "domain/AddToPlaylist": { - "candidate_correct": 165, - "error_counts": { - "false_abstention": { - "candidate": 12, - "parent": 13 - }, - "json_parse_ok": { - "candidate": 206, - "parent": 206 - }, - "missed_abstention": { - "candidate": 20, - "parent": 21 - }, - "schema_ok": { - "candidate": 206, - "parent": 206 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 15, - "parent": 13 - } - }, - "metrics": { - "absent_accuracy": { - "candidate": 0.9152542372881356, - "delta_percentage_points": 0.4237288135593209, - "parent": 0.9110169491525424 - }, - "exact": { - "candidate": 0.8009708737864077, - "delta_percentage_points": -1.4563106796116498, - "parent": 0.8155339805825242 - }, - "f1": { - "candidate": 0.8741496598639455, - "delta_percentage_points": -1.0204081632653184, - "parent": 0.8843537414965987 - }, - "field_accuracy": { - "candidate": 0.8992395437262357, - "delta_percentage_points": -0.38022813688213253, - "parent": 0.903041825095057 - }, - "precision": { - "candidate": 0.8624161073825504, - "delta_percentage_points": -1.0067114093959662, - "parent": 0.87248322147651 - }, - "recall": { - "candidate": 0.8862068965517241, - "delta_percentage_points": -1.0344827586206917, - "parent": 0.896551724137931 - }, - "schema_valid_rate": { - "candidate": 1.0, - "delta_percentage_points": 0.0, - "parent": 1.0 - } - }, - "parent_correct": 168, - "requests": 206 - }, - "domain/BookRestaurant": { - "candidate_correct": 162, - "error_counts": { - "false_abstention": { - "candidate": 16, - "parent": 21 - }, - "json_parse_ok": { - "candidate": 208, - "parent": 208 - }, - "missed_abstention": { - "candidate": 23, - "parent": 23 - }, - "schema_ok": { - "candidate": 208, - "parent": 208 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 3, - "parent": 4 - } - }, - "metrics": { - "absent_accuracy": { - "candidate": 0.908, - "delta_percentage_points": 0.0, - "parent": 0.908 - }, - "exact": { - "candidate": 0.7788461538461539, - "delta_percentage_points": 2.8846153846153855, - "parent": 0.75 - }, - "f1": { - "candidate": 0.8775193798449612, - "delta_percentage_points": 0.5644379844961134, - "parent": 0.8718750000000001 - }, - "field_accuracy": { - "candidate": 0.8963093145869947, - "delta_percentage_points": 0.7029876977152849, - "parent": 0.8892794376098418 - }, - "precision": { - "candidate": 0.8680981595092024, - "delta_percentage_points": -0.10607189954704443, - "parent": 0.8691588785046729 - }, - "recall": { - "candidate": 0.8871473354231975, - "delta_percentage_points": 1.253918495297801, - "parent": 0.8746081504702194 - }, - "schema_valid_rate": { - "candidate": 1.0, - "delta_percentage_points": 0.0, - "parent": 1.0 - } - }, - "parent_correct": 156, - "requests": 208 - }, - "domain/GetWeather": { - "candidate_correct": 164, - "error_counts": { - "false_abstention": { - "candidate": 15, - "parent": 15 - }, - "json_parse_ok": { - "candidate": 205, - "parent": 205 - }, - "missed_abstention": { - "candidate": 20, - "parent": 16 - }, - "schema_ok": { - "candidate": 204, - "parent": 204 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 3, - "parent": 2 - } - }, - "metrics": { - "absent_accuracy": { - "candidate": 0.9186991869918699, - "delta_percentage_points": -1.6260162601625994, - "parent": 0.9349593495934959 - }, - "exact": { - "candidate": 0.7961165048543689, - "delta_percentage_points": -1.9417475728155331, - "parent": 0.8155339805825242 - }, - "f1": { - "candidate": 0.8853046594982079, - "delta_percentage_points": -1.1992637799089434, - "parent": 0.8972972972972973 - }, - "field_accuracy": { - "candidate": 0.900952380952381, - "delta_percentage_points": -1.1428571428571455, - "parent": 0.9123809523809524 - }, - "precision": { - "candidate": 0.8853046594982079, - "delta_percentage_points": -1.6869253545270357, - "parent": 0.9021739130434783 - }, - "recall": { - "candidate": 0.8853046594982079, - "delta_percentage_points": -0.7168458781361964, - "parent": 0.8924731182795699 - }, - "schema_valid_rate": { - "candidate": 0.9902912621359223, - "delta_percentage_points": 0.0, - "parent": 0.9902912621359223 - } - }, - "parent_correct": 168, - "requests": 206 - }, - "domain/PlayMusic": { - "candidate_correct": 153, - "error_counts": { - "false_abstention": { - "candidate": 45, - "parent": 44 - }, - "json_parse_ok": { - "candidate": 207, - "parent": 208 - }, - "missed_abstention": { - "candidate": 15, - "parent": 15 - }, - "schema_ok": { - "candidate": 207, - "parent": 208 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 1, - "parent": 2 - } - }, - "metrics": { - "absent_accuracy": { - "candidate": 0.94140625, - "delta_percentage_points": 0.0, - "parent": 0.94140625 - }, - "exact": { - "candidate": 0.7355769230769231, - "delta_percentage_points": 1.9230769230769273, - "parent": 0.7163461538461539 - }, - "f1": { - "candidate": 0.8320000000000001, - "delta_percentage_points": 0.031683168316842814, - "parent": 0.8316831683168316 - }, - "field_accuracy": { - "candidate": 0.858508604206501, - "delta_percentage_points": -0.3824091778202643, - "parent": 0.8623326959847036 - }, - "precision": { - "candidate": 0.8927038626609443, - "delta_percentage_points": 1.0350921484473696, - "parent": 0.8823529411764706 - }, - "recall": { - "candidate": 0.7790262172284644, - "delta_percentage_points": -0.7490636704119868, - "parent": 0.7865168539325843 - }, - "schema_valid_rate": { - "candidate": 0.9951923076923077, - "delta_percentage_points": -0.48076923076922906, - "parent": 1.0 - } - }, - "parent_correct": 149, - "requests": 208 - }, - "domain/RateBook": { - "candidate_correct": 193, - "error_counts": { - "false_abstention": { - "candidate": 3, - "parent": 2 - }, - "json_parse_ok": { - "candidate": 204, - "parent": 203 - }, - "missed_abstention": { - "candidate": 5, - "parent": 11 - }, - "schema_ok": { - "candidate": 204, - "parent": 203 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 1, - "parent": 2 - } - }, - "metrics": { - "absent_accuracy": { - "candidate": 0.9796747967479674, - "delta_percentage_points": 2.4390243902438935, - "parent": 0.9552845528455285 - }, - "exact": { - "candidate": 0.946078431372549, - "delta_percentage_points": 2.450980392156865, - "parent": 0.9215686274509803 - }, - "f1": { - "candidate": 0.9782608695652175, - "delta_percentage_points": 1.2211486849168174, - "parent": 0.9660493827160493 - }, - "field_accuracy": { - "candidate": 0.9805996472663139, - "delta_percentage_points": 1.4109347442680775, - "parent": 0.9664902998236331 - }, - "precision": { - "candidate": 0.9752321981424149, - "delta_percentage_points": 1.8045653799907302, - "parent": 0.9571865443425076 - }, - "recall": { - "candidate": 0.9813084112149533, - "delta_percentage_points": 0.6230529595015577, - "parent": 0.9750778816199377 - }, - "schema_valid_rate": { - "candidate": 1.0, - "delta_percentage_points": 0.4901960784313708, - "parent": 0.9950980392156863 - } - }, - "parent_correct": 188, - "requests": 204 }, - "domain/SearchCreativeWork": { - "candidate_correct": 197, - "error_counts": { - "false_abstention": { - "candidate": 2, - "parent": 2 - }, - "json_parse_ok": { - "candidate": 208, - "parent": 208 - }, - "missed_abstention": { - "candidate": 2, - "parent": 3 - }, - "schema_ok": { - "candidate": 208, - "parent": 208 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 0, - "parent": 0 - } + "improved_requests": 96, + "regressed_requests": 24 + }, + "playground_60": { + "metrics": { + "all": { + "absent_accuracy": 0.9322033898305084, + "count": 60, + "counts": { + "absent_correct": 55, + "absent_total": 59, + "exact": 49, + "false_abstention": 2, + "field_correct": 136, + "fields": 147, + "fn": 9, + "fp": 10, + "json_parse_ok": 60, + "missed_abstention": 4, + "natural_stop": 60, + "nonrepetitive_nonempty": 60, + "predicted_values": 109, + "present_correct": 81, + "present_total": 88, + "schema_ok": 60, + "token_limited": 0, + "tp": 99, + "unsupported_values": 6 + }, + "exact": 0.8166666666666667, + "f1": 0.912442396313364, + "false_abstention_rate": 0.022727272727272728, + "field_accuracy": 0.9251700680272109, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.06779661016949153, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.046341597845215195, + "precision": 0.908256880733945, + "present_accuracy": 0.9204545454545454, + "recall": 0.9166666666666666, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.05504587155963303 }, - "metrics": { - "absent_accuracy": { - "candidate": 0.9487179487179487, - "delta_percentage_points": 2.564102564102555, - "parent": 0.9230769230769231 - }, - "exact": { - "candidate": 0.9471153846153846, - "delta_percentage_points": 0.0, - "parent": 0.9471153846153846 - }, - "f1": { - "candidate": 0.9635627530364372, - "delta_percentage_points": -0.20938126201284213, - "parent": 0.9656565656565657 - }, - "field_accuracy": { - "candidate": 0.9615384615384616, - "delta_percentage_points": 0.0, - "parent": 0.9615384615384616 - }, - "precision": { - "candidate": 0.9635627530364372, - "delta_percentage_points": -0.014692438291763477, - "parent": 0.9637096774193549 - }, - "recall": { - "candidate": 0.9635627530364372, - "delta_percentage_points": -0.4048582995951455, - "parent": 0.9676113360323887 - }, - "schema_valid_rate": { - "candidate": 1.0, - "delta_percentage_points": 0.0, - "parent": 1.0 - } + "domain/playground": { + "absent_accuracy": 0.9322033898305084, + "count": 60, + "counts": { + "absent_correct": 55, + "absent_total": 59, + "exact": 49, + "false_abstention": 2, + "field_correct": 136, + "fields": 147, + "fn": 9, + "fp": 10, + "json_parse_ok": 60, + "missed_abstention": 4, + "natural_stop": 60, + "nonrepetitive_nonempty": 60, + "predicted_values": 109, + "present_correct": 81, + "present_total": 88, + "schema_ok": 60, + "token_limited": 0, + "tp": 99, + "unsupported_values": 6 + }, + "exact": 0.8166666666666667, + "f1": 0.912442396313364, + "false_abstention_rate": 0.022727272727272728, + "field_accuracy": 0.9251700680272109, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.06779661016949153, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.046341597845215195, + "precision": 0.908256880733945, + "present_accuracy": 0.9204545454545454, + "recall": 0.9166666666666666, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.05504587155963303 }, - "parent_correct": 197, - "requests": 208 - }, - "domain/SearchScreeningEvent": { - "candidate_correct": 165, - "error_counts": { - "false_abstention": { - "candidate": 14, - "parent": 14 - }, - "json_parse_ok": { - "candidate": 204, - "parent": 204 - }, - "missed_abstention": { - "candidate": 25, - "parent": 26 - }, - "schema_ok": { - "candidate": 204, - "parent": 204 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 4, - "parent": 7 - } + "macro_exact": 0.8166666666666667, + "paired_requests": { + "accuracy": 0.0, + "both_correct": 0, + "count": 0 }, - "metrics": { - "absent_accuracy": { - "candidate": 0.9011857707509882, - "delta_percentage_points": 0.39525691699604515, - "parent": 0.8972332015810277 - }, - "exact": { - "candidate": 0.8088235294117647, - "delta_percentage_points": 0.0, - "parent": 0.8088235294117647 - }, - "f1": { - "candidate": 0.9112426035502958, - "delta_percentage_points": -0.21432232213577374, - "parent": 0.9133858267716536 - }, - "field_accuracy": { - "candidate": 0.9161676646706587, - "delta_percentage_points": 0.0, - "parent": 0.9161676646706587 - }, - "precision": { - "candidate": 0.8918918918918919, - "delta_percentage_points": -0.04158004158004713, - "parent": 0.8923076923076924 - }, - "recall": { - "candidate": 0.9314516129032258, - "delta_percentage_points": -0.40322580645161255, - "parent": 0.9354838709677419 - }, - "schema_valid_rate": { - "candidate": 1.0, - "delta_percentage_points": 0.0, - "parent": 1.0 - } + "panel/playground60": { + "absent_accuracy": 0.9322033898305084, + "count": 60, + "counts": { + "absent_correct": 55, + "absent_total": 59, + "exact": 49, + "false_abstention": 2, + "field_correct": 136, + "fields": 147, + "fn": 9, + "fp": 10, + "json_parse_ok": 60, + "missed_abstention": 4, + "natural_stop": 60, + "nonrepetitive_nonempty": 60, + "predicted_values": 109, + "present_correct": 81, + "present_total": 88, + "schema_ok": 60, + "token_limited": 0, + "tp": 99, + "unsupported_values": 6 + }, + "exact": 0.8166666666666667, + "f1": 0.912442396313364, + "false_abstention_rate": 0.022727272727272728, + "field_accuracy": 0.9251700680272109, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.06779661016949153, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.046341597845215195, + "precision": 0.908256880733945, + "present_accuracy": 0.9204545454545454, + "recall": 0.9166666666666666, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.05504587155963303 }, - "parent_correct": 165, - "requests": 204 + "variant/canonical": { + "absent_accuracy": 0.9322033898305084, + "count": 60, + "counts": { + "absent_correct": 55, + "absent_total": 59, + "exact": 49, + "false_abstention": 2, + "field_correct": 136, + "fields": 147, + "fn": 9, + "fp": 10, + "json_parse_ok": 60, + "missed_abstention": 4, + "natural_stop": 60, + "nonrepetitive_nonempty": 60, + "predicted_values": 109, + "present_correct": 81, + "present_total": 88, + "schema_ok": 60, + "token_limited": 0, + "tp": 99, + "unsupported_values": 6 + }, + "exact": 0.8166666666666667, + "f1": 0.912442396313364, + "false_abstention_rate": 0.022727272727272728, + "field_accuracy": 0.9251700680272109, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.06779661016949153, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.046341597845215195, + "precision": 0.908256880733945, + "present_accuracy": 0.9204545454545454, + "recall": 0.9166666666666666, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.05504587155963303 + } }, - "variant/alias": { - "candidate_correct": 21, - "error_counts": { - "false_abstention": { - "candidate": 1, - "parent": 2 - }, - "json_parse_ok": { - "candidate": 26, - "parent": 27 - }, - "missed_abstention": { - "candidate": 6, - "parent": 5 - }, - "schema_ok": { - "candidate": 26, - "parent": 26 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 2, - "parent": 2 - } + "previous_metrics": { + "all": { + "absent_accuracy": 0.8983050847457628, + "count": 60, + "counts": { + "absent_correct": 53, + "absent_total": 59, + "exact": 45, + "false_abstention": 3, + "field_correct": 129, + "fields": 147, + "fn": 22, + "fp": 13, + "json_parse_ok": 58, + "missed_abstention": 6, + "natural_stop": 60, + "nonrepetitive_nonempty": 60, + "predicted_values": 99, + "present_correct": 76, + "present_total": 88, + "schema_ok": 58, + "token_limited": 0, + "tp": 86, + "unsupported_values": 7 + }, + "exact": 0.75, + "f1": 0.8309178743961353, + "false_abstention_rate": 0.03409090909090909, + "field_accuracy": 0.8775510204081632, + "json_parse_rate": 0.9666666666666667, + "missed_abstention_rate": 0.1016949152542373, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.07145761149877217, + "precision": 0.8686868686868687, + "present_accuracy": 0.8636363636363636, + "recall": 0.7962962962962963, + "schema_valid_rate": 0.9666666666666667, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.0707070707070707 }, - "metrics": { - "absent_accuracy": { - "candidate": 0.7931034482758621, - "delta_percentage_points": -3.4482758620689613, - "parent": 0.8275862068965517 - }, - "exact": { - "candidate": 0.75, - "delta_percentage_points": 7.14285714285714, - "parent": 0.6785714285714286 - }, - "f1": { - "candidate": 0.853932584269663, - "delta_percentage_points": 1.8767749104827813, - "parent": 0.8351648351648352 - }, - "field_accuracy": { - "candidate": 0.8133333333333334, - "delta_percentage_points": -1.3333333333333308, - "parent": 0.8266666666666667 - }, - "precision": { - "candidate": 0.8837209302325582, - "delta_percentage_points": 3.9276485788113713, - "parent": 0.8444444444444444 - }, - "recall": { - "candidate": 0.8260869565217391, - "delta_percentage_points": 0.0, - "parent": 0.8260869565217391 - }, - "schema_valid_rate": { - "candidate": 0.9285714285714286, - "delta_percentage_points": 0.0, - "parent": 0.9285714285714286 - } + "domain/playground": { + "absent_accuracy": 0.8983050847457628, + "count": 60, + "counts": { + "absent_correct": 53, + "absent_total": 59, + "exact": 45, + "false_abstention": 3, + "field_correct": 129, + "fields": 147, + "fn": 22, + "fp": 13, + "json_parse_ok": 58, + "missed_abstention": 6, + "natural_stop": 60, + "nonrepetitive_nonempty": 60, + "predicted_values": 99, + "present_correct": 76, + "present_total": 88, + "schema_ok": 58, + "token_limited": 0, + "tp": 86, + "unsupported_values": 7 + }, + "exact": 0.75, + "f1": 0.8309178743961353, + "false_abstention_rate": 0.03409090909090909, + "field_accuracy": 0.8775510204081632, + "json_parse_rate": 0.9666666666666667, + "missed_abstention_rate": 0.1016949152542373, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.07145761149877217, + "precision": 0.8686868686868687, + "present_accuracy": 0.8636363636363636, + "recall": 0.7962962962962963, + "schema_valid_rate": 0.9666666666666667, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.0707070707070707 }, - "parent_correct": 19, - "requests": 28 - }, - "variant/canonical": { - "candidate_correct": 1163, - "error_counts": { - "false_abstention": { - "candidate": 100, - "parent": 103 - }, - "json_parse_ok": { - "candidate": 1388, - "parent": 1388 - }, - "missed_abstention": { - "candidate": 97, - "parent": 102 - }, - "schema_ok": { - "candidate": 1388, - "parent": 1388 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 23, - "parent": 24 - } + "macro_exact": 0.75, + "paired_requests": { + "accuracy": 0.0, + "both_correct": 0, + "count": 0 }, - "metrics": { - "absent_accuracy": { - "candidate": 0.9339237057220708, - "delta_percentage_points": 0.34059945504086864, - "parent": 0.9305177111716622 - }, - "exact": { - "candidate": 0.8378962536023055, - "delta_percentage_points": 0.5043227665706085, - "parent": 0.8328530259365994 - }, - "f1": { - "candidate": 0.9075898801597869, - "delta_percentage_points": -0.1646212467309116, - "parent": 0.909236092627096 - }, - "field_accuracy": { - "candidate": 0.9187331939049895, - "delta_percentage_points": 0.02987750224081598, - "parent": 0.9184344188825814 - }, - "precision": { - "candidate": 0.908315565031983, - "delta_percentage_points": -0.11626032321278013, - "parent": 0.9094781682641108 - }, - "recall": { - "candidate": 0.9068653539116551, - "delta_percentage_points": -0.21287919105907882, - "parent": 0.9089941458222459 - }, - "schema_valid_rate": { - "candidate": 1.0, - "delta_percentage_points": 0.0, - "parent": 1.0 - } + "panel/playground60": { + "absent_accuracy": 0.8983050847457628, + "count": 60, + "counts": { + "absent_correct": 53, + "absent_total": 59, + "exact": 45, + "false_abstention": 3, + "field_correct": 129, + "fields": 147, + "fn": 22, + "fp": 13, + "json_parse_ok": 58, + "missed_abstention": 6, + "natural_stop": 60, + "nonrepetitive_nonempty": 60, + "predicted_values": 99, + "present_correct": 76, + "present_total": 88, + "schema_ok": 58, + "token_limited": 0, + "tp": 86, + "unsupported_values": 7 + }, + "exact": 0.75, + "f1": 0.8309178743961353, + "false_abstention_rate": 0.03409090909090909, + "field_accuracy": 0.8775510204081632, + "json_parse_rate": 0.9666666666666667, + "missed_abstention_rate": 0.1016949152542373, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.07145761149877217, + "precision": 0.8686868686868687, + "present_accuracy": 0.8636363636363636, + "recall": 0.7962962962962963, + "schema_valid_rate": 0.9666666666666667, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.0707070707070707 }, - "parent_correct": 1156, - "requests": 1388 + "variant/canonical": { + "absent_accuracy": 0.8983050847457628, + "count": 60, + "counts": { + "absent_correct": 53, + "absent_total": 59, + "exact": 45, + "false_abstention": 3, + "field_correct": 129, + "fields": 147, + "fn": 22, + "fp": 13, + "json_parse_ok": 58, + "missed_abstention": 6, + "natural_stop": 60, + "nonrepetitive_nonempty": 60, + "predicted_values": 99, + "present_correct": 76, + "present_total": 88, + "schema_ok": 58, + "token_limited": 0, + "tp": 86, + "unsupported_values": 7 + }, + "exact": 0.75, + "f1": 0.8309178743961353, + "false_abstention_rate": 0.03409090909090909, + "field_accuracy": 0.8775510204081632, + "json_parse_rate": 0.9666666666666667, + "missed_abstention_rate": 0.1016949152542373, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.07145761149877217, + "precision": 0.8686868686868687, + "present_accuracy": 0.8636363636363636, + "recall": 0.7962962962962963, + "schema_valid_rate": 0.9666666666666667, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.0707070707070707 + } }, - "variant/nullable": { - "candidate_correct": 15, - "error_counts": { - "false_abstention": { - "candidate": 6, - "parent": 6 - }, - "json_parse_ok": { - "candidate": 28, - "parent": 27 - }, - "missed_abstention": { - "candidate": 7, - "parent": 8 - }, - "schema_ok": { - "candidate": 27, - "parent": 27 - }, - "token_limited": { - "candidate": 0, - "parent": 0 - }, - "unsupported_values": { - "candidate": 2, - "parent": 4 - } - }, - "metrics": { - "absent_accuracy": { - "candidate": 0.7586206896551724, - "delta_percentage_points": 3.4482758620689613, - "parent": 0.7241379310344828 - }, - "exact": { - "candidate": 0.5357142857142857, - "delta_percentage_points": -3.57142857142857, - "parent": 0.5714285714285714 - }, - "f1": { - "candidate": 0.8043478260869565, - "delta_percentage_points": 1.3139034878165368, - "parent": 0.7912087912087912 - }, - "field_accuracy": { - "candidate": 0.7866666666666666, - "delta_percentage_points": 2.6666666666666616, - "parent": 0.76 - }, - "precision": { - "candidate": 0.8043478260869565, - "delta_percentage_points": 0.43478260869564966, - "parent": 0.8 - }, - "recall": { - "candidate": 0.8043478260869565, - "delta_percentage_points": 2.1739130434782594, - "parent": 0.782608695652174 - }, - "schema_valid_rate": { - "candidate": 0.9642857142857143, - "delta_percentage_points": 0.0, - "parent": 0.9642857142857143 + "subgroups": { + "original1444": { + "parent105_64": {}, + "replay128_200": {}, + "replay128_300": {} + }, + "playground60": { + "parent105_64": { + "added6": { + "absent_accuracy": 0.6666666666666666, + "count": 6, + "counts": { + "absent_correct": 4, + "absent_total": 6, + "exact": 1, + "false_abstention": 0, + "field_correct": 5, + "fields": 12, + "fn": 15, + "fp": 4, + "json_parse_ok": 4, + "missed_abstention": 2, + "natural_stop": 6, + "nonrepetitive_nonempty": 6, + "predicted_values": 13, + "present_correct": 1, + "present_total": 6, + "schema_ok": 4, + "token_limited": 0, + "tp": 9, + "unsupported_values": 4 + }, + "exact": 0.16666666666666666, + "f1": 0.48648648648648646, + "false_abstention_rate": 0.0, + "field_accuracy": 0.4166666666666667, + "json_parse_rate": 0.6666666666666666, + "missed_abstention_rate": 0.3333333333333333, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.16294185755153498, + "precision": 0.6923076923076923, + "present_accuracy": 0.16666666666666666, + "recall": 0.375, + "schema_valid_rate": 0.6666666666666666, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.3076923076923077 + }, + "array3": { + "absent_accuracy": 1.0, + "count": 2, + "counts": { + "absent_correct": 2, + "absent_total": 2, + "exact": 1, + "false_abstention": 0, + "field_correct": 3, + "fields": 4, + "fn": 1, + "fp": 1, + "json_parse_ok": 2, + "missed_abstention": 0, + "natural_stop": 2, + "nonrepetitive_nonempty": 2, + "predicted_values": 6, + "present_correct": 1, + "present_total": 2, + "schema_ok": 2, + "token_limited": 0, + "tp": 5, + "unsupported_values": 1 + }, + "exact": 0.5, + "f1": 0.8333333333333334, + "false_abstention_rate": 0.0, + "field_accuracy": 0.75, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.12317727319896221, + "precision": 0.8333333333333334, + "present_accuracy": 0.5, + "recall": 0.8333333333333334, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.16666666666666666 + }, + "array4": { + "absent_accuracy": 0.5, + "count": 2, + "counts": { + "absent_correct": 1, + "absent_total": 2, + "exact": 0, + "false_abstention": 0, + "field_correct": 1, + "fields": 4, + "fn": 6, + "fp": 2, + "json_parse_ok": 1, + "missed_abstention": 1, + "natural_stop": 2, + "nonrepetitive_nonempty": 2, + "predicted_values": 4, + "present_correct": 0, + "present_total": 2, + "schema_ok": 1, + "token_limited": 0, + "tp": 2, + "unsupported_values": 2 + }, + "exact": 0.0, + "f1": 0.3333333333333333, + "false_abstention_rate": 0.0, + "field_accuracy": 0.25, + "json_parse_rate": 0.5, + "missed_abstention_rate": 0.5, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.1759970560669899, + "precision": 0.5, + "present_accuracy": 0.0, + "recall": 0.25, + "schema_valid_rate": 0.5, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.5 + }, + "array5": { + "absent_accuracy": 0.5, + "count": 2, + "counts": { + "absent_correct": 1, + "absent_total": 2, + "exact": 0, + "false_abstention": 0, + "field_correct": 1, + "fields": 4, + "fn": 8, + "fp": 1, + "json_parse_ok": 1, + "missed_abstention": 1, + "natural_stop": 2, + "nonrepetitive_nonempty": 2, + "predicted_values": 3, + "present_correct": 0, + "present_total": 2, + "schema_ok": 1, + "token_limited": 0, + "tp": 2, + "unsupported_values": 1 + }, + "exact": 0.0, + "f1": 0.30769230769230765, + "false_abstention_rate": 0.0, + "field_accuracy": 0.25, + "json_parse_rate": 0.5, + "missed_abstention_rate": 0.5, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.1896512433886528, + "precision": 0.6666666666666666, + "present_accuracy": 0.0, + "recall": 0.2, + "schema_valid_rate": 0.5, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.3333333333333333 + }, + "original54": { + "absent_accuracy": 0.9245283018867925, + "count": 54, + "counts": { + "absent_correct": 49, + "absent_total": 53, + "exact": 44, + "false_abstention": 3, + "field_correct": 124, + "fields": 135, + "fn": 7, + "fp": 9, + "json_parse_ok": 54, + "missed_abstention": 4, + "natural_stop": 54, + "nonrepetitive_nonempty": 54, + "predicted_values": 86, + "present_correct": 75, + "present_total": 82, + "schema_ok": 54, + "token_limited": 0, + "tp": 77, + "unsupported_values": 3 + }, + "exact": 0.8148148148148148, + "f1": 0.9058823529411765, + "false_abstention_rate": 0.036585365853658534, + "field_accuracy": 0.9185185185185185, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.07547169811320754, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.06129269527068741, + "precision": 0.8953488372093024, + "present_accuracy": 0.9146341463414634, + "recall": 0.9166666666666666, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.03488372093023256 + } + }, + "replay128_200": { + "added6": { + "absent_accuracy": 1.0, + "count": 6, + "counts": { + "absent_correct": 6, + "absent_total": 6, + "exact": 3, + "false_abstention": 0, + "field_correct": 9, + "fields": 12, + "fn": 5, + "fp": 3, + "json_parse_ok": 6, + "missed_abstention": 0, + "natural_stop": 6, + "nonrepetitive_nonempty": 6, + "predicted_values": 22, + "present_correct": 3, + "present_total": 6, + "schema_ok": 6, + "token_limited": 0, + "tp": 19, + "unsupported_values": 3 + }, + "exact": 0.5, + "f1": 0.8260869565217391, + "false_abstention_rate": 0.0, + "field_accuracy": 0.75, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.10507260775193572, + "precision": 0.8636363636363636, + "present_accuracy": 0.5, + "recall": 0.7916666666666666, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.13636363636363635 + }, + "array3": { + "absent_accuracy": 1.0, + "count": 2, + "counts": { + "absent_correct": 2, + "absent_total": 2, + "exact": 1, + "false_abstention": 0, + "field_correct": 3, + "fields": 4, + "fn": 1, + "fp": 1, + "json_parse_ok": 2, + "missed_abstention": 0, + "natural_stop": 2, + "nonrepetitive_nonempty": 2, + "predicted_values": 6, + "present_correct": 1, + "present_total": 2, + "schema_ok": 2, + "token_limited": 0, + "tp": 5, + "unsupported_values": 1 + }, + "exact": 0.5, + "f1": 0.8333333333333334, + "false_abstention_rate": 0.0, + "field_accuracy": 0.75, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.0854416680522263, + "precision": 0.8333333333333334, + "present_accuracy": 0.5, + "recall": 0.8333333333333334, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.16666666666666666 + }, + "array4": { + "absent_accuracy": 1.0, + "count": 2, + "counts": { + "absent_correct": 2, + "absent_total": 2, + "exact": 1, + "false_abstention": 0, + "field_correct": 3, + "fields": 4, + "fn": 1, + "fp": 1, + "json_parse_ok": 2, + "missed_abstention": 0, + "natural_stop": 2, + "nonrepetitive_nonempty": 2, + "predicted_values": 8, + "present_correct": 1, + "present_total": 2, + "schema_ok": 2, + "token_limited": 0, + "tp": 7, + "unsupported_values": 1 + }, + "exact": 0.5, + "f1": 0.875, + "false_abstention_rate": 0.0, + "field_accuracy": 0.75, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.1231730468571186, + "precision": 0.875, + "present_accuracy": 0.5, + "recall": 0.875, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.125 + }, + "array5": { + "absent_accuracy": 1.0, + "count": 2, + "counts": { + "absent_correct": 2, + "absent_total": 2, + "exact": 1, + "false_abstention": 0, + "field_correct": 3, + "fields": 4, + "fn": 3, + "fp": 1, + "json_parse_ok": 2, + "missed_abstention": 0, + "natural_stop": 2, + "nonrepetitive_nonempty": 2, + "predicted_values": 8, + "present_correct": 1, + "present_total": 2, + "schema_ok": 2, + "token_limited": 0, + "tp": 7, + "unsupported_values": 1 + }, + "exact": 0.5, + "f1": 0.7777777777777777, + "false_abstention_rate": 0.0, + "field_accuracy": 0.75, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.10660310834646225, + "precision": 0.875, + "present_accuracy": 0.5, + "recall": 0.7, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.125 + }, + "original54": { + "absent_accuracy": 0.9056603773584906, + "count": 54, + "counts": { + "absent_correct": 48, + "absent_total": 53, + "exact": 45, + "false_abstention": 1, + "field_correct": 126, + "fields": 135, + "fn": 4, + "fp": 9, + "json_parse_ok": 54, + "missed_abstention": 5, + "natural_stop": 54, + "nonrepetitive_nonempty": 54, + "predicted_values": 89, + "present_correct": 78, + "present_total": 82, + "schema_ok": 54, + "token_limited": 0, + "tp": 80, + "unsupported_values": 3 + }, + "exact": 0.8333333333333334, + "f1": 0.9248554913294796, + "false_abstention_rate": 0.012195121951219513, + "field_accuracy": 0.9333333333333333, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.09433962264150944, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.038012701103407716, + "precision": 0.898876404494382, + "present_accuracy": 0.9512195121951219, + "recall": 0.9523809523809523, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.033707865168539325 + } + }, + "replay128_300": { + "added6": { + "absent_accuracy": 1.0, + "count": 6, + "counts": { + "absent_correct": 6, + "absent_total": 6, + "exact": 3, + "false_abstention": 0, + "field_correct": 9, + "fields": 12, + "fn": 5, + "fp": 3, + "json_parse_ok": 6, + "missed_abstention": 0, + "natural_stop": 6, + "nonrepetitive_nonempty": 6, + "predicted_values": 22, + "present_correct": 3, + "present_total": 6, + "schema_ok": 6, + "token_limited": 0, + "tp": 19, + "unsupported_values": 3 + }, + "exact": 0.5, + "f1": 0.8260869565217391, + "false_abstention_rate": 0.0, + "field_accuracy": 0.75, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.10285404759148757, + "precision": 0.8636363636363636, + "present_accuracy": 0.5, + "recall": 0.7916666666666666, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.13636363636363635 + }, + "array3": { + "absent_accuracy": 1.0, + "count": 2, + "counts": { + "absent_correct": 2, + "absent_total": 2, + "exact": 1, + "false_abstention": 0, + "field_correct": 3, + "fields": 4, + "fn": 1, + "fp": 1, + "json_parse_ok": 2, + "missed_abstention": 0, + "natural_stop": 2, + "nonrepetitive_nonempty": 2, + "predicted_values": 6, + "present_correct": 1, + "present_total": 2, + "schema_ok": 2, + "token_limited": 0, + "tp": 5, + "unsupported_values": 1 + }, + "exact": 0.5, + "f1": 0.8333333333333334, + "false_abstention_rate": 0.0, + "field_accuracy": 0.75, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.08301506005227566, + "precision": 0.8333333333333334, + "present_accuracy": 0.5, + "recall": 0.8333333333333334, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.16666666666666666 + }, + "array4": { + "absent_accuracy": 1.0, + "count": 2, + "counts": { + "absent_correct": 2, + "absent_total": 2, + "exact": 1, + "false_abstention": 0, + "field_correct": 3, + "fields": 4, + "fn": 1, + "fp": 1, + "json_parse_ok": 2, + "missed_abstention": 0, + "natural_stop": 2, + "nonrepetitive_nonempty": 2, + "predicted_values": 8, + "present_correct": 1, + "present_total": 2, + "schema_ok": 2, + "token_limited": 0, + "tp": 7, + "unsupported_values": 1 + }, + "exact": 0.5, + "f1": 0.875, + "false_abstention_rate": 0.0, + "field_accuracy": 0.75, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.1220800168812275, + "precision": 0.875, + "present_accuracy": 0.5, + "recall": 0.875, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.125 + }, + "array5": { + "absent_accuracy": 1.0, + "count": 2, + "counts": { + "absent_correct": 2, + "absent_total": 2, + "exact": 1, + "false_abstention": 0, + "field_correct": 3, + "fields": 4, + "fn": 3, + "fp": 1, + "json_parse_ok": 2, + "missed_abstention": 0, + "natural_stop": 2, + "nonrepetitive_nonempty": 2, + "predicted_values": 8, + "present_correct": 1, + "present_total": 2, + "schema_ok": 2, + "token_limited": 0, + "tp": 7, + "unsupported_values": 1 + }, + "exact": 0.5, + "f1": 0.7777777777777777, + "false_abstention_rate": 0.0, + "field_accuracy": 0.75, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.0, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.10346706584095955, + "precision": 0.875, + "present_accuracy": 0.5, + "recall": 0.7, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.125 + }, + "original54": { + "absent_accuracy": 0.9245283018867925, + "count": 54, + "counts": { + "absent_correct": 49, + "absent_total": 53, + "exact": 46, + "false_abstention": 2, + "field_correct": 127, + "fields": 135, + "fn": 4, + "fp": 7, + "json_parse_ok": 54, + "missed_abstention": 4, + "natural_stop": 54, + "nonrepetitive_nonempty": 54, + "predicted_values": 87, + "present_correct": 78, + "present_total": 82, + "schema_ok": 54, + "token_limited": 0, + "tp": 80, + "unsupported_values": 3 + }, + "exact": 0.8518518518518519, + "f1": 0.935672514619883, + "false_abstention_rate": 0.024390243902439025, + "field_accuracy": 0.9407407407407408, + "json_parse_rate": 1.0, + "missed_abstention_rate": 0.07547169811320754, + "natural_stop_rate": 1.0, + "nonrepetitive_nonempty_rate": 1.0, + "output_nll": 0.040062436762296046, + "precision": 0.9195402298850575, + "present_accuracy": 0.9512195121951219, + "recall": 0.9523809523809523, + "schema_valid_rate": 1.0, + "token_limited_rate": 0.0, + "unsupported_value_rate": 0.034482758620689655 + } } - }, - "parent_correct": 16, - "requests": 28 + } } }, - "historical_protocol_note": "Earlier extraction-plus-assembly scores are historical and do not describe these native JSON weights.", - "selection": { - "method": "Owner-selected main checkpoint after development/playground and observed original1444 matched comparison", - "strict_retention_gates_passed": false, - "reason": "Modest net complete-request gain with per-domain tradeoffs; unsupported values and nullable false abstentions exceeded some strict development gates. No uniform no-regression or broad reliability claim." + "protocol": { + "constrained_decoding": false, + "json_assembly": false, + "max_tokens": 256, + "one_generation_per_request": true, + "output_repair": false, + "prompt_truncation": false, + "repetition_penalty": 1, + "temperature": 0 + }, + "scope": "Previously observed development, SNIPS-derived held-out and playground research panels, not fresh blind external benchmarks. Source text, schema and fixed instruction only; gold values excluded. One raw generation; no JSON assembly, repair or constrained decoding.", + "selection": "Owner-selected aggregate improvement; strict zero-regression gate did not pass.", + "training_prefix": { + "records": 19200, + "serialized_positions": 6153194, + "supervised_assistant_positions": 633561, + "components": { + "original_json": 13640, + "new_prose": 4871, + "cardinality_replay": 689 + } } }