| { |
| "model": "OpenSML-JSONLite-175M-Instruct", |
| "checkpoint": "main", |
| "selected_stage": "joint JSON extraction SFT", |
| "step": 300, |
| "model_sha256": "781597baeb9b51d535fc27cd6f941dbc202e01d62c8d963587a3f3d565128560", |
| "output_mode": "native-json", |
| "development": { |
| "metrics": { |
| "all": { |
| "absent_accuracy": 0.9211356466876972, |
| "count": 240, |
| "counts": { |
| "absent_correct": 292, |
| "absent_total": 317, |
| "exact": 178, |
| "false_abstention": 25, |
| "field_correct": 632, |
| "fields": 714, |
| "fn": 65, |
| "fp": 47, |
| "json_parse_ok": 240, |
| "missed_abstention": 25, |
| "natural_stop": 240, |
| "nonrepetitive_nonempty": 240, |
| "predicted_values": 504, |
| "present_correct": 340, |
| "present_total": 397, |
| "schema_ok": 239, |
| "token_limited": 0, |
| "tp": 457, |
| "unsupported_values": 6 |
| }, |
| "exact": 0.7416666666666667, |
| "f1": 0.8908382066276802, |
| "false_abstention_rate": 0.06297229219143577, |
| "field_accuracy": 0.8851540616246498, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.07886435331230283, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.04828121321412861, |
| "precision": 0.9067460317460317, |
| "present_accuracy": 0.8564231738035264, |
| "recall": 0.8754789272030651, |
| "schema_valid_rate": 0.9958333333333333, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.011904761904761904 |
| }, |
| "domain/AddToPlaylist": { |
| "absent_accuracy": 0.8974358974358975, |
| "count": 28, |
| "counts": { |
| "absent_correct": 35, |
| "absent_total": 39, |
| "exact": 19, |
| "false_abstention": 1, |
| "field_correct": 81, |
| "fields": 92, |
| "fn": 7, |
| "fp": 10, |
| "json_parse_ok": 28, |
| "missed_abstention": 4, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 56, |
| "present_correct": 46, |
| "present_total": 53, |
| "schema_ok": 28, |
| "token_limited": 0, |
| "tp": 46, |
| "unsupported_values": 2 |
| }, |
| "exact": 0.6785714285714286, |
| "f1": 0.8440366972477065, |
| "false_abstention_rate": 0.018867924528301886, |
| "field_accuracy": 0.8804347826086957, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.10256410256410256, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.0638373876364702, |
| "precision": 0.8214285714285714, |
| "present_accuracy": 0.8679245283018868, |
| "recall": 0.8679245283018868, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.03571428571428571 |
| }, |
| "domain/BookRestaurant": { |
| "absent_accuracy": 0.9454545454545454, |
| "count": 28, |
| "counts": { |
| "absent_correct": 52, |
| "absent_total": 55, |
| "exact": 22, |
| "false_abstention": 1, |
| "field_correct": 99, |
| "fields": 105, |
| "fn": 3, |
| "fp": 5, |
| "json_parse_ok": 28, |
| "missed_abstention": 3, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 60, |
| "present_correct": 47, |
| "present_total": 50, |
| "schema_ok": 28, |
| "token_limited": 0, |
| "tp": 55, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.7857142857142857, |
| "f1": 0.9322033898305084, |
| "false_abstention_rate": 0.02, |
| "field_accuracy": 0.9428571428571428, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.05454545454545454, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.01643673108524776, |
| "precision": 0.9166666666666666, |
| "present_accuracy": 0.94, |
| "recall": 0.9482758620689655, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.016666666666666666 |
| }, |
| "domain/GetWeather": { |
| "absent_accuracy": 0.9090909090909091, |
| "count": 28, |
| "counts": { |
| "absent_correct": 30, |
| "absent_total": 33, |
| "exact": 20, |
| "false_abstention": 5, |
| "field_correct": 64, |
| "fields": 73, |
| "fn": 6, |
| "fp": 3, |
| "json_parse_ok": 28, |
| "missed_abstention": 3, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 59, |
| "present_correct": 34, |
| "present_total": 40, |
| "schema_ok": 28, |
| "token_limited": 0, |
| "tp": 56, |
| "unsupported_values": 0 |
| }, |
| "exact": 0.7142857142857143, |
| "f1": 0.9256198347107438, |
| "false_abstention_rate": 0.125, |
| "field_accuracy": 0.8767123287671232, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.09090909090909091, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.037961417684398056, |
| "precision": 0.9491525423728814, |
| "present_accuracy": 0.85, |
| "recall": 0.9032258064516129, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "domain/PlayMusic": { |
| "absent_accuracy": 0.8, |
| "count": 28, |
| "counts": { |
| "absent_correct": 28, |
| "absent_total": 35, |
| "exact": 19, |
| "false_abstention": 5, |
| "field_correct": 62, |
| "fields": 74, |
| "fn": 5, |
| "fp": 7, |
| "json_parse_ok": 28, |
| "missed_abstention": 7, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 56, |
| "present_correct": 34, |
| "present_total": 39, |
| "schema_ok": 28, |
| "token_limited": 0, |
| "tp": 49, |
| "unsupported_values": 0 |
| }, |
| "exact": 0.6785714285714286, |
| "f1": 0.8909090909090909, |
| "false_abstention_rate": 0.1282051282051282, |
| "field_accuracy": 0.8378378378378378, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.2, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.028358824200820112, |
| "precision": 0.875, |
| "present_accuracy": 0.8717948717948718, |
| "recall": 0.9074074074074074, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "domain/RateBook": { |
| "absent_accuracy": 0.9761904761904762, |
| "count": 28, |
| "counts": { |
| "absent_correct": 41, |
| "absent_total": 42, |
| "exact": 23, |
| "false_abstention": 2, |
| "field_correct": 104, |
| "fields": 110, |
| "fn": 5, |
| "fp": 4, |
| "json_parse_ok": 28, |
| "missed_abstention": 1, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 67, |
| "present_correct": 63, |
| "present_total": 68, |
| "schema_ok": 28, |
| "token_limited": 0, |
| "tp": 63, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.8214285714285714, |
| "f1": 0.9333333333333335, |
| "false_abstention_rate": 0.029411764705882353, |
| "field_accuracy": 0.9454545454545454, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.023809523809523808, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.010672787178918952, |
| "precision": 0.9402985074626866, |
| "present_accuracy": 0.9264705882352942, |
| "recall": 0.9264705882352942, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.014925373134328358 |
| }, |
| "domain/SearchCreativeWork": { |
| "absent_accuracy": 1.0, |
| "count": 28, |
| "counts": { |
| "absent_correct": 9, |
| "absent_total": 9, |
| "exact": 26, |
| "false_abstention": 0, |
| "field_correct": 43, |
| "fields": 45, |
| "fn": 2, |
| "fp": 2, |
| "json_parse_ok": 28, |
| "missed_abstention": 0, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 36, |
| "present_correct": 34, |
| "present_total": 36, |
| "schema_ok": 28, |
| "token_limited": 0, |
| "tp": 34, |
| "unsupported_values": 0 |
| }, |
| "exact": 0.9285714285714286, |
| "f1": 0.9444444444444444, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.9555555555555556, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.011042242548553207, |
| "precision": 0.9444444444444444, |
| "present_accuracy": 0.9444444444444444, |
| "recall": 0.9444444444444444, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "domain/SearchScreeningEvent": { |
| "absent_accuracy": 0.9454545454545454, |
| "count": 28, |
| "counts": { |
| "absent_correct": 52, |
| "absent_total": 55, |
| "exact": 23, |
| "false_abstention": 1, |
| "field_correct": 100, |
| "fields": 108, |
| "fn": 8, |
| "fp": 6, |
| "json_parse_ok": 28, |
| "missed_abstention": 3, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 64, |
| "present_correct": 48, |
| "present_total": 53, |
| "schema_ok": 28, |
| "token_limited": 0, |
| "tp": 58, |
| "unsupported_values": 0 |
| }, |
| "exact": 0.8214285714285714, |
| "f1": 0.8923076923076922, |
| "false_abstention_rate": 0.018867924528301886, |
| "field_accuracy": 0.9259259259259259, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.05454545454545454, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.018851709860817727, |
| "precision": 0.90625, |
| "present_accuracy": 0.9056603773584906, |
| "recall": 0.8787878787878788, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "domain/cardinality": { |
| "absent_accuracy": 1.0, |
| "count": 8, |
| "counts": { |
| "absent_correct": 8, |
| "absent_total": 8, |
| "exact": 8, |
| "false_abstention": 0, |
| "field_correct": 16, |
| "fields": 16, |
| "fn": 0, |
| "fp": 0, |
| "json_parse_ok": 8, |
| "missed_abstention": 0, |
| "natural_stop": 8, |
| "nonrepetitive_nonempty": 8, |
| "predicted_values": 12, |
| "present_correct": 8, |
| "present_total": 8, |
| "schema_ok": 8, |
| "token_limited": 0, |
| "tp": 12, |
| "unsupported_values": 0 |
| }, |
| "exact": 1.0, |
| "f1": 1.0, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 1.0, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.012686168280197307, |
| "precision": 1.0, |
| "present_accuracy": 1.0, |
| "recall": 1.0, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "domain/mixsnips": { |
| "absent_accuracy": 1.0, |
| "count": 8, |
| "counts": { |
| "absent_correct": 8, |
| "absent_total": 8, |
| "exact": 1, |
| "false_abstention": 4, |
| "field_correct": 12, |
| "fields": 24, |
| "fn": 14, |
| "fp": 4, |
| "json_parse_ok": 8, |
| "missed_abstention": 0, |
| "natural_stop": 8, |
| "nonrepetitive_nonempty": 8, |
| "predicted_values": 12, |
| "present_correct": 4, |
| "present_total": 16, |
| "schema_ok": 7, |
| "token_limited": 0, |
| "tp": 8, |
| "unsupported_values": 0 |
| }, |
| "exact": 0.125, |
| "f1": 0.4705882352941177, |
| "false_abstention_rate": 0.25, |
| "field_accuracy": 0.5, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.378448877716437, |
| "precision": 0.6666666666666666, |
| "present_accuracy": 0.25, |
| "recall": 0.36363636363636365, |
| "schema_valid_rate": 0.875, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "domain/observed_lists": { |
| "absent_accuracy": 1.0, |
| "count": 12, |
| "counts": { |
| "absent_correct": 12, |
| "absent_total": 12, |
| "exact": 9, |
| "false_abstention": 0, |
| "field_correct": 21, |
| "fields": 24, |
| "fn": 4, |
| "fp": 0, |
| "json_parse_ok": 12, |
| "missed_abstention": 0, |
| "natural_stop": 12, |
| "nonrepetitive_nonempty": 12, |
| "predicted_values": 49, |
| "present_correct": 9, |
| "present_total": 12, |
| "schema_ok": 12, |
| "token_limited": 0, |
| "tp": 49, |
| "unsupported_values": 0 |
| }, |
| "exact": 0.75, |
| "f1": 0.9607843137254902, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.875, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.04184045293368399, |
| "precision": 1.0, |
| "present_accuracy": 0.75, |
| "recall": 0.9245283018867925, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "domain/playground": { |
| "absent_accuracy": 1.0, |
| "count": 4, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 2, |
| "exact": 3, |
| "false_abstention": 0, |
| "field_correct": 8, |
| "fields": 9, |
| "fn": 3, |
| "fp": 1, |
| "json_parse_ok": 4, |
| "missed_abstention": 0, |
| "natural_stop": 4, |
| "nonrepetitive_nonempty": 4, |
| "predicted_values": 9, |
| "present_correct": 6, |
| "present_total": 7, |
| "schema_ok": 4, |
| "token_limited": 0, |
| "tp": 8, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.75, |
| "f1": 0.7999999999999999, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.8888888888888888, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.03125538246240467, |
| "precision": 0.8888888888888888, |
| "present_accuracy": 0.8571428571428571, |
| "recall": 0.7272727272727273, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.1111111111111111 |
| }, |
| "domain/roles": { |
| "absent_accuracy": 0.8181818181818182, |
| "count": 8, |
| "counts": { |
| "absent_correct": 9, |
| "absent_total": 11, |
| "exact": 5, |
| "false_abstention": 2, |
| "field_correct": 16, |
| "fields": 22, |
| "fn": 4, |
| "fp": 3, |
| "json_parse_ok": 8, |
| "missed_abstention": 2, |
| "natural_stop": 8, |
| "nonrepetitive_nonempty": 8, |
| "predicted_values": 22, |
| "present_correct": 7, |
| "present_total": 11, |
| "schema_ok": 8, |
| "token_limited": 0, |
| "tp": 19, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.625, |
| "f1": 0.8444444444444444, |
| "false_abstention_rate": 0.18181818181818182, |
| "field_accuracy": 0.7272727272727273, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.18181818181818182, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.21559197976603173, |
| "precision": 0.8636363636363636, |
| "present_accuracy": 0.6363636363636364, |
| "recall": 0.8260869565217391, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.045454545454545456 |
| }, |
| "domain/single_lists": { |
| "absent_accuracy": 0.75, |
| "count": 4, |
| "counts": { |
| "absent_correct": 6, |
| "absent_total": 8, |
| "exact": 0, |
| "false_abstention": 4, |
| "field_correct": 6, |
| "fields": 12, |
| "fn": 4, |
| "fp": 2, |
| "json_parse_ok": 4, |
| "missed_abstention": 2, |
| "natural_stop": 4, |
| "nonrepetitive_nonempty": 4, |
| "predicted_values": 2, |
| "present_correct": 0, |
| "present_total": 4, |
| "schema_ok": 4, |
| "token_limited": 0, |
| "tp": 0, |
| "unsupported_values": 0 |
| }, |
| "exact": 0.0, |
| "f1": 0.0, |
| "false_abstention_rate": 1.0, |
| "field_accuracy": 0.5, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.25, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.21651429869234562, |
| "precision": 0.0, |
| "present_accuracy": 0.0, |
| "recall": 0.0, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "macro_exact": 0.6675824175824177, |
| "paired_requests": { |
| "accuracy": 0.7045454545454546, |
| "both_correct": 31, |
| "count": 44 |
| }, |
| "panel/legacy": { |
| "absent_accuracy": 0.9076923076923077, |
| "count": 128, |
| "counts": { |
| "absent_correct": 118, |
| "absent_total": 130, |
| "exact": 96, |
| "false_abstention": 11, |
| "field_correct": 267, |
| "fields": 310, |
| "fn": 36, |
| "fp": 24, |
| "json_parse_ok": 128, |
| "missed_abstention": 12, |
| "natural_stop": 128, |
| "nonrepetitive_nonempty": 128, |
| "predicted_values": 235, |
| "present_correct": 149, |
| "present_total": 180, |
| "schema_ok": 127, |
| "token_limited": 0, |
| "tp": 211, |
| "unsupported_values": 4 |
| }, |
| "exact": 0.75, |
| "f1": 0.8755186721991701, |
| "false_abstention_rate": 0.06111111111111111, |
| "field_accuracy": 0.8612903225806452, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.09230769230769231, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.06681022582381502, |
| "precision": 0.8978723404255319, |
| "present_accuracy": 0.8277777777777777, |
| "recall": 0.854251012145749, |
| "schema_valid_rate": 0.9921875, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.01702127659574468 |
| }, |
| "panel/new_prose": { |
| "absent_accuracy": 0.93048128342246, |
| "count": 112, |
| "counts": { |
| "absent_correct": 174, |
| "absent_total": 187, |
| "exact": 82, |
| "false_abstention": 14, |
| "field_correct": 365, |
| "fields": 404, |
| "fn": 29, |
| "fp": 23, |
| "json_parse_ok": 112, |
| "missed_abstention": 13, |
| "natural_stop": 112, |
| "nonrepetitive_nonempty": 112, |
| "predicted_values": 269, |
| "present_correct": 191, |
| "present_total": 217, |
| "schema_ok": 112, |
| "token_limited": 0, |
| "tp": 246, |
| "unsupported_values": 2 |
| }, |
| "exact": 0.7321428571428571, |
| "f1": 0.9044117647058822, |
| "false_abstention_rate": 0.06451612903225806, |
| "field_accuracy": 0.9034653465346535, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.06951871657754011, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.027105198803058426, |
| "precision": 0.9144981412639405, |
| "present_accuracy": 0.880184331797235, |
| "recall": 0.8945454545454545, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.007434944237918215 |
| }, |
| "variant/alias": { |
| "absent_accuracy": 0.9078947368421053, |
| "count": 46, |
| "counts": { |
| "absent_correct": 69, |
| "absent_total": 76, |
| "exact": 29, |
| "false_abstention": 11, |
| "field_correct": 131, |
| "fields": 157, |
| "fn": 20, |
| "fp": 14, |
| "json_parse_ok": 46, |
| "missed_abstention": 7, |
| "natural_stop": 46, |
| "nonrepetitive_nonempty": 46, |
| "predicted_values": 101, |
| "present_correct": 62, |
| "present_total": 81, |
| "schema_ok": 46, |
| "token_limited": 0, |
| "tp": 87, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.6304347826086957, |
| "f1": 0.8365384615384616, |
| "false_abstention_rate": 0.13580246913580246, |
| "field_accuracy": 0.8343949044585988, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.09210526315789473, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.06243703520158306, |
| "precision": 0.8613861386138614, |
| "present_accuracy": 0.7654320987654321, |
| "recall": 0.8130841121495327, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.009900990099009901 |
| }, |
| "variant/canonical": { |
| "absent_accuracy": 0.9235294117647059, |
| "count": 149, |
| "counts": { |
| "absent_correct": 157, |
| "absent_total": 170, |
| "exact": 118, |
| "false_abstention": 8, |
| "field_correct": 369, |
| "fields": 405, |
| "fn": 27, |
| "fp": 24, |
| "json_parse_ok": 149, |
| "missed_abstention": 13, |
| "natural_stop": 149, |
| "nonrepetitive_nonempty": 149, |
| "predicted_values": 304, |
| "present_correct": 212, |
| "present_total": 235, |
| "schema_ok": 149, |
| "token_limited": 0, |
| "tp": 280, |
| "unsupported_values": 4 |
| }, |
| "exact": 0.7919463087248322, |
| "f1": 0.9165302782324058, |
| "false_abstention_rate": 0.03404255319148936, |
| "field_accuracy": 0.9111111111111111, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.07647058823529412, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.043735888344098986, |
| "precision": 0.9210526315789473, |
| "present_accuracy": 0.902127659574468, |
| "recall": 0.9120521172638436, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.013157894736842105 |
| }, |
| "variant/nullable": { |
| "absent_accuracy": 0.9295774647887324, |
| "count": 45, |
| "counts": { |
| "absent_correct": 66, |
| "absent_total": 71, |
| "exact": 31, |
| "false_abstention": 6, |
| "field_correct": 132, |
| "fields": 152, |
| "fn": 18, |
| "fp": 9, |
| "json_parse_ok": 45, |
| "missed_abstention": 5, |
| "natural_stop": 45, |
| "nonrepetitive_nonempty": 45, |
| "predicted_values": 99, |
| "present_correct": 66, |
| "present_total": 81, |
| "schema_ok": 44, |
| "token_limited": 0, |
| "tp": 90, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.6888888888888889, |
| "f1": 0.8695652173913043, |
| "false_abstention_rate": 0.07407407407407407, |
| "field_accuracy": 0.868421052631579, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.07042253521126761, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.04886089308549547, |
| "precision": 0.9090909090909091, |
| "present_accuracy": 0.8148148148148148, |
| "recall": 0.8333333333333334, |
| "schema_valid_rate": 0.9777777777777777, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.010101010101010102 |
| } |
| }, |
| "retention": { |
| "gained": [ |
| "snips:AddToPlaylist:train:954:view0:native:canonical", |
| "snips:AddToPlaylist:train:1715:view0:native:canonical", |
| "snips:AddToPlaylist:train:1564:view0:native:canonical", |
| "focus110:development:roles:61", |
| "focus110:development:roles:36", |
| "focus110:development:roles:33", |
| "focus109:development:lists:43", |
| "focus109:development:lists:38", |
| "focus109:development:lists:40", |
| "cardinality105:development:16:alias", |
| "cardinality105:development:37:alias", |
| "cardinality105:development:10:alias", |
| "mixsnips:dev:738:focused102:canonical", |
| "joint128:development:PlayMusic:1:alias", |
| "joint128:development:PlayMusic:5:nullable", |
| "joint128:development:PlayMusic:7:alias", |
| "joint128:development:PlayMusic:8:nullable", |
| "joint128:development:PlayMusic:11:nullable", |
| "joint128:development:GetWeather:1:alias", |
| "joint128:development:GetWeather:3:canonical", |
| "joint128:development:GetWeather:7:alias", |
| "joint128:development:GetWeather:8:nullable", |
| "joint128:development:GetWeather:9:canonical", |
| "joint128:development:GetWeather:13:alias", |
| "joint128:development:BookRestaurant:0:canonical", |
| "joint128:development:BookRestaurant:1:alias", |
| "joint128:development:BookRestaurant:3:canonical", |
| "joint128:development:BookRestaurant:5:nullable", |
| "joint128:development:BookRestaurant:9:canonical", |
| "joint128:development:AddToPlaylist:1:alias", |
| "joint128:development:AddToPlaylist:2:nullable", |
| "joint128:development:AddToPlaylist:4:alias", |
| "joint128:development:AddToPlaylist:9:canonical", |
| "joint128:development:AddToPlaylist:11:nullable", |
| "joint128:development:AddToPlaylist:13:alias", |
| "joint128:development:AddToPlaylist:14:nullable", |
| "joint128:development:RateBook:1:alias", |
| "joint128:development:RateBook:6:canonical", |
| "joint128:development:RateBook:10:alias", |
| "joint128:development:RateBook:11:nullable", |
| "joint128:development:RateBook:12:canonical", |
| "joint128:development:RateBook:13:alias", |
| "joint128:development:SearchCreativeWork:1:alias", |
| "joint128:development:SearchCreativeWork:15:canonical", |
| "joint128:development:SearchScreeningEvent:0:canonical", |
| "joint128:development:SearchScreeningEvent:1:alias", |
| "joint128:development:SearchScreeningEvent:3:canonical", |
| "joint128:development:SearchScreeningEvent:5:nullable", |
| "joint128:development:SearchScreeningEvent:7:alias", |
| "joint128:development:SearchScreeningEvent:9:canonical", |
| "joint128:development:SearchScreeningEvent:11:nullable", |
| "joint128:development:SearchScreeningEvent:13:alias" |
| ], |
| "lost_previously_correct": [ |
| "snips:BookRestaurant:train:246:view1:native:canonical", |
| "snips:RateBook:train:1057:view0:native:canonical", |
| "snips:RateBook:train:1057:view1:native:canonical", |
| "focus109:development:lists:80", |
| "focus109:development:lists:77", |
| "focus109:development:lists:83" |
| ], |
| "new_prose_correct_delta": 37, |
| "paired_correct_delta": -1, |
| "panels": { |
| "all": { |
| "error_deltas": { |
| "false_abstention": -3, |
| "missed_abstention": -2, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.01538461538461533, |
| "exact": 0.0546875, |
| "f1": 0.04434984103033901, |
| "field_accuracy": 0.029032258064516148, |
| "json_parse_rate": 0.0234375, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.0048490846115784025, |
| "present_accuracy": 0.03888888888888886, |
| "recall": 0.07692307692307687, |
| "schema_valid_rate": 0.0625 |
| }, |
| "passed": true |
| }, |
| "domain/AddToPlaylist": { |
| "error_deltas": { |
| "false_abstention": -2, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": -1 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": 0.24999999999999994, |
| "f1": 0.11988304093567259, |
| "field_accuracy": 0.09375, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.07777777777777783, |
| "present_accuracy": 0.16666666666666663, |
| "recall": 0.16666666666666663, |
| "schema_valid_rate": 0.0 |
| }, |
| "passed": true |
| }, |
| "domain/BookRestaurant": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": 1, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": -0.07692307692307687, |
| "exact": -0.08333333333333337, |
| "f1": -0.026890756302521024, |
| "field_accuracy": -0.03448275862068961, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": -0.04678362573099415, |
| "present_accuracy": 0.0, |
| "recall": 0.0, |
| "schema_valid_rate": 0.0 |
| }, |
| "passed": false |
| }, |
| "domain/GetWeather": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": 0.0, |
| "f1": 0.0, |
| "field_accuracy": 0.0, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.0, |
| "present_accuracy": 0.0, |
| "recall": 0.0, |
| "schema_valid_rate": 0.0 |
| }, |
| "passed": true |
| }, |
| "domain/PlayMusic": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": 0.0, |
| "f1": 0.0, |
| "field_accuracy": 0.0, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.0, |
| "present_accuracy": 0.0, |
| "recall": 0.0, |
| "schema_valid_rate": 0.0 |
| }, |
| "passed": true |
| }, |
| "domain/RateBook": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": -0.16666666666666663, |
| "f1": -0.09523809523809523, |
| "field_accuracy": -0.05882352941176472, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": -0.09523809523809523, |
| "present_accuracy": -0.09523809523809523, |
| "recall": -0.09523809523809523, |
| "schema_valid_rate": 0.0 |
| }, |
| "passed": false |
| }, |
| "domain/SearchCreativeWork": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": 0.0, |
| "f1": 0.0, |
| "field_accuracy": 0.0, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.0, |
| "present_accuracy": 0.0, |
| "recall": 0.0, |
| "schema_valid_rate": 0.0 |
| }, |
| "passed": true |
| }, |
| "domain/SearchScreeningEvent": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": 0.0, |
| "f1": 0.0, |
| "field_accuracy": 0.0, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.0, |
| "present_accuracy": 0.0, |
| "recall": 0.0, |
| "schema_valid_rate": 0.0 |
| }, |
| "passed": true |
| }, |
| "domain/cardinality": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": -2, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.25, |
| "exact": 0.375, |
| "f1": 0.33333333333333337, |
| "field_accuracy": 0.3125, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.33333333333333337, |
| "present_accuracy": 0.375, |
| "recall": 0.33333333333333337, |
| "schema_valid_rate": 0.375 |
| }, |
| "passed": true |
| }, |
| "domain/mixsnips": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": 0.125, |
| "f1": -0.05882352941176466, |
| "field_accuracy": 0.0, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": -0.08333333333333337, |
| "present_accuracy": 0.0, |
| "recall": -0.04545454545454547, |
| "schema_valid_rate": 0.125 |
| }, |
| "passed": false |
| }, |
| "domain/observed_lists": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": -2, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.16666666666666663, |
| "exact": 0.0, |
| "f1": 0.07657378740970067, |
| "field_accuracy": 0.08333333333333337, |
| "json_parse_rate": 0.16666666666666663, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.0, |
| "present_accuracy": 0.0, |
| "recall": 0.13207547169811318, |
| "schema_valid_rate": 0.16666666666666663 |
| }, |
| "passed": true |
| }, |
| "domain/playground": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": 0.0, |
| "f1": 0.0, |
| "field_accuracy": 0.0, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.0, |
| "present_accuracy": 0.0, |
| "recall": 0.0, |
| "schema_valid_rate": 0.0 |
| }, |
| "passed": true |
| }, |
| "domain/roles": { |
| "error_deltas": { |
| "false_abstention": -1, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": 1 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": 0.375, |
| "f1": 0.23333333333333328, |
| "field_accuracy": 0.13636363636363635, |
| "json_parse_rate": 0.125, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.0174825174825175, |
| "present_accuracy": 0.2727272727272727, |
| "recall": 0.34782608695652173, |
| "schema_valid_rate": 0.25 |
| }, |
| "passed": false |
| }, |
| "domain/single_lists": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": 1, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": -0.125, |
| "exact": 0.0, |
| "f1": 0.0, |
| "field_accuracy": -0.08333333333333337, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.0, |
| "present_accuracy": 0.0, |
| "recall": 0.0, |
| "schema_valid_rate": 0.0 |
| }, |
| "passed": false |
| }, |
| "variant/alias": { |
| "error_deltas": { |
| "false_abstention": 0, |
| "missed_abstention": -2, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.1333333333333333, |
| "exact": 0.4545454545454545, |
| "f1": 0.380392156862745, |
| "field_accuracy": 0.20689655172413796, |
| "json_parse_rate": 0.09090909090909094, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": 0.3857142857142857, |
| "present_accuracy": 0.2857142857142857, |
| "recall": 0.35000000000000003, |
| "schema_valid_rate": 0.4545454545454546 |
| }, |
| "passed": true |
| }, |
| "variant/canonical": { |
| "error_deltas": { |
| "false_abstention": -4, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": 0.028037383177570097, |
| "f1": 0.029791422893334252, |
| "field_accuracy": 0.01945525291828798, |
| "json_parse_rate": 0.01869158878504673, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": -0.012469023085952768, |
| "present_accuracy": 0.0326797385620915, |
| "recall": 0.06796116504854377, |
| "schema_valid_rate": 0.01869158878504673 |
| }, |
| "passed": false |
| }, |
| "variant/nullable": { |
| "error_deltas": { |
| "false_abstention": 1, |
| "missed_abstention": 0, |
| "token_limited": 0, |
| "unsupported_values": 0 |
| }, |
| "metric_deltas": { |
| "absent_accuracy": 0.0, |
| "exact": -0.09999999999999998, |
| "f1": -0.0637560637560638, |
| "field_accuracy": -0.08333333333333337, |
| "json_parse_rate": 0.0, |
| "natural_stop_rate": 0.0, |
| "nonrepetitive_nonempty_rate": 0.0, |
| "precision": -0.01388888888888884, |
| "present_accuracy": -0.15384615384615385, |
| "recall": -0.09523809523809523, |
| "schema_valid_rate": 0.09999999999999998 |
| }, |
| "passed": false |
| } |
| }, |
| "passed": false, |
| "scope": "Finite observed request/metric retention gates; cannot guarantee universal no-regression." |
| } |
| }, |
| "heldout_1444": { |
| "metrics": { |
| "all": { |
| "absent_accuracy": 0.9521625163826999, |
| "count": 1444, |
| "counts": { |
| "absent_correct": 1453, |
| "absent_total": 1526, |
| "exact": 1271, |
| "false_abstention": 53, |
| "field_correct": 3293, |
| "fields": 3497, |
| "fn": 131, |
| "fp": 151, |
| "json_parse_ok": 1444, |
| "missed_abstention": 73, |
| "natural_stop": 1444, |
| "nonrepetitive_nonempty": 1444, |
| "predicted_values": 1991, |
| "present_correct": 1840, |
| "present_total": 1971, |
| "schema_ok": 1444, |
| "token_limited": 0, |
| "tp": 1840, |
| "unsupported_values": 24 |
| }, |
| "exact": 0.8801939058171745, |
| "f1": 0.928823826350328, |
| "false_abstention_rate": 0.026889903602232368, |
| "field_accuracy": 0.9416642836717186, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.04783748361730013, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.024015552303623188, |
| "precision": 0.9241587142139628, |
| "present_accuracy": 0.9335362760020294, |
| "recall": 0.9335362760020294, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.012054244098442994 |
| }, |
| "domain/AddToPlaylist": { |
| "absent_accuracy": 0.9364406779661016, |
| "count": 206, |
| "counts": { |
| "absent_correct": 221, |
| "absent_total": 236, |
| "exact": 172, |
| "false_abstention": 8, |
| "field_correct": 481, |
| "fields": 526, |
| "fn": 30, |
| "fp": 37, |
| "json_parse_ok": 206, |
| "missed_abstention": 15, |
| "natural_stop": 206, |
| "nonrepetitive_nonempty": 206, |
| "predicted_values": 297, |
| "present_correct": 260, |
| "present_total": 290, |
| "schema_ok": 206, |
| "token_limited": 0, |
| "tp": 260, |
| "unsupported_values": 14 |
| }, |
| "exact": 0.8349514563106796, |
| "f1": 0.8858603066439523, |
| "false_abstention_rate": 0.027586206896551724, |
| "field_accuracy": 0.9144486692015209, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0635593220338983, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.041144459200014236, |
| "precision": 0.8754208754208754, |
| "present_accuracy": 0.896551724137931, |
| "recall": 0.896551724137931, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.04713804713804714 |
| }, |
| "domain/BookRestaurant": { |
| "absent_accuracy": 0.932, |
| "count": 208, |
| "counts": { |
| "absent_correct": 233, |
| "absent_total": 250, |
| "exact": 172, |
| "false_abstention": 8, |
| "field_correct": 528, |
| "fields": 569, |
| "fn": 24, |
| "fp": 33, |
| "json_parse_ok": 208, |
| "missed_abstention": 17, |
| "natural_stop": 208, |
| "nonrepetitive_nonempty": 208, |
| "predicted_values": 328, |
| "present_correct": 295, |
| "present_total": 319, |
| "schema_ok": 208, |
| "token_limited": 0, |
| "tp": 295, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.8269230769230769, |
| "f1": 0.9119010819165378, |
| "false_abstention_rate": 0.025078369905956112, |
| "field_accuracy": 0.9279437609841827, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.068, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.027245943393189427, |
| "precision": 0.899390243902439, |
| "present_accuracy": 0.9247648902821317, |
| "recall": 0.9247648902821317, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.003048780487804878 |
| }, |
| "domain/GetWeather": { |
| "absent_accuracy": 0.943089430894309, |
| "count": 206, |
| "counts": { |
| "absent_correct": 232, |
| "absent_total": 246, |
| "exact": 178, |
| "false_abstention": 5, |
| "field_correct": 493, |
| "fields": 525, |
| "fn": 18, |
| "fp": 27, |
| "json_parse_ok": 206, |
| "missed_abstention": 14, |
| "natural_stop": 206, |
| "nonrepetitive_nonempty": 206, |
| "predicted_values": 288, |
| "present_correct": 261, |
| "present_total": 279, |
| "schema_ok": 206, |
| "token_limited": 0, |
| "tp": 261, |
| "unsupported_values": 5 |
| }, |
| "exact": 0.8640776699029126, |
| "f1": 0.9206349206349206, |
| "false_abstention_rate": 0.017921146953405017, |
| "field_accuracy": 0.939047619047619, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.056910569105691054, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.022860561515487453, |
| "precision": 0.90625, |
| "present_accuracy": 0.9354838709677419, |
| "recall": 0.9354838709677419, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.017361111111111112 |
| }, |
| "domain/PlayMusic": { |
| "absent_accuracy": 0.9296875, |
| "count": 208, |
| "counts": { |
| "absent_correct": 238, |
| "absent_total": 256, |
| "exact": 164, |
| "false_abstention": 24, |
| "field_correct": 468, |
| "fields": 523, |
| "fn": 37, |
| "fp": 31, |
| "json_parse_ok": 208, |
| "missed_abstention": 18, |
| "natural_stop": 208, |
| "nonrepetitive_nonempty": 208, |
| "predicted_values": 261, |
| "present_correct": 230, |
| "present_total": 267, |
| "schema_ok": 208, |
| "token_limited": 0, |
| "tp": 230, |
| "unsupported_values": 0 |
| }, |
| "exact": 0.7884615384615384, |
| "f1": 0.8712121212121211, |
| "false_abstention_rate": 0.0898876404494382, |
| "field_accuracy": 0.8948374760994264, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0703125, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.03961348310237484, |
| "precision": 0.8812260536398467, |
| "present_accuracy": 0.8614232209737828, |
| "recall": 0.8614232209737828, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "domain/RateBook": { |
| "absent_accuracy": 0.983739837398374, |
| "count": 204, |
| "counts": { |
| "absent_correct": 242, |
| "absent_total": 246, |
| "exact": 195, |
| "false_abstention": 1, |
| "field_correct": 558, |
| "fields": 567, |
| "fn": 5, |
| "fp": 8, |
| "json_parse_ok": 204, |
| "missed_abstention": 4, |
| "natural_stop": 204, |
| "nonrepetitive_nonempty": 204, |
| "predicted_values": 324, |
| "present_correct": 316, |
| "present_total": 321, |
| "schema_ok": 204, |
| "token_limited": 0, |
| "tp": 316, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.9558823529411765, |
| "f1": 0.9798449612403102, |
| "false_abstention_rate": 0.003115264797507788, |
| "field_accuracy": 0.9841269841269841, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.016260162601626018, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.008335007565976554, |
| "precision": 0.9753086419753086, |
| "present_accuracy": 0.9844236760124611, |
| "recall": 0.9844236760124611, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0030864197530864196 |
| }, |
| "domain/SearchCreativeWork": { |
| "absent_accuracy": 1.0, |
| "count": 208, |
| "counts": { |
| "absent_correct": 39, |
| "absent_total": 39, |
| "exact": 199, |
| "false_abstention": 2, |
| "field_correct": 277, |
| "fields": 286, |
| "fn": 9, |
| "fp": 7, |
| "json_parse_ok": 208, |
| "missed_abstention": 0, |
| "natural_stop": 208, |
| "nonrepetitive_nonempty": 208, |
| "predicted_values": 245, |
| "present_correct": 238, |
| "present_total": 247, |
| "schema_ok": 208, |
| "token_limited": 0, |
| "tp": 238, |
| "unsupported_values": 2 |
| }, |
| "exact": 0.9567307692307693, |
| "f1": 0.9674796747967479, |
| "false_abstention_rate": 0.008097165991902834, |
| "field_accuracy": 0.9685314685314685, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.01792181849521932, |
| "precision": 0.9714285714285714, |
| "present_accuracy": 0.9635627530364372, |
| "recall": 0.9635627530364372, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.00816326530612245 |
| }, |
| "domain/SearchScreeningEvent": { |
| "absent_accuracy": 0.9802371541501976, |
| "count": 204, |
| "counts": { |
| "absent_correct": 248, |
| "absent_total": 253, |
| "exact": 191, |
| "false_abstention": 5, |
| "field_correct": 488, |
| "fields": 501, |
| "fn": 8, |
| "fp": 8, |
| "json_parse_ok": 204, |
| "missed_abstention": 5, |
| "natural_stop": 204, |
| "nonrepetitive_nonempty": 204, |
| "predicted_values": 248, |
| "present_correct": 240, |
| "present_total": 248, |
| "schema_ok": 204, |
| "token_limited": 0, |
| "tp": 240, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.9362745098039216, |
| "f1": 0.967741935483871, |
| "false_abstention_rate": 0.020161290322580645, |
| "field_accuracy": 0.9740518962075848, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.019762845849802372, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.01058128802694281, |
| "precision": 0.967741935483871, |
| "present_accuracy": 0.967741935483871, |
| "recall": 0.967741935483871, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.004032258064516129 |
| }, |
| "macro_exact": 0.8804716247962964, |
| "paired_requests": { |
| "accuracy": 0.80835734870317, |
| "both_correct": 561, |
| "count": 694 |
| }, |
| "panel/original1444": { |
| "absent_accuracy": 0.9521625163826999, |
| "count": 1444, |
| "counts": { |
| "absent_correct": 1453, |
| "absent_total": 1526, |
| "exact": 1271, |
| "false_abstention": 53, |
| "field_correct": 3293, |
| "fields": 3497, |
| "fn": 131, |
| "fp": 151, |
| "json_parse_ok": 1444, |
| "missed_abstention": 73, |
| "natural_stop": 1444, |
| "nonrepetitive_nonempty": 1444, |
| "predicted_values": 1991, |
| "present_correct": 1840, |
| "present_total": 1971, |
| "schema_ok": 1444, |
| "token_limited": 0, |
| "tp": 1840, |
| "unsupported_values": 24 |
| }, |
| "exact": 0.8801939058171745, |
| "f1": 0.928823826350328, |
| "false_abstention_rate": 0.026889903602232368, |
| "field_accuracy": 0.9416642836717186, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.04783748361730013, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.024015552303623188, |
| "precision": 0.9241587142139628, |
| "present_accuracy": 0.9335362760020294, |
| "recall": 0.9335362760020294, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.012054244098442994 |
| }, |
| "variant/alias": { |
| "absent_accuracy": 0.9310344827586207, |
| "count": 28, |
| "counts": { |
| "absent_correct": 27, |
| "absent_total": 29, |
| "exact": 23, |
| "false_abstention": 1, |
| "field_correct": 70, |
| "fields": 75, |
| "fn": 3, |
| "fp": 4, |
| "json_parse_ok": 28, |
| "missed_abstention": 2, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 47, |
| "present_correct": 43, |
| "present_total": 46, |
| "schema_ok": 28, |
| "token_limited": 0, |
| "tp": 43, |
| "unsupported_values": 2 |
| }, |
| "exact": 0.8214285714285714, |
| "f1": 0.924731182795699, |
| "false_abstention_rate": 0.021739130434782608, |
| "field_accuracy": 0.9333333333333333, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.06896551724137931, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.051775694317517, |
| "precision": 0.9148936170212766, |
| "present_accuracy": 0.9347826086956522, |
| "recall": 0.9347826086956522, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0425531914893617 |
| }, |
| "variant/canonical": { |
| "absent_accuracy": 0.952316076294278, |
| "count": 1388, |
| "counts": { |
| "absent_correct": 1398, |
| "absent_total": 1468, |
| "exact": 1224, |
| "false_abstention": 50, |
| "field_correct": 3154, |
| "fields": 3347, |
| "fn": 123, |
| "fp": 143, |
| "json_parse_ok": 1388, |
| "missed_abstention": 70, |
| "natural_stop": 1388, |
| "nonrepetitive_nonempty": 1388, |
| "predicted_values": 1899, |
| "present_correct": 1756, |
| "present_total": 1879, |
| "schema_ok": 1388, |
| "token_limited": 0, |
| "tp": 1756, |
| "unsupported_values": 21 |
| }, |
| "exact": 0.8818443804034583, |
| "f1": 0.9295923769190046, |
| "false_abstention_rate": 0.026609898882384245, |
| "field_accuracy": 0.9423364206752316, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.047683923705722074, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.022789464743013806, |
| "precision": 0.9246972090573986, |
| "present_accuracy": 0.9345396487493347, |
| "recall": 0.9345396487493347, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.011058451816745656 |
| }, |
| "variant/nullable": { |
| "absent_accuracy": 0.9655172413793104, |
| "count": 28, |
| "counts": { |
| "absent_correct": 28, |
| "absent_total": 29, |
| "exact": 24, |
| "false_abstention": 2, |
| "field_correct": 69, |
| "fields": 75, |
| "fn": 5, |
| "fp": 4, |
| "json_parse_ok": 28, |
| "missed_abstention": 1, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 45, |
| "present_correct": 41, |
| "present_total": 46, |
| "schema_ok": 28, |
| "token_limited": 0, |
| "tp": 41, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.8571428571428571, |
| "f1": 0.9010989010989011, |
| "false_abstention_rate": 0.043478260869565216, |
| "field_accuracy": 0.92, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.034482758620689655, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.05703432222279454, |
| "precision": 0.9111111111111111, |
| "present_accuracy": 0.8913043478260869, |
| "recall": 0.8913043478260869, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.022222222222222223 |
| } |
| }, |
| "previous_metrics": { |
| "all": { |
| "absent_accuracy": 0.927916120576671, |
| "count": 1444, |
| "counts": { |
| "absent_correct": 1416, |
| "absent_total": 1526, |
| "exact": 1199, |
| "false_abstention": 107, |
| "field_correct": 3195, |
| "fields": 3497, |
| "fn": 192, |
| "fp": 186, |
| "json_parse_ok": 1442, |
| "missed_abstention": 110, |
| "natural_stop": 1444, |
| "nonrepetitive_nonempty": 1444, |
| "predicted_values": 1965, |
| "present_correct": 1779, |
| "present_total": 1971, |
| "schema_ok": 1441, |
| "token_limited": 0, |
| "tp": 1779, |
| "unsupported_values": 27 |
| }, |
| "exact": 0.8303324099722992, |
| "f1": 0.9039634146341463, |
| "false_abstention_rate": 0.05428716387620497, |
| "field_accuracy": 0.9136402630826422, |
| "json_parse_rate": 0.9986149584487535, |
| "missed_abstention_rate": 0.07208387942332896, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.030197499356996216, |
| "precision": 0.9053435114503817, |
| "present_accuracy": 0.9025875190258752, |
| "recall": 0.9025875190258752, |
| "schema_valid_rate": 0.9979224376731302, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.013740458015267175 |
| }, |
| "domain/AddToPlaylist": { |
| "absent_accuracy": 0.9152542372881356, |
| "count": 206, |
| "counts": { |
| "absent_correct": 216, |
| "absent_total": 236, |
| "exact": 165, |
| "false_abstention": 12, |
| "field_correct": 473, |
| "fields": 526, |
| "fn": 33, |
| "fp": 41, |
| "json_parse_ok": 206, |
| "missed_abstention": 20, |
| "natural_stop": 206, |
| "nonrepetitive_nonempty": 206, |
| "predicted_values": 298, |
| "present_correct": 257, |
| "present_total": 290, |
| "schema_ok": 206, |
| "token_limited": 0, |
| "tp": 257, |
| "unsupported_values": 15 |
| }, |
| "exact": 0.8009708737864077, |
| "f1": 0.8741496598639455, |
| "false_abstention_rate": 0.041379310344827586, |
| "field_accuracy": 0.8992395437262357, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0847457627118644, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.04620553536913629, |
| "precision": 0.8624161073825504, |
| "present_accuracy": 0.8862068965517241, |
| "recall": 0.8862068965517241, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.050335570469798654 |
| }, |
| "domain/BookRestaurant": { |
| "absent_accuracy": 0.908, |
| "count": 208, |
| "counts": { |
| "absent_correct": 227, |
| "absent_total": 250, |
| "exact": 162, |
| "false_abstention": 16, |
| "field_correct": 510, |
| "fields": 569, |
| "fn": 36, |
| "fp": 43, |
| "json_parse_ok": 208, |
| "missed_abstention": 23, |
| "natural_stop": 208, |
| "nonrepetitive_nonempty": 208, |
| "predicted_values": 326, |
| "present_correct": 283, |
| "present_total": 319, |
| "schema_ok": 208, |
| "token_limited": 0, |
| "tp": 283, |
| "unsupported_values": 3 |
| }, |
| "exact": 0.7788461538461539, |
| "f1": 0.8775193798449612, |
| "false_abstention_rate": 0.050156739811912224, |
| "field_accuracy": 0.8963093145869947, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.092, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.03511890941109204, |
| "precision": 0.8680981595092024, |
| "present_accuracy": 0.8871473354231975, |
| "recall": 0.8871473354231975, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.009202453987730062 |
| }, |
| "domain/GetWeather": { |
| "absent_accuracy": 0.9186991869918699, |
| "count": 206, |
| "counts": { |
| "absent_correct": 226, |
| "absent_total": 246, |
| "exact": 164, |
| "false_abstention": 15, |
| "field_correct": 473, |
| "fields": 525, |
| "fn": 32, |
| "fp": 32, |
| "json_parse_ok": 205, |
| "missed_abstention": 20, |
| "natural_stop": 206, |
| "nonrepetitive_nonempty": 206, |
| "predicted_values": 279, |
| "present_correct": 247, |
| "present_total": 279, |
| "schema_ok": 204, |
| "token_limited": 0, |
| "tp": 247, |
| "unsupported_values": 3 |
| }, |
| "exact": 0.7961165048543689, |
| "f1": 0.8853046594982079, |
| "false_abstention_rate": 0.053763440860215055, |
| "field_accuracy": 0.900952380952381, |
| "json_parse_rate": 0.9951456310679612, |
| "missed_abstention_rate": 0.08130081300813008, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.030885711256629398, |
| "precision": 0.8853046594982079, |
| "present_accuracy": 0.8853046594982079, |
| "recall": 0.8853046594982079, |
| "schema_valid_rate": 0.9902912621359223, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.010752688172043012 |
| }, |
| "domain/PlayMusic": { |
| "absent_accuracy": 0.94140625, |
| "count": 208, |
| "counts": { |
| "absent_correct": 241, |
| "absent_total": 256, |
| "exact": 153, |
| "false_abstention": 45, |
| "field_correct": 449, |
| "fields": 523, |
| "fn": 59, |
| "fp": 25, |
| "json_parse_ok": 207, |
| "missed_abstention": 15, |
| "natural_stop": 208, |
| "nonrepetitive_nonempty": 208, |
| "predicted_values": 233, |
| "present_correct": 208, |
| "present_total": 267, |
| "schema_ok": 207, |
| "token_limited": 0, |
| "tp": 208, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.7355769230769231, |
| "f1": 0.8320000000000001, |
| "false_abstention_rate": 0.16853932584269662, |
| "field_accuracy": 0.858508604206501, |
| "json_parse_rate": 0.9951923076923077, |
| "missed_abstention_rate": 0.05859375, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.0454767054272032, |
| "precision": 0.8927038626609443, |
| "present_accuracy": 0.7790262172284644, |
| "recall": 0.7790262172284644, |
| "schema_valid_rate": 0.9951923076923077, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.004291845493562232 |
| }, |
| "domain/RateBook": { |
| "absent_accuracy": 0.9796747967479674, |
| "count": 204, |
| "counts": { |
| "absent_correct": 241, |
| "absent_total": 246, |
| "exact": 193, |
| "false_abstention": 3, |
| "field_correct": 556, |
| "fields": 567, |
| "fn": 6, |
| "fp": 8, |
| "json_parse_ok": 204, |
| "missed_abstention": 5, |
| "natural_stop": 204, |
| "nonrepetitive_nonempty": 204, |
| "predicted_values": 323, |
| "present_correct": 315, |
| "present_total": 321, |
| "schema_ok": 204, |
| "token_limited": 0, |
| "tp": 315, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.946078431372549, |
| "f1": 0.9782608695652175, |
| "false_abstention_rate": 0.009345794392523364, |
| "field_accuracy": 0.9805996472663139, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.02032520325203252, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.010863955307286233, |
| "precision": 0.9752321981424149, |
| "present_accuracy": 0.9813084112149533, |
| "recall": 0.9813084112149533, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0030959752321981426 |
| }, |
| "domain/SearchCreativeWork": { |
| "absent_accuracy": 0.9487179487179487, |
| "count": 208, |
| "counts": { |
| "absent_correct": 37, |
| "absent_total": 39, |
| "exact": 197, |
| "false_abstention": 2, |
| "field_correct": 275, |
| "fields": 286, |
| "fn": 9, |
| "fp": 9, |
| "json_parse_ok": 208, |
| "missed_abstention": 2, |
| "natural_stop": 208, |
| "nonrepetitive_nonempty": 208, |
| "predicted_values": 247, |
| "present_correct": 238, |
| "present_total": 247, |
| "schema_ok": 208, |
| "token_limited": 0, |
| "tp": 238, |
| "unsupported_values": 0 |
| }, |
| "exact": 0.9471153846153846, |
| "f1": 0.9635627530364372, |
| "false_abstention_rate": 0.008097165991902834, |
| "field_accuracy": 0.9615384615384616, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.05128205128205128, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.019084845725116723, |
| "precision": 0.9635627530364372, |
| "present_accuracy": 0.9635627530364372, |
| "recall": 0.9635627530364372, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "domain/SearchScreeningEvent": { |
| "absent_accuracy": 0.9011857707509882, |
| "count": 204, |
| "counts": { |
| "absent_correct": 228, |
| "absent_total": 253, |
| "exact": 165, |
| "false_abstention": 14, |
| "field_correct": 459, |
| "fields": 501, |
| "fn": 17, |
| "fp": 28, |
| "json_parse_ok": 204, |
| "missed_abstention": 25, |
| "natural_stop": 204, |
| "nonrepetitive_nonempty": 204, |
| "predicted_values": 259, |
| "present_correct": 231, |
| "present_total": 248, |
| "schema_ok": 204, |
| "token_limited": 0, |
| "tp": 231, |
| "unsupported_values": 4 |
| }, |
| "exact": 0.8088235294117647, |
| "f1": 0.9112426035502958, |
| "false_abstention_rate": 0.056451612903225805, |
| "field_accuracy": 0.9161676646706587, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.09881422924901186, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.023404948954503554, |
| "precision": 0.8918918918918919, |
| "present_accuracy": 0.9314516129032258, |
| "recall": 0.9314516129032258, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.015444015444015444 |
| }, |
| "macro_exact": 0.8305039715662217, |
| "paired_requests": { |
| "accuracy": 0.7377521613832853, |
| "both_correct": 512, |
| "count": 694 |
| }, |
| "panel/original1444": { |
| "absent_accuracy": 0.927916120576671, |
| "count": 1444, |
| "counts": { |
| "absent_correct": 1416, |
| "absent_total": 1526, |
| "exact": 1199, |
| "false_abstention": 107, |
| "field_correct": 3195, |
| "fields": 3497, |
| "fn": 192, |
| "fp": 186, |
| "json_parse_ok": 1442, |
| "missed_abstention": 110, |
| "natural_stop": 1444, |
| "nonrepetitive_nonempty": 1444, |
| "predicted_values": 1965, |
| "present_correct": 1779, |
| "present_total": 1971, |
| "schema_ok": 1441, |
| "token_limited": 0, |
| "tp": 1779, |
| "unsupported_values": 27 |
| }, |
| "exact": 0.8303324099722992, |
| "f1": 0.9039634146341463, |
| "false_abstention_rate": 0.05428716387620497, |
| "field_accuracy": 0.9136402630826422, |
| "json_parse_rate": 0.9986149584487535, |
| "missed_abstention_rate": 0.07208387942332896, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.030197499356996216, |
| "precision": 0.9053435114503817, |
| "present_accuracy": 0.9025875190258752, |
| "recall": 0.9025875190258752, |
| "schema_valid_rate": 0.9979224376731302, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.013740458015267175 |
| }, |
| "variant/alias": { |
| "absent_accuracy": 0.7931034482758621, |
| "count": 28, |
| "counts": { |
| "absent_correct": 23, |
| "absent_total": 29, |
| "exact": 21, |
| "false_abstention": 1, |
| "field_correct": 61, |
| "fields": 75, |
| "fn": 8, |
| "fp": 5, |
| "json_parse_ok": 26, |
| "missed_abstention": 6, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 43, |
| "present_correct": 38, |
| "present_total": 46, |
| "schema_ok": 26, |
| "token_limited": 0, |
| "tp": 38, |
| "unsupported_values": 2 |
| }, |
| "exact": 0.75, |
| "f1": 0.853932584269663, |
| "false_abstention_rate": 0.021739130434782608, |
| "field_accuracy": 0.8133333333333334, |
| "json_parse_rate": 0.9285714285714286, |
| "missed_abstention_rate": 0.20689655172413793, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.05955322041908013, |
| "precision": 0.8837209302325582, |
| "present_accuracy": 0.8260869565217391, |
| "recall": 0.8260869565217391, |
| "schema_valid_rate": 0.9285714285714286, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.046511627906976744 |
| }, |
| "variant/canonical": { |
| "absent_accuracy": 0.9339237057220708, |
| "count": 1388, |
| "counts": { |
| "absent_correct": 1371, |
| "absent_total": 1468, |
| "exact": 1163, |
| "false_abstention": 100, |
| "field_correct": 3075, |
| "fields": 3347, |
| "fn": 175, |
| "fp": 172, |
| "json_parse_ok": 1388, |
| "missed_abstention": 97, |
| "natural_stop": 1388, |
| "nonrepetitive_nonempty": 1388, |
| "predicted_values": 1876, |
| "present_correct": 1704, |
| "present_total": 1879, |
| "schema_ok": 1388, |
| "token_limited": 0, |
| "tp": 1704, |
| "unsupported_values": 23 |
| }, |
| "exact": 0.8378962536023055, |
| "f1": 0.9075898801597869, |
| "false_abstention_rate": 0.05321979776476849, |
| "field_accuracy": 0.9187331939049895, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.06607629427792916, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.028413429219706483, |
| "precision": 0.908315565031983, |
| "present_accuracy": 0.9068653539116551, |
| "recall": 0.9068653539116551, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.012260127931769723 |
| }, |
| "variant/nullable": { |
| "absent_accuracy": 0.7586206896551724, |
| "count": 28, |
| "counts": { |
| "absent_correct": 22, |
| "absent_total": 29, |
| "exact": 15, |
| "false_abstention": 6, |
| "field_correct": 59, |
| "fields": 75, |
| "fn": 9, |
| "fp": 9, |
| "json_parse_ok": 28, |
| "missed_abstention": 7, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 46, |
| "present_correct": 37, |
| "present_total": 46, |
| "schema_ok": 27, |
| "token_limited": 0, |
| "tp": 37, |
| "unsupported_values": 2 |
| }, |
| "exact": 0.5357142857142857, |
| "f1": 0.8043478260869565, |
| "false_abstention_rate": 0.13043478260869565, |
| "field_accuracy": 0.7866666666666666, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.2413793103448276, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.08928068367198908, |
| "precision": 0.8043478260869565, |
| "present_accuracy": 0.8043478260869565, |
| "recall": 0.8043478260869565, |
| "schema_valid_rate": 0.9642857142857143, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.043478260869565216 |
| } |
| }, |
| "improved_requests": 96, |
| "regressed_requests": 24 |
| }, |
| "playground_60": { |
| "metrics": { |
| "all": { |
| "absent_accuracy": 0.9322033898305084, |
| "count": 60, |
| "counts": { |
| "absent_correct": 55, |
| "absent_total": 59, |
| "exact": 49, |
| "false_abstention": 2, |
| "field_correct": 136, |
| "fields": 147, |
| "fn": 9, |
| "fp": 10, |
| "json_parse_ok": 60, |
| "missed_abstention": 4, |
| "natural_stop": 60, |
| "nonrepetitive_nonempty": 60, |
| "predicted_values": 109, |
| "present_correct": 81, |
| "present_total": 88, |
| "schema_ok": 60, |
| "token_limited": 0, |
| "tp": 99, |
| "unsupported_values": 6 |
| }, |
| "exact": 0.8166666666666667, |
| "f1": 0.912442396313364, |
| "false_abstention_rate": 0.022727272727272728, |
| "field_accuracy": 0.9251700680272109, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.06779661016949153, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.046341597845215195, |
| "precision": 0.908256880733945, |
| "present_accuracy": 0.9204545454545454, |
| "recall": 0.9166666666666666, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.05504587155963303 |
| }, |
| "domain/playground": { |
| "absent_accuracy": 0.9322033898305084, |
| "count": 60, |
| "counts": { |
| "absent_correct": 55, |
| "absent_total": 59, |
| "exact": 49, |
| "false_abstention": 2, |
| "field_correct": 136, |
| "fields": 147, |
| "fn": 9, |
| "fp": 10, |
| "json_parse_ok": 60, |
| "missed_abstention": 4, |
| "natural_stop": 60, |
| "nonrepetitive_nonempty": 60, |
| "predicted_values": 109, |
| "present_correct": 81, |
| "present_total": 88, |
| "schema_ok": 60, |
| "token_limited": 0, |
| "tp": 99, |
| "unsupported_values": 6 |
| }, |
| "exact": 0.8166666666666667, |
| "f1": 0.912442396313364, |
| "false_abstention_rate": 0.022727272727272728, |
| "field_accuracy": 0.9251700680272109, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.06779661016949153, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.046341597845215195, |
| "precision": 0.908256880733945, |
| "present_accuracy": 0.9204545454545454, |
| "recall": 0.9166666666666666, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.05504587155963303 |
| }, |
| "macro_exact": 0.8166666666666667, |
| "paired_requests": { |
| "accuracy": 0.0, |
| "both_correct": 0, |
| "count": 0 |
| }, |
| "panel/playground60": { |
| "absent_accuracy": 0.9322033898305084, |
| "count": 60, |
| "counts": { |
| "absent_correct": 55, |
| "absent_total": 59, |
| "exact": 49, |
| "false_abstention": 2, |
| "field_correct": 136, |
| "fields": 147, |
| "fn": 9, |
| "fp": 10, |
| "json_parse_ok": 60, |
| "missed_abstention": 4, |
| "natural_stop": 60, |
| "nonrepetitive_nonempty": 60, |
| "predicted_values": 109, |
| "present_correct": 81, |
| "present_total": 88, |
| "schema_ok": 60, |
| "token_limited": 0, |
| "tp": 99, |
| "unsupported_values": 6 |
| }, |
| "exact": 0.8166666666666667, |
| "f1": 0.912442396313364, |
| "false_abstention_rate": 0.022727272727272728, |
| "field_accuracy": 0.9251700680272109, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.06779661016949153, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.046341597845215195, |
| "precision": 0.908256880733945, |
| "present_accuracy": 0.9204545454545454, |
| "recall": 0.9166666666666666, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.05504587155963303 |
| }, |
| "variant/canonical": { |
| "absent_accuracy": 0.9322033898305084, |
| "count": 60, |
| "counts": { |
| "absent_correct": 55, |
| "absent_total": 59, |
| "exact": 49, |
| "false_abstention": 2, |
| "field_correct": 136, |
| "fields": 147, |
| "fn": 9, |
| "fp": 10, |
| "json_parse_ok": 60, |
| "missed_abstention": 4, |
| "natural_stop": 60, |
| "nonrepetitive_nonempty": 60, |
| "predicted_values": 109, |
| "present_correct": 81, |
| "present_total": 88, |
| "schema_ok": 60, |
| "token_limited": 0, |
| "tp": 99, |
| "unsupported_values": 6 |
| }, |
| "exact": 0.8166666666666667, |
| "f1": 0.912442396313364, |
| "false_abstention_rate": 0.022727272727272728, |
| "field_accuracy": 0.9251700680272109, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.06779661016949153, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.046341597845215195, |
| "precision": 0.908256880733945, |
| "present_accuracy": 0.9204545454545454, |
| "recall": 0.9166666666666666, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.05504587155963303 |
| } |
| }, |
| "previous_metrics": { |
| "all": { |
| "absent_accuracy": 0.8983050847457628, |
| "count": 60, |
| "counts": { |
| "absent_correct": 53, |
| "absent_total": 59, |
| "exact": 45, |
| "false_abstention": 3, |
| "field_correct": 129, |
| "fields": 147, |
| "fn": 22, |
| "fp": 13, |
| "json_parse_ok": 58, |
| "missed_abstention": 6, |
| "natural_stop": 60, |
| "nonrepetitive_nonempty": 60, |
| "predicted_values": 99, |
| "present_correct": 76, |
| "present_total": 88, |
| "schema_ok": 58, |
| "token_limited": 0, |
| "tp": 86, |
| "unsupported_values": 7 |
| }, |
| "exact": 0.75, |
| "f1": 0.8309178743961353, |
| "false_abstention_rate": 0.03409090909090909, |
| "field_accuracy": 0.8775510204081632, |
| "json_parse_rate": 0.9666666666666667, |
| "missed_abstention_rate": 0.1016949152542373, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.07145761149877217, |
| "precision": 0.8686868686868687, |
| "present_accuracy": 0.8636363636363636, |
| "recall": 0.7962962962962963, |
| "schema_valid_rate": 0.9666666666666667, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0707070707070707 |
| }, |
| "domain/playground": { |
| "absent_accuracy": 0.8983050847457628, |
| "count": 60, |
| "counts": { |
| "absent_correct": 53, |
| "absent_total": 59, |
| "exact": 45, |
| "false_abstention": 3, |
| "field_correct": 129, |
| "fields": 147, |
| "fn": 22, |
| "fp": 13, |
| "json_parse_ok": 58, |
| "missed_abstention": 6, |
| "natural_stop": 60, |
| "nonrepetitive_nonempty": 60, |
| "predicted_values": 99, |
| "present_correct": 76, |
| "present_total": 88, |
| "schema_ok": 58, |
| "token_limited": 0, |
| "tp": 86, |
| "unsupported_values": 7 |
| }, |
| "exact": 0.75, |
| "f1": 0.8309178743961353, |
| "false_abstention_rate": 0.03409090909090909, |
| "field_accuracy": 0.8775510204081632, |
| "json_parse_rate": 0.9666666666666667, |
| "missed_abstention_rate": 0.1016949152542373, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.07145761149877217, |
| "precision": 0.8686868686868687, |
| "present_accuracy": 0.8636363636363636, |
| "recall": 0.7962962962962963, |
| "schema_valid_rate": 0.9666666666666667, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0707070707070707 |
| }, |
| "macro_exact": 0.75, |
| "paired_requests": { |
| "accuracy": 0.0, |
| "both_correct": 0, |
| "count": 0 |
| }, |
| "panel/playground60": { |
| "absent_accuracy": 0.8983050847457628, |
| "count": 60, |
| "counts": { |
| "absent_correct": 53, |
| "absent_total": 59, |
| "exact": 45, |
| "false_abstention": 3, |
| "field_correct": 129, |
| "fields": 147, |
| "fn": 22, |
| "fp": 13, |
| "json_parse_ok": 58, |
| "missed_abstention": 6, |
| "natural_stop": 60, |
| "nonrepetitive_nonempty": 60, |
| "predicted_values": 99, |
| "present_correct": 76, |
| "present_total": 88, |
| "schema_ok": 58, |
| "token_limited": 0, |
| "tp": 86, |
| "unsupported_values": 7 |
| }, |
| "exact": 0.75, |
| "f1": 0.8309178743961353, |
| "false_abstention_rate": 0.03409090909090909, |
| "field_accuracy": 0.8775510204081632, |
| "json_parse_rate": 0.9666666666666667, |
| "missed_abstention_rate": 0.1016949152542373, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.07145761149877217, |
| "precision": 0.8686868686868687, |
| "present_accuracy": 0.8636363636363636, |
| "recall": 0.7962962962962963, |
| "schema_valid_rate": 0.9666666666666667, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0707070707070707 |
| }, |
| "variant/canonical": { |
| "absent_accuracy": 0.8983050847457628, |
| "count": 60, |
| "counts": { |
| "absent_correct": 53, |
| "absent_total": 59, |
| "exact": 45, |
| "false_abstention": 3, |
| "field_correct": 129, |
| "fields": 147, |
| "fn": 22, |
| "fp": 13, |
| "json_parse_ok": 58, |
| "missed_abstention": 6, |
| "natural_stop": 60, |
| "nonrepetitive_nonempty": 60, |
| "predicted_values": 99, |
| "present_correct": 76, |
| "present_total": 88, |
| "schema_ok": 58, |
| "token_limited": 0, |
| "tp": 86, |
| "unsupported_values": 7 |
| }, |
| "exact": 0.75, |
| "f1": 0.8309178743961353, |
| "false_abstention_rate": 0.03409090909090909, |
| "field_accuracy": 0.8775510204081632, |
| "json_parse_rate": 0.9666666666666667, |
| "missed_abstention_rate": 0.1016949152542373, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.07145761149877217, |
| "precision": 0.8686868686868687, |
| "present_accuracy": 0.8636363636363636, |
| "recall": 0.7962962962962963, |
| "schema_valid_rate": 0.9666666666666667, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0707070707070707 |
| } |
| }, |
| "subgroups": { |
| "original1444": { |
| "parent105_64": {}, |
| "replay128_200": {}, |
| "replay128_300": {} |
| }, |
| "playground60": { |
| "parent105_64": { |
| "added6": { |
| "absent_accuracy": 0.6666666666666666, |
| "count": 6, |
| "counts": { |
| "absent_correct": 4, |
| "absent_total": 6, |
| "exact": 1, |
| "false_abstention": 0, |
| "field_correct": 5, |
| "fields": 12, |
| "fn": 15, |
| "fp": 4, |
| "json_parse_ok": 4, |
| "missed_abstention": 2, |
| "natural_stop": 6, |
| "nonrepetitive_nonempty": 6, |
| "predicted_values": 13, |
| "present_correct": 1, |
| "present_total": 6, |
| "schema_ok": 4, |
| "token_limited": 0, |
| "tp": 9, |
| "unsupported_values": 4 |
| }, |
| "exact": 0.16666666666666666, |
| "f1": 0.48648648648648646, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.4166666666666667, |
| "json_parse_rate": 0.6666666666666666, |
| "missed_abstention_rate": 0.3333333333333333, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.16294185755153498, |
| "precision": 0.6923076923076923, |
| "present_accuracy": 0.16666666666666666, |
| "recall": 0.375, |
| "schema_valid_rate": 0.6666666666666666, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.3076923076923077 |
| }, |
| "array3": { |
| "absent_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 2, |
| "exact": 1, |
| "false_abstention": 0, |
| "field_correct": 3, |
| "fields": 4, |
| "fn": 1, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_abstention": 0, |
| "natural_stop": 2, |
| "nonrepetitive_nonempty": 2, |
| "predicted_values": 6, |
| "present_correct": 1, |
| "present_total": 2, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 5, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.5, |
| "f1": 0.8333333333333334, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.75, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.12317727319896221, |
| "precision": 0.8333333333333334, |
| "present_accuracy": 0.5, |
| "recall": 0.8333333333333334, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.16666666666666666 |
| }, |
| "array4": { |
| "absent_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 1, |
| "absent_total": 2, |
| "exact": 0, |
| "false_abstention": 0, |
| "field_correct": 1, |
| "fields": 4, |
| "fn": 6, |
| "fp": 2, |
| "json_parse_ok": 1, |
| "missed_abstention": 1, |
| "natural_stop": 2, |
| "nonrepetitive_nonempty": 2, |
| "predicted_values": 4, |
| "present_correct": 0, |
| "present_total": 2, |
| "schema_ok": 1, |
| "token_limited": 0, |
| "tp": 2, |
| "unsupported_values": 2 |
| }, |
| "exact": 0.0, |
| "f1": 0.3333333333333333, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.25, |
| "json_parse_rate": 0.5, |
| "missed_abstention_rate": 0.5, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.1759970560669899, |
| "precision": 0.5, |
| "present_accuracy": 0.0, |
| "recall": 0.25, |
| "schema_valid_rate": 0.5, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.5 |
| }, |
| "array5": { |
| "absent_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 1, |
| "absent_total": 2, |
| "exact": 0, |
| "false_abstention": 0, |
| "field_correct": 1, |
| "fields": 4, |
| "fn": 8, |
| "fp": 1, |
| "json_parse_ok": 1, |
| "missed_abstention": 1, |
| "natural_stop": 2, |
| "nonrepetitive_nonempty": 2, |
| "predicted_values": 3, |
| "present_correct": 0, |
| "present_total": 2, |
| "schema_ok": 1, |
| "token_limited": 0, |
| "tp": 2, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.0, |
| "f1": 0.30769230769230765, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.25, |
| "json_parse_rate": 0.5, |
| "missed_abstention_rate": 0.5, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.1896512433886528, |
| "precision": 0.6666666666666666, |
| "present_accuracy": 0.0, |
| "recall": 0.2, |
| "schema_valid_rate": 0.5, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.3333333333333333 |
| }, |
| "original54": { |
| "absent_accuracy": 0.9245283018867925, |
| "count": 54, |
| "counts": { |
| "absent_correct": 49, |
| "absent_total": 53, |
| "exact": 44, |
| "false_abstention": 3, |
| "field_correct": 124, |
| "fields": 135, |
| "fn": 7, |
| "fp": 9, |
| "json_parse_ok": 54, |
| "missed_abstention": 4, |
| "natural_stop": 54, |
| "nonrepetitive_nonempty": 54, |
| "predicted_values": 86, |
| "present_correct": 75, |
| "present_total": 82, |
| "schema_ok": 54, |
| "token_limited": 0, |
| "tp": 77, |
| "unsupported_values": 3 |
| }, |
| "exact": 0.8148148148148148, |
| "f1": 0.9058823529411765, |
| "false_abstention_rate": 0.036585365853658534, |
| "field_accuracy": 0.9185185185185185, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.07547169811320754, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.06129269527068741, |
| "precision": 0.8953488372093024, |
| "present_accuracy": 0.9146341463414634, |
| "recall": 0.9166666666666666, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.03488372093023256 |
| } |
| }, |
| "replay128_200": { |
| "added6": { |
| "absent_accuracy": 1.0, |
| "count": 6, |
| "counts": { |
| "absent_correct": 6, |
| "absent_total": 6, |
| "exact": 3, |
| "false_abstention": 0, |
| "field_correct": 9, |
| "fields": 12, |
| "fn": 5, |
| "fp": 3, |
| "json_parse_ok": 6, |
| "missed_abstention": 0, |
| "natural_stop": 6, |
| "nonrepetitive_nonempty": 6, |
| "predicted_values": 22, |
| "present_correct": 3, |
| "present_total": 6, |
| "schema_ok": 6, |
| "token_limited": 0, |
| "tp": 19, |
| "unsupported_values": 3 |
| }, |
| "exact": 0.5, |
| "f1": 0.8260869565217391, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.75, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.10507260775193572, |
| "precision": 0.8636363636363636, |
| "present_accuracy": 0.5, |
| "recall": 0.7916666666666666, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.13636363636363635 |
| }, |
| "array3": { |
| "absent_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 2, |
| "exact": 1, |
| "false_abstention": 0, |
| "field_correct": 3, |
| "fields": 4, |
| "fn": 1, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_abstention": 0, |
| "natural_stop": 2, |
| "nonrepetitive_nonempty": 2, |
| "predicted_values": 6, |
| "present_correct": 1, |
| "present_total": 2, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 5, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.5, |
| "f1": 0.8333333333333334, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.75, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.0854416680522263, |
| "precision": 0.8333333333333334, |
| "present_accuracy": 0.5, |
| "recall": 0.8333333333333334, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.16666666666666666 |
| }, |
| "array4": { |
| "absent_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 2, |
| "exact": 1, |
| "false_abstention": 0, |
| "field_correct": 3, |
| "fields": 4, |
| "fn": 1, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_abstention": 0, |
| "natural_stop": 2, |
| "nonrepetitive_nonempty": 2, |
| "predicted_values": 8, |
| "present_correct": 1, |
| "present_total": 2, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 7, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.5, |
| "f1": 0.875, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.75, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.1231730468571186, |
| "precision": 0.875, |
| "present_accuracy": 0.5, |
| "recall": 0.875, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.125 |
| }, |
| "array5": { |
| "absent_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 2, |
| "exact": 1, |
| "false_abstention": 0, |
| "field_correct": 3, |
| "fields": 4, |
| "fn": 3, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_abstention": 0, |
| "natural_stop": 2, |
| "nonrepetitive_nonempty": 2, |
| "predicted_values": 8, |
| "present_correct": 1, |
| "present_total": 2, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 7, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.5, |
| "f1": 0.7777777777777777, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.75, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.10660310834646225, |
| "precision": 0.875, |
| "present_accuracy": 0.5, |
| "recall": 0.7, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.125 |
| }, |
| "original54": { |
| "absent_accuracy": 0.9056603773584906, |
| "count": 54, |
| "counts": { |
| "absent_correct": 48, |
| "absent_total": 53, |
| "exact": 45, |
| "false_abstention": 1, |
| "field_correct": 126, |
| "fields": 135, |
| "fn": 4, |
| "fp": 9, |
| "json_parse_ok": 54, |
| "missed_abstention": 5, |
| "natural_stop": 54, |
| "nonrepetitive_nonempty": 54, |
| "predicted_values": 89, |
| "present_correct": 78, |
| "present_total": 82, |
| "schema_ok": 54, |
| "token_limited": 0, |
| "tp": 80, |
| "unsupported_values": 3 |
| }, |
| "exact": 0.8333333333333334, |
| "f1": 0.9248554913294796, |
| "false_abstention_rate": 0.012195121951219513, |
| "field_accuracy": 0.9333333333333333, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.09433962264150944, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.038012701103407716, |
| "precision": 0.898876404494382, |
| "present_accuracy": 0.9512195121951219, |
| "recall": 0.9523809523809523, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.033707865168539325 |
| } |
| }, |
| "replay128_300": { |
| "added6": { |
| "absent_accuracy": 1.0, |
| "count": 6, |
| "counts": { |
| "absent_correct": 6, |
| "absent_total": 6, |
| "exact": 3, |
| "false_abstention": 0, |
| "field_correct": 9, |
| "fields": 12, |
| "fn": 5, |
| "fp": 3, |
| "json_parse_ok": 6, |
| "missed_abstention": 0, |
| "natural_stop": 6, |
| "nonrepetitive_nonempty": 6, |
| "predicted_values": 22, |
| "present_correct": 3, |
| "present_total": 6, |
| "schema_ok": 6, |
| "token_limited": 0, |
| "tp": 19, |
| "unsupported_values": 3 |
| }, |
| "exact": 0.5, |
| "f1": 0.8260869565217391, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.75, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.10285404759148757, |
| "precision": 0.8636363636363636, |
| "present_accuracy": 0.5, |
| "recall": 0.7916666666666666, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.13636363636363635 |
| }, |
| "array3": { |
| "absent_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 2, |
| "exact": 1, |
| "false_abstention": 0, |
| "field_correct": 3, |
| "fields": 4, |
| "fn": 1, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_abstention": 0, |
| "natural_stop": 2, |
| "nonrepetitive_nonempty": 2, |
| "predicted_values": 6, |
| "present_correct": 1, |
| "present_total": 2, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 5, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.5, |
| "f1": 0.8333333333333334, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.75, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.08301506005227566, |
| "precision": 0.8333333333333334, |
| "present_accuracy": 0.5, |
| "recall": 0.8333333333333334, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.16666666666666666 |
| }, |
| "array4": { |
| "absent_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 2, |
| "exact": 1, |
| "false_abstention": 0, |
| "field_correct": 3, |
| "fields": 4, |
| "fn": 1, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_abstention": 0, |
| "natural_stop": 2, |
| "nonrepetitive_nonempty": 2, |
| "predicted_values": 8, |
| "present_correct": 1, |
| "present_total": 2, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 7, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.5, |
| "f1": 0.875, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.75, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.1220800168812275, |
| "precision": 0.875, |
| "present_accuracy": 0.5, |
| "recall": 0.875, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.125 |
| }, |
| "array5": { |
| "absent_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 2, |
| "exact": 1, |
| "false_abstention": 0, |
| "field_correct": 3, |
| "fields": 4, |
| "fn": 3, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_abstention": 0, |
| "natural_stop": 2, |
| "nonrepetitive_nonempty": 2, |
| "predicted_values": 8, |
| "present_correct": 1, |
| "present_total": 2, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 7, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.5, |
| "f1": 0.7777777777777777, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.75, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.10346706584095955, |
| "precision": 0.875, |
| "present_accuracy": 0.5, |
| "recall": 0.7, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.125 |
| }, |
| "original54": { |
| "absent_accuracy": 0.9245283018867925, |
| "count": 54, |
| "counts": { |
| "absent_correct": 49, |
| "absent_total": 53, |
| "exact": 46, |
| "false_abstention": 2, |
| "field_correct": 127, |
| "fields": 135, |
| "fn": 4, |
| "fp": 7, |
| "json_parse_ok": 54, |
| "missed_abstention": 4, |
| "natural_stop": 54, |
| "nonrepetitive_nonempty": 54, |
| "predicted_values": 87, |
| "present_correct": 78, |
| "present_total": 82, |
| "schema_ok": 54, |
| "token_limited": 0, |
| "tp": 80, |
| "unsupported_values": 3 |
| }, |
| "exact": 0.8518518518518519, |
| "f1": 0.935672514619883, |
| "false_abstention_rate": 0.024390243902439025, |
| "field_accuracy": 0.9407407407407408, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.07547169811320754, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.040062436762296046, |
| "precision": 0.9195402298850575, |
| "present_accuracy": 0.9512195121951219, |
| "recall": 0.9523809523809523, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.034482758620689655 |
| } |
| } |
| } |
| } |
| }, |
| "protocol": { |
| "constrained_decoding": false, |
| "json_assembly": false, |
| "max_tokens": 256, |
| "one_generation_per_request": true, |
| "output_repair": false, |
| "prompt_truncation": false, |
| "repetition_penalty": 1, |
| "temperature": 0 |
| }, |
| "scope": "Previously observed development, SNIPS-derived held-out and playground research panels, not fresh blind external benchmarks. Source text, schema and fixed instruction only; gold values excluded. One raw generation; no JSON assembly, repair or constrained decoding.", |
| "selection": "Owner-selected aggregate improvement; strict zero-regression gate did not pass.", |
| "training_prefix": { |
| "records": 19200, |
| "serialized_positions": 6153194, |
| "supervised_assistant_positions": 633561, |
| "components": { |
| "original_json": 13640, |
| "new_prose": 4871, |
| "cardinality_replay": 689 |
| } |
| } |
| } |
|
|
