| { |
| "model": "OpenSML-JSONLite-175M-Instruct", |
| "checkpoint": "main", |
| "internal_checkpoint": "153/500", |
| "model_sha256": "25b406a7b86e3e8ac4dd0b7f32e27590d9d11349a0148a7815c32304150ac725", |
| "training_prefix": { |
| "update": 500, |
| "presentations": 32000, |
| "distinct_source_texts": 21392, |
| "source_groups": 19589, |
| "components": { |
| "atomic": 2844, |
| "composite": 5689, |
| "grounded_qa": 2667, |
| "massive": 2667, |
| "prior_authored": 2133, |
| "schema_variation": 2667, |
| "snips": 13333 |
| }, |
| "supervised_positions_metadata": 1027385, |
| "counter_scope": "Checkpoint metadata tokens counts supervised assistant/EOS positions, not serialized source/schema positions.", |
| "train_sha256": "dd69dac2fa72ef5286df8cd513730f751c4d45835947e04173db130f906451a4", |
| "batches_sha256": "bff71b6a94d13dae8f5c766145f4f1d59074665f92a1ff1916d138af27cd0268" |
| }, |
| "panels": { |
| "original1444": { |
| "metrics": { |
| "all": { |
| "absent_accuracy": 0.9613368283093053, |
| "count": 1444, |
| "counts": { |
| "absent_correct": 1467, |
| "absent_total": 1526, |
| "exact": 1293, |
| "false_abstention": 49, |
| "field_correct": 3319, |
| "fields": 3497, |
| "fn": 119, |
| "fp": 128, |
| "json_parse_ok": 1444, |
| "missed_abstention": 59, |
| "natural_stop": 1444, |
| "nonrepetitive_nonempty": 1444, |
| "predicted_values": 1980, |
| "present_correct": 1852, |
| "present_total": 1971, |
| "schema_ok": 1443, |
| "token_limited": 0, |
| "tp": 1852, |
| "unsupported_values": 20 |
| }, |
| "exact": 0.8954293628808865, |
| "f1": 0.9374841812199443, |
| "false_abstention_rate": 0.024860476915271434, |
| "field_accuracy": 0.9490992279096369, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.03866317169069463, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.0213116571704629, |
| "precision": 0.9353535353535354, |
| "present_accuracy": 0.9396245560629122, |
| "recall": 0.9396245560629122, |
| "schema_valid_rate": 0.9993074792243767, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.010101010101010102 |
| }, |
| "domain/AddToPlaylist": { |
| "absent_accuracy": 0.9449152542372882, |
| "count": 206, |
| "counts": { |
| "absent_correct": 223, |
| "absent_total": 236, |
| "exact": 179, |
| "false_abstention": 5, |
| "field_correct": 492, |
| "fields": 526, |
| "fn": 21, |
| "fp": 29, |
| "json_parse_ok": 206, |
| "missed_abstention": 13, |
| "natural_stop": 206, |
| "nonrepetitive_nonempty": 206, |
| "predicted_values": 298, |
| "present_correct": 269, |
| "present_total": 290, |
| "schema_ok": 206, |
| "token_limited": 0, |
| "tp": 269, |
| "unsupported_values": 6 |
| }, |
| "exact": 0.8689320388349514, |
| "f1": 0.9149659863945578, |
| "false_abstention_rate": 0.017241379310344827, |
| "field_accuracy": 0.935361216730038, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.05508474576271186, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.033199904241485556, |
| "precision": 0.9026845637583892, |
| "present_accuracy": 0.9275862068965517, |
| "recall": 0.9275862068965517, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.020134228187919462 |
| }, |
| "domain/BookRestaurant": { |
| "absent_accuracy": 0.94, |
| "count": 208, |
| "counts": { |
| "absent_correct": 235, |
| "absent_total": 250, |
| "exact": 173, |
| "false_abstention": 7, |
| "field_correct": 532, |
| "fields": 569, |
| "fn": 22, |
| "fp": 30, |
| "json_parse_ok": 208, |
| "missed_abstention": 15, |
| "natural_stop": 208, |
| "nonrepetitive_nonempty": 208, |
| "predicted_values": 327, |
| "present_correct": 297, |
| "present_total": 319, |
| "schema_ok": 208, |
| "token_limited": 0, |
| "tp": 297, |
| "unsupported_values": 3 |
| }, |
| "exact": 0.8317307692307693, |
| "f1": 0.9195046439628484, |
| "false_abstention_rate": 0.0219435736677116, |
| "field_accuracy": 0.9349736379613357, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.06, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.021212414096645753, |
| "precision": 0.908256880733945, |
| "present_accuracy": 0.9310344827586207, |
| "recall": 0.9310344827586207, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.009174311926605505 |
| }, |
| "domain/GetWeather": { |
| "absent_accuracy": 0.9512195121951219, |
| "count": 206, |
| "counts": { |
| "absent_correct": 234, |
| "absent_total": 246, |
| "exact": 186, |
| "false_abstention": 5, |
| "field_correct": 500, |
| "fields": 525, |
| "fn": 13, |
| "fp": 19, |
| "json_parse_ok": 206, |
| "missed_abstention": 12, |
| "natural_stop": 206, |
| "nonrepetitive_nonempty": 206, |
| "predicted_values": 285, |
| "present_correct": 266, |
| "present_total": 279, |
| "schema_ok": 205, |
| "token_limited": 0, |
| "tp": 266, |
| "unsupported_values": 4 |
| }, |
| "exact": 0.9029126213592233, |
| "f1": 0.9432624113475178, |
| "false_abstention_rate": 0.017921146953405017, |
| "field_accuracy": 0.9523809523809523, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.04878048780487805, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.018259013971190636, |
| "precision": 0.9333333333333333, |
| "present_accuracy": 0.953405017921147, |
| "recall": 0.953405017921147, |
| "schema_valid_rate": 0.9951456310679612, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.014035087719298246 |
| }, |
| "domain/PlayMusic": { |
| "absent_accuracy": 0.9453125, |
| "count": 208, |
| "counts": { |
| "absent_correct": 242, |
| "absent_total": 256, |
| "exact": 169, |
| "false_abstention": 22, |
| "field_correct": 471, |
| "fields": 523, |
| "fn": 38, |
| "fp": 30, |
| "json_parse_ok": 208, |
| "missed_abstention": 14, |
| "natural_stop": 208, |
| "nonrepetitive_nonempty": 208, |
| "predicted_values": 259, |
| "present_correct": 229, |
| "present_total": 267, |
| "schema_ok": 208, |
| "token_limited": 0, |
| "tp": 229, |
| "unsupported_values": 5 |
| }, |
| "exact": 0.8125, |
| "f1": 0.8707224334600762, |
| "false_abstention_rate": 0.08239700374531835, |
| "field_accuracy": 0.9005736137667304, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0546875, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.03351788995986191, |
| "precision": 0.8841698841698842, |
| "present_accuracy": 0.8576779026217228, |
| "recall": 0.8576779026217228, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.019305019305019305 |
| }, |
| "domain/RateBook": { |
| "absent_accuracy": 0.9959349593495935, |
| "count": 204, |
| "counts": { |
| "absent_correct": 245, |
| "absent_total": 246, |
| "exact": 199, |
| "false_abstention": 0, |
| "field_correct": 562, |
| "fields": 567, |
| "fn": 4, |
| "fp": 5, |
| "json_parse_ok": 204, |
| "missed_abstention": 1, |
| "natural_stop": 204, |
| "nonrepetitive_nonempty": 204, |
| "predicted_values": 322, |
| "present_correct": 317, |
| "present_total": 321, |
| "schema_ok": 204, |
| "token_limited": 0, |
| "tp": 317, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.9754901960784313, |
| "f1": 0.9860031104199067, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.9911816578483245, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0040650406504065045, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.006796475639252914, |
| "precision": 0.984472049689441, |
| "present_accuracy": 0.9875389408099688, |
| "recall": 0.9875389408099688, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.003105590062111801 |
| }, |
| "domain/SearchCreativeWork": { |
| "absent_accuracy": 1.0, |
| "count": 208, |
| "counts": { |
| "absent_correct": 39, |
| "absent_total": 39, |
| "exact": 200, |
| "false_abstention": 0, |
| "field_correct": 278, |
| "fields": 286, |
| "fn": 8, |
| "fp": 8, |
| "json_parse_ok": 208, |
| "missed_abstention": 0, |
| "natural_stop": 208, |
| "nonrepetitive_nonempty": 208, |
| "predicted_values": 247, |
| "present_correct": 239, |
| "present_total": 247, |
| "schema_ok": 208, |
| "token_limited": 0, |
| "tp": 239, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.9615384615384616, |
| "f1": 0.9676113360323887, |
| "false_abstention_rate": 0.0, |
| "field_accuracy": 0.972027972027972, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.014501851789401162, |
| "precision": 0.9676113360323887, |
| "present_accuracy": 0.9676113360323887, |
| "recall": 0.9676113360323887, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.004048582995951417 |
| }, |
| "domain/SearchScreeningEvent": { |
| "absent_accuracy": 0.9841897233201581, |
| "count": 204, |
| "counts": { |
| "absent_correct": 249, |
| "absent_total": 253, |
| "exact": 187, |
| "false_abstention": 10, |
| "field_correct": 484, |
| "fields": 501, |
| "fn": 13, |
| "fp": 7, |
| "json_parse_ok": 204, |
| "missed_abstention": 4, |
| "natural_stop": 204, |
| "nonrepetitive_nonempty": 204, |
| "predicted_values": 242, |
| "present_correct": 235, |
| "present_total": 248, |
| "schema_ok": 204, |
| "token_limited": 0, |
| "tp": 235, |
| "unsupported_values": 0 |
| }, |
| "exact": 0.9166666666666666, |
| "f1": 0.9591836734693877, |
| "false_abstention_rate": 0.04032258064516129, |
| "field_accuracy": 0.9660678642714571, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.015810276679841896, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.021503560568531862, |
| "precision": 0.9710743801652892, |
| "present_accuracy": 0.9475806451612904, |
| "recall": 0.9475806451612904, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.0 |
| }, |
| "macro_exact": 0.895681536244072, |
| "paired_requests": { |
| "accuracy": 0.8328530259365994, |
| "both_correct": 578, |
| "count": 694 |
| }, |
| "variant/alias": { |
| "absent_accuracy": 1.0, |
| "count": 28, |
| "counts": { |
| "absent_correct": 29, |
| "absent_total": 29, |
| "exact": 24, |
| "false_abstention": 1, |
| "field_correct": 70, |
| "fields": 75, |
| "fn": 5, |
| "fp": 3, |
| "json_parse_ok": 28, |
| "missed_abstention": 0, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 44, |
| "present_correct": 41, |
| "present_total": 46, |
| "schema_ok": 27, |
| "token_limited": 0, |
| "tp": 41, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.8571428571428571, |
| "f1": 0.9111111111111111, |
| "false_abstention_rate": 0.021739130434782608, |
| "field_accuracy": 0.9333333333333333, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.0, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.0613402272782488, |
| "precision": 0.9318181818181818, |
| "present_accuracy": 0.8913043478260869, |
| "recall": 0.8913043478260869, |
| "schema_valid_rate": 0.9642857142857143, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.022727272727272728 |
| }, |
| "variant/canonical": { |
| "absent_accuracy": 0.9625340599455041, |
| "count": 1388, |
| "counts": { |
| "absent_correct": 1413, |
| "absent_total": 1468, |
| "exact": 1247, |
| "false_abstention": 47, |
| "field_correct": 3181, |
| "fields": 3347, |
| "fn": 111, |
| "fp": 119, |
| "json_parse_ok": 1388, |
| "missed_abstention": 55, |
| "natural_stop": 1388, |
| "nonrepetitive_nonempty": 1388, |
| "predicted_values": 1887, |
| "present_correct": 1768, |
| "present_total": 1879, |
| "schema_ok": 1388, |
| "token_limited": 0, |
| "tp": 1768, |
| "unsupported_values": 18 |
| }, |
| "exact": 0.8984149855907781, |
| "f1": 0.9389272437599575, |
| "false_abstention_rate": 0.025013304949441192, |
| "field_accuracy": 0.9504033462802509, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.03746594005449591, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.019958412440256805, |
| "precision": 0.9369369369369369, |
| "present_accuracy": 0.940926024481107, |
| "recall": 0.940926024481107, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.009538950715421303 |
| }, |
| "variant/nullable": { |
| "absent_accuracy": 0.8620689655172413, |
| "count": 28, |
| "counts": { |
| "absent_correct": 25, |
| "absent_total": 29, |
| "exact": 22, |
| "false_abstention": 1, |
| "field_correct": 68, |
| "fields": 75, |
| "fn": 3, |
| "fp": 6, |
| "json_parse_ok": 28, |
| "missed_abstention": 4, |
| "natural_stop": 28, |
| "nonrepetitive_nonempty": 28, |
| "predicted_values": 49, |
| "present_correct": 43, |
| "present_total": 46, |
| "schema_ok": 28, |
| "token_limited": 0, |
| "tp": 43, |
| "unsupported_values": 1 |
| }, |
| "exact": 0.7857142857142857, |
| "f1": 0.9052631578947369, |
| "false_abstention_rate": 0.021739130434782608, |
| "field_accuracy": 0.9066666666666666, |
| "json_parse_rate": 1.0, |
| "missed_abstention_rate": 0.13793103448275862, |
| "natural_stop_rate": 1.0, |
| "nonrepetitive_nonempty_rate": 1.0, |
| "output_nll": 0.048365361545750475, |
| "precision": 0.8775510204081632, |
| "present_accuracy": 0.9347826086956522, |
| "recall": 0.9347826086956522, |
| "schema_valid_rate": 1.0, |
| "token_limited_rate": 0.0, |
| "unsupported_value_rate": 0.02040816326530612 |
| } |
| }, |
| "identity": { |
| "assembly": false, |
| "context": 2048, |
| "input_files": { |
| "reports/model-design/SFT-128_joint_extraction_20261016/release-comparison/original1444.jsonl": "083513ab76421287337ec56cb2b293a986894db7f8955be256342602b3307241" |
| }, |
| "max_output_tokens": 256, |
| "mlx": "0.32.3", |
| "model": "153/500", |
| "model_calls_per_request": 1, |
| "raw_greedy": true, |
| "references_in_input": false, |
| "repair": false, |
| "repetition_penalty": 1, |
| "requests_sha256": "083513ab76421287337ec56cb2b293a986894db7f8955be256342602b3307241", |
| "runner_sha256": "843fc245da9134d1252ab4ca9182742c66a5ec20abfe15dfdef4bff5cc5cab0e", |
| "scorer_sha256": "1f4d2c69ebc640f993b77cb09ecf280d2df1dbf3f20d1060f7c8a5ce6780dd97", |
| "selected_by_development": true, |
| "serialization": "native-sft-chat-v1", |
| "shared_decoder_sha256": "5a3826b2124f1eea2832a4d753a227c7f5f6c62bf882ffdb800cd6e20aeb6ce2", |
| "step": 500, |
| "structural_scorer_sha256": "334544f6a42d7b2c65840bb90d488e87c15d185900e8ceec41e24e24a3984505", |
| "teachers_at_inference": false, |
| "temperature": 0, |
| "tokenizer_sha256": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb", |
| "truncation": false, |
| "weight_sha256": "25b406a7b86e3e8ac4dd0b7f32e27590d9d11349a0148a7815c32304150ac725" |
| } |
| }, |
| "playground108": { |
| "metrics": { |
| "all": { |
| "complete_request_accuracy": 0.7222222222222222, |
| "count": 108, |
| "counts": { |
| "absent_correct": 62, |
| "absent_total": 67, |
| "exact": 78, |
| "false_abstentions": 1, |
| "field_correct": 189, |
| "fields": 225, |
| "fn": 72, |
| "fp": 32, |
| "json_parse_ok": 104, |
| "missed_absences": 4, |
| "natural_stop": 108, |
| "optional_omitted_correct": 3, |
| "optional_omitted_total": 3, |
| "predicted_strings": 170, |
| "schema_ok": 94, |
| "token_limited": 0, |
| "tp": 222, |
| "unsupported_strings": 8 |
| }, |
| "json_parse_rate": 0.9629629629629629, |
| "missing_value_accuracy": 0.9253731343283582, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 1.0, |
| "schema_valid_rate": 0.8703703703703703, |
| "structural_path_f1": 0.8102189781021898, |
| "structural_path_precision": 0.8740157480314961, |
| "structural_path_recall": 0.7551020408163265, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.84, |
| "unsupported_string_rate": 0.047058823529411764 |
| }, |
| "difficulty/Hard": { |
| "complete_request_accuracy": 0.4444444444444444, |
| "count": 18, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 8, |
| "false_abstentions": 0, |
| "field_correct": 12, |
| "fields": 25, |
| "fn": 31, |
| "fp": 9, |
| "json_parse_ok": 14, |
| "missed_absences": 0, |
| "natural_stop": 18, |
| "optional_omitted_correct": 2, |
| "optional_omitted_total": 2, |
| "predicted_strings": 24, |
| "schema_ok": 9, |
| "token_limited": 0, |
| "tp": 22, |
| "unsupported_strings": 1 |
| }, |
| "json_parse_rate": 0.7777777777777778, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 1.0, |
| "schema_valid_rate": 0.5, |
| "structural_path_f1": 0.5238095238095238, |
| "structural_path_precision": 0.7096774193548387, |
| "structural_path_recall": 0.41509433962264153, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.48, |
| "unsupported_string_rate": 0.041666666666666664 |
| }, |
| "difficulty/Medium": { |
| "complete_request_accuracy": 0.5625, |
| "count": 16, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 3, |
| "exact": 9, |
| "false_abstentions": 0, |
| "field_correct": 19, |
| "fields": 29, |
| "fn": 20, |
| "fp": 10, |
| "json_parse_ok": 16, |
| "missed_absences": 0, |
| "natural_stop": 16, |
| "optional_omitted_correct": 1, |
| "optional_omitted_total": 1, |
| "predicted_strings": 29, |
| "schema_ok": 13, |
| "token_limited": 0, |
| "tp": 24, |
| "unsupported_strings": 2 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.6666666666666666, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 1.0, |
| "schema_valid_rate": 0.8125, |
| "structural_path_f1": 0.6153846153846154, |
| "structural_path_precision": 0.7058823529411765, |
| "structural_path_recall": 0.5454545454545454, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.6551724137931034, |
| "unsupported_string_rate": 0.06896551724137931 |
| }, |
| "difficulty/Original": { |
| "complete_request_accuracy": 0.8166666666666667, |
| "count": 60, |
| "counts": { |
| "absent_correct": 55, |
| "absent_total": 59, |
| "exact": 49, |
| "false_abstentions": 1, |
| "field_correct": 136, |
| "fields": 147, |
| "fn": 17, |
| "fp": 11, |
| "json_parse_ok": 60, |
| "missed_absences": 4, |
| "natural_stop": 60, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 105, |
| "schema_ok": 60, |
| "token_limited": 0, |
| "tp": 150, |
| "unsupported_strings": 5 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.9322033898305084, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.9146341463414634, |
| "structural_path_precision": 0.9316770186335404, |
| "structural_path_recall": 0.8982035928143712, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.9251700680272109, |
| "unsupported_string_rate": 0.047619047619047616 |
| }, |
| "difficulty/Simple": { |
| "complete_request_accuracy": 0.8571428571428571, |
| "count": 14, |
| "counts": { |
| "absent_correct": 5, |
| "absent_total": 5, |
| "exact": 12, |
| "false_abstentions": 0, |
| "field_correct": 22, |
| "fields": 24, |
| "fn": 4, |
| "fp": 2, |
| "json_parse_ok": 14, |
| "missed_absences": 0, |
| "natural_stop": 14, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 12, |
| "schema_ok": 12, |
| "token_limited": 0, |
| "tp": 26, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 1.0, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.8571428571428571, |
| "structural_path_f1": 0.896551724137931, |
| "structural_path_precision": 0.9285714285714286, |
| "structural_path_recall": 0.8666666666666667, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.9166666666666666, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/01 · Flat strings": { |
| "complete_request_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 2, |
| "false_abstentions": 0, |
| "field_correct": 4, |
| "fields": 4, |
| "fn": 0, |
| "fp": 0, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 4, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 1.0, |
| "structural_path_precision": 1.0, |
| "structural_path_recall": 1.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 1.0, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/02 · Integers and decimals": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 3, |
| "fields": 4, |
| "fn": 1, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 1, |
| "schema_ok": 1, |
| "token_limited": 0, |
| "tp": 3, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5, |
| "structural_path_f1": 0.75, |
| "structural_path_precision": 0.75, |
| "structural_path_recall": 0.75, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.75, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/03 · Booleans": { |
| "complete_request_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 2, |
| "false_abstentions": 0, |
| "field_correct": 4, |
| "fields": 4, |
| "fn": 0, |
| "fp": 0, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 0, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 1.0, |
| "structural_path_precision": 1.0, |
| "structural_path_recall": 1.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 1.0, |
| "unsupported_string_rate": null |
| }, |
| "family/04 · Nullable values": { |
| "complete_request_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 2, |
| "exact": 2, |
| "false_abstentions": 0, |
| "field_correct": 4, |
| "fields": 4, |
| "fn": 0, |
| "fp": 0, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 2, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 1.0, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 1.0, |
| "structural_path_precision": 1.0, |
| "structural_path_recall": 1.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 1.0, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/05 · String arrays": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 1, |
| "fields": 2, |
| "fn": 5, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 4, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 3, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.5, |
| "structural_path_precision": 0.75, |
| "structural_path_recall": 0.375, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/06 · Numeric and boolean arrays": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 1, |
| "fields": 2, |
| "fn": 3, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 1, |
| "schema_ok": 1, |
| "token_limited": 0, |
| "tp": 3, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5, |
| "structural_path_f1": 0.6, |
| "structural_path_precision": 0.75, |
| "structural_path_recall": 0.5, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/07 · Nested objects": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 1, |
| "fields": 2, |
| "fn": 3, |
| "fp": 2, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 5, |
| "schema_ok": 1, |
| "token_limited": 0, |
| "tp": 3, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5, |
| "structural_path_f1": 0.5454545454545454, |
| "structural_path_precision": 0.6, |
| "structural_path_recall": 0.5, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/08 · Deeper nesting": { |
| "complete_request_accuracy": 0.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 0, |
| "false_abstentions": 0, |
| "field_correct": 0, |
| "fields": 2, |
| "fn": 5, |
| "fp": 3, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 4, |
| "schema_ok": 0, |
| "token_limited": 0, |
| "tp": 1, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.0, |
| "structural_path_f1": 0.2, |
| "structural_path_precision": 0.25, |
| "structural_path_recall": 0.16666666666666666, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.0, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/09 · Arrays of objects": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 1, |
| "fields": 2, |
| "fn": 6, |
| "fp": 3, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 7, |
| "schema_ok": 1, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 1 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5, |
| "structural_path_f1": 0.47058823529411764, |
| "structural_path_precision": 0.5714285714285714, |
| "structural_path_recall": 0.4, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5, |
| "unsupported_string_rate": 0.14285714285714285 |
| }, |
| "family/10 · Multiple arrays and objects": { |
| "complete_request_accuracy": 0.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 0, |
| "false_abstentions": 0, |
| "field_correct": 1, |
| "fields": 5, |
| "fn": 8, |
| "fp": 2, |
| "json_parse_ok": 1, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 3, |
| "schema_ok": 1, |
| "token_limited": 0, |
| "tp": 1, |
| "unsupported_strings": 1 |
| }, |
| "json_parse_rate": 0.5, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5, |
| "structural_path_f1": 0.16666666666666666, |
| "structural_path_precision": 0.3333333333333333, |
| "structural_path_recall": 0.1111111111111111, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.2, |
| "unsupported_string_rate": 0.3333333333333333 |
| }, |
| "family/11 · Arrays of arrays": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 1, |
| "fields": 2, |
| "fn": 4, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 1, |
| "schema_ok": 1, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5, |
| "structural_path_f1": 0.6153846153846154, |
| "structural_path_precision": 0.8, |
| "structural_path_recall": 0.5, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/12 · Mixed arrays and tuples": { |
| "complete_request_accuracy": 0.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 0, |
| "false_abstentions": 0, |
| "field_correct": 0, |
| "fields": 2, |
| "fn": 7, |
| "fp": 1, |
| "json_parse_ok": 1, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 1, |
| "schema_ok": 0, |
| "token_limited": 0, |
| "tp": 0, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 0.5, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.0, |
| "structural_path_f1": 0.0, |
| "structural_path_precision": 0.0, |
| "structural_path_recall": 0.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.0, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/13 · Enum and const": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 4, |
| "fields": 6, |
| "fn": 2, |
| "fp": 2, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 6, |
| "schema_ok": 1, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 1 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5, |
| "structural_path_f1": 0.6666666666666666, |
| "structural_path_precision": 0.6666666666666666, |
| "structural_path_recall": 0.6666666666666666, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.6666666666666666, |
| "unsupported_string_rate": 0.16666666666666666 |
| }, |
| "family/14 · Optional properties": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 1, |
| "fields": 3, |
| "fn": 2, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 1, |
| "optional_omitted_total": 1, |
| "predicted_strings": 2, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 1, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 1.0, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.4, |
| "structural_path_precision": 0.5, |
| "structural_path_recall": 0.3333333333333333, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.3333333333333333, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/15 · Local schema references": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 2, |
| "fields": 3, |
| "fn": 1, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 6, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 5, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.8333333333333334, |
| "structural_path_precision": 0.8333333333333334, |
| "structural_path_recall": 0.8333333333333334, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.6666666666666666, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/16 · Union schemas": { |
| "complete_request_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 1, |
| "absent_total": 1, |
| "exact": 2, |
| "false_abstentions": 0, |
| "field_correct": 4, |
| "fields": 4, |
| "fn": 0, |
| "fp": 0, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 3, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 1.0, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 1.0, |
| "structural_path_precision": 1.0, |
| "structural_path_recall": 1.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 1.0, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/17 · Conditional schemas": { |
| "complete_request_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 2, |
| "false_abstentions": 0, |
| "field_correct": 4, |
| "fields": 4, |
| "fn": 0, |
| "fp": 0, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 2, |
| "optional_omitted_total": 2, |
| "predicted_strings": 4, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 1.0, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 1.0, |
| "structural_path_precision": 1.0, |
| "structural_path_recall": 1.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 1.0, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/18 · Value constraints": { |
| "complete_request_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 2, |
| "false_abstentions": 0, |
| "field_correct": 4, |
| "fields": 4, |
| "fn": 0, |
| "fp": 0, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 2, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 1.0, |
| "structural_path_precision": 1.0, |
| "structural_path_recall": 1.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 1.0, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/19 · Escaping and Unicode": { |
| "complete_request_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 2, |
| "false_abstentions": 0, |
| "field_correct": 4, |
| "fields": 4, |
| "fn": 0, |
| "fp": 0, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 4, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 1.0, |
| "structural_path_precision": 1.0, |
| "structural_path_recall": 1.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 1.0, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/20 · allOf composition": { |
| "complete_request_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 2, |
| "false_abstentions": 0, |
| "field_correct": 2, |
| "fields": 2, |
| "fn": 0, |
| "fp": 0, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 1, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 2, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 1.0, |
| "structural_path_precision": 1.0, |
| "structural_path_recall": 1.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 1.0, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/21 · Arrays inside objects": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 1, |
| "fields": 2, |
| "fn": 3, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 3, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.6666666666666666, |
| "structural_path_precision": 0.8, |
| "structural_path_recall": 0.5714285714285714, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/22 · Single-field objects": { |
| "complete_request_accuracy": 1.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 2, |
| "false_abstentions": 0, |
| "field_correct": 2, |
| "fields": 2, |
| "fn": 0, |
| "fp": 0, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 1, |
| "schema_ok": 2, |
| "token_limited": 0, |
| "tp": 2, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 1.0, |
| "structural_path_precision": 1.0, |
| "structural_path_recall": 1.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 1.0, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/23 · Dynamic object keys": { |
| "complete_request_accuracy": 0.0, |
| "count": 2, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 0, |
| "false_abstentions": 0, |
| "field_correct": 0, |
| "fields": 4, |
| "fn": 4, |
| "fp": 0, |
| "json_parse_ok": 0, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 0, |
| "schema_ok": 0, |
| "token_limited": 0, |
| "tp": 0, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 0.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.0, |
| "structural_path_f1": 0.0, |
| "structural_path_precision": null, |
| "structural_path_recall": 0.0, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.0, |
| "unsupported_string_rate": null |
| }, |
| "family/24 · Empty containers": { |
| "complete_request_accuracy": 0.5, |
| "count": 2, |
| "counts": { |
| "absent_correct": 4, |
| "absent_total": 5, |
| "exact": 1, |
| "false_abstentions": 0, |
| "field_correct": 4, |
| "fields": 5, |
| "fn": 1, |
| "fp": 1, |
| "json_parse_ok": 2, |
| "missed_absences": 0, |
| "natural_stop": 2, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 0, |
| "schema_ok": 1, |
| "token_limited": 0, |
| "tp": 4, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.8, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5, |
| "structural_path_f1": 0.8, |
| "structural_path_precision": 0.8, |
| "structural_path_recall": 0.8, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.8, |
| "unsupported_string_rate": null |
| }, |
| "family/Original extraction": { |
| "complete_request_accuracy": 0.8166666666666667, |
| "count": 60, |
| "counts": { |
| "absent_correct": 55, |
| "absent_total": 59, |
| "exact": 49, |
| "false_abstentions": 1, |
| "field_correct": 136, |
| "fields": 147, |
| "fn": 17, |
| "fp": 11, |
| "json_parse_ok": 60, |
| "missed_absences": 4, |
| "natural_stop": 60, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 105, |
| "schema_ok": 60, |
| "token_limited": 0, |
| "tp": 150, |
| "unsupported_strings": 5 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.9322033898305084, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.9146341463414634, |
| "structural_path_precision": 0.9316770186335404, |
| "structural_path_recall": 0.8982035928143712, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.9251700680272109, |
| "unsupported_string_rate": 0.047619047619047616 |
| }, |
| "macro_family_accuracy": 0.6126666666666667, |
| "paired_schema_views": { |
| "both_correct": 0, |
| "count": 0, |
| "incomplete_groups": 108 |
| }, |
| "schema_view/playground-v2": { |
| "complete_request_accuracy": 0.7222222222222222, |
| "count": 108, |
| "counts": { |
| "absent_correct": 62, |
| "absent_total": 67, |
| "exact": 78, |
| "false_abstentions": 1, |
| "field_correct": 189, |
| "fields": 225, |
| "fn": 72, |
| "fp": 32, |
| "json_parse_ok": 104, |
| "missed_absences": 4, |
| "natural_stop": 108, |
| "optional_omitted_correct": 3, |
| "optional_omitted_total": 3, |
| "predicted_strings": 170, |
| "schema_ok": 94, |
| "token_limited": 0, |
| "tp": 222, |
| "unsupported_strings": 8 |
| }, |
| "json_parse_rate": 0.9629629629629629, |
| "missing_value_accuracy": 0.9253731343283582, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 1.0, |
| "schema_valid_rate": 0.8703703703703703, |
| "structural_path_f1": 0.8102189781021898, |
| "structural_path_precision": 0.8740157480314961, |
| "structural_path_recall": 0.7551020408163265, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.84, |
| "unsupported_string_rate": 0.047058823529411764 |
| }, |
| "tier/challenge48": { |
| "complete_request_accuracy": 0.6041666666666666, |
| "count": 48, |
| "counts": { |
| "absent_correct": 7, |
| "absent_total": 8, |
| "exact": 29, |
| "false_abstentions": 0, |
| "field_correct": 53, |
| "fields": 78, |
| "fn": 55, |
| "fp": 21, |
| "json_parse_ok": 44, |
| "missed_absences": 0, |
| "natural_stop": 48, |
| "optional_omitted_correct": 3, |
| "optional_omitted_total": 3, |
| "predicted_strings": 65, |
| "schema_ok": 34, |
| "token_limited": 0, |
| "tp": 72, |
| "unsupported_strings": 3 |
| }, |
| "json_parse_rate": 0.9166666666666666, |
| "missing_value_accuracy": 0.875, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 1.0, |
| "schema_valid_rate": 0.7083333333333334, |
| "structural_path_f1": 0.6545454545454545, |
| "structural_path_precision": 0.7741935483870968, |
| "structural_path_recall": 0.5669291338582677, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.6794871794871795, |
| "unsupported_string_rate": 0.046153846153846156 |
| }, |
| "tier/original60": { |
| "complete_request_accuracy": 0.8166666666666667, |
| "count": 60, |
| "counts": { |
| "absent_correct": 55, |
| "absent_total": 59, |
| "exact": 49, |
| "false_abstentions": 1, |
| "field_correct": 136, |
| "fields": 147, |
| "fn": 17, |
| "fp": 11, |
| "json_parse_ok": 60, |
| "missed_absences": 4, |
| "natural_stop": 60, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 105, |
| "schema_ok": 60, |
| "token_limited": 0, |
| "tp": 150, |
| "unsupported_strings": 5 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.9322033898305084, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.9146341463414634, |
| "structural_path_precision": 0.9316770186335404, |
| "structural_path_recall": 0.8982035928143712, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.9251700680272109, |
| "unsupported_string_rate": 0.047619047619047616 |
| } |
| }, |
| "identity": { |
| "assembly": false, |
| "context": 2048, |
| "input_files": { |
| "reports/model-design/SFT-145_paired_foundation_20261018/full-eval-500/playground108-v2/references.jsonl": "a79abf4275ed465f1fefdb793e7ed5c4cb6fcbb3e42844d49fc8ab52f58f1ba3", |
| "reports/model-design/SFT-145_paired_foundation_20261018/full-eval-500/playground108-v2/requests.jsonl": "b3a674665ac7e71aff6e6dc662670679a4859a20141130b6aad59f3854fcbf0b" |
| }, |
| "max_output_tokens": 256, |
| "mlx": "0.32.3", |
| "model": "153/500", |
| "model_calls_per_request": 1, |
| "raw_greedy": true, |
| "references_in_input": false, |
| "repair": false, |
| "repetition_penalty": 1, |
| "requests_sha256": "b3a674665ac7e71aff6e6dc662670679a4859a20141130b6aad59f3854fcbf0b", |
| "runner_sha256": "843fc245da9134d1252ab4ca9182742c66a5ec20abfe15dfdef4bff5cc5cab0e", |
| "scorer_sha256": "a8b6dff1b9f7e2cc7d526fa3614ab261cb4c43438dcf7436a1a579f403d46503", |
| "selected_by_development": true, |
| "serialization": "native-sft-chat-v1", |
| "shared_decoder_sha256": "5a3826b2124f1eea2832a4d753a227c7f5f6c62bf882ffdb800cd6e20aeb6ce2", |
| "step": 500, |
| "structural_scorer_sha256": "334544f6a42d7b2c65840bb90d488e87c15d185900e8ceec41e24e24a3984505", |
| "teachers_at_inference": false, |
| "temperature": 0, |
| "tokenizer_sha256": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb", |
| "truncation": false, |
| "weight_sha256": "25b406a7b86e3e8ac4dd0b7f32e27590d9d11349a0148a7815c32304150ac725" |
| } |
| }, |
| "dedicated384": { |
| "metrics": { |
| "all": { |
| "complete_request_accuracy": 0.4765625, |
| "count": 384, |
| "counts": { |
| "absent_correct": 57, |
| "absent_total": 70, |
| "exact": 183, |
| "false_abstentions": 12, |
| "field_correct": 668, |
| "fields": 1036, |
| "fn": 433, |
| "fp": 349, |
| "json_parse_ok": 380, |
| "missed_absences": 13, |
| "natural_stop": 384, |
| "optional_omitted_correct": 28, |
| "optional_omitted_total": 36, |
| "predicted_strings": 925, |
| "schema_ok": 313, |
| "token_limited": 0, |
| "tp": 945, |
| "unsupported_strings": 83 |
| }, |
| "json_parse_rate": 0.9895833333333334, |
| "missing_value_accuracy": 0.8142857142857143, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 0.7777777777777778, |
| "schema_valid_rate": 0.8151041666666666, |
| "structural_path_f1": 0.7073353293413174, |
| "structural_path_precision": 0.7302936630602782, |
| "structural_path_recall": 0.6857764876632801, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.6447876447876448, |
| "unsupported_string_rate": 0.08972972972972973 |
| }, |
| "difficulty/hard": { |
| "complete_request_accuracy": 0.3359375, |
| "count": 128, |
| "counts": { |
| "absent_correct": 16, |
| "absent_total": 20, |
| "exact": 43, |
| "false_abstentions": 8, |
| "field_correct": 194, |
| "fields": 342, |
| "fn": 197, |
| "fp": 158, |
| "json_parse_ok": 126, |
| "missed_absences": 4, |
| "natural_stop": 128, |
| "optional_omitted_correct": 10, |
| "optional_omitted_total": 12, |
| "predicted_strings": 333, |
| "schema_ok": 97, |
| "token_limited": 0, |
| "tp": 289, |
| "unsupported_strings": 40 |
| }, |
| "json_parse_rate": 0.984375, |
| "missing_value_accuracy": 0.8, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 0.8333333333333334, |
| "schema_valid_rate": 0.7578125, |
| "structural_path_f1": 0.6195069667738478, |
| "structural_path_precision": 0.6465324384787472, |
| "structural_path_recall": 0.5946502057613169, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5672514619883041, |
| "unsupported_string_rate": 0.12012012012012012 |
| }, |
| "difficulty/medium": { |
| "complete_request_accuracy": 0.53125, |
| "count": 128, |
| "counts": { |
| "absent_correct": 21, |
| "absent_total": 26, |
| "exact": 68, |
| "false_abstentions": 0, |
| "field_correct": 239, |
| "fields": 351, |
| "fn": 121, |
| "fp": 102, |
| "json_parse_ok": 128, |
| "missed_absences": 5, |
| "natural_stop": 128, |
| "optional_omitted_correct": 9, |
| "optional_omitted_total": 12, |
| "predicted_strings": 317, |
| "schema_ok": 108, |
| "token_limited": 0, |
| "tp": 359, |
| "unsupported_strings": 19 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.8076923076923077, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 0.75, |
| "schema_valid_rate": 0.84375, |
| "structural_path_f1": 0.7630180658873539, |
| "structural_path_precision": 0.7787418655097614, |
| "structural_path_recall": 0.7479166666666667, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.6809116809116809, |
| "unsupported_string_rate": 0.05993690851735016 |
| }, |
| "difficulty/simple": { |
| "complete_request_accuracy": 0.5625, |
| "count": 128, |
| "counts": { |
| "absent_correct": 20, |
| "absent_total": 24, |
| "exact": 72, |
| "false_abstentions": 4, |
| "field_correct": 235, |
| "fields": 343, |
| "fn": 115, |
| "fp": 89, |
| "json_parse_ok": 126, |
| "missed_absences": 4, |
| "natural_stop": 128, |
| "optional_omitted_correct": 9, |
| "optional_omitted_total": 12, |
| "predicted_strings": 275, |
| "schema_ok": 108, |
| "token_limited": 0, |
| "tp": 297, |
| "unsupported_strings": 24 |
| }, |
| "json_parse_rate": 0.984375, |
| "missing_value_accuracy": 0.8333333333333334, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 0.75, |
| "schema_valid_rate": 0.84375, |
| "structural_path_f1": 0.7443609022556391, |
| "structural_path_precision": 0.7694300518134715, |
| "structural_path_recall": 0.720873786407767, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.685131195335277, |
| "unsupported_string_rate": 0.08727272727272728 |
| }, |
| "family/boolean_null": { |
| "complete_request_accuracy": 0.3333333333333333, |
| "count": 24, |
| "counts": { |
| "absent_correct": 7, |
| "absent_total": 12, |
| "exact": 8, |
| "false_abstentions": 3, |
| "field_correct": 68, |
| "fields": 120, |
| "fn": 72, |
| "fp": 28, |
| "json_parse_ok": 20, |
| "missed_absences": 5, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 27, |
| "schema_ok": 13, |
| "token_limited": 0, |
| "tp": 96, |
| "unsupported_strings": 6 |
| }, |
| "json_parse_rate": 0.8333333333333334, |
| "missing_value_accuracy": 0.5833333333333334, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5416666666666666, |
| "structural_path_f1": 0.6575342465753424, |
| "structural_path_precision": 0.7741935483870968, |
| "structural_path_recall": 0.5714285714285714, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5666666666666667, |
| "unsupported_string_rate": 0.2222222222222222 |
| }, |
| "family/composition": { |
| "complete_request_accuracy": 0.9166666666666666, |
| "count": 24, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 4, |
| "exact": 22, |
| "false_abstentions": 0, |
| "field_correct": 22, |
| "fields": 24, |
| "fn": 2, |
| "fp": 2, |
| "json_parse_ok": 24, |
| "missed_absences": 2, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 30, |
| "schema_ok": 24, |
| "token_limited": 0, |
| "tp": 30, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.5, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.9375, |
| "structural_path_precision": 0.9375, |
| "structural_path_recall": 0.9375, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.9166666666666666, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/conditionals": { |
| "complete_request_accuracy": 0.75, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 18, |
| "false_abstentions": 0, |
| "field_correct": 42, |
| "fields": 54, |
| "fn": 6, |
| "fp": 6, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 18, |
| "optional_omitted_total": 24, |
| "predicted_strings": 48, |
| "schema_ok": 18, |
| "token_limited": 0, |
| "tp": 42, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 0.75, |
| "schema_valid_rate": 0.75, |
| "structural_path_f1": 0.875, |
| "structural_path_precision": 0.875, |
| "structural_path_recall": 0.875, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.7777777777777778, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/constraints": { |
| "complete_request_accuracy": 0.3333333333333333, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 8, |
| "false_abstentions": 0, |
| "field_correct": 55, |
| "fields": 96, |
| "fn": 41, |
| "fp": 41, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 72, |
| "schema_ok": 8, |
| "token_limited": 0, |
| "tp": 55, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.3333333333333333, |
| "structural_path_f1": 0.5729166666666666, |
| "structural_path_precision": 0.5729166666666666, |
| "structural_path_recall": 0.5729166666666666, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5729166666666666, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/corrections": { |
| "complete_request_accuracy": 0.5, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 12, |
| "false_abstentions": 0, |
| "field_correct": 54, |
| "fields": 72, |
| "fn": 18, |
| "fp": 18, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 72, |
| "schema_ok": 24, |
| "token_limited": 0, |
| "tp": 54, |
| "unsupported_strings": 5 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.75, |
| "structural_path_precision": 0.75, |
| "structural_path_recall": 0.75, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.75, |
| "unsupported_string_rate": 0.06944444444444445 |
| }, |
| "family/exact_copy": { |
| "complete_request_accuracy": 0.5, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 12, |
| "false_abstentions": 0, |
| "field_correct": 70, |
| "fields": 96, |
| "fn": 26, |
| "fp": 21, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 91, |
| "schema_ok": 20, |
| "token_limited": 0, |
| "tp": 70, |
| "unsupported_strings": 1 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.8333333333333334, |
| "structural_path_f1": 0.7486631016042781, |
| "structural_path_precision": 0.7692307692307693, |
| "structural_path_recall": 0.7291666666666666, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.7291666666666666, |
| "unsupported_string_rate": 0.01098901098901099 |
| }, |
| "family/flat_fields": { |
| "complete_request_accuracy": 0.4583333333333333, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 11, |
| "false_abstentions": 0, |
| "field_correct": 50, |
| "fields": 72, |
| "fn": 22, |
| "fp": 22, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 72, |
| "schema_ok": 24, |
| "token_limited": 0, |
| "tp": 50, |
| "unsupported_strings": 4 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.6944444444444444, |
| "structural_path_precision": 0.6944444444444444, |
| "structural_path_recall": 0.6944444444444444, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.6944444444444444, |
| "unsupported_string_rate": 0.05555555555555555 |
| }, |
| "family/lists": { |
| "complete_request_accuracy": 0.5833333333333334, |
| "count": 24, |
| "counts": { |
| "absent_correct": 28, |
| "absent_total": 28, |
| "exact": 14, |
| "false_abstentions": 0, |
| "field_correct": 38, |
| "fields": 48, |
| "fn": 24, |
| "fp": 16, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 52, |
| "schema_ok": 24, |
| "token_limited": 0, |
| "tp": 64, |
| "unsupported_strings": 10 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 1.0, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.7619047619047619, |
| "structural_path_precision": 0.8, |
| "structural_path_recall": 0.7272727272727273, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.7916666666666666, |
| "unsupported_string_rate": 0.19230769230769232 |
| }, |
| "family/maps_matrices_tuples": { |
| "complete_request_accuracy": 0.8333333333333334, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 20, |
| "false_abstentions": 0, |
| "field_correct": 20, |
| "fields": 24, |
| "fn": 10, |
| "fp": 2, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 8, |
| "schema_ok": 20, |
| "token_limited": 0, |
| "tp": 62, |
| "unsupported_strings": 0 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.8333333333333334, |
| "structural_path_f1": 0.9117647058823529, |
| "structural_path_precision": 0.96875, |
| "structural_path_recall": 0.8611111111111112, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.8333333333333334, |
| "unsupported_string_rate": 0.0 |
| }, |
| "family/missing_optional": { |
| "complete_request_accuracy": 0.16666666666666666, |
| "count": 24, |
| "counts": { |
| "absent_correct": 17, |
| "absent_total": 20, |
| "exact": 4, |
| "false_abstentions": 8, |
| "field_correct": 27, |
| "fields": 62, |
| "fn": 33, |
| "fp": 24, |
| "json_parse_ok": 24, |
| "missed_absences": 3, |
| "natural_stop": 24, |
| "optional_omitted_correct": 10, |
| "optional_omitted_total": 12, |
| "predicted_strings": 25, |
| "schema_ok": 23, |
| "token_limited": 0, |
| "tp": 27, |
| "unsupported_strings": 1 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.85, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 0.8333333333333334, |
| "schema_valid_rate": 0.9583333333333334, |
| "structural_path_f1": 0.4864864864864865, |
| "structural_path_precision": 0.5294117647058824, |
| "structural_path_recall": 0.45, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.43548387096774194, |
| "unsupported_string_rate": 0.04 |
| }, |
| "family/nested_objects": { |
| "complete_request_accuracy": 0.5416666666666666, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 13, |
| "false_abstentions": 0, |
| "field_correct": 13, |
| "fields": 24, |
| "fn": 27, |
| "fp": 20, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 57, |
| "schema_ok": 16, |
| "token_limited": 0, |
| "tp": 37, |
| "unsupported_strings": 3 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.6666666666666666, |
| "structural_path_f1": 0.6115702479338843, |
| "structural_path_precision": 0.6491228070175439, |
| "structural_path_recall": 0.578125, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5416666666666666, |
| "unsupported_string_rate": 0.05263157894736842 |
| }, |
| "family/numeric_types": { |
| "complete_request_accuracy": 0.25, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 6, |
| "false_abstentions": 1, |
| "field_correct": 86, |
| "fields": 120, |
| "fn": 41, |
| "fp": 32, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 31, |
| "schema_ok": 13, |
| "token_limited": 0, |
| "tp": 103, |
| "unsupported_strings": 4 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5416666666666666, |
| "structural_path_f1": 0.7383512544802867, |
| "structural_path_precision": 0.762962962962963, |
| "structural_path_recall": 0.7152777777777778, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.7166666666666667, |
| "unsupported_string_rate": 0.12903225806451613 |
| }, |
| "family/parallel_groups": { |
| "complete_request_accuracy": 0.16666666666666666, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 4, |
| "false_abstentions": 0, |
| "field_correct": 34, |
| "fields": 72, |
| "fn": 42, |
| "fp": 39, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 84, |
| "schema_ok": 14, |
| "token_limited": 0, |
| "tp": 46, |
| "unsupported_strings": 14 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 0.5833333333333334, |
| "structural_path_f1": 0.5317919075144508, |
| "structural_path_precision": 0.5411764705882353, |
| "structural_path_recall": 0.5227272727272727, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.4722222222222222, |
| "unsupported_string_rate": 0.16666666666666666 |
| }, |
| "family/record_arrays": { |
| "complete_request_accuracy": 0.4583333333333333, |
| "count": 24, |
| "counts": { |
| "absent_correct": 3, |
| "absent_total": 6, |
| "exact": 11, |
| "false_abstentions": 0, |
| "field_correct": 11, |
| "fields": 24, |
| "fn": 19, |
| "fp": 28, |
| "json_parse_ok": 24, |
| "missed_absences": 3, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 80, |
| "schema_ok": 24, |
| "token_limited": 0, |
| "tp": 83, |
| "unsupported_strings": 14 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.5, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.7793427230046949, |
| "structural_path_precision": 0.7477477477477478, |
| "structural_path_recall": 0.8137254901960784, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.4583333333333333, |
| "unsupported_string_rate": 0.175 |
| }, |
| "family/references": { |
| "complete_request_accuracy": 0.625, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 15, |
| "false_abstentions": 0, |
| "field_correct": 23, |
| "fields": 32, |
| "fn": 9, |
| "fp": 9, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 80, |
| "schema_ok": 24, |
| "token_limited": 0, |
| "tp": 71, |
| "unsupported_strings": 9 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.8875, |
| "structural_path_precision": 0.8875, |
| "structural_path_recall": 0.8875, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.71875, |
| "unsupported_string_rate": 0.1125 |
| }, |
| "family/role_binding": { |
| "complete_request_accuracy": 0.20833333333333334, |
| "count": 24, |
| "counts": { |
| "absent_correct": 0, |
| "absent_total": 0, |
| "exact": 5, |
| "false_abstentions": 0, |
| "field_correct": 55, |
| "fields": 96, |
| "fn": 41, |
| "fp": 41, |
| "json_parse_ok": 24, |
| "missed_absences": 0, |
| "natural_stop": 24, |
| "optional_omitted_correct": 0, |
| "optional_omitted_total": 0, |
| "predicted_strings": 96, |
| "schema_ok": 24, |
| "token_limited": 0, |
| "tp": 55, |
| "unsupported_strings": 12 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": null, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": null, |
| "schema_valid_rate": 1.0, |
| "structural_path_f1": 0.5729166666666666, |
| "structural_path_precision": 0.5729166666666666, |
| "structural_path_recall": 0.5729166666666666, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.5729166666666666, |
| "unsupported_string_rate": 0.125 |
| }, |
| "macro_family_accuracy": 0.4765625, |
| "paired_schema_views": { |
| "both_correct": 64, |
| "count": 192, |
| "incomplete_groups": 0 |
| }, |
| "schema_view/canonical": { |
| "complete_request_accuracy": 0.6041666666666666, |
| "count": 192, |
| "counts": { |
| "absent_correct": 31, |
| "absent_total": 35, |
| "exact": 116, |
| "false_abstentions": 1, |
| "field_correct": 422, |
| "fields": 516, |
| "fn": 114, |
| "fp": 107, |
| "json_parse_ok": 192, |
| "missed_absences": 4, |
| "natural_stop": 192, |
| "optional_omitted_correct": 16, |
| "optional_omitted_total": 18, |
| "predicted_strings": 473, |
| "schema_ok": 179, |
| "token_limited": 0, |
| "tp": 575, |
| "unsupported_strings": 42 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.8857142857142857, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 0.8888888888888888, |
| "schema_valid_rate": 0.9322916666666666, |
| "structural_path_f1": 0.8388037928519329, |
| "structural_path_precision": 0.843108504398827, |
| "structural_path_recall": 0.8345428156748912, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.8178294573643411, |
| "unsupported_string_rate": 0.08879492600422834 |
| }, |
| "schema_view/renamed": { |
| "complete_request_accuracy": 0.3489583333333333, |
| "count": 192, |
| "counts": { |
| "absent_correct": 26, |
| "absent_total": 35, |
| "exact": 67, |
| "false_abstentions": 11, |
| "field_correct": 246, |
| "fields": 520, |
| "fn": 319, |
| "fp": 242, |
| "json_parse_ok": 188, |
| "missed_absences": 9, |
| "natural_stop": 192, |
| "optional_omitted_correct": 12, |
| "optional_omitted_total": 18, |
| "predicted_strings": 452, |
| "schema_ok": 134, |
| "token_limited": 0, |
| "tp": 370, |
| "unsupported_strings": 41 |
| }, |
| "json_parse_rate": 0.9791666666666666, |
| "missing_value_accuracy": 0.7428571428571429, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 0.6666666666666666, |
| "schema_valid_rate": 0.6979166666666666, |
| "structural_path_f1": 0.568793235972329, |
| "structural_path_precision": 0.6045751633986928, |
| "structural_path_recall": 0.5370101596516691, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.47307692307692306, |
| "unsupported_string_rate": 0.09070796460176991 |
| }, |
| "tier/advanced": { |
| "complete_request_accuracy": 0.78125, |
| "count": 96, |
| "counts": { |
| "absent_correct": 2, |
| "absent_total": 4, |
| "exact": 75, |
| "false_abstentions": 0, |
| "field_correct": 107, |
| "fields": 134, |
| "fn": 27, |
| "fp": 19, |
| "json_parse_ok": 96, |
| "missed_absences": 2, |
| "natural_stop": 96, |
| "optional_omitted_correct": 18, |
| "optional_omitted_total": 24, |
| "predicted_strings": 166, |
| "schema_ok": 86, |
| "token_limited": 0, |
| "tp": 205, |
| "unsupported_strings": 9 |
| }, |
| "json_parse_rate": 1.0, |
| "missing_value_accuracy": 0.5, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 0.75, |
| "schema_valid_rate": 0.8958333333333334, |
| "structural_path_f1": 0.8991228070175439, |
| "structural_path_precision": 0.9151785714285714, |
| "structural_path_recall": 0.8836206896551724, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.7985074626865671, |
| "unsupported_string_rate": 0.05421686746987952 |
| }, |
| "tier/core": { |
| "complete_request_accuracy": 0.375, |
| "count": 288, |
| "counts": { |
| "absent_correct": 55, |
| "absent_total": 66, |
| "exact": 108, |
| "false_abstentions": 12, |
| "field_correct": 561, |
| "fields": 902, |
| "fn": 406, |
| "fp": 330, |
| "json_parse_ok": 284, |
| "missed_absences": 11, |
| "natural_stop": 288, |
| "optional_omitted_correct": 10, |
| "optional_omitted_total": 12, |
| "predicted_strings": 759, |
| "schema_ok": 227, |
| "token_limited": 0, |
| "tp": 740, |
| "unsupported_strings": 74 |
| }, |
| "json_parse_rate": 0.9861111111111112, |
| "missing_value_accuracy": 0.8333333333333334, |
| "natural_stop_rate": 1.0, |
| "optional_omission_accuracy": 0.8333333333333334, |
| "schema_valid_rate": 0.7881944444444444, |
| "structural_path_f1": 0.6678700361010831, |
| "structural_path_precision": 0.6915887850467289, |
| "structural_path_recall": 0.6457242582897034, |
| "token_limit_rate": 0.0, |
| "top_level_field_accuracy": 0.6219512195121951, |
| "unsupported_string_rate": 0.09749670619235837 |
| } |
| }, |
| "identity": { |
| "assembly": false, |
| "context": 2048, |
| "input_files": { |
| "pretraining/data/jsonlite_eval_v1/references.jsonl": "e034d717d5c15fbf1cc714a1ff66bbf5fb759b0081527d5120bd6ba034a3d406", |
| "pretraining/data/jsonlite_eval_v1/requests.jsonl": "4e68add1a3fd4f5920c659e0519ccdc2b143df08b5f553063c9b006b30bb4790" |
| }, |
| "max_output_tokens": 256, |
| "mlx": "0.32.3", |
| "model": "153/500", |
| "model_calls_per_request": 1, |
| "raw_greedy": true, |
| "references_in_input": false, |
| "repair": false, |
| "repetition_penalty": 1, |
| "requests_sha256": "4e68add1a3fd4f5920c659e0519ccdc2b143df08b5f553063c9b006b30bb4790", |
| "runner_sha256": "843fc245da9134d1252ab4ca9182742c66a5ec20abfe15dfdef4bff5cc5cab0e", |
| "scorer_sha256": "334544f6a42d7b2c65840bb90d488e87c15d185900e8ceec41e24e24a3984505", |
| "selected_by_development": true, |
| "serialization": "native-sft-chat-v1", |
| "shared_decoder_sha256": "5a3826b2124f1eea2832a4d753a227c7f5f6c62bf882ffdb800cd6e20aeb6ce2", |
| "step": 500, |
| "structural_scorer_sha256": "334544f6a42d7b2c65840bb90d488e87c15d185900e8ceec41e24e24a3984505", |
| "teachers_at_inference": false, |
| "temperature": 0, |
| "tokenizer_sha256": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb", |
| "truncation": false, |
| "weight_sha256": "25b406a7b86e3e8ac4dd0b7f32e27590d9d11349a0148a7815c32304150ac725" |
| } |
| }, |
| "massive2974": { |
| "metrics": { |
| "complete_exact": 0.5262273032952253, |
| "correct": 1565, |
| "count": 2974, |
| "counts": { |
| "exact": 1565, |
| "fields_correct": 1565, |
| "fields_total": 2974, |
| "fn": 3142, |
| "fp": 2507, |
| "gold_mentions": 2815, |
| "json_parse_valid": 2973, |
| "malformed_items": 0, |
| "missing_correct": 853, |
| "missing_total": 994, |
| "natural_stop": 2974, |
| "predicted_mentions": 2421, |
| "schema_valid": 2966, |
| "slot_fn": 1503, |
| "slot_fp": 1109, |
| "slot_tp": 1312, |
| "token_limit": 0, |
| "tp": 3482, |
| "unannotated_correct": 853, |
| "unannotated_reference": 994, |
| "unsupported_mentions": 117 |
| }, |
| "diagnostic_counts": { |
| "exact": 1565, |
| "fields_correct": 1565, |
| "fields_total": 2974, |
| "fn": 3142, |
| "fp": 2507, |
| "gold_mentions": 2815, |
| "json_parse_valid": 2973, |
| "malformed_items": 0, |
| "missing_correct": 853, |
| "missing_total": 994, |
| "natural_stop": 2974, |
| "predicted_mentions": 2421, |
| "schema_valid": 2966, |
| "slot_fn": 1503, |
| "slot_fp": 1109, |
| "slot_tp": 1312, |
| "token_limit": 0, |
| "tp": 3482, |
| "unannotated_correct": 853, |
| "unannotated_reference": 994, |
| "unsupported_mentions": 117 |
| }, |
| "field_accuracy": 0.5262273032952253, |
| "json_parse_rate": 0.9996637525218561, |
| "leaf_f1": 0.55212875604535, |
| "leaf_precision": 0.5813992319251962, |
| "leaf_recall": 0.5256642512077294, |
| "missing_field_accuracy": 0.858148893360161, |
| "natural_stop_rate": 1.0, |
| "schema_valid_rate": 0.9973100201748487, |
| "slot_f1": 0.5011459129106188, |
| "slot_precision": 0.5419248244527055, |
| "slot_recall": 0.4660746003552398 |
| }, |
| "identity": { |
| "assembly": false, |
| "context": 2048, |
| "input_files": { |
| "pretraining/data/massive_slot_list_139_20261017/references.jsonl": "1b6b2a18348939d959177f5a6a635f76a1b0645536e321195011dd2cd35b1461", |
| "pretraining/data/massive_slot_list_139_20261017/requests.jsonl": "05acf77b73133416111348d7778db332168ceeeab00dc644386ea24af9a99b2f" |
| }, |
| "max_output_tokens": 384, |
| "mlx": "0.32.3", |
| "model": "153/500", |
| "model_calls_per_request": 1, |
| "raw_greedy": true, |
| "references_in_input": false, |
| "repair": false, |
| "repetition_penalty": 1, |
| "requests_sha256": "05acf77b73133416111348d7778db332168ceeeab00dc644386ea24af9a99b2f", |
| "runner_sha256": "843fc245da9134d1252ab4ca9182742c66a5ec20abfe15dfdef4bff5cc5cab0e", |
| "scorer_sha256": "07be6f71f4e2339e25ea9696be617fb440fb923fac81ba51aaa63a64bff0c096", |
| "selected_by_development": true, |
| "serialization": "native-sft-chat-v1", |
| "shared_decoder_sha256": "5a3826b2124f1eea2832a4d753a227c7f5f6c62bf882ffdb800cd6e20aeb6ce2", |
| "step": 500, |
| "structural_scorer_sha256": "334544f6a42d7b2c65840bb90d488e87c15d185900e8ceec41e24e24a3984505", |
| "teachers_at_inference": false, |
| "temperature": 0, |
| "tokenizer_sha256": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb", |
| "truncation": false, |
| "weight_sha256": "25b406a7b86e3e8ac4dd0b7f32e27590d9d11349a0148a7815c32304150ac725" |
| } |
| } |
| }, |
| "scope": "Separate previously observed research panels; not pooled, fresh blind benchmarks or official leaderboard scores. Raw greedy, no assembly, repair, constraints or reference hints." |
| } |
|
|
