{ "model": "OpenSML-JSONLite-175M-Instruct", "checkpoint": "main", "internal_checkpoint": "153/500", "model_sha256": "25b406a7b86e3e8ac4dd0b7f32e27590d9d11349a0148a7815c32304150ac725", "training_prefix": { "update": 500, "presentations": 32000, "distinct_source_texts": 21392, "source_groups": 19589, "components": { "atomic": 2844, "composite": 5689, "grounded_qa": 2667, "massive": 2667, "prior_authored": 2133, "schema_variation": 2667, "snips": 13333 }, "supervised_positions_metadata": 1027385, "counter_scope": "Checkpoint metadata tokens counts supervised assistant/EOS positions, not serialized source/schema positions.", "train_sha256": "dd69dac2fa72ef5286df8cd513730f751c4d45835947e04173db130f906451a4", "batches_sha256": "bff71b6a94d13dae8f5c766145f4f1d59074665f92a1ff1916d138af27cd0268" }, "panels": { "original1444": { "metrics": { "all": { "absent_accuracy": 0.9613368283093053, "count": 1444, "counts": { "absent_correct": 1467, "absent_total": 1526, "exact": 1293, "false_abstention": 49, "field_correct": 3319, "fields": 3497, "fn": 119, "fp": 128, "json_parse_ok": 1444, "missed_abstention": 59, "natural_stop": 1444, "nonrepetitive_nonempty": 1444, "predicted_values": 1980, "present_correct": 1852, "present_total": 1971, "schema_ok": 1443, "token_limited": 0, "tp": 1852, "unsupported_values": 20 }, "exact": 0.8954293628808865, "f1": 0.9374841812199443, "false_abstention_rate": 0.024860476915271434, "field_accuracy": 0.9490992279096369, "json_parse_rate": 1.0, "missed_abstention_rate": 0.03866317169069463, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.0213116571704629, "precision": 0.9353535353535354, "present_accuracy": 0.9396245560629122, "recall": 0.9396245560629122, "schema_valid_rate": 0.9993074792243767, "token_limited_rate": 0.0, "unsupported_value_rate": 0.010101010101010102 }, "domain/AddToPlaylist": { "absent_accuracy": 0.9449152542372882, "count": 206, "counts": { "absent_correct": 223, "absent_total": 236, "exact": 179, "false_abstention": 5, "field_correct": 492, "fields": 526, "fn": 21, "fp": 29, "json_parse_ok": 206, "missed_abstention": 13, "natural_stop": 206, "nonrepetitive_nonempty": 206, "predicted_values": 298, "present_correct": 269, "present_total": 290, "schema_ok": 206, "token_limited": 0, "tp": 269, "unsupported_values": 6 }, "exact": 0.8689320388349514, "f1": 0.9149659863945578, "false_abstention_rate": 0.017241379310344827, "field_accuracy": 0.935361216730038, "json_parse_rate": 1.0, "missed_abstention_rate": 0.05508474576271186, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.033199904241485556, "precision": 0.9026845637583892, "present_accuracy": 0.9275862068965517, "recall": 0.9275862068965517, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.020134228187919462 }, "domain/BookRestaurant": { "absent_accuracy": 0.94, "count": 208, "counts": { "absent_correct": 235, "absent_total": 250, "exact": 173, "false_abstention": 7, "field_correct": 532, "fields": 569, "fn": 22, "fp": 30, "json_parse_ok": 208, "missed_abstention": 15, "natural_stop": 208, "nonrepetitive_nonempty": 208, "predicted_values": 327, "present_correct": 297, "present_total": 319, "schema_ok": 208, "token_limited": 0, "tp": 297, "unsupported_values": 3 }, "exact": 0.8317307692307693, "f1": 0.9195046439628484, "false_abstention_rate": 0.0219435736677116, "field_accuracy": 0.9349736379613357, "json_parse_rate": 1.0, "missed_abstention_rate": 0.06, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.021212414096645753, "precision": 0.908256880733945, "present_accuracy": 0.9310344827586207, "recall": 0.9310344827586207, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.009174311926605505 }, "domain/GetWeather": { "absent_accuracy": 0.9512195121951219, "count": 206, "counts": { "absent_correct": 234, "absent_total": 246, "exact": 186, "false_abstention": 5, "field_correct": 500, "fields": 525, "fn": 13, "fp": 19, "json_parse_ok": 206, "missed_abstention": 12, "natural_stop": 206, "nonrepetitive_nonempty": 206, "predicted_values": 285, "present_correct": 266, "present_total": 279, "schema_ok": 205, "token_limited": 0, "tp": 266, "unsupported_values": 4 }, "exact": 0.9029126213592233, "f1": 0.9432624113475178, "false_abstention_rate": 0.017921146953405017, "field_accuracy": 0.9523809523809523, "json_parse_rate": 1.0, "missed_abstention_rate": 0.04878048780487805, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.018259013971190636, "precision": 0.9333333333333333, "present_accuracy": 0.953405017921147, "recall": 0.953405017921147, "schema_valid_rate": 0.9951456310679612, "token_limited_rate": 0.0, "unsupported_value_rate": 0.014035087719298246 }, "domain/PlayMusic": { "absent_accuracy": 0.9453125, "count": 208, "counts": { "absent_correct": 242, "absent_total": 256, "exact": 169, "false_abstention": 22, "field_correct": 471, "fields": 523, "fn": 38, "fp": 30, "json_parse_ok": 208, "missed_abstention": 14, "natural_stop": 208, "nonrepetitive_nonempty": 208, "predicted_values": 259, "present_correct": 229, "present_total": 267, "schema_ok": 208, "token_limited": 0, "tp": 229, "unsupported_values": 5 }, "exact": 0.8125, "f1": 0.8707224334600762, "false_abstention_rate": 0.08239700374531835, "field_accuracy": 0.9005736137667304, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0546875, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.03351788995986191, "precision": 0.8841698841698842, "present_accuracy": 0.8576779026217228, "recall": 0.8576779026217228, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.019305019305019305 }, "domain/RateBook": { "absent_accuracy": 0.9959349593495935, "count": 204, "counts": { "absent_correct": 245, "absent_total": 246, "exact": 199, "false_abstention": 0, "field_correct": 562, "fields": 567, "fn": 4, "fp": 5, "json_parse_ok": 204, "missed_abstention": 1, "natural_stop": 204, "nonrepetitive_nonempty": 204, "predicted_values": 322, "present_correct": 317, "present_total": 321, "schema_ok": 204, "token_limited": 0, "tp": 317, "unsupported_values": 1 }, "exact": 0.9754901960784313, "f1": 0.9860031104199067, "false_abstention_rate": 0.0, "field_accuracy": 0.9911816578483245, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0040650406504065045, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.006796475639252914, "precision": 0.984472049689441, "present_accuracy": 0.9875389408099688, "recall": 0.9875389408099688, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.003105590062111801 }, "domain/SearchCreativeWork": { "absent_accuracy": 1.0, "count": 208, "counts": { "absent_correct": 39, "absent_total": 39, "exact": 200, "false_abstention": 0, "field_correct": 278, "fields": 286, "fn": 8, "fp": 8, "json_parse_ok": 208, "missed_abstention": 0, "natural_stop": 208, "nonrepetitive_nonempty": 208, "predicted_values": 247, "present_correct": 239, "present_total": 247, "schema_ok": 208, "token_limited": 0, "tp": 239, "unsupported_values": 1 }, "exact": 0.9615384615384616, "f1": 0.9676113360323887, "false_abstention_rate": 0.0, "field_accuracy": 0.972027972027972, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.014501851789401162, "precision": 0.9676113360323887, "present_accuracy": 0.9676113360323887, "recall": 0.9676113360323887, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.004048582995951417 }, "domain/SearchScreeningEvent": { "absent_accuracy": 0.9841897233201581, "count": 204, "counts": { "absent_correct": 249, "absent_total": 253, "exact": 187, "false_abstention": 10, "field_correct": 484, "fields": 501, "fn": 13, "fp": 7, "json_parse_ok": 204, "missed_abstention": 4, "natural_stop": 204, "nonrepetitive_nonempty": 204, "predicted_values": 242, "present_correct": 235, "present_total": 248, "schema_ok": 204, "token_limited": 0, "tp": 235, "unsupported_values": 0 }, "exact": 0.9166666666666666, "f1": 0.9591836734693877, "false_abstention_rate": 0.04032258064516129, "field_accuracy": 0.9660678642714571, "json_parse_rate": 1.0, "missed_abstention_rate": 0.015810276679841896, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.021503560568531862, "precision": 0.9710743801652892, "present_accuracy": 0.9475806451612904, "recall": 0.9475806451612904, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.0 }, "macro_exact": 0.895681536244072, "paired_requests": { "accuracy": 0.8328530259365994, "both_correct": 578, "count": 694 }, "variant/alias": { "absent_accuracy": 1.0, "count": 28, "counts": { "absent_correct": 29, "absent_total": 29, "exact": 24, "false_abstention": 1, "field_correct": 70, "fields": 75, "fn": 5, "fp": 3, "json_parse_ok": 28, "missed_abstention": 0, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 44, "present_correct": 41, "present_total": 46, "schema_ok": 27, "token_limited": 0, "tp": 41, "unsupported_values": 1 }, "exact": 0.8571428571428571, "f1": 0.9111111111111111, "false_abstention_rate": 0.021739130434782608, "field_accuracy": 0.9333333333333333, "json_parse_rate": 1.0, "missed_abstention_rate": 0.0, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.0613402272782488, "precision": 0.9318181818181818, "present_accuracy": 0.8913043478260869, "recall": 0.8913043478260869, "schema_valid_rate": 0.9642857142857143, "token_limited_rate": 0.0, "unsupported_value_rate": 0.022727272727272728 }, "variant/canonical": { "absent_accuracy": 0.9625340599455041, "count": 1388, "counts": { "absent_correct": 1413, "absent_total": 1468, "exact": 1247, "false_abstention": 47, "field_correct": 3181, "fields": 3347, "fn": 111, "fp": 119, "json_parse_ok": 1388, "missed_abstention": 55, "natural_stop": 1388, "nonrepetitive_nonempty": 1388, "predicted_values": 1887, "present_correct": 1768, "present_total": 1879, "schema_ok": 1388, "token_limited": 0, "tp": 1768, "unsupported_values": 18 }, "exact": 0.8984149855907781, "f1": 0.9389272437599575, "false_abstention_rate": 0.025013304949441192, "field_accuracy": 0.9504033462802509, "json_parse_rate": 1.0, "missed_abstention_rate": 0.03746594005449591, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.019958412440256805, "precision": 0.9369369369369369, "present_accuracy": 0.940926024481107, "recall": 0.940926024481107, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.009538950715421303 }, "variant/nullable": { "absent_accuracy": 0.8620689655172413, "count": 28, "counts": { "absent_correct": 25, "absent_total": 29, "exact": 22, "false_abstention": 1, "field_correct": 68, "fields": 75, "fn": 3, "fp": 6, "json_parse_ok": 28, "missed_abstention": 4, "natural_stop": 28, "nonrepetitive_nonempty": 28, "predicted_values": 49, "present_correct": 43, "present_total": 46, "schema_ok": 28, "token_limited": 0, "tp": 43, "unsupported_values": 1 }, "exact": 0.7857142857142857, "f1": 0.9052631578947369, "false_abstention_rate": 0.021739130434782608, "field_accuracy": 0.9066666666666666, "json_parse_rate": 1.0, "missed_abstention_rate": 0.13793103448275862, "natural_stop_rate": 1.0, "nonrepetitive_nonempty_rate": 1.0, "output_nll": 0.048365361545750475, "precision": 0.8775510204081632, "present_accuracy": 0.9347826086956522, "recall": 0.9347826086956522, "schema_valid_rate": 1.0, "token_limited_rate": 0.0, "unsupported_value_rate": 0.02040816326530612 } }, "identity": { "assembly": false, "context": 2048, "input_files": { "reports/model-design/SFT-128_joint_extraction_20261016/release-comparison/original1444.jsonl": "083513ab76421287337ec56cb2b293a986894db7f8955be256342602b3307241" }, "max_output_tokens": 256, "mlx": "0.32.3", "model": "153/500", "model_calls_per_request": 1, "raw_greedy": true, "references_in_input": false, "repair": false, "repetition_penalty": 1, "requests_sha256": "083513ab76421287337ec56cb2b293a986894db7f8955be256342602b3307241", "runner_sha256": "843fc245da9134d1252ab4ca9182742c66a5ec20abfe15dfdef4bff5cc5cab0e", "scorer_sha256": "1f4d2c69ebc640f993b77cb09ecf280d2df1dbf3f20d1060f7c8a5ce6780dd97", "selected_by_development": true, "serialization": "native-sft-chat-v1", "shared_decoder_sha256": "5a3826b2124f1eea2832a4d753a227c7f5f6c62bf882ffdb800cd6e20aeb6ce2", "step": 500, "structural_scorer_sha256": "334544f6a42d7b2c65840bb90d488e87c15d185900e8ceec41e24e24a3984505", "teachers_at_inference": false, "temperature": 0, "tokenizer_sha256": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb", "truncation": false, "weight_sha256": "25b406a7b86e3e8ac4dd0b7f32e27590d9d11349a0148a7815c32304150ac725" } }, "playground108": { "metrics": { "all": { "complete_request_accuracy": 0.7222222222222222, "count": 108, "counts": { "absent_correct": 62, "absent_total": 67, "exact": 78, "false_abstentions": 1, "field_correct": 189, "fields": 225, "fn": 72, "fp": 32, "json_parse_ok": 104, "missed_absences": 4, "natural_stop": 108, "optional_omitted_correct": 3, "optional_omitted_total": 3, "predicted_strings": 170, "schema_ok": 94, "token_limited": 0, "tp": 222, "unsupported_strings": 8 }, "json_parse_rate": 0.9629629629629629, "missing_value_accuracy": 0.9253731343283582, "natural_stop_rate": 1.0, "optional_omission_accuracy": 1.0, "schema_valid_rate": 0.8703703703703703, "structural_path_f1": 0.8102189781021898, "structural_path_precision": 0.8740157480314961, "structural_path_recall": 0.7551020408163265, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.84, "unsupported_string_rate": 0.047058823529411764 }, "difficulty/Hard": { "complete_request_accuracy": 0.4444444444444444, "count": 18, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 8, "false_abstentions": 0, "field_correct": 12, "fields": 25, "fn": 31, "fp": 9, "json_parse_ok": 14, "missed_absences": 0, "natural_stop": 18, "optional_omitted_correct": 2, "optional_omitted_total": 2, "predicted_strings": 24, "schema_ok": 9, "token_limited": 0, "tp": 22, "unsupported_strings": 1 }, "json_parse_rate": 0.7777777777777778, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": 1.0, "schema_valid_rate": 0.5, "structural_path_f1": 0.5238095238095238, "structural_path_precision": 0.7096774193548387, "structural_path_recall": 0.41509433962264153, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.48, "unsupported_string_rate": 0.041666666666666664 }, "difficulty/Medium": { "complete_request_accuracy": 0.5625, "count": 16, "counts": { "absent_correct": 2, "absent_total": 3, "exact": 9, "false_abstentions": 0, "field_correct": 19, "fields": 29, "fn": 20, "fp": 10, "json_parse_ok": 16, "missed_absences": 0, "natural_stop": 16, "optional_omitted_correct": 1, "optional_omitted_total": 1, "predicted_strings": 29, "schema_ok": 13, "token_limited": 0, "tp": 24, "unsupported_strings": 2 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.6666666666666666, "natural_stop_rate": 1.0, "optional_omission_accuracy": 1.0, "schema_valid_rate": 0.8125, "structural_path_f1": 0.6153846153846154, "structural_path_precision": 0.7058823529411765, "structural_path_recall": 0.5454545454545454, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.6551724137931034, "unsupported_string_rate": 0.06896551724137931 }, "difficulty/Original": { "complete_request_accuracy": 0.8166666666666667, "count": 60, "counts": { "absent_correct": 55, "absent_total": 59, "exact": 49, "false_abstentions": 1, "field_correct": 136, "fields": 147, "fn": 17, "fp": 11, "json_parse_ok": 60, "missed_absences": 4, "natural_stop": 60, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 105, "schema_ok": 60, "token_limited": 0, "tp": 150, "unsupported_strings": 5 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.9322033898305084, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.9146341463414634, "structural_path_precision": 0.9316770186335404, "structural_path_recall": 0.8982035928143712, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.9251700680272109, "unsupported_string_rate": 0.047619047619047616 }, "difficulty/Simple": { "complete_request_accuracy": 0.8571428571428571, "count": 14, "counts": { "absent_correct": 5, "absent_total": 5, "exact": 12, "false_abstentions": 0, "field_correct": 22, "fields": 24, "fn": 4, "fp": 2, "json_parse_ok": 14, "missed_absences": 0, "natural_stop": 14, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 12, "schema_ok": 12, "token_limited": 0, "tp": 26, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": 1.0, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.8571428571428571, "structural_path_f1": 0.896551724137931, "structural_path_precision": 0.9285714285714286, "structural_path_recall": 0.8666666666666667, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.9166666666666666, "unsupported_string_rate": 0.0 }, "family/01 · Flat strings": { "complete_request_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 2, "false_abstentions": 0, "field_correct": 4, "fields": 4, "fn": 0, "fp": 0, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 4, "schema_ok": 2, "token_limited": 0, "tp": 4, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 1.0, "structural_path_precision": 1.0, "structural_path_recall": 1.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 1.0, "unsupported_string_rate": 0.0 }, "family/02 · Integers and decimals": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 1, "false_abstentions": 0, "field_correct": 3, "fields": 4, "fn": 1, "fp": 1, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 1, "schema_ok": 1, "token_limited": 0, "tp": 3, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5, "structural_path_f1": 0.75, "structural_path_precision": 0.75, "structural_path_recall": 0.75, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.75, "unsupported_string_rate": 0.0 }, "family/03 · Booleans": { "complete_request_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 2, "false_abstentions": 0, "field_correct": 4, "fields": 4, "fn": 0, "fp": 0, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 0, "schema_ok": 2, "token_limited": 0, "tp": 4, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 1.0, "structural_path_precision": 1.0, "structural_path_recall": 1.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 1.0, "unsupported_string_rate": null }, "family/04 · Nullable values": { "complete_request_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 2, "absent_total": 2, "exact": 2, "false_abstentions": 0, "field_correct": 4, "fields": 4, "fn": 0, "fp": 0, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 2, "schema_ok": 2, "token_limited": 0, "tp": 4, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": 1.0, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 1.0, "structural_path_precision": 1.0, "structural_path_recall": 1.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 1.0, "unsupported_string_rate": 0.0 }, "family/05 · String arrays": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 1, "false_abstentions": 0, "field_correct": 1, "fields": 2, "fn": 5, "fp": 1, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 4, "schema_ok": 2, "token_limited": 0, "tp": 3, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.5, "structural_path_precision": 0.75, "structural_path_recall": 0.375, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5, "unsupported_string_rate": 0.0 }, "family/06 · Numeric and boolean arrays": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 1, "false_abstentions": 0, "field_correct": 1, "fields": 2, "fn": 3, "fp": 1, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 1, "schema_ok": 1, "token_limited": 0, "tp": 3, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5, "structural_path_f1": 0.6, "structural_path_precision": 0.75, "structural_path_recall": 0.5, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5, "unsupported_string_rate": 0.0 }, "family/07 · Nested objects": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 1, "false_abstentions": 0, "field_correct": 1, "fields": 2, "fn": 3, "fp": 2, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 5, "schema_ok": 1, "token_limited": 0, "tp": 3, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5, "structural_path_f1": 0.5454545454545454, "structural_path_precision": 0.6, "structural_path_recall": 0.5, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5, "unsupported_string_rate": 0.0 }, "family/08 · Deeper nesting": { "complete_request_accuracy": 0.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 0, "false_abstentions": 0, "field_correct": 0, "fields": 2, "fn": 5, "fp": 3, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 4, "schema_ok": 0, "token_limited": 0, "tp": 1, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.0, "structural_path_f1": 0.2, "structural_path_precision": 0.25, "structural_path_recall": 0.16666666666666666, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.0, "unsupported_string_rate": 0.0 }, "family/09 · Arrays of objects": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 1, "false_abstentions": 0, "field_correct": 1, "fields": 2, "fn": 6, "fp": 3, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 7, "schema_ok": 1, "token_limited": 0, "tp": 4, "unsupported_strings": 1 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5, "structural_path_f1": 0.47058823529411764, "structural_path_precision": 0.5714285714285714, "structural_path_recall": 0.4, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5, "unsupported_string_rate": 0.14285714285714285 }, "family/10 · Multiple arrays and objects": { "complete_request_accuracy": 0.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 0, "false_abstentions": 0, "field_correct": 1, "fields": 5, "fn": 8, "fp": 2, "json_parse_ok": 1, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 3, "schema_ok": 1, "token_limited": 0, "tp": 1, "unsupported_strings": 1 }, "json_parse_rate": 0.5, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5, "structural_path_f1": 0.16666666666666666, "structural_path_precision": 0.3333333333333333, "structural_path_recall": 0.1111111111111111, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.2, "unsupported_string_rate": 0.3333333333333333 }, "family/11 · Arrays of arrays": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 1, "false_abstentions": 0, "field_correct": 1, "fields": 2, "fn": 4, "fp": 1, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 1, "schema_ok": 1, "token_limited": 0, "tp": 4, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5, "structural_path_f1": 0.6153846153846154, "structural_path_precision": 0.8, "structural_path_recall": 0.5, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5, "unsupported_string_rate": 0.0 }, "family/12 · Mixed arrays and tuples": { "complete_request_accuracy": 0.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 0, "false_abstentions": 0, "field_correct": 0, "fields": 2, "fn": 7, "fp": 1, "json_parse_ok": 1, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 1, "schema_ok": 0, "token_limited": 0, "tp": 0, "unsupported_strings": 0 }, "json_parse_rate": 0.5, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.0, "structural_path_f1": 0.0, "structural_path_precision": 0.0, "structural_path_recall": 0.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.0, "unsupported_string_rate": 0.0 }, "family/13 · Enum and const": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 1, "false_abstentions": 0, "field_correct": 4, "fields": 6, "fn": 2, "fp": 2, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 6, "schema_ok": 1, "token_limited": 0, "tp": 4, "unsupported_strings": 1 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5, "structural_path_f1": 0.6666666666666666, "structural_path_precision": 0.6666666666666666, "structural_path_recall": 0.6666666666666666, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.6666666666666666, "unsupported_string_rate": 0.16666666666666666 }, "family/14 · Optional properties": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 1, "false_abstentions": 0, "field_correct": 1, "fields": 3, "fn": 2, "fp": 1, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 1, "optional_omitted_total": 1, "predicted_strings": 2, "schema_ok": 2, "token_limited": 0, "tp": 1, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": 1.0, "schema_valid_rate": 1.0, "structural_path_f1": 0.4, "structural_path_precision": 0.5, "structural_path_recall": 0.3333333333333333, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.3333333333333333, "unsupported_string_rate": 0.0 }, "family/15 · Local schema references": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 1, "false_abstentions": 0, "field_correct": 2, "fields": 3, "fn": 1, "fp": 1, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 6, "schema_ok": 2, "token_limited": 0, "tp": 5, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.8333333333333334, "structural_path_precision": 0.8333333333333334, "structural_path_recall": 0.8333333333333334, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.6666666666666666, "unsupported_string_rate": 0.0 }, "family/16 · Union schemas": { "complete_request_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 1, "absent_total": 1, "exact": 2, "false_abstentions": 0, "field_correct": 4, "fields": 4, "fn": 0, "fp": 0, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 3, "schema_ok": 2, "token_limited": 0, "tp": 4, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": 1.0, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 1.0, "structural_path_precision": 1.0, "structural_path_recall": 1.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 1.0, "unsupported_string_rate": 0.0 }, "family/17 · Conditional schemas": { "complete_request_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 2, "false_abstentions": 0, "field_correct": 4, "fields": 4, "fn": 0, "fp": 0, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 2, "optional_omitted_total": 2, "predicted_strings": 4, "schema_ok": 2, "token_limited": 0, "tp": 4, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": 1.0, "schema_valid_rate": 1.0, "structural_path_f1": 1.0, "structural_path_precision": 1.0, "structural_path_recall": 1.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 1.0, "unsupported_string_rate": 0.0 }, "family/18 · Value constraints": { "complete_request_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 2, "false_abstentions": 0, "field_correct": 4, "fields": 4, "fn": 0, "fp": 0, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 2, "schema_ok": 2, "token_limited": 0, "tp": 4, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 1.0, "structural_path_precision": 1.0, "structural_path_recall": 1.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 1.0, "unsupported_string_rate": 0.0 }, "family/19 · Escaping and Unicode": { "complete_request_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 2, "false_abstentions": 0, "field_correct": 4, "fields": 4, "fn": 0, "fp": 0, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 4, "schema_ok": 2, "token_limited": 0, "tp": 4, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 1.0, "structural_path_precision": 1.0, "structural_path_recall": 1.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 1.0, "unsupported_string_rate": 0.0 }, "family/20 · allOf composition": { "complete_request_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 2, "false_abstentions": 0, "field_correct": 2, "fields": 2, "fn": 0, "fp": 0, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 1, "schema_ok": 2, "token_limited": 0, "tp": 2, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 1.0, "structural_path_precision": 1.0, "structural_path_recall": 1.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 1.0, "unsupported_string_rate": 0.0 }, "family/21 · Arrays inside objects": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 1, "false_abstentions": 0, "field_correct": 1, "fields": 2, "fn": 3, "fp": 1, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 3, "schema_ok": 2, "token_limited": 0, "tp": 4, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.6666666666666666, "structural_path_precision": 0.8, "structural_path_recall": 0.5714285714285714, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5, "unsupported_string_rate": 0.0 }, "family/22 · Single-field objects": { "complete_request_accuracy": 1.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 2, "false_abstentions": 0, "field_correct": 2, "fields": 2, "fn": 0, "fp": 0, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 1, "schema_ok": 2, "token_limited": 0, "tp": 2, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 1.0, "structural_path_precision": 1.0, "structural_path_recall": 1.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 1.0, "unsupported_string_rate": 0.0 }, "family/23 · Dynamic object keys": { "complete_request_accuracy": 0.0, "count": 2, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 0, "false_abstentions": 0, "field_correct": 0, "fields": 4, "fn": 4, "fp": 0, "json_parse_ok": 0, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 0, "schema_ok": 0, "token_limited": 0, "tp": 0, "unsupported_strings": 0 }, "json_parse_rate": 0.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.0, "structural_path_f1": 0.0, "structural_path_precision": null, "structural_path_recall": 0.0, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.0, "unsupported_string_rate": null }, "family/24 · Empty containers": { "complete_request_accuracy": 0.5, "count": 2, "counts": { "absent_correct": 4, "absent_total": 5, "exact": 1, "false_abstentions": 0, "field_correct": 4, "fields": 5, "fn": 1, "fp": 1, "json_parse_ok": 2, "missed_absences": 0, "natural_stop": 2, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 0, "schema_ok": 1, "token_limited": 0, "tp": 4, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.8, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5, "structural_path_f1": 0.8, "structural_path_precision": 0.8, "structural_path_recall": 0.8, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.8, "unsupported_string_rate": null }, "family/Original extraction": { "complete_request_accuracy": 0.8166666666666667, "count": 60, "counts": { "absent_correct": 55, "absent_total": 59, "exact": 49, "false_abstentions": 1, "field_correct": 136, "fields": 147, "fn": 17, "fp": 11, "json_parse_ok": 60, "missed_absences": 4, "natural_stop": 60, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 105, "schema_ok": 60, "token_limited": 0, "tp": 150, "unsupported_strings": 5 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.9322033898305084, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.9146341463414634, "structural_path_precision": 0.9316770186335404, "structural_path_recall": 0.8982035928143712, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.9251700680272109, "unsupported_string_rate": 0.047619047619047616 }, "macro_family_accuracy": 0.6126666666666667, "paired_schema_views": { "both_correct": 0, "count": 0, "incomplete_groups": 108 }, "schema_view/playground-v2": { "complete_request_accuracy": 0.7222222222222222, "count": 108, "counts": { "absent_correct": 62, "absent_total": 67, "exact": 78, "false_abstentions": 1, "field_correct": 189, "fields": 225, "fn": 72, "fp": 32, "json_parse_ok": 104, "missed_absences": 4, "natural_stop": 108, "optional_omitted_correct": 3, "optional_omitted_total": 3, "predicted_strings": 170, "schema_ok": 94, "token_limited": 0, "tp": 222, "unsupported_strings": 8 }, "json_parse_rate": 0.9629629629629629, "missing_value_accuracy": 0.9253731343283582, "natural_stop_rate": 1.0, "optional_omission_accuracy": 1.0, "schema_valid_rate": 0.8703703703703703, "structural_path_f1": 0.8102189781021898, "structural_path_precision": 0.8740157480314961, "structural_path_recall": 0.7551020408163265, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.84, "unsupported_string_rate": 0.047058823529411764 }, "tier/challenge48": { "complete_request_accuracy": 0.6041666666666666, "count": 48, "counts": { "absent_correct": 7, "absent_total": 8, "exact": 29, "false_abstentions": 0, "field_correct": 53, "fields": 78, "fn": 55, "fp": 21, "json_parse_ok": 44, "missed_absences": 0, "natural_stop": 48, "optional_omitted_correct": 3, "optional_omitted_total": 3, "predicted_strings": 65, "schema_ok": 34, "token_limited": 0, "tp": 72, "unsupported_strings": 3 }, "json_parse_rate": 0.9166666666666666, "missing_value_accuracy": 0.875, "natural_stop_rate": 1.0, "optional_omission_accuracy": 1.0, "schema_valid_rate": 0.7083333333333334, "structural_path_f1": 0.6545454545454545, "structural_path_precision": 0.7741935483870968, "structural_path_recall": 0.5669291338582677, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.6794871794871795, "unsupported_string_rate": 0.046153846153846156 }, "tier/original60": { "complete_request_accuracy": 0.8166666666666667, "count": 60, "counts": { "absent_correct": 55, "absent_total": 59, "exact": 49, "false_abstentions": 1, "field_correct": 136, "fields": 147, "fn": 17, "fp": 11, "json_parse_ok": 60, "missed_absences": 4, "natural_stop": 60, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 105, "schema_ok": 60, "token_limited": 0, "tp": 150, "unsupported_strings": 5 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.9322033898305084, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.9146341463414634, "structural_path_precision": 0.9316770186335404, "structural_path_recall": 0.8982035928143712, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.9251700680272109, "unsupported_string_rate": 0.047619047619047616 } }, "identity": { "assembly": false, "context": 2048, "input_files": { "reports/model-design/SFT-145_paired_foundation_20261018/full-eval-500/playground108-v2/references.jsonl": "a79abf4275ed465f1fefdb793e7ed5c4cb6fcbb3e42844d49fc8ab52f58f1ba3", "reports/model-design/SFT-145_paired_foundation_20261018/full-eval-500/playground108-v2/requests.jsonl": "b3a674665ac7e71aff6e6dc662670679a4859a20141130b6aad59f3854fcbf0b" }, "max_output_tokens": 256, "mlx": "0.32.3", "model": "153/500", "model_calls_per_request": 1, "raw_greedy": true, "references_in_input": false, "repair": false, "repetition_penalty": 1, "requests_sha256": "b3a674665ac7e71aff6e6dc662670679a4859a20141130b6aad59f3854fcbf0b", "runner_sha256": "843fc245da9134d1252ab4ca9182742c66a5ec20abfe15dfdef4bff5cc5cab0e", "scorer_sha256": "a8b6dff1b9f7e2cc7d526fa3614ab261cb4c43438dcf7436a1a579f403d46503", "selected_by_development": true, "serialization": "native-sft-chat-v1", "shared_decoder_sha256": "5a3826b2124f1eea2832a4d753a227c7f5f6c62bf882ffdb800cd6e20aeb6ce2", "step": 500, "structural_scorer_sha256": "334544f6a42d7b2c65840bb90d488e87c15d185900e8ceec41e24e24a3984505", "teachers_at_inference": false, "temperature": 0, "tokenizer_sha256": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb", "truncation": false, "weight_sha256": "25b406a7b86e3e8ac4dd0b7f32e27590d9d11349a0148a7815c32304150ac725" } }, "dedicated384": { "metrics": { "all": { "complete_request_accuracy": 0.4765625, "count": 384, "counts": { "absent_correct": 57, "absent_total": 70, "exact": 183, "false_abstentions": 12, "field_correct": 668, "fields": 1036, "fn": 433, "fp": 349, "json_parse_ok": 380, "missed_absences": 13, "natural_stop": 384, "optional_omitted_correct": 28, "optional_omitted_total": 36, "predicted_strings": 925, "schema_ok": 313, "token_limited": 0, "tp": 945, "unsupported_strings": 83 }, "json_parse_rate": 0.9895833333333334, "missing_value_accuracy": 0.8142857142857143, "natural_stop_rate": 1.0, "optional_omission_accuracy": 0.7777777777777778, "schema_valid_rate": 0.8151041666666666, "structural_path_f1": 0.7073353293413174, "structural_path_precision": 0.7302936630602782, "structural_path_recall": 0.6857764876632801, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.6447876447876448, "unsupported_string_rate": 0.08972972972972973 }, "difficulty/hard": { "complete_request_accuracy": 0.3359375, "count": 128, "counts": { "absent_correct": 16, "absent_total": 20, "exact": 43, "false_abstentions": 8, "field_correct": 194, "fields": 342, "fn": 197, "fp": 158, "json_parse_ok": 126, "missed_absences": 4, "natural_stop": 128, "optional_omitted_correct": 10, "optional_omitted_total": 12, "predicted_strings": 333, "schema_ok": 97, "token_limited": 0, "tp": 289, "unsupported_strings": 40 }, "json_parse_rate": 0.984375, "missing_value_accuracy": 0.8, "natural_stop_rate": 1.0, "optional_omission_accuracy": 0.8333333333333334, "schema_valid_rate": 0.7578125, "structural_path_f1": 0.6195069667738478, "structural_path_precision": 0.6465324384787472, "structural_path_recall": 0.5946502057613169, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5672514619883041, "unsupported_string_rate": 0.12012012012012012 }, "difficulty/medium": { "complete_request_accuracy": 0.53125, "count": 128, "counts": { "absent_correct": 21, "absent_total": 26, "exact": 68, "false_abstentions": 0, "field_correct": 239, "fields": 351, "fn": 121, "fp": 102, "json_parse_ok": 128, "missed_absences": 5, "natural_stop": 128, "optional_omitted_correct": 9, "optional_omitted_total": 12, "predicted_strings": 317, "schema_ok": 108, "token_limited": 0, "tp": 359, "unsupported_strings": 19 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.8076923076923077, "natural_stop_rate": 1.0, "optional_omission_accuracy": 0.75, "schema_valid_rate": 0.84375, "structural_path_f1": 0.7630180658873539, "structural_path_precision": 0.7787418655097614, "structural_path_recall": 0.7479166666666667, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.6809116809116809, "unsupported_string_rate": 0.05993690851735016 }, "difficulty/simple": { "complete_request_accuracy": 0.5625, "count": 128, "counts": { "absent_correct": 20, "absent_total": 24, "exact": 72, "false_abstentions": 4, "field_correct": 235, "fields": 343, "fn": 115, "fp": 89, "json_parse_ok": 126, "missed_absences": 4, "natural_stop": 128, "optional_omitted_correct": 9, "optional_omitted_total": 12, "predicted_strings": 275, "schema_ok": 108, "token_limited": 0, "tp": 297, "unsupported_strings": 24 }, "json_parse_rate": 0.984375, "missing_value_accuracy": 0.8333333333333334, "natural_stop_rate": 1.0, "optional_omission_accuracy": 0.75, "schema_valid_rate": 0.84375, "structural_path_f1": 0.7443609022556391, "structural_path_precision": 0.7694300518134715, "structural_path_recall": 0.720873786407767, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.685131195335277, "unsupported_string_rate": 0.08727272727272728 }, "family/boolean_null": { "complete_request_accuracy": 0.3333333333333333, "count": 24, "counts": { "absent_correct": 7, "absent_total": 12, "exact": 8, "false_abstentions": 3, "field_correct": 68, "fields": 120, "fn": 72, "fp": 28, "json_parse_ok": 20, "missed_absences": 5, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 27, "schema_ok": 13, "token_limited": 0, "tp": 96, "unsupported_strings": 6 }, "json_parse_rate": 0.8333333333333334, "missing_value_accuracy": 0.5833333333333334, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5416666666666666, "structural_path_f1": 0.6575342465753424, "structural_path_precision": 0.7741935483870968, "structural_path_recall": 0.5714285714285714, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5666666666666667, "unsupported_string_rate": 0.2222222222222222 }, "family/composition": { "complete_request_accuracy": 0.9166666666666666, "count": 24, "counts": { "absent_correct": 2, "absent_total": 4, "exact": 22, "false_abstentions": 0, "field_correct": 22, "fields": 24, "fn": 2, "fp": 2, "json_parse_ok": 24, "missed_absences": 2, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 30, "schema_ok": 24, "token_limited": 0, "tp": 30, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.5, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.9375, "structural_path_precision": 0.9375, "structural_path_recall": 0.9375, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.9166666666666666, "unsupported_string_rate": 0.0 }, "family/conditionals": { "complete_request_accuracy": 0.75, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 18, "false_abstentions": 0, "field_correct": 42, "fields": 54, "fn": 6, "fp": 6, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 18, "optional_omitted_total": 24, "predicted_strings": 48, "schema_ok": 18, "token_limited": 0, "tp": 42, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": 0.75, "schema_valid_rate": 0.75, "structural_path_f1": 0.875, "structural_path_precision": 0.875, "structural_path_recall": 0.875, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.7777777777777778, "unsupported_string_rate": 0.0 }, "family/constraints": { "complete_request_accuracy": 0.3333333333333333, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 8, "false_abstentions": 0, "field_correct": 55, "fields": 96, "fn": 41, "fp": 41, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 72, "schema_ok": 8, "token_limited": 0, "tp": 55, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.3333333333333333, "structural_path_f1": 0.5729166666666666, "structural_path_precision": 0.5729166666666666, "structural_path_recall": 0.5729166666666666, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5729166666666666, "unsupported_string_rate": 0.0 }, "family/corrections": { "complete_request_accuracy": 0.5, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 12, "false_abstentions": 0, "field_correct": 54, "fields": 72, "fn": 18, "fp": 18, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 72, "schema_ok": 24, "token_limited": 0, "tp": 54, "unsupported_strings": 5 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.75, "structural_path_precision": 0.75, "structural_path_recall": 0.75, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.75, "unsupported_string_rate": 0.06944444444444445 }, "family/exact_copy": { "complete_request_accuracy": 0.5, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 12, "false_abstentions": 0, "field_correct": 70, "fields": 96, "fn": 26, "fp": 21, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 91, "schema_ok": 20, "token_limited": 0, "tp": 70, "unsupported_strings": 1 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.8333333333333334, "structural_path_f1": 0.7486631016042781, "structural_path_precision": 0.7692307692307693, "structural_path_recall": 0.7291666666666666, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.7291666666666666, "unsupported_string_rate": 0.01098901098901099 }, "family/flat_fields": { "complete_request_accuracy": 0.4583333333333333, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 11, "false_abstentions": 0, "field_correct": 50, "fields": 72, "fn": 22, "fp": 22, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 72, "schema_ok": 24, "token_limited": 0, "tp": 50, "unsupported_strings": 4 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.6944444444444444, "structural_path_precision": 0.6944444444444444, "structural_path_recall": 0.6944444444444444, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.6944444444444444, "unsupported_string_rate": 0.05555555555555555 }, "family/lists": { "complete_request_accuracy": 0.5833333333333334, "count": 24, "counts": { "absent_correct": 28, "absent_total": 28, "exact": 14, "false_abstentions": 0, "field_correct": 38, "fields": 48, "fn": 24, "fp": 16, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 52, "schema_ok": 24, "token_limited": 0, "tp": 64, "unsupported_strings": 10 }, "json_parse_rate": 1.0, "missing_value_accuracy": 1.0, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.7619047619047619, "structural_path_precision": 0.8, "structural_path_recall": 0.7272727272727273, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.7916666666666666, "unsupported_string_rate": 0.19230769230769232 }, "family/maps_matrices_tuples": { "complete_request_accuracy": 0.8333333333333334, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 20, "false_abstentions": 0, "field_correct": 20, "fields": 24, "fn": 10, "fp": 2, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 8, "schema_ok": 20, "token_limited": 0, "tp": 62, "unsupported_strings": 0 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.8333333333333334, "structural_path_f1": 0.9117647058823529, "structural_path_precision": 0.96875, "structural_path_recall": 0.8611111111111112, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.8333333333333334, "unsupported_string_rate": 0.0 }, "family/missing_optional": { "complete_request_accuracy": 0.16666666666666666, "count": 24, "counts": { "absent_correct": 17, "absent_total": 20, "exact": 4, "false_abstentions": 8, "field_correct": 27, "fields": 62, "fn": 33, "fp": 24, "json_parse_ok": 24, "missed_absences": 3, "natural_stop": 24, "optional_omitted_correct": 10, "optional_omitted_total": 12, "predicted_strings": 25, "schema_ok": 23, "token_limited": 0, "tp": 27, "unsupported_strings": 1 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.85, "natural_stop_rate": 1.0, "optional_omission_accuracy": 0.8333333333333334, "schema_valid_rate": 0.9583333333333334, "structural_path_f1": 0.4864864864864865, "structural_path_precision": 0.5294117647058824, "structural_path_recall": 0.45, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.43548387096774194, "unsupported_string_rate": 0.04 }, "family/nested_objects": { "complete_request_accuracy": 0.5416666666666666, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 13, "false_abstentions": 0, "field_correct": 13, "fields": 24, "fn": 27, "fp": 20, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 57, "schema_ok": 16, "token_limited": 0, "tp": 37, "unsupported_strings": 3 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.6666666666666666, "structural_path_f1": 0.6115702479338843, "structural_path_precision": 0.6491228070175439, "structural_path_recall": 0.578125, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5416666666666666, "unsupported_string_rate": 0.05263157894736842 }, "family/numeric_types": { "complete_request_accuracy": 0.25, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 6, "false_abstentions": 1, "field_correct": 86, "fields": 120, "fn": 41, "fp": 32, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 31, "schema_ok": 13, "token_limited": 0, "tp": 103, "unsupported_strings": 4 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5416666666666666, "structural_path_f1": 0.7383512544802867, "structural_path_precision": 0.762962962962963, "structural_path_recall": 0.7152777777777778, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.7166666666666667, "unsupported_string_rate": 0.12903225806451613 }, "family/parallel_groups": { "complete_request_accuracy": 0.16666666666666666, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 4, "false_abstentions": 0, "field_correct": 34, "fields": 72, "fn": 42, "fp": 39, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 84, "schema_ok": 14, "token_limited": 0, "tp": 46, "unsupported_strings": 14 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 0.5833333333333334, "structural_path_f1": 0.5317919075144508, "structural_path_precision": 0.5411764705882353, "structural_path_recall": 0.5227272727272727, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.4722222222222222, "unsupported_string_rate": 0.16666666666666666 }, "family/record_arrays": { "complete_request_accuracy": 0.4583333333333333, "count": 24, "counts": { "absent_correct": 3, "absent_total": 6, "exact": 11, "false_abstentions": 0, "field_correct": 11, "fields": 24, "fn": 19, "fp": 28, "json_parse_ok": 24, "missed_absences": 3, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 80, "schema_ok": 24, "token_limited": 0, "tp": 83, "unsupported_strings": 14 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.5, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.7793427230046949, "structural_path_precision": 0.7477477477477478, "structural_path_recall": 0.8137254901960784, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.4583333333333333, "unsupported_string_rate": 0.175 }, "family/references": { "complete_request_accuracy": 0.625, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 15, "false_abstentions": 0, "field_correct": 23, "fields": 32, "fn": 9, "fp": 9, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 80, "schema_ok": 24, "token_limited": 0, "tp": 71, "unsupported_strings": 9 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.8875, "structural_path_precision": 0.8875, "structural_path_recall": 0.8875, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.71875, "unsupported_string_rate": 0.1125 }, "family/role_binding": { "complete_request_accuracy": 0.20833333333333334, "count": 24, "counts": { "absent_correct": 0, "absent_total": 0, "exact": 5, "false_abstentions": 0, "field_correct": 55, "fields": 96, "fn": 41, "fp": 41, "json_parse_ok": 24, "missed_absences": 0, "natural_stop": 24, "optional_omitted_correct": 0, "optional_omitted_total": 0, "predicted_strings": 96, "schema_ok": 24, "token_limited": 0, "tp": 55, "unsupported_strings": 12 }, "json_parse_rate": 1.0, "missing_value_accuracy": null, "natural_stop_rate": 1.0, "optional_omission_accuracy": null, "schema_valid_rate": 1.0, "structural_path_f1": 0.5729166666666666, "structural_path_precision": 0.5729166666666666, "structural_path_recall": 0.5729166666666666, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.5729166666666666, "unsupported_string_rate": 0.125 }, "macro_family_accuracy": 0.4765625, "paired_schema_views": { "both_correct": 64, "count": 192, "incomplete_groups": 0 }, "schema_view/canonical": { "complete_request_accuracy": 0.6041666666666666, "count": 192, "counts": { "absent_correct": 31, "absent_total": 35, "exact": 116, "false_abstentions": 1, "field_correct": 422, "fields": 516, "fn": 114, "fp": 107, "json_parse_ok": 192, "missed_absences": 4, "natural_stop": 192, "optional_omitted_correct": 16, "optional_omitted_total": 18, "predicted_strings": 473, "schema_ok": 179, "token_limited": 0, "tp": 575, "unsupported_strings": 42 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.8857142857142857, "natural_stop_rate": 1.0, "optional_omission_accuracy": 0.8888888888888888, "schema_valid_rate": 0.9322916666666666, "structural_path_f1": 0.8388037928519329, "structural_path_precision": 0.843108504398827, "structural_path_recall": 0.8345428156748912, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.8178294573643411, "unsupported_string_rate": 0.08879492600422834 }, "schema_view/renamed": { "complete_request_accuracy": 0.3489583333333333, "count": 192, "counts": { "absent_correct": 26, "absent_total": 35, "exact": 67, "false_abstentions": 11, "field_correct": 246, "fields": 520, "fn": 319, "fp": 242, "json_parse_ok": 188, "missed_absences": 9, "natural_stop": 192, "optional_omitted_correct": 12, "optional_omitted_total": 18, "predicted_strings": 452, "schema_ok": 134, "token_limited": 0, "tp": 370, "unsupported_strings": 41 }, "json_parse_rate": 0.9791666666666666, "missing_value_accuracy": 0.7428571428571429, "natural_stop_rate": 1.0, "optional_omission_accuracy": 0.6666666666666666, "schema_valid_rate": 0.6979166666666666, "structural_path_f1": 0.568793235972329, "structural_path_precision": 0.6045751633986928, "structural_path_recall": 0.5370101596516691, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.47307692307692306, "unsupported_string_rate": 0.09070796460176991 }, "tier/advanced": { "complete_request_accuracy": 0.78125, "count": 96, "counts": { "absent_correct": 2, "absent_total": 4, "exact": 75, "false_abstentions": 0, "field_correct": 107, "fields": 134, "fn": 27, "fp": 19, "json_parse_ok": 96, "missed_absences": 2, "natural_stop": 96, "optional_omitted_correct": 18, "optional_omitted_total": 24, "predicted_strings": 166, "schema_ok": 86, "token_limited": 0, "tp": 205, "unsupported_strings": 9 }, "json_parse_rate": 1.0, "missing_value_accuracy": 0.5, "natural_stop_rate": 1.0, "optional_omission_accuracy": 0.75, "schema_valid_rate": 0.8958333333333334, "structural_path_f1": 0.8991228070175439, "structural_path_precision": 0.9151785714285714, "structural_path_recall": 0.8836206896551724, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.7985074626865671, "unsupported_string_rate": 0.05421686746987952 }, "tier/core": { "complete_request_accuracy": 0.375, "count": 288, "counts": { "absent_correct": 55, "absent_total": 66, "exact": 108, "false_abstentions": 12, "field_correct": 561, "fields": 902, "fn": 406, "fp": 330, "json_parse_ok": 284, "missed_absences": 11, "natural_stop": 288, "optional_omitted_correct": 10, "optional_omitted_total": 12, "predicted_strings": 759, "schema_ok": 227, "token_limited": 0, "tp": 740, "unsupported_strings": 74 }, "json_parse_rate": 0.9861111111111112, "missing_value_accuracy": 0.8333333333333334, "natural_stop_rate": 1.0, "optional_omission_accuracy": 0.8333333333333334, "schema_valid_rate": 0.7881944444444444, "structural_path_f1": 0.6678700361010831, "structural_path_precision": 0.6915887850467289, "structural_path_recall": 0.6457242582897034, "token_limit_rate": 0.0, "top_level_field_accuracy": 0.6219512195121951, "unsupported_string_rate": 0.09749670619235837 } }, "identity": { "assembly": false, "context": 2048, "input_files": { "pretraining/data/jsonlite_eval_v1/references.jsonl": "e034d717d5c15fbf1cc714a1ff66bbf5fb759b0081527d5120bd6ba034a3d406", "pretraining/data/jsonlite_eval_v1/requests.jsonl": "4e68add1a3fd4f5920c659e0519ccdc2b143df08b5f553063c9b006b30bb4790" }, "max_output_tokens": 256, "mlx": "0.32.3", "model": "153/500", "model_calls_per_request": 1, "raw_greedy": true, "references_in_input": false, "repair": false, "repetition_penalty": 1, "requests_sha256": "4e68add1a3fd4f5920c659e0519ccdc2b143df08b5f553063c9b006b30bb4790", "runner_sha256": "843fc245da9134d1252ab4ca9182742c66a5ec20abfe15dfdef4bff5cc5cab0e", "scorer_sha256": "334544f6a42d7b2c65840bb90d488e87c15d185900e8ceec41e24e24a3984505", "selected_by_development": true, "serialization": "native-sft-chat-v1", "shared_decoder_sha256": "5a3826b2124f1eea2832a4d753a227c7f5f6c62bf882ffdb800cd6e20aeb6ce2", "step": 500, "structural_scorer_sha256": "334544f6a42d7b2c65840bb90d488e87c15d185900e8ceec41e24e24a3984505", "teachers_at_inference": false, "temperature": 0, "tokenizer_sha256": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb", "truncation": false, "weight_sha256": "25b406a7b86e3e8ac4dd0b7f32e27590d9d11349a0148a7815c32304150ac725" } }, "massive2974": { "metrics": { "complete_exact": 0.5262273032952253, "correct": 1565, "count": 2974, "counts": { "exact": 1565, "fields_correct": 1565, "fields_total": 2974, "fn": 3142, "fp": 2507, "gold_mentions": 2815, "json_parse_valid": 2973, "malformed_items": 0, "missing_correct": 853, "missing_total": 994, "natural_stop": 2974, "predicted_mentions": 2421, "schema_valid": 2966, "slot_fn": 1503, "slot_fp": 1109, "slot_tp": 1312, "token_limit": 0, "tp": 3482, "unannotated_correct": 853, "unannotated_reference": 994, "unsupported_mentions": 117 }, "diagnostic_counts": { "exact": 1565, "fields_correct": 1565, "fields_total": 2974, "fn": 3142, "fp": 2507, "gold_mentions": 2815, "json_parse_valid": 2973, "malformed_items": 0, "missing_correct": 853, "missing_total": 994, "natural_stop": 2974, "predicted_mentions": 2421, "schema_valid": 2966, "slot_fn": 1503, "slot_fp": 1109, "slot_tp": 1312, "token_limit": 0, "tp": 3482, "unannotated_correct": 853, "unannotated_reference": 994, "unsupported_mentions": 117 }, "field_accuracy": 0.5262273032952253, "json_parse_rate": 0.9996637525218561, "leaf_f1": 0.55212875604535, "leaf_precision": 0.5813992319251962, "leaf_recall": 0.5256642512077294, "missing_field_accuracy": 0.858148893360161, "natural_stop_rate": 1.0, "schema_valid_rate": 0.9973100201748487, "slot_f1": 0.5011459129106188, "slot_precision": 0.5419248244527055, "slot_recall": 0.4660746003552398 }, "identity": { "assembly": false, "context": 2048, "input_files": { "pretraining/data/massive_slot_list_139_20261017/references.jsonl": "1b6b2a18348939d959177f5a6a635f76a1b0645536e321195011dd2cd35b1461", "pretraining/data/massive_slot_list_139_20261017/requests.jsonl": "05acf77b73133416111348d7778db332168ceeeab00dc644386ea24af9a99b2f" }, "max_output_tokens": 384, "mlx": "0.32.3", "model": "153/500", "model_calls_per_request": 1, "raw_greedy": true, "references_in_input": false, "repair": false, "repetition_penalty": 1, "requests_sha256": "05acf77b73133416111348d7778db332168ceeeab00dc644386ea24af9a99b2f", "runner_sha256": "843fc245da9134d1252ab4ca9182742c66a5ec20abfe15dfdef4bff5cc5cab0e", "scorer_sha256": "07be6f71f4e2339e25ea9696be617fb440fb923fac81ba51aaa63a64bff0c096", "selected_by_development": true, "serialization": "native-sft-chat-v1", "shared_decoder_sha256": "5a3826b2124f1eea2832a4d753a227c7f5f6c62bf882ffdb800cd6e20aeb6ce2", "step": 500, "structural_scorer_sha256": "334544f6a42d7b2c65840bb90d488e87c15d185900e8ceec41e24e24a3984505", "teachers_at_inference": false, "temperature": 0, "tokenizer_sha256": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb", "truncation": false, "weight_sha256": "25b406a7b86e3e8ac4dd0b7f32e27590d9d11349a0148a7815c32304150ac725" } } }, "scope": "Separate previously observed research panels; not pooled, fresh blind benchmarks or official leaderboard scores. Raw greedy, no assembly, repair, constraints or reference hints." }