| { |
| "scope": "Reused offline text development tests; not independent blind accuracy or natural audio acceptance.", |
| "precision": "BF16", |
| "backend": "Transformers SDPA RTX", |
| "batch": 16, |
| "max_new_tokens": 256, |
| "metrics": { |
| "core": { |
| "all": { |
| "n": 8, |
| "lexical_exact": 8, |
| "exact": 8, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "user_exact": { |
| "n": 8, |
| "lexical_exact": 8, |
| "exact": 8, |
| "cer": 0.0, |
| "truncated": 0 |
| } |
| }, |
| "basic48": { |
| "all": { |
| "n": 48, |
| "lexical_exact": 46, |
| "exact": 42, |
| "cer": 0.004524886877828055, |
| "truncated": 0 |
| }, |
| "empty": { |
| "n": 4, |
| "lexical_exact": 4, |
| "exact": 4, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "english": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "filler": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "identifier": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "identity": { |
| "n": 6, |
| "lexical_exact": 6, |
| "exact": 6, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "instruction": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "list": { |
| "n": 2, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.05405405405405406, |
| "truncated": 0 |
| }, |
| "mixed": { |
| "n": 5, |
| "lexical_exact": 5, |
| "exact": 4, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "name": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "negation": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "number": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "punctuation": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 0, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "question": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "quote": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair": { |
| "n": 3, |
| "lexical_exact": 2, |
| "exact": 1, |
| "cer": 0.034482758620689655, |
| "truncated": 0 |
| }, |
| "restart": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "short": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| } |
| }, |
| "format20": { |
| "all": { |
| "n": 20, |
| "lexical_exact": 20, |
| "exact": 19, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "format_identity": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "heldout_list": { |
| "n": 5, |
| "lexical_exact": 5, |
| "exact": 4, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "heldout_number": { |
| "n": 6, |
| "lexical_exact": 6, |
| "exact": 6, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "number_identity": { |
| "n": 4, |
| "lexical_exact": 4, |
| "exact": 4, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "reported_list": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "reported_number": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| } |
| }, |
| "names60": { |
| "all": { |
| "n": 60, |
| "lexical_exact": 60, |
| "exact": 60, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "fresh_chinese_name_preservation": { |
| "n": 60, |
| "lexical_exact": 60, |
| "exact": 60, |
| "cer": 0.0, |
| "truncated": 0 |
| } |
| }, |
| "fresh_dev": { |
| "all": { |
| "n": 94, |
| "lexical_exact": 81, |
| "exact": 80, |
| "cer": 0.008744038155802861, |
| "truncated": 0 |
| }, |
| "fragment_complete_sentence_contrast": { |
| "n": 8, |
| "lexical_exact": 5, |
| "exact": 5, |
| "cer": 0.015810276679841896, |
| "truncated": 0 |
| }, |
| "fragment_dangling_no_autocomplete": { |
| "n": 8, |
| "lexical_exact": 8, |
| "exact": 8, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "grammar_homophone_fix_positive": { |
| "n": 1, |
| "lexical_exact": 0, |
| "exact": 0, |
| "cer": 0.058823529411764705, |
| "truncated": 0 |
| }, |
| "layout_newline_render_positive": { |
| "n": 6, |
| "lexical_exact": 4, |
| "exact": 4, |
| "cer": 0.013761467889908258, |
| "truncated": 0 |
| }, |
| "math_equation_correct": { |
| "n": 5, |
| "lexical_exact": 5, |
| "exact": 5, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "math_equation_wrong_rhs": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_address_room_hundred": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_address_room_thousand": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_coarse_idiom_preserve": { |
| "n": 4, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.037037037037037035, |
| "truncated": 0 |
| }, |
| "num_doc_page_hundred": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_internal_zero_hundred_vs_thousand": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_technical_model_version": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_unit_time_positive": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_bare_initialism_allcaps": { |
| "n": 2, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.3333333333333333, |
| "truncated": 0 |
| }, |
| "repair_bare_ordinary_titlecase": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_clause_comma_restoration": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_double_negation_and_contrast": { |
| "n": 10, |
| "lexical_exact": 10, |
| "exact": 10, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_explicit_display_paraphrase": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_list_compact_numbered": { |
| "n": 2, |
| "lexical_exact": 0, |
| "exact": 0, |
| "cer": 0.023809523809523808, |
| "truncated": 0 |
| }, |
| "repair_list_headless_numbered": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_list_weekday_fidelity": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_negation_and_comma_contrast": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_perturbed_nonce_fidelity": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "slot_correction_quote_negation_negative": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "slot_long_range_replacement_positive": { |
| "n": 12, |
| "lexical_exact": 11, |
| "exact": 11, |
| "cer": 0.007263922518159807, |
| "truncated": 0 |
| }, |
| "spell_explicit_display": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "spell_phonetic_near_repair": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "tech_compound_homophone_positive": { |
| "n": 5, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.013986013986013986, |
| "truncated": 0 |
| } |
| }, |
| "fresh_diagnostic": { |
| "all": { |
| "n": 97, |
| "lexical_exact": 74, |
| "exact": 73, |
| "cer": 0.04445234708392603, |
| "truncated": 0 |
| }, |
| "fragment_complete_sentence_contrast": { |
| "n": 9, |
| "lexical_exact": 7, |
| "exact": 7, |
| "cer": 0.0064516129032258064, |
| "truncated": 0 |
| }, |
| "fragment_dangling_no_autocomplete": { |
| "n": 9, |
| "lexical_exact": 8, |
| "exact": 8, |
| "cer": 0.004651162790697674, |
| "truncated": 0 |
| }, |
| "grammar_homophone_fix_positive": { |
| "n": 2, |
| "lexical_exact": 0, |
| "exact": 0, |
| "cer": 0.046511627906976744, |
| "truncated": 0 |
| }, |
| "layout_newline_render_positive": { |
| "n": 8, |
| "lexical_exact": 5, |
| "exact": 5, |
| "cer": 0.06666666666666667, |
| "truncated": 0 |
| }, |
| "math_equation_correct": { |
| "n": 5, |
| "lexical_exact": 4, |
| "exact": 4, |
| "cer": 0.014492753623188406, |
| "truncated": 0 |
| }, |
| "math_equation_wrong_rhs": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_address_room_hundred": { |
| "n": 1, |
| "lexical_exact": 0, |
| "exact": 0, |
| "cer": 0.06666666666666667, |
| "truncated": 0 |
| }, |
| "num_address_room_thousand": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_coarse_idiom_preserve": { |
| "n": 4, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.018518518518518517, |
| "truncated": 0 |
| }, |
| "num_code_line_hundred": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_internal_zero_hundred_vs_thousand": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_technical_model_version": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_unit_time_contrast_negative": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "num_unit_time_positive": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_clause_comma_restoration": { |
| "n": 2, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.06578947368421052, |
| "truncated": 0 |
| }, |
| "repair_double_negation_and_contrast": { |
| "n": 10, |
| "lexical_exact": 10, |
| "exact": 10, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_explicit_display_paraphrase": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_known_entity_casing": { |
| "n": 1, |
| "lexical_exact": 0, |
| "exact": 0, |
| "cer": 0.041666666666666664, |
| "truncated": 0 |
| }, |
| "repair_list_headless_numbered": { |
| "n": 4, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.08633093525179857, |
| "truncated": 0 |
| }, |
| "repair_list_weekday_fidelity": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repair_mid_sentence_ordinary_lowercase": { |
| "n": 2, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.022222222222222223, |
| "truncated": 0 |
| }, |
| "repair_negation_and_comma_contrast": { |
| "n": 3, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.011111111111111112, |
| "truncated": 0 |
| }, |
| "repair_perturbed_nonce_fidelity": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "slot_correction_quote_negation_negative": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "slot_long_range_replacement_positive": { |
| "n": 13, |
| "lexical_exact": 9, |
| "exact": 9, |
| "cer": 0.05021834061135371, |
| "truncated": 0 |
| }, |
| "spell_explicit_display": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "spell_phonetic_near_repair": { |
| "n": 3, |
| "lexical_exact": 0, |
| "exact": 0, |
| "cer": 0.8771929824561403, |
| "truncated": 0 |
| }, |
| "tech_compound_homophone_positive": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| } |
| }, |
| "public400": { |
| "all": { |
| "n": 400, |
| "lexical_exact": 232, |
| "exact": 118, |
| "cer": 0.09368478857770456, |
| "truncated": 0 |
| }, |
| "test_en": { |
| "n": 100, |
| "lexical_exact": 67, |
| "exact": 63, |
| "cer": 0.07500479570305006, |
| "truncated": 0 |
| }, |
| "test_identity": { |
| "n": 100, |
| "lexical_exact": 88, |
| "exact": 37, |
| "cer": 0.019711902956785442, |
| "truncated": 0 |
| }, |
| "test_zh": { |
| "n": 200, |
| "lexical_exact": 77, |
| "exact": 18, |
| "cer": 0.16945200155460552, |
| "truncated": 0 |
| } |
| }, |
| "intent": { |
| "all": { |
| "n": 326, |
| "lexical_exact": 318, |
| "exact": 318, |
| "cer": 0.0026373626373626374, |
| "truncated": 0 |
| }, |
| "literal_identity": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "meaning_identity": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "question_identity": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "question_list": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repeated_display": { |
| "n": 16, |
| "lexical_exact": 16, |
| "exact": 16, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "repeated_letters": { |
| "n": 16, |
| "lexical_exact": 16, |
| "exact": 16, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "reported_acronym": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "reported_context": { |
| "n": 2, |
| "lexical_exact": 2, |
| "exact": 2, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "reported_display": { |
| "n": 3, |
| "lexical_exact": 3, |
| "exact": 3, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "reported_person": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "reported_repair": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "reported_repetition": { |
| "n": 1, |
| "lexical_exact": 1, |
| "exact": 1, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "reported_word": { |
| "n": 4, |
| "lexical_exact": 4, |
| "exact": 4, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "semantic_case": { |
| "n": 42, |
| "lexical_exact": 42, |
| "exact": 42, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "unseen_acronym_word": { |
| "n": 12, |
| "lexical_exact": 12, |
| "exact": 12, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "unseen_brand_word": { |
| "n": 12, |
| "lexical_exact": 7, |
| "exact": 7, |
| "cer": 0.0457516339869281, |
| "truncated": 0 |
| }, |
| "unseen_display": { |
| "n": 78, |
| "lexical_exact": 78, |
| "exact": 78, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "unseen_ordinary_word": { |
| "n": 36, |
| "lexical_exact": 36, |
| "exact": 36, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "unseen_person_word": { |
| "n": 18, |
| "lexical_exact": 18, |
| "exact": 18, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "unseen_repair": { |
| "n": 78, |
| "lexical_exact": 75, |
| "exact": 75, |
| "cer": 0.004901960784313725, |
| "truncated": 0 |
| } |
| }, |
| "nonce": { |
| "all": { |
| "n": 200, |
| "lexical_exact": 197, |
| "exact": 197, |
| "cer": 0.002153183635804368, |
| "truncated": 0 |
| }, |
| "unseen_nonce_word": { |
| "n": 200, |
| "lexical_exact": 197, |
| "exact": 197, |
| "cer": 0.002153183635804368, |
| "truncated": 0 |
| } |
| }, |
| "math236": { |
| "all": { |
| "n": 236, |
| "lexical_exact": 235, |
| "exact": 235, |
| "cer": 0.0013136288998357964, |
| "truncated": 0 |
| }, |
| "math_equation_correct": { |
| "n": 53, |
| "lexical_exact": 53, |
| "exact": 53, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "math_equation_wrong_rhs": { |
| "n": 51, |
| "lexical_exact": 51, |
| "exact": 51, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "math_expression_no_eq": { |
| "n": 42, |
| "lexical_exact": 42, |
| "exact": 42, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "math_non_math_contrast": { |
| "n": 5, |
| "lexical_exact": 4, |
| "exact": 4, |
| "cer": 0.034482758620689655, |
| "truncated": 0 |
| }, |
| "math_question_preserve": { |
| "n": 29, |
| "lexical_exact": 29, |
| "exact": 29, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "math_self_corrected": { |
| "n": 14, |
| "lexical_exact": 14, |
| "exact": 14, |
| "cer": 0.0, |
| "truncated": 0 |
| }, |
| "math_unfinished_dangling": { |
| "n": 42, |
| "lexical_exact": 42, |
| "exact": 42, |
| "cer": 0.0, |
| "truncated": 0 |
| } |
| } |
| }, |
| "live_service": { |
| "engine": "vLLM 0.23.0 BF16, max-num-seqs=1", |
| "model_enabled_smoke_cases": 23, |
| "disabled_passthrough_cases": 1, |
| "scope": "Targeted deployment smoke only; not a broad accuracy benchmark." |
| }, |
| "precision_control": { |
| "diagnostic_batch1": { |
| "lexical_exact": 75, |
| "strict_exact": 75, |
| "n": 97 |
| }, |
| "diagnostic_FP32_batch16": { |
| "lexical_exact": 75, |
| "strict_exact": 75, |
| "n": 97 |
| } |
| }, |
| "metric_definition": { |
| "lexical_exact": "Exact equality after removing non-alphanumeric characters; letter case and digits retained.", |
| "strict_exact": "Output and reference equal after stripping leading and trailing whitespace.", |
| "cer": "Edit distance over lexical characters divided by total reference lexical characters." |
| }, |
| "limits": [ |
| "Repeated diagnostic use informed development; not blind testing.", |
| "Not comparable to upstream 94.8% token accuracy.", |
| "A known 501 versus 5001 transcription changes with BF16 batch shape; FP32 fixed that observed case, not a general guarantee.", |
| "Some formatting, corrections and technical-name cases still fail." |
| ] |
| } |
|
|
