{
  "scope": "Reused offline text development tests; not independent blind accuracy or natural audio acceptance.",
  "precision": "BF16",
  "backend": "Transformers SDPA RTX",
  "batch": 16,
  "max_new_tokens": 256,
  "metrics": {
    "core": {
      "all": {
        "n": 8,
        "lexical_exact": 8,
        "exact": 8,
        "cer": 0.0,
        "truncated": 0
      },
      "user_exact": {
        "n": 8,
        "lexical_exact": 8,
        "exact": 8,
        "cer": 0.0,
        "truncated": 0
      }
    },
    "basic48": {
      "all": {
        "n": 48,
        "lexical_exact": 46,
        "exact": 42,
        "cer": 0.004524886877828055,
        "truncated": 0
      },
      "empty": {
        "n": 4,
        "lexical_exact": 4,
        "exact": 4,
        "cer": 0.0,
        "truncated": 0
      },
      "english": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "filler": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "identifier": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "identity": {
        "n": 6,
        "lexical_exact": 6,
        "exact": 6,
        "cer": 0.0,
        "truncated": 0
      },
      "instruction": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "list": {
        "n": 2,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.05405405405405406,
        "truncated": 0
      },
      "mixed": {
        "n": 5,
        "lexical_exact": 5,
        "exact": 4,
        "cer": 0.0,
        "truncated": 0
      },
      "name": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "negation": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "number": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "punctuation": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 0,
        "cer": 0.0,
        "truncated": 0
      },
      "question": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "quote": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "repair": {
        "n": 3,
        "lexical_exact": 2,
        "exact": 1,
        "cer": 0.034482758620689655,
        "truncated": 0
      },
      "restart": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "short": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      }
    },
    "format20": {
      "all": {
        "n": 20,
        "lexical_exact": 20,
        "exact": 19,
        "cer": 0.0,
        "truncated": 0
      },
      "format_identity": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "heldout_list": {
        "n": 5,
        "lexical_exact": 5,
        "exact": 4,
        "cer": 0.0,
        "truncated": 0
      },
      "heldout_number": {
        "n": 6,
        "lexical_exact": 6,
        "exact": 6,
        "cer": 0.0,
        "truncated": 0
      },
      "number_identity": {
        "n": 4,
        "lexical_exact": 4,
        "exact": 4,
        "cer": 0.0,
        "truncated": 0
      },
      "reported_list": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "reported_number": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      }
    },
    "names60": {
      "all": {
        "n": 60,
        "lexical_exact": 60,
        "exact": 60,
        "cer": 0.0,
        "truncated": 0
      },
      "fresh_chinese_name_preservation": {
        "n": 60,
        "lexical_exact": 60,
        "exact": 60,
        "cer": 0.0,
        "truncated": 0
      }
    },
    "fresh_dev": {
      "all": {
        "n": 94,
        "lexical_exact": 81,
        "exact": 80,
        "cer": 0.008744038155802861,
        "truncated": 0
      },
      "fragment_complete_sentence_contrast": {
        "n": 8,
        "lexical_exact": 5,
        "exact": 5,
        "cer": 0.015810276679841896,
        "truncated": 0
      },
      "fragment_dangling_no_autocomplete": {
        "n": 8,
        "lexical_exact": 8,
        "exact": 8,
        "cer": 0.0,
        "truncated": 0
      },
      "grammar_homophone_fix_positive": {
        "n": 1,
        "lexical_exact": 0,
        "exact": 0,
        "cer": 0.058823529411764705,
        "truncated": 0
      },
      "layout_newline_render_positive": {
        "n": 6,
        "lexical_exact": 4,
        "exact": 4,
        "cer": 0.013761467889908258,
        "truncated": 0
      },
      "math_equation_correct": {
        "n": 5,
        "lexical_exact": 5,
        "exact": 5,
        "cer": 0.0,
        "truncated": 0
      },
      "math_equation_wrong_rhs": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "num_address_room_hundred": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "num_address_room_thousand": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "num_coarse_idiom_preserve": {
        "n": 4,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.037037037037037035,
        "truncated": 0
      },
      "num_doc_page_hundred": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "num_internal_zero_hundred_vs_thousand": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "num_technical_model_version": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "num_unit_time_positive": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_bare_initialism_allcaps": {
        "n": 2,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.3333333333333333,
        "truncated": 0
      },
      "repair_bare_ordinary_titlecase": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_clause_comma_restoration": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_double_negation_and_contrast": {
        "n": 10,
        "lexical_exact": 10,
        "exact": 10,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_explicit_display_paraphrase": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_list_compact_numbered": {
        "n": 2,
        "lexical_exact": 0,
        "exact": 0,
        "cer": 0.023809523809523808,
        "truncated": 0
      },
      "repair_list_headless_numbered": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_list_weekday_fidelity": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_negation_and_comma_contrast": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_perturbed_nonce_fidelity": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "slot_correction_quote_negation_negative": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "slot_long_range_replacement_positive": {
        "n": 12,
        "lexical_exact": 11,
        "exact": 11,
        "cer": 0.007263922518159807,
        "truncated": 0
      },
      "spell_explicit_display": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "spell_phonetic_near_repair": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "tech_compound_homophone_positive": {
        "n": 5,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.013986013986013986,
        "truncated": 0
      }
    },
    "fresh_diagnostic": {
      "all": {
        "n": 97,
        "lexical_exact": 74,
        "exact": 73,
        "cer": 0.04445234708392603,
        "truncated": 0
      },
      "fragment_complete_sentence_contrast": {
        "n": 9,
        "lexical_exact": 7,
        "exact": 7,
        "cer": 0.0064516129032258064,
        "truncated": 0
      },
      "fragment_dangling_no_autocomplete": {
        "n": 9,
        "lexical_exact": 8,
        "exact": 8,
        "cer": 0.004651162790697674,
        "truncated": 0
      },
      "grammar_homophone_fix_positive": {
        "n": 2,
        "lexical_exact": 0,
        "exact": 0,
        "cer": 0.046511627906976744,
        "truncated": 0
      },
      "layout_newline_render_positive": {
        "n": 8,
        "lexical_exact": 5,
        "exact": 5,
        "cer": 0.06666666666666667,
        "truncated": 0
      },
      "math_equation_correct": {
        "n": 5,
        "lexical_exact": 4,
        "exact": 4,
        "cer": 0.014492753623188406,
        "truncated": 0
      },
      "math_equation_wrong_rhs": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "num_address_room_hundred": {
        "n": 1,
        "lexical_exact": 0,
        "exact": 0,
        "cer": 0.06666666666666667,
        "truncated": 0
      },
      "num_address_room_thousand": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "num_coarse_idiom_preserve": {
        "n": 4,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.018518518518518517,
        "truncated": 0
      },
      "num_code_line_hundred": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "num_internal_zero_hundred_vs_thousand": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "num_technical_model_version": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "num_unit_time_contrast_negative": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "num_unit_time_positive": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_clause_comma_restoration": {
        "n": 2,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.06578947368421052,
        "truncated": 0
      },
      "repair_double_negation_and_contrast": {
        "n": 10,
        "lexical_exact": 10,
        "exact": 10,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_explicit_display_paraphrase": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_known_entity_casing": {
        "n": 1,
        "lexical_exact": 0,
        "exact": 0,
        "cer": 0.041666666666666664,
        "truncated": 0
      },
      "repair_list_headless_numbered": {
        "n": 4,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.08633093525179857,
        "truncated": 0
      },
      "repair_list_weekday_fidelity": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "repair_mid_sentence_ordinary_lowercase": {
        "n": 2,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.022222222222222223,
        "truncated": 0
      },
      "repair_negation_and_comma_contrast": {
        "n": 3,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.011111111111111112,
        "truncated": 0
      },
      "repair_perturbed_nonce_fidelity": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "slot_correction_quote_negation_negative": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "slot_long_range_replacement_positive": {
        "n": 13,
        "lexical_exact": 9,
        "exact": 9,
        "cer": 0.05021834061135371,
        "truncated": 0
      },
      "spell_explicit_display": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "spell_phonetic_near_repair": {
        "n": 3,
        "lexical_exact": 0,
        "exact": 0,
        "cer": 0.8771929824561403,
        "truncated": 0
      },
      "tech_compound_homophone_positive": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      }
    },
    "public400": {
      "all": {
        "n": 400,
        "lexical_exact": 232,
        "exact": 118,
        "cer": 0.09368478857770456,
        "truncated": 0
      },
      "test_en": {
        "n": 100,
        "lexical_exact": 67,
        "exact": 63,
        "cer": 0.07500479570305006,
        "truncated": 0
      },
      "test_identity": {
        "n": 100,
        "lexical_exact": 88,
        "exact": 37,
        "cer": 0.019711902956785442,
        "truncated": 0
      },
      "test_zh": {
        "n": 200,
        "lexical_exact": 77,
        "exact": 18,
        "cer": 0.16945200155460552,
        "truncated": 0
      }
    },
    "intent": {
      "all": {
        "n": 326,
        "lexical_exact": 318,
        "exact": 318,
        "cer": 0.0026373626373626374,
        "truncated": 0
      },
      "literal_identity": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "meaning_identity": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "question_identity": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "question_list": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "repeated_display": {
        "n": 16,
        "lexical_exact": 16,
        "exact": 16,
        "cer": 0.0,
        "truncated": 0
      },
      "repeated_letters": {
        "n": 16,
        "lexical_exact": 16,
        "exact": 16,
        "cer": 0.0,
        "truncated": 0
      },
      "reported_acronym": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "reported_context": {
        "n": 2,
        "lexical_exact": 2,
        "exact": 2,
        "cer": 0.0,
        "truncated": 0
      },
      "reported_display": {
        "n": 3,
        "lexical_exact": 3,
        "exact": 3,
        "cer": 0.0,
        "truncated": 0
      },
      "reported_person": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "reported_repair": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "reported_repetition": {
        "n": 1,
        "lexical_exact": 1,
        "exact": 1,
        "cer": 0.0,
        "truncated": 0
      },
      "reported_word": {
        "n": 4,
        "lexical_exact": 4,
        "exact": 4,
        "cer": 0.0,
        "truncated": 0
      },
      "semantic_case": {
        "n": 42,
        "lexical_exact": 42,
        "exact": 42,
        "cer": 0.0,
        "truncated": 0
      },
      "unseen_acronym_word": {
        "n": 12,
        "lexical_exact": 12,
        "exact": 12,
        "cer": 0.0,
        "truncated": 0
      },
      "unseen_brand_word": {
        "n": 12,
        "lexical_exact": 7,
        "exact": 7,
        "cer": 0.0457516339869281,
        "truncated": 0
      },
      "unseen_display": {
        "n": 78,
        "lexical_exact": 78,
        "exact": 78,
        "cer": 0.0,
        "truncated": 0
      },
      "unseen_ordinary_word": {
        "n": 36,
        "lexical_exact": 36,
        "exact": 36,
        "cer": 0.0,
        "truncated": 0
      },
      "unseen_person_word": {
        "n": 18,
        "lexical_exact": 18,
        "exact": 18,
        "cer": 0.0,
        "truncated": 0
      },
      "unseen_repair": {
        "n": 78,
        "lexical_exact": 75,
        "exact": 75,
        "cer": 0.004901960784313725,
        "truncated": 0
      }
    },
    "nonce": {
      "all": {
        "n": 200,
        "lexical_exact": 197,
        "exact": 197,
        "cer": 0.002153183635804368,
        "truncated": 0
      },
      "unseen_nonce_word": {
        "n": 200,
        "lexical_exact": 197,
        "exact": 197,
        "cer": 0.002153183635804368,
        "truncated": 0
      }
    },
    "math236": {
      "all": {
        "n": 236,
        "lexical_exact": 235,
        "exact": 235,
        "cer": 0.0013136288998357964,
        "truncated": 0
      },
      "math_equation_correct": {
        "n": 53,
        "lexical_exact": 53,
        "exact": 53,
        "cer": 0.0,
        "truncated": 0
      },
      "math_equation_wrong_rhs": {
        "n": 51,
        "lexical_exact": 51,
        "exact": 51,
        "cer": 0.0,
        "truncated": 0
      },
      "math_expression_no_eq": {
        "n": 42,
        "lexical_exact": 42,
        "exact": 42,
        "cer": 0.0,
        "truncated": 0
      },
      "math_non_math_contrast": {
        "n": 5,
        "lexical_exact": 4,
        "exact": 4,
        "cer": 0.034482758620689655,
        "truncated": 0
      },
      "math_question_preserve": {
        "n": 29,
        "lexical_exact": 29,
        "exact": 29,
        "cer": 0.0,
        "truncated": 0
      },
      "math_self_corrected": {
        "n": 14,
        "lexical_exact": 14,
        "exact": 14,
        "cer": 0.0,
        "truncated": 0
      },
      "math_unfinished_dangling": {
        "n": 42,
        "lexical_exact": 42,
        "exact": 42,
        "cer": 0.0,
        "truncated": 0
      }
    }
  },
  "live_service": {
    "engine": "vLLM 0.23.0 BF16, max-num-seqs=1",
    "model_enabled_smoke_cases": 23,
    "disabled_passthrough_cases": 1,
    "scope": "Targeted deployment smoke only; not a broad accuracy benchmark."
  },
  "precision_control": {
    "diagnostic_batch1": {
      "lexical_exact": 75,
      "strict_exact": 75,
      "n": 97
    },
    "diagnostic_FP32_batch16": {
      "lexical_exact": 75,
      "strict_exact": 75,
      "n": 97
    }
  },
  "metric_definition": {
    "lexical_exact": "Exact equality after removing non-alphanumeric characters; letter case and digits retained.",
    "strict_exact": "Output and reference equal after stripping leading and trailing whitespace.",
    "cer": "Edit distance over lexical characters divided by total reference lexical characters."
  },
  "limits": [
    "Repeated diagnostic use informed development; not blind testing.",
    "Not comparable to upstream 94.8% token accuracy.",
    "A known 501 versus 5001 transcription changes with BF16 batch shape; FP32 fixed that observed case, not a general guarantee.",
    "Some formatting, corrections and technical-name cases still fail."
  ]
}