File size: 2,763 Bytes
2737bcb 40189a8 2737bcb 40189a8 2737bcb 40189a8 2737bcb 40189a8 2737bcb 40189a8 782bc12 40189a8 2737bcb 40189a8 2737bcb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 | {
"revision": "format-numbers-v5",
"weights_sha256": "54602b3a2e0c484fe7ae796a4290969d3a1cc58e099c83e59948fe94889150a0",
"public400": {
"previous_step400": {
"all": {
"n": 400,
"lexical_exact": 239,
"exact": 110,
"cer": 0.10115321252059307,
"truncated": 0
},
"test_en": {
"n": 100,
"lexical_exact": 64,
"exact": 57,
"cer": 0.09015921734126223,
"truncated": 0
},
"test_identity": {
"n": 100,
"lexical_exact": 92,
"exact": 38,
"cer": 0.014404852160727824,
"truncated": 0
},
"test_zh": {
"n": 200,
"lexical_exact": 83,
"exact": 15,
"cer": 0.16789739603575593,
"truncated": 0
}
},
"current": {
"all": {
"n": 400,
"lexical_exact": 232,
"exact": 108,
"cer": 0.10247116968698518,
"truncated": 0
},
"test_en": {
"n": 100,
"lexical_exact": 60,
"exact": 54,
"cer": 0.09246115480529446,
"truncated": 0
},
"test_identity": {
"n": 100,
"lexical_exact": 91,
"exact": 38,
"cer": 0.01592115238817286,
"truncated": 0
},
"test_zh": {
"n": 200,
"lexical_exact": 81,
"exact": 16,
"cer": 0.16712009327633112,
"truncated": 0
}
},
"decoding": {
"dtype": "bfloat16",
"temperature": 0,
"batch_size": 16,
"max_new_tokens": 256,
"backend": "Transformers 5.3 / PyTorch 2.10, CUDA, SDPA"
},
"scope": "Raw model outputs on the same reused 400 public text cases. Lexical metrics remove punctuation and retain case, numbers, negation and order; not semantic accuracy or a new blind benchmark."
},
"development20": {
"offline_exact": 18,
"sequential_orin_vllm_exact": 19,
"cases": 20,
"numbered_list_cases": 6,
"numbered_list_cases_rendered_as_numbered_lines": 6,
"scope": "Diagnostic probes, excluded from training inputs after lexical normalization. Runtime/reference mismatch retained equivalent Chinese quantity spelling. Backend/batching may change greedy BF16 output; results are not a new broad quality benchmark."
},
"token_metrics": {
"status": "not_recomputed_for_current_weights",
"historical_file": "history/step400/token-metrics.json"
},
"deployment_content_guard": "none; external configuration, not model weights",
"limits": [
"English lexical sentence matches decreased from 64/100 to 60/100.",
"Overall lexical CER increased from 10.115% to 10.247%.",
"Number/name correction and paragraph/list grouping are not universally reliable."
]
}
|