S1-mini-Chinese-1106-FP8 / evaluation-summary.json
WIlfLin's picture
Release verified FP8 W8A16 Marlin checkpoint with Chinese regression results
2cf7deb verified
Raw History Blame Contribute Delete
4.04 kB
{
"all_passed": true,
"failed": [],
"baseline": {
"core": {
"n": 8,
"exact": 8,
"lexical": 8,
"truncated": 0,
"cer": 0.0
},
"basic48": {
"n": 48,
"exact": 42,
"lexical": 46,
"truncated": 0,
"cer": 0.004524886877828055
},
"format20": {
"n": 20,
"exact": 19,
"lexical": 20,
"truncated": 0,
"cer": 0.0
},
"names60": {
"n": 60,
"exact": 60,
"lexical": 60,
"truncated": 0,
"cer": 0.0
},
"public400": {
"n": 400,
"exact": 120,
"lexical": 234,
"truncated": 0,
"cer": 0.09236683141131247
},
"intent": {
"n": 326,
"exact": 318,
"lexical": 318,
"truncated": 0,
"cer": 0.0026373626373626374
},
"nonce": {
"n": 200,
"exact": 196,
"lexical": 196,
"truncated": 0,
"cer": 0.0030759766225776685
},
"math236": {
"n": 236,
"exact": 235,
"lexical": 235,
"truncated": 0,
"cer": 0.0013136288998357964
},
"dev432": {
"n": 432,
"exact": 414,
"lexical": 414,
"truncated": 0,
"cer": 0.011299435028248588
},
"heldout432": {
"n": 432,
"exact": 430,
"lexical": 430,
"truncated": 0,
"cer": 0.0009416195856873823
},
"critical5": {
"n": 5,
"exact": 5,
"lexical": 5,
"truncated": 0,
"cer": 0.0
}
},
"candidate": {
"core": {
"n": 8,
"exact": 8,
"lexical": 8,
"truncated": 0,
"cer": 0.0
},
"basic48": {
"n": 48,
"exact": 42,
"lexical": 46,
"truncated": 0,
"cer": 0.004524886877828055
},
"format20": {
"n": 20,
"exact": 19,
"lexical": 20,
"truncated": 0,
"cer": 0.0
},
"names60": {
"n": 60,
"exact": 60,
"lexical": 60,
"truncated": 0,
"cer": 0.0
},
"public400": {
"n": 400,
"exact": 117,
"lexical": 233,
"truncated": 0,
"cer": 0.09368478857770456
},
"intent": {
"n": 326,
"exact": 318,
"lexical": 318,
"truncated": 0,
"cer": 0.0024175824175824176
},
"nonce": {
"n": 200,
"exact": 197,
"lexical": 197,
"truncated": 0,
"cer": 0.0015379883112888342
},
"math236": {
"n": 236,
"exact": 235,
"lexical": 235,
"truncated": 0,
"cer": 0.0013136288998357964
},
"dev432": {
"n": 432,
"exact": 414,
"lexical": 414,
"truncated": 0,
"cer": 0.011299435028248588
},
"heldout432": {
"n": 432,
"exact": 429,
"lexical": 429,
"truncated": 0,
"cer": 0.001569365976145637
},
"critical5": {
"n": 5,
"exact": 5,
"lexical": 5,
"truncated": 0,
"cer": 0.0
}
},
"scope": "Actual serialized compressed checkpoint. Reused development/regression tests; no natural-audio, independent blind or Orin acceptance claim.",
"new_numeric_value_errors_count": 0,
"new_operator_errors_count": 0,
"new_layout_errors_count": 0,
"new_name_errors_count": 0,
"latex_outputs_count": 0,
"sequences": 8,
"other_profile_rejections": {},
"effective_rope_theta": 1000000,
"request_chunk": 64,
"runtime": "vLLM0.17 Blackwell SM120, eager, max-num-seqs8, prefix cache disabled; BF16 KV; compressed-tensors FP8 W8A16 Marlin with BF16 activations.",
"single_request_critical": {
"requested_n": 5,
"early_rejection": false,
"effective_rope_parameters": {
"rope_theta": 1000000,
"rope_type": "default"
},
"n": 5,
"strict_exact": 5,
"lexical_exact": 5,
"truncated": 0,
"seconds": 0.554567532002693,
"engine": "vLLM 0.17.0 eager; Blackwell SM120",
"sequences": 1,
"request_chunk": 64,
"scope": "Real serialized checkpoint, not ModelOpt fake quant. Runtime is not the production Orin vLLM version."
}
}