Download evaluation-summary.json from WIlfLin/S1-mini-Chinese-1106-FP8: direct link, hf CLI and curl.
- Browser
- Download file 4.04 kB
-
https://huggingface.co/WIlfLin/S1-mini-Chinese-1106-FP8/resolve/main/evaluation-summary.json
- Command line
-
hf download hf://WIlfLin/S1-mini-Chinese-1106-FP8/evaluation-summary.json
-
curl -L -o evaluation-summary.json https://huggingface.co/WIlfLin/S1-mini-Chinese-1106-FP8/resolve/main/evaluation-summary.json
4.04 kB
| { | |
| "all_passed": true, | |
| "failed": [], | |
| "baseline": { | |
| "core": { | |
| "n": 8, | |
| "exact": 8, | |
| "lexical": 8, | |
| "truncated": 0, | |
| "cer": 0.0 | |
| }, | |
| "basic48": { | |
| "n": 48, | |
| "exact": 42, | |
| "lexical": 46, | |
| "truncated": 0, | |
| "cer": 0.004524886877828055 | |
| }, | |
| "format20": { | |
| "n": 20, | |
| "exact": 19, | |
| "lexical": 20, | |
| "truncated": 0, | |
| "cer": 0.0 | |
| }, | |
| "names60": { | |
| "n": 60, | |
| "exact": 60, | |
| "lexical": 60, | |
| "truncated": 0, | |
| "cer": 0.0 | |
| }, | |
| "public400": { | |
| "n": 400, | |
| "exact": 120, | |
| "lexical": 234, | |
| "truncated": 0, | |
| "cer": 0.09236683141131247 | |
| }, | |
| "intent": { | |
| "n": 326, | |
| "exact": 318, | |
| "lexical": 318, | |
| "truncated": 0, | |
| "cer": 0.0026373626373626374 | |
| }, | |
| "nonce": { | |
| "n": 200, | |
| "exact": 196, | |
| "lexical": 196, | |
| "truncated": 0, | |
| "cer": 0.0030759766225776685 | |
| }, | |
| "math236": { | |
| "n": 236, | |
| "exact": 235, | |
| "lexical": 235, | |
| "truncated": 0, | |
| "cer": 0.0013136288998357964 | |
| }, | |
| "dev432": { | |
| "n": 432, | |
| "exact": 414, | |
| "lexical": 414, | |
| "truncated": 0, | |
| "cer": 0.011299435028248588 | |
| }, | |
| "heldout432": { | |
| "n": 432, | |
| "exact": 430, | |
| "lexical": 430, | |
| "truncated": 0, | |
| "cer": 0.0009416195856873823 | |
| }, | |
| "critical5": { | |
| "n": 5, | |
| "exact": 5, | |
| "lexical": 5, | |
| "truncated": 0, | |
| "cer": 0.0 | |
| } | |
| }, | |
| "candidate": { | |
| "core": { | |
| "n": 8, | |
| "exact": 8, | |
| "lexical": 8, | |
| "truncated": 0, | |
| "cer": 0.0 | |
| }, | |
| "basic48": { | |
| "n": 48, | |
| "exact": 42, | |
| "lexical": 46, | |
| "truncated": 0, | |
| "cer": 0.004524886877828055 | |
| }, | |
| "format20": { | |
| "n": 20, | |
| "exact": 19, | |
| "lexical": 20, | |
| "truncated": 0, | |
| "cer": 0.0 | |
| }, | |
| "names60": { | |
| "n": 60, | |
| "exact": 60, | |
| "lexical": 60, | |
| "truncated": 0, | |
| "cer": 0.0 | |
| }, | |
| "public400": { | |
| "n": 400, | |
| "exact": 117, | |
| "lexical": 233, | |
| "truncated": 0, | |
| "cer": 0.09368478857770456 | |
| }, | |
| "intent": { | |
| "n": 326, | |
| "exact": 318, | |
| "lexical": 318, | |
| "truncated": 0, | |
| "cer": 0.0024175824175824176 | |
| }, | |
| "nonce": { | |
| "n": 200, | |
| "exact": 197, | |
| "lexical": 197, | |
| "truncated": 0, | |
| "cer": 0.0015379883112888342 | |
| }, | |
| "math236": { | |
| "n": 236, | |
| "exact": 235, | |
| "lexical": 235, | |
| "truncated": 0, | |
| "cer": 0.0013136288998357964 | |
| }, | |
| "dev432": { | |
| "n": 432, | |
| "exact": 414, | |
| "lexical": 414, | |
| "truncated": 0, | |
| "cer": 0.011299435028248588 | |
| }, | |
| "heldout432": { | |
| "n": 432, | |
| "exact": 429, | |
| "lexical": 429, | |
| "truncated": 0, | |
| "cer": 0.001569365976145637 | |
| }, | |
| "critical5": { | |
| "n": 5, | |
| "exact": 5, | |
| "lexical": 5, | |
| "truncated": 0, | |
| "cer": 0.0 | |
| } | |
| }, | |
| "scope": "Actual serialized compressed checkpoint. Reused development/regression tests; no natural-audio, independent blind or Orin acceptance claim.", | |
| "new_numeric_value_errors_count": 0, | |
| "new_operator_errors_count": 0, | |
| "new_layout_errors_count": 0, | |
| "new_name_errors_count": 0, | |
| "latex_outputs_count": 0, | |
| "sequences": 8, | |
| "other_profile_rejections": {}, | |
| "effective_rope_theta": 1000000, | |
| "request_chunk": 64, | |
| "runtime": "vLLM0.17 Blackwell SM120, eager, max-num-seqs8, prefix cache disabled; BF16 KV; compressed-tensors FP8 W8A16 Marlin with BF16 activations.", | |
| "single_request_critical": { | |
| "requested_n": 5, | |
| "early_rejection": false, | |
| "effective_rope_parameters": { | |
| "rope_theta": 1000000, | |
| "rope_type": "default" | |
| }, | |
| "n": 5, | |
| "strict_exact": 5, | |
| "lexical_exact": 5, | |
| "truncated": 0, | |
| "seconds": 0.554567532002693, | |
| "engine": "vLLM 0.17.0 eager; Blackwell SM120", | |
| "sequences": 1, | |
| "request_chunk": 64, | |
| "scope": "Real serialized checkpoint, not ModelOpt fake quant. Runtime is not the production Orin vLLM version." | |
| } | |
| } | |