| { |
| "all_passed": true, |
| "failed": [], |
| "baseline": { |
| "core": { |
| "n": 8, |
| "exact": 8, |
| "lexical": 8, |
| "truncated": 0, |
| "cer": 0.0 |
| }, |
| "basic48": { |
| "n": 48, |
| "exact": 42, |
| "lexical": 46, |
| "truncated": 0, |
| "cer": 0.004524886877828055 |
| }, |
| "format20": { |
| "n": 20, |
| "exact": 19, |
| "lexical": 20, |
| "truncated": 0, |
| "cer": 0.0 |
| }, |
| "names60": { |
| "n": 60, |
| "exact": 60, |
| "lexical": 60, |
| "truncated": 0, |
| "cer": 0.0 |
| }, |
| "public400": { |
| "n": 400, |
| "exact": 120, |
| "lexical": 234, |
| "truncated": 0, |
| "cer": 0.09236683141131247 |
| }, |
| "intent": { |
| "n": 326, |
| "exact": 318, |
| "lexical": 318, |
| "truncated": 0, |
| "cer": 0.0026373626373626374 |
| }, |
| "nonce": { |
| "n": 200, |
| "exact": 196, |
| "lexical": 196, |
| "truncated": 0, |
| "cer": 0.0030759766225776685 |
| }, |
| "math236": { |
| "n": 236, |
| "exact": 235, |
| "lexical": 235, |
| "truncated": 0, |
| "cer": 0.0013136288998357964 |
| }, |
| "dev432": { |
| "n": 432, |
| "exact": 414, |
| "lexical": 414, |
| "truncated": 0, |
| "cer": 0.011299435028248588 |
| }, |
| "heldout432": { |
| "n": 432, |
| "exact": 430, |
| "lexical": 430, |
| "truncated": 0, |
| "cer": 0.0009416195856873823 |
| }, |
| "critical5": { |
| "n": 5, |
| "exact": 5, |
| "lexical": 5, |
| "truncated": 0, |
| "cer": 0.0 |
| } |
| }, |
| "candidate": { |
| "core": { |
| "n": 8, |
| "exact": 8, |
| "lexical": 8, |
| "truncated": 0, |
| "cer": 0.0 |
| }, |
| "basic48": { |
| "n": 48, |
| "exact": 42, |
| "lexical": 46, |
| "truncated": 0, |
| "cer": 0.004524886877828055 |
| }, |
| "format20": { |
| "n": 20, |
| "exact": 19, |
| "lexical": 20, |
| "truncated": 0, |
| "cer": 0.0 |
| }, |
| "names60": { |
| "n": 60, |
| "exact": 60, |
| "lexical": 60, |
| "truncated": 0, |
| "cer": 0.0 |
| }, |
| "public400": { |
| "n": 400, |
| "exact": 120, |
| "lexical": 237, |
| "truncated": 0, |
| "cer": 0.09225700164744646 |
| }, |
| "intent": { |
| "n": 326, |
| "exact": 318, |
| "lexical": 318, |
| "truncated": 0, |
| "cer": 0.002197802197802198 |
| }, |
| "nonce": { |
| "n": 200, |
| "exact": 197, |
| "lexical": 197, |
| "truncated": 0, |
| "cer": 0.002153183635804368 |
| }, |
| "math236": { |
| "n": 236, |
| "exact": 235, |
| "lexical": 235, |
| "truncated": 0, |
| "cer": 0.0013136288998357964 |
| }, |
| "dev432": { |
| "n": 432, |
| "exact": 414, |
| "lexical": 414, |
| "truncated": 0, |
| "cer": 0.011299435028248588 |
| }, |
| "heldout432": { |
| "n": 432, |
| "exact": 431, |
| "lexical": 431, |
| "truncated": 0, |
| "cer": 0.0006277463904582549 |
| }, |
| "critical5": { |
| "n": 5, |
| "exact": 5, |
| "lexical": 5, |
| "truncated": 0, |
| "cer": 0.0 |
| } |
| }, |
| "scope": "Actual serialized compressed checkpoint. Reused development/regression tests; no natural-audio, independent blind or Orin acceptance claim.", |
| "new_numeric_value_errors_count": 0, |
| "new_operator_errors_count": 0, |
| "new_layout_errors_count": 0, |
| "new_name_errors_count": 0, |
| "latex_outputs_count": 0, |
| "effective_rope_theta": 1000000, |
| "request_chunk": 64, |
| "runtime": "vLLM0.17 Blackwell SM120, eager, max-num-seqs8, prefix cache disabled; BF16 KV. FP8 uses native CUTLASS instead of FlashInfer FP8 kernel.", |
| "single_request_critical": { |
| "effective_rope_parameters": { |
| "rope_theta": 1000000, |
| "rope_type": "default" |
| }, |
| "n": 5, |
| "strict_exact": 5, |
| "lexical_exact": 5, |
| "truncated": 0, |
| "seconds": 0.5074490859988146, |
| "engine": "vLLM 0.17.0 eager; Blackwell SM120", |
| "sequences": 1, |
| "request_chunk": 64, |
| "scope": "Real serialized checkpoint, not ModelOpt fake quant. Runtime is not the production Orin vLLM version." |
| } |
| } |
|
|
