{
  "all_passed": true,
  "failed": [],
  "baseline": {
    "core": {
      "n": 8,
      "exact": 8,
      "lexical": 8,
      "truncated": 0,
      "cer": 0.0
    },
    "basic48": {
      "n": 48,
      "exact": 42,
      "lexical": 46,
      "truncated": 0,
      "cer": 0.004524886877828055
    },
    "format20": {
      "n": 20,
      "exact": 19,
      "lexical": 20,
      "truncated": 0,
      "cer": 0.0
    },
    "names60": {
      "n": 60,
      "exact": 60,
      "lexical": 60,
      "truncated": 0,
      "cer": 0.0
    },
    "public400": {
      "n": 400,
      "exact": 120,
      "lexical": 234,
      "truncated": 0,
      "cer": 0.09236683141131247
    },
    "intent": {
      "n": 326,
      "exact": 318,
      "lexical": 318,
      "truncated": 0,
      "cer": 0.0026373626373626374
    },
    "nonce": {
      "n": 200,
      "exact": 196,
      "lexical": 196,
      "truncated": 0,
      "cer": 0.0030759766225776685
    },
    "math236": {
      "n": 236,
      "exact": 235,
      "lexical": 235,
      "truncated": 0,
      "cer": 0.0013136288998357964
    },
    "dev432": {
      "n": 432,
      "exact": 414,
      "lexical": 414,
      "truncated": 0,
      "cer": 0.011299435028248588
    },
    "heldout432": {
      "n": 432,
      "exact": 430,
      "lexical": 430,
      "truncated": 0,
      "cer": 0.0009416195856873823
    },
    "critical5": {
      "n": 5,
      "exact": 5,
      "lexical": 5,
      "truncated": 0,
      "cer": 0.0
    }
  },
  "candidate": {
    "core": {
      "n": 8,
      "exact": 8,
      "lexical": 8,
      "truncated": 0,
      "cer": 0.0
    },
    "basic48": {
      "n": 48,
      "exact": 42,
      "lexical": 46,
      "truncated": 0,
      "cer": 0.004524886877828055
    },
    "format20": {
      "n": 20,
      "exact": 19,
      "lexical": 20,
      "truncated": 0,
      "cer": 0.0
    },
    "names60": {
      "n": 60,
      "exact": 60,
      "lexical": 60,
      "truncated": 0,
      "cer": 0.0
    },
    "public400": {
      "n": 400,
      "exact": 120,
      "lexical": 237,
      "truncated": 0,
      "cer": 0.09225700164744646
    },
    "intent": {
      "n": 326,
      "exact": 318,
      "lexical": 318,
      "truncated": 0,
      "cer": 0.002197802197802198
    },
    "nonce": {
      "n": 200,
      "exact": 197,
      "lexical": 197,
      "truncated": 0,
      "cer": 0.002153183635804368
    },
    "math236": {
      "n": 236,
      "exact": 235,
      "lexical": 235,
      "truncated": 0,
      "cer": 0.0013136288998357964
    },
    "dev432": {
      "n": 432,
      "exact": 414,
      "lexical": 414,
      "truncated": 0,
      "cer": 0.011299435028248588
    },
    "heldout432": {
      "n": 432,
      "exact": 431,
      "lexical": 431,
      "truncated": 0,
      "cer": 0.0006277463904582549
    },
    "critical5": {
      "n": 5,
      "exact": 5,
      "lexical": 5,
      "truncated": 0,
      "cer": 0.0
    }
  },
  "scope": "Actual serialized compressed checkpoint. Reused development/regression tests; no natural-audio, independent blind or Orin acceptance claim.",
  "new_numeric_value_errors_count": 0,
  "new_operator_errors_count": 0,
  "new_layout_errors_count": 0,
  "new_name_errors_count": 0,
  "latex_outputs_count": 0,
  "effective_rope_theta": 1000000,
  "request_chunk": 64,
  "runtime": "vLLM0.17 Blackwell SM120, eager, max-num-seqs8, prefix cache disabled; BF16 KV. FP8 uses native CUTLASS instead of FlashInfer FP8 kernel.",
  "single_request_critical": {
    "effective_rope_parameters": {
      "rope_theta": 1000000,
      "rope_type": "default"
    },
    "n": 5,
    "strict_exact": 5,
    "lexical_exact": 5,
    "truncated": 0,
    "seconds": 0.5074490859988146,
    "engine": "vLLM 0.17.0 eager; Blackwell SM120",
    "sequences": 1,
    "request_chunk": 64,
    "scope": "Real serialized checkpoint, not ModelOpt fake quant. Runtime is not the production Orin vLLM version."
  }
}