{
  "source_revision": "3febc62259ee92ea36e2fb6e8303371c4eb3922c",
  "exported_checkpoint": {
    "expert_tensors_checked": 96,
    "expert_samples_bitwise_match_runtime_conversion": true,
    "nonexpert_tensors_bitwise_unchanged": 1418,
    "mtp_bitwise_unchanged": true,
    "text_images_streaming_passed": true,
    "context_limit": 262144,
    "served_model": "seekr-org/Inkling-2-Small-FP8-ROCm",
    "accuracy": {
      "gsm8k": {
        "passed": 48,
        "total": 50
      },
      "humaneval": {
        "passed": 20,
        "total": 20
      }
    },
    "accuracy_generation": {
      "temperature": 0,
      "reasoning_effort": 0,
      "max_tokens": 2048,
      "concurrency": 8
    },
    "vision_checks": [
      {
        "name": "arithmetic",
        "passed": true,
        "text": "42"
      },
      {
        "name": "capital",
        "passed": true,
        "text": "Paris"
      },
      {
        "name": "image_red",
        "passed": true,
        "text": "Red"
      },
      {
        "name": "image_blue",
        "passed": true,
        "text": "Blue"
      },
      {
        "name": "image_ocr",
        "passed": true,
        "text": "5739"
      },
      {
        "name": "two_images",
        "passed": true,
        "text": "Red, blue"
      }
    ]
  },
  "prior_quantization_comparison": {
    "model": "thinkingmachines/Inkling-2-Small-MXFP8",
    "revision": "3febc62259ee92ea36e2fb6e8303371c4eb3922c",
    "hardware": "8 AMD MI300X gfx942, 192 GiB each",
    "max_model_len": 262144,
    "speculative_config": {
      "method": "dspark",
      "num_speculative_tokens": 7,
      "draft_sample_method": "probabilistic"
    },
    "workload": {
      "requests_per_concurrency": 24,
      "input_tokens": 2016,
      "output_tokens": 512,
      "domains": [
        "math",
        "programming",
        "prose"
      ],
      "temperature": 1,
      "ignore_eos": true,
      "prefix_caching": false,
      "reasoning": "model default; completion counts include reasoning"
    },
    "variants": {
      "mxfp8-final": {
        "c1": {
          "completed": 24,
          "failed": 0,
          "duration": 87.02415829710662,
          "output_throughput": 141.2021700692341,
          "total_output_tokens": 12288,
          "mean_ttft_ms": 353.02664975946146,
          "mean_tpot_ms": 6.4045614900100265,
          "spec_decode_acceptance_length": 3.054865938430983
        },
        "c1_by_domain": {
          "math": {
            "output_tok_s": 192.68324620967573,
            "min_request_tok_s": 162.16978953567997,
            "max_request_tok_s": 254.20072339484943
          },
          "programming": {
            "output_tok_s": 138.22244793740694,
            "min_request_tok_s": 120.47456621151565,
            "max_request_tok_s": 154.9191445331261
          },
          "prose": {
            "output_tok_s": 113.3774818838921,
            "min_request_tok_s": 101.28198891060704,
            "max_request_tok_s": 124.1347560614812
          }
        },
        "c8": {
          "completed": 24,
          "failed": 0,
          "duration": 35.22780516371131,
          "output_throughput": 348.81537305247883,
          "total_output_tokens": 12288,
          "mean_ttft_ms": 1618.4657968891163,
          "mean_tpot_ms": 17.592307901440037,
          "spec_decode_acceptance_length": 3.04
        },
        "accuracy_short_budget": {
          "gsm8k": {
            "passed": 47,
            "total": 50
          },
          "humaneval": {
            "passed": 19,
            "total": 20
          }
        },
        "accuracy_2048_budget": {
          "gsm8k": {
            "passed": 48,
            "total": 50
          },
          "humaneval": {
            "passed": 20,
            "total": 20
          }
        },
        "prose": {
          "checks": {
            "word_count_180_to_220": true,
            "Mara": true,
            "lighthouse": true,
            "map": true,
            "dialogue": true,
            "exact_ending": true
          },
          "word_count": 196
        },
        "vision_checks": true,
        "long_context": {
          "http_status": 200,
          "seconds": 51.59381696023047,
          "usage": {
            "prompt_tokens": 261040,
            "total_tokens": 261046,
            "completion_tokens": 6,
            "completion_tokens_details": {
              "reasoning_tokens": 0
            }
          },
          "content": "739124"
        }
      },
      "row-fp8-final": {
        "c1": {
          "completed": 24,
          "failed": 0,
          "duration": 49.65525996312499,
          "output_throughput": 247.46623034750638,
          "total_output_tokens": 12288,
          "mean_ttft_ms": 128.29210500543317,
          "mean_tpot_ms": 3.797367814167683,
          "spec_decode_acceptance_length": 3.117617237008872
        },
        "c1_by_domain": {
          "math": {
            "output_tok_s": 326.52381715114535,
            "min_request_tok_s": 265.20409125347084,
            "max_request_tok_s": 397.12908010147316
          },
          "programming": {
            "output_tok_s": 248.068111866624,
            "min_request_tok_s": 207.0873267282028,
            "max_request_tok_s": 276.0874995644794
          },
          "prose": {
            "output_tok_s": 198.8921654365731,
            "min_request_tok_s": 173.67162954038446,
            "max_request_tok_s": 234.15402964596927
          }
        },
        "c8": {
          "completed": 24,
          "failed": 0,
          "duration": 15.359448773786426,
          "output_throughput": 800.0287107289691,
          "total_output_tokens": 12288,
          "mean_ttft_ms": 375.8305724089344,
          "mean_tpot_ms": 8.250392456882222,
          "spec_decode_acceptance_length": 3.0183733463988243
        },
        "accuracy_short_budget": {
          "gsm8k": {
            "passed": 48,
            "total": 50
          },
          "humaneval": {
            "passed": 19,
            "total": 20
          }
        },
        "accuracy_2048_budget": {
          "gsm8k": {
            "passed": 48,
            "total": 50
          },
          "humaneval": {
            "passed": 19,
            "total": 20
          }
        },
        "prose": {
          "checks": {
            "word_count_180_to_220": true,
            "Mara": true,
            "lighthouse": true,
            "map": true,
            "dialogue": true,
            "exact_ending": true
          },
          "word_count": 192
        },
        "vision_checks": true,
        "long_context": {
          "http_status": 200,
          "seconds": 27.074019331485033,
          "usage": {
            "prompt_tokens": 261040,
            "total_tokens": 261046,
            "completion_tokens": 6,
            "completion_tokens_details": {
              "reasoning_tokens": 0
            }
          },
          "content": "739124"
        }
      }
    },
    "accuracy_notes": [
      "Fixed Random(42) subset: 50 GSM8K and 20 HumanEval; temperature=0, reasoning_effort=0, concurrency=8.",
      "The short paired run uses 512 math /768 code max_tokens. Extended paired run uses 2048 for both. Both sets retained.",
      "GSM8K sample0 has an incorrect official reference: 430+750+300+700=2180, while reference says2280. Raw scores retain the original label.",
      "Prose checks measure instruction following only; no automated literary-quality score."
    ]
  },
  "notes": [
    "The historical paired score for row FP8 is retained separately. Greedy output varies with batching; the fresh exported-checkpoint check does not replace that paired measurement.",
    "GSM8K scores retain a known incorrect official reference label."
  ],
  "verification_seconds": 6.884046273306012
}