| { |
| "source_revision": "3febc62259ee92ea36e2fb6e8303371c4eb3922c", |
| "exported_checkpoint": { |
| "expert_tensors_checked": 96, |
| "expert_samples_bitwise_match_runtime_conversion": true, |
| "nonexpert_tensors_bitwise_unchanged": 1418, |
| "mtp_bitwise_unchanged": true, |
| "text_images_streaming_passed": true, |
| "context_limit": 262144, |
| "served_model": "seekr-org/Inkling-2-Small-FP8-ROCm", |
| "accuracy": { |
| "gsm8k": { |
| "passed": 48, |
| "total": 50 |
| }, |
| "humaneval": { |
| "passed": 20, |
| "total": 20 |
| } |
| }, |
| "accuracy_generation": { |
| "temperature": 0, |
| "reasoning_effort": 0, |
| "max_tokens": 2048, |
| "concurrency": 8 |
| }, |
| "vision_checks": [ |
| { |
| "name": "arithmetic", |
| "passed": true, |
| "text": "42" |
| }, |
| { |
| "name": "capital", |
| "passed": true, |
| "text": "Paris" |
| }, |
| { |
| "name": "image_red", |
| "passed": true, |
| "text": "Red" |
| }, |
| { |
| "name": "image_blue", |
| "passed": true, |
| "text": "Blue" |
| }, |
| { |
| "name": "image_ocr", |
| "passed": true, |
| "text": "5739" |
| }, |
| { |
| "name": "two_images", |
| "passed": true, |
| "text": "Red, blue" |
| } |
| ] |
| }, |
| "prior_quantization_comparison": { |
| "model": "thinkingmachines/Inkling-2-Small-MXFP8", |
| "revision": "3febc62259ee92ea36e2fb6e8303371c4eb3922c", |
| "hardware": "8 AMD MI300X gfx942, 192 GiB each", |
| "max_model_len": 262144, |
| "speculative_config": { |
| "method": "dspark", |
| "num_speculative_tokens": 7, |
| "draft_sample_method": "probabilistic" |
| }, |
| "workload": { |
| "requests_per_concurrency": 24, |
| "input_tokens": 2016, |
| "output_tokens": 512, |
| "domains": [ |
| "math", |
| "programming", |
| "prose" |
| ], |
| "temperature": 1, |
| "ignore_eos": true, |
| "prefix_caching": false, |
| "reasoning": "model default; completion counts include reasoning" |
| }, |
| "variants": { |
| "mxfp8-final": { |
| "c1": { |
| "completed": 24, |
| "failed": 0, |
| "duration": 87.02415829710662, |
| "output_throughput": 141.2021700692341, |
| "total_output_tokens": 12288, |
| "mean_ttft_ms": 353.02664975946146, |
| "mean_tpot_ms": 6.4045614900100265, |
| "spec_decode_acceptance_length": 3.054865938430983 |
| }, |
| "c1_by_domain": { |
| "math": { |
| "output_tok_s": 192.68324620967573, |
| "min_request_tok_s": 162.16978953567997, |
| "max_request_tok_s": 254.20072339484943 |
| }, |
| "programming": { |
| "output_tok_s": 138.22244793740694, |
| "min_request_tok_s": 120.47456621151565, |
| "max_request_tok_s": 154.9191445331261 |
| }, |
| "prose": { |
| "output_tok_s": 113.3774818838921, |
| "min_request_tok_s": 101.28198891060704, |
| "max_request_tok_s": 124.1347560614812 |
| } |
| }, |
| "c8": { |
| "completed": 24, |
| "failed": 0, |
| "duration": 35.22780516371131, |
| "output_throughput": 348.81537305247883, |
| "total_output_tokens": 12288, |
| "mean_ttft_ms": 1618.4657968891163, |
| "mean_tpot_ms": 17.592307901440037, |
| "spec_decode_acceptance_length": 3.04 |
| }, |
| "accuracy_short_budget": { |
| "gsm8k": { |
| "passed": 47, |
| "total": 50 |
| }, |
| "humaneval": { |
| "passed": 19, |
| "total": 20 |
| } |
| }, |
| "accuracy_2048_budget": { |
| "gsm8k": { |
| "passed": 48, |
| "total": 50 |
| }, |
| "humaneval": { |
| "passed": 20, |
| "total": 20 |
| } |
| }, |
| "prose": { |
| "checks": { |
| "word_count_180_to_220": true, |
| "Mara": true, |
| "lighthouse": true, |
| "map": true, |
| "dialogue": true, |
| "exact_ending": true |
| }, |
| "word_count": 196 |
| }, |
| "vision_checks": true, |
| "long_context": { |
| "http_status": 200, |
| "seconds": 51.59381696023047, |
| "usage": { |
| "prompt_tokens": 261040, |
| "total_tokens": 261046, |
| "completion_tokens": 6, |
| "completion_tokens_details": { |
| "reasoning_tokens": 0 |
| } |
| }, |
| "content": "739124" |
| } |
| }, |
| "row-fp8-final": { |
| "c1": { |
| "completed": 24, |
| "failed": 0, |
| "duration": 49.65525996312499, |
| "output_throughput": 247.46623034750638, |
| "total_output_tokens": 12288, |
| "mean_ttft_ms": 128.29210500543317, |
| "mean_tpot_ms": 3.797367814167683, |
| "spec_decode_acceptance_length": 3.117617237008872 |
| }, |
| "c1_by_domain": { |
| "math": { |
| "output_tok_s": 326.52381715114535, |
| "min_request_tok_s": 265.20409125347084, |
| "max_request_tok_s": 397.12908010147316 |
| }, |
| "programming": { |
| "output_tok_s": 248.068111866624, |
| "min_request_tok_s": 207.0873267282028, |
| "max_request_tok_s": 276.0874995644794 |
| }, |
| "prose": { |
| "output_tok_s": 198.8921654365731, |
| "min_request_tok_s": 173.67162954038446, |
| "max_request_tok_s": 234.15402964596927 |
| } |
| }, |
| "c8": { |
| "completed": 24, |
| "failed": 0, |
| "duration": 15.359448773786426, |
| "output_throughput": 800.0287107289691, |
| "total_output_tokens": 12288, |
| "mean_ttft_ms": 375.8305724089344, |
| "mean_tpot_ms": 8.250392456882222, |
| "spec_decode_acceptance_length": 3.0183733463988243 |
| }, |
| "accuracy_short_budget": { |
| "gsm8k": { |
| "passed": 48, |
| "total": 50 |
| }, |
| "humaneval": { |
| "passed": 19, |
| "total": 20 |
| } |
| }, |
| "accuracy_2048_budget": { |
| "gsm8k": { |
| "passed": 48, |
| "total": 50 |
| }, |
| "humaneval": { |
| "passed": 19, |
| "total": 20 |
| } |
| }, |
| "prose": { |
| "checks": { |
| "word_count_180_to_220": true, |
| "Mara": true, |
| "lighthouse": true, |
| "map": true, |
| "dialogue": true, |
| "exact_ending": true |
| }, |
| "word_count": 192 |
| }, |
| "vision_checks": true, |
| "long_context": { |
| "http_status": 200, |
| "seconds": 27.074019331485033, |
| "usage": { |
| "prompt_tokens": 261040, |
| "total_tokens": 261046, |
| "completion_tokens": 6, |
| "completion_tokens_details": { |
| "reasoning_tokens": 0 |
| } |
| }, |
| "content": "739124" |
| } |
| } |
| }, |
| "accuracy_notes": [ |
| "Fixed Random(42) subset: 50 GSM8K and 20 HumanEval; temperature=0, reasoning_effort=0, concurrency=8.", |
| "The short paired run uses 512 math /768 code max_tokens. Extended paired run uses 2048 for both. Both sets retained.", |
| "GSM8K sample0 has an incorrect official reference: 430+750+300+700=2180, while reference says2280. Raw scores retain the original label.", |
| "Prose checks measure instruction following only; no automated literary-quality score." |
| ] |
| }, |
| "notes": [ |
| "The historical paired score for row FP8 is retained separately. Greedy output varies with batching; the fresh exported-checkpoint check does not replace that paired measurement.", |
| "GSM8K scores retain a known incorrect official reference label." |
| ], |
| "verification_seconds": 6.884046273306012 |
| } |
|
|
