File size: 1,818 Bytes
b9ba72c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
{
  "runtime": "Kapitsa 0.2.31",
  "runtime_commit": "84249c0a79d6d6dd930998cce69a58b41dee586b",
  "host": "M4 Max, 40-core GPU, 64 GiB",
  "context_limit": 131072,
  "native_mtp_head": "mtp.safetensors",
  "sampler": "greedy; thinking off",
  "text_prompts": 4,
  "vision_cases": "8 generated OCR, chart, spatial and two-image cases; two seeds; unique chart maxima",
  "vision_fixture_seeds": [
    7319,
    173
  ],
  "normalized_grading": "Remove one enclosing Markdown JSON fence, parse JSON, require exact object/array equality. No flattening or visual-answer rewriting.",
  "decode_measurement": "Mean server-reported decode for one code and one prose output, 256-token cap; one run per mode.",
  "results": {
    "ar": {
      "text_parity_passed": 0,
      "text_total": 4,
      "vision_bare_json_passed": 4,
      "vision_normalized_json_passed": 8,
      "vision_total": 8,
      "code_prose_mean_decode_tps": 20.2
    },
    "mtp1": {
      "text_parity_passed": 4,
      "text_total": 4,
      "vision_bare_json_passed": 0,
      "vision_normalized_json_passed": 0,
      "vision_total": 0,
      "code_prose_mean_decode_tps": 29.9
    },
    "mtp2": {
      "text_parity_passed": 4,
      "text_total": 4,
      "vision_bare_json_passed": 4,
      "vision_normalized_json_passed": 8,
      "vision_total": 8,
      "code_prose_mean_decode_tps": 39.6
    },
    "mtp3": {
      "text_parity_passed": 4,
      "text_total": 4,
      "vision_bare_json_passed": 0,
      "vision_normalized_json_passed": 0,
      "vision_total": 0,
      "code_prose_mean_decode_tps": 45.5
    },
    "dflash2": {
      "text_parity_passed": 4,
      "text_total": 4,
      "vision_bare_json_passed": 4,
      "vision_normalized_json_passed": 8,
      "vision_total": 8,
      "code_prose_mean_decode_tps": 90.3
    }
  }
}