File size: 2,166 Bytes
7a52f8a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | {
"model": "d3",
"bundle": "/Users/andaloes/mlx-work/coreai/bundles/d3-CoreAI",
"reference": "upstream BF16 d3_runtime.D3 on CPU (torch), measured in a separate process",
"device": "macOS 27, coreai-core 1.0.0b3, coreai-torch 0.4.3",
"cases": [
{
"case": 1,
"kind": "choice",
"options": 3,
"reference_choice": "billing",
"coreai_choice": "billing",
"selected_answer_matches": true,
"max_probability_difference": 1.0777304399578469e-05,
"coreai_seconds": 3.4
},
{
"case": 2,
"kind": "noul",
"options": 2,
"reference_choice": "true",
"coreai_choice": "true",
"selected_answer_matches": true,
"max_probability_difference": 0.001235121800388872,
"coreai_seconds": 2.9
},
{
"case": 3,
"kind": "choice",
"options": 3,
"reference_choice": "p1",
"coreai_choice": "p1",
"selected_answer_matches": true,
"max_probability_difference": 0.0007533040125704549,
"coreai_seconds": 2.9
},
{
"case": 4,
"kind": "choice",
"options": 3,
"reference_choice": "privacy",
"coreai_choice": "privacy",
"selected_answer_matches": true,
"max_probability_difference": 4.279321103917155e-05,
"coreai_seconds": 2.9
},
{
"case": 5,
"kind": "noul",
"options": 2,
"reference_choice": "true",
"coreai_choice": "true",
"selected_answer_matches": true,
"max_probability_difference": 0.00031628914563774924,
"coreai_seconds": 2.9
},
{
"case": 6,
"kind": "choice",
"options": 3,
"reference_choice": "patch_now",
"coreai_choice": "patch_now",
"selected_answer_matches": true,
"max_probability_difference": 6.207629011623084e-05,
"coreai_seconds": 2.9
}
],
"summary": {
"cases": 6,
"all_selected_answers_match": true,
"max_probability_difference": 0.001235121800388872,
"media_cases": 0,
"note": "Text-only smoke cases. Media paths and long-prompt capacity are not measured. Small conversion smoke tests, not a quality benchmark."
}
}
|