| { |
| "model": "d3-lite", |
| "bundle": "/Users/andaloes/mlx-work/coreai/bundles/d3-lite-CoreAI", |
| "reference": "upstream BF16 d3_runtime.D3 on CPU (torch)", |
| "device": "macOS 27, coreai-core 1.0.0b3, coreai-torch 0.4.3", |
| "cases": [ |
| { |
| "case": 1, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "billing", |
| "coreai_choice": "billing", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.00019077396836220117, |
| "reference_seconds": 2.43, |
| "coreai_seconds": 0.18 |
| }, |
| { |
| "case": 2, |
| "kind": "noul", |
| "options": 2, |
| "reference_choice": "true", |
| "coreai_choice": "true", |
| "selected_answer_matches": true, |
| "max_probability_difference": 8.098354932573049e-05, |
| "reference_seconds": 2.44, |
| "coreai_seconds": 0.11 |
| }, |
| { |
| "case": 3, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "p1", |
| "coreai_choice": "p1", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.010075154316128265, |
| "reference_seconds": 2.53, |
| "coreai_seconds": 0.13 |
| }, |
| { |
| "case": 4, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "privacy", |
| "coreai_choice": "privacy", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.00021470569841079354, |
| "reference_seconds": 2.41, |
| "coreai_seconds": 0.13 |
| }, |
| { |
| "case": 5, |
| "kind": "noul", |
| "options": 2, |
| "reference_choice": "true", |
| "coreai_choice": "true", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.0019319451525090514, |
| "reference_seconds": 2.36, |
| "coreai_seconds": 0.13 |
| }, |
| { |
| "case": 6, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "patch_now", |
| "coreai_choice": "patch_now", |
| "selected_answer_matches": true, |
| "max_probability_difference": 8.864512209827424e-05, |
| "reference_seconds": 2.45, |
| "coreai_seconds": 0.13 |
| } |
| ], |
| "summary": { |
| "cases": 6, |
| "all_selected_answers_match": true, |
| "max_probability_difference": 0.010075154316128265, |
| "media_cases": 0, |
| "note": "Text-only smoke cases. Media (image/video) paths and long-prompt capacity are not measured here. Small conversion smoke tests, not a quality benchmark." |
| } |
| } |
|
|
