| { |
| "model": "d3-edge", |
| "bundle": "/Users/andaloes/mlx-work/coreai/bundles/d3-edge-CoreAI", |
| "reference": "upstream BF16 d3_runtime.D3 on CPU (torch)", |
| "device": "macOS 27, coreai-core 1.0.0b3, coreai-torch 0.4.3", |
| "cases": [ |
| { |
| "case": 1, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "billing", |
| "coreai_choice": "billing", |
| "selected_answer_matches": true, |
| "max_probability_difference": 3.27192645783736e-05, |
| "reference_seconds": 1.19, |
| "coreai_seconds": 3.85 |
| }, |
| { |
| "case": 2, |
| "kind": "noul", |
| "options": 2, |
| "reference_choice": "false", |
| "coreai_choice": "false", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.01056208509985801, |
| "reference_seconds": 0.92, |
| "coreai_seconds": 0.06 |
| }, |
| { |
| "case": 3, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "p3", |
| "coreai_choice": "p3", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.0027891119161109756, |
| "reference_seconds": 1.02, |
| "coreai_seconds": 0.06 |
| }, |
| { |
| "case": 4, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "privacy", |
| "coreai_choice": "privacy", |
| "selected_answer_matches": true, |
| "max_probability_difference": 2.8309543013259808e-05, |
| "reference_seconds": 0.9, |
| "coreai_seconds": 0.06 |
| }, |
| { |
| "case": 5, |
| "kind": "noul", |
| "options": 2, |
| "reference_choice": "true", |
| "coreai_choice": "true", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.0011074866374130021, |
| "reference_seconds": 2.32, |
| "coreai_seconds": 0.07 |
| }, |
| { |
| "case": 6, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "patch_now", |
| "coreai_choice": "patch_now", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.004848614349495128, |
| "reference_seconds": 1.2, |
| "coreai_seconds": 0.07 |
| } |
| ], |
| "summary": { |
| "cases": 6, |
| "all_selected_answers_match": true, |
| "max_probability_difference": 0.01056208509985801, |
| "media_cases": 0, |
| "note": "Text-only smoke cases. Media (image/video) paths and long-prompt capacity are not measured here. Small conversion smoke tests, not a quality benchmark." |
| } |
| } |
|
|
