| { |
| "source_model": "google/embeddinggemma-2", |
| "source_revision": "914f7f89142e33e77833254d9c9b90c3cef7303b", |
| "mlx_vlm_revision": "3d87e88402f307efbf68e568971aa887ee7d9ed0", |
| "variant": "bf16", |
| "hardware": "Apple M3 Ultra", |
| "mlx_version": "0.32.3", |
| "transformers_version": "5.18.0.dev0", |
| "validation_scope": "Six multilingual text inputs and synthetic image, audio, two-frame video, and text+image; numerical smoke checks, not a retrieval benchmark.", |
| "tensor_count": 1376, |
| "tensor_dtypes": [ |
| "BF16" |
| ], |
| "weight_bytes": 1488914930, |
| "cases": { |
| "audio": { |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "finite": true, |
| "minimum_cosine_vs_torch_fp32": 0.9999469530459295, |
| "maximum_absolute_error_vs_torch_fp32": 0.0012578400660175314, |
| "minimum_cosine_vs_source_bf16": 1.0000000000000002, |
| "minimum_truncated_cosine_vs_torch_fp32": 0.999949991961834, |
| "bit_identical_to_source_bf16": true |
| }, |
| "image": { |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "finite": true, |
| "minimum_cosine_vs_torch_fp32": 0.9999538918775841, |
| "maximum_absolute_error_vs_torch_fp32": 0.001134838495517842, |
| "minimum_cosine_vs_source_bf16": 1.0000000000000002, |
| "minimum_truncated_cosine_vs_torch_fp32": 0.999955921952359, |
| "bit_identical_to_source_bf16": true |
| }, |
| "text": { |
| "shape": [ |
| 6, |
| 768 |
| ], |
| "finite": true, |
| "minimum_cosine_vs_torch_fp32": 0.9999371785819806, |
| "maximum_absolute_error_vs_torch_fp32": 0.0014684630640885471, |
| "minimum_cosine_vs_source_bf16": 1.0, |
| "minimum_truncated_cosine_vs_torch_fp32": 0.9999395054380499, |
| "bit_identical_to_source_bf16": true, |
| "retrieval_scores_venus_mars": [ |
| 0.6774646639823914, |
| 0.8348115086555481 |
| ] |
| }, |
| "text_image": { |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "finite": true, |
| "minimum_cosine_vs_torch_fp32": 0.9999051411129661, |
| "maximum_absolute_error_vs_torch_fp32": 0.0018641552341061365, |
| "minimum_cosine_vs_source_bf16": 1.0000000000000004, |
| "minimum_truncated_cosine_vs_torch_fp32": 0.9999094392794228, |
| "bit_identical_to_source_bf16": true |
| }, |
| "video": { |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "finite": true, |
| "minimum_cosine_vs_torch_fp32": 0.9999002035102345, |
| "maximum_absolute_error_vs_torch_fp32": 0.0016884156983105539, |
| "minimum_cosine_vs_source_bf16": 1.0, |
| "minimum_truncated_cosine_vs_torch_fp32": 0.9999016834128542, |
| "bit_identical_to_source_bf16": true |
| } |
| } |
| } |
|
|
