| { |
| "cases": [ |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.2304, |
| "fact_detail": "privacy/locality concepts present", |
| "facts_correct": true, |
| "fixture": null, |
| "generation_tokens": 26, |
| "id": "text_privacy", |
| "passed": true, |
| "peak_memory_gb": 3.231338698, |
| "prompt": "In one concise sentence, explain how running a vision-language model entirely on-device can improve privacy for clinical documents.", |
| "prompt_tokens": 30, |
| "response": "Running a vision-language model entirely on-device reduces the need for cloud storage and transmission of sensitive clinical data, thereby enhancing privacy." |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.0389, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": null, |
| "generation_tokens": 2, |
| "id": "text_fact_extraction", |
| "passed": true, |
| "peak_memory_gb": 3.26314305, |
| "prompt": "A synthetic note states: \"The follow-up appointment is scheduled for Tuesday at 10:30 AM.\" What day is the follow-up? Answer with only the day.", |
| "prompt_tokens": 44, |
| "response": "Tuesday" |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.9995, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": "synthetic_clinical_document.png", |
| "generation_tokens": 32, |
| "id": "image_clinical_document", |
| "passed": true, |
| "peak_memory_gb": 5.850670814, |
| "prompt": "This is synthetic test data. In one concise sentence, report the exact patient name, record ID, medication with dose and frequency, and allergy shown in the image.", |
| "prompt_tokens": 1161, |
| "response": "The synthetic test data includes patient Alex Rivera, record ID SYN-2048, medication Metformin 500 mg twice daily, and allergy to Penicillin." |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.7322, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": "synthetic_clinic_chart.png", |
| "generation_tokens": 5, |
| "id": "image_chart", |
| "passed": true, |
| "peak_memory_gb": 5.850670814, |
| "prompt": "Which category has the tallest bar, and what exact value is printed above it? Answer concisely.", |
| "prompt_tokens": 1053, |
| "response": "Screening, 42" |
| } |
| ], |
| "device": { |
| "architecture": "applegpu_g15d", |
| "device_name": "Apple M3 Ultra", |
| "max_buffer_length": 373662154752, |
| "max_recommended_working_set_size": 498216206336, |
| "memory_size": 549755813888, |
| "resource_limit": 499000 |
| }, |
| "load_seconds": 0.8468, |
| "mlx_vlm_revision": "dd79a5d8caf3edafd6fa9e6326d7ce4977ddcbfc", |
| "model_path": "OpenMed/North-Micro-Vision-Instruct-8bit-mlx", |
| "passed": true, |
| "runtime_versions": { |
| "huggingface-hub": "1.27.0", |
| "mlx": "0.32.0", |
| "mlx-lm": "0.31.3", |
| "mlx-metal": "0.32.0", |
| "mlx-vlm": "0.6.10", |
| "transformers": "5.15.0" |
| }, |
| "schema_version": 1, |
| "source_model": "CohereLabs/North-Micro-Vision-Instruct", |
| "source_revision": "373bda96ac70bf89f99f7048f420cf00dc07c149", |
| "variant": "8bit", |
| "weights": { |
| "actual": { |
| "sha256": "912eb8d8463c14539790711b4516beee01510963b34e1d892ecfaa4838906018", |
| "size": 3145292807 |
| }, |
| "embedding_health": { |
| "healthy": true, |
| "tensors": { |
| "language_model.model.embed_tokens.biases": { |
| "dtype": "mlx.core.bfloat16", |
| "healthy": true, |
| "nonzero_count": 8388608, |
| "shape": [ |
| 262144, |
| 32 |
| ], |
| "size": 8388608 |
| }, |
| "language_model.model.embed_tokens.scales": { |
| "dtype": "mlx.core.bfloat16", |
| "healthy": true, |
| "nonzero_count": 8388608, |
| "shape": [ |
| 262144, |
| 32 |
| ], |
| "size": 8388608 |
| }, |
| "language_model.model.embed_tokens.weight": { |
| "dtype": "mlx.core.uint32", |
| "healthy": true, |
| "nonzero_count": 134217728, |
| "shape": [ |
| 262144, |
| 512 |
| ], |
| "size": 134217728 |
| } |
| } |
| }, |
| "payload_valid": true, |
| "reference": { |
| "sha256": "82c8ba545f0c9e533f8c08cfaa8420d27ca3c460470e236c28164ef9afc025e4", |
| "size": 3145292807 |
| }, |
| "reference_hash_match": false, |
| "reference_size_match": true |
| } |
| } |
|
|
