| { |
| "cases": [ |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.2482, |
| "fact_detail": "privacy/locality concepts present", |
| "facts_correct": true, |
| "fixture": null, |
| "generation_tokens": 32, |
| "id": "text_privacy", |
| "passed": true, |
| "peak_memory_gb": 2.258253002, |
| "prompt": "In one concise sentence, explain how running a vision-language model entirely on-device can improve privacy for clinical documents.", |
| "prompt_tokens": 30, |
| "response": "Running a vision-language model entirely on-Device ensures that sensitive clinical data remains private and secure, as it eliminates the need for cloud-based storage and transmission." |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.0382, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": null, |
| "generation_tokens": 2, |
| "id": "text_fact_extraction", |
| "passed": true, |
| "peak_memory_gb": 2.290057354, |
| "prompt": "A synthetic note states: \"The follow-up appointment is scheduled for Tuesday at 10:30 AM.\" What day is the follow-up? Answer with only the day.", |
| "prompt_tokens": 44, |
| "response": "Tuesday" |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 1.0533, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": "synthetic_clinical_document.png", |
| "generation_tokens": 43, |
| "id": "image_clinical_document", |
| "passed": true, |
| "peak_memory_gb": 4.877585118, |
| "prompt": "This is synthetic test data. In one concise sentence, report the exact patient name, record ID, medication with dose and frequency, and allergy shown in the image.", |
| "prompt_tokens": 1161, |
| "response": "The image shows synthetic test data for a patient named Alex Rivera, record ID SYN-2048, prescribed Metformin 500 mg twice daily with a penicillin allergy, and a follow-up of 14 days." |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.7262, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": "synthetic_clinic_chart.png", |
| "generation_tokens": 5, |
| "id": "image_chart", |
| "passed": true, |
| "peak_memory_gb": 4.877585118, |
| "prompt": "Which category has the tallest bar, and what exact value is printed above it? Answer concisely.", |
| "prompt_tokens": 1053, |
| "response": "Screening, 42" |
| } |
| ], |
| "device": { |
| "architecture": "applegpu_g15d", |
| "device_name": "Apple M3 Ultra", |
| "max_buffer_length": 373662154752, |
| "max_recommended_working_set_size": 498216206336, |
| "memory_size": 549755813888, |
| "resource_limit": 499000 |
| }, |
| "load_seconds": 0.8291, |
| "mlx_vlm_revision": "dd79a5d8caf3edafd6fa9e6326d7ce4977ddcbfc", |
| "model_path": "OpenMed/North-Micro-Vision-Instruct-4bit-mlx", |
| "passed": true, |
| "runtime_versions": { |
| "huggingface-hub": "1.27.0", |
| "mlx": "0.32.0", |
| "mlx-lm": "0.31.3", |
| "mlx-metal": "0.32.0", |
| "mlx-vlm": "0.6.10", |
| "transformers": "5.15.0" |
| }, |
| "schema_version": 1, |
| "source_model": "CohereLabs/North-Micro-Vision-Instruct", |
| "source_revision": "373bda96ac70bf89f99f7048f420cf00dc07c149", |
| "variant": "4bit", |
| "weights": { |
| "actual": { |
| "sha256": "ce3248aa6f570f742cd8d8f13bf08d90f05a26796c98459627aa015586351fd2", |
| "size": 2172214057 |
| }, |
| "embedding_health": { |
| "healthy": true, |
| "tensors": { |
| "language_model.model.embed_tokens.biases": { |
| "dtype": "mlx.core.bfloat16", |
| "healthy": true, |
| "nonzero_count": 8388608, |
| "shape": [ |
| 262144, |
| 32 |
| ], |
| "size": 8388608 |
| }, |
| "language_model.model.embed_tokens.scales": { |
| "dtype": "mlx.core.bfloat16", |
| "healthy": true, |
| "nonzero_count": 8388608, |
| "shape": [ |
| 262144, |
| 32 |
| ], |
| "size": 8388608 |
| }, |
| "language_model.model.embed_tokens.weight": { |
| "dtype": "mlx.core.uint32", |
| "healthy": true, |
| "nonzero_count": 67108864, |
| "shape": [ |
| 262144, |
| 256 |
| ], |
| "size": 67108864 |
| } |
| } |
| }, |
| "payload_valid": true, |
| "reference": { |
| "sha256": "96d91be7705ac4adcbec731d981f388db9cacd36076bc1fb89e32ea552c303e7", |
| "size": 2172214057 |
| }, |
| "reference_hash_match": false, |
| "reference_size_match": true |
| } |
| } |
|
|
