{
  "cases": [
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.2482,
      "fact_detail": "privacy/locality concepts present",
      "facts_correct": true,
      "fixture": null,
      "generation_tokens": 32,
      "id": "text_privacy",
      "passed": true,
      "peak_memory_gb": 2.258253002,
      "prompt": "In one concise sentence, explain how running a vision-language model entirely on-device can improve privacy for clinical documents.",
      "prompt_tokens": 30,
      "response": "Running a vision-language model entirely on-Device ensures that sensitive clinical data remains private and secure, as it eliminates the need for cloud-based storage and transmission."
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.0382,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": null,
      "generation_tokens": 2,
      "id": "text_fact_extraction",
      "passed": true,
      "peak_memory_gb": 2.290057354,
      "prompt": "A synthetic note states: \"The follow-up appointment is scheduled for Tuesday at 10:30 AM.\" What day is the follow-up? Answer with only the day.",
      "prompt_tokens": 44,
      "response": "Tuesday"
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 1.0533,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": "synthetic_clinical_document.png",
      "generation_tokens": 43,
      "id": "image_clinical_document",
      "passed": true,
      "peak_memory_gb": 4.877585118,
      "prompt": "This is synthetic test data. In one concise sentence, report the exact patient name, record ID, medication with dose and frequency, and allergy shown in the image.",
      "prompt_tokens": 1161,
      "response": "The image shows synthetic test data for a patient named Alex Rivera, record ID SYN-2048, prescribed Metformin 500 mg twice daily with a penicillin allergy, and a follow-up of 14 days."
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.7262,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": "synthetic_clinic_chart.png",
      "generation_tokens": 5,
      "id": "image_chart",
      "passed": true,
      "peak_memory_gb": 4.877585118,
      "prompt": "Which category has the tallest bar, and what exact value is printed above it? Answer concisely.",
      "prompt_tokens": 1053,
      "response": "Screening, 42"
    }
  ],
  "device": {
    "architecture": "applegpu_g15d",
    "device_name": "Apple M3 Ultra",
    "max_buffer_length": 373662154752,
    "max_recommended_working_set_size": 498216206336,
    "memory_size": 549755813888,
    "resource_limit": 499000
  },
  "load_seconds": 0.8291,
  "mlx_vlm_revision": "dd79a5d8caf3edafd6fa9e6326d7ce4977ddcbfc",
  "model_path": "OpenMed/North-Micro-Vision-Instruct-4bit-mlx",
  "passed": true,
  "runtime_versions": {
    "huggingface-hub": "1.27.0",
    "mlx": "0.32.0",
    "mlx-lm": "0.31.3",
    "mlx-metal": "0.32.0",
    "mlx-vlm": "0.6.10",
    "transformers": "5.15.0"
  },
  "schema_version": 1,
  "source_model": "CohereLabs/North-Micro-Vision-Instruct",
  "source_revision": "373bda96ac70bf89f99f7048f420cf00dc07c149",
  "variant": "4bit",
  "weights": {
    "actual": {
      "sha256": "ce3248aa6f570f742cd8d8f13bf08d90f05a26796c98459627aa015586351fd2",
      "size": 2172214057
    },
    "embedding_health": {
      "healthy": true,
      "tensors": {
        "language_model.model.embed_tokens.biases": {
          "dtype": "mlx.core.bfloat16",
          "healthy": true,
          "nonzero_count": 8388608,
          "shape": [
            262144,
            32
          ],
          "size": 8388608
        },
        "language_model.model.embed_tokens.scales": {
          "dtype": "mlx.core.bfloat16",
          "healthy": true,
          "nonzero_count": 8388608,
          "shape": [
            262144,
            32
          ],
          "size": 8388608
        },
        "language_model.model.embed_tokens.weight": {
          "dtype": "mlx.core.uint32",
          "healthy": true,
          "nonzero_count": 67108864,
          "shape": [
            262144,
            256
          ],
          "size": 67108864
        }
      }
    },
    "payload_valid": true,
    "reference": {
      "sha256": "96d91be7705ac4adcbec731d981f388db9cacd36076bc1fb89e32ea552c303e7",
      "size": 2172214057
    },
    "reference_hash_match": false,
    "reference_size_match": true
  }
}