{
  "model": "gemma4-e4b-efe-t-MLX-4bit",
  "results": [
    {
      "prompt": "What is 17 + 25? Give only the number.",
      "thinking": false,
      "output": "42",
      "finish_reason": "stop",
      "generated_tokens": 3,
      "tokens_per_second": 25.32718694288921,
      "peak_memory_gb": 6.603434614,
      "passed": true
    },
    {
      "prompt": "What is 9 multiplied by 8? Give only the number.",
      "thinking": false,
      "output": "72",
      "finish_reason": "stop",
      "generated_tokens": 3,
      "tokens_per_second": 25.399746072864932,
      "peak_memory_gb": 6.603434614,
      "passed": true
    },
    {
      "prompt": "What is 17 + 25? Give only the number.",
      "thinking": true,
      "output": "42",
      "finish_reason": "stop",
      "generated_tokens": 3,
      "tokens_per_second": 24.52999507269902,
      "peak_memory_gb": 6.624006072,
      "passed": true
    },
    {
      "prompt": "What is 9 multiplied by 8? Give only the number.",
      "thinking": true,
      "output": "72",
      "finish_reason": "stop",
      "generated_tokens": 3,
      "tokens_per_second": 25.771479991715882,
      "peak_memory_gb": 6.624006072,
      "passed": true
    },
    {
      "prompt": "Translate to Turkish: The meeting is tomorrow. Reply only with the translation.",
      "thinking": false,
      "output": "Toplantı yarın.",
      "finish_reason": "stop",
      "generated_tokens": 7,
      "tokens_per_second": 20.10139234177456,
      "peak_memory_gb": 6.624006072,
      "passed": true
    },
    {
      "prompt": "Return only JSON with key ok and boolean value true.",
      "thinking": false,
      "output": "```json\n{\n  \"ok\": true\n}\n```",
      "finish_reason": "stop",
      "generated_tokens": 15,
      "tokens_per_second": 18.893826300522242,
      "peak_memory_gb": 6.624006072,
      "passed": false
    }
  ],
  "all_passed": false,
  "elapsed_seconds": 16.34895896911621
}