| { |
| "model": "gemma4-e4b-efe-t-MLX-4bit", |
| "results": [ |
| { |
| "prompt": "What is 17 + 25? Give only the number.", |
| "thinking": false, |
| "output": "42", |
| "finish_reason": "stop", |
| "generated_tokens": 3, |
| "tokens_per_second": 25.32718694288921, |
| "peak_memory_gb": 6.603434614, |
| "passed": true |
| }, |
| { |
| "prompt": "What is 9 multiplied by 8? Give only the number.", |
| "thinking": false, |
| "output": "72", |
| "finish_reason": "stop", |
| "generated_tokens": 3, |
| "tokens_per_second": 25.399746072864932, |
| "peak_memory_gb": 6.603434614, |
| "passed": true |
| }, |
| { |
| "prompt": "What is 17 + 25? Give only the number.", |
| "thinking": true, |
| "output": "42", |
| "finish_reason": "stop", |
| "generated_tokens": 3, |
| "tokens_per_second": 24.52999507269902, |
| "peak_memory_gb": 6.624006072, |
| "passed": true |
| }, |
| { |
| "prompt": "What is 9 multiplied by 8? Give only the number.", |
| "thinking": true, |
| "output": "72", |
| "finish_reason": "stop", |
| "generated_tokens": 3, |
| "tokens_per_second": 25.771479991715882, |
| "peak_memory_gb": 6.624006072, |
| "passed": true |
| }, |
| { |
| "prompt": "Translate to Turkish: The meeting is tomorrow. Reply only with the translation.", |
| "thinking": false, |
| "output": "Toplantı yarın.", |
| "finish_reason": "stop", |
| "generated_tokens": 7, |
| "tokens_per_second": 20.10139234177456, |
| "peak_memory_gb": 6.624006072, |
| "passed": true |
| }, |
| { |
| "prompt": "Return only JSON with key ok and boolean value true.", |
| "thinking": false, |
| "output": "```json\n{\n \"ok\": true\n}\n```", |
| "finish_reason": "stop", |
| "generated_tokens": 15, |
| "tokens_per_second": 18.893826300522242, |
| "peak_memory_gb": 6.624006072, |
| "passed": false |
| } |
| ], |
| "all_passed": false, |
| "elapsed_seconds": 16.34895896911621 |
| } |
|
|
