| { |
| "format": "4bit", |
| "strict_reload": true, |
| "finite_prefill_decode_logits": true, |
| "cache_offsets_correct": true, |
| "source_sidecars_identical": true, |
| "native_thinking_template_preserved": true, |
| "temperature": 0.0, |
| "max_tokens": 512, |
| "smoke_tests": [ |
| { |
| "task": "arithmetic", |
| "prompt": "What is 17 multiplied by 23? Give the answer.", |
| "text": "<think>1. The user asks for the product of 17 and 23.\n2. I need to calculate 17 * 23.\n3. Calculation: 17 * 20 = 340, 17 * 3 = 51, sum = 391.\n4. Or, standard multiplication: 17 * 23 = 391.\n5. The user specifically asks to \"Give the answer.\" So I should just provide the number clearly.</think>391", |
| "answer": "391", |
| "checks": { |
| "expected_391": true |
| }, |
| "generated_tokens": 100, |
| "seconds": 0.8707173750735819 |
| }, |
| { |
| "task": "retrieval", |
| "prompt": "The passcode is 941731. Repeat the passcode exactly.", |
| "text": "<think>1. The user asks to repeat the passcode exactly, which is \"941731\".\n2. I should just output the string \"941731\" without any additional text, as per the instruction to \"Repeat the passcode exactly.\"\n\nFinal check: The instruction is clear and direct. I will just output the number.</think>941731", |
| "answer": "941731", |
| "checks": { |
| "expected_passcode": true |
| }, |
| "generated_tokens": 69, |
| "seconds": 0.59464908298105 |
| }, |
| { |
| "task": "code", |
| "prompt": "Write a Python function add(a, b) that returns a + b.", |
| "text": "<think>We need to write a Python function add(a, b) that returns a + b. This is straightforward. The user likely expects a simple implementation.\n\nWe should provide the code, maybe with a brief explanation. The instruction says \"Write a Python function add(a, b) that returns a + b.\" So we just need to output the function definition.\n\nWe can include type hints or not, but simple is fine.\n\nLet's respond with the code and perhaps a short note.\n\nWe should also consider that the user might be asking for a specific implementation, but it's just basic addition.\n\nI'll write:\n\ndef add(a, b):\n return a + b\n\nMaybe with type hints: def add(a: float, b: float) -> float: but not necessary.\n\nI'll keep it simple.\n\nAlso, we might include a usage example.\n\nBut the instruction is just to write the function, so the answer should be the code.\n\nI'll provide it in a code block.\n\nLet's do that.</think>Here's a simple Python function that returns the sum of two numbers:\n\n```python\ndef add(a, b):\n return a + b\n```\n\nYou can use it like this:\n\n```python\nresult = add(3, 5) # result will be 8\n```", |
| "answer": "Here's a simple Python function that returns the sum of two numbers:\n\n```python\ndef add(a, b):\n return a + b\n```\n\nYou can use it like this:\n\n```python\nresult = add(3, 5) # result will be 8\n```", |
| "checks": { |
| "add_function": true |
| }, |
| "generated_tokens": 257, |
| "seconds": 2.1881555418949574 |
| } |
| ], |
| "all_smoke_checks_pass": true, |
| "device": { |
| "device_name": "Apple M5 Max", |
| "max_recommended_working_set_size": 115448725504, |
| "memory_size": 137438953472, |
| "architecture": "applegpu_g17s", |
| "max_buffer_length": 86586540032, |
| "resource_limit": 499000 |
| }, |
| "mlx_version": "0.32.4", |
| "mlx_vlm_version": "0.7.10", |
| "converter_revision": "6d9ba328abbd6129c7461eda86280aa9fbfd9986", |
| "python": "3.12.15", |
| "peak_active_gb": 22.48750338, |
| "seconds": 4.975497207837179, |
| "swap_before": "vm.swapusage: total = 10240.00M used = 8744.50M free = 1495.50M (encrypted)", |
| "swap_after": "vm.swapusage: total = 10240.00M used = 8744.50M free = 1495.50M (encrypted)", |
| "limits": "Three synthetic smoke prompts and finite/cache checks; not a quality benchmark or proof of parity to BF16/Transformers." |
| } |
