File size: 9,096 Bytes
d5c844b 5987826 d5c844b 5987826 d5c844b 5987826 d5c844b 5987826 d5c844b 5987826 d5c844b 5987826 d5c844b 5987826 d5c844b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | {
"xnnpack": {
"bytes": 1789728256,
"context_length": 8192,
"device": "Samsung Galaxy S20+ SM-G986B, Exynos 990, Mali-G77, Android 13",
"runtime": "Android shared libraries shipped with React Native ExecuTorch",
"compute_dtype": "float32",
"delegates": {
"XnnpackBackend": 364
},
"portable_operators": [
"aten::sym_size.int",
"quantized_decomposed::embedding_byte.dtype_out",
"aten::select_copy.int_out",
"aten::_local_scalar_dense.",
"executorch_prim::add.Scalar",
"aten::slice_copy.Tensor_out",
"aten::mean.out",
"aten::unsqueeze_copy.out",
"aten::alias_copy.out",
"executorch_prim::et_view.default",
"llama::update_cache.out",
"aten::cat.out",
"llama::custom_sdpa.out",
"aten::copy_."
],
"first_token": 597,
"reference_first_token": 597,
"cosine_similarity": 0.9592449069023132,
"prefill_bound_checked": 128,
"generated_text": "The user is asking for the capital of France and wants just the city name as a reply. This is a straightforward factual question. The capital of France is Paris. I should provide just \"Paris\" as requested.</think>Paris<|im_end|>",
"short_prompt": {
"prompt_tokens": 35,
"prefill_ms": 600.089,
"generated": 46,
"decode_ms": 3070.132,
"tokens_per_second": 14.983,
"peak_rss_mib": 1915.4
},
"long_context": {
"prompt_tokens": 8000,
"prefill_ms": 177152.107,
"generated": 90,
"decode_ms": 16435.307,
"tokens_per_second": 5.476,
"peak_rss_mib": 1915.3,
"generated_text": "The user has provided a long list of background text (all identical: \"The weather is pleasant and the sky is blue.\") and then at the end asks: \"What is the capital of France? Reply with just the city name.\"\n\nThis is a straightforward question. The answer is \"Paris\". The user wants just the city name, no extra text.\n\nI need to provide the answer \"Paris\" as the only content.</think>Paris<|im_end|>",
"last_decode_position": 8089
},
"test_scope": "Generation, shape/finite checks and long-context smoke test; not a quality benchmark.",
"mac_validation": {
"bytes": 1789728256,
"context_length": 8192,
"delegates": {
"XnnpackBackend": 364
},
"portable_operators": [
"aten::sym_size.int",
"quantized_decomposed::embedding_byte.dtype_out",
"aten::select_copy.int_out",
"aten::_local_scalar_dense.",
"executorch_prim::add.Scalar",
"aten::slice_copy.Tensor_out",
"aten::mean.out",
"aten::unsqueeze_copy.out",
"aten::alias_copy.out",
"executorch_prim::et_view.default",
"llama::update_cache.out",
"aten::cat.out",
"llama::custom_sdpa.out",
"aten::copy_."
],
"prefill_seconds": 4.127708917018026,
"first_token": 597,
"reference_first_token": 597,
"cosine_similarity": 0.9592365622520447,
"reference_top1_rank": 1,
"max_abs_error": 10.013851165771484,
"decode_tokens_per_second": 23.283732161212995,
"prefill_bound_checked": 128,
"generated_text": "The user is asking for the capital of France and wants just the city name as a reply. This is a straightforward factual question. The capital of France is Paris. I should provide just \"Paris\" as requested.</think>Paris",
"long_context": {
"prompt_tokens": 7800,
"prefill_seconds": 269.0508715000469,
"generated_text": "The user has provided a long list of background text (all about \"The weather is pleasant and the sky is blue\") and then at the end asks: \"What is the capital of France? Reply with just the city name.\"\n\nThis is a straightforward question. The capital of France is Paris. The user wants just the city name as the answer.\n\nI need to provide a concise answer: \"Paris\"\n\nLet me verify: The question is \"What is the capital of France?\" and the instruction is \"Reply with just the city name.\"\n\nSo the answer should be \"Paris\" with no additional text.</think>Paris<|im_end|>",
"last_cache_position_checked": 8191,
"peak_rss_mib": 2017.765625
},
"device": "Apple M1 Pro Mac, 16 GB RAM"
}
},
"mlx": {
"bytes": 1665769984,
"context_length": 8192,
"delegates": {
"MLXBackend": 1
},
"portable_operators": [
"quantized_decomposed::embedding_4bit.dtype_out"
],
"prefill_seconds": 0.888007125002332,
"first_token": 597,
"reference_first_token": 597,
"cosine_similarity": 0.9547369480133057,
"reference_top1_rank": 1,
"max_abs_error": 16.562870025634766,
"decode_tokens_per_second": 71.28213884705849,
"prefill_bound_checked": 128,
"generated_text": "The user is asking for the capital of France and wants a simple answer with just the city name. This is a straightforward factual question. The capital of France is Paris. I should provide just \"Paris\" as requested.</think>Paris",
"device": "Apple M1 Pro Mac, 16 GB RAM",
"runtime": "Source-built software-mansion-labs ExecuTorch runtime with native MLX backend",
"long_context": {
"prompt_tokens": 8000,
"prefill_seconds": 16.85774962499272,
"generated_text": "The user has provided a long list of background text (all saying \"The weather is pleasant and the sky is blue\") and then at the end asks: \"Ignore the background text. What is the capital of France? Reply with just the city name.\"\n\nSo I need to:\n1. Ignore the background text (the long list)\n2. Answer the question \"What is the capital of France?\"\n3. Reply with just the city name\n\nThe capital of France is Paris.\n\nSo my response should be simply \"Paris\".</think>Paris<|im_end|>",
"last_cache_position_checked": 8191,
"peak_rss_mib": 2136.0625
}
},
"vulkan": {
"bytes": 1623052544,
"context_length": 8192,
"device": "Samsung Galaxy S20+ SM-G986B, Exynos 990, Mali-G77, Android 13",
"runtime": "Android shared libraries shipped with React Native ExecuTorch",
"compute_dtype": "float16",
"delegates": {
"VulkanBackend": 1
},
"portable_operators": [],
"first_token": 597,
"reference_first_token": 597,
"cosine_similarity": 0.9498623609542847,
"prefill_bound_checked": 128,
"generated_text": "The user is asking for the capital of France and wants a simple answer with just the city name. This is a straightforward factual question. The capital of France is Paris. I should provide just \"Paris\" as requested.</think>Paris<|im_end|>",
"short_prompt": {
"prompt_tokens": 35,
"prefill_ms": 1257.702,
"generated": 47,
"decode_ms": 6588.921,
"tokens_per_second": 7.133,
"peak_rss_mib": 2305.3
},
"long_context": {
"prompt_tokens": 8000,
"prefill_ms": 285873.588,
"generated": 120,
"decode_ms": 24415.921,
"tokens_per_second": 4.915,
"peak_rss_mib": 2401.0,
"generated_text": "The user has provided a long list of background text (all identical: \"This is background text. The weather is pleasant and the sky is blue.\") and then at the end asks: \"Ignore the background text. What is the capital of France? Reply with just the city name.\"\n\nSo I need to:\n1. Ignore the background text (the long list)\n2. Answer the question: \"What is the capital of France?\"\n3. Reply with just the city name\n\nThe capital of France is \"Paris\".\n\nSo my response should be simply \"Paris\".</think>Paris<|im_end|>",
"last_decode_position": 8119
},
"test_scope": "Generation, shape/finite checks and long-context smoke test; not a quality benchmark.",
"tool_roundtrip": {
"generated_call": "The user wants me to multiply 17 by 23 using the calculate function. This is a straightforward arithmetic operation. I need to use the calculate function with:\n- operation: \"multiply\"\n- a: 17\n- b: 23\n\nLet me make this call.</think><|tool_call_start|>[calculate(operation='multiply', a=17, b=23)]<|tool_call_end|><|im_end|>",
"tool_result": "Result: 391",
"final_answer": "The calculation has been performed. The result of multiplying 17 by 23 is 391. I should provide this answer to the user.</think>The result of multiplying 17 by 23 is 391.<|im_end|>",
"tool_prompt": {
"prompt_tokens": 406,
"prefill_ms": 15040.201,
"generated": 76,
"decode_ms": 13034.403,
"tokens_per_second": 5.831,
"peak_rss_mib": 2412.6
},
"followup_prompt": {
"prompt_tokens": 438,
"prefill_ms": 17350.5,
"generated": 42,
"decode_ms": 11261.632,
"tokens_per_second": 3.729,
"peak_rss_mib": 2409.3
}
},
"maximum_explicit_3d_texture_extents": [
2688,
16384,
128
],
"required_max_image_dimension_3d": 16384,
"quantized_operator_counts": {
"et_vk.linear_dq8ca_q4gsw.default": 211,
"et_vk.embedding_q4gsw.default": 1
}
}
}
|