diff --git "a/build/webgpu/bench.json" "b/build/webgpu/bench.json" --- "a/build/webgpu/bench.json" +++ "b/build/webgpu/bench.json" @@ -142,6 +142,10 @@ { "type": "bandwidth", "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" + }, + { + "type": "gflops", + "value": "4 * dim(shapes.queryT, 0) * dim(shapes.queryT, 1) * dim(shapes.keyT, 1) * dim(shapes.queryT, 2)" } ] } @@ -163,6 +167,10 @@ { "type": "bandwidth", "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim + 3 * args.heads * args.headDim) * 4" + }, + { + "type": "gflops", + "value": "4 * dim(shapes.queryT, 0) * dim(shapes.queryT, 1) * dim(shapes.keyT, 1) * dim(shapes.queryT, 2)" } ] } @@ -183,6 +191,10 @@ { "type": "bandwidth", "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" + }, + { + "type": "gflops", + "value": "4 * dim(shapes.queryT, 0) * dim(shapes.queryT, 1) * dim(shapes.keyT, 1) * dim(shapes.queryT, 2)" } ] } @@ -204,6 +216,10 @@ { "type": "bandwidth", "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim + 3 * args.heads * args.headDim) * 4" + }, + { + "type": "gflops", + "value": "4 * dim(shapes.queryT, 0) * dim(shapes.queryT, 1) * dim(shapes.keyT, 1) * dim(shapes.queryT, 2)" } ] } @@ -224,6 +240,10 @@ { "type": "bandwidth", "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" + }, + { + "type": "gflops", + "value": "4 * dim(shapes.queryT, 0) * dim(shapes.queryT, 1) * dim(shapes.keyT, 1) * dim(shapes.queryT, 2)" } ] } @@ -245,6 +265,10 @@ { "type": "bandwidth", "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim + 3 * args.heads * args.headDim) * 4" + }, + { + "type": "gflops", + "value": "4 * dim(shapes.queryT, 0) * dim(shapes.queryT, 1) * dim(shapes.keyT, 1) * dim(shapes.queryT, 2)" } ] } @@ -266,6 +290,10 @@ { "type": "bandwidth", "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" + }, + { + "type": "gflops", + "value": "4 * dim(shapes.queryT, 0) * dim(shapes.queryT, 1) * dim(shapes.keyT, 1) * dim(shapes.queryT, 2)" } ] } @@ -332,6 +360,10 @@ { "type": "bandwidth", "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" + }, + { + "type": "gflops", + "value": "4 * dim(shapes.queryT, 0) * dim(shapes.queryT, 1) * dim(shapes.keyT, 1) * dim(shapes.queryT, 2)" } ] } @@ -387,12 +419,16 @@ { "type": "bandwidth", "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 2" + }, + { + "type": "gflops", + "value": "4 * dim(shapes.queryT, 0) * dim(shapes.queryT, 1) * dim(shapes.keyT, 1) * dim(shapes.queryT, 2)" } ] } }, { - "name": "prefill-lpq4-d80-s512-h8-BLINDSPOT", + "name": "prefill-lpq4-d80-s512-h8", "preset": "smoke", "vars": { "batch": 1, "qSeq": 512, "kvSeq": 512, "heads": 8, "headDim": 80 }, "attrs": { "num_heads": 8, "scale": 0.111803398875 }, @@ -410,7 +446,7 @@ } }, { - "name": "prefill-cluster-d128-s512-h8-HEALTHY", + "name": "prefill-cluster-d128-s512-h8-control", "preset": "smoke", "vars": { "batch": 1, "qSeq": 512, "kvSeq": 512, "heads": 8, "headDim": 128 }, "attrs": { "num_heads": 8, "scale": 0.08838834764831845 }, @@ -427,7 +463,7 @@ } }, { - "name": "prefill-online-noncluster-qseq16-h16-d64-FALLOFF", + "name": "prefill-online-noncluster-qseq16-h16-d64-falloff", "preset": "smoke", "vars": { "batch": 1, "qSeq": 16, "kvSeq": 512, "heads": 16, "headDim": 64 }, "attrs": { "num_heads": 16, "scale": 0.125 }, @@ -444,7 +480,7 @@ } }, { - "name": "prefill-cluster-qseq64-h16-d64-HEALTHY", + "name": "prefill-cluster-qseq64-h16-d64-control", "preset": "smoke", "vars": { "batch": 1, "qSeq": 64, "kvSeq": 512, "heads": 16, "headDim": 64 }, "attrs": { "num_heads": 16, "scale": 0.125 }, @@ -461,7 +497,7 @@ } }, { - "name": "decode-splitk-longkv-kv16384-h4-d64-LONGCONTEXT", + "name": "decode-splitk-longkv-kv16384-h4-d64-long-context", "preset": "model", "vars": { "batch": 1, "qSeq": 1, "kvSeq": 16384, "heads": 4, "headDim": 64 }, "attrs": { "num_heads": 4, "scale": 0.125 }, @@ -476,12 +512,16 @@ { "type": "bandwidth", "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" + }, + { + "type": "gflops", + "value": "4 * dim(shapes.queryT, 0) * dim(shapes.queryT, 1) * dim(shapes.keyT, 1) * dim(shapes.queryT, 2)" } ] } }, { - "name": "scalar-fallback-odd-headdim6-prefill-h8-q128-kv2048-SLOW", + "name": "scalar-fallback-odd-headdim6-prefill-h8-q128-kv2048", "preset": "stress", "vars": { "batch": 1, "qSeq": 128, "kvSeq": 2048, "heads": 8, "headDim": 6 }, "attrs": { "num_heads": 8, "scale": 0.408248290463863 }, @@ -498,7 +538,7 @@ } }, { - "name": "scalar-fallback-shortq16-longkv4096-d128-h4-b2-SLOW", + "name": "scalar-fallback-shortq16-longkv4096-d128-h4-b2", "preset": "stress", "vars": { "batch": 2, "qSeq": 16, "kvSeq": 4096, "heads": 4, "headDim": 128 }, "attrs": { "num_heads": 4, "scale": 0.08838834764831845 }, @@ -664,6 +704,6692 @@ { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } ] } + }, + { + "name": "value-float32-b2-q1-kv65-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 2], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 2], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 2], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 2], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1040 }] } + }, + { + "name": "value-float32-b1-q31-kv512-h8-d1", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 8], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 8], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 8], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 8], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 507904 }] } + }, + { + "name": "value-float32-b1-q128-kv2048-h3-d1", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 3], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 3], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 3], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 3], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3145728 }] } + }, + { + "name": "value-float32-b1-q31-kv65-h3-d2", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [1, 31, 6], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 6], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 6], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 6], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 48360 }] } + }, + { + "name": "value-float32-b1-q128-kv512-h2-d2", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 4], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 4], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 4], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 4], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1048576 }] } + }, + { + "name": "value-float32-b2-q1-kv2048-h8-d2", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [2, 1, 16], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2048, 16], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2048, 16], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 16], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 262144 }] } + }, + { + "name": "value-float32-b1-q128-kv65-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 24], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 24], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 24], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 24], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 798720 }] } + }, + { + "name": "value-float32-b2-q1-kv512-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 512, 9], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 512, 9], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 36864 }] } + }, + { + "name": "value-float32-b1-q31-kv2048-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 6], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 6], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 6], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 6], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1523712 }] } + }, + { + "name": "value-float32-b2-q1-kv65-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [2, 1, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 6240 }] } + }, + { + "name": "value-float32-b1-q31-kv512-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 48], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 48], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3047424 }] } + }, + { + "name": "value-float32-b1-q128-kv2048-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [1, 128, 18], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 18], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 18], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 18], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 18874368 }] } + }, + { + "name": "value-float32-b1-q31-kv65-h3-d8", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 24], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 24], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 193440 }] } + }, + { + "name": "value-float32-b1-q128-kv512-h2-d8", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 16], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 16], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 16], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 16], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4194304 }] } + }, + { + "name": "value-float32-b2-q1-kv2048-h8-d8", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 64], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2048, 64], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2048, 64], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 64], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1048576 }] } + }, + { + "name": "value-float32-b1-q128-kv65-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 128, 120], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 120], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 120], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 120], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3993600 }] } + }, + { + "name": "value-float32-b2-q1-kv512-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 512, 45], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 512, 45], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 184320 }] } + }, + { + "name": "value-float32-b1-q31-kv2048-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 31, 30], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 30], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 30], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 30], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7618560 }] } + }, + { + "name": "value-float32-b2-q1-kv65-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 32], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 32], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 32], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 32], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 16640 }] } + }, + { + "name": "value-float32-b1-q31-kv512-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 128], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 128], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 8126464 }] } + }, + { + "name": "value-float32-b1-q128-kv2048-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 48], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 48], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 50331648 }] } + }, + { + "name": "value-float32-b1-q31-kv65-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [1, 31, 72], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 72], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 72], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 72], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 580320 }] } + }, + { + "name": "value-float32-b1-q128-kv512-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 48], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 48], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 12582912 }] } + }, + { + "name": "value-float32-b2-q1-kv2048-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [2, 1, 192], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2048, 192], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2048, 192], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 192], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3145728 }] } + }, + { + "name": "value-float32-b1-q128-kv65-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 248], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 248], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 248], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 248], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 8253440 }] } + }, + { + "name": "value-float32-b2-q1-kv512-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 512, 93], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 512, 93], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 380928 }] } + }, + { + "name": "value-float32-b1-q31-kv2048-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 62], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 62], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 62], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 15745024 }] } + }, + { + "name": "value-float32-b2-q1-kv65-h2-d32", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [2, 1, 64], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 64], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 64], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 64], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 33280 }] } + }, + { + "name": "value-float32-b1-q31-kv512-h8-d32", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 256], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 256], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 256], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 256], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 16252928 }] } + }, + { + "name": "value-float32-b1-q128-kv2048-h3-d32", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [1, 128, 96], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 96], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 96], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 96], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 100663296 }] } + }, + { + "name": "value-float32-b1-q31-kv65-h3-d33", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 99], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 99], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 99], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 99], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 797940 }] } + }, + { + "name": "value-float32-b1-q128-kv512-h2-d33", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 66], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 66], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 66], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 66], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 17301504 }] } + }, + { + "name": "value-float32-b2-q1-kv2048-h8-d33", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 264], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2048, 264], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2048, 264], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 264], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4325376 }] } + }, + { + "name": "value-float32-b1-q128-kv65-h8-d64", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 128, 512], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 512], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 512], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 512], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 17039360 }] } + }, + { + "name": "value-float32-b2-q1-kv512-h3-d64", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 192], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 512, 192], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 512, 192], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 192], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 786432 }] } + }, + { + "name": "value-float32-b1-q31-kv2048-h2-d64", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 128], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 128], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 32505856 }] } + }, + { + "name": "value-float32-b2-q3-kv63-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 63, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 63, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 18144 }] } + }, + { + "name": "value-float32-b2-q3-kv64-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 64, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 64, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 18432 }] } + }, + { + "name": "value-float32-b2-q3-kv65-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 18720 }] } + }, + { + "name": "value-float32-b2-q3-kv127-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 127, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 127, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 36576 }] } + }, + { + "name": "value-float32-b2-q3-kv128-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 128, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 128, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 36864 }] } + }, + { + "name": "value-float32-b2-q3-kv129-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 129, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 129, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 37152 }] } + }, + { + "name": "value-float32-b2-q3-kv2047-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 589536 }] } + }, + { + "name": "value-float32-b2-q3-kv2048-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2048, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2048, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 589824 }] } + }, + { + "name": "value-float32-b2-q3-kv2049-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2049, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2049, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 590112 }] } + }, + { + "name": "value-float32-b2-q3-kv4095-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 4095, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 4095, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1179360 }] } + }, + { + "name": "value-float32-b2-q3-kv4096-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 4096, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 4096, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1179648 }] } + }, + { + "name": "value-float32-b1-q128-kv2048-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0.4082482904638631 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 48], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 48], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 50331648 }] } + }, + { + "name": "value-float32-b1-q3-kv65-h2-d6-bias", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 }, + "biasT": { "shape": [36], "dtype": "float32", "dist": "normal", "seed": 1534, "scale": 0.1 } + }, + "outputs": { "outputT": { "shape": [1, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9360 }] } + }, + { + "name": "value-float32-b1-q3-kv65-h2-d6-causal", + "preset": "model", + "attrs": { "num_heads": 2, "unidirectional": 1 }, + "inputs": { + "queryT": { "shape": [1, 3, 12], "dtype": "float32", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 12], "dtype": "float32", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 12], "dtype": "float32", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 3, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9360 }] } + }, + { + "name": "value-float16-b2-q1-kv65-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 2], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 2], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 2], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 2], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1040 }] } + }, + { + "name": "value-float16-b1-q31-kv512-h8-d1", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 8], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 8], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 8], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 8], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 507904 }] } + }, + { + "name": "value-float16-b1-q128-kv2048-h3-d1", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 3], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 3], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 3], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 3], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3145728 }] } + }, + { + "name": "value-float16-b1-q31-kv65-h3-d2", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [1, 31, 6], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 6], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 6], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 6], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 48360 }] } + }, + { + "name": "value-float16-b1-q128-kv512-h2-d2", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 4], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 4], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 4], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 4], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1048576 }] } + }, + { + "name": "value-float16-b2-q1-kv2048-h8-d2", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [2, 1, 16], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2048, 16], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2048, 16], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 16], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 262144 }] } + }, + { + "name": "value-float16-b1-q128-kv65-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 24], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 24], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 24], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 24], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 798720 }] } + }, + { + "name": "value-float16-b2-q1-kv512-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 512, 9], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 512, 9], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 36864 }] } + }, + { + "name": "value-float16-b1-q31-kv2048-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 6], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 6], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 6], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 6], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1523712 }] } + }, + { + "name": "value-float16-b2-q1-kv65-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [2, 1, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 6240 }] } + }, + { + "name": "value-float16-b1-q31-kv512-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 48], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 48], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3047424 }] } + }, + { + "name": "value-float16-b1-q128-kv2048-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [1, 128, 18], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 18], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 18], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 18], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 18874368 }] } + }, + { + "name": "value-float16-b1-q31-kv65-h3-d8", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 24], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 24], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 193440 }] } + }, + { + "name": "value-float16-b1-q128-kv512-h2-d8", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 16], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 16], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 16], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 16], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4194304 }] } + }, + { + "name": "value-float16-b2-q1-kv2048-h8-d8", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 64], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2048, 64], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2048, 64], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 64], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1048576 }] } + }, + { + "name": "value-float16-b1-q128-kv65-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 128, 120], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 120], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 120], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 120], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3993600 }] } + }, + { + "name": "value-float16-b2-q1-kv512-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 512, 45], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 512, 45], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 184320 }] } + }, + { + "name": "value-float16-b1-q31-kv2048-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 31, 30], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 30], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 30], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 30], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7618560 }] } + }, + { + "name": "value-float16-b2-q1-kv65-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 32], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 32], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 32], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 32], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 16640 }] } + }, + { + "name": "value-float16-b1-q31-kv512-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 128], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 128], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 8126464 }] } + }, + { + "name": "value-float16-b1-q128-kv2048-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 48], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 48], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 50331648 }] } + }, + { + "name": "value-float16-b1-q31-kv65-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [1, 31, 72], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 72], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 72], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 72], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 580320 }] } + }, + { + "name": "value-float16-b1-q128-kv512-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 48], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 48], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 12582912 }] } + }, + { + "name": "value-float16-b2-q1-kv2048-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [2, 1, 192], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2048, 192], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2048, 192], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 192], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3145728 }] } + }, + { + "name": "value-float16-b1-q128-kv65-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 128, 248], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 248], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 248], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 248], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 8253440 }] } + }, + { + "name": "value-float16-b2-q1-kv512-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 512, 93], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 512, 93], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 380928 }] } + }, + { + "name": "value-float16-b1-q31-kv2048-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 62], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 62], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 62], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 15745024 }] } + }, + { + "name": "value-float16-b2-q1-kv65-h2-d32", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [2, 1, 64], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 64], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 64], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 64], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 33280 }] } + }, + { + "name": "value-float16-b1-q31-kv512-h8-d32", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 256], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 256], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 256], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 256], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 16252928 }] } + }, + { + "name": "value-float16-b1-q128-kv2048-h3-d32", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [1, 128, 96], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 96], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 96], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 96], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 100663296 }] } + }, + { + "name": "value-float16-b1-q31-kv65-h3-d33", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [1, 31, 99], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 99], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 99], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 99], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 797940 }] } + }, + { + "name": "value-float16-b1-q128-kv512-h2-d33", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 66], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 66], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 66], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 66], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 17301504 }] } + }, + { + "name": "value-float16-b2-q1-kv2048-h8-d33", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 264], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2048, 264], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2048, 264], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 264], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4325376 }] } + }, + { + "name": "value-float16-b1-q128-kv65-h8-d64", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 128, 512], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 512], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 512], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 512], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 17039360 }] } + }, + { + "name": "value-float16-b2-q1-kv512-h3-d64", + "preset": "model", + "attrs": { "num_heads": 3, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 1, 192], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 512, 192], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 512, 192], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 192], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 786432 }] } + }, + { + "name": "value-float16-b1-q31-kv2048-h2-d64", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 128], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 128], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 32505856 }] } + }, + { + "name": "value-float16-b2-q3-kv63-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 63, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 63, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 18144 }] } + }, + { + "name": "value-float16-b2-q3-kv64-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 64, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 64, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 18432 }] } + }, + { + "name": "value-float16-b2-q3-kv65-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 18720 }] } + }, + { + "name": "value-float16-b2-q3-kv127-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 127, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 127, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 36576 }] } + }, + { + "name": "value-float16-b2-q3-kv128-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 128, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 128, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 36864 }] } + }, + { + "name": "value-float16-b2-q3-kv129-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 129, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 129, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 37152 }] } + }, + { + "name": "value-float16-b2-q3-kv2047-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 589536 }] } + }, + { + "name": "value-float16-b2-q3-kv2048-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2048, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2048, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 589824 }] } + }, + { + "name": "value-float16-b2-q3-kv2049-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 2049, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 2049, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 590112 }] } + }, + { + "name": "value-float16-b2-q3-kv4095-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 4095, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 4095, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1179360 }] } + }, + { + "name": "value-float16-b2-q3-kv4096-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "shape": [2, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [2, 4096, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [2, 4096, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1179648 }] } + }, + { + "name": "value-float16-b1-q128-kv2048-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8, "scale": 0.4082482904638631 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 2048, 48], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 2048, 48], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 50331648 }] } + }, + { + "name": "value-float16-b1-q3-kv65-h2-d6-bias", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 }, + "biasT": { "shape": [36], "dtype": "float16", "dist": "normal", "seed": 1534, "scale": 0.1 } + }, + "outputs": { "outputT": { "shape": [1, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9360 }] } + }, + { + "name": "value-float16-b1-q3-kv65-h2-d6-causal", + "preset": "model", + "attrs": { "num_heads": 2, "unidirectional": 1 }, + "inputs": { + "queryT": { "shape": [1, 3, 12], "dtype": "float16", "dist": "normal", "seed": 1531, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 12], "dtype": "float16", "dist": "normal", "seed": 1532, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 12], "dtype": "float16", "dist": "normal", "seed": 1533, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 3, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9360 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv65-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 9], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 9], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4680 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv65-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 24], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 24], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 193440 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv65-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 6], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 6], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 6], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 6], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 199680 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv66-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 66, 9], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 66, 9], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4752 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv66-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 66, 24], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 66, 24], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 196416 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv66-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 6], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 66, 6], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 66, 6], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 6], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 202752 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv129-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 129, 9], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 129, 9], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9288 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv129-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 129, 24], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 129, 24], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 383904 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv129-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 6], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 129, 6], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 129, 6], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 6], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 396288 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv96-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 96, 18], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 96, 18], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 13824 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv96-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 96, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 96, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 571392 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv96-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 96, 12], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 96, 12], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 589824 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv97-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 97, 18], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 97, 18], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 13968 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv97-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 97, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 97, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 577344 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv97-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 97, 12], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 97, 12], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 595968 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv98-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 98, 18], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 98, 18], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 14112 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv98-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 98, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 98, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 583296 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv98-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 98, 12], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 98, 12], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 602112 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv129-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 129, 18], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 129, 18], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 18576 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv129-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 129, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 129, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 767808 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv129-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 129, 12], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 129, 12], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 792576 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv285-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 285, 45], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 285, 45], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 102600 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv285-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 285, 120], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 285, 120], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4240800 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv285-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 285, 30], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 285, 30], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4377600 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv286-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 286, 45], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 286, 45], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 102960 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv286-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 286, 120], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 286, 120], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4255680 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv286-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 286, 30], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 286, 30], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4392960 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv287-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 287, 45], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 287, 45], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 103320 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv287-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 287, 120], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 287, 120], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4270560 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv287-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 287, 30], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 287, 30], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4408320 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv321-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 321, 45], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 321, 45], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 115560 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv321-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 321, 120], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 321, 120], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4776480 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv321-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 321, 30], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 321, 30], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4930560 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv304-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 304, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 304, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 116736 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv304-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 304, 128], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 304, 128], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4825088 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv304-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 304, 32], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 304, 32], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4980736 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv305-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 305, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 305, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 117120 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv305-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 305, 128], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 305, 128], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4840960 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv305-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 305, 32], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 305, 32], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4997120 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv306-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 306, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 306, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 117504 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv306-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 306, 128], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 306, 128], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4856832 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv306-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 306, 32], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 306, 32], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5013504 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv321-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 321, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 321, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 123264 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv321-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 321, 128], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 321, 128], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5094912 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv321-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 321, 32], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 321, 32], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5259264 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv552-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 552, 72], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 552, 72], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 317952 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv552-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 552, 192], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 552, 192], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 13142016 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv552-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 552, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 552, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 13565952 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv553-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 553, 72], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 553, 72], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 318528 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv553-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 553, 192], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 553, 192], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 13165824 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv553-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 553, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 553, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 13590528 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv554-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 554, 72], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 554, 72], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 319104 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv554-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 554, 192], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 554, 192], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 13189632 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv554-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 554, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 554, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 13615104 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv577-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 577, 72], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 577, 72], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 332352 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv577-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 577, 192], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 577, 192], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 13737216 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv577-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 577, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 577, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 14180352 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv868-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 868, 93], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 868, 93], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 645792 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv868-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 868, 248], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 868, 248], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 26692736 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv868-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 868, 62], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 868, 62], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 27553792 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv869-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 869, 93], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 869, 93], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 646536 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv869-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 869, 248], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 869, 248], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 26723488 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv869-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 869, 62], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 869, 62], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 27585536 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv870-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 870, 93], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 870, 93], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 647280 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv870-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 870, 248], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 870, 248], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 26754240 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv870-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 870, 62], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 870, 62], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 27617280 }] } + }, + { + "name": "value-cost-float32-b2-q1-kv897-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 897, 93], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 897, 93], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 667368 }] } + }, + { + "name": "value-cost-float32-b1-q31-kv897-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 897, 248], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 897, 248], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 27584544 }] } + }, + { + "name": "value-cost-float32-b1-q128-kv897-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 897, 62], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 897, 62], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 28474368 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv84-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 84, 9], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 84, 9], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 6048 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv84-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 84, 24], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 84, 24], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 249984 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv84-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 6], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 84, 6], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 84, 6], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 6], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 258048 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv85-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 85, 9], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 85, 9], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 6120 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv85-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 85, 24], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 85, 24], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 252960 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv85-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 6], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 85, 6], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 85, 6], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 6], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 261120 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv86-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 86, 9], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 86, 9], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 6192 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv86-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 86, 24], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 86, 24], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 255936 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv86-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 6], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 86, 6], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 86, 6], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 6], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 264192 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv129-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 129, 9], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 129, 9], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9288 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv129-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 129, 24], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 129, 24], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 383904 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv129-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 6], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 129, 6], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 129, 6], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 6], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 396288 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv174-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 174, 18], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 174, 18], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 25056 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv174-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 174, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 174, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1035648 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv174-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 174, 12], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 174, 12], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1069056 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv175-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 175, 18], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 175, 18], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 25200 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv175-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 175, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 175, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1041600 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv175-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 175, 12], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 175, 12], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1075200 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv176-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 176, 18], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 176, 18], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 25344 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv176-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 176, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 176, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1047552 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv176-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 176, 12], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 176, 12], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1081344 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv193-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 193, 18], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 193, 18], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 27792 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv193-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 193, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 193, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1148736 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv193-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 193, 12], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 193, 12], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1185792 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv510-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 510, 45], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 510, 45], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 183600 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv510-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 510, 120], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 510, 120], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7588800 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv510-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 510, 30], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 510, 30], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7833600 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv511-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 511, 45], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 511, 45], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 183960 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv511-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 511, 120], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 511, 120], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7603680 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv511-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 511, 30], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 511, 30], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7848960 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv512-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 512, 45], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 512, 45], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 184320 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv512-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 120], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 120], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7618560 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv512-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 512, 30], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 512, 30], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7864320 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv513-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 513, 45], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 513, 45], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 184680 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv513-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 513, 120], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 513, 120], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7633440 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv513-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 513, 30], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 513, 30], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7879680 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv560-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 560, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 560, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 215040 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv560-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 560, 128], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 560, 128], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 8888320 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv560-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 560, 32], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 560, 32], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9175040 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv561-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 561, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 561, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 215424 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv561-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 561, 128], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 561, 128], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 8904192 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv561-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 561, 32], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 561, 32], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9191424 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv562-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 562, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 562, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 215808 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv562-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 562, 128], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 562, 128], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 8920064 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv562-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 562, 32], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 562, 32], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9207808 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv577-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 577, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 577, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 221568 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv577-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 577, 128], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 577, 128], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9158144 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv577-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 577, 32], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 577, 32], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9453568 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv1008-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 1008, 72], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 1008, 72], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 580608 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv1008-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1008, 192], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1008, 192], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 23998464 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv1008-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1008, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1008, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 24772608 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv1009-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 1009, 72], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 1009, 72], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 581184 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv1009-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1009, 192], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1009, 192], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 24022272 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv1009-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1009, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1009, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 24797184 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv1010-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 1010, 72], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 1010, 72], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 581760 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv1010-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1010, 192], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1010, 192], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 24046080 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv1010-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1010, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1010, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 24821760 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv1025-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 1025, 72], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 1025, 72], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 590400 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv1025-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1025, 192], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1025, 192], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 24403200 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv1025-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1025, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1025, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 25190400 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv1581-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 1581, 93], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 1581, 93], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1176264 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv1581-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1581, 248], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1581, 248], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 48618912 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv1581-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1581, 62], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1581, 62], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 50187264 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv1582-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 1582, 93], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 1582, 93], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1177008 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv1582-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1582, 248], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1582, 248], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 48649664 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv1582-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1582, 62], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1582, 62], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 50219008 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv1583-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 1583, 93], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 1583, 93], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1177752 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv1583-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1583, 248], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1583, 248], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 48680416 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv1583-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1583, 62], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1583, 62], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 50250752 }] } + }, + { + "name": "value-cost-float16-b2-q1-kv1601-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 1601, 93], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 1601, 93], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1191144 }] } + }, + { + "name": "value-cost-float16-b1-q31-kv1601-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1601, 248], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1601, 248], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 49233952 }] } + }, + { + "name": "value-cost-float16-b1-q128-kv1601-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 1601, 62], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 1601, 62], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 50822144 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv65-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 18], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 18], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9360 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv65-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 386880 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv65-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 12], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 12], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 399360 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv66-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 66, 18], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 66, 18], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9504 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv66-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 66, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 66, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 392832 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv66-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 66, 12], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 66, 12], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 405504 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv120-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 120, 45], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 120, 45], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 43200 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv120-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 120, 120], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 120, 120], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1785600 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv120-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 120, 30], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 120, 30], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1843200 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv121-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 121, 45], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 121, 45], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 43560 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv121-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 121, 120], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 121, 120], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1800480 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv121-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 121, 30], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 121, 30], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1858560 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv122-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 122, 45], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 122, 45], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 43920 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv122-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 122, 120], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 122, 120], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1815360 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv122-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 122, 30], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 122, 30], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1873920 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv129-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 129, 45], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 129, 45], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 46440 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv129-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 129, 120], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 129, 120], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1919520 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv129-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 129, 30], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 129, 30], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1981440 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv144-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 144, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 144, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 55296 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv144-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 144, 128], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 144, 128], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2285568 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv144-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 144, 32], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 144, 32], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2359296 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv145-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 145, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 145, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 55680 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv145-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 145, 128], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 145, 128], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2301440 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv145-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 145, 32], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 145, 32], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2375680 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv146-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 146, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 146, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 56064 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv146-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 146, 128], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 146, 128], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2317312 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv146-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 146, 32], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 146, 32], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2392064 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv193-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 193, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 193, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 74112 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv193-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 193, 128], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 193, 128], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3063296 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv193-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 193, 32], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 193, 32], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3162112 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv240-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 240, 72], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 240, 72], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 138240 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv240-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 240, 192], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 240, 192], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5713920 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv240-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 240, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 240, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5898240 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv241-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 241, 72], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 241, 72], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 138816 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv241-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 241, 192], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 241, 192], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5737728 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv241-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 241, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 241, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5922816 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv242-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 242, 72], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 242, 72], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 139392 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv242-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 242, 192], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 242, 192], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5761536 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv242-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 242, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 242, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5947392 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv257-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 257, 72], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 257, 72], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 148032 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv257-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 257, 192], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 257, 192], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 6118656 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv257-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 257, 48], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 257, 48], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 6316032 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv372-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 372, 93], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 372, 93], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 276768 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv372-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 372, 248], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 372, 248], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 11439744 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv372-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 372, 62], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 372, 62], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 11808768 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv373-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 373, 93], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 373, 93], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 277512 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv373-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 373, 248], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 373, 248], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 11470496 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv373-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 373, 62], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 373, 62], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 11840512 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv374-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 374, 93], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 374, 93], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 278256 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv374-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 374, 248], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 374, 248], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 11501248 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv374-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 374, 62], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 374, 62], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 11872256 }] } + }, + { + "name": "value-cost-sg32-float32-b2-q1-kv385-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 385, 93], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 385, 93], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 286440 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q31-kv385-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 385, 248], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 385, 248], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 11839520 }] } + }, + { + "name": "value-cost-sg32-float32-b1-q128-kv385-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float32", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 385, 62], "dtype": "float32", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 385, 62], "dtype": "float32", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 12221440 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv65-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 65, 9], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 65, 9], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4680 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv65-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 24], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 24], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 193440 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv65-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 6], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 65, 6], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 65, 6], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 6], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 199680 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv66-h3-d3", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 9], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 66, 9], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 66, 9], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 9], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4752 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv66-h8-d3", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 24], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 66, 24], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 66, 24], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 24], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 196416 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv66-h2-d3", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 6], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 66, 6], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 66, 6], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 6], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 202752 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv72-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 72, 18], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 72, 18], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10368 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv72-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 72, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 72, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 428544 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv72-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 72, 12], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 72, 12], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 442368 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv73-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 73, 18], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 73, 18], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10512 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv73-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 73, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 73, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 434496 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv73-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 73, 12], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 73, 12], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 448512 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv74-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 74, 18], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 74, 18], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10656 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv74-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 74, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 74, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 440448 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv74-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 74, 12], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 74, 12], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 454656 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv129-h3-d6", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 18], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 129, 18], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 129, 18], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 18], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 18576 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv129-h8-d6", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 129, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 129, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 767808 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv129-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 12], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 129, 12], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 129, 12], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 792576 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv210-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 210, 45], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 210, 45], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 75600 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv210-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 210, 120], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 210, 120], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3124800 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv210-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 210, 30], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 210, 30], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3225600 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv211-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 211, 45], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 211, 45], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 75960 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv211-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 211, 120], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 211, 120], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3139680 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv211-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 211, 30], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 211, 30], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3240960 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv212-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 212, 45], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 212, 45], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 76320 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv212-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 212, 120], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 212, 120], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3154560 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv212-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 212, 30], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 212, 30], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3256320 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv257-h3-d15", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 45], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 257, 45], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 257, 45], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 45], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 92520 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv257-h8-d15", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 120], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 257, 120], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 257, 120], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 120], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3824160 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv257-h2-d15", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 30], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 257, 30], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 257, 30], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 30], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3947520 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv224-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 224, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 224, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 86016 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv224-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 224, 128], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 224, 128], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3555328 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv224-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 224, 32], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 224, 32], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3670016 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv225-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 225, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 225, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 86400 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv225-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 225, 128], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 225, 128], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3571200 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv225-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 225, 32], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 225, 32], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3686400 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv226-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 226, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 226, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 86784 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv226-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 226, 128], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 226, 128], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3587072 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv226-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 226, 32], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 226, 32], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3702784 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv257-h3-d16", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 257, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 257, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 98688 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv257-h8-d16", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 128], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 257, 128], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 257, 128], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 128], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4079104 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv257-h2-d16", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 32], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 257, 32], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 257, 32], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 32], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4210688 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv408-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 408, 72], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 408, 72], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 235008 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv408-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 408, 192], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 408, 192], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9713664 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv408-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 408, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 408, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10027008 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv409-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 409, 72], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 409, 72], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 235584 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv409-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 409, 192], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 409, 192], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9737472 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv409-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 409, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 409, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10051584 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv410-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 410, 72], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 410, 72], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 236160 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv410-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 410, 192], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 410, 192], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9761280 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv410-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 410, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 410, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10076160 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv449-h3-d24", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 72], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 449, 72], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 449, 72], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 72], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 258624 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv449-h8-d24", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 192], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 449, 192], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 449, 192], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 192], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10689792 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv449-h2-d24", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 48], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 449, 48], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 449, 48], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 11034624 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv620-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 620, 93], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 620, 93], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 461280 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv620-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 620, 248], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 620, 248], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 19066240 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv620-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 620, 62], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 620, 62], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 19681280 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv621-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 621, 93], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 621, 93], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 462024 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv621-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 621, 248], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 621, 248], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 19096992 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv621-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 621, 62], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 621, 62], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 19713024 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv622-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 622, 93], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 622, 93], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 462768 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv622-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 622, 248], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 622, 248], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 19127744 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv622-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 622, 62], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 622, 62], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 19744768 }] } + }, + { + "name": "value-cost-sg32-float16-b2-q1-kv641-h3-d31", + "preset": "model", + "attrs": { "num_heads": 3 }, + "inputs": { + "queryT": { "shape": [2, 1, 93], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [2, 641, 93], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [2, 641, 93], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 1, 93], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 476904 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q31-kv641-h8-d31", + "preset": "model", + "attrs": { "num_heads": 8 }, + "inputs": { + "queryT": { "shape": [1, 31, 248], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 641, 248], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 641, 248], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 31, 248], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 19712032 }] } + }, + { + "name": "value-cost-sg32-float16-b1-q128-kv641-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2 }, + "inputs": { + "queryT": { "shape": [1, 128, 62], "dtype": "float16", "dist": "normal", "seed": 15391, "scale": 0.2 }, + "keyT": { "shape": [1, 641, 62], "dtype": "float16", "dist": "normal", "seed": 15392, "scale": 0.2 }, + "valueT": { "shape": [1, 641, 62], "dtype": "float16", "dist": "normal", "seed": 15393, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [1, 128, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20347904 }] } + }, + { + "name": "query-long-value-stress-float32-d1-uniform-probability", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 2], "dtype": "float32", "dist": "constant", "value": 0 }, + "keyT": { "shape": [2, 2047, 2], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 2], "dtype": "float32", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 2], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163760 }] } + }, + { + "name": "query-long-value-stress-float32-d1-peaked", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 2], "dtype": "float32", "dist": "normal", "seed": 15351, "scale": 3 }, + "keyT": { "shape": [2, 2047, 2], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 3 }, + "valueT": { "shape": [2, 2047, 2], "dtype": "float32", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 2], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163760 }] } + }, + { + "name": "query-long-value-stress-float32-d1-negative-scale", + "preset": "model", + "attrs": { "num_heads": 2, "scale": -0.37 }, + "inputs": { + "queryT": { "shape": [2, 5, 2], "dtype": "float32", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 2], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 2], "dtype": "float32", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 2], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163760 }] } + }, + { + "name": "query-long-value-stress-float32-d1-constant-value", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 2], "dtype": "float32", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 2], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 2], "dtype": "float32", "dist": "constant", "value": 0.25 } + }, + "outputs": { "outputT": { "shape": [2, 5, 2], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163760 }] } + }, + { + "name": "query-long-value-stress-float32-d6-uniform-probability", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 12], "dtype": "float32", "dist": "constant", "value": 0 }, + "keyT": { "shape": [2, 2047, 12], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 12], "dtype": "float32", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 982560 }] } + }, + { + "name": "query-long-value-stress-float32-d6-peaked", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 12], "dtype": "float32", "dist": "normal", "seed": 15351, "scale": 3 }, + "keyT": { "shape": [2, 2047, 12], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 3 }, + "valueT": { "shape": [2, 2047, 12], "dtype": "float32", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 982560 }] } + }, + { + "name": "query-long-value-stress-float32-d6-negative-scale", + "preset": "model", + "attrs": { "num_heads": 2, "scale": -0.37 }, + "inputs": { + "queryT": { "shape": [2, 5, 12], "dtype": "float32", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 12], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 12], "dtype": "float32", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 982560 }] } + }, + { + "name": "query-long-value-stress-float32-d6-constant-value", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 12], "dtype": "float32", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 12], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 12], "dtype": "float32", "dist": "constant", "value": 0.25 } + }, + "outputs": { "outputT": { "shape": [2, 5, 12], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 982560 }] } + }, + { + "name": "query-long-value-stress-float32-d31-uniform-probability", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 62], "dtype": "float32", "dist": "constant", "value": 0 }, + "keyT": { "shape": [2, 2047, 62], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 62], "dtype": "float32", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5076560 }] } + }, + { + "name": "query-long-value-stress-float32-d31-peaked", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 62], "dtype": "float32", "dist": "normal", "seed": 15351, "scale": 3 }, + "keyT": { "shape": [2, 2047, 62], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 3 }, + "valueT": { "shape": [2, 2047, 62], "dtype": "float32", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5076560 }] } + }, + { + "name": "query-long-value-stress-float32-d31-negative-scale", + "preset": "model", + "attrs": { "num_heads": 2, "scale": -0.37 }, + "inputs": { + "queryT": { "shape": [2, 5, 62], "dtype": "float32", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 62], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 62], "dtype": "float32", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5076560 }] } + }, + { + "name": "query-long-value-stress-float32-d31-constant-value", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 62], "dtype": "float32", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 62], "dtype": "float32", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 62], "dtype": "float32", "dist": "constant", "value": 0.25 } + }, + "outputs": { "outputT": { "shape": [2, 5, 62], "dtype": "float32" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5076560 }] } + }, + { + "name": "query-long-value-stress-float16-d1-uniform-probability", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 2], "dtype": "float16", "dist": "constant", "value": 0 }, + "keyT": { "shape": [2, 2047, 2], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 2], "dtype": "float16", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 2], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163760 }] } + }, + { + "name": "query-long-value-stress-float16-d1-peaked", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 2], "dtype": "float16", "dist": "normal", "seed": 15351, "scale": 3 }, + "keyT": { "shape": [2, 2047, 2], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 3 }, + "valueT": { "shape": [2, 2047, 2], "dtype": "float16", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 2], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163760 }] } + }, + { + "name": "query-long-value-stress-float16-d1-negative-scale", + "preset": "model", + "attrs": { "num_heads": 2, "scale": -0.37 }, + "inputs": { + "queryT": { "shape": [2, 5, 2], "dtype": "float16", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 2], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 2], "dtype": "float16", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 2], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163760 }] } + }, + { + "name": "query-long-value-stress-float16-d1-constant-value", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 2], "dtype": "float16", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 2], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 2], "dtype": "float16", "dist": "constant", "value": 0.25 } + }, + "outputs": { "outputT": { "shape": [2, 5, 2], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163760 }] } + }, + { + "name": "query-long-value-stress-float16-d6-uniform-probability", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 12], "dtype": "float16", "dist": "constant", "value": 0 }, + "keyT": { "shape": [2, 2047, 12], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 12], "dtype": "float16", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 982560 }] } + }, + { + "name": "query-long-value-stress-float16-d6-peaked", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 12], "dtype": "float16", "dist": "normal", "seed": 15351, "scale": 3 }, + "keyT": { "shape": [2, 2047, 12], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 3 }, + "valueT": { "shape": [2, 2047, 12], "dtype": "float16", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 982560 }] } + }, + { + "name": "query-long-value-stress-float16-d6-negative-scale", + "preset": "model", + "attrs": { "num_heads": 2, "scale": -0.37 }, + "inputs": { + "queryT": { "shape": [2, 5, 12], "dtype": "float16", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 12], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 12], "dtype": "float16", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 982560 }] } + }, + { + "name": "query-long-value-stress-float16-d6-constant-value", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 12], "dtype": "float16", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 12], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 12], "dtype": "float16", "dist": "constant", "value": 0.25 } + }, + "outputs": { "outputT": { "shape": [2, 5, 12], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 982560 }] } + }, + { + "name": "query-long-value-stress-float16-d31-uniform-probability", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 62], "dtype": "float16", "dist": "constant", "value": 0 }, + "keyT": { "shape": [2, 2047, 62], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 62], "dtype": "float16", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5076560 }] } + }, + { + "name": "query-long-value-stress-float16-d31-peaked", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 62], "dtype": "float16", "dist": "normal", "seed": 15351, "scale": 3 }, + "keyT": { "shape": [2, 2047, 62], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 3 }, + "valueT": { "shape": [2, 2047, 62], "dtype": "float16", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5076560 }] } + }, + { + "name": "query-long-value-stress-float16-d31-negative-scale", + "preset": "model", + "attrs": { "num_heads": 2, "scale": -0.37 }, + "inputs": { + "queryT": { "shape": [2, 5, 62], "dtype": "float16", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 62], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 62], "dtype": "float16", "dist": "normal", "seed": 15353, "scale": 1 } + }, + "outputs": { "outputT": { "shape": [2, 5, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5076560 }] } + }, + { + "name": "query-long-value-stress-float16-d31-constant-value", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "shape": [2, 5, 62], "dtype": "float16", "dist": "normal", "seed": 15351, "scale": 0.2 }, + "keyT": { "shape": [2, 2047, 62], "dtype": "float16", "dist": "normal", "seed": 15352, "scale": 0.2 }, + "valueT": { "shape": [2, 2047, 62], "dtype": "float16", "dist": "constant", "value": 0.25 } + }, + "outputs": { "outputT": { "shape": [2, 5, 62], "dtype": "float16" } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5076560 }] } + }, + { + "name": "capacity-float32-b2-q1-kv2048-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2048, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2048, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 32768 }] } + }, + { + "name": "capacity-float32-b2-q1-kv2049-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 32784 }] } + }, + { + "name": "capacity-float32-b2-q1-kv4032-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4032, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4032, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 64512 }] } + }, + { + "name": "capacity-float32-b2-q1-kv4033-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 64528 }] } + }, + { + "name": "capacity-float32-b2-q1-kv8128-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 130048 }] } + }, + { + "name": "capacity-float32-b2-q1-kv8129-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8129, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8129, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 130064 }] } + }, + { + "name": "capacity-float32-b2-q5-kv2048-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2048, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2048, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163840 }] } + }, + { + "name": "capacity-float32-b2-q5-kv2049-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163920 }] } + }, + { + "name": "capacity-float32-b2-q5-kv4032-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4032, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4032, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 322560 }] } + }, + { + "name": "capacity-float32-b2-q5-kv4033-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 322640 }] } + }, + { + "name": "capacity-float32-b2-q5-kv8128-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 650240 }] } + }, + { + "name": "capacity-float32-b2-q5-kv8129-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8129, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8129, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 650320 }] } + }, + { + "name": "capacity-float32-b2-q1-kv2048-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2048, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2048, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 196608 }] } + }, + { + "name": "capacity-float32-b2-q1-kv2049-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 196704 }] } + }, + { + "name": "capacity-float32-b2-q1-kv4032-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4032, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4032, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 387072 }] } + }, + { + "name": "capacity-float32-b2-q1-kv4033-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 387168 }] } + }, + { + "name": "capacity-float32-b2-q1-kv8128-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 780288 }] } + }, + { + "name": "capacity-float32-b2-q1-kv8129-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8129, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8129, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 780384 }] } + }, + { + "name": "capacity-float32-b2-q5-kv2048-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2048, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2048, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 983040 }] } + }, + { + "name": "capacity-float32-b2-q5-kv2049-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 983520 }] } + }, + { + "name": "capacity-float32-b2-q5-kv4032-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4032, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4032, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1935360 }] } + }, + { + "name": "capacity-float32-b2-q5-kv4033-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1935840 }] } + }, + { + "name": "capacity-float32-b2-q5-kv8128-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901440 }] } + }, + { + "name": "capacity-float32-b2-q5-kv8129-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8129, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8129, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901920 }] } + }, + { + "name": "capacity-float32-b2-q1-kv2048-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2048, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2048, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1015808 }] } + }, + { + "name": "capacity-float32-b2-q1-kv2049-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1016304 }] } + }, + { + "name": "capacity-float32-b2-q1-kv4032-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4032, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4032, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1999872 }] } + }, + { + "name": "capacity-float32-b2-q1-kv4033-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2000368 }] } + }, + { + "name": "capacity-float32-b2-q1-kv8128-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4031488 }] } + }, + { + "name": "capacity-float32-b2-q1-kv8129-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8129, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8129, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4031984 }] } + }, + { + "name": "capacity-float32-b2-q5-kv2048-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2048, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2048, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5079040 }] } + }, + { + "name": "capacity-float32-b2-q5-kv2049-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5081520 }] } + }, + { + "name": "capacity-float32-b2-q5-kv4032-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4032, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4032, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9999360 }] } + }, + { + "name": "capacity-float32-b2-q5-kv4033-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10001840 }] } + }, + { + "name": "capacity-float32-b2-q5-kv8128-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20157440 }] } + }, + { + "name": "capacity-float32-b2-q5-kv8129-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8129, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8129, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20159920 }] } + }, + { + "name": "capacity-float16-b2-q1-kv2048-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2048, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2048, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 32768 }] } + }, + { + "name": "capacity-float16-b2-q1-kv2049-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 32784 }] } + }, + { + "name": "capacity-float16-b2-q1-kv4032-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4032, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4032, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 64512 }] } + }, + { + "name": "capacity-float16-b2-q1-kv4033-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 64528 }] } + }, + { + "name": "capacity-float16-b2-q1-kv8128-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 130048 }] } + }, + { + "name": "capacity-float16-b2-q1-kv8129-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8129, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8129, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 130064 }] } + }, + { + "name": "capacity-float16-b2-q5-kv2048-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2048, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2048, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163840 }] } + }, + { + "name": "capacity-float16-b2-q5-kv2049-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 163920 }] } + }, + { + "name": "capacity-float16-b2-q5-kv4032-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4032, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4032, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 322560 }] } + }, + { + "name": "capacity-float16-b2-q5-kv4033-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 322640 }] } + }, + { + "name": "capacity-float16-b2-q5-kv8128-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 650240 }] } + }, + { + "name": "capacity-float16-b2-q5-kv8129-h2-d1", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 2], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8129, 2], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8129, 2], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 2] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 650320 }] } + }, + { + "name": "capacity-float16-b2-q1-kv2048-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2048, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2048, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 196608 }] } + }, + { + "name": "capacity-float16-b2-q1-kv2049-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 196704 }] } + }, + { + "name": "capacity-float16-b2-q1-kv4032-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4032, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4032, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 387072 }] } + }, + { + "name": "capacity-float16-b2-q1-kv4033-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 387168 }] } + }, + { + "name": "capacity-float16-b2-q1-kv8128-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 780288 }] } + }, + { + "name": "capacity-float16-b2-q1-kv8129-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8129, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8129, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 780384 }] } + }, + { + "name": "capacity-float16-b2-q5-kv2048-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2048, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2048, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 983040 }] } + }, + { + "name": "capacity-float16-b2-q5-kv2049-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 983520 }] } + }, + { + "name": "capacity-float16-b2-q5-kv4032-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4032, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4032, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1935360 }] } + }, + { + "name": "capacity-float16-b2-q5-kv4033-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1935840 }] } + }, + { + "name": "capacity-float16-b2-q5-kv8128-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901440 }] } + }, + { + "name": "capacity-float16-b2-q5-kv8129-h2-d6", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8129, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8129, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901920 }] } + }, + { + "name": "capacity-float16-b2-q1-kv2048-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2048, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2048, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1015808 }] } + }, + { + "name": "capacity-float16-b2-q1-kv2049-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1016304 }] } + }, + { + "name": "capacity-float16-b2-q1-kv4032-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4032, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4032, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1999872 }] } + }, + { + "name": "capacity-float16-b2-q1-kv4033-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2000368 }] } + }, + { + "name": "capacity-float16-b2-q1-kv8128-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4031488 }] } + }, + { + "name": "capacity-float16-b2-q1-kv8129-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8129, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8129, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4031984 }] } + }, + { + "name": "capacity-float16-b2-q5-kv2048-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2048, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2048, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5079040 }] } + }, + { + "name": "capacity-float16-b2-q5-kv2049-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5081520 }] } + }, + { + "name": "capacity-float16-b2-q5-kv4032-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4032, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4032, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9999360 }] } + }, + { + "name": "capacity-float16-b2-q5-kv4033-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10001840 }] } + }, + { + "name": "capacity-float16-b2-q5-kv8128-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20157440 }] } + }, + { + "name": "capacity-float16-b2-q5-kv8129-h2-d31", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8129, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8129, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20159920 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv2049-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 98352 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv4033-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 193584 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv8128-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 390144 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv2049-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 491760 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv4033-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 967920 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1950720 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv2049-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 262272 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv4033-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 516224 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv8128-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1040384 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv2049-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1311360 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv4033-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2581120 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5201920 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv2049-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 491760 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv4033-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 967920 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv8128-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1950720 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv2049-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2458800 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv4033-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4839600 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9753600 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv2049-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 524544 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv4033-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1032448 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv8128-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2080768 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv2049-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2622720 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv4033-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5162240 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10403840 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv2049-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 786816 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv4033-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1548672 }] } + }, + { + "name": "capacity-general-float32-b2-q1-kv8128-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 1, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 1, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3121152 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv2049-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3934080 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv4033-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7743360 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 15605760 }] } + }, + { + "name": "capacity-general-float32-b2-q31-kv2049-h2-d6-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 31, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 31, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 6097824 }] } + }, + { + "name": "capacity-general-float32-b2-q31-kv4033-h2-d6-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 31, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 31, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 12002208 }] } + }, + { + "name": "capacity-general-float32-b2-q128-kv2049-h2-d6-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 128, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 128, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 25178112 }] } + }, + { + "name": "capacity-general-float32-b2-q128-kv4033-h2-d6-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 128, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 128, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 49557504 }] } + }, + { + "name": "capacity-general-float32-b2-q31-kv2049-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 31, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 31, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 8130432 }] } + }, + { + "name": "capacity-general-float32-b2-q31-kv4033-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 31, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 31, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 16002944 }] } + }, + { + "name": "capacity-general-float32-b2-q128-kv2049-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 128, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 128, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 33570816 }] } + }, + { + "name": "capacity-general-float32-b2-q128-kv4033-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 128, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 128, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 66076672 }] } + }, + { + "name": "capacity-general-float32-b2-q31-kv2049-h2-d31-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 31, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 31, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 31505424 }] } + }, + { + "name": "capacity-general-float32-b2-q31-kv4033-h2-d31-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 31, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 31, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 62011408 }] } + }, + { + "name": "capacity-general-float32-b2-q128-kv2049-h2-d31-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 128, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 2049, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 2049, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 128, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 130086912 }] } + }, + { + "name": "capacity-general-float32-b2-q128-kv4033-h2-d31-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 128, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 4033, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 4033, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 128, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 256047104 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d6-peaked", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 3 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 3 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901440 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d6-uniform", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 12], "dist": "constant", "value": 0 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901440 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d6-negative-scale", + "preset": "model", + "attrs": { "num_heads": 2, "scale": -0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901440 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d6-constant-value", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 12], "dist": "constant", "value": 0.25 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901440 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d31-peaked", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 3 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 3 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20157440 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d31-uniform", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 62], "dist": "constant", "value": 0 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20157440 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d31-negative-scale", + "preset": "model", + "attrs": { "num_heads": 2, "scale": -0.37 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20157440 }] } + }, + { + "name": "capacity-general-float32-b2-q5-kv8128-h2-d31-constant-value", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float32", "shape": [2, 8128, 62], "dist": "constant", "value": 0.25 } + }, + "outputs": { "outputT": { "dtype": "float32", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20157440 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv2049-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 98352 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv4033-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 193584 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv8128-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 390144 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv2049-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 491760 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv4033-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 967920 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d3-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 6], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 6], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 6], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 6] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1950720 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv2049-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 262272 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv4033-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 516224 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv8128-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1040384 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv2049-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1311360 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv4033-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2581120 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5201920 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv2049-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 491760 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv4033-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 967920 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv8128-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1950720 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv2049-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2458800 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv4033-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 4839600 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d15-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 30], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 30], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 30], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 30] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 9753600 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv2049-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 524544 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv4033-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1032448 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv8128-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2080768 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv2049-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 2622720 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv4033-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 5162240 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d16-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 32], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 32], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 32], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 32] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 10403840 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv2049-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 786816 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv4033-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 1548672 }] } + }, + { + "name": "capacity-general-float16-b2-q1-kv8128-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 1, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 1, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3121152 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv2049-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3934080 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv4033-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 7743360 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d24-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 48], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 48], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 48], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 48] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 15605760 }] } + }, + { + "name": "capacity-general-float16-b2-q31-kv2049-h2-d6-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 31, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 31, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 6097824 }] } + }, + { + "name": "capacity-general-float16-b2-q31-kv4033-h2-d6-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 31, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 31, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 12002208 }] } + }, + { + "name": "capacity-general-float16-b2-q128-kv2049-h2-d6-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 128, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 128, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 25178112 }] } + }, + { + "name": "capacity-general-float16-b2-q128-kv4033-h2-d6-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 128, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 128, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 49557504 }] } + }, + { + "name": "capacity-general-float16-b2-q31-kv2049-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 31, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 31, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 8130432 }] } + }, + { + "name": "capacity-general-float16-b2-q31-kv4033-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 31, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 31, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 16002944 }] } + }, + { + "name": "capacity-general-float16-b2-q128-kv2049-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 128, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 128, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 33570816 }] } + }, + { + "name": "capacity-general-float16-b2-q128-kv4033-h2-d8-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 128, 16], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 16], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 16], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 128, 16] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 66076672 }] } + }, + { + "name": "capacity-general-float16-b2-q31-kv2049-h2-d31-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 31, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 31, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 31505424 }] } + }, + { + "name": "capacity-general-float16-b2-q31-kv4033-h2-d31-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 31, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 31, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 62011408 }] } + }, + { + "name": "capacity-general-float16-b2-q128-kv2049-h2-d31-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 128, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 2049, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 2049, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 128, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 130086912 }] } + }, + { + "name": "capacity-general-float16-b2-q128-kv4033-h2-d31-ordinary", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 128, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 4033, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 4033, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 128, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 256047104 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d6-peaked", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 3 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 3 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901440 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d6-uniform", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 12], "dist": "constant", "value": 0 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901440 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d6-negative-scale", + "preset": "model", + "attrs": { "num_heads": 2, "scale": -0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901440 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d6-constant-value", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 12], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 12], "dist": "constant", "value": 0.25 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 12] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 3901440 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d31-peaked", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 3 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 3 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20157440 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d31-uniform", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 62], "dist": "constant", "value": 0 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20157440 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d31-negative-scale", + "preset": "model", + "attrs": { "num_heads": 2, "scale": -0.37 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17103, "scale": 1 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20157440 }] } + }, + { + "name": "capacity-general-float16-b2-q5-kv8128-h2-d31-constant-value", + "preset": "model", + "attrs": { "num_heads": 2, "scale": 0 }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [2, 5, 62], "dist": "normal", "seed": 17101, "scale": 0.2 }, + "keyT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "normal", "seed": 17102, "scale": 0.2 }, + "valueT": { "dtype": "float16", "shape": [2, 8128, 62], "dist": "constant", "value": 0.25 } + }, + "outputs": { "outputT": { "dtype": "float16", "shape": [2, 5, 62] } }, + "bench": { "metrics": [{ "type": "gflops", "value": 20157440 }] } } ] }