File size: 2,512 Bytes
2bff5be | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | {
"q4": {
"medians": {
"32": {
"prefill_speedup": 1.1565667783539286,
"decode_speedup": 1.1574511540902834,
"reference_top1": 31,
"decode_steps": 32
},
"256": {
"prefill_speedup": 1.1330960539402803,
"decode_speedup": 1.2261791684237928,
"reference_top1": 32,
"decode_steps": 32
},
"896": {
"prefill_speedup": 1.3813925427562908,
"decode_speedup": 1.6319104406097984,
"reference_top1": 32,
"decode_steps": 32
}
},
"followup": {
"fmt": "q4",
"prompt": 770,
"reused": 757,
"warmMs": 143.26299599999948,
"coldMs": 8303.308422999999,
"ratio": 57.95850048396328,
"exact": true
},
"fp32_before": {
"n": 1308,
"modelNll": 2.5563980281587257,
"fullNll": 2.5285575156082536,
"coarseKl": 0.0350820307014913,
"top1": 0.8899082568807339
},
"fp32_after": {
"n": 1308,
"modelNll": 2.5561584536429423,
"fullNll": 2.5285575156082536,
"coarseKl": 0.03506060461156311,
"top1": 0.8906727828746177
},
"weights": "b87dd88be8872fad5a7623b0f9c76721043dfdad182f9303750da888911ec83e",
"node": "v24.21.0"
},
"q8": {
"medians": {
"32": {
"prefill_speedup": 0.9555009223589261,
"decode_speedup": 1.1868181229684265,
"reference_top1": 32,
"decode_steps": 32
},
"256": {
"prefill_speedup": 1.367436934903951,
"decode_speedup": 1.2760796271775725,
"reference_top1": 32,
"decode_steps": 32
},
"896": {
"prefill_speedup": 1.402068146663775,
"decode_speedup": 1.6925790537638363,
"reference_top1": 32,
"decode_steps": 32
}
},
"followup": {
"fmt": "q8",
"prompt": 770,
"reused": 757,
"warmMs": 165.48757300000034,
"coldMs": 8979.35544,
"ratio": 54.25999836253554,
"exact": true
},
"fp32_before": {
"n": 1308,
"modelNll": 2.5275877610792397,
"fullNll": 2.5285575156082536,
"coarseKl": 0.000283973873353186,
"top1": 0.981651376146789
},
"fp32_after": {
"n": 1308,
"modelNll": 2.5272502054028125,
"fullNll": 2.5285575156082536,
"coarseKl": 0.000290142925742153,
"top1": 0.9801223241590215
},
"weights": "fe5f24b98b333e23112ccbe885516f9557ab6bb0182540cce2ef82ed80362af7",
"node": "v24.21.0"
}
}
|