phi-4-FlipQuant / ppl_summary.json
edgeai-lab's picture
FlipQuant MixFP4 maps (8x64, 16x64, 256x64), model card, PPL summary and license files
548b81d verified
Raw History Blame Contribute Delete
3.8 kB
{
"model": "microsoft/phi-4",
"revision": "2db69c1c3e91a05d2c64a3185acfbaf36f744e25",
"flipquant_model_key": "phi4-14b",
"evaluation": {
"mode": "native sm_120 kernels (FlipQuant evaluation.ppl --mode native)",
"convention": "FlipQuant paper: per-token (row) FourOverSix activations",
"quantized_modules": 160,
"transformers": "5.16.1",
"torch": "2.9.0+cu128",
"gpu": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition",
"corpora": {
"wiki": {
"dataset": "Salesforce/wikitext, wikitext-2-raw-v1, test split",
"revision": "b08601e04326c79dfdd32d625aee71d232d685c3",
"total_tokens": 289305,
"used_tokens": 288768,
"windows": 141,
"window_tokens": 2048
},
"c4": {
"dataset": "allenai/c4, validation shard en/c4-validation.00000-of-00008.json.gz",
"revision": "1588ec454efa1a09f29cd18ddd04fe05fc8653a2",
"path": "en/c4-validation.00000-of-00008.json.gz",
"seed": 0,
"windows": 256,
"window_tokens": 2048
}
},
"c4_documents": "the documents are drawn with the recorded seed from the listed shard"
},
"ppl": {
"bf16": {
"wiki": 6.46151240363403,
"c4": 10.3098094444712
},
"fo6": {
"wiki": 6.661734651161868,
"c4": 10.544115699896242
},
"flipquant_8x64": {
"wiki": 6.63447200402568,
"c4": 10.518660263628771
},
"flipquant_16x64": {
"wiki": 6.632533270171243,
"c4": 10.515677655076827
},
"flipquant_256x64": {
"wiki": 6.638175028125632,
"c4": 10.51814315351229
}
},
"paired_vs_fourover6": {
"bf16": {
"c4": {
"delta_nll": -0.022472135489806533,
"two_se": 0.0013891787909336083,
"n": 256,
"significant": true,
"verdict": "better"
},
"wiki": {
"delta_nll": -0.03051650016865832,
"two_se": 0.002170510242467142,
"n": 141,
"significant": true,
"verdict": "better"
}
},
"flipquant_8x64": {
"c4": {
"delta_nll": -0.0024171029217541218,
"two_se": 0.0008386692348550485,
"n": 256,
"significant": true,
"verdict": "better"
},
"wiki": {
"delta_nll": -0.004100821542401685,
"two_se": 0.0014164293893722355,
"n": 141,
"significant": true,
"verdict": "better"
}
},
"flipquant_16x64": {
"c4": {
"delta_nll": -0.0027006971649825573,
"two_se": 0.0008759992843983449,
"n": 256,
"significant": true,
"verdict": "better"
},
"wiki": {
"delta_nll": -0.0043930855203182136,
"two_se": 0.0013564826324927554,
"n": 141,
"significant": true,
"verdict": "better"
}
},
"flipquant_256x64": {
"c4": {
"delta_nll": -0.00246626534499228,
"two_se": 0.0007803954698395105,
"n": 256,
"significant": true,
"verdict": "better"
},
"wiki": {
"delta_nll": -0.0035428282216931066,
"two_se": 0.001415833063163873,
"n": 141,
"significant": true,
"verdict": "better"
}
}
},
"maps": {
"flipquant_8x64.pt": {
"sha256": "9131bff768fce81dbb5e4ad559d1c3dc95c356145a5c705f301aacb9c2a58e94",
"e0m3_tiles": 129730,
"tiles": 26624000
},
"flipquant_16x64.pt": {
"sha256": "8ea582050b40943a5abca4c44dc3edd0f86922f422fe77991ba7b04d75cb08fb",
"e0m3_tiles": 79591,
"tiles": 13312000
},
"flipquant_256x64.pt": {
"sha256": "145f060338481ac66aff4e9f73842c1f329ca731d23db88cbf2e2d194c8e38f7",
"e0m3_tiles": 11059,
"tiles": 832000
}
},
"paper_setting_maps_ppl": {
"8x64": {
"wiki": 6.607846736907959,
"c4": 10.497916221618652
},
"16x64": {
"wiki": 6.613272666931152,
"c4": 10.504680633544922
},
"256x64": {
"wiki": 6.625688076019287,
"c4": 10.514570236206055
}
},
"paper_setting_note": "maps calibrated at the paper's settings (20 epochs, 128 windows, full-vocabulary teacher), evaluated under the same convention; for comparison only"
}