{ "model": "microsoft/phi-4", "revision": "2db69c1c3e91a05d2c64a3185acfbaf36f744e25", "flipquant_model_key": "phi4-14b", "evaluation": { "mode": "native sm_120 kernels (FlipQuant evaluation.ppl --mode native)", "convention": "FlipQuant paper: per-token (row) FourOverSix activations", "quantized_modules": 160, "transformers": "5.16.1", "torch": "2.9.0+cu128", "gpu": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", "corpora": { "wiki": { "dataset": "Salesforce/wikitext, wikitext-2-raw-v1, test split", "revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "total_tokens": 289305, "used_tokens": 288768, "windows": 141, "window_tokens": 2048 }, "c4": { "dataset": "allenai/c4, validation shard en/c4-validation.00000-of-00008.json.gz", "revision": "1588ec454efa1a09f29cd18ddd04fe05fc8653a2", "path": "en/c4-validation.00000-of-00008.json.gz", "seed": 0, "windows": 256, "window_tokens": 2048 } }, "c4_documents": "the documents are drawn with the recorded seed from the listed shard" }, "ppl": { "bf16": { "wiki": 6.46151240363403, "c4": 10.3098094444712 }, "fo6": { "wiki": 6.661734651161868, "c4": 10.544115699896242 }, "flipquant_8x64": { "wiki": 6.63447200402568, "c4": 10.518660263628771 }, "flipquant_16x64": { "wiki": 6.632533270171243, "c4": 10.515677655076827 }, "flipquant_256x64": { "wiki": 6.638175028125632, "c4": 10.51814315351229 } }, "paired_vs_fourover6": { "bf16": { "c4": { "delta_nll": -0.022472135489806533, "two_se": 0.0013891787909336083, "n": 256, "significant": true, "verdict": "better" }, "wiki": { "delta_nll": -0.03051650016865832, "two_se": 0.002170510242467142, "n": 141, "significant": true, "verdict": "better" } }, "flipquant_8x64": { "c4": { "delta_nll": -0.0024171029217541218, "two_se": 0.0008386692348550485, "n": 256, "significant": true, "verdict": "better" }, "wiki": { "delta_nll": -0.004100821542401685, "two_se": 0.0014164293893722355, "n": 141, "significant": true, "verdict": "better" } }, "flipquant_16x64": { "c4": { "delta_nll": -0.0027006971649825573, "two_se": 0.0008759992843983449, "n": 256, "significant": true, "verdict": "better" }, "wiki": { "delta_nll": -0.0043930855203182136, "two_se": 0.0013564826324927554, "n": 141, "significant": true, "verdict": "better" } }, "flipquant_256x64": { "c4": { "delta_nll": -0.00246626534499228, "two_se": 0.0007803954698395105, "n": 256, "significant": true, "verdict": "better" }, "wiki": { "delta_nll": -0.0035428282216931066, "two_se": 0.001415833063163873, "n": 141, "significant": true, "verdict": "better" } } }, "maps": { "flipquant_8x64.pt": { "sha256": "9131bff768fce81dbb5e4ad559d1c3dc95c356145a5c705f301aacb9c2a58e94", "e0m3_tiles": 129730, "tiles": 26624000 }, "flipquant_16x64.pt": { "sha256": "8ea582050b40943a5abca4c44dc3edd0f86922f422fe77991ba7b04d75cb08fb", "e0m3_tiles": 79591, "tiles": 13312000 }, "flipquant_256x64.pt": { "sha256": "145f060338481ac66aff4e9f73842c1f329ca731d23db88cbf2e2d194c8e38f7", "e0m3_tiles": 11059, "tiles": 832000 } }, "paper_setting_maps_ppl": { "8x64": { "wiki": 6.607846736907959, "c4": 10.497916221618652 }, "16x64": { "wiki": 6.613272666931152, "c4": 10.504680633544922 }, "256x64": { "wiki": 6.625688076019287, "c4": 10.514570236206055 } }, "paper_setting_note": "maps calibrated at the paper's settings (20 epochs, 128 windows, full-vocabulary teacher), evaluated under the same convention; for comparison only" }