Document routing attribution and validated per-layer alternative; preserve original weights
8e55f71 verified | { | |
| "complete": true, | |
| "split": "validation", | |
| "provenance": { | |
| "source": { | |
| "repo": "RecursiveSI/flint", | |
| "revision": "2a247be0c0aec895ccb0ac7ca92229ec6df37288" | |
| }, | |
| "scripts": { | |
| "flint_quantize.py": "6cdb630599183e25e416d0e4c44fdab68acdff2ed6d8cde656beaab80c2283ae", | |
| "flint_reference.py": "2534d51af8e99699692f1a2cf5fc022bd7beaa7de07770dbffb9690aa4f62ff6", | |
| "flint_nvfp4.py": "31aec4a310b7bddaa90da083b80352e5607ce754c10052458859f7306b5cc566" | |
| }, | |
| "packages": { | |
| "torch": "2.8.0+cu128", | |
| "triton": "3.4.0", | |
| "safetensors": "0.8.0", | |
| "transformers": "5.20.2", | |
| "huggingface_hub": "2.2.0", | |
| "datasets": "5.1.0" | |
| }, | |
| "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", | |
| "cuda": "12.8", | |
| "created_utc": "2026-10-21T19:16:32Z" | |
| }, | |
| "script_sha256": "e0e0aa655a97d9c78837196a94b280c54944ab5542686242dbaa7604d33dca1c", | |
| "normalization_script_sha256": "f06c3d51f9018814648a1fe42c39274dd103528b30185d914f76adbd68fef3ac", | |
| "flashinfer_version": "0.6.16.post1", | |
| "corpus_sha256": "b6fdc596cb03b87ddea6a2f07453d27574c415fdb7dd034a61abf8146f9eb573", | |
| "results": { | |
| "bf16": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "3": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "4": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "5": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "6": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "7": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "8": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "9": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "10": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "11": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "12": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "13": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "14": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "15": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "16": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "17": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "18": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "19": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "20": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "21": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "22": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "23": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "24": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "25": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "26": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "27": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "28": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "29": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "30": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "31": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "32": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "33": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "34": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "35": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "36": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "37": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "38": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "39": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "40": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "41": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "42": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "43": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "44": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "45": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "46": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "47": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "48": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| } | |
| }, | |
| "tensor_bytes": 79570574912, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.1118895109538194, | |
| "teacher_nll": 2.1118895109538194, | |
| "kl": 0.0, | |
| "top1_agree": 1.0, | |
| "perplexity": 8.26384116031041, | |
| "teacher_perplexity": 8.26384116031041, | |
| "delta_nll": 0.0 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.3611519390191786, | |
| "teacher_nll": 1.3611519390191786, | |
| "kl": 0.0, | |
| "top1_agree": 1.0, | |
| "perplexity": 3.9006840649318666, | |
| "teacher_perplexity": 3.9006840649318666, | |
| "delta_nll": 0.0 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.4229323645068903, | |
| "teacher_nll": 1.4229323645068903, | |
| "kl": 0.0, | |
| "top1_agree": 1.0, | |
| "perplexity": 4.149269789846104, | |
| "teacher_perplexity": 4.149269789846104, | |
| "delta_nll": 0.0 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.551584229335389, | |
| "teacher_nll": 3.551584229335389, | |
| "kl": 0.0, | |
| "top1_agree": 1.0, | |
| "perplexity": 34.868513476436284, | |
| "teacher_perplexity": 34.868513476436284, | |
| "delta_nll": 0.0 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-policy-validation-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.730275568886409, | |
| "teacher_nll": 3.730275568886409, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "prose-policy-validation-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4023956112518126, | |
| "teacher_nll": 3.4023956112518126, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "prose-policy-validation-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5666212368360806, | |
| "teacher_nll": 3.5666212368360806, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "prose-policy-validation-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5070445003672543, | |
| "teacher_nll": 3.5070445003672543, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "code-policy-validation-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4573161135663042, | |
| "teacher_nll": 1.4573161135663042, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "code-policy-validation-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.2908055956110414, | |
| "teacher_nll": 1.2908055956110414, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "code-policy-validation-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.255632556547321, | |
| "teacher_nll": 1.255632556547321, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "code-policy-validation-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4408534903520485, | |
| "teacher_nll": 1.4408534903520485, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "math-policy-validation-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.482822512299441, | |
| "teacher_nll": 1.482822512299441, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "math-policy-validation-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.408211328462221, | |
| "teacher_nll": 1.408211328462221, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "math-policy-validation-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4567602635012151, | |
| "teacher_nll": 1.4567602635012151, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "math-policy-validation-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.3439353537646843, | |
| "teacher_nll": 1.3439353537646843, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 9683609088 | |
| } | |
| }, | |
| "preferred-fp8-45-48": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "3": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "4": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "5": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "6": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "7": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "8": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "9": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "10": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "11": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "12": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "13": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "14": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "15": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "16": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "17": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "18": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "19": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "20": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "21": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "22": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "23": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "24": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "25": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "26": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "27": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "28": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "29": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "30": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "31": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "32": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "33": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "34": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "35": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "36": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "37": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "38": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "39": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "40": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "41": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "42": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "43": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "44": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 76350135872, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.1111584732335515, | |
| "teacher_nll": 2.1118895109538194, | |
| "kl": 0.0007934031345190516, | |
| "top1_agree": 0.9902930402930403, | |
| "perplexity": 8.257802188335017, | |
| "teacher_perplexity": 8.26384116031041, | |
| "delta_nll": -0.0007310377202678353 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.3587105913039965, | |
| "teacher_nll": 1.3611519390191786, | |
| "kl": 0.0006417182103448726, | |
| "top1_agree": 0.9924297924297925, | |
| "perplexity": 3.891172753735319, | |
| "teacher_perplexity": 3.9006840649318666, | |
| "delta_nll": -0.0024413477151821183 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.4229313316508236, | |
| "teacher_nll": 1.4229323645068903, | |
| "kl": 0.0005105946148616081, | |
| "top1_agree": 0.9940781440781441, | |
| "perplexity": 4.1492655042498425, | |
| "teacher_perplexity": 4.149269789846104, | |
| "delta_nll": -1.032856066673915e-06 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.551833496745835, | |
| "teacher_nll": 3.551584229335389, | |
| "kl": 0.001227896578350674, | |
| "top1_agree": 0.9843711843711843, | |
| "perplexity": 34.87720614385099, | |
| "teacher_perplexity": 34.868513476436284, | |
| "delta_nll": 0.00024926741044595246 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-policy-validation-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.7294776180579405, | |
| "teacher_nll": 3.730275568886409, | |
| "kl": 0.0015413437680010394, | |
| "top1_agree": 0.9841269841269841 | |
| }, | |
| { | |
| "id": "prose-policy-validation-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4025479271298362, | |
| "teacher_nll": 3.4023956112518126, | |
| "kl": 0.001127076467842555, | |
| "top1_agree": 0.9846153846153847 | |
| }, | |
| { | |
| "id": "prose-policy-validation-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5673533270792794, | |
| "teacher_nll": 3.5666212368360806, | |
| "kl": 0.001155838767869423, | |
| "top1_agree": 0.9829059829059829 | |
| }, | |
| { | |
| "id": "prose-policy-validation-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.507955114716284, | |
| "teacher_nll": 3.5070445003672543, | |
| "kl": 0.001087327309689679, | |
| "top1_agree": 0.9858363858363859 | |
| }, | |
| { | |
| "id": "code-policy-validation-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4538545538654257, | |
| "teacher_nll": 1.4573161135663042, | |
| "kl": 0.0006527559892575783, | |
| "top1_agree": 0.9934065934065934 | |
| }, | |
| { | |
| "id": "code-policy-validation-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.290072934415023, | |
| "teacher_nll": 1.2908055956110414, | |
| "kl": 0.0007901271593658043, | |
| "top1_agree": 0.9916971916971917 | |
| }, | |
| { | |
| "id": "code-policy-validation-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.2532000170231448, | |
| "teacher_nll": 1.255632556547321, | |
| "kl": 0.0005299052457815271, | |
| "top1_agree": 0.9921855921855922 | |
| }, | |
| { | |
| "id": "code-policy-validation-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4377148599123926, | |
| "teacher_nll": 1.4408534903520485, | |
| "kl": 0.0005940844469745809, | |
| "top1_agree": 0.9924297924297925 | |
| }, | |
| { | |
| "id": "math-policy-validation-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4838312496050228, | |
| "teacher_nll": 1.482822512299441, | |
| "kl": 0.00048286072080852566, | |
| "top1_agree": 0.9946275946275946 | |
| }, | |
| { | |
| "id": "math-policy-validation-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4076686153481732, | |
| "teacher_nll": 1.408211328462221, | |
| "kl": 0.000718450903783351, | |
| "top1_agree": 0.9926739926739927 | |
| }, | |
| { | |
| "id": "math-policy-validation-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4570507105453547, | |
| "teacher_nll": 1.4567602635012151, | |
| "kl": 0.0004748969137559444, | |
| "top1_agree": 0.9946275946275946 | |
| }, | |
| { | |
| "id": "math-policy-validation-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.3431747511047438, | |
| "teacher_nll": 1.3439353537646843, | |
| "kl": 0.0003661699210986113, | |
| "top1_agree": 0.9943833943833944 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 9683609088 | |
| } | |
| }, | |
| "conservative-nvfp4": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "3": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "4": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "5": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "6": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "7": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "8": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "9": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "10": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "11": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "12": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "13": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "14": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "15": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "16": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "17": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "18": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "19": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "20": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "21": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "22": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "23": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "24": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "25": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "26": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "27": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "28": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "29": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "30": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "31": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "32": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "33": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "34": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "35": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "36": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "37": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "38": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "39": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "40": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "41": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "42": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "43": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "44": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "45": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "46": { | |
| "gate": "bf16", | |
| "up": "fp8", | |
| "down": "bf16" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "nvfp4", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 77574480448, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.110927108768336, | |
| "teacher_nll": 2.1118895109538194, | |
| "kl": 0.0007221096060126012, | |
| "top1_agree": 0.990964590964591, | |
| "perplexity": 8.255891847348888, | |
| "teacher_perplexity": 8.26384116031041, | |
| "delta_nll": -0.0009624021854834908 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.3591037988226056, | |
| "teacher_nll": 1.3611519390191786, | |
| "kl": 0.0004896586170060144, | |
| "top1_agree": 0.9945054945054945, | |
| "perplexity": 3.8927030929690236, | |
| "teacher_perplexity": 3.9006840649318666, | |
| "delta_nll": -0.0020481401965730672 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.4222320152580812, | |
| "teacher_nll": 1.4229323645068903, | |
| "kl": 0.0004681396002452525, | |
| "top1_agree": 0.9937118437118437, | |
| "perplexity": 4.146364869213906, | |
| "teacher_perplexity": 4.149269789846104, | |
| "delta_nll": -0.000700349248809129 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.551445512224321, | |
| "teacher_nll": 3.551584229335389, | |
| "kl": 0.0012085306007865366, | |
| "top1_agree": 0.9846764346764347, | |
| "perplexity": 34.863676952441665, | |
| "teacher_perplexity": 34.868513476436284, | |
| "delta_nll": -0.00013871711106805407 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-policy-validation-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.728131878565228, | |
| "teacher_nll": 3.730275568886409, | |
| "kl": 0.001546875980565545, | |
| "top1_agree": 0.9843711843711843 | |
| }, | |
| { | |
| "id": "prose-policy-validation-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.402904140672695, | |
| "teacher_nll": 3.4023956112518126, | |
| "kl": 0.0011176317939318725, | |
| "top1_agree": 0.9875457875457876 | |
| }, | |
| { | |
| "id": "prose-policy-validation-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.566692111023471, | |
| "teacher_nll": 3.5666212368360806, | |
| "kl": 0.0010729805890457098, | |
| "top1_agree": 0.9833943833943833 | |
| }, | |
| { | |
| "id": "prose-policy-validation-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.508053918635889, | |
| "teacher_nll": 3.5070445003672543, | |
| "kl": 0.001096634039603019, | |
| "top1_agree": 0.9833943833943833 | |
| }, | |
| { | |
| "id": "code-policy-validation-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.454416419763996, | |
| "teacher_nll": 1.4573161135663042, | |
| "kl": 0.0005280497261597087, | |
| "top1_agree": 0.9938949938949939 | |
| }, | |
| { | |
| "id": "code-policy-validation-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.2902200173109006, | |
| "teacher_nll": 1.2908055956110414, | |
| "kl": 0.0004691741009378153, | |
| "top1_agree": 0.9951159951159951 | |
| }, | |
| { | |
| "id": "code-policy-validation-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.2530585611550653, | |
| "teacher_nll": 1.255632556547321, | |
| "kl": 0.0004382003377895616, | |
| "top1_agree": 0.9943833943833944 | |
| }, | |
| { | |
| "id": "code-policy-validation-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4387201970604604, | |
| "teacher_nll": 1.4408534903520485, | |
| "kl": 0.000523210303136972, | |
| "top1_agree": 0.9946275946275946 | |
| }, | |
| { | |
| "id": "math-policy-validation-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4823827999706525, | |
| "teacher_nll": 1.482822512299441, | |
| "kl": 0.0005088003770185586, | |
| "top1_agree": 0.9946275946275946 | |
| }, | |
| { | |
| "id": "math-policy-validation-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4077888519044905, | |
| "teacher_nll": 1.408211328462221, | |
| "kl": 0.0005599377982168298, | |
| "top1_agree": 0.991941391941392 | |
| }, | |
| { | |
| "id": "math-policy-validation-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4556590544871795, | |
| "teacher_nll": 1.4567602635012151, | |
| "kl": 0.00041746762484512197, | |
| "top1_agree": 0.9929181929181929 | |
| }, | |
| { | |
| "id": "math-policy-validation-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.3430973546700018, | |
| "teacher_nll": 1.3439353537646843, | |
| "kl": 0.0003863526009004995, | |
| "top1_agree": 0.9953601953601954 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 9683609088 | |
| } | |
| }, | |
| "smaller-fp8-39-48": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "3": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "4": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "5": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "6": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "7": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "8": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "9": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "10": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "11": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "12": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "13": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "14": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "15": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "16": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "17": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "18": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "19": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "20": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "21": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "22": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "23": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "24": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "25": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "26": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "27": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "28": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "29": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "30": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "31": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "32": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "33": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "34": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "35": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "36": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "37": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "38": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 71519477312, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.1108048758445643, | |
| "teacher_nll": 2.1118895109538194, | |
| "kl": 0.0029436703867441604, | |
| "top1_agree": 0.9822954822954822, | |
| "perplexity": 8.254882767222709, | |
| "teacher_perplexity": 8.26384116031041, | |
| "delta_nll": -0.00108463510925505 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.3591565501777243, | |
| "teacher_nll": 1.3611519390191786, | |
| "kl": 0.0022441253232654817, | |
| "top1_agree": 0.9860195360195361, | |
| "perplexity": 3.892908443748471, | |
| "teacher_perplexity": 3.9006840649318666, | |
| "delta_nll": -0.0019953888414543375 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.4214664039448794, | |
| "teacher_nll": 1.4229323645068903, | |
| "kl": 0.0019512432102018443, | |
| "top1_agree": 0.9892551892551893, | |
| "perplexity": 4.143191580269336, | |
| "teacher_perplexity": 4.149269789846104, | |
| "delta_nll": -0.0014659605620108707 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.551791673411089, | |
| "teacher_nll": 3.551584229335389, | |
| "kl": 0.004635642626765156, | |
| "top1_agree": 0.9716117216117216, | |
| "perplexity": 34.87574749328646, | |
| "teacher_perplexity": 34.868513476436284, | |
| "delta_nll": 0.00020744407569983636 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-policy-validation-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.7271026909424125, | |
| "teacher_nll": 3.730275568886409, | |
| "kl": 0.005908135372931558, | |
| "top1_agree": 0.9724053724053724 | |
| }, | |
| { | |
| "id": "prose-policy-validation-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4017877338861227, | |
| "teacher_nll": 3.4023956112518126, | |
| "kl": 0.004090809205294529, | |
| "top1_agree": 0.9726495726495726 | |
| }, | |
| { | |
| "id": "prose-policy-validation-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.567241637960022, | |
| "teacher_nll": 3.5666212368360806, | |
| "kl": 0.0043217347036496765, | |
| "top1_agree": 0.9675213675213675 | |
| }, | |
| { | |
| "id": "prose-policy-validation-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.511034630855798, | |
| "teacher_nll": 3.5070445003672543, | |
| "kl": 0.004221891225184859, | |
| "top1_agree": 0.9738705738705739 | |
| }, | |
| { | |
| "id": "code-policy-validation-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4563412683787362, | |
| "teacher_nll": 1.4573161135663042, | |
| "kl": 0.0023459550880250476, | |
| "top1_agree": 0.9858363858363859 | |
| }, | |
| { | |
| "id": "code-policy-validation-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.2917616325711447, | |
| "teacher_nll": 1.2908055956110414, | |
| "kl": 0.0023298357348429705, | |
| "top1_agree": 0.9858363858363859 | |
| }, | |
| { | |
| "id": "code-policy-validation-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.2537396895579802, | |
| "teacher_nll": 1.255632556547321, | |
| "kl": 0.0018885908009295643, | |
| "top1_agree": 0.9851037851037852 | |
| }, | |
| { | |
| "id": "code-policy-validation-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4347836102030362, | |
| "teacher_nll": 1.4408534903520485, | |
| "kl": 0.002412119669264344, | |
| "top1_agree": 0.9873015873015873 | |
| }, | |
| { | |
| "id": "math-policy-validation-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4846272683987833, | |
| "teacher_nll": 1.482822512299441, | |
| "kl": 0.0018748843605756323, | |
| "top1_agree": 0.98998778998779 | |
| }, | |
| { | |
| "id": "math-policy-validation-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4076982973259446, | |
| "teacher_nll": 1.408211328462221, | |
| "kl": 0.0022242125599066273, | |
| "top1_agree": 0.9875457875457876 | |
| }, | |
| { | |
| "id": "math-policy-validation-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4541759463020296, | |
| "teacher_nll": 1.4567602635012151, | |
| "kl": 0.002292718319620332, | |
| "top1_agree": 0.9921855921855922 | |
| }, | |
| { | |
| "id": "math-policy-validation-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.3393641037527604, | |
| "teacher_nll": 1.3439353537646843, | |
| "kl": 0.0014131576007047851, | |
| "top1_agree": 0.9873015873015873 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 9683609088 | |
| } | |
| }, | |
| "mixed-late-projections": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "3": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "4": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "5": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "6": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "7": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "8": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "9": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "10": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "11": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "12": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "13": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "14": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "15": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "16": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "17": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "18": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "19": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "20": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "21": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "22": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "23": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "24": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "25": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "26": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "27": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "28": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "29": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "30": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "31": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "32": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "33": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "34": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "35": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "36": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "37": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "38": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "39": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "40": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "nvfp4", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 73012191808, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.110575082263764, | |
| "teacher_nll": 2.1118895109538194, | |
| "kl": 0.002768995274312409, | |
| "top1_agree": 0.9835571835571836, | |
| "perplexity": 8.252986066085763, | |
| "teacher_perplexity": 8.26384116031041, | |
| "delta_nll": -0.0013144286900552693 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.35876675370412, | |
| "teacher_nll": 1.3611519390191786, | |
| "kl": 0.002667256963742904, | |
| "top1_agree": 0.9891941391941392, | |
| "perplexity": 3.8913912974733766, | |
| "teacher_perplexity": 3.9006840649318666, | |
| "delta_nll": -0.0023851853150587132 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.422199620781364, | |
| "teacher_nll": 1.4229323645068903, | |
| "kl": 0.0017168798307650546, | |
| "top1_agree": 0.9890720390720391, | |
| "perplexity": 4.146230552069269, | |
| "teacher_perplexity": 4.149269789846104, | |
| "delta_nll": -0.0007327437255262303 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5507588723058086, | |
| "teacher_nll": 3.551584229335389, | |
| "kl": 0.003922849028429269, | |
| "top1_agree": 0.9724053724053724, | |
| "perplexity": 34.83974637692444, | |
| "teacher_perplexity": 34.868513476436284, | |
| "delta_nll": -0.0008253570295804202 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-policy-validation-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.72932381542611, | |
| "teacher_nll": 3.730275568886409, | |
| "kl": 0.004572560680946005, | |
| "top1_agree": 0.9709401709401709 | |
| }, | |
| { | |
| "id": "prose-policy-validation-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.401089091877361, | |
| "teacher_nll": 3.4023956112518126, | |
| "kl": 0.0037138127636560155, | |
| "top1_agree": 0.9748473748473748 | |
| }, | |
| { | |
| "id": "prose-policy-validation-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.565276440071972, | |
| "teacher_nll": 3.5666212368360806, | |
| "kl": 0.003613682586122054, | |
| "top1_agree": 0.9709401709401709 | |
| }, | |
| { | |
| "id": "prose-policy-validation-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5073461418477896, | |
| "teacher_nll": 3.5070445003672543, | |
| "kl": 0.0037913400829930007, | |
| "top1_agree": 0.9728937728937729 | |
| }, | |
| { | |
| "id": "code-policy-validation-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4552863815037467, | |
| "teacher_nll": 1.4573161135663042, | |
| "kl": 0.004951566845742629, | |
| "top1_agree": 0.9897435897435898 | |
| }, | |
| { | |
| "id": "code-policy-validation-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.2907511532670557, | |
| "teacher_nll": 1.2908055956110414, | |
| "kl": 0.0021100598511369093, | |
| "top1_agree": 0.9894993894993895 | |
| }, | |
| { | |
| "id": "code-policy-validation-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.2536105592608888, | |
| "teacher_nll": 1.255632556547321, | |
| "kl": 0.0018023577269726185, | |
| "top1_agree": 0.9865689865689866 | |
| }, | |
| { | |
| "id": "code-policy-validation-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4354189207847885, | |
| "teacher_nll": 1.4408534903520485, | |
| "kl": 0.0018050434311194588, | |
| "top1_agree": 0.990964590964591 | |
| }, | |
| { | |
| "id": "math-policy-validation-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4831841726151724, | |
| "teacher_nll": 1.482822512299441, | |
| "kl": 0.0015588753443934542, | |
| "top1_agree": 0.9894993894993895 | |
| }, | |
| { | |
| "id": "math-policy-validation-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4087224536471896, | |
| "teacher_nll": 1.408211328462221, | |
| "kl": 0.002210616383142087, | |
| "top1_agree": 0.9863247863247864 | |
| }, | |
| { | |
| "id": "math-policy-validation-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4548888978504, | |
| "teacher_nll": 1.4567602635012151, | |
| "kl": 0.00180718969958533, | |
| "top1_agree": 0.9921855921855922 | |
| }, | |
| { | |
| "id": "math-policy-validation-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.342002959012694, | |
| "teacher_nll": 1.3439353537646843, | |
| "kl": 0.0012908378959393472, | |
| "top1_agree": 0.9882783882783883 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 9683609088 | |
| } | |
| }, | |
| "all-fp8": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "24": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "31": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "36": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "37": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 41730416192, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.112549294860663, | |
| "teacher_nll": 2.1118895109538194, | |
| "kl": 0.06278559984953898, | |
| "top1_agree": 0.9388888888888889, | |
| "perplexity": 8.269295308798538, | |
| "teacher_perplexity": 8.26384116031041, | |
| "delta_nll": 0.0006597839068436251 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.3609414568459741, | |
| "teacher_nll": 1.3611519390191786, | |
| "kl": 0.07343230811703919, | |
| "top1_agree": 0.9515873015873015, | |
| "perplexity": 3.8998631268723396, | |
| "teacher_perplexity": 3.9006840649318666, | |
| "delta_nll": -0.00021048217320451812 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.429890595134507, | |
| "teacher_nll": 1.4229323645068903, | |
| "kl": 0.023749441677142703, | |
| "top1_agree": 0.9608669108669109, | |
| "perplexity": 4.178242046907618, | |
| "teacher_perplexity": 4.149269789846104, | |
| "delta_nll": 0.0069582306276165795 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.546815832601508, | |
| "teacher_nll": 3.551584229335389, | |
| "kl": 0.09117504975443504, | |
| "top1_agree": 0.9042124542124542, | |
| "perplexity": 34.70264235461032, | |
| "teacher_perplexity": 34.868513476436284, | |
| "delta_nll": -0.004768396733881186 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-policy-validation-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.727741913102631, | |
| "teacher_nll": 3.730275568886409, | |
| "kl": 0.1131111862895253, | |
| "top1_agree": 0.8991452991452992 | |
| }, | |
| { | |
| "id": "prose-policy-validation-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4063639006189663, | |
| "teacher_nll": 3.4023956112518126, | |
| "kl": 0.0909493915618412, | |
| "top1_agree": 0.8993894993894994 | |
| }, | |
| { | |
| "id": "prose-policy-validation-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.557529046101739, | |
| "teacher_nll": 3.5666212368360806, | |
| "kl": 0.07937738505329696, | |
| "top1_agree": 0.9023199023199023 | |
| }, | |
| { | |
| "id": "prose-policy-validation-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.495628470582694, | |
| "teacher_nll": 3.5070445003672543, | |
| "kl": 0.08126223611307669, | |
| "top1_agree": 0.915995115995116 | |
| }, | |
| { | |
| "id": "code-policy-validation-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4691444356217345, | |
| "teacher_nll": 1.4573161135663042, | |
| "kl": 0.10382261592826564, | |
| "top1_agree": 0.9474969474969475 | |
| }, | |
| { | |
| "id": "code-policy-validation-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.2884563401796296, | |
| "teacher_nll": 1.2908055956110414, | |
| "kl": 0.06478373968703115, | |
| "top1_agree": 0.9557997557997558 | |
| }, | |
| { | |
| "id": "code-policy-validation-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.2366658761533333, | |
| "teacher_nll": 1.255632556547321, | |
| "kl": 0.07255809160452041, | |
| "top1_agree": 0.9492063492063492 | |
| }, | |
| { | |
| "id": "code-policy-validation-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4494991754291986, | |
| "teacher_nll": 1.4408534903520485, | |
| "kl": 0.05256478524833954, | |
| "top1_agree": 0.9538461538461539 | |
| }, | |
| { | |
| "id": "math-policy-validation-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.48816221584185, | |
| "teacher_nll": 1.482822512299441, | |
| "kl": 0.02462935806849064, | |
| "top1_agree": 0.9631257631257631 | |
| }, | |
| { | |
| "id": "math-policy-validation-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4196465551634847, | |
| "teacher_nll": 1.408211328462221, | |
| "kl": 0.02701483388523479, | |
| "top1_agree": 0.9592185592185593 | |
| }, | |
| { | |
| "id": "math-policy-validation-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4596746856913025, | |
| "teacher_nll": 1.4567602635012151, | |
| "kl": 0.026232065173077496, | |
| "top1_agree": 0.958974358974359 | |
| }, | |
| { | |
| "id": "math-policy-validation-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.3520789238413902, | |
| "teacher_nll": 1.3439353537646843, | |
| "kl": 0.017121509581767887, | |
| "top1_agree": 0.9621489621489622 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 9683609088 | |
| } | |
| } | |
| } | |
| } | |