{
  "complete": true,
  "split": "validation",
  "provenance": {
    "source": {
      "repo": "RecursiveSI/flint",
      "revision": "2a247be0c0aec895ccb0ac7ca92229ec6df37288"
    },
    "scripts": {
      "flint_quantize.py": "6cdb630599183e25e416d0e4c44fdab68acdff2ed6d8cde656beaab80c2283ae",
      "flint_reference.py": "2534d51af8e99699692f1a2cf5fc022bd7beaa7de07770dbffb9690aa4f62ff6",
      "flint_nvfp4.py": "31aec4a310b7bddaa90da083b80352e5607ce754c10052458859f7306b5cc566"
    },
    "packages": {
      "torch": "2.8.0+cu128",
      "triton": "3.4.0",
      "safetensors": "0.8.0",
      "transformers": "5.20.2",
      "huggingface_hub": "2.2.0",
      "datasets": "5.1.0"
    },
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
    "cuda": "12.8",
    "created_utc": "2026-10-21T19:16:32Z"
  },
  "script_sha256": "e0e0aa655a97d9c78837196a94b280c54944ab5542686242dbaa7604d33dca1c",
  "normalization_script_sha256": "f06c3d51f9018814648a1fe42c39274dd103528b30185d914f76adbd68fef3ac",
  "flashinfer_version": "0.6.16.post1",
  "corpus_sha256": "b6fdc596cb03b87ddea6a2f07453d27574c415fdb7dd034a61abf8146f9eb573",
  "results": {
    "bf16": {
      "precision_by_layer": {
        "2": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "3": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "4": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "5": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "6": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "7": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "8": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "9": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "10": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "11": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "12": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "13": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "14": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "15": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "16": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "17": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "18": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "19": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "20": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "21": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "22": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "23": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "24": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "25": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "26": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "27": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "28": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "29": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "30": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "31": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "32": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "33": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "34": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "35": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "36": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "37": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "38": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "39": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "40": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "41": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "42": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "43": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "44": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "45": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "46": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "47": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "48": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        }
      },
      "tensor_bytes": 79570574912,
      "aggregate": {
        "all": {
          "tokens": 49140,
          "nll": 2.1118895109538194,
          "teacher_nll": 2.1118895109538194,
          "kl": 0.0,
          "top1_agree": 1.0,
          "perplexity": 8.26384116031041,
          "teacher_perplexity": 8.26384116031041,
          "delta_nll": 0.0
        },
        "code": {
          "tokens": 16380,
          "nll": 1.3611519390191786,
          "teacher_nll": 1.3611519390191786,
          "kl": 0.0,
          "top1_agree": 1.0,
          "perplexity": 3.9006840649318666,
          "teacher_perplexity": 3.9006840649318666,
          "delta_nll": 0.0
        },
        "math": {
          "tokens": 16380,
          "nll": 1.4229323645068903,
          "teacher_nll": 1.4229323645068903,
          "kl": 0.0,
          "top1_agree": 1.0,
          "perplexity": 4.149269789846104,
          "teacher_perplexity": 4.149269789846104,
          "delta_nll": 0.0
        },
        "prose": {
          "tokens": 16380,
          "nll": 3.551584229335389,
          "teacher_nll": 3.551584229335389,
          "kl": 0.0,
          "top1_agree": 1.0,
          "perplexity": 34.868513476436284,
          "teacher_perplexity": 34.868513476436284,
          "delta_nll": 0.0
        }
      },
      "sequences": [
        {
          "id": "prose-policy-validation-0",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.730275568886409,
          "teacher_nll": 3.730275568886409,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "prose-policy-validation-1",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.4023956112518126,
          "teacher_nll": 3.4023956112518126,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "prose-policy-validation-2",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.5666212368360806,
          "teacher_nll": 3.5666212368360806,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "prose-policy-validation-3",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.5070445003672543,
          "teacher_nll": 3.5070445003672543,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "code-policy-validation-0",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4573161135663042,
          "teacher_nll": 1.4573161135663042,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "code-policy-validation-1",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.2908055956110414,
          "teacher_nll": 1.2908055956110414,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "code-policy-validation-2",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.255632556547321,
          "teacher_nll": 1.255632556547321,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "code-policy-validation-3",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4408534903520485,
          "teacher_nll": 1.4408534903520485,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "math-policy-validation-0",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.482822512299441,
          "teacher_nll": 1.482822512299441,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "math-policy-validation-1",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.408211328462221,
          "teacher_nll": 1.408211328462221,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "math-policy-validation-2",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4567602635012151,
          "teacher_nll": 1.4567602635012151,
          "kl": 0.0,
          "top1_agree": 1.0
        },
        {
          "id": "math-policy-validation-3",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.3439353537646843,
          "teacher_nll": 1.3439353537646843,
          "kl": 0.0,
          "top1_agree": 1.0
        }
      ],
      "memory": {
        "swap_kib": 0,
        "cuda_peak_bytes": 9683609088
      }
    },
    "preferred-fp8-45-48": {
      "precision_by_layer": {
        "2": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "3": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "4": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "5": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "6": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "7": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "8": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "9": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "10": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "11": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "12": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "13": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "14": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "15": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "16": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "17": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "18": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "19": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "20": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "21": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "22": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "23": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "24": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "25": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "26": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "27": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "28": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "29": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "30": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "31": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "32": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "33": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "34": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "35": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "36": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "37": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "38": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "39": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "40": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "41": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "42": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "43": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "44": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "45": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "46": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "47": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "48": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        }
      },
      "tensor_bytes": 76350135872,
      "aggregate": {
        "all": {
          "tokens": 49140,
          "nll": 2.1111584732335515,
          "teacher_nll": 2.1118895109538194,
          "kl": 0.0007934031345190516,
          "top1_agree": 0.9902930402930403,
          "perplexity": 8.257802188335017,
          "teacher_perplexity": 8.26384116031041,
          "delta_nll": -0.0007310377202678353
        },
        "code": {
          "tokens": 16380,
          "nll": 1.3587105913039965,
          "teacher_nll": 1.3611519390191786,
          "kl": 0.0006417182103448726,
          "top1_agree": 0.9924297924297925,
          "perplexity": 3.891172753735319,
          "teacher_perplexity": 3.9006840649318666,
          "delta_nll": -0.0024413477151821183
        },
        "math": {
          "tokens": 16380,
          "nll": 1.4229313316508236,
          "teacher_nll": 1.4229323645068903,
          "kl": 0.0005105946148616081,
          "top1_agree": 0.9940781440781441,
          "perplexity": 4.1492655042498425,
          "teacher_perplexity": 4.149269789846104,
          "delta_nll": -1.032856066673915e-06
        },
        "prose": {
          "tokens": 16380,
          "nll": 3.551833496745835,
          "teacher_nll": 3.551584229335389,
          "kl": 0.001227896578350674,
          "top1_agree": 0.9843711843711843,
          "perplexity": 34.87720614385099,
          "teacher_perplexity": 34.868513476436284,
          "delta_nll": 0.00024926741044595246
        }
      },
      "sequences": [
        {
          "id": "prose-policy-validation-0",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.7294776180579405,
          "teacher_nll": 3.730275568886409,
          "kl": 0.0015413437680010394,
          "top1_agree": 0.9841269841269841
        },
        {
          "id": "prose-policy-validation-1",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.4025479271298362,
          "teacher_nll": 3.4023956112518126,
          "kl": 0.001127076467842555,
          "top1_agree": 0.9846153846153847
        },
        {
          "id": "prose-policy-validation-2",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.5673533270792794,
          "teacher_nll": 3.5666212368360806,
          "kl": 0.001155838767869423,
          "top1_agree": 0.9829059829059829
        },
        {
          "id": "prose-policy-validation-3",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.507955114716284,
          "teacher_nll": 3.5070445003672543,
          "kl": 0.001087327309689679,
          "top1_agree": 0.9858363858363859
        },
        {
          "id": "code-policy-validation-0",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4538545538654257,
          "teacher_nll": 1.4573161135663042,
          "kl": 0.0006527559892575783,
          "top1_agree": 0.9934065934065934
        },
        {
          "id": "code-policy-validation-1",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.290072934415023,
          "teacher_nll": 1.2908055956110414,
          "kl": 0.0007901271593658043,
          "top1_agree": 0.9916971916971917
        },
        {
          "id": "code-policy-validation-2",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.2532000170231448,
          "teacher_nll": 1.255632556547321,
          "kl": 0.0005299052457815271,
          "top1_agree": 0.9921855921855922
        },
        {
          "id": "code-policy-validation-3",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4377148599123926,
          "teacher_nll": 1.4408534903520485,
          "kl": 0.0005940844469745809,
          "top1_agree": 0.9924297924297925
        },
        {
          "id": "math-policy-validation-0",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4838312496050228,
          "teacher_nll": 1.482822512299441,
          "kl": 0.00048286072080852566,
          "top1_agree": 0.9946275946275946
        },
        {
          "id": "math-policy-validation-1",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4076686153481732,
          "teacher_nll": 1.408211328462221,
          "kl": 0.000718450903783351,
          "top1_agree": 0.9926739926739927
        },
        {
          "id": "math-policy-validation-2",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4570507105453547,
          "teacher_nll": 1.4567602635012151,
          "kl": 0.0004748969137559444,
          "top1_agree": 0.9946275946275946
        },
        {
          "id": "math-policy-validation-3",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.3431747511047438,
          "teacher_nll": 1.3439353537646843,
          "kl": 0.0003661699210986113,
          "top1_agree": 0.9943833943833944
        }
      ],
      "memory": {
        "swap_kib": 0,
        "cuda_peak_bytes": 9683609088
      }
    },
    "conservative-nvfp4": {
      "precision_by_layer": {
        "2": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "3": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "4": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "5": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "6": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "7": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "8": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "9": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "10": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "11": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "12": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "13": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "14": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "15": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "16": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "17": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "18": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "19": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "20": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "21": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "22": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "23": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "24": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "25": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "26": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "27": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "28": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "29": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "30": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "31": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "32": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "33": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "34": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "35": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "36": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "37": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "38": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "39": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "40": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "41": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "42": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "43": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "44": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "45": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "46": {
          "gate": "bf16",
          "up": "fp8",
          "down": "bf16"
        },
        "47": {
          "gate": "fp8",
          "up": "nvfp4",
          "down": "fp8"
        },
        "48": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        }
      },
      "tensor_bytes": 77574480448,
      "aggregate": {
        "all": {
          "tokens": 49140,
          "nll": 2.110927108768336,
          "teacher_nll": 2.1118895109538194,
          "kl": 0.0007221096060126012,
          "top1_agree": 0.990964590964591,
          "perplexity": 8.255891847348888,
          "teacher_perplexity": 8.26384116031041,
          "delta_nll": -0.0009624021854834908
        },
        "code": {
          "tokens": 16380,
          "nll": 1.3591037988226056,
          "teacher_nll": 1.3611519390191786,
          "kl": 0.0004896586170060144,
          "top1_agree": 0.9945054945054945,
          "perplexity": 3.8927030929690236,
          "teacher_perplexity": 3.9006840649318666,
          "delta_nll": -0.0020481401965730672
        },
        "math": {
          "tokens": 16380,
          "nll": 1.4222320152580812,
          "teacher_nll": 1.4229323645068903,
          "kl": 0.0004681396002452525,
          "top1_agree": 0.9937118437118437,
          "perplexity": 4.146364869213906,
          "teacher_perplexity": 4.149269789846104,
          "delta_nll": -0.000700349248809129
        },
        "prose": {
          "tokens": 16380,
          "nll": 3.551445512224321,
          "teacher_nll": 3.551584229335389,
          "kl": 0.0012085306007865366,
          "top1_agree": 0.9846764346764347,
          "perplexity": 34.863676952441665,
          "teacher_perplexity": 34.868513476436284,
          "delta_nll": -0.00013871711106805407
        }
      },
      "sequences": [
        {
          "id": "prose-policy-validation-0",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.728131878565228,
          "teacher_nll": 3.730275568886409,
          "kl": 0.001546875980565545,
          "top1_agree": 0.9843711843711843
        },
        {
          "id": "prose-policy-validation-1",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.402904140672695,
          "teacher_nll": 3.4023956112518126,
          "kl": 0.0011176317939318725,
          "top1_agree": 0.9875457875457876
        },
        {
          "id": "prose-policy-validation-2",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.566692111023471,
          "teacher_nll": 3.5666212368360806,
          "kl": 0.0010729805890457098,
          "top1_agree": 0.9833943833943833
        },
        {
          "id": "prose-policy-validation-3",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.508053918635889,
          "teacher_nll": 3.5070445003672543,
          "kl": 0.001096634039603019,
          "top1_agree": 0.9833943833943833
        },
        {
          "id": "code-policy-validation-0",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.454416419763996,
          "teacher_nll": 1.4573161135663042,
          "kl": 0.0005280497261597087,
          "top1_agree": 0.9938949938949939
        },
        {
          "id": "code-policy-validation-1",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.2902200173109006,
          "teacher_nll": 1.2908055956110414,
          "kl": 0.0004691741009378153,
          "top1_agree": 0.9951159951159951
        },
        {
          "id": "code-policy-validation-2",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.2530585611550653,
          "teacher_nll": 1.255632556547321,
          "kl": 0.0004382003377895616,
          "top1_agree": 0.9943833943833944
        },
        {
          "id": "code-policy-validation-3",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4387201970604604,
          "teacher_nll": 1.4408534903520485,
          "kl": 0.000523210303136972,
          "top1_agree": 0.9946275946275946
        },
        {
          "id": "math-policy-validation-0",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4823827999706525,
          "teacher_nll": 1.482822512299441,
          "kl": 0.0005088003770185586,
          "top1_agree": 0.9946275946275946
        },
        {
          "id": "math-policy-validation-1",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4077888519044905,
          "teacher_nll": 1.408211328462221,
          "kl": 0.0005599377982168298,
          "top1_agree": 0.991941391941392
        },
        {
          "id": "math-policy-validation-2",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4556590544871795,
          "teacher_nll": 1.4567602635012151,
          "kl": 0.00041746762484512197,
          "top1_agree": 0.9929181929181929
        },
        {
          "id": "math-policy-validation-3",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.3430973546700018,
          "teacher_nll": 1.3439353537646843,
          "kl": 0.0003863526009004995,
          "top1_agree": 0.9953601953601954
        }
      ],
      "memory": {
        "swap_kib": 0,
        "cuda_peak_bytes": 9683609088
      }
    },
    "smaller-fp8-39-48": {
      "precision_by_layer": {
        "2": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "3": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "4": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "5": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "6": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "7": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "8": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "9": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "10": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "11": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "12": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "13": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "14": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "15": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "16": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "17": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "18": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "19": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "20": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "21": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "22": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "23": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "24": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "25": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "26": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "27": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "28": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "29": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "30": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "31": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "32": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "33": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "34": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "35": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "36": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "37": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "38": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "39": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "40": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "41": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "42": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "43": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "44": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "45": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "46": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "47": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "48": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        }
      },
      "tensor_bytes": 71519477312,
      "aggregate": {
        "all": {
          "tokens": 49140,
          "nll": 2.1108048758445643,
          "teacher_nll": 2.1118895109538194,
          "kl": 0.0029436703867441604,
          "top1_agree": 0.9822954822954822,
          "perplexity": 8.254882767222709,
          "teacher_perplexity": 8.26384116031041,
          "delta_nll": -0.00108463510925505
        },
        "code": {
          "tokens": 16380,
          "nll": 1.3591565501777243,
          "teacher_nll": 1.3611519390191786,
          "kl": 0.0022441253232654817,
          "top1_agree": 0.9860195360195361,
          "perplexity": 3.892908443748471,
          "teacher_perplexity": 3.9006840649318666,
          "delta_nll": -0.0019953888414543375
        },
        "math": {
          "tokens": 16380,
          "nll": 1.4214664039448794,
          "teacher_nll": 1.4229323645068903,
          "kl": 0.0019512432102018443,
          "top1_agree": 0.9892551892551893,
          "perplexity": 4.143191580269336,
          "teacher_perplexity": 4.149269789846104,
          "delta_nll": -0.0014659605620108707
        },
        "prose": {
          "tokens": 16380,
          "nll": 3.551791673411089,
          "teacher_nll": 3.551584229335389,
          "kl": 0.004635642626765156,
          "top1_agree": 0.9716117216117216,
          "perplexity": 34.87574749328646,
          "teacher_perplexity": 34.868513476436284,
          "delta_nll": 0.00020744407569983636
        }
      },
      "sequences": [
        {
          "id": "prose-policy-validation-0",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.7271026909424125,
          "teacher_nll": 3.730275568886409,
          "kl": 0.005908135372931558,
          "top1_agree": 0.9724053724053724
        },
        {
          "id": "prose-policy-validation-1",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.4017877338861227,
          "teacher_nll": 3.4023956112518126,
          "kl": 0.004090809205294529,
          "top1_agree": 0.9726495726495726
        },
        {
          "id": "prose-policy-validation-2",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.567241637960022,
          "teacher_nll": 3.5666212368360806,
          "kl": 0.0043217347036496765,
          "top1_agree": 0.9675213675213675
        },
        {
          "id": "prose-policy-validation-3",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.511034630855798,
          "teacher_nll": 3.5070445003672543,
          "kl": 0.004221891225184859,
          "top1_agree": 0.9738705738705739
        },
        {
          "id": "code-policy-validation-0",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4563412683787362,
          "teacher_nll": 1.4573161135663042,
          "kl": 0.0023459550880250476,
          "top1_agree": 0.9858363858363859
        },
        {
          "id": "code-policy-validation-1",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.2917616325711447,
          "teacher_nll": 1.2908055956110414,
          "kl": 0.0023298357348429705,
          "top1_agree": 0.9858363858363859
        },
        {
          "id": "code-policy-validation-2",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.2537396895579802,
          "teacher_nll": 1.255632556547321,
          "kl": 0.0018885908009295643,
          "top1_agree": 0.9851037851037852
        },
        {
          "id": "code-policy-validation-3",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4347836102030362,
          "teacher_nll": 1.4408534903520485,
          "kl": 0.002412119669264344,
          "top1_agree": 0.9873015873015873
        },
        {
          "id": "math-policy-validation-0",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4846272683987833,
          "teacher_nll": 1.482822512299441,
          "kl": 0.0018748843605756323,
          "top1_agree": 0.98998778998779
        },
        {
          "id": "math-policy-validation-1",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4076982973259446,
          "teacher_nll": 1.408211328462221,
          "kl": 0.0022242125599066273,
          "top1_agree": 0.9875457875457876
        },
        {
          "id": "math-policy-validation-2",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4541759463020296,
          "teacher_nll": 1.4567602635012151,
          "kl": 0.002292718319620332,
          "top1_agree": 0.9921855921855922
        },
        {
          "id": "math-policy-validation-3",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.3393641037527604,
          "teacher_nll": 1.3439353537646843,
          "kl": 0.0014131576007047851,
          "top1_agree": 0.9873015873015873
        }
      ],
      "memory": {
        "swap_kib": 0,
        "cuda_peak_bytes": 9683609088
      }
    },
    "mixed-late-projections": {
      "precision_by_layer": {
        "2": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "3": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "4": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "5": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "6": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "7": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "8": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "9": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "10": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "11": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "12": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "13": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "14": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "15": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "16": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "17": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "18": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "19": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "20": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "21": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "22": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "23": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "24": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "25": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "26": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "27": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "28": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "29": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "30": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "31": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "32": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "33": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "34": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "35": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "36": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "37": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "38": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "39": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "40": {
          "gate": "bf16",
          "up": "bf16",
          "down": "bf16"
        },
        "41": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "42": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "43": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "44": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "45": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "46": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "47": {
          "gate": "fp8",
          "up": "nvfp4",
          "down": "fp8"
        },
        "48": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        }
      },
      "tensor_bytes": 73012191808,
      "aggregate": {
        "all": {
          "tokens": 49140,
          "nll": 2.110575082263764,
          "teacher_nll": 2.1118895109538194,
          "kl": 0.002768995274312409,
          "top1_agree": 0.9835571835571836,
          "perplexity": 8.252986066085763,
          "teacher_perplexity": 8.26384116031041,
          "delta_nll": -0.0013144286900552693
        },
        "code": {
          "tokens": 16380,
          "nll": 1.35876675370412,
          "teacher_nll": 1.3611519390191786,
          "kl": 0.002667256963742904,
          "top1_agree": 0.9891941391941392,
          "perplexity": 3.8913912974733766,
          "teacher_perplexity": 3.9006840649318666,
          "delta_nll": -0.0023851853150587132
        },
        "math": {
          "tokens": 16380,
          "nll": 1.422199620781364,
          "teacher_nll": 1.4229323645068903,
          "kl": 0.0017168798307650546,
          "top1_agree": 0.9890720390720391,
          "perplexity": 4.146230552069269,
          "teacher_perplexity": 4.149269789846104,
          "delta_nll": -0.0007327437255262303
        },
        "prose": {
          "tokens": 16380,
          "nll": 3.5507588723058086,
          "teacher_nll": 3.551584229335389,
          "kl": 0.003922849028429269,
          "top1_agree": 0.9724053724053724,
          "perplexity": 34.83974637692444,
          "teacher_perplexity": 34.868513476436284,
          "delta_nll": -0.0008253570295804202
        }
      },
      "sequences": [
        {
          "id": "prose-policy-validation-0",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.72932381542611,
          "teacher_nll": 3.730275568886409,
          "kl": 0.004572560680946005,
          "top1_agree": 0.9709401709401709
        },
        {
          "id": "prose-policy-validation-1",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.401089091877361,
          "teacher_nll": 3.4023956112518126,
          "kl": 0.0037138127636560155,
          "top1_agree": 0.9748473748473748
        },
        {
          "id": "prose-policy-validation-2",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.565276440071972,
          "teacher_nll": 3.5666212368360806,
          "kl": 0.003613682586122054,
          "top1_agree": 0.9709401709401709
        },
        {
          "id": "prose-policy-validation-3",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.5073461418477896,
          "teacher_nll": 3.5070445003672543,
          "kl": 0.0037913400829930007,
          "top1_agree": 0.9728937728937729
        },
        {
          "id": "code-policy-validation-0",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4552863815037467,
          "teacher_nll": 1.4573161135663042,
          "kl": 0.004951566845742629,
          "top1_agree": 0.9897435897435898
        },
        {
          "id": "code-policy-validation-1",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.2907511532670557,
          "teacher_nll": 1.2908055956110414,
          "kl": 0.0021100598511369093,
          "top1_agree": 0.9894993894993895
        },
        {
          "id": "code-policy-validation-2",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.2536105592608888,
          "teacher_nll": 1.255632556547321,
          "kl": 0.0018023577269726185,
          "top1_agree": 0.9865689865689866
        },
        {
          "id": "code-policy-validation-3",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4354189207847885,
          "teacher_nll": 1.4408534903520485,
          "kl": 0.0018050434311194588,
          "top1_agree": 0.990964590964591
        },
        {
          "id": "math-policy-validation-0",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4831841726151724,
          "teacher_nll": 1.482822512299441,
          "kl": 0.0015588753443934542,
          "top1_agree": 0.9894993894993895
        },
        {
          "id": "math-policy-validation-1",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4087224536471896,
          "teacher_nll": 1.408211328462221,
          "kl": 0.002210616383142087,
          "top1_agree": 0.9863247863247864
        },
        {
          "id": "math-policy-validation-2",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4548888978504,
          "teacher_nll": 1.4567602635012151,
          "kl": 0.00180718969958533,
          "top1_agree": 0.9921855921855922
        },
        {
          "id": "math-policy-validation-3",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.342002959012694,
          "teacher_nll": 1.3439353537646843,
          "kl": 0.0012908378959393472,
          "top1_agree": 0.9882783882783883
        }
      ],
      "memory": {
        "swap_kib": 0,
        "cuda_peak_bytes": 9683609088
      }
    },
    "all-fp8": {
      "precision_by_layer": {
        "2": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "3": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "4": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "5": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "6": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "7": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "8": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "9": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "10": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "11": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "12": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "13": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "14": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "15": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "16": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "17": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "18": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "19": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "20": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "21": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "22": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "23": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "24": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "25": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "26": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "27": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "28": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "29": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "30": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "31": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "32": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "33": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "34": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "35": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "36": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "37": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "38": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "39": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "40": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "41": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "42": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "43": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "44": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "45": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "46": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "47": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        },
        "48": {
          "gate": "fp8",
          "up": "fp8",
          "down": "fp8"
        }
      },
      "tensor_bytes": 41730416192,
      "aggregate": {
        "all": {
          "tokens": 49140,
          "nll": 2.112549294860663,
          "teacher_nll": 2.1118895109538194,
          "kl": 0.06278559984953898,
          "top1_agree": 0.9388888888888889,
          "perplexity": 8.269295308798538,
          "teacher_perplexity": 8.26384116031041,
          "delta_nll": 0.0006597839068436251
        },
        "code": {
          "tokens": 16380,
          "nll": 1.3609414568459741,
          "teacher_nll": 1.3611519390191786,
          "kl": 0.07343230811703919,
          "top1_agree": 0.9515873015873015,
          "perplexity": 3.8998631268723396,
          "teacher_perplexity": 3.9006840649318666,
          "delta_nll": -0.00021048217320451812
        },
        "math": {
          "tokens": 16380,
          "nll": 1.429890595134507,
          "teacher_nll": 1.4229323645068903,
          "kl": 0.023749441677142703,
          "top1_agree": 0.9608669108669109,
          "perplexity": 4.178242046907618,
          "teacher_perplexity": 4.149269789846104,
          "delta_nll": 0.0069582306276165795
        },
        "prose": {
          "tokens": 16380,
          "nll": 3.546815832601508,
          "teacher_nll": 3.551584229335389,
          "kl": 0.09117504975443504,
          "top1_agree": 0.9042124542124542,
          "perplexity": 34.70264235461032,
          "teacher_perplexity": 34.868513476436284,
          "delta_nll": -0.004768396733881186
        }
      },
      "sequences": [
        {
          "id": "prose-policy-validation-0",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.727741913102631,
          "teacher_nll": 3.730275568886409,
          "kl": 0.1131111862895253,
          "top1_agree": 0.8991452991452992
        },
        {
          "id": "prose-policy-validation-1",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.4063639006189663,
          "teacher_nll": 3.4023956112518126,
          "kl": 0.0909493915618412,
          "top1_agree": 0.8993894993894994
        },
        {
          "id": "prose-policy-validation-2",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.557529046101739,
          "teacher_nll": 3.5666212368360806,
          "kl": 0.07937738505329696,
          "top1_agree": 0.9023199023199023
        },
        {
          "id": "prose-policy-validation-3",
          "domain": "prose",
          "tokens": 4095,
          "nll": 3.495628470582694,
          "teacher_nll": 3.5070445003672543,
          "kl": 0.08126223611307669,
          "top1_agree": 0.915995115995116
        },
        {
          "id": "code-policy-validation-0",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4691444356217345,
          "teacher_nll": 1.4573161135663042,
          "kl": 0.10382261592826564,
          "top1_agree": 0.9474969474969475
        },
        {
          "id": "code-policy-validation-1",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.2884563401796296,
          "teacher_nll": 1.2908055956110414,
          "kl": 0.06478373968703115,
          "top1_agree": 0.9557997557997558
        },
        {
          "id": "code-policy-validation-2",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.2366658761533333,
          "teacher_nll": 1.255632556547321,
          "kl": 0.07255809160452041,
          "top1_agree": 0.9492063492063492
        },
        {
          "id": "code-policy-validation-3",
          "domain": "code",
          "tokens": 4095,
          "nll": 1.4494991754291986,
          "teacher_nll": 1.4408534903520485,
          "kl": 0.05256478524833954,
          "top1_agree": 0.9538461538461539
        },
        {
          "id": "math-policy-validation-0",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.48816221584185,
          "teacher_nll": 1.482822512299441,
          "kl": 0.02462935806849064,
          "top1_agree": 0.9631257631257631
        },
        {
          "id": "math-policy-validation-1",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4196465551634847,
          "teacher_nll": 1.408211328462221,
          "kl": 0.02701483388523479,
          "top1_agree": 0.9592185592185593
        },
        {
          "id": "math-policy-validation-2",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.4596746856913025,
          "teacher_nll": 1.4567602635012151,
          "kl": 0.026232065173077496,
          "top1_agree": 0.958974358974359
        },
        {
          "id": "math-policy-validation-3",
          "domain": "math",
          "tokens": 4095,
          "nll": 1.3520789238413902,
          "teacher_nll": 1.3439353537646843,
          "kl": 0.017121509581767887,
          "top1_agree": 0.9621489621489622
        }
      ],
      "memory": {
        "swap_kib": 0,
        "cuda_peak_bytes": 9683609088
      }
    }
  }
}