{
  "model": "diffusion-v2",
  "step": 15000,
  "source_tokens": 3932160000,
  "stored_parameters": 51392512,
  "optimized_parameters": 50867200,
  "training_duration_minutes": 439.5,
  "training_loss": 5.127222061157227,
  "protocol": "EXPERIMENTAL: continuation PLL; single-mask left-to-right greedy generation; NOT AR likelihood",
  "manifest": {
    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/diffusion-v2-step15000-20260923",
    "local_model": "diffusion",
    "kind": "diffusion",
    "tokenizer": "/mnt/d/Projects/tiny_llm/tokenizer",
    "revision": "main",
    "tokenizer_revision": null,
    "protocol": "EXPERIMENTAL: continuation PLL; single-mask left-to-right greedy generation; NOT AR likelihood",
    "harness_version": "0.4.12",
    "smoke_only": false,
    "tasks": [
      {
        "task": "hellaswag",
        "shots": 0,
        "description": "Commonsense completion; acc and acc_norm",
        "unsafe_code": false,
        "rolling": false
      },
      {
        "task": "arc_easy",
        "shots": 0,
        "description": "AI2 ARC-Easy science questions; acc and acc_norm",
        "unsafe_code": false,
        "rolling": false
      },
      {
        "task": "arc_challenge",
        "shots": 0,
        "description": "AI2 ARC-Challenge; acc and acc_norm",
        "unsafe_code": false,
        "rolling": false
      },
      {
        "task": "piqa",
        "shots": 0,
        "description": "Physical commonsense; acc and acc_norm",
        "unsafe_code": false,
        "rolling": false
      },
      {
        "task": "winogrande",
        "shots": 0,
        "description": "Pronoun resolution; acc",
        "unsafe_code": false,
        "rolling": false
      },
      {
        "task": "openbookqa",
        "shots": 0,
        "description": "Science multiple choice; acc and acc_norm",
        "unsafe_code": false,
        "rolling": false
      },
      {
        "task": "boolq",
        "shots": 0,
        "description": "Yes/no reading comprehension; acc",
        "unsafe_code": false,
        "rolling": false
      },
      {
        "task": "lambada_openai",
        "shots": 0,
        "description": "Complete final word; acc and perplexity",
        "unsafe_code": false,
        "rolling": false
      }
    ],
    "options": {
      "checkpoint": "runs/eval/diffusion-v2-step15000-20260923",
      "model": null,
      "tokenizer": null,
      "revision": "main",
      "tokenizer_revision": null,
      "suite": "core",
      "tasks": null,
      "num_fewshot": null,
      "device": "cuda:0",
      "dtype": "bfloat16",
      "batch_size": 1,
      "max_length": 2048,
      "max_gen_toks": null,
      "limit": null,
      "seed": 1234,
      "threads": 4,
      "bootstrap_iters": 1000,
      "trust_remote_code": false,
      "allow_code_execution": false,
      "apply_chat_template": false,
      "log_samples": false,
      "output": "runs/eval/diffusion-v2-core-full-20260923",
      "list": false,
      "dry_run": false
    },
    "checkpoint_stamp": {
      "model.safetensors": [
        205580928,
        1790122976166062900
      ],
      "config.json": [
        677,
        1790122975189825600
      ]
    },
    "checkpoint_sha256": {
      "config.json": "8e3daed6864d9b1f39be95c7b311a93c47efd4e76ee33fb4dbeede6d4701343d",
      "model.safetensors": "d0d24e0cc492cd5c9a613710c454af9a6e49265d2bae7abb69a673a53efe2bfb"
    },
    "source_sha256": {
      "src/tiny_llm/__init__.py": "8c7f075d51006a53fe1d645ef5a84d65db58ff4251e7544074fe29ee7549c14c",
      "src/tiny_llm/benchmarks.py": "5d9f461366389bd1f926c6a6468192f3e2313c87caf5b1de27f407d563a858a4",
      "src/tiny_llm/checkpoint.py": "ac9df5f8c981a47eb1f1e59d0687ff41a1f6c38ca74ced50f71ac446a58d6ef5",
      "src/tiny_llm/continuation.py": "548b7e76401cd545af3e7ff1aedf041fc96a330c8c9cc80266e0e20830f0e95d",
      "src/tiny_llm/data.py": "973e4ff3e7b6e80c74bd99173f89cb8ce94c6ef3a21ff54acf1a42eab9654d89",
      "src/tiny_llm/diffusion_diagnostics.py": "2ae667a076b1d3f057a29d20f00559068eb73a073b3f99030e1d5b4c2f478fa5",
      "src/tiny_llm/evaluation/__init__.py": "e49e80f7be3ab6e200f0db14b028e465de9caa8f6f705430eabce0c41b8322ba",
      "src/tiny_llm/evaluation/adapters.py": "8e64e5542bbcb0f9ad448b320e712916a9559823987e298b22321a8d8e2f727f",
      "src/tiny_llm/evaluation/catalog.py": "5681c9433c67890644488f7f73280e18662f28a5de8c18374443bd7c84ba4be9",
      "src/tiny_llm/evaluation/cli.py": "31dccd47b5ae441bbc523bc6d113f22ed7134b2b42612224fee7f1d88911801f",
      "src/tiny_llm/losses.py": "592a05287369f6e5077b3faca404a9392de4772cf06389fb8aa7e36924e74f86",
      "src/tiny_llm/models/__init__.py": "55f88bbd8cf8bfd70e5e17c1d0050299412528385c3714bb712623f40b0a681d",
      "src/tiny_llm/models/diffusion/__init__.py": "c92eaf5986d604d8aaf1894a862a217cefa32ab25359175b88110e45a71c1b25",
      "src/tiny_llm/models/diffusion/configuration_diffusion_lm.py": "3b303aba60f0a831c966f5cfa7f5441daa2584814e335646195bd6ceecfa6d0c",
      "src/tiny_llm/models/diffusion/modeling_diffusion_lm.py": "4fc21c2aed1d5a231afe6f7634d9d8c3682b4e2abfc8fecef27b52136de77b2f",
      "src/tiny_llm/models/looped/__init__.py": "4b46b426021c59ed313cb2f49afc1926ad2793d771ef90117e3fef0462957cff",
      "src/tiny_llm/models/looped/configuration_looped_lm.py": "b8d99c52b98e0a97a4cae76ca4c20affd8f4d019ec38e0a9beefd5fad864dc25",
      "src/tiny_llm/models/looped/modeling_looped_lm.py": "9bdb3c5e3aaae663d09e98055f95df1adbc42e0a4bf5dffb7f6c39242af41b9e",
      "src/tiny_llm/models/moe_t5/__init__.py": "18e33fe38b2049521f573512ac8796afa209e1b25c2052784e1c927a7160d66c",
      "src/tiny_llm/models/moe_t5/configuration_alicet5_moe.py": "4647b1e9ab9ce07887bffc1abc3beb3e691962a380b4cc237aae376490bb56b7",
      "src/tiny_llm/models/moe_t5/modeling_alicet5_moe.py": "c7bbba2f01dc889cc604c7bc161354397239b28503d05f7e00665658be84751d",
      "src/tiny_llm/models/prefixlm/__init__.py": "c94c3c861acba8d2147a3a6c46bccfc71ac8adc8d70757bede009598029d31ea",
      "src/tiny_llm/models/prefixlm/configuration_prefix_lm.py": "20b3a62ae3628b5a64566ef19d2947c90f10c6159daad00dd13d18fc5910c42c",
      "src/tiny_llm/models/prefixlm/modeling_prefix_lm.py": "f62a38211df175696f64945173d1e41d9f7fcf6152a7648020da8c91788e98fd",
      "src/tiny_llm/models/q50m/__init__.py": "9d0dfec6fa0cfa6967e2f17a73d8fbeca69ba93b4a93651eda4d95626d5f5eb5",
      "src/tiny_llm/models/q50m/configuration_q50m.py": "796806b2d5d982b633796bc7de53c3073b1b6e52d259f44ef798b63f55eaea92",
      "src/tiny_llm/models/q50m/modeling_q50m.py": "b7a712050167fad24471193b0ca7baf4ca7e5ebdf8036c91b506d3fff5b49c8b",
      "src/tiny_llm/telemetry.py": "f5a35de68da1a2304159fd3613fa3773bc368982ed18712791c847285b895c09",
      "src/tiny_llm/ul2.py": "f4d948d89d5e106965612971dd473a54b57719f811761f18457c6221d763b876"
    },
    "packages": {
      "lm_eval": "0.4.12",
      "torch": "2.11.0+cu128",
      "transformers": "5.17.0",
      "datasets": "5.0.1",
      "accelerate": "1.15.0"
    },
    "git_commit": "ca42011e5a82513978e4a466c3f24a5dfd4905de",
    "git_dirty": true
  },
  "tasks": {
    "hellaswag": {
      "samples": 10042,
      "dataset": "Rowan/hellaswag",
      "config": "default",
      "split": "validation",
      "primary": "acc_norm",
      "metrics": {
        "acc": {
          "value": 0.27743477394941246,
          "stderr": 0.004468178273665403,
          "ci95": [
            0.26876410971590897,
            0.2862756529238705
          ]
        },
        "acc_norm": {
          "value": 0.2795259908384784,
          "stderr": 0.004478491697891533,
          "ci95": [
            0.2708343265409323,
            0.2883862705438426
          ]
        }
      }
    },
    "arc_easy": {
      "samples": 2376,
      "dataset": "allenai/ai2_arc",
      "config": "ARC-Easy",
      "split": "test",
      "primary": "acc_norm",
      "metrics": {
        "acc": {
          "value": 0.37836700336700335,
          "stderr": 0.009951575683332024,
          "ci95": [
            0.3590774700626608,
            0.39804920833110835
          ]
        },
        "acc_norm": {
          "value": 0.3362794612794613,
          "stderr": 0.0096941780727253,
          "ci95": [
            0.3175609903739864,
            0.3555264763913292
          ]
        }
      }
    },
    "arc_challenge": {
      "samples": 1172,
      "dataset": "allenai/ai2_arc",
      "config": "ARC-Challenge",
      "split": "test",
      "primary": "acc_norm",
      "metrics": {
        "acc": {
          "value": 0.2030716723549488,
          "stderr": 0.011755899303705644,
          "ci95": [
            0.18102762485103946,
            0.22705584186219357
          ]
        },
        "acc_norm": {
          "value": 0.2226962457337884,
          "stderr": 0.012158314774829888,
          "ci95": [
            0.19980419390295,
            0.2474001931137188
          ]
        }
      }
    },
    "piqa": {
      "samples": 1838,
      "dataset": "baber/piqa",
      "config": "default",
      "split": "validation",
      "primary": "acc_norm",
      "metrics": {
        "acc": {
          "value": 0.5696409140369967,
          "stderr": 0.011552114834700413,
          "ci95": [
            0.5468832486835327,
            0.592108084568306
          ]
        },
        "acc_norm": {
          "value": 0.5386289445048966,
          "stderr": 0.011630956681145891,
          "ci95": [
            0.5157819596226829,
            0.5613147955404465
          ]
        }
      }
    },
    "winogrande": {
      "samples": 1267,
      "dataset": "allenai/winogrande",
      "config": "winogrande_xl",
      "split": "validation",
      "primary": "acc",
      "metrics": {
        "acc": {
          "value": 0.5011838989739542,
          "stderr": 0.014052446290528871,
          "ci95": [
            0.47369054474157507,
            0.5286700959026042
          ]
        }
      }
    },
    "openbookqa": {
      "samples": 500,
      "dataset": "allenai/openbookqa",
      "config": "main",
      "split": "test",
      "primary": "acc_norm",
      "metrics": {
        "acc": {
          "value": 0.142,
          "stderr": 0.01562563024781025,
          "ci95": [
            0.114129360445696,
            0.1753296673247754
          ]
        },
        "acc_norm": {
          "value": 0.26,
          "stderr": 0.019635965529725526,
          "ci95": [
            0.22348572318538285,
            0.3001739602361623
          ]
        }
      }
    },
    "boolq": {
      "samples": 3270,
      "dataset": "aps/super_glue",
      "config": "boolq",
      "split": "validation",
      "primary": "acc",
      "metrics": {
        "acc": {
          "value": 0.5296636085626911,
          "stderr": 0.008729651343606093,
          "ci95": [
            0.5125316201633632,
            0.5467259836150076
          ]
        }
      }
    },
    "lambada_openai": {
      "samples": 5153,
      "dataset": "EleutherAI/lambada_openai",
      "config": "default",
      "split": "test",
      "primary": "acc",
      "metrics": {
        "acc": {
          "value": 0.42208422278284496,
          "stderr": 0.0068808787917354055,
          "ci95": [
            0.4086621968305708,
            0.4356223315164309
          ]
        },
        "perplexity": {
          "value": 112.36630812906695,
          "stderr": 7.78227994736911
        }
      }
    },
    "arithmark3": {
      "label": "ArithMark-3",
      "dataset": "AxiomicLabs/Arithmark-3.0",
      "config": "default",
      "split": "train",
      "samples": 1000,
      "shots": 0,
      "primary": "acc_norm",
      "metrics": {
        "acc": {
          "value": 0.332,
          "stderr": 0.014899597242811565,
          "ci95": [
            0.30350363489239063,
            0.36178215595875596
          ],
          "ci_method": "Wilson 95%; item independence approximation"
        },
        "acc_norm": {
          "value": 0.332,
          "stderr": 0.014899597242811565,
          "ci95": [
            0.30350363489239063,
            0.36178215595875596
          ],
          "ci_method": "Wilson 95%; item independence approximation"
        }
      },
      "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/arithmark3.json",
      "result_sha256": "68430957620991dec536d3717f684c1b92d6921b9e324940f542ceb62109ec0f",
      "categories": {
        "elementary_school_math_continuation::addition::grades_1_2::easy": {
          "count": 128,
          "acc_norm,none": 0.21875,
          "acc_norm_stderr,none": 0.03668319762764659
        },
        "elementary_school_math_continuation::comparison::grades_2_3::medium": {
          "count": 44,
          "acc_norm,none": 0.18181818181818182,
          "acc_norm_stderr,none": 0.05881787629278459
        },
        "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
          "count": 48,
          "acc_norm,none": 0.2708333333333333,
          "acc_norm_stderr,none": 0.06482097094483913
        },
        "elementary_school_math_continuation::data::grades_2_3::easy": {
          "count": 43,
          "acc_norm,none": 0.16279069767441862,
          "acc_norm_stderr,none": 0.056964877739143674
        },
        "elementary_school_math_continuation::division::grades_3_4::medium": {
          "count": 54,
          "acc_norm,none": 0.37037037037037035,
          "acc_norm_stderr,none": 0.06633194954624339
        },
        "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
          "count": 50,
          "acc_norm,none": 0.08,
          "acc_norm_stderr,none": 0.03875617133214439
        },
        "elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
          "count": 52,
          "acc_norm,none": 0.46153846153846156,
          "acc_norm_stderr,none": 0.06980655484407924
        },
        "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
          "count": 45,
          "acc_norm,none": 0.4666666666666667,
          "acc_norm_stderr,none": 0.0752101433090355
        },
        "elementary_school_math_continuation::measurement::grades_2_3::easy": {
          "count": 76,
          "acc_norm,none": 0.32894736842105265,
          "acc_norm_stderr,none": 0.05425138981075869
        },
        "elementary_school_math_continuation::money::grades_3_4::medium": {
          "count": 64,
          "acc_norm,none": 0.375,
          "acc_norm_stderr,none": 0.060993754559283325
        },
        "elementary_school_math_continuation::multiplication::grades_3_4::medium": {
          "count": 74,
          "acc_norm,none": 0.5675675675675675,
          "acc_norm_stderr,none": 0.057983774751431
        },
        "elementary_school_math_continuation::patterns::grades_3_4::medium": {
          "count": 53,
          "acc_norm,none": 0.16981132075471697,
          "acc_norm_stderr,none": 0.05206789873629056
        },
        "elementary_school_math_continuation::subtraction::grades_1_2::easy": {
          "count": 117,
          "acc_norm,none": 0.23931623931623933,
          "acc_norm_stderr,none": 0.03961495460787807
        },
        "elementary_school_math_continuation::time::grades_2_3::easy": {
          "count": 55,
          "acc_norm,none": 0.9818181818181818,
          "acc_norm_stderr,none": 0.018181818181818184
        },
        "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
          "count": 46,
          "acc_norm,none": 0.21739130434782608,
          "acc_norm_stderr,none": 0.06148754619013457
        },
        "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
          "count": 19,
          "acc_norm,none": 0.3157894736842105,
          "acc_norm_stderr,none": 0.10956136839295436
        },
        "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
          "count": 32,
          "acc_norm,none": 0.28125,
          "acc_norm_stderr,none": 0.08075219711382271
        }
      }
    },
    "balanced_copa": {
      "label": "Balanced COPA",
      "dataset": "pkavumba/balanced-copa",
      "config": "default",
      "split": "train",
      "samples": 1000,
      "shots": 0,
      "primary": "acc",
      "metrics": {
        "acc": {
          "value": 0.522,
          "stderr": 0.01580397942816194,
          "ci95": [
            0.4910152521271681,
            0.5528163704994674
          ],
          "ci_method": "Wilson 95%; item independence approximation"
        }
      },
      "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/balanced_copa.json",
      "result_sha256": "1375e8cff071a9464a7c0eded68c0afd7b944bb716d680493dfbe3301963b482",
      "categories": {}
    },
    "commonsense_qa": {
      "label": "CommonsenseQA",
      "dataset": "tau/commonsense_qa",
      "config": "default",
      "split": "validation",
      "samples": 1221,
      "shots": 0,
      "primary": "acc",
      "metrics": {
        "acc": {
          "value": 0.21457821457821458,
          "stderr": 0.011753423094216953,
          "ci95": [
            0.19246524693116476,
            0.2384815135817527
          ],
          "ci_method": "Wilson 95%; item independence approximation"
        }
      },
      "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/commonsense_qa.json",
      "result_sha256": "28a06065e6b827e085a8a03ce1bba594ae9edf1ae0cc7d5814b01be992f064bb",
      "categories": {}
    },
    "sciq": {
      "label": "SciQ (with support)",
      "dataset": "allenai/sciq",
      "config": "default",
      "split": "test",
      "samples": 1000,
      "shots": 0,
      "primary": "acc_norm",
      "metrics": {
        "acc": {
          "value": 0.723,
          "stderr": 0.014158794845306273,
          "ci95": [
            0.6944497560637553,
            0.7498435096516871
          ],
          "ci_method": "Wilson 95%; item independence approximation"
        },
        "acc_norm": {
          "value": 0.688,
          "stderr": 0.014658474370509057,
          "ci95": [
            0.6586108249018996,
            0.7159503139075316
          ],
          "ci_method": "Wilson 95%; item independence approximation"
        }
      },
      "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/sciq.json",
      "result_sha256": "0f1d7377097498ae7ea47e0c1b70941aedbcea59f3e2b3ca0c0101083e6af4ee",
      "categories": {}
    },
    "truthfulqa_mc2": {
      "label": "TruthfulQA MC2",
      "dataset": "truthfulqa/truthful_qa",
      "config": "multiple_choice",
      "split": "validation",
      "samples": 817,
      "shots": 0,
      "primary": "acc",
      "metrics": {
        "acc": {
          "value": 0.45836334441298165,
          "stderr": 0.0158524239677764
        }
      },
      "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/truthfulqa_mc2.json",
      "result_sha256": "bfc8d62f669c951f26cc2d300dbc4bd7c7787001573e0fc310c2db27c9405541",
      "categories": {}
    },
    "bananamind_base": {
      "label": "BananaMind Base 1.1",
      "dataset": "BananaMind/BananaMind-Base-Bench-1.1",
      "config": "default",
      "split": "test",
      "samples": 350,
      "shots": 0,
      "primary": "raw_accuracy",
      "metrics": {
        "raw_accuracy": {
          "value": 0.39714285714285713,
          "stderr": 0.02619195222772307,
          "ci95": [
            0.34726441786546686,
            0.4492546213676228
          ],
          "ci_method": "Wilson 95%; item independence approximation"
        }
      },
      "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/bananamind_base.json",
      "result_sha256": "5d9eb7e2d3daa94d14dd7d35a35003c17ab1f376931e739ca027198bb566dcb7",
      "categories": {
        "code_completion": {
          "count": 50,
          "raw_accuracy,none": 0.28,
          "raw_accuracy_stderr,none": 0.06414269805898186
        },
        "commonsense": {
          "count": 50,
          "raw_accuracy,none": 0.26,
          "raw_accuracy_stderr,none": 0.06266203485560375
        },
        "context_tracking": {
          "count": 50,
          "raw_accuracy,none": 0.48,
          "raw_accuracy_stderr,none": 0.07137140569598169
        },
        "language_completion": {
          "count": 50,
          "raw_accuracy,none": 0.4,
          "raw_accuracy_stderr,none": 0.06998542122237651
        },
        "logical_reasoning": {
          "count": 50,
          "raw_accuracy,none": 0.44,
          "raw_accuracy_stderr,none": 0.07091242083423346
        },
        "quantitative": {
          "count": 50,
          "raw_accuracy,none": 0.36,
          "raw_accuracy_stderr,none": 0.06857142857142857
        },
        "world_knowledge": {
          "count": 50,
          "raw_accuracy,none": 0.56,
          "raw_accuracy_stderr,none": 0.07091242083423346
        }
      }
    },
    "mmlu_continuation": {
      "label": "MMLU continuation",
      "dataset": "cais/mmlu",
      "config": "57 subjects",
      "split": "test",
      "samples": 14042,
      "shots": 0,
      "primary": "acc",
      "metrics": {
        "acc": {
          "value": 0.24483691781797465,
          "stderr": 0.0036208715458871405
        }
      },
      "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/mmlu_continuation.json",
      "result_sha256": "423e99cb8501f47e672d81cfa27934e7e8986de4b85e52c5248b7ac37a01370a",
      "categories": {}
    },
    "blimp": {
      "label": "BLiMP",
      "dataset": "nyu-mll/blimp",
      "config": "67 minimal-pair subsets",
      "split": "train",
      "samples": 67000,
      "shots": 0,
      "primary": "acc",
      "metrics": {
        "acc": {
          "value": 0.6420746268656716,
          "stderr": 0.0016890890018278603
        }
      },
      "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/blimp.json",
      "result_sha256": "585fbe3ad36be3a0616a31f078a27c405f136571da4859f6f6cd216cad0b0f09",
      "categories": {}
    }
  },
  "continuation": {
    "model": "diffusion",
    "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/diffusion-v2-step15000-20260923",
    "checkpoint_sha256": "d0d24e0cc492cd5c9a613710c454af9a6e49265d2bae7abb69a673a53efe2bfb",
    "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc",
    "blocks": 291,
    "scored_tokens": 148992,
    "dropped_tail_tokens": 335,
    "method": "Percentile bootstrap over nonoverlapping blocks; 10000 resamples; seed 2026",
    "device": "RTX 5070 Ti",
    "measurements": {
      "experimental_bfloat16_pll": {
        "nll": 2.764107143243973,
        "nll_ci95": [
          2.7211559490245905,
          2.8069273205348404
        ],
        "ppl": 15.864868609926912,
        "ppl_ci95": [
          15.197880069666004,
          16.558959594011032
        ],
        "block_nll": [
          2.7829585056751966,
          3.0317970663309097,
          2.5357599072158337,
          2.961236920207739,
          3.2387401442974806,
          3.1967140398919582,
          3.5564816743135452,
          3.2850491665303707,
          3.090988576412201,
          3.3609489277005196,
          2.591668199747801,
          2.860096715390682,
          2.708660712465644,
          2.5630124360322952,
          3.1861327439546585,
          3.1876195669174194,
          3.1324607878923416,
          2.7504578065127134,
          2.322118042036891,
          2.8061939422041178,
          2.745770934969187,
          2.9064043685793877,
          3.0713150948286057,
          1.9669531853869557,
          2.5212070886045694,
          2.7240037880837917,
          2.711319748312235,
          2.69622303545475,
          2.4561769291758537,
          2.418418664485216,
          2.9869874715805054,
          2.976748116314411,
          2.61490149050951,
          2.3598167710006237,
          2.571508329361677,
          3.2832227535545826,
          3.2237844727933407,
          3.27628917619586,
          2.325753504410386,
          2.8533600866794586,
          2.594134012237191,
          2.9107855036854744,
          2.4254464991390705,
          2.9700102768838406,
          2.4324689880013466,
          2.8034669384360313,
          2.989572286605835,
          2.574267504736781,
          2.8986573331058025,
          2.6539596505463123,
          2.5357956485822797,
          2.6015634983778,
          2.691170619800687,
          3.18845184892416,
          2.527509342879057,
          3.337221387773752,
          2.426599685102701,
          2.2993762604892254,
          2.5453836545348167,
          2.3985878052189946,
          2.3124238550662994,
          1.941805656068027,
          2.2805505823343992,
          1.8020723229274154,
          2.239573931787163,
          2.3718604035675526,
          2.2049689581617713,
          1.9899997469037771,
          2.428840108215809,
          2.531139859929681,
          2.892642453312874,
          2.466876283288002,
          3.055173859000206,
          2.871721997857094,
          2.957256019115448,
          2.7795100696384907,
          2.8655768148601055,
          3.252855319529772,
          2.9252156391739845,
          3.1850139684975147,
          2.5851380806416273,
          2.557009832933545,
          2.72511207126081,
          3.1816058568656445,
          3.1726122945547104,
          3.3162384908646345,
          2.998203292489052,
          3.166694551706314,
          3.1640372052788734,
          3.2403770945966244,
          2.8558538034558296,
          2.686061516404152,
          2.843296468257904,
          2.8421897776424885,
          2.775616243481636,
          2.861169770359993,
          2.598568834364414,
          2.4869228303432465,
          2.3929356765002012,
          3.16889875754714,
          3.0112213380634785,
          2.9401434790343046,
          1.9148766044527292,
          2.914667997509241,
          2.468462001532316,
          2.4090462122112513,
          2.695859983563423,
          2.4624281525611877,
          2.516034297645092,
          2.794669073075056,
          2.378003466874361,
          2.65300477668643,
          2.0976478420197964,
          2.430451113730669,
          2.6419609524309635,
          2.386194022372365,
          2.2934875106438994,
          2.975379640236497,
          2.562062126584351,
          2.6109477784484625,
          2.939204316586256,
          2.8489961437880993,
          2.9326409585773945,
          2.6114140758290887,
          2.8763786368072033,
          2.9938401505351067,
          3.027923382818699,
          2.5406803395599127,
          3.021192528307438,
          2.6221686378121376,
          2.8193065896630287,
          2.560953389853239,
          2.935681402683258,
          2.651722924783826,
          2.997558591887355,
          2.4939030203968287,
          2.600422801449895,
          2.754160091280937,
          2.704239597544074,
          3.2729596719145775,
          2.6149044558405876,
          2.5219095926731825,
          3.5004509277641773,
          2.8379627130925655,
          2.86722194775939,
          3.356606185436249,
          3.5334784854203463,
          3.3040704764425755,
          2.527249839156866,
          2.564117144793272,
          2.9029220640659332,
          2.633116714656353,
          2.3410460390150547,
          2.4432906024158,
          2.187035361304879,
          2.547063034027815,
          2.86611901037395,
          2.591669477522373,
          3.3610201440751553,
          2.3358615674078465,
          2.6958967745304108,
          2.375181168317795,
          2.379982866346836,
          2.2954897060990334,
          2.2123583629727364,
          2.4340793676674366,
          2.2295648148283362,
          2.6996378619223833,
          2.062216225080192,
          3.127024333924055,
          3.0209955647587776,
          3.4968365654349327,
          3.2891967333853245,
          3.7504169791936874,
          3.920315720140934,
          3.1434715762734413,
          3.397133741527796,
          2.5951910726726055,
          2.5684335939586163,
          3.7280097790062428,
          2.798967409878969,
          3.4091082997620106,
          3.200954582542181,
          3.34494037553668,
          3.300129771232605,
          3.177050609141588,
          3.1355742812156677,
          3.4357525892555714,
          3.080030655488372,
          3.220784742385149,
          3.7006002590060234,
          3.4562263805419207,
          2.4568815734237432,
          3.3255615197122097,
          3.443592596799135,
          2.7211416959762573,
          2.9859204441308975,
          2.4545959513634443,
          2.9261309653520584,
          2.821535898372531,
          2.481567721813917,
          2.4241346083581448,
          2.4054175838828087,
          2.536921963095665,
          2.763008141890168,
          2.811686184257269,
          2.6349865794181824,
          2.9586063902825117,
          2.298954177647829,
          2.9774351995438337,
          2.8946317601948977,
          2.534884534776211,
          2.7397540509700775,
          2.65107442997396,
          2.276937797665596,
          2.6924358792603016,
          2.8996455036103725,
          2.5960876122117043,
          1.8635172508656979,
          2.311602182686329,
          2.6205743737518787,
          2.710319995880127,
          2.312985338270664,
          2.3018196411430836,
          2.902193432673812,
          2.29562268871814,
          1.9515556334517896,
          2.7122908234596252,
          3.0253245178610086,
          2.458246875554323,
          2.6150007657706738,
          2.6408553197979927,
          2.363369856029749,
          2.6554111279547215,
          2.7695462368428707,
          3.1155146658420563,
          3.22971373423934,
          3.0301930978894234,
          2.495756993070245,
          2.8292034193873405,
          2.699986157938838,
          2.6294361613690853,
          2.2457251269370317,
          2.2291543846949935,
          2.319860439747572,
          2.4007096588611603,
          2.764710668474436,
          3.095868729054928,
          3.3412937577813864,
          2.6099788956344128,
          2.7982289530336857,
          3.3710330463945866,
          2.6359564252197742,
          2.645110823214054,
          3.234867185354233,
          2.7725971899926662,
          2.7251428347080946,
          2.580301821231842,
          3.114419549703598,
          2.9577612336724997,
          2.5457747634500265,
          2.733024090528488,
          2.451223786920309,
          2.302502987906337,
          2.4830800890922546,
          2.8186698192730546,
          2.3596253879368305,
          3.400495909154415,
          2.752669269219041,
          2.3634354025125504,
          2.8680862337350845,
          2.8643765076994896,
          2.3219218626618385,
          2.606395379640162,
          2.5968867540359497,
          2.208502020686865,
          2.6173937618732452,
          2.5596893802285194,
          3.2839341573417187,
          2.7687441669404507,
          2.819413423538208,
          3.018079172819853,
          2.272316951304674,
          2.629175551235676,
          2.979037892073393,
          2.7783413901925087,
          2.8200975321233273,
          3.000820927321911,
          3.51092467084527,
          3.3711718004196882,
          2.6552017582580447
        ]
      }
    },
    "diagnostics": {
      "0.001": {
        "real_context": {
          "masked_ce": 2.4482657188592953,
          "accuracy": 0.5116279069767442,
          "count": 43,
          "weighted_source_loss": 3.212751034880057,
          "most_common_prediction": " the"
        },
        "shuffled_context": {
          "masked_ce": 6.62493404122286,
          "accuracy": 0.023255813953488372,
          "count": 43,
          "weighted_source_loss": 8.693608513567597,
          "most_common_prediction": " the"
        }
      },
      "0.01": {
        "real_context": {
          "masked_ce": 2.7691402160468637,
          "accuracy": 0.4783861671469741,
          "count": 347,
          "weighted_source_loss": 2.932408615015447,
          "most_common_prediction": " the"
        },
        "shuffled_context": {
          "masked_ce": 7.788403865583349,
          "accuracy": 0.020172910662824207,
          "count": 347,
          "weighted_source_loss": 8.247607853263617,
          "most_common_prediction": " the"
        }
      },
      "0.15": {
        "real_context": {
          "masked_ce": 3.583461505108476,
          "accuracy": 0.3727547931382442,
          "count": 4955,
          "weighted_source_loss": 3.612477978070577,
          "most_common_prediction": " the"
        },
        "shuffled_context": {
          "masked_ce": 7.765570407101412,
          "accuracy": 0.022199798183652877,
          "count": 4955,
          "weighted_source_loss": 7.828450798988342,
          "most_common_prediction": " the"
        }
      },
      "0.5": {
        "real_context": {
          "masked_ce": 5.209607536318694,
          "accuracy": 0.19628775931093048,
          "count": 16486,
          "weighted_source_loss": 5.242040395736694,
          "most_common_prediction": " the"
        },
        "shuffled_context": {
          "masked_ce": 7.614242529646367,
          "accuracy": 0.027053257309232075,
          "count": 16486,
          "weighted_source_loss": 7.6616456508636475,
          "most_common_prediction": " the"
        }
      },
      "0.9": {
        "real_context": {
          "masked_ce": 6.848565318257388,
          "accuracy": 0.06728191904454925,
          "count": 29473,
          "weighted_source_loss": 6.844338840908474,
          "most_common_prediction": " the"
        },
        "shuffled_context": {
          "masked_ce": 7.424731295380518,
          "accuracy": 0.03874732806297289,
          "count": 29473,
          "weighted_source_loss": 7.420149246851603,
          "most_common_prediction": " the"
        }
      },
      "1.0": {
        "real_context": {
          "masked_ce": 8.18438184261322,
          "accuracy": 0.05010986328125,
          "count": 32768,
          "weighted_source_loss": 8.18438184261322,
          "most_common_prediction": " the"
        },
        "shuffled_context": {
          "masked_ce": 8.18438184261322,
          "accuracy": 0.05010986328125,
          "count": 32768,
          "weighted_source_loss": 8.18438184261322,
          "most_common_prediction": " the"
        }
      }
    },
    "uniform_nll": 10.39720770839918
  },
  "final_development_probe": {
    "step": 15000,
    "data_sha256": "8d231e5879a69adc4fbf5234e13e136dd89bfc9fa67c50228523e2c37c56644d",
    "metrics": {
      "0.15": {
        "real_ce": 3.861273639721408,
        "real_accuracy": 0.33048489666136727,
        "replaced_ce": 9.796257079689983,
        "replaced_accuracy": 0.015103338632750398,
        "context_gain": 5.934983439968575
      },
      "0.5": {
        "real_ce": 5.344470073507008,
        "real_accuracy": 0.1708104814137721,
        "replaced_ce": 8.847738731147166,
        "replaced_accuracy": 0.022242535039609993,
        "context_gain": 3.5032686576401577
      },
      "0.9": {
        "real_ce": 6.942881706040416,
        "real_accuracy": 0.06130268199233716,
        "replaced_ce": 7.976276613836334,
        "replaced_accuracy": 0.03221103312650459,
        "context_gain": 1.033394907795918
      }
    }
  },
  "tokenizer_sha256": "6155a4212832ddd656acd1cd45ab4acb327dbb080e3acdbec84f086adf17414e",
  "tensorboard_eval_scalars": 82,
  "updated": "2026-10-01",
  "additional_evaluation": {
    "path": "evaluation/comparison-20261001/results.json",
    "weight_sha256": "d0d24e0cc492cd5c9a613710c454af9a6e49265d2bae7abb69a673a53efe2bfb",
    "date": "2026-10-01",
    "dtype": "bfloat16",
    "tasks": [
      "arithmark3",
      "balanced_copa",
      "commonsense_qa",
      "sciq",
      "truthfulqa_mc2",
      "bananamind_base",
      "mmlu_continuation",
      "blimp"
    ]
  }
}