| { |
| "model": "diffusion-v2", |
| "step": 15000, |
| "source_tokens": 3932160000, |
| "stored_parameters": 51392512, |
| "optimized_parameters": 50867200, |
| "training_duration_minutes": 439.5, |
| "training_loss": 5.127222061157227, |
| "protocol": "EXPERIMENTAL: continuation PLL; single-mask left-to-right greedy generation; NOT AR likelihood", |
| "manifest": { |
| "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/diffusion-v2-step15000-20260923", |
| "local_model": "diffusion", |
| "kind": "diffusion", |
| "tokenizer": "/mnt/d/Projects/tiny_llm/tokenizer", |
| "revision": "main", |
| "tokenizer_revision": null, |
| "protocol": "EXPERIMENTAL: continuation PLL; single-mask left-to-right greedy generation; NOT AR likelihood", |
| "harness_version": "0.4.12", |
| "smoke_only": false, |
| "tasks": [ |
| { |
| "task": "hellaswag", |
| "shots": 0, |
| "description": "Commonsense completion; acc and acc_norm", |
| "unsafe_code": false, |
| "rolling": false |
| }, |
| { |
| "task": "arc_easy", |
| "shots": 0, |
| "description": "AI2 ARC-Easy science questions; acc and acc_norm", |
| "unsafe_code": false, |
| "rolling": false |
| }, |
| { |
| "task": "arc_challenge", |
| "shots": 0, |
| "description": "AI2 ARC-Challenge; acc and acc_norm", |
| "unsafe_code": false, |
| "rolling": false |
| }, |
| { |
| "task": "piqa", |
| "shots": 0, |
| "description": "Physical commonsense; acc and acc_norm", |
| "unsafe_code": false, |
| "rolling": false |
| }, |
| { |
| "task": "winogrande", |
| "shots": 0, |
| "description": "Pronoun resolution; acc", |
| "unsafe_code": false, |
| "rolling": false |
| }, |
| { |
| "task": "openbookqa", |
| "shots": 0, |
| "description": "Science multiple choice; acc and acc_norm", |
| "unsafe_code": false, |
| "rolling": false |
| }, |
| { |
| "task": "boolq", |
| "shots": 0, |
| "description": "Yes/no reading comprehension; acc", |
| "unsafe_code": false, |
| "rolling": false |
| }, |
| { |
| "task": "lambada_openai", |
| "shots": 0, |
| "description": "Complete final word; acc and perplexity", |
| "unsafe_code": false, |
| "rolling": false |
| } |
| ], |
| "options": { |
| "checkpoint": "runs/eval/diffusion-v2-step15000-20260923", |
| "model": null, |
| "tokenizer": null, |
| "revision": "main", |
| "tokenizer_revision": null, |
| "suite": "core", |
| "tasks": null, |
| "num_fewshot": null, |
| "device": "cuda:0", |
| "dtype": "bfloat16", |
| "batch_size": 1, |
| "max_length": 2048, |
| "max_gen_toks": null, |
| "limit": null, |
| "seed": 1234, |
| "threads": 4, |
| "bootstrap_iters": 1000, |
| "trust_remote_code": false, |
| "allow_code_execution": false, |
| "apply_chat_template": false, |
| "log_samples": false, |
| "output": "runs/eval/diffusion-v2-core-full-20260923", |
| "list": false, |
| "dry_run": false |
| }, |
| "checkpoint_stamp": { |
| "model.safetensors": [ |
| 205580928, |
| 1790122976166062900 |
| ], |
| "config.json": [ |
| 677, |
| 1790122975189825600 |
| ] |
| }, |
| "checkpoint_sha256": { |
| "config.json": "8e3daed6864d9b1f39be95c7b311a93c47efd4e76ee33fb4dbeede6d4701343d", |
| "model.safetensors": "d0d24e0cc492cd5c9a613710c454af9a6e49265d2bae7abb69a673a53efe2bfb" |
| }, |
| "source_sha256": { |
| "src/tiny_llm/__init__.py": "8c7f075d51006a53fe1d645ef5a84d65db58ff4251e7544074fe29ee7549c14c", |
| "src/tiny_llm/benchmarks.py": "5d9f461366389bd1f926c6a6468192f3e2313c87caf5b1de27f407d563a858a4", |
| "src/tiny_llm/checkpoint.py": "ac9df5f8c981a47eb1f1e59d0687ff41a1f6c38ca74ced50f71ac446a58d6ef5", |
| "src/tiny_llm/continuation.py": "548b7e76401cd545af3e7ff1aedf041fc96a330c8c9cc80266e0e20830f0e95d", |
| "src/tiny_llm/data.py": "973e4ff3e7b6e80c74bd99173f89cb8ce94c6ef3a21ff54acf1a42eab9654d89", |
| "src/tiny_llm/diffusion_diagnostics.py": "2ae667a076b1d3f057a29d20f00559068eb73a073b3f99030e1d5b4c2f478fa5", |
| "src/tiny_llm/evaluation/__init__.py": "e49e80f7be3ab6e200f0db14b028e465de9caa8f6f705430eabce0c41b8322ba", |
| "src/tiny_llm/evaluation/adapters.py": "8e64e5542bbcb0f9ad448b320e712916a9559823987e298b22321a8d8e2f727f", |
| "src/tiny_llm/evaluation/catalog.py": "5681c9433c67890644488f7f73280e18662f28a5de8c18374443bd7c84ba4be9", |
| "src/tiny_llm/evaluation/cli.py": "31dccd47b5ae441bbc523bc6d113f22ed7134b2b42612224fee7f1d88911801f", |
| "src/tiny_llm/losses.py": "592a05287369f6e5077b3faca404a9392de4772cf06389fb8aa7e36924e74f86", |
| "src/tiny_llm/models/__init__.py": "55f88bbd8cf8bfd70e5e17c1d0050299412528385c3714bb712623f40b0a681d", |
| "src/tiny_llm/models/diffusion/__init__.py": "c92eaf5986d604d8aaf1894a862a217cefa32ab25359175b88110e45a71c1b25", |
| "src/tiny_llm/models/diffusion/configuration_diffusion_lm.py": "3b303aba60f0a831c966f5cfa7f5441daa2584814e335646195bd6ceecfa6d0c", |
| "src/tiny_llm/models/diffusion/modeling_diffusion_lm.py": "4fc21c2aed1d5a231afe6f7634d9d8c3682b4e2abfc8fecef27b52136de77b2f", |
| "src/tiny_llm/models/looped/__init__.py": "4b46b426021c59ed313cb2f49afc1926ad2793d771ef90117e3fef0462957cff", |
| "src/tiny_llm/models/looped/configuration_looped_lm.py": "b8d99c52b98e0a97a4cae76ca4c20affd8f4d019ec38e0a9beefd5fad864dc25", |
| "src/tiny_llm/models/looped/modeling_looped_lm.py": "9bdb3c5e3aaae663d09e98055f95df1adbc42e0a4bf5dffb7f6c39242af41b9e", |
| "src/tiny_llm/models/moe_t5/__init__.py": "18e33fe38b2049521f573512ac8796afa209e1b25c2052784e1c927a7160d66c", |
| "src/tiny_llm/models/moe_t5/configuration_alicet5_moe.py": "4647b1e9ab9ce07887bffc1abc3beb3e691962a380b4cc237aae376490bb56b7", |
| "src/tiny_llm/models/moe_t5/modeling_alicet5_moe.py": "c7bbba2f01dc889cc604c7bc161354397239b28503d05f7e00665658be84751d", |
| "src/tiny_llm/models/prefixlm/__init__.py": "c94c3c861acba8d2147a3a6c46bccfc71ac8adc8d70757bede009598029d31ea", |
| "src/tiny_llm/models/prefixlm/configuration_prefix_lm.py": "20b3a62ae3628b5a64566ef19d2947c90f10c6159daad00dd13d18fc5910c42c", |
| "src/tiny_llm/models/prefixlm/modeling_prefix_lm.py": "f62a38211df175696f64945173d1e41d9f7fcf6152a7648020da8c91788e98fd", |
| "src/tiny_llm/models/q50m/__init__.py": "9d0dfec6fa0cfa6967e2f17a73d8fbeca69ba93b4a93651eda4d95626d5f5eb5", |
| "src/tiny_llm/models/q50m/configuration_q50m.py": "796806b2d5d982b633796bc7de53c3073b1b6e52d259f44ef798b63f55eaea92", |
| "src/tiny_llm/models/q50m/modeling_q50m.py": "b7a712050167fad24471193b0ca7baf4ca7e5ebdf8036c91b506d3fff5b49c8b", |
| "src/tiny_llm/telemetry.py": "f5a35de68da1a2304159fd3613fa3773bc368982ed18712791c847285b895c09", |
| "src/tiny_llm/ul2.py": "f4d948d89d5e106965612971dd473a54b57719f811761f18457c6221d763b876" |
| }, |
| "packages": { |
| "lm_eval": "0.4.12", |
| "torch": "2.11.0+cu128", |
| "transformers": "5.17.0", |
| "datasets": "5.0.1", |
| "accelerate": "1.15.0" |
| }, |
| "git_commit": "ca42011e5a82513978e4a466c3f24a5dfd4905de", |
| "git_dirty": true |
| }, |
| "tasks": { |
| "hellaswag": { |
| "samples": 10042, |
| "dataset": "Rowan/hellaswag", |
| "config": "default", |
| "split": "validation", |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.27743477394941246, |
| "stderr": 0.004468178273665403, |
| "ci95": [ |
| 0.26876410971590897, |
| 0.2862756529238705 |
| ] |
| }, |
| "acc_norm": { |
| "value": 0.2795259908384784, |
| "stderr": 0.004478491697891533, |
| "ci95": [ |
| 0.2708343265409323, |
| 0.2883862705438426 |
| ] |
| } |
| } |
| }, |
| "arc_easy": { |
| "samples": 2376, |
| "dataset": "allenai/ai2_arc", |
| "config": "ARC-Easy", |
| "split": "test", |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.37836700336700335, |
| "stderr": 0.009951575683332024, |
| "ci95": [ |
| 0.3590774700626608, |
| 0.39804920833110835 |
| ] |
| }, |
| "acc_norm": { |
| "value": 0.3362794612794613, |
| "stderr": 0.0096941780727253, |
| "ci95": [ |
| 0.3175609903739864, |
| 0.3555264763913292 |
| ] |
| } |
| } |
| }, |
| "arc_challenge": { |
| "samples": 1172, |
| "dataset": "allenai/ai2_arc", |
| "config": "ARC-Challenge", |
| "split": "test", |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.2030716723549488, |
| "stderr": 0.011755899303705644, |
| "ci95": [ |
| 0.18102762485103946, |
| 0.22705584186219357 |
| ] |
| }, |
| "acc_norm": { |
| "value": 0.2226962457337884, |
| "stderr": 0.012158314774829888, |
| "ci95": [ |
| 0.19980419390295, |
| 0.2474001931137188 |
| ] |
| } |
| } |
| }, |
| "piqa": { |
| "samples": 1838, |
| "dataset": "baber/piqa", |
| "config": "default", |
| "split": "validation", |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.5696409140369967, |
| "stderr": 0.011552114834700413, |
| "ci95": [ |
| 0.5468832486835327, |
| 0.592108084568306 |
| ] |
| }, |
| "acc_norm": { |
| "value": 0.5386289445048966, |
| "stderr": 0.011630956681145891, |
| "ci95": [ |
| 0.5157819596226829, |
| 0.5613147955404465 |
| ] |
| } |
| } |
| }, |
| "winogrande": { |
| "samples": 1267, |
| "dataset": "allenai/winogrande", |
| "config": "winogrande_xl", |
| "split": "validation", |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.5011838989739542, |
| "stderr": 0.014052446290528871, |
| "ci95": [ |
| 0.47369054474157507, |
| 0.5286700959026042 |
| ] |
| } |
| } |
| }, |
| "openbookqa": { |
| "samples": 500, |
| "dataset": "allenai/openbookqa", |
| "config": "main", |
| "split": "test", |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.142, |
| "stderr": 0.01562563024781025, |
| "ci95": [ |
| 0.114129360445696, |
| 0.1753296673247754 |
| ] |
| }, |
| "acc_norm": { |
| "value": 0.26, |
| "stderr": 0.019635965529725526, |
| "ci95": [ |
| 0.22348572318538285, |
| 0.3001739602361623 |
| ] |
| } |
| } |
| }, |
| "boolq": { |
| "samples": 3270, |
| "dataset": "aps/super_glue", |
| "config": "boolq", |
| "split": "validation", |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.5296636085626911, |
| "stderr": 0.008729651343606093, |
| "ci95": [ |
| 0.5125316201633632, |
| 0.5467259836150076 |
| ] |
| } |
| } |
| }, |
| "lambada_openai": { |
| "samples": 5153, |
| "dataset": "EleutherAI/lambada_openai", |
| "config": "default", |
| "split": "test", |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.42208422278284496, |
| "stderr": 0.0068808787917354055, |
| "ci95": [ |
| 0.4086621968305708, |
| 0.4356223315164309 |
| ] |
| }, |
| "perplexity": { |
| "value": 112.36630812906695, |
| "stderr": 7.78227994736911 |
| } |
| } |
| }, |
| "arithmark3": { |
| "label": "ArithMark-3", |
| "dataset": "AxiomicLabs/Arithmark-3.0", |
| "config": "default", |
| "split": "train", |
| "samples": 1000, |
| "shots": 0, |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.332, |
| "stderr": 0.014899597242811565, |
| "ci95": [ |
| 0.30350363489239063, |
| 0.36178215595875596 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| }, |
| "acc_norm": { |
| "value": 0.332, |
| "stderr": 0.014899597242811565, |
| "ci95": [ |
| 0.30350363489239063, |
| 0.36178215595875596 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/arithmark3.json", |
| "result_sha256": "68430957620991dec536d3717f684c1b92d6921b9e324940f542ceb62109ec0f", |
| "categories": { |
| "elementary_school_math_continuation::addition::grades_1_2::easy": { |
| "count": 128, |
| "acc_norm,none": 0.21875, |
| "acc_norm_stderr,none": 0.03668319762764659 |
| }, |
| "elementary_school_math_continuation::comparison::grades_2_3::medium": { |
| "count": 44, |
| "acc_norm,none": 0.18181818181818182, |
| "acc_norm_stderr,none": 0.05881787629278459 |
| }, |
| "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": { |
| "count": 48, |
| "acc_norm,none": 0.2708333333333333, |
| "acc_norm_stderr,none": 0.06482097094483913 |
| }, |
| "elementary_school_math_continuation::data::grades_2_3::easy": { |
| "count": 43, |
| "acc_norm,none": 0.16279069767441862, |
| "acc_norm_stderr,none": 0.056964877739143674 |
| }, |
| "elementary_school_math_continuation::division::grades_3_4::medium": { |
| "count": 54, |
| "acc_norm,none": 0.37037037037037035, |
| "acc_norm_stderr,none": 0.06633194954624339 |
| }, |
| "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": { |
| "count": 50, |
| "acc_norm,none": 0.08, |
| "acc_norm_stderr,none": 0.03875617133214439 |
| }, |
| "elementary_school_math_continuation::geometry_area::grades_4_5::medium": { |
| "count": 52, |
| "acc_norm,none": 0.46153846153846156, |
| "acc_norm_stderr,none": 0.06980655484407924 |
| }, |
| "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": { |
| "count": 45, |
| "acc_norm,none": 0.4666666666666667, |
| "acc_norm_stderr,none": 0.0752101433090355 |
| }, |
| "elementary_school_math_continuation::measurement::grades_2_3::easy": { |
| "count": 76, |
| "acc_norm,none": 0.32894736842105265, |
| "acc_norm_stderr,none": 0.05425138981075869 |
| }, |
| "elementary_school_math_continuation::money::grades_3_4::medium": { |
| "count": 64, |
| "acc_norm,none": 0.375, |
| "acc_norm_stderr,none": 0.060993754559283325 |
| }, |
| "elementary_school_math_continuation::multiplication::grades_3_4::medium": { |
| "count": 74, |
| "acc_norm,none": 0.5675675675675675, |
| "acc_norm_stderr,none": 0.057983774751431 |
| }, |
| "elementary_school_math_continuation::patterns::grades_3_4::medium": { |
| "count": 53, |
| "acc_norm,none": 0.16981132075471697, |
| "acc_norm_stderr,none": 0.05206789873629056 |
| }, |
| "elementary_school_math_continuation::subtraction::grades_1_2::easy": { |
| "count": 117, |
| "acc_norm,none": 0.23931623931623933, |
| "acc_norm_stderr,none": 0.03961495460787807 |
| }, |
| "elementary_school_math_continuation::time::grades_2_3::easy": { |
| "count": 55, |
| "acc_norm,none": 0.9818181818181818, |
| "acc_norm_stderr,none": 0.018181818181818184 |
| }, |
| "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": { |
| "count": 46, |
| "acc_norm,none": 0.21739130434782608, |
| "acc_norm_stderr,none": 0.06148754619013457 |
| }, |
| "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": { |
| "count": 19, |
| "acc_norm,none": 0.3157894736842105, |
| "acc_norm_stderr,none": 0.10956136839295436 |
| }, |
| "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": { |
| "count": 32, |
| "acc_norm,none": 0.28125, |
| "acc_norm_stderr,none": 0.08075219711382271 |
| } |
| } |
| }, |
| "balanced_copa": { |
| "label": "Balanced COPA", |
| "dataset": "pkavumba/balanced-copa", |
| "config": "default", |
| "split": "train", |
| "samples": 1000, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.522, |
| "stderr": 0.01580397942816194, |
| "ci95": [ |
| 0.4910152521271681, |
| 0.5528163704994674 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/balanced_copa.json", |
| "result_sha256": "1375e8cff071a9464a7c0eded68c0afd7b944bb716d680493dfbe3301963b482", |
| "categories": {} |
| }, |
| "commonsense_qa": { |
| "label": "CommonsenseQA", |
| "dataset": "tau/commonsense_qa", |
| "config": "default", |
| "split": "validation", |
| "samples": 1221, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.21457821457821458, |
| "stderr": 0.011753423094216953, |
| "ci95": [ |
| 0.19246524693116476, |
| 0.2384815135817527 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/commonsense_qa.json", |
| "result_sha256": "28a06065e6b827e085a8a03ce1bba594ae9edf1ae0cc7d5814b01be992f064bb", |
| "categories": {} |
| }, |
| "sciq": { |
| "label": "SciQ (with support)", |
| "dataset": "allenai/sciq", |
| "config": "default", |
| "split": "test", |
| "samples": 1000, |
| "shots": 0, |
| "primary": "acc_norm", |
| "metrics": { |
| "acc": { |
| "value": 0.723, |
| "stderr": 0.014158794845306273, |
| "ci95": [ |
| 0.6944497560637553, |
| 0.7498435096516871 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| }, |
| "acc_norm": { |
| "value": 0.688, |
| "stderr": 0.014658474370509057, |
| "ci95": [ |
| 0.6586108249018996, |
| 0.7159503139075316 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/sciq.json", |
| "result_sha256": "0f1d7377097498ae7ea47e0c1b70941aedbcea59f3e2b3ca0c0101083e6af4ee", |
| "categories": {} |
| }, |
| "truthfulqa_mc2": { |
| "label": "TruthfulQA MC2", |
| "dataset": "truthfulqa/truthful_qa", |
| "config": "multiple_choice", |
| "split": "validation", |
| "samples": 817, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.45836334441298165, |
| "stderr": 0.0158524239677764 |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/truthfulqa_mc2.json", |
| "result_sha256": "bfc8d62f669c951f26cc2d300dbc4bd7c7787001573e0fc310c2db27c9405541", |
| "categories": {} |
| }, |
| "bananamind_base": { |
| "label": "BananaMind Base 1.1", |
| "dataset": "BananaMind/BananaMind-Base-Bench-1.1", |
| "config": "default", |
| "split": "test", |
| "samples": 350, |
| "shots": 0, |
| "primary": "raw_accuracy", |
| "metrics": { |
| "raw_accuracy": { |
| "value": 0.39714285714285713, |
| "stderr": 0.02619195222772307, |
| "ci95": [ |
| 0.34726441786546686, |
| 0.4492546213676228 |
| ], |
| "ci_method": "Wilson 95%; item independence approximation" |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/bananamind_base.json", |
| "result_sha256": "5d9eb7e2d3daa94d14dd7d35a35003c17ab1f376931e739ca027198bb566dcb7", |
| "categories": { |
| "code_completion": { |
| "count": 50, |
| "raw_accuracy,none": 0.28, |
| "raw_accuracy_stderr,none": 0.06414269805898186 |
| }, |
| "commonsense": { |
| "count": 50, |
| "raw_accuracy,none": 0.26, |
| "raw_accuracy_stderr,none": 0.06266203485560375 |
| }, |
| "context_tracking": { |
| "count": 50, |
| "raw_accuracy,none": 0.48, |
| "raw_accuracy_stderr,none": 0.07137140569598169 |
| }, |
| "language_completion": { |
| "count": 50, |
| "raw_accuracy,none": 0.4, |
| "raw_accuracy_stderr,none": 0.06998542122237651 |
| }, |
| "logical_reasoning": { |
| "count": 50, |
| "raw_accuracy,none": 0.44, |
| "raw_accuracy_stderr,none": 0.07091242083423346 |
| }, |
| "quantitative": { |
| "count": 50, |
| "raw_accuracy,none": 0.36, |
| "raw_accuracy_stderr,none": 0.06857142857142857 |
| }, |
| "world_knowledge": { |
| "count": 50, |
| "raw_accuracy,none": 0.56, |
| "raw_accuracy_stderr,none": 0.07091242083423346 |
| } |
| } |
| }, |
| "mmlu_continuation": { |
| "label": "MMLU continuation", |
| "dataset": "cais/mmlu", |
| "config": "57 subjects", |
| "split": "test", |
| "samples": 14042, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.24483691781797465, |
| "stderr": 0.0036208715458871405 |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/mmlu_continuation.json", |
| "result_sha256": "423e99cb8501f47e672d81cfa27934e7e8986de4b85e52c5248b7ac37a01370a", |
| "categories": {} |
| }, |
| "blimp": { |
| "label": "BLiMP", |
| "dataset": "nyu-mll/blimp", |
| "config": "67 minimal-pair subsets", |
| "split": "train", |
| "samples": 67000, |
| "shots": 0, |
| "primary": "acc", |
| "metrics": { |
| "acc": { |
| "value": 0.6420746268656716, |
| "stderr": 0.0016890890018278603 |
| } |
| }, |
| "result_path": "runs/eval/cria-backfill-20261001/diffusion-v2/attempt-001/blimp.json", |
| "result_sha256": "585fbe3ad36be3a0616a31f078a27c405f136571da4859f6f6cd216cad0b0f09", |
| "categories": {} |
| } |
| }, |
| "continuation": { |
| "model": "diffusion", |
| "checkpoint": "/mnt/d/Projects/tiny_llm/runs/eval/diffusion-v2-step15000-20260923", |
| "checkpoint_sha256": "d0d24e0cc492cd5c9a613710c454af9a6e49265d2bae7abb69a673a53efe2bfb", |
| "corpus_sha256": "aca2f46735043bcfd0a44eca981d04627b9cdf74c4c9a04bf0856d04066f58fc", |
| "blocks": 291, |
| "scored_tokens": 148992, |
| "dropped_tail_tokens": 335, |
| "method": "Percentile bootstrap over nonoverlapping blocks; 10000 resamples; seed 2026", |
| "device": "RTX 5070 Ti", |
| "measurements": { |
| "experimental_bfloat16_pll": { |
| "nll": 2.764107143243973, |
| "nll_ci95": [ |
| 2.7211559490245905, |
| 2.8069273205348404 |
| ], |
| "ppl": 15.864868609926912, |
| "ppl_ci95": [ |
| 15.197880069666004, |
| 16.558959594011032 |
| ], |
| "block_nll": [ |
| 2.7829585056751966, |
| 3.0317970663309097, |
| 2.5357599072158337, |
| 2.961236920207739, |
| 3.2387401442974806, |
| 3.1967140398919582, |
| 3.5564816743135452, |
| 3.2850491665303707, |
| 3.090988576412201, |
| 3.3609489277005196, |
| 2.591668199747801, |
| 2.860096715390682, |
| 2.708660712465644, |
| 2.5630124360322952, |
| 3.1861327439546585, |
| 3.1876195669174194, |
| 3.1324607878923416, |
| 2.7504578065127134, |
| 2.322118042036891, |
| 2.8061939422041178, |
| 2.745770934969187, |
| 2.9064043685793877, |
| 3.0713150948286057, |
| 1.9669531853869557, |
| 2.5212070886045694, |
| 2.7240037880837917, |
| 2.711319748312235, |
| 2.69622303545475, |
| 2.4561769291758537, |
| 2.418418664485216, |
| 2.9869874715805054, |
| 2.976748116314411, |
| 2.61490149050951, |
| 2.3598167710006237, |
| 2.571508329361677, |
| 3.2832227535545826, |
| 3.2237844727933407, |
| 3.27628917619586, |
| 2.325753504410386, |
| 2.8533600866794586, |
| 2.594134012237191, |
| 2.9107855036854744, |
| 2.4254464991390705, |
| 2.9700102768838406, |
| 2.4324689880013466, |
| 2.8034669384360313, |
| 2.989572286605835, |
| 2.574267504736781, |
| 2.8986573331058025, |
| 2.6539596505463123, |
| 2.5357956485822797, |
| 2.6015634983778, |
| 2.691170619800687, |
| 3.18845184892416, |
| 2.527509342879057, |
| 3.337221387773752, |
| 2.426599685102701, |
| 2.2993762604892254, |
| 2.5453836545348167, |
| 2.3985878052189946, |
| 2.3124238550662994, |
| 1.941805656068027, |
| 2.2805505823343992, |
| 1.8020723229274154, |
| 2.239573931787163, |
| 2.3718604035675526, |
| 2.2049689581617713, |
| 1.9899997469037771, |
| 2.428840108215809, |
| 2.531139859929681, |
| 2.892642453312874, |
| 2.466876283288002, |
| 3.055173859000206, |
| 2.871721997857094, |
| 2.957256019115448, |
| 2.7795100696384907, |
| 2.8655768148601055, |
| 3.252855319529772, |
| 2.9252156391739845, |
| 3.1850139684975147, |
| 2.5851380806416273, |
| 2.557009832933545, |
| 2.72511207126081, |
| 3.1816058568656445, |
| 3.1726122945547104, |
| 3.3162384908646345, |
| 2.998203292489052, |
| 3.166694551706314, |
| 3.1640372052788734, |
| 3.2403770945966244, |
| 2.8558538034558296, |
| 2.686061516404152, |
| 2.843296468257904, |
| 2.8421897776424885, |
| 2.775616243481636, |
| 2.861169770359993, |
| 2.598568834364414, |
| 2.4869228303432465, |
| 2.3929356765002012, |
| 3.16889875754714, |
| 3.0112213380634785, |
| 2.9401434790343046, |
| 1.9148766044527292, |
| 2.914667997509241, |
| 2.468462001532316, |
| 2.4090462122112513, |
| 2.695859983563423, |
| 2.4624281525611877, |
| 2.516034297645092, |
| 2.794669073075056, |
| 2.378003466874361, |
| 2.65300477668643, |
| 2.0976478420197964, |
| 2.430451113730669, |
| 2.6419609524309635, |
| 2.386194022372365, |
| 2.2934875106438994, |
| 2.975379640236497, |
| 2.562062126584351, |
| 2.6109477784484625, |
| 2.939204316586256, |
| 2.8489961437880993, |
| 2.9326409585773945, |
| 2.6114140758290887, |
| 2.8763786368072033, |
| 2.9938401505351067, |
| 3.027923382818699, |
| 2.5406803395599127, |
| 3.021192528307438, |
| 2.6221686378121376, |
| 2.8193065896630287, |
| 2.560953389853239, |
| 2.935681402683258, |
| 2.651722924783826, |
| 2.997558591887355, |
| 2.4939030203968287, |
| 2.600422801449895, |
| 2.754160091280937, |
| 2.704239597544074, |
| 3.2729596719145775, |
| 2.6149044558405876, |
| 2.5219095926731825, |
| 3.5004509277641773, |
| 2.8379627130925655, |
| 2.86722194775939, |
| 3.356606185436249, |
| 3.5334784854203463, |
| 3.3040704764425755, |
| 2.527249839156866, |
| 2.564117144793272, |
| 2.9029220640659332, |
| 2.633116714656353, |
| 2.3410460390150547, |
| 2.4432906024158, |
| 2.187035361304879, |
| 2.547063034027815, |
| 2.86611901037395, |
| 2.591669477522373, |
| 3.3610201440751553, |
| 2.3358615674078465, |
| 2.6958967745304108, |
| 2.375181168317795, |
| 2.379982866346836, |
| 2.2954897060990334, |
| 2.2123583629727364, |
| 2.4340793676674366, |
| 2.2295648148283362, |
| 2.6996378619223833, |
| 2.062216225080192, |
| 3.127024333924055, |
| 3.0209955647587776, |
| 3.4968365654349327, |
| 3.2891967333853245, |
| 3.7504169791936874, |
| 3.920315720140934, |
| 3.1434715762734413, |
| 3.397133741527796, |
| 2.5951910726726055, |
| 2.5684335939586163, |
| 3.7280097790062428, |
| 2.798967409878969, |
| 3.4091082997620106, |
| 3.200954582542181, |
| 3.34494037553668, |
| 3.300129771232605, |
| 3.177050609141588, |
| 3.1355742812156677, |
| 3.4357525892555714, |
| 3.080030655488372, |
| 3.220784742385149, |
| 3.7006002590060234, |
| 3.4562263805419207, |
| 2.4568815734237432, |
| 3.3255615197122097, |
| 3.443592596799135, |
| 2.7211416959762573, |
| 2.9859204441308975, |
| 2.4545959513634443, |
| 2.9261309653520584, |
| 2.821535898372531, |
| 2.481567721813917, |
| 2.4241346083581448, |
| 2.4054175838828087, |
| 2.536921963095665, |
| 2.763008141890168, |
| 2.811686184257269, |
| 2.6349865794181824, |
| 2.9586063902825117, |
| 2.298954177647829, |
| 2.9774351995438337, |
| 2.8946317601948977, |
| 2.534884534776211, |
| 2.7397540509700775, |
| 2.65107442997396, |
| 2.276937797665596, |
| 2.6924358792603016, |
| 2.8996455036103725, |
| 2.5960876122117043, |
| 1.8635172508656979, |
| 2.311602182686329, |
| 2.6205743737518787, |
| 2.710319995880127, |
| 2.312985338270664, |
| 2.3018196411430836, |
| 2.902193432673812, |
| 2.29562268871814, |
| 1.9515556334517896, |
| 2.7122908234596252, |
| 3.0253245178610086, |
| 2.458246875554323, |
| 2.6150007657706738, |
| 2.6408553197979927, |
| 2.363369856029749, |
| 2.6554111279547215, |
| 2.7695462368428707, |
| 3.1155146658420563, |
| 3.22971373423934, |
| 3.0301930978894234, |
| 2.495756993070245, |
| 2.8292034193873405, |
| 2.699986157938838, |
| 2.6294361613690853, |
| 2.2457251269370317, |
| 2.2291543846949935, |
| 2.319860439747572, |
| 2.4007096588611603, |
| 2.764710668474436, |
| 3.095868729054928, |
| 3.3412937577813864, |
| 2.6099788956344128, |
| 2.7982289530336857, |
| 3.3710330463945866, |
| 2.6359564252197742, |
| 2.645110823214054, |
| 3.234867185354233, |
| 2.7725971899926662, |
| 2.7251428347080946, |
| 2.580301821231842, |
| 3.114419549703598, |
| 2.9577612336724997, |
| 2.5457747634500265, |
| 2.733024090528488, |
| 2.451223786920309, |
| 2.302502987906337, |
| 2.4830800890922546, |
| 2.8186698192730546, |
| 2.3596253879368305, |
| 3.400495909154415, |
| 2.752669269219041, |
| 2.3634354025125504, |
| 2.8680862337350845, |
| 2.8643765076994896, |
| 2.3219218626618385, |
| 2.606395379640162, |
| 2.5968867540359497, |
| 2.208502020686865, |
| 2.6173937618732452, |
| 2.5596893802285194, |
| 3.2839341573417187, |
| 2.7687441669404507, |
| 2.819413423538208, |
| 3.018079172819853, |
| 2.272316951304674, |
| 2.629175551235676, |
| 2.979037892073393, |
| 2.7783413901925087, |
| 2.8200975321233273, |
| 3.000820927321911, |
| 3.51092467084527, |
| 3.3711718004196882, |
| 2.6552017582580447 |
| ] |
| } |
| }, |
| "diagnostics": { |
| "0.001": { |
| "real_context": { |
| "masked_ce": 2.4482657188592953, |
| "accuracy": 0.5116279069767442, |
| "count": 43, |
| "weighted_source_loss": 3.212751034880057, |
| "most_common_prediction": " the" |
| }, |
| "shuffled_context": { |
| "masked_ce": 6.62493404122286, |
| "accuracy": 0.023255813953488372, |
| "count": 43, |
| "weighted_source_loss": 8.693608513567597, |
| "most_common_prediction": " the" |
| } |
| }, |
| "0.01": { |
| "real_context": { |
| "masked_ce": 2.7691402160468637, |
| "accuracy": 0.4783861671469741, |
| "count": 347, |
| "weighted_source_loss": 2.932408615015447, |
| "most_common_prediction": " the" |
| }, |
| "shuffled_context": { |
| "masked_ce": 7.788403865583349, |
| "accuracy": 0.020172910662824207, |
| "count": 347, |
| "weighted_source_loss": 8.247607853263617, |
| "most_common_prediction": " the" |
| } |
| }, |
| "0.15": { |
| "real_context": { |
| "masked_ce": 3.583461505108476, |
| "accuracy": 0.3727547931382442, |
| "count": 4955, |
| "weighted_source_loss": 3.612477978070577, |
| "most_common_prediction": " the" |
| }, |
| "shuffled_context": { |
| "masked_ce": 7.765570407101412, |
| "accuracy": 0.022199798183652877, |
| "count": 4955, |
| "weighted_source_loss": 7.828450798988342, |
| "most_common_prediction": " the" |
| } |
| }, |
| "0.5": { |
| "real_context": { |
| "masked_ce": 5.209607536318694, |
| "accuracy": 0.19628775931093048, |
| "count": 16486, |
| "weighted_source_loss": 5.242040395736694, |
| "most_common_prediction": " the" |
| }, |
| "shuffled_context": { |
| "masked_ce": 7.614242529646367, |
| "accuracy": 0.027053257309232075, |
| "count": 16486, |
| "weighted_source_loss": 7.6616456508636475, |
| "most_common_prediction": " the" |
| } |
| }, |
| "0.9": { |
| "real_context": { |
| "masked_ce": 6.848565318257388, |
| "accuracy": 0.06728191904454925, |
| "count": 29473, |
| "weighted_source_loss": 6.844338840908474, |
| "most_common_prediction": " the" |
| }, |
| "shuffled_context": { |
| "masked_ce": 7.424731295380518, |
| "accuracy": 0.03874732806297289, |
| "count": 29473, |
| "weighted_source_loss": 7.420149246851603, |
| "most_common_prediction": " the" |
| } |
| }, |
| "1.0": { |
| "real_context": { |
| "masked_ce": 8.18438184261322, |
| "accuracy": 0.05010986328125, |
| "count": 32768, |
| "weighted_source_loss": 8.18438184261322, |
| "most_common_prediction": " the" |
| }, |
| "shuffled_context": { |
| "masked_ce": 8.18438184261322, |
| "accuracy": 0.05010986328125, |
| "count": 32768, |
| "weighted_source_loss": 8.18438184261322, |
| "most_common_prediction": " the" |
| } |
| } |
| }, |
| "uniform_nll": 10.39720770839918 |
| }, |
| "final_development_probe": { |
| "step": 15000, |
| "data_sha256": "8d231e5879a69adc4fbf5234e13e136dd89bfc9fa67c50228523e2c37c56644d", |
| "metrics": { |
| "0.15": { |
| "real_ce": 3.861273639721408, |
| "real_accuracy": 0.33048489666136727, |
| "replaced_ce": 9.796257079689983, |
| "replaced_accuracy": 0.015103338632750398, |
| "context_gain": 5.934983439968575 |
| }, |
| "0.5": { |
| "real_ce": 5.344470073507008, |
| "real_accuracy": 0.1708104814137721, |
| "replaced_ce": 8.847738731147166, |
| "replaced_accuracy": 0.022242535039609993, |
| "context_gain": 3.5032686576401577 |
| }, |
| "0.9": { |
| "real_ce": 6.942881706040416, |
| "real_accuracy": 0.06130268199233716, |
| "replaced_ce": 7.976276613836334, |
| "replaced_accuracy": 0.03221103312650459, |
| "context_gain": 1.033394907795918 |
| } |
| } |
| }, |
| "tokenizer_sha256": "6155a4212832ddd656acd1cd45ab4acb327dbb080e3acdbec84f086adf17414e", |
| "tensorboard_eval_scalars": 82, |
| "updated": "2026-10-01", |
| "additional_evaluation": { |
| "path": "evaluation/comparison-20261001/results.json", |
| "weight_sha256": "d0d24e0cc492cd5c9a613710c454af9a6e49265d2bae7abb69a673a53efe2bfb", |
| "date": "2026-10-01", |
| "dtype": "bfloat16", |
| "tasks": [ |
| "arithmark3", |
| "balanced_copa", |
| "commonsense_qa", |
| "sciq", |
| "truthfulqa_mc2", |
| "bananamind_base", |
| "mmlu_continuation", |
| "blimp" |
| ] |
| } |
| } |
|
|
