| { |
| "elapsed_seconds": 2117.840530227404, |
| "evaluated_at_unix": 1793123924.65349, |
| "held_out_tokens": 101191373, |
| "model": "rondo", |
| "objective": "masked-diffusion ELBO", |
| "optimizer_step": 2352, |
| "pass": 7, |
| "stage": "rondo-pass-7", |
| "training_tokens": 616562688, |
| "unique_training_tokens": 88080384, |
| "weights": { |
| "ema": { |
| "downstream": { |
| "arc_easy": { |
| "accuracy": 0.399, |
| "accuracy_norm": 0.379, |
| "chance": 0.25043333333333334, |
| "elapsed_seconds": 22.55707299383357, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.5026144015188456 |
| }, |
| "hellaswag": { |
| "accuracy": 0.258, |
| "accuracy_norm": 0.267, |
| "chance": 0.25, |
| "elapsed_seconds": 46.325808217749, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.2376718155048758 |
| }, |
| "lambada": { |
| "accuracy": 0.26, |
| "accuracy_norm": null, |
| "chance": 0.0, |
| "elapsed_seconds": 20.816600896883756, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.3992670085982348, |
| "perplexity": 644.3548869575394 |
| }, |
| "mean_accuracy": 0.4483333333333333, |
| "mean_gold_bits_per_byte": 1.440707633064486, |
| "piqa": { |
| "accuracy": 0.565, |
| "accuracy_norm": 0.549, |
| "chance": 0.5, |
| "elapsed_seconds": 17.97771777305752, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.504813773789007 |
| }, |
| "sciq": { |
| "accuracy": 0.718, |
| "accuracy_norm": 0.6, |
| "chance": 0.25, |
| "elapsed_seconds": 56.67296985397115, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.4959449151871012 |
| }, |
| "winogrande": { |
| "accuracy": 0.49, |
| "accuracy_norm": null, |
| "chance": 0.5, |
| "elapsed_seconds": 16.03261487232521, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.5039338837888518 |
| } |
| }, |
| "generalization_gap": 0.12874157542601106, |
| "held_out": { |
| "elapsed_seconds": 843.1091586039402, |
| "loss": 3.810009565082785, |
| "loss_95ci": [ |
| 3.7958800360865435, |
| 3.824440118452423 |
| ], |
| "masked_ce_by_ratio": { |
| "0.0-0.1": 1.8569215830841097, |
| "0.1-0.2": 2.0926027476867586, |
| "0.2-0.3": 2.4039280896118735, |
| "0.3-0.4": 2.77021456808516, |
| "0.4-0.5": 3.2149921125797234, |
| "0.5-0.6": 3.755035922527032, |
| "0.6-0.7": 4.368418108977536, |
| "0.7-0.8": 5.078711144805392, |
| "0.8-0.9": 5.853509515575058, |
| "0.9-1.0": 6.829785668435118 |
| }, |
| "objective": "masked-diffusion ELBO (upper bound on NLL)", |
| "perplexity": 45.150870736069685, |
| "tokens": 101191373, |
| "tokens_per_second": 120021.6744977097, |
| "windows": 49410 |
| }, |
| "memorization": { |
| "held_out": { |
| "continuation_tokens": 32, |
| "exact_match": 0.00390625, |
| "infill_exact_match": 0.0, |
| "infill_token_accuracy": 0.051025390625, |
| "prefix_tokens": 64, |
| "token_accuracy": 0.0362548828125, |
| "windows": 256 |
| }, |
| "train": { |
| "continuation_tokens": 32, |
| "exact_match": 0.0, |
| "infill_exact_match": 0.0, |
| "infill_token_accuracy": 0.05078125, |
| "prefix_tokens": 64, |
| "token_accuracy": 0.0321044921875, |
| "windows": 256 |
| } |
| }, |
| "memorization_gap": -0.00390625, |
| "train_probe": { |
| "elapsed_seconds": 81.37673071911559, |
| "loss": 3.681267989656774, |
| "loss_95ci": [ |
| 3.6352494597473455, |
| 3.729074858372841 |
| ], |
| "masked_ce_by_ratio": { |
| "0.0-0.1": 1.7252946729235736, |
| "0.1-0.2": 1.9484836446175906, |
| "0.2-0.3": 2.2960490385393233, |
| "0.3-0.4": 2.625906797500246, |
| "0.4-0.5": 3.047773674520883, |
| "0.5-0.6": 3.60300183416652, |
| "0.6-0.7": 4.21702739636985, |
| "0.7-0.8": 4.9430757709797035, |
| "0.8-0.9": 5.762470387557617, |
| "0.9-1.0": 6.762598411642805 |
| }, |
| "objective": "masked-diffusion ELBO (upper bound on NLL)", |
| "perplexity": 39.69669717535225, |
| "tokens": 10000384, |
| "tokens_per_second": 122889.97003968956, |
| "windows": 4883 |
| } |
| }, |
| "normal": { |
| "downstream": null, |
| "generalization_gap": 0.13213724887151423, |
| "held_out": { |
| "elapsed_seconds": 840.4453170802444, |
| "loss": 3.9165963032634092, |
| "loss_95ci": [ |
| 3.9026294513848856, |
| 3.930944980468664 |
| ], |
| "masked_ce_by_ratio": { |
| "0.0-0.1": 1.962205365840126, |
| "0.1-0.2": 2.2046403235621645, |
| "0.2-0.3": 2.521042147891554, |
| "0.3-0.4": 2.89163803690958, |
| "0.4-0.5": 3.339016312871816, |
| "0.5-0.6": 3.8776453257371855, |
| "0.6-0.7": 4.485782267280772, |
| "0.7-0.8": 5.184232049039673, |
| "0.8-0.9": 5.939894414170329, |
| "0.9-1.0": 6.884795573489192 |
| }, |
| "objective": "masked-diffusion ELBO (upper bound on NLL)", |
| "perplexity": 50.229188566092304, |
| "tokens": 101191373, |
| "tokens_per_second": 120402.09034841752, |
| "windows": 49410 |
| }, |
| "memorization": { |
| "held_out": { |
| "continuation_tokens": 32, |
| "exact_match": 0.00390625, |
| "infill_exact_match": 0.0, |
| "infill_token_accuracy": 0.0523681640625, |
| "prefix_tokens": 64, |
| "token_accuracy": 0.03466796875, |
| "windows": 256 |
| }, |
| "train": { |
| "continuation_tokens": 32, |
| "exact_match": 0.0, |
| "infill_exact_match": 0.0, |
| "infill_token_accuracy": 0.052734375, |
| "prefix_tokens": 64, |
| "token_accuracy": 0.0333251953125, |
| "windows": 256 |
| } |
| }, |
| "memorization_gap": -0.00390625, |
| "train_probe": { |
| "elapsed_seconds": 117.16330459574237, |
| "loss": 3.784459054391895, |
| "loss_95ci": [ |
| 3.7387899501117454, |
| 3.831598950549923 |
| ], |
| "masked_ce_by_ratio": { |
| "0.0-0.1": 1.8213217645513433, |
| "0.1-0.2": 2.052082935279734, |
| "0.2-0.3": 2.406457547782629, |
| "0.3-0.4": 2.742347328385029, |
| "0.4-0.5": 3.16785829172493, |
| "0.5-0.6": 3.723176236144619, |
| "0.6-0.7": 4.332039606894565, |
| "0.7-0.8": 5.048179276594368, |
| "0.8-0.9": 5.847315192536721, |
| "0.9-1.0": 6.820226933626609 |
| }, |
| "objective": "masked-diffusion ELBO (upper bound on NLL)", |
| "perplexity": 44.01185609790593, |
| "tokens": 10000384, |
| "tokens_per_second": 85354.23300413981, |
| "windows": 4883 |
| } |
| } |
| } |
| } |
|
|
