| { |
| "elapsed_seconds": 1349.062306778971, |
| "evaluated_at_unix": 1793127069.5127065, |
| "held_out_tokens": 101191373, |
| "model": "monody", |
| "objective": "next-token NLL", |
| "optimizer_step": 2352, |
| "pass": 7, |
| "stage": "monody-pass-7", |
| "training_tokens": 616562688, |
| "unique_training_tokens": 88080384, |
| "weights": { |
| "ema": { |
| "downstream": { |
| "arc_easy": { |
| "accuracy": 0.475, |
| "accuracy_norm": 0.433, |
| "chance": 0.25043333333333334, |
| "elapsed_seconds": 7.513357853982598, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.2626438911998918 |
| }, |
| "hellaswag": { |
| "accuracy": 0.264, |
| "accuracy_norm": 0.279, |
| "chance": 0.25, |
| "elapsed_seconds": 8.59502274915576, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.1487319894695152 |
| }, |
| "lambada": { |
| "accuracy": 0.217, |
| "accuracy_norm": null, |
| "chance": 0.0, |
| "elapsed_seconds": 3.444128228817135, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.2708033027176069, |
| "perplexity": 355.8159137348601 |
| }, |
| "mean_accuracy": 0.45649999999999996, |
| "mean_gold_bits_per_byte": 1.3049644023078721, |
| "piqa": { |
| "accuracy": 0.564, |
| "accuracy_norm": 0.564, |
| "chance": 0.5, |
| "elapsed_seconds": 5.2499302718788385, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.3786440568762732 |
| }, |
| "sciq": { |
| "accuracy": 0.726, |
| "accuracy_norm": 0.636, |
| "chance": 0.25, |
| "elapsed_seconds": 8.13590192887932, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.408172402357161 |
| }, |
| "winogrande": { |
| "accuracy": 0.493, |
| "accuracy_norm": null, |
| "chance": 0.5, |
| "elapsed_seconds": 4.800854941364378, |
| "examples": 1000, |
| "gold_bits_per_byte": 1.3607907712267844 |
| } |
| }, |
| "generalization_gap": 1.5875091574716764, |
| "held_out": { |
| "elapsed_seconds": 534.5517211458646, |
| "loss": 3.6317696267661397, |
| "loss_95ci": [ |
| 3.6271775943594977, |
| 3.636572309692387 |
| ], |
| "objective": "next-token NLL", |
| "perplexity": 37.77961331207202, |
| "tokens": 101191373, |
| "tokens_per_second": 189301.3697216918, |
| "windows": 49410 |
| }, |
| "memorization": { |
| "held_out": { |
| "continuation_tokens": 32, |
| "exact_match": 0.0, |
| "prefix_tokens": 64, |
| "token_accuracy": 0.0364990234375, |
| "windows": 256 |
| }, |
| "train": { |
| "continuation_tokens": 32, |
| "exact_match": 0.00390625, |
| "prefix_tokens": 64, |
| "token_accuracy": 0.0614013671875, |
| "windows": 256 |
| } |
| }, |
| "memorization_gap": 0.00390625, |
| "train_probe": { |
| "elapsed_seconds": 54.62738667288795, |
| "loss": 2.0442604692944633, |
| "loss_95ci": [ |
| 2.036346034331813, |
| 2.052173914243236 |
| ], |
| "objective": "next-token NLL", |
| "perplexity": 7.72344470088251, |
| "tokens": 10000384, |
| "tokens_per_second": 183065.39281995854, |
| "windows": 4883 |
| } |
| }, |
| "normal": { |
| "downstream": null, |
| "generalization_gap": 1.5069047089964025, |
| "held_out": { |
| "elapsed_seconds": 637.9913326767273, |
| "loss": 3.740345572211005, |
| "loss_95ci": [ |
| 3.7358146551118874, |
| 3.745179232388174 |
| ], |
| "objective": "next-token NLL", |
| "perplexity": 42.11254057450087, |
| "tokens": 101191373, |
| "tokens_per_second": 158609.32244243208, |
| "windows": 49410 |
| }, |
| "memorization": { |
| "held_out": { |
| "continuation_tokens": 32, |
| "exact_match": 0.0, |
| "prefix_tokens": 64, |
| "token_accuracy": 0.0335693359375, |
| "windows": 256 |
| }, |
| "train": { |
| "continuation_tokens": 32, |
| "exact_match": 0.00390625, |
| "prefix_tokens": 64, |
| "token_accuracy": 0.0570068359375, |
| "windows": 256 |
| } |
| }, |
| "memorization_gap": 0.00390625, |
| "train_probe": { |
| "elapsed_seconds": 67.42786940094084, |
| "loss": 2.2334408632146023, |
| "loss_95ci": [ |
| 2.223836386290301, |
| 2.2425792741355606 |
| ], |
| "objective": "next-token NLL", |
| "perplexity": 9.331920762841568, |
| "tokens": 10000384, |
| "tokens_per_second": 148312.32380390863, |
| "windows": 4883 |
| } |
| } |
| } |
| } |
|
|
