{
  "elapsed_seconds": 2117.840530227404,
  "evaluated_at_unix": 1793123924.65349,
  "held_out_tokens": 101191373,
  "model": "rondo",
  "objective": "masked-diffusion ELBO",
  "optimizer_step": 2352,
  "pass": 7,
  "stage": "rondo-pass-7",
  "training_tokens": 616562688,
  "unique_training_tokens": 88080384,
  "weights": {
    "ema": {
      "downstream": {
        "arc_easy": {
          "accuracy": 0.399,
          "accuracy_norm": 0.379,
          "chance": 0.25043333333333334,
          "elapsed_seconds": 22.55707299383357,
          "examples": 1000,
          "gold_bits_per_byte": 1.5026144015188456
        },
        "hellaswag": {
          "accuracy": 0.258,
          "accuracy_norm": 0.267,
          "chance": 0.25,
          "elapsed_seconds": 46.325808217749,
          "examples": 1000,
          "gold_bits_per_byte": 1.2376718155048758
        },
        "lambada": {
          "accuracy": 0.26,
          "accuracy_norm": null,
          "chance": 0.0,
          "elapsed_seconds": 20.816600896883756,
          "examples": 1000,
          "gold_bits_per_byte": 1.3992670085982348,
          "perplexity": 644.3548869575394
        },
        "mean_accuracy": 0.4483333333333333,
        "mean_gold_bits_per_byte": 1.440707633064486,
        "piqa": {
          "accuracy": 0.565,
          "accuracy_norm": 0.549,
          "chance": 0.5,
          "elapsed_seconds": 17.97771777305752,
          "examples": 1000,
          "gold_bits_per_byte": 1.504813773789007
        },
        "sciq": {
          "accuracy": 0.718,
          "accuracy_norm": 0.6,
          "chance": 0.25,
          "elapsed_seconds": 56.67296985397115,
          "examples": 1000,
          "gold_bits_per_byte": 1.4959449151871012
        },
        "winogrande": {
          "accuracy": 0.49,
          "accuracy_norm": null,
          "chance": 0.5,
          "elapsed_seconds": 16.03261487232521,
          "examples": 1000,
          "gold_bits_per_byte": 1.5039338837888518
        }
      },
      "generalization_gap": 0.12874157542601106,
      "held_out": {
        "elapsed_seconds": 843.1091586039402,
        "loss": 3.810009565082785,
        "loss_95ci": [
          3.7958800360865435,
          3.824440118452423
        ],
        "masked_ce_by_ratio": {
          "0.0-0.1": 1.8569215830841097,
          "0.1-0.2": 2.0926027476867586,
          "0.2-0.3": 2.4039280896118735,
          "0.3-0.4": 2.77021456808516,
          "0.4-0.5": 3.2149921125797234,
          "0.5-0.6": 3.755035922527032,
          "0.6-0.7": 4.368418108977536,
          "0.7-0.8": 5.078711144805392,
          "0.8-0.9": 5.853509515575058,
          "0.9-1.0": 6.829785668435118
        },
        "objective": "masked-diffusion ELBO (upper bound on NLL)",
        "perplexity": 45.150870736069685,
        "tokens": 101191373,
        "tokens_per_second": 120021.6744977097,
        "windows": 49410
      },
      "memorization": {
        "held_out": {
          "continuation_tokens": 32,
          "exact_match": 0.00390625,
          "infill_exact_match": 0.0,
          "infill_token_accuracy": 0.051025390625,
          "prefix_tokens": 64,
          "token_accuracy": 0.0362548828125,
          "windows": 256
        },
        "train": {
          "continuation_tokens": 32,
          "exact_match": 0.0,
          "infill_exact_match": 0.0,
          "infill_token_accuracy": 0.05078125,
          "prefix_tokens": 64,
          "token_accuracy": 0.0321044921875,
          "windows": 256
        }
      },
      "memorization_gap": -0.00390625,
      "train_probe": {
        "elapsed_seconds": 81.37673071911559,
        "loss": 3.681267989656774,
        "loss_95ci": [
          3.6352494597473455,
          3.729074858372841
        ],
        "masked_ce_by_ratio": {
          "0.0-0.1": 1.7252946729235736,
          "0.1-0.2": 1.9484836446175906,
          "0.2-0.3": 2.2960490385393233,
          "0.3-0.4": 2.625906797500246,
          "0.4-0.5": 3.047773674520883,
          "0.5-0.6": 3.60300183416652,
          "0.6-0.7": 4.21702739636985,
          "0.7-0.8": 4.9430757709797035,
          "0.8-0.9": 5.762470387557617,
          "0.9-1.0": 6.762598411642805
        },
        "objective": "masked-diffusion ELBO (upper bound on NLL)",
        "perplexity": 39.69669717535225,
        "tokens": 10000384,
        "tokens_per_second": 122889.97003968956,
        "windows": 4883
      }
    },
    "normal": {
      "downstream": null,
      "generalization_gap": 0.13213724887151423,
      "held_out": {
        "elapsed_seconds": 840.4453170802444,
        "loss": 3.9165963032634092,
        "loss_95ci": [
          3.9026294513848856,
          3.930944980468664
        ],
        "masked_ce_by_ratio": {
          "0.0-0.1": 1.962205365840126,
          "0.1-0.2": 2.2046403235621645,
          "0.2-0.3": 2.521042147891554,
          "0.3-0.4": 2.89163803690958,
          "0.4-0.5": 3.339016312871816,
          "0.5-0.6": 3.8776453257371855,
          "0.6-0.7": 4.485782267280772,
          "0.7-0.8": 5.184232049039673,
          "0.8-0.9": 5.939894414170329,
          "0.9-1.0": 6.884795573489192
        },
        "objective": "masked-diffusion ELBO (upper bound on NLL)",
        "perplexity": 50.229188566092304,
        "tokens": 101191373,
        "tokens_per_second": 120402.09034841752,
        "windows": 49410
      },
      "memorization": {
        "held_out": {
          "continuation_tokens": 32,
          "exact_match": 0.00390625,
          "infill_exact_match": 0.0,
          "infill_token_accuracy": 0.0523681640625,
          "prefix_tokens": 64,
          "token_accuracy": 0.03466796875,
          "windows": 256
        },
        "train": {
          "continuation_tokens": 32,
          "exact_match": 0.0,
          "infill_exact_match": 0.0,
          "infill_token_accuracy": 0.052734375,
          "prefix_tokens": 64,
          "token_accuracy": 0.0333251953125,
          "windows": 256
        }
      },
      "memorization_gap": -0.00390625,
      "train_probe": {
        "elapsed_seconds": 117.16330459574237,
        "loss": 3.784459054391895,
        "loss_95ci": [
          3.7387899501117454,
          3.831598950549923
        ],
        "masked_ce_by_ratio": {
          "0.0-0.1": 1.8213217645513433,
          "0.1-0.2": 2.052082935279734,
          "0.2-0.3": 2.406457547782629,
          "0.3-0.4": 2.742347328385029,
          "0.4-0.5": 3.16785829172493,
          "0.5-0.6": 3.723176236144619,
          "0.6-0.7": 4.332039606894565,
          "0.7-0.8": 5.048179276594368,
          "0.8-0.9": 5.847315192536721,
          "0.9-1.0": 6.820226933626609
        },
        "objective": "masked-diffusion ELBO (upper bound on NLL)",
        "perplexity": 44.01185609790593,
        "tokens": 10000384,
        "tokens_per_second": 85354.23300413981,
        "windows": 4883
      }
    }
  }
}