{
  "meta": {
    "checkpoint": "o1.pt",
    "checkpoint_bytes": 13487357,
    "parameter_count": 3318465,
    "carrier_bytes": 32512,
    "active_crystals": 3768,
    "vocab_size": 4096,
    "invariant_16d_norm": 124.371132,
    "device": "mps"
  },
  "symbolic_and_identity": {
    "organ_hits": 12,
    "organ_total": 16,
    "organ_details": [
      {
        "prompt": "seven larger three is",
        "expected": "seven",
        "answer": "seven",
        "ok": true
      },
      {
        "prompt": "9 smaller 4 is",
        "expected": "4",
        "answer": "4",
        "ok": true
      },
      {
        "prompt": "median of 2, 9 and 5 is",
        "expected": "5",
        "answer": "3",
        "ok": false
      },
      {
        "prompt": "closer to 10 between 3 and 8 is",
        "expected": "8",
        "answer": "8",
        "ok": true
      },
      {
        "prompt": "count of 4, 4, 7, 4 is",
        "expected": "3",
        "answer": "4",
        "ok": false
      },
      {
        "prompt": "distance between 15 and 9 is",
        "expected": "6",
        "answer": "9",
        "ok": false
      },
      {
        "prompt": "8 add 5 =",
        "expected": "13",
        "answer": "13",
        "ok": true
      },
      {
        "prompt": "14 subtract 6 =",
        "expected": "8",
        "answer": "8",
        "ok": true
      },
      {
        "prompt": "7 times 6 =",
        "expected": "42",
        "answer": "42",
        "ok": true
      },
      {
        "prompt": "20 divide 4 =",
        "expected": "5",
        "answer": "10",
        "ok": false
      },
      {
        "prompt": "17 mod 5 =",
        "expected": "2",
        "answer": "2",
        "ok": true
      },
      {
        "prompt": "parity of 14 is",
        "expected": "even",
        "answer": "even",
        "ok": true
      },
      {
        "prompt": "3 glorp 4 =",
        "expected": "10",
        "answer": "10",
        "ok": true
      },
      {
        "prompt": "6 hypot 8 =",
        "expected": "10",
        "answer": "10",
        "ok": true
      },
      {
        "prompt": "3 phasor 4 =",
        "expected": "5",
        "answer": "5",
        "ok": true
      },
      {
        "prompt": "3 reladd 4 =",
        "expected": "0.538462",
        "answer": "0.538462",
        "ok": true
      }
    ],
    "identity_probes": {
      "Who created o1?": "o1 was created by David Tom Foss at Wirkung",
      "Author:": "David Tom Foss (Wirkung Inc.,",
      "Contact email:": "david@wirku.ng (David Tom Fo",
      "X profile:": "https://x.com/fossdt (Da"
    },
    "weltbuch_verify": {
      "valid": true,
      "checked_count": 48,
      "total_triples": 48,
      "tip_fnv64": "0x7da8573a337604fe",
      "tip_sha256": "007e8dba7436ff65c05d05afadc594b80ecba55f224c71731811f72a4d33773a"
    },
    "weltbuch_triples": 48
  },
  "wikitext2": {
    "utf8_bytes": 1287656,
    "crystal_tokens": 546887,
    "bytes_per_token": 2.3545,
    "static_fresh_bpb": 1.962,
    "static_continuous_bpb": 2.3223,
    "static_continuous_ppl_tok": 44.26,
    "online_gssm_bpb": 2.5635,
    "online_gssm_ppl_tok": 65.61,
    "online_early_bpb": 2.4985,
    "online_late_bpb": 2.3846,
    "online_updates": 134,
    "online_tok_per_sec": 3437.3,
    "carrier_bytes_after_online": 32512
  },
  "arc_easy": {
    "n": 2376,
    "acc": 0.2588,
    "acc_norm": 0.2643,
    "correct_raw": 615,
    "correct_norm": 628,
    "eval_sec": 13.65,
    "tok_per_sec": 52693.9
  },
  "arc_challenge": {
    "n": 1172,
    "acc": 0.1826,
    "acc_norm": 0.2381,
    "correct_raw": 214,
    "correct_norm": 279,
    "eval_sec": 7.94,
    "tok_per_sec": 50226.6
  },
  "hellaswag": {
    "n": 10042,
    "acc": 0.2616,
    "acc_norm": 0.2577,
    "correct_raw": 2627,
    "correct_norm": 2588,
    "eval_sec": 144.65,
    "tok_per_sec": 46978.6
  },
  "mmlu": {
    "n": 14042,
    "acc": 0.2347,
    "acc_norm": 0.252,
    "correct_raw": 3296,
    "correct_norm": 3538,
    "eval_sec": 164.29,
    "tok_per_sec": 50069.3
  },
  "mmlu_dev_online": {
    "n": 14042,
    "acc": 0.235,
    "acc_norm": 0.2508,
    "correct_raw": 3300,
    "correct_norm": 3522,
    "eval_sec": 164.26,
    "tok_per_sec": 50079.9,
    "dev_adapt_steps": 3,
    "dev_adapt_items": 285,
    "dev_adapt_sec": 18.15
  },
  "gsm8k": {
    "n": 1319,
    "exact_match": 0.0152,
    "correct": 20,
    "gen_tok_per_sec": 2878.3
  },
  "humaneval": {
    "n": 164,
    "compiled": 80,
    "pass_at_1": 0.0,
    "passed": 0,
    "gen_tok_per_sec": 759.3
  },
  "mbpp": {
    "n": 500,
    "compiled": 241,
    "pass_at_1": 0.0,
    "passed": 0,
    "gen_tok_per_sec": 2602.0
  },
  "niah": {
    "lengths": [
      {
        "T": 128,
        "trials": 5,
        "mc10_acc": 1.0,
        "greedy_em": 0.6,
        "carrier_bytes": 32512,
        "stream_tok_per_sec": 1484.0
      },
      {
        "T": 512,
        "trials": 5,
        "mc10_acc": 1.0,
        "greedy_em": 0.4,
        "carrier_bytes": 32512,
        "stream_tok_per_sec": 3309.6
      },
      {
        "T": 1024,
        "trials": 5,
        "mc10_acc": 1.0,
        "greedy_em": 0.4,
        "carrier_bytes": 32512,
        "stream_tok_per_sec": 3936.5
      },
      {
        "T": 2048,
        "trials": 5,
        "mc10_acc": 1.0,
        "greedy_em": 0.2,
        "carrier_bytes": 32512,
        "stream_tok_per_sec": 4447.1
      },
      {
        "T": 4096,
        "trials": 5,
        "mc10_acc": 0.2,
        "greedy_em": 0.0,
        "carrier_bytes": 32512,
        "stream_tok_per_sec": 4720.4
      },
      {
        "T": 8192,
        "trials": 5,
        "mc10_acc": 0.8,
        "greedy_em": 0.0,
        "carrier_bytes": 32512,
        "stream_tok_per_sec": 4878.0
      },
      {
        "T": 16384,
        "trials": 5,
        "mc10_acc": 0.2,
        "greedy_em": 0.0,
        "carrier_bytes": 32512,
        "stream_tok_per_sec": 4946.9
      },
      {
        "T": 32768,
        "trials": 5,
        "mc10_acc": 0.0,
        "greedy_em": 0.0,
        "carrier_bytes": 32512,
        "stream_tok_per_sec": 5005.4
      }
    ],
    "constant_carrier_bytes_all_T": true,
    "rss_delta_mb": 0.0
  },
  "ane_hardware": {
    "ane_hardware_available": true,
    "backend": "direct_private_ane_pingpong",
    "compile_ms": 756.38,
    "load_ms": 20.89,
    "tokens_512_ane_ms": 152.578,
    "tokens_per_sec_ane": 3355.7,
    "us_per_token_ane": 298.0,
    "recompile_count": 0,
    "pytorch_parity_mean_cosine": 0.997528,
    "pytorch_parity_min_cosine": 0.977215,
    "path_a_us_per_step": 131.12,
    "path_a_loss_drop": [
      0.05118,
      0.04095
    ],
    "path_a_ane_energy_uj": 4000.0,
    "path_b_ane_us_per_tok": 3.33,
    "path_b_crystal_flash_us": 8.25,
    "hcgm_total_sram_bytes": 2806914,
    "hcgm_total_sram_mb": 2.6769,
    "hcgm_mean_cosine": 0.997844
  }
}