| { |
| "meta": { |
| "checkpoint": "o1.pt", |
| "checkpoint_bytes": 13487357, |
| "parameter_count": 3318465, |
| "carrier_bytes": 32512, |
| "active_crystals": 3768, |
| "vocab_size": 4096, |
| "invariant_16d_norm": 124.371132, |
| "device": "mps" |
| }, |
| "symbolic_and_identity": { |
| "organ_hits": 12, |
| "organ_total": 16, |
| "organ_details": [ |
| { |
| "prompt": "seven larger three is", |
| "expected": "seven", |
| "answer": "seven", |
| "ok": true |
| }, |
| { |
| "prompt": "9 smaller 4 is", |
| "expected": "4", |
| "answer": "4", |
| "ok": true |
| }, |
| { |
| "prompt": "median of 2, 9 and 5 is", |
| "expected": "5", |
| "answer": "3", |
| "ok": false |
| }, |
| { |
| "prompt": "closer to 10 between 3 and 8 is", |
| "expected": "8", |
| "answer": "8", |
| "ok": true |
| }, |
| { |
| "prompt": "count of 4, 4, 7, 4 is", |
| "expected": "3", |
| "answer": "4", |
| "ok": false |
| }, |
| { |
| "prompt": "distance between 15 and 9 is", |
| "expected": "6", |
| "answer": "9", |
| "ok": false |
| }, |
| { |
| "prompt": "8 add 5 =", |
| "expected": "13", |
| "answer": "13", |
| "ok": true |
| }, |
| { |
| "prompt": "14 subtract 6 =", |
| "expected": "8", |
| "answer": "8", |
| "ok": true |
| }, |
| { |
| "prompt": "7 times 6 =", |
| "expected": "42", |
| "answer": "42", |
| "ok": true |
| }, |
| { |
| "prompt": "20 divide 4 =", |
| "expected": "5", |
| "answer": "10", |
| "ok": false |
| }, |
| { |
| "prompt": "17 mod 5 =", |
| "expected": "2", |
| "answer": "2", |
| "ok": true |
| }, |
| { |
| "prompt": "parity of 14 is", |
| "expected": "even", |
| "answer": "even", |
| "ok": true |
| }, |
| { |
| "prompt": "3 glorp 4 =", |
| "expected": "10", |
| "answer": "10", |
| "ok": true |
| }, |
| { |
| "prompt": "6 hypot 8 =", |
| "expected": "10", |
| "answer": "10", |
| "ok": true |
| }, |
| { |
| "prompt": "3 phasor 4 =", |
| "expected": "5", |
| "answer": "5", |
| "ok": true |
| }, |
| { |
| "prompt": "3 reladd 4 =", |
| "expected": "0.538462", |
| "answer": "0.538462", |
| "ok": true |
| } |
| ], |
| "identity_probes": { |
| "Who created o1?": "o1 was created by David Tom Foss at Wirkung", |
| "Author:": "David Tom Foss (Wirkung Inc.,", |
| "Contact email:": "david@wirku.ng (David Tom Fo", |
| "X profile:": "https://x.com/fossdt (Da" |
| }, |
| "weltbuch_verify": { |
| "valid": true, |
| "checked_count": 48, |
| "total_triples": 48, |
| "tip_fnv64": "0x7da8573a337604fe", |
| "tip_sha256": "007e8dba7436ff65c05d05afadc594b80ecba55f224c71731811f72a4d33773a" |
| }, |
| "weltbuch_triples": 48 |
| }, |
| "wikitext2": { |
| "utf8_bytes": 1287656, |
| "crystal_tokens": 546887, |
| "bytes_per_token": 2.3545, |
| "static_fresh_bpb": 1.962, |
| "static_continuous_bpb": 2.3223, |
| "static_continuous_ppl_tok": 44.26, |
| "online_gssm_bpb": 2.5635, |
| "online_gssm_ppl_tok": 65.61, |
| "online_early_bpb": 2.4985, |
| "online_late_bpb": 2.3846, |
| "online_updates": 134, |
| "online_tok_per_sec": 3437.3, |
| "carrier_bytes_after_online": 32512 |
| }, |
| "arc_easy": { |
| "n": 2376, |
| "acc": 0.2588, |
| "acc_norm": 0.2643, |
| "correct_raw": 615, |
| "correct_norm": 628, |
| "eval_sec": 13.65, |
| "tok_per_sec": 52693.9 |
| }, |
| "arc_challenge": { |
| "n": 1172, |
| "acc": 0.1826, |
| "acc_norm": 0.2381, |
| "correct_raw": 214, |
| "correct_norm": 279, |
| "eval_sec": 7.94, |
| "tok_per_sec": 50226.6 |
| }, |
| "hellaswag": { |
| "n": 10042, |
| "acc": 0.2616, |
| "acc_norm": 0.2577, |
| "correct_raw": 2627, |
| "correct_norm": 2588, |
| "eval_sec": 144.65, |
| "tok_per_sec": 46978.6 |
| }, |
| "mmlu": { |
| "n": 14042, |
| "acc": 0.2347, |
| "acc_norm": 0.252, |
| "correct_raw": 3296, |
| "correct_norm": 3538, |
| "eval_sec": 164.29, |
| "tok_per_sec": 50069.3 |
| }, |
| "mmlu_dev_online": { |
| "n": 14042, |
| "acc": 0.235, |
| "acc_norm": 0.2508, |
| "correct_raw": 3300, |
| "correct_norm": 3522, |
| "eval_sec": 164.26, |
| "tok_per_sec": 50079.9, |
| "dev_adapt_steps": 3, |
| "dev_adapt_items": 285, |
| "dev_adapt_sec": 18.15 |
| }, |
| "gsm8k": { |
| "n": 1319, |
| "exact_match": 0.0152, |
| "correct": 20, |
| "gen_tok_per_sec": 2878.3 |
| }, |
| "humaneval": { |
| "n": 164, |
| "compiled": 80, |
| "pass_at_1": 0.0, |
| "passed": 0, |
| "gen_tok_per_sec": 759.3 |
| }, |
| "mbpp": { |
| "n": 500, |
| "compiled": 241, |
| "pass_at_1": 0.0, |
| "passed": 0, |
| "gen_tok_per_sec": 2602.0 |
| }, |
| "niah": { |
| "lengths": [ |
| { |
| "T": 128, |
| "trials": 5, |
| "mc10_acc": 1.0, |
| "greedy_em": 0.6, |
| "carrier_bytes": 32512, |
| "stream_tok_per_sec": 1484.0 |
| }, |
| { |
| "T": 512, |
| "trials": 5, |
| "mc10_acc": 1.0, |
| "greedy_em": 0.4, |
| "carrier_bytes": 32512, |
| "stream_tok_per_sec": 3309.6 |
| }, |
| { |
| "T": 1024, |
| "trials": 5, |
| "mc10_acc": 1.0, |
| "greedy_em": 0.4, |
| "carrier_bytes": 32512, |
| "stream_tok_per_sec": 3936.5 |
| }, |
| { |
| "T": 2048, |
| "trials": 5, |
| "mc10_acc": 1.0, |
| "greedy_em": 0.2, |
| "carrier_bytes": 32512, |
| "stream_tok_per_sec": 4447.1 |
| }, |
| { |
| "T": 4096, |
| "trials": 5, |
| "mc10_acc": 0.2, |
| "greedy_em": 0.0, |
| "carrier_bytes": 32512, |
| "stream_tok_per_sec": 4720.4 |
| }, |
| { |
| "T": 8192, |
| "trials": 5, |
| "mc10_acc": 0.8, |
| "greedy_em": 0.0, |
| "carrier_bytes": 32512, |
| "stream_tok_per_sec": 4878.0 |
| }, |
| { |
| "T": 16384, |
| "trials": 5, |
| "mc10_acc": 0.2, |
| "greedy_em": 0.0, |
| "carrier_bytes": 32512, |
| "stream_tok_per_sec": 4946.9 |
| }, |
| { |
| "T": 32768, |
| "trials": 5, |
| "mc10_acc": 0.0, |
| "greedy_em": 0.0, |
| "carrier_bytes": 32512, |
| "stream_tok_per_sec": 5005.4 |
| } |
| ], |
| "constant_carrier_bytes_all_T": true, |
| "rss_delta_mb": 0.0 |
| }, |
| "ane_hardware": { |
| "ane_hardware_available": true, |
| "backend": "direct_private_ane_pingpong", |
| "compile_ms": 756.38, |
| "load_ms": 20.89, |
| "tokens_512_ane_ms": 152.578, |
| "tokens_per_sec_ane": 3355.7, |
| "us_per_token_ane": 298.0, |
| "recompile_count": 0, |
| "pytorch_parity_mean_cosine": 0.997528, |
| "pytorch_parity_min_cosine": 0.977215, |
| "path_a_us_per_step": 131.12, |
| "path_a_loss_drop": [ |
| 0.05118, |
| 0.04095 |
| ], |
| "path_a_ane_energy_uj": 4000.0, |
| "path_b_ane_us_per_tok": 3.33, |
| "path_b_crystal_flash_us": 8.25, |
| "hcgm_total_sram_bytes": 2806914, |
| "hcgm_total_sram_mb": 2.6769, |
| "hcgm_mean_cosine": 0.997844 |
| } |
| } |
|
|
