| { |
| "repo": "mume-math-125m", |
| "version": "v0.1.0", |
| "run": "math0_shared32k_d125m_ctx1024", |
| "torch": "2.14.0+cu130", |
| "transformers": "5.18.0", |
| "hf_load_seconds": 0.3, |
| "hf_param_counts": { |
| "total": 134105856, |
| "embedding": 24576000, |
| "head": 24576000, |
| "non_embedding": 84953856 |
| }, |
| "tokenizer_class": "MumeTokenizer", |
| "orig_param_counts": { |
| "total": 134105856, |
| "embedding": 24576000, |
| "token_embedding": 24576000, |
| "position_embedding": 0, |
| "value_embedding": 0, |
| "head": 24576000, |
| "non_embedding": 84953856 |
| }, |
| "hf_head_equals_wte": false, |
| "logits": [ |
| { |
| "input": "owm_val", |
| "tokens": 619, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.18924713134765625, |
| "mean_abs_diff_vs_fp32_master": 0.02142287604510784, |
| "argmax_agree_vs_same_bf16_weights": 1.0, |
| "argmax_agree_vs_fp32_master": 0.9983844757080078, |
| "logit_abs_max": 30.520248413085938 |
| }, |
| { |
| "input": "owm_val_clean", |
| "tokens": 619, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.18924713134765625, |
| "mean_abs_diff_vs_fp32_master": 0.02142287604510784, |
| "argmax_agree_vs_same_bf16_weights": 1.0, |
| "argmax_agree_vs_fp32_master": 0.9983844757080078, |
| "logit_abs_max": 30.520248413085938 |
| }, |
| { |
| "input": "gsm8k_test", |
| "tokens": 121, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.17910480499267578, |
| "mean_abs_diff_vs_fp32_master": 0.02021566778421402, |
| "argmax_agree_vs_same_bf16_weights": 1.0, |
| "argmax_agree_vs_fp32_master": 0.9834710955619812, |
| "logit_abs_max": 22.752124786376953 |
| }, |
| { |
| "input": "math_test", |
| "tokens": 104, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.1441974639892578, |
| "mean_abs_diff_vs_fp32_master": 0.021314028650522232, |
| "argmax_agree_vs_same_bf16_weights": 1.0, |
| "argmax_agree_vs_fp32_master": 1.0, |
| "logit_abs_max": 25.434036254882812 |
| }, |
| { |
| "input": "math_test_clean", |
| "tokens": 104, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.1441974639892578, |
| "mean_abs_diff_vs_fp32_master": 0.021314028650522232, |
| "argmax_agree_vs_same_bf16_weights": 1.0, |
| "argmax_agree_vs_fp32_master": 1.0, |
| "logit_abs_max": 25.434036254882812 |
| }, |
| { |
| "input": "random_1024", |
| "tokens": 1024, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.2052001953125, |
| "mean_abs_diff_vs_fp32_master": 0.024474455043673515, |
| "argmax_agree_vs_same_bf16_weights": 1.0, |
| "argmax_agree_vs_fp32_master": 0.994140625, |
| "logit_abs_max": 17.417985916137695 |
| } |
| ], |
| "logits_max_abs_diff_impl": 0.0, |
| "logits_max_abs_diff_bf16_storage": 0.2052001953125, |
| "left_padding_max_abs_diff": 3.147125244140625e-05, |
| "tokenizer_ids_identical": "100/100", |
| "tokenizer_roundtrip_exact": "100/100", |
| "bpb_sample": { |
| "records_per_set": 20, |
| "max_chars_per_record": 20000, |
| "block": 1024, |
| "stride": 512, |
| "per_set": { |
| "owm_val": { |
| "records": 20, |
| "tokens": 19797, |
| "utf8_bytes": 67807, |
| "original_fp32_master": 1.04333, |
| "hf_bf16_weights_fp32_compute": 1.04337, |
| "rel_diff": 3.7595580306251906e-05 |
| }, |
| "owm_val_clean": { |
| "records": 20, |
| "tokens": 21497, |
| "utf8_bytes": 80082, |
| "original_fp32_master": 1.13213, |
| "hf_bf16_weights_fp32_compute": 1.13217, |
| "rel_diff": 3.843563384466128e-05 |
| }, |
| "gsm8k_test": { |
| "records": 20, |
| "tokens": 3636, |
| "utf8_bytes": 10910, |
| "original_fp32_master": 1.04602, |
| "hf_bf16_weights_fp32_compute": 1.04612, |
| "rel_diff": 9.461596812371703e-05 |
| }, |
| "math_test": { |
| "records": 20, |
| "tokens": 4062, |
| "utf8_bytes": 10074, |
| "original_fp32_master": 0.8763, |
| "hf_bf16_weights_fp32_compute": 0.87655, |
| "rel_diff": 0.00027754500238853655 |
| }, |
| "math_test_clean": { |
| "records": 20, |
| "tokens": 3983, |
| "utf8_bytes": 9958, |
| "original_fp32_master": 0.89883, |
| "hf_bf16_weights_fp32_compute": 0.89907, |
| "rel_diff": 0.0002619447947951707 |
| } |
| }, |
| "pooled_original_fp32_master": 1.0658, |
| "pooled_hf_bf16_weights_fp32_compute": 1.06587, |
| "pooled_rel_diff": 6.305839549519214e-05, |
| "seconds": 107.1 |
| }, |
| "full_set": { |
| "set": "gsm8k_test", |
| "records": 1319, |
| "tokens": 225351, |
| "utf8_bytes": 652230, |
| "frozen_bytes": 652230, |
| "hf_bpb": 1.07, |
| "frozen_bpb_gpu_bf16_autocast": 1.07016, |
| "rel_diff": -0.00014918026252958935, |
| "note": "", |
| "seconds": 217.3 |
| } |
| } |
