File size: 5,146 Bytes
6307c7e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | {
"repo": "mume-math-125m",
"version": "v0.1.0",
"run": "math0_shared32k_d125m_ctx1024",
"torch": "2.14.0+cu130",
"transformers": "5.18.0",
"hf_load_seconds": 0.3,
"hf_param_counts": {
"total": 134105856,
"embedding": 24576000,
"head": 24576000,
"non_embedding": 84953856
},
"tokenizer_class": "MumeTokenizer",
"orig_param_counts": {
"total": 134105856,
"embedding": 24576000,
"token_embedding": 24576000,
"position_embedding": 0,
"value_embedding": 0,
"head": 24576000,
"non_embedding": 84953856
},
"hf_head_equals_wte": false,
"logits": [
{
"input": "owm_val",
"tokens": 619,
"max_abs_diff_vs_same_bf16_weights": 0.0,
"mean_abs_diff_vs_same_bf16_weights": 0.0,
"max_abs_diff_vs_fp32_master": 0.18924713134765625,
"mean_abs_diff_vs_fp32_master": 0.02142287604510784,
"argmax_agree_vs_same_bf16_weights": 1.0,
"argmax_agree_vs_fp32_master": 0.9983844757080078,
"logit_abs_max": 30.520248413085938
},
{
"input": "owm_val_clean",
"tokens": 619,
"max_abs_diff_vs_same_bf16_weights": 0.0,
"mean_abs_diff_vs_same_bf16_weights": 0.0,
"max_abs_diff_vs_fp32_master": 0.18924713134765625,
"mean_abs_diff_vs_fp32_master": 0.02142287604510784,
"argmax_agree_vs_same_bf16_weights": 1.0,
"argmax_agree_vs_fp32_master": 0.9983844757080078,
"logit_abs_max": 30.520248413085938
},
{
"input": "gsm8k_test",
"tokens": 121,
"max_abs_diff_vs_same_bf16_weights": 0.0,
"mean_abs_diff_vs_same_bf16_weights": 0.0,
"max_abs_diff_vs_fp32_master": 0.17910480499267578,
"mean_abs_diff_vs_fp32_master": 0.02021566778421402,
"argmax_agree_vs_same_bf16_weights": 1.0,
"argmax_agree_vs_fp32_master": 0.9834710955619812,
"logit_abs_max": 22.752124786376953
},
{
"input": "math_test",
"tokens": 104,
"max_abs_diff_vs_same_bf16_weights": 0.0,
"mean_abs_diff_vs_same_bf16_weights": 0.0,
"max_abs_diff_vs_fp32_master": 0.1441974639892578,
"mean_abs_diff_vs_fp32_master": 0.021314028650522232,
"argmax_agree_vs_same_bf16_weights": 1.0,
"argmax_agree_vs_fp32_master": 1.0,
"logit_abs_max": 25.434036254882812
},
{
"input": "math_test_clean",
"tokens": 104,
"max_abs_diff_vs_same_bf16_weights": 0.0,
"mean_abs_diff_vs_same_bf16_weights": 0.0,
"max_abs_diff_vs_fp32_master": 0.1441974639892578,
"mean_abs_diff_vs_fp32_master": 0.021314028650522232,
"argmax_agree_vs_same_bf16_weights": 1.0,
"argmax_agree_vs_fp32_master": 1.0,
"logit_abs_max": 25.434036254882812
},
{
"input": "random_1024",
"tokens": 1024,
"max_abs_diff_vs_same_bf16_weights": 0.0,
"mean_abs_diff_vs_same_bf16_weights": 0.0,
"max_abs_diff_vs_fp32_master": 0.2052001953125,
"mean_abs_diff_vs_fp32_master": 0.024474455043673515,
"argmax_agree_vs_same_bf16_weights": 1.0,
"argmax_agree_vs_fp32_master": 0.994140625,
"logit_abs_max": 17.417985916137695
}
],
"logits_max_abs_diff_impl": 0.0,
"logits_max_abs_diff_bf16_storage": 0.2052001953125,
"left_padding_max_abs_diff": 3.147125244140625e-05,
"tokenizer_ids_identical": "100/100",
"tokenizer_roundtrip_exact": "100/100",
"bpb_sample": {
"records_per_set": 20,
"max_chars_per_record": 20000,
"block": 1024,
"stride": 512,
"per_set": {
"owm_val": {
"records": 20,
"tokens": 19797,
"utf8_bytes": 67807,
"original_fp32_master": 1.04333,
"hf_bf16_weights_fp32_compute": 1.04337,
"rel_diff": 3.7595580306251906e-05
},
"owm_val_clean": {
"records": 20,
"tokens": 21497,
"utf8_bytes": 80082,
"original_fp32_master": 1.13213,
"hf_bf16_weights_fp32_compute": 1.13217,
"rel_diff": 3.843563384466128e-05
},
"gsm8k_test": {
"records": 20,
"tokens": 3636,
"utf8_bytes": 10910,
"original_fp32_master": 1.04602,
"hf_bf16_weights_fp32_compute": 1.04612,
"rel_diff": 9.461596812371703e-05
},
"math_test": {
"records": 20,
"tokens": 4062,
"utf8_bytes": 10074,
"original_fp32_master": 0.8763,
"hf_bf16_weights_fp32_compute": 0.87655,
"rel_diff": 0.00027754500238853655
},
"math_test_clean": {
"records": 20,
"tokens": 3983,
"utf8_bytes": 9958,
"original_fp32_master": 0.89883,
"hf_bf16_weights_fp32_compute": 0.89907,
"rel_diff": 0.0002619447947951707
}
},
"pooled_original_fp32_master": 1.0658,
"pooled_hf_bf16_weights_fp32_compute": 1.06587,
"pooled_rel_diff": 6.305839549519214e-05,
"seconds": 107.1
},
"full_set": {
"set": "gsm8k_test",
"records": 1319,
"tokens": 225351,
"utf8_bytes": 652230,
"frozen_bytes": 652230,
"hf_bpb": 1.07,
"frozen_bpb_gpu_bf16_autocast": 1.07016,
"rel_diff": -0.00014918026252958935,
"note": "",
"seconds": 217.3
}
} |