| { |
| "size": "700m", |
| "version": "v0.1.0", |
| "run": "f10_slp1_uni8k_d700m_plus_all_v2_3x", |
| "torch": "2.14.0+cu130", |
| "transformers": "5.18.0", |
| "hf_load_seconds": 0.4, |
| "hf_param_counts": { |
| "total": 704128512, |
| "embedding": 12288000, |
| "head": 12288000, |
| "non_embedding": 679552512 |
| }, |
| "orig_param_counts": { |
| "total": 704128512, |
| "embedding": 12288000, |
| "token_embedding": 12288000, |
| "position_embedding": 0, |
| "value_embedding": 0, |
| "head": 12288000, |
| "non_embedding": 679552512 |
| }, |
| "tied_head": false, |
| "hf_head_equals_wte": false, |
| "logits": [ |
| { |
| "input": "dcs_gold", |
| "tokens": 19, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.10451602935791016, |
| "mean_abs_diff_vs_fp32_master": 0.017697827890515327, |
| "argmax_agree_vs_fp32_master": 1.0, |
| "logit_abs_max": 13.280336380004883 |
| }, |
| { |
| "input": "gita", |
| "tokens": 27, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.13284730911254883, |
| "mean_abs_diff_vs_fp32_master": 0.0162887554615736, |
| "argmax_agree_vs_fp32_master": 1.0, |
| "logit_abs_max": 24.75530242919922 |
| }, |
| { |
| "input": "ood", |
| "tokens": 512, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.15660560131072998, |
| "mean_abs_diff_vs_fp32_master": 0.018970483914017677, |
| "argmax_agree_vs_fp32_master": 0.99609375, |
| "logit_abs_max": 26.321086883544922 |
| }, |
| { |
| "input": "prose", |
| "tokens": 61, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.13342761993408203, |
| "mean_abs_diff_vs_fp32_master": 0.016097350046038628, |
| "argmax_agree_vs_fp32_master": 1.0, |
| "logit_abs_max": 17.176036834716797 |
| }, |
| { |
| "input": "vedic", |
| "tokens": 22, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.10383224487304688, |
| "mean_abs_diff_vs_fp32_master": 0.016606679186224937, |
| "argmax_agree_vs_fp32_master": 1.0, |
| "logit_abs_max": 17.724536895751953 |
| }, |
| { |
| "input": "random_512", |
| "tokens": 512, |
| "max_abs_diff_vs_same_bf16_weights": 0.0, |
| "mean_abs_diff_vs_same_bf16_weights": 0.0, |
| "max_abs_diff_vs_fp32_master": 0.20844125747680664, |
| "mean_abs_diff_vs_fp32_master": 0.013787541538476944, |
| "argmax_agree_vs_fp32_master": 0.99609375, |
| "logit_abs_max": 20.799428939819336 |
| } |
| ], |
| "logits_max_abs_diff_impl": 0.0, |
| "logits_max_abs_diff_bf16_storage": 0.20844125747680664, |
| "left_padding_max_abs_diff": 3.528594970703125e-05, |
| "tokenizer_ids_identical": "95/100", |
| "tokenizer_first_difference": { |
| "set": "ood", |
| "index": 2 |
| }, |
| "tokenizer_ids_identical_fence_latin_false": "100/100", |
| "tokenizer_roundtrip_exact": "100/100", |
| "bpb_sample": { |
| "records_per_set": 20, |
| "block": 512, |
| "stride": 256, |
| "original_fp32_master": { |
| "pooled": 0.48407, |
| "dcs_gold": 0.66705, |
| "gita": 0.10244, |
| "ood": 0.49614, |
| "prose": 0.48516, |
| "vedic": 0.67513 |
| }, |
| "hf_bf16_weights_fp32_compute": { |
| "pooled": 0.48405, |
| "dcs_gold": 0.66683, |
| "gita": 0.1024, |
| "ood": 0.49612, |
| "prose": 0.48514, |
| "vedic": 0.6754 |
| }, |
| "pooled_rel_diff": -3.880216049873875e-05, |
| "seconds": 104.1 |
| }, |
| "bpb_full": { |
| "note": "every record, HF bf16 weights via transformers + trust_remote_code, extbench_bpb.py --fence_latin_false --no_bos --cfgs 512:256, CUDA bf16; ref = eval/bpb*.json", |
| "std": { |
| "hf_ex_gita": 0.53059, |
| "ref_ex_gita": 0.53067, |
| "rel_diff": -0.00016257954652671732, |
| "hf": { |
| "dcs_gold": 0.55111, |
| "gita": 0.07577, |
| "ood": 0.52853, |
| "prose": 0.51859, |
| "vedic": 0.66687 |
| }, |
| "ref": { |
| "dcs_gold": 0.55126, |
| "gita": 0.07576, |
| "ood": 0.52863, |
| "prose": 0.5186, |
| "vedic": 0.66705 |
| }, |
| "bytes_equal": true |
| }, |
| "clean_v1": { |
| "hf_ex_gita": 0.55404, |
| "ref_ex_gita": 0.55415, |
| "rel_diff": -0.00019493170581936874, |
| "hf": { |
| "dcs_gold": 0.55453, |
| "gita": 0.08184, |
| "ood": 0.55967, |
| "prose": 0.53144, |
| "vedic": 0.66953 |
| }, |
| "ref": { |
| "dcs_gold": 0.55472, |
| "gita": 0.08182, |
| "ood": 0.55979, |
| "prose": 0.53148, |
| "vedic": 0.66967 |
| }, |
| "bytes_equal": true |
| } |
| } |
| } |
