| { |
| "ckpt": "/content/init/udgam3-base-v1/model_ema_076294.pt", |
| "config": { |
| "sequence_len": 8193, |
| "vocab_size": 32768, |
| "n_layer": 28, |
| "n_head": 10, |
| "n_kv_head": 2, |
| "n_embd": 640, |
| "head_dim": 64, |
| "global_head_dim": 128, |
| "intermediate_size": 1792, |
| "ple_dim": 0, |
| "window_pattern": "SSSL", |
| "sliding_window": 512, |
| "rope_theta_sliding": 10000.0, |
| "rope_theta_full": 1000000.0, |
| "final_logit_softcap": 15.0, |
| "rms_norm_eps": 1e-06, |
| "loop_start": -1, |
| "loop_end": -1, |
| "n_loops": 1, |
| "mtp": false, |
| "loop_loss": false, |
| "loop_loss_stride": 4, |
| "loop_loss_weight": -1.0, |
| "loop_reinject": false, |
| "abacus": 0, |
| "abacus_offset": 0, |
| "digit_ids": [ |
| 291, |
| 292, |
| 293, |
| 294, |
| 295, |
| 296, |
| 297, |
| 298, |
| 299, |
| 300 |
| ], |
| "digit_space_ids": [ |
| 1150, |
| 958, |
| 994, |
| 1106, |
| 1189, |
| 1182, |
| 1348, |
| 1506, |
| 1500, |
| 1564 |
| ], |
| "tie_embeddings": true |
| }, |
| "bpb2048_web": 0.8501416386046416, |
| "bpb2048_task": 0.634110372765497, |
| "bpb2048_reason": 0.6844764738368659, |
| "bpb2048_fresh": 0.9209135476121134, |
| "long_bpb_L8192": { |
| "0-512": 1.173462, |
| "512-1024": 1.035295, |
| "1024-2048": 1.012183, |
| "2048-3072": 1.082558, |
| "3072-4096": 1.27177, |
| "4096-6144": 1.366089, |
| "6144-8192": 1.347903, |
| "n_docs": 400, |
| "per_src_first_half_second_half": { |
| "pre1929": [ |
| 1.20924, |
| 1.47077, |
| 60 |
| ], |
| "gutenberg": [ |
| 1.25172, |
| 1.53167, |
| 112 |
| ], |
| "arxiv": [ |
| 0.91582, |
| 1.12526, |
| 35 |
| ], |
| "loc": [ |
| 1.41106, |
| 1.63034, |
| 59 |
| ], |
| "doab": [ |
| 0.88866, |
| 1.10496, |
| 54 |
| ], |
| "hansard_commons": [ |
| 0.93938, |
| 1.13722, |
| 80 |
| ] |
| } |
| }, |
| "long_bpb_L16384": { |
| "0-512": 1.173462, |
| "512-1024": 1.035295, |
| "1024-2048": 1.012183, |
| "2048-3072": 1.082558, |
| "3072-4096": 1.27177, |
| "4096-6144": 1.366089, |
| "6144-8192": 1.347903, |
| "8192-12288": 1.368215, |
| "12288-16384": 1.388605, |
| "n_docs": 400, |
| "per_src_first_half_second_half": { |
| "pre1929": [ |
| 1.33858, |
| 1.49199, |
| 60 |
| ], |
| "gutenberg": [ |
| 1.3932, |
| 1.5649, |
| 112 |
| ], |
| "arxiv": [ |
| 1.0123, |
| 1.13282, |
| 35 |
| ], |
| "loc": [ |
| 1.52222, |
| 1.59875, |
| 59 |
| ], |
| "doab": [ |
| 0.99651, |
| 1.11498, |
| 54 |
| ], |
| "hansard_commons": [ |
| 1.0388, |
| 1.17896, |
| 80 |
| ] |
| } |
| }, |
| "long_bpb_L32768": { |
| "0-512": 1.173462, |
| "512-1024": 1.035295, |
| "1024-2048": 1.012183, |
| "2048-3072": 1.082558, |
| "3072-4096": 1.27177, |
| "4096-6144": 1.366089, |
| "6144-8192": 1.347903, |
| "8192-12288": 1.368215, |
| "12288-16384": 1.388605, |
| "16384-24576": 1.403997, |
| "24576-32768": 1.436084, |
| "n_docs": 400, |
| "per_src_first_half_second_half": { |
| "pre1929": [ |
| 1.41471, |
| 1.50851, |
| 60 |
| ], |
| "gutenberg": [ |
| 1.47915, |
| 1.60659, |
| 112 |
| ], |
| "arxiv": [ |
| 1.06798, |
| 1.26401, |
| 35 |
| ], |
| "loc": [ |
| 1.56117, |
| 1.61969, |
| 59 |
| ], |
| "doab": [ |
| 1.05531, |
| 1.19035, |
| 54 |
| ], |
| "hansard_commons": [ |
| 1.10888, |
| 1.20284, |
| 80 |
| ] |
| } |
| }, |
| "needle": { |
| "1024": { |
| "exact": 1.0, |
| "mean_logprob_key_token": -0.0224, |
| "distance_tokens_approx": 920 |
| }, |
| "2048": { |
| "exact": 1.0, |
| "mean_logprob_key_token": -0.0422, |
| "distance_tokens_approx": 1944 |
| }, |
| "4096": { |
| "exact": 0.0, |
| "mean_logprob_key_token": -2.6239, |
| "distance_tokens_approx": 3992 |
| }, |
| "8192": { |
| "exact": 0.0, |
| "mean_logprob_key_token": -3.6672, |
| "distance_tokens_approx": 8088 |
| }, |
| "12288": { |
| "exact": 0.0, |
| "mean_logprob_key_token": -3.7545, |
| "distance_tokens_approx": 12184 |
| }, |
| "16384": { |
| "exact": 0.0, |
| "mean_logprob_key_token": -3.9069, |
| "distance_tokens_approx": 16280 |
| }, |
| "24576": { |
| "exact": 0.0, |
| "mean_logprob_key_token": -4.1492, |
| "distance_tokens_approx": 24472 |
| }, |
| "32768": { |
| "exact": 0.0, |
| "mean_logprob_key_token": -4.1836, |
| "distance_tokens_approx": 32664 |
| } |
| }, |
| "copy": { |
| "2048": 0.9475, |
| "8192": 0.043, |
| "16384": 0.0225, |
| "32768": 0.023 |
| }, |
| "seconds": 597.9 |
| } |
