| { |
| "ckpt": "/content/cache/base_checkpoints/udgam3-ctx32k/model_001900.pt", |
| "config": { |
| "sequence_len": 32768, |
| "vocab_size": 32768, |
| "n_layer": 28, |
| "n_head": 10, |
| "n_kv_head": 2, |
| "n_embd": 640, |
| "head_dim": 64, |
| "global_head_dim": 128, |
| "intermediate_size": 1792, |
| "ple_dim": 0, |
| "window_pattern": "SSSL", |
| "sliding_window": 512, |
| "rope_theta_sliding": 10000.0, |
| "rope_theta_full": 1000000.0, |
| "final_logit_softcap": 15.0, |
| "rms_norm_eps": 1e-06, |
| "loop_start": -1, |
| "loop_end": -1, |
| "n_loops": 1, |
| "mtp": false, |
| "loop_loss": false, |
| "loop_loss_stride": 4, |
| "loop_loss_weight": -1.0, |
| "loop_reinject": false, |
| "abacus": 0, |
| "abacus_offset": 0, |
| "digit_ids": [ |
| 291, |
| 292, |
| 293, |
| 294, |
| 295, |
| 296, |
| 297, |
| 298, |
| 299, |
| 300 |
| ], |
| "digit_space_ids": [ |
| 1150, |
| 958, |
| 994, |
| 1106, |
| 1189, |
| 1182, |
| 1348, |
| 1506, |
| 1500, |
| 1564 |
| ], |
| "tie_embeddings": true |
| }, |
| "bpb2048_web": 0.8535457788260455, |
| "bpb2048_task": 0.6393049352887717, |
| "bpb2048_reason": 0.6919760354978337, |
| "bpb2048_fresh": 0.9205743961059951, |
| "long_bpb_L8192": { |
| "0-512": 1.026994, |
| "512-1024": 0.935882, |
| "1024-2048": 0.944786, |
| "2048-3072": 0.930071, |
| "3072-4096": 0.921109, |
| "4096-6144": 0.928579, |
| "6144-8192": 0.925182, |
| "n_docs": 400, |
| "per_src_first_half_second_half": { |
| "pre1929": [ |
| 1.06866, |
| 1.07843, |
| 60 |
| ], |
| "gutenberg": [ |
| 1.0833, |
| 1.06462, |
| 112 |
| ], |
| "arxiv": [ |
| 0.76825, |
| 0.6845, |
| 35 |
| ], |
| "loc": [ |
| 1.21512, |
| 1.12338, |
| 59 |
| ], |
| "doab": [ |
| 0.68966, |
| 0.69926, |
| 54 |
| ], |
| "hansard_commons": [ |
| 0.75608, |
| 0.75785, |
| 80 |
| ] |
| } |
| }, |
| "long_bpb_L16384": { |
| "0-512": 1.026994, |
| "512-1024": 0.935882, |
| "1024-2048": 0.944786, |
| "2048-3072": 0.930071, |
| "3072-4096": 0.921109, |
| "4096-6144": 0.928579, |
| "6144-8192": 0.925182, |
| "8192-12288": 0.928408, |
| "12288-16384": 0.938268, |
| "n_docs": 400, |
| "per_src_first_half_second_half": { |
| "pre1929": [ |
| 1.07349, |
| 1.08106, |
| 60 |
| ], |
| "gutenberg": [ |
| 1.07386, |
| 1.08415, |
| 112 |
| ], |
| "arxiv": [ |
| 0.72967, |
| 0.6699, |
| 35 |
| ], |
| "loc": [ |
| 1.16862, |
| 1.104, |
| 59 |
| ], |
| "doab": [ |
| 0.69445, |
| 0.68398, |
| 54 |
| ], |
| "hansard_commons": [ |
| 0.75697, |
| 0.77872, |
| 80 |
| ] |
| } |
| }, |
| "long_bpb_L32768": { |
| "0-512": 1.026994, |
| "512-1024": 0.935882, |
| "1024-2048": 0.944786, |
| "2048-3072": 0.930071, |
| "3072-4096": 0.921109, |
| "4096-6144": 0.928579, |
| "6144-8192": 0.925182, |
| "8192-12288": 0.928408, |
| "12288-16384": 0.938268, |
| "16384-24576": 0.943074, |
| "24576-32768": 0.940369, |
| "n_docs": 400, |
| "per_src_first_half_second_half": { |
| "pre1929": [ |
| 1.07725, |
| 1.07459, |
| 60 |
| ], |
| "gutenberg": [ |
| 1.07901, |
| 1.09185, |
| 112 |
| ], |
| "arxiv": [ |
| 0.70206, |
| 0.68107, |
| 35 |
| ], |
| "loc": [ |
| 1.13573, |
| 1.09291, |
| 59 |
| ], |
| "doab": [ |
| 0.68925, |
| 0.69537, |
| 54 |
| ], |
| "hansard_commons": [ |
| 0.76784, |
| 0.80331, |
| 80 |
| ] |
| } |
| }, |
| "needle": { |
| "1024": { |
| "exact": 1.0, |
| "mean_logprob_key_token": -0.0238, |
| "distance_tokens_approx": 920 |
| }, |
| "2048": { |
| "exact": 1.0, |
| "mean_logprob_key_token": -0.0283, |
| "distance_tokens_approx": 1944 |
| }, |
| "4096": { |
| "exact": 1.0, |
| "mean_logprob_key_token": -0.0521, |
| "distance_tokens_approx": 3992 |
| }, |
| "8192": { |
| "exact": 1.0, |
| "mean_logprob_key_token": -0.0644, |
| "distance_tokens_approx": 8088 |
| }, |
| "12288": { |
| "exact": 0.99, |
| "mean_logprob_key_token": -0.1275, |
| "distance_tokens_approx": 12184 |
| }, |
| "16384": { |
| "exact": 0.96, |
| "mean_logprob_key_token": -0.1908, |
| "distance_tokens_approx": 16280 |
| }, |
| "24576": { |
| "exact": 0.0, |
| "mean_logprob_key_token": -2.4617, |
| "distance_tokens_approx": 24472 |
| }, |
| "32768": { |
| "exact": 0.0, |
| "mean_logprob_key_token": -2.9218, |
| "distance_tokens_approx": 32664 |
| } |
| }, |
| "copy": { |
| "2048": 0.947, |
| "8192": 0.8895, |
| "16384": 0.749, |
| "32768": 0.192 |
| }, |
| "seconds": 598.7 |
| } |
