{
 "ckpt": "/content/init/udgam3-base-v1/model_ema_076294.pt",
 "config": {
  "sequence_len": 8193,
  "vocab_size": 32768,
  "n_layer": 28,
  "n_head": 10,
  "n_kv_head": 2,
  "n_embd": 640,
  "head_dim": 64,
  "global_head_dim": 128,
  "intermediate_size": 1792,
  "ple_dim": 0,
  "window_pattern": "SSSL",
  "sliding_window": 512,
  "rope_theta_sliding": 10000.0,
  "rope_theta_full": 1000000.0,
  "final_logit_softcap": 15.0,
  "rms_norm_eps": 1e-06,
  "loop_start": -1,
  "loop_end": -1,
  "n_loops": 1,
  "mtp": false,
  "loop_loss": false,
  "loop_loss_stride": 4,
  "loop_loss_weight": -1.0,
  "loop_reinject": false,
  "abacus": 0,
  "abacus_offset": 0,
  "digit_ids": [
   291,
   292,
   293,
   294,
   295,
   296,
   297,
   298,
   299,
   300
  ],
  "digit_space_ids": [
   1150,
   958,
   994,
   1106,
   1189,
   1182,
   1348,
   1506,
   1500,
   1564
  ],
  "tie_embeddings": true
 },
 "bpb2048_web": 0.8501416386046416,
 "bpb2048_task": 0.634110372765497,
 "bpb2048_reason": 0.6844764738368659,
 "bpb2048_fresh": 0.9209135476121134,
 "long_bpb_L8192": {
  "0-512": 1.173462,
  "512-1024": 1.035295,
  "1024-2048": 1.012183,
  "2048-3072": 1.082558,
  "3072-4096": 1.27177,
  "4096-6144": 1.366089,
  "6144-8192": 1.347903,
  "n_docs": 400,
  "per_src_first_half_second_half": {
   "pre1929": [
    1.20924,
    1.47077,
    60
   ],
   "gutenberg": [
    1.25172,
    1.53167,
    112
   ],
   "arxiv": [
    0.91582,
    1.12526,
    35
   ],
   "loc": [
    1.41106,
    1.63034,
    59
   ],
   "doab": [
    0.88866,
    1.10496,
    54
   ],
   "hansard_commons": [
    0.93938,
    1.13722,
    80
   ]
  }
 },
 "long_bpb_L16384": {
  "0-512": 1.173462,
  "512-1024": 1.035295,
  "1024-2048": 1.012183,
  "2048-3072": 1.082558,
  "3072-4096": 1.27177,
  "4096-6144": 1.366089,
  "6144-8192": 1.347903,
  "8192-12288": 1.368215,
  "12288-16384": 1.388605,
  "n_docs": 400,
  "per_src_first_half_second_half": {
   "pre1929": [
    1.33858,
    1.49199,
    60
   ],
   "gutenberg": [
    1.3932,
    1.5649,
    112
   ],
   "arxiv": [
    1.0123,
    1.13282,
    35
   ],
   "loc": [
    1.52222,
    1.59875,
    59
   ],
   "doab": [
    0.99651,
    1.11498,
    54
   ],
   "hansard_commons": [
    1.0388,
    1.17896,
    80
   ]
  }
 },
 "long_bpb_L32768": {
  "0-512": 1.173462,
  "512-1024": 1.035295,
  "1024-2048": 1.012183,
  "2048-3072": 1.082558,
  "3072-4096": 1.27177,
  "4096-6144": 1.366089,
  "6144-8192": 1.347903,
  "8192-12288": 1.368215,
  "12288-16384": 1.388605,
  "16384-24576": 1.403997,
  "24576-32768": 1.436084,
  "n_docs": 400,
  "per_src_first_half_second_half": {
   "pre1929": [
    1.41471,
    1.50851,
    60
   ],
   "gutenberg": [
    1.47915,
    1.60659,
    112
   ],
   "arxiv": [
    1.06798,
    1.26401,
    35
   ],
   "loc": [
    1.56117,
    1.61969,
    59
   ],
   "doab": [
    1.05531,
    1.19035,
    54
   ],
   "hansard_commons": [
    1.10888,
    1.20284,
    80
   ]
  }
 },
 "needle": {
  "1024": {
   "exact": 1.0,
   "mean_logprob_key_token": -0.0224,
   "distance_tokens_approx": 920
  },
  "2048": {
   "exact": 1.0,
   "mean_logprob_key_token": -0.0422,
   "distance_tokens_approx": 1944
  },
  "4096": {
   "exact": 0.0,
   "mean_logprob_key_token": -2.6239,
   "distance_tokens_approx": 3992
  },
  "8192": {
   "exact": 0.0,
   "mean_logprob_key_token": -3.6672,
   "distance_tokens_approx": 8088
  },
  "12288": {
   "exact": 0.0,
   "mean_logprob_key_token": -3.7545,
   "distance_tokens_approx": 12184
  },
  "16384": {
   "exact": 0.0,
   "mean_logprob_key_token": -3.9069,
   "distance_tokens_approx": 16280
  },
  "24576": {
   "exact": 0.0,
   "mean_logprob_key_token": -4.1492,
   "distance_tokens_approx": 24472
  },
  "32768": {
   "exact": 0.0,
   "mean_logprob_key_token": -4.1836,
   "distance_tokens_approx": 32664
  }
 },
 "copy": {
  "2048": 0.9475,
  "8192": 0.043,
  "16384": 0.0225,
  "32768": 0.023
 },
 "seconds": 597.9
}