bpb@2048 web: 0.850142
bpb@2048 task: 0.634110
bpb@2048 reason: 0.684476
bpb@2048 fresh: 0.920914
long docs L=8192: {'0-512': 1.173462, '512-1024': 1.035295, '1024-2048': 1.012183, '2048-3072': 1.082558, '3072-4096': 1.27177, '4096-6144': 1.366089, '6144-8192': 1.347903, 'n_docs': 400, 'per_src_first_half_second_half': {'pre1929': [1.20924, 1.47077, 60], 'gutenberg': [1.25172, 1.53167, 112], 'arxiv': [0.91582, 1.12526, 35], 'loc': [1.41106, 1.63034, 59], 'doab': [0.88866, 1.10496, 54], 'hansard_commons': [0.93938, 1.13722, 80]}}
long docs L=16384: {'0-512': 1.173462, '512-1024': 1.035295, '1024-2048': 1.012183, '2048-3072': 1.082558, '3072-4096': 1.27177, '4096-6144': 1.366089, '6144-8192': 1.347903, '8192-12288': 1.368215, '12288-16384': 1.388605, 'n_docs': 400, 'per_src_first_half_second_half': {'pre1929': [1.33858, 1.49199, 60], 'gutenberg': [1.3932, 1.5649, 112], 'arxiv': [1.0123, 1.13282, 35], 'loc': [1.52222, 1.59875, 59], 'doab': [0.99651, 1.11498, 54], 'hansard_commons': [1.0388, 1.17896, 80]}}
long docs L=32768: {'0-512': 1.173462, '512-1024': 1.035295, '1024-2048': 1.012183, '2048-3072': 1.082558, '3072-4096': 1.27177, '4096-6144': 1.366089, '6144-8192': 1.347903, '8192-12288': 1.368215, '12288-16384': 1.388605, '16384-24576': 1.403997, '24576-32768': 1.436084, 'n_docs': 400, 'per_src_first_half_second_half': {'pre1929': [1.41471, 1.50851, 60], 'gutenberg': [1.47915, 1.60659, 112], 'arxiv': [1.06798, 1.26401, 35], 'loc': [1.56117, 1.61969, 59], 'doab': [1.05531, 1.19035, 54], 'hansard_commons': [1.10888, 1.20284, 80]}}
needle ctx 1024: {'exact': 1.0, 'mean_logprob_key_token': -0.0224, 'distance_tokens_approx': 920}
needle ctx 2048: {'exact': 1.0, 'mean_logprob_key_token': -0.0422, 'distance_tokens_approx': 1944}
needle ctx 4096: {'exact': 0.0, 'mean_logprob_key_token': -2.6239, 'distance_tokens_approx': 3992}
needle ctx 8192: {'exact': 0.0, 'mean_logprob_key_token': -3.6672, 'distance_tokens_approx': 8088}
needle ctx 12288: {'exact': 0.0, 'mean_logprob_key_token': -3.7545, 'distance_tokens_approx': 12184}
needle ctx 16384: {'exact': 0.0, 'mean_logprob_key_token': -3.9069, 'distance_tokens_approx': 16280}
needle ctx 24576: {'exact': 0.0, 'mean_logprob_key_token': -4.1492, 'distance_tokens_approx': 24472}
needle ctx 32768: {'exact': 0.0, 'mean_logprob_key_token': -4.1836, 'distance_tokens_approx': 32664}
copy ctx 2048: next-token acc 0.9475
copy ctx 8192: next-token acc 0.043
copy ctx 16384: next-token acc 0.0225
copy ctx 32768: next-token acc 0.023
wrote /content/eval_base2k.json